309 lines
12 KiB
Diff
309 lines
12 KiB
Diff
|
commit 8715f5df5f531eb997dcc20c1427cabf5a3284ab
|
||
|
|
Author: Victor Costan <pwnall@chromium.org>
|
||
|
|
Date: Tue Jan 30 06:22:31 2018 +0000
|
||
|
|
|
||
|
|
sqlite: Refactor BUILD.gn.
|
||
|
|
|
||
|
|
This CL contains the following refactorings.
|
||
|
|
|
||
|
|
1) The preprocessor defines ("compile-time options" in the SQLite
|
||
|
|
documentation [1]) used to build the SQLite library bundled with Chromium
|
||
|
|
are extracted in a "chromium_sqlite3_compile_options" configuration.
|
||
|
|
|
||
|
|
2) The "chromium_sqlite3_compile_options" configuration is injected into
|
||
|
|
all the targets that depend on //third_party/sqlite (when using
|
||
|
|
Chromium's bundled SQLite library), so sqlite.h is parsed with the
|
||
|
|
same preprocessor defines used to compile the library. This will
|
||
|
|
become important when we start disabling the SQLite features we don't
|
||
|
|
use.
|
||
|
|
|
||
|
|
3) The SQLite shell is compiled with the same preprocessor defines and
|
||
|
|
disabled warnings as the SQLite library. The shell is only built on
|
||
|
|
Linux for the purpose of debugging SQLite issues, and does not ship
|
||
|
|
with Chrome.
|
||
|
|
|
||
|
|
4) The configuration used when we rely on the system's SQLite library
|
||
|
|
(so the SQLite bundled with Chromium is not built) is renamed from
|
||
|
|
"sqlite_config" to "system_sqlite_config".
|
||
|
|
|
||
|
|
[1] https://www.sqlite.org/compile.html
|
||
|
|
|
||
|
|
Bug: 807093
|
||
|
|
Change-Id: Ibf495ef3c4635a9b40c35e9998694293899d10d9
|
||
|
|
Reviewed-on: https://chromium-review.googlesource.com/892096
|
||
|
|
Reviewed-by: Chris Mumford <cmumford@chromium.org>
|
||
|
|
Commit-Queue: Victor Costan <pwnall@chromium.org>
|
||
|
|
Cr-Commit-Position: refs/heads/master@{#532792}
|
||
|
|
|
||
|
|
diff --git a/third_party/sqlite/BUILD.gn b/third_party/sqlite/BUILD.gn
|
||
|
|
index 795bd8b..c23f10a 100644
|
||
|
|
--- a/third_party/sqlite/BUILD.gn
|
||
|
|
+++ b/third_party/sqlite/BUILD.gn
|
||
|
|
@@ -11,6 +11,98 @@ declare_args() {
|
||
|
|
use_system_sqlite = is_ios
|
||
|
|
}
|
||
|
|
|
||
|
|
+# Compile-time options passed to SQLite.
|
||
|
|
+#
|
||
|
|
+# These options are used when building our own SQLite library, which happens
|
||
|
|
+# everywhere except on iOS. These compile-time options are exported via a
|
||
|
|
+# public_config to all targets using SQLite, because they're needed by the
|
||
|
|
+# sqlite.h header. To avoid name clashes (macro names are resolved using a
|
||
|
|
+# global namespace), this block should only contain preprocessor macros that
|
||
|
|
+# are unambiguously connected to SQLite.
|
||
|
|
+#
|
||
|
|
+# The vast majority of the macros here are documented at
|
||
|
|
+# https://www.sqlite.org/compile.html
|
||
|
|
+config("chromium_sqlite3_compile_options") {
|
||
|
|
+ defines = [
|
||
|
|
+ "SQLITE_ENABLE_FTS3",
|
||
|
|
+
|
||
|
|
+ # New unicode61 tokenizer with built-in tables.
|
||
|
|
+ "SQLITE_DISABLE_FTS3_UNICODE",
|
||
|
|
+
|
||
|
|
+ # Chromium currently does not enable fts4, disable extra code.
|
||
|
|
+ "SQLITE_DISABLE_FTS4_DEFERRED",
|
||
|
|
+ "SQLITE_ENABLE_ICU",
|
||
|
|
+ "SQLITE_ENABLE_MEMORY_MANAGEMENT",
|
||
|
|
+ "SQLITE_SECURE_DELETE",
|
||
|
|
+
|
||
|
|
+ # Custom flag to tweak pcache pools.
|
||
|
|
+ # TODO(shess): This shouldn't use faux-SQLite naming.
|
||
|
|
+ "SQLITE_SEPARATE_CACHE_POOLS",
|
||
|
|
+
|
||
|
|
+ # TODO(shess): SQLite adds mutexes to protect structures which cross
|
||
|
|
+ # threads. In theory Chromium should be able to turn this to "2" which
|
||
|
|
+ # should give a slight speed boost. "2" is safe as long as a single
|
||
|
|
+ # connection is not used by more than one thread at a time.
|
||
|
|
+ "SQLITE_THREADSAFE=1",
|
||
|
|
+
|
||
|
|
+ # SQLite can spawn threads to sort in parallel if configured
|
||
|
|
+ # appropriately. Chromium doesn't configure SQLite for that, and would
|
||
|
|
+ # prefer to control distribution to worker threads.
|
||
|
|
+ "SQLITE_MAX_WORKER_THREADS=0",
|
||
|
|
+
|
||
|
|
+ # Allow 256MB mmap footprint per connection. Should not be too open-ended
|
||
|
|
+ # as that could cause memory fragmentation. 50MB encompasses the 99th
|
||
|
|
+ # percentile of Chrome databases in the wild.
|
||
|
|
+ # TODO(shess): A 64-bit-specific value could be 1G or more.
|
||
|
|
+ # TODO(shess): Figure out if exceeding this is costly.
|
||
|
|
+ "SQLITE_MAX_MMAP_SIZE=268435456",
|
||
|
|
+
|
||
|
|
+ # Use a read-only memory map when mmap'ed I/O is enabled to prevent memory
|
||
|
|
+ # stompers from directly corrupting the database.
|
||
|
|
+ # TODO(shess): Upstream the ability to use this define.
|
||
|
|
+ "SQLITE_MMAP_READ_ONLY=1",
|
||
|
|
+
|
||
|
|
+ # By default SQLite pre-allocates 100 pages of pcache data, which will not
|
||
|
|
+ # be released until the handle is closed. This is contrary to Chromium's
|
||
|
|
+ # memory-usage goals.
|
||
|
|
+ "SQLITE_DEFAULT_PCACHE_INITSZ=0",
|
||
|
|
+
|
||
|
|
+ # NOTE(shess): Some defines can affect the amalgamation. Those should be
|
||
|
|
+ # added to google_generate_amalgamation.sh, and the amalgamation
|
||
|
|
+ # re-generated. Usually this involves disabling features which include
|
||
|
|
+ # keywords or syntax, for instance SQLITE_OMIT_VIRTUALTABLE omits the
|
||
|
|
+ # virtual table syntax entirely. Missing an item usually results in
|
||
|
|
+ # syntax working but execution failing. Review:
|
||
|
|
+ # src/src/parse.py
|
||
|
|
+ # src/tool/mkkeywordhash.c
|
||
|
|
+ ]
|
||
|
|
+
|
||
|
|
+ # Pull in config.h on Linux. This allows use of preprocessor macros which
|
||
|
|
+ # are not available to the build config.
|
||
|
|
+ if (is_linux) {
|
||
|
|
+ defines += [ "_HAVE_SQLITE_CONFIG_H" ]
|
||
|
|
+ }
|
||
|
|
+
|
||
|
|
+ if (using_sanitizer) {
|
||
|
|
+ # Limit max length of data blobs and queries for fuzzing builds by 128 MB.
|
||
|
|
+ defines += [
|
||
|
|
+ "SQLITE_MAX_LENGTH=128000000",
|
||
|
|
+ "SQLITE_MAX_SQL_LENGTH=128000000",
|
||
|
|
+ "SQLITE_PRINTF_PRECISION_LIMIT=1280000",
|
||
|
|
+ ]
|
||
|
|
+
|
||
|
|
+ # During fuzz testing, valid SQL queries generated by fuzzing engine may
|
||
|
|
+ # lead to large memory allocations. If that happens, fuzzer reports an
|
||
|
|
+ # out-of-memory error. However, such errors are not valid bugs.
|
||
|
|
+ # To avoid hitting those irrelevant OOMs, we limit max number of memory
|
||
|
|
+ # pages, so fuzzer will not crash when reaching the limit.
|
||
|
|
+ # Apply this for fuzzing builds only, not for all builds with sanitizers.
|
||
|
|
+ if (use_fuzzing_engine) {
|
||
|
|
+ defines += [ "SQLITE_MAX_PAGE_COUNT=16384" ]
|
||
|
|
+ }
|
||
|
|
+ }
|
||
|
|
+}
|
||
|
|
+
|
||
|
|
if (!use_system_sqlite) {
|
||
|
|
config("sqlite_warnings") {
|
||
|
|
cflags = []
|
||
|
|
@@ -46,59 +138,8 @@ if (!use_system_sqlite) {
|
||
|
|
]
|
||
|
|
|
||
|
|
cflags = []
|
||
|
|
- defines = [
|
||
|
|
- "SQLITE_ENABLE_FTS3",
|
||
|
|
-
|
||
|
|
- # New unicode61 tokenizer with built-in tables.
|
||
|
|
- "SQLITE_DISABLE_FTS3_UNICODE",
|
||
|
|
-
|
||
|
|
- # Chromium currently does not enable fts4, disable extra code.
|
||
|
|
- "SQLITE_DISABLE_FTS4_DEFERRED",
|
||
|
|
- "SQLITE_ENABLE_ICU",
|
||
|
|
- "SQLITE_ENABLE_MEMORY_MANAGEMENT",
|
||
|
|
- "SQLITE_SECURE_DELETE",
|
||
|
|
-
|
||
|
|
- # Custom flag to tweak pcache pools.
|
||
|
|
- # TODO(shess): This shouldn't use faux-SQLite naming.
|
||
|
|
- "SQLITE_SEPARATE_CACHE_POOLS",
|
||
|
|
-
|
||
|
|
- # TODO(shess): SQLite adds mutexes to protect structures which cross
|
||
|
|
- # threads. In theory Chromium should be able to turn this to "2" which
|
||
|
|
- # should give a slight speed boost. "2" is safe as long as a single
|
||
|
|
- # connection is not used by more than one thread at a time.
|
||
|
|
- "SQLITE_THREADSAFE=1",
|
||
|
|
-
|
||
|
|
- # SQLite can spawn threads to sort in parallel if configured
|
||
|
|
- # appropriately. Chromium doesn't configure SQLite for that, and would
|
||
|
|
- # prefer to control distribution to worker threads.
|
||
|
|
- "SQLITE_MAX_WORKER_THREADS=0",
|
||
|
|
-
|
||
|
|
- # Allow 256MB mmap footprint per connection. Should not be too open-ended
|
||
|
|
- # as that could cause memory fragmentation. 50MB encompasses the 99th
|
||
|
|
- # percentile of Chrome databases in the wild.
|
||
|
|
- # TODO(shess): A 64-bit-specific value could be 1G or more.
|
||
|
|
- # TODO(shess): Figure out if exceeding this is costly.
|
||
|
|
- "SQLITE_MAX_MMAP_SIZE=268435456",
|
||
|
|
-
|
||
|
|
- # Use a read-only memory map when mmap'ed I/O is enabled to prevent memory
|
||
|
|
- # stompers from directly corrupting the database.
|
||
|
|
- # TODO(shess): Upstream the ability to use this define.
|
||
|
|
- "SQLITE_MMAP_READ_ONLY=1",
|
||
|
|
-
|
||
|
|
- # By default SQLite pre-allocates 100 pages of pcache data, which will not
|
||
|
|
- # be released until the handle is closed. This is contrary to Chromium's
|
||
|
|
- # memory-usage goals.
|
||
|
|
- "SQLITE_DEFAULT_PCACHE_INITSZ=0",
|
||
|
|
-
|
||
|
|
- # NOTE(shess): Some defines can affect the amalgamation. Those should be
|
||
|
|
- # added to google_generate_amalgamation.sh, and the amalgamation
|
||
|
|
- # re-generated. Usually this involves disabling features which include
|
||
|
|
- # keywords or syntax, for instance SQLITE_OMIT_VIRTUALTABLE omits the
|
||
|
|
- # virtual table syntax entirely. Missing an item usually results in
|
||
|
|
- # syntax working but execution failing. Review:
|
||
|
|
- # src/src/parse.py
|
||
|
|
- # src/tool/mkkeywordhash.c
|
||
|
|
- ]
|
||
|
|
+ defines = []
|
||
|
|
+
|
||
|
|
if (is_component_build) {
|
||
|
|
if (is_win) {
|
||
|
|
defines += [ "SQLITE_API=__declspec(dllexport)" ]
|
||
|
|
@@ -106,6 +147,14 @@ if (!use_system_sqlite) {
|
||
|
|
defines += [ "SQLITE_API=__attribute__((visibility(\"default\")))" ]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
+
|
||
|
|
+ if (is_linux || is_android) {
|
||
|
|
+ defines += [
|
||
|
|
+ # Linux provides fdatasync(), a faster equivalent of fsync().
|
||
|
|
+ "fdatasync=fdatasync",
|
||
|
|
+ ]
|
||
|
|
+ }
|
||
|
|
+
|
||
|
|
if (is_posix) {
|
||
|
|
defines += [
|
||
|
|
# Allow xSleep() call on Unix to use usleep() rather than sleep(), so it
|
||
|
|
@@ -118,42 +167,12 @@ if (!use_system_sqlite) {
|
||
|
|
"USE_PREAD=1",
|
||
|
|
]
|
||
|
|
}
|
||
|
|
- if (is_linux || is_android) {
|
||
|
|
- defines += [
|
||
|
|
- # Linux provides fdatasync(), a faster equivalent of fsync().
|
||
|
|
- "fdatasync=fdatasync",
|
||
|
|
- ]
|
||
|
|
- }
|
||
|
|
-
|
||
|
|
- # Pull in config.h on Linux. This allows use of preprocessor macros which
|
||
|
|
- # are not available to the build config.
|
||
|
|
- if (is_linux) {
|
||
|
|
- defines += [ "_HAVE_SQLITE_CONFIG_H" ]
|
||
|
|
- }
|
||
|
|
-
|
||
|
|
- if (using_sanitizer) {
|
||
|
|
- # Limit max length of data blobs and queries for fuzzing builds by 128 MB.
|
||
|
|
- defines += [
|
||
|
|
- "SQLITE_MAX_LENGTH=128000000",
|
||
|
|
- "SQLITE_MAX_SQL_LENGTH=128000000",
|
||
|
|
- "SQLITE_PRINTF_PRECISION_LIMIT=1280000",
|
||
|
|
- ]
|
||
|
|
-
|
||
|
|
- # During fuzz testing, valid SQL queries generated by fuzzing engine may
|
||
|
|
- # lead to large memory allocations. If that happens, fuzzer reports an
|
||
|
|
- # out-of-memory error. However, such errors are not valid bugs.
|
||
|
|
- # To avoid hitting those irrelevant OOMs, we limit max number of memory
|
||
|
|
- # pages, so fuzzer will not crash when reaching the limit.
|
||
|
|
- # Apply this for fuzzing builds only, not for all builds with sanitizers.
|
||
|
|
- if (use_libfuzzer || use_afl) {
|
||
|
|
- defines += [ "SQLITE_MAX_PAGE_COUNT=16384" ]
|
||
|
|
- }
|
||
|
|
- }
|
||
|
|
|
||
|
|
include_dirs = [ "amalgamation" ]
|
||
|
|
|
||
|
|
configs -= [ "//build/config/compiler:chromium_code" ]
|
||
|
|
configs += [
|
||
|
|
+ ":chromium_sqlite3_compile_options",
|
||
|
|
"//build/config/compiler:no_chromium_code",
|
||
|
|
|
||
|
|
# Must be after no_chromium_code for warning flags to be ordered
|
||
|
|
@@ -195,7 +214,10 @@ if (!use_system_sqlite) {
|
||
|
|
public_deps = [
|
||
|
|
":chromium_sqlite3",
|
||
|
|
]
|
||
|
|
- public_configs = [ ":sqlite_export" ]
|
||
|
|
+ public_configs = [
|
||
|
|
+ ":chromium_sqlite3_compile_options",
|
||
|
|
+ ":sqlite_export",
|
||
|
|
+ ]
|
||
|
|
}
|
||
|
|
|
||
|
|
if (is_linux) {
|
||
|
|
@@ -216,6 +238,16 @@ if (!use_system_sqlite) {
|
||
|
|
"//build/config:exe_and_shlib_deps",
|
||
|
|
"//third_party/icu",
|
||
|
|
]
|
||
|
|
+
|
||
|
|
+ configs -= [ "//build/config/compiler:chromium_code" ]
|
||
|
|
+ configs += [
|
||
|
|
+ ":chromium_sqlite3_compile_options",
|
||
|
|
+ "//build/config/compiler:no_chromium_code",
|
||
|
|
+
|
||
|
|
+ # Must be after no_chromium_code for warning flags to be ordered
|
||
|
|
+ # correctly.
|
||
|
|
+ ":sqlite_warnings",
|
||
|
|
+ ]
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
@@ -225,7 +257,7 @@ if (use_system_sqlite) {
|
||
|
|
# version shipped with Chromium. Export a "sqlite" target so the change
|
||
|
|
# can be localized to this file.
|
||
|
|
|
||
|
|
- config("sqlite_config") {
|
||
|
|
+ config("system_sqlite_config") {
|
||
|
|
defines = [ "USE_SYSTEM_SQLITE" ]
|
||
|
|
if (is_ios) {
|
||
|
|
libs = [ "sqlite3" ]
|
||
|
|
@@ -235,7 +267,7 @@ if (use_system_sqlite) {
|
||
|
|
}
|
||
|
|
|
||
|
|
source_set("sqlite") {
|
||
|
|
- public_configs = [ ":sqlite_config" ]
|
||
|
|
+ public_configs = [ ":system_sqlite_config" ]
|
||
|
|
if (is_ios) {
|
||
|
|
public_deps = [
|
||
|
|
":sqlite_recover",
|