Increase leveldb version to 1.19.

------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=129930720
A zippy change broke test assumptions about the size of compressed output.
2026-09-01 11:37:52 +08:00 · 2016-08-11 07:33:30 -07:00 · 2016-07-06 09:16:11 -07:00 · 2016-07-06 09:15:53 -07:00 · 2016-04-15 10:58:27 -07:00 · 2016-04-12 15:38:09 -07:00
27 changed files with 622 additions and 233 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -0,0 +1,13 @@
+language: cpp
+compiler:
+- clang
+- gcc
+os:
+- linux
+- osx
+sudo: false
+before_install:
+- echo $LANG
+- echo $LC_ALL
+script:
+- make -j 4 check
--- a/469
+++ b/469
@@ -20,214 +20,395 @@ $(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \
 # this file is generated by the previous line to set build flags and sources
 include build_config.mk

+TESTS = \
+	db/autocompact_test \
+	db/c_test \
+	db/corruption_test \
+	db/db_test \
+	db/dbformat_test \
+	db/fault_injection_test \
+	db/filename_test \
+	db/log_test \
+	db/recovery_test \
+	db/skiplist_test \
+	db/version_edit_test \
+	db/version_set_test \
+	db/write_batch_test \
+	helpers/memenv/memenv_test \
+	issues/issue178_test \
+	issues/issue200_test \
+	table/filter_block_test \
+	table/table_test \
+	util/arena_test \
+	util/bloom_test \
+	util/cache_test \
+	util/coding_test \
+	util/crc32c_test \
+	util/env_test \
+	util/hash_test
+
+UTILS = \
+	db/db_bench \
+	db/leveldbutil
+
+# Put the object files in a subdirectory, but the application at the top of the object dir.
+PROGNAMES := $(notdir $(TESTS) $(UTILS))
+
+# On Linux may need libkyotocabinet-dev for dependency.
+BENCHMARKS = \
+	doc/bench/db_bench_sqlite3 \
+	doc/bench/db_bench_tree_db
+
 CFLAGS += -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT)

 LDFLAGS += $(PLATFORM_LDFLAGS)
 LIBS += $(PLATFORM_LIBS)

-LIBOBJECTS = $(SOURCES:.cc=.o)
-MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
+SIMULATOR_OUTDIR=out-ios-x86
+DEVICE_OUTDIR=out-ios-arm

-TESTUTIL = ./util/testutil.o
-TESTHARNESS = ./util/testharness.o $(TESTUTIL)
-
-# Note: iOS should probably be using libtool, not ar.
 ifeq ($(PLATFORM), IOS)
+# Note: iOS should probably be using libtool, not ar.
 AR=xcrun ar
+SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path)
+DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path)
+DEVICE_CFLAGS = -isysroot "$(DEVICESDK)" -arch armv6 -arch armv7 -arch armv7s -arch arm64
+SIMULATOR_CFLAGS = -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64
+STATIC_OUTDIR=out-ios-universal
+else
+STATIC_OUTDIR=out-static
+SHARED_OUTDIR=out-shared
+STATIC_PROGRAMS := $(addprefix $(STATIC_OUTDIR)/, $(PROGNAMES))
+SHARED_PROGRAMS := $(addprefix $(SHARED_OUTDIR)/, db_bench)
 endif

-TESTS = \
-	arena_test \
-	autocompact_test \
-	bloom_test \
-	c_test \
-	cache_test \
-	coding_test \
-	corruption_test \
-	crc32c_test \
-	db_test \
-	dbformat_test \
-	env_test \
-	fault_injection_test \
-	filename_test \
-	filter_block_test \
-	hash_test \
-	issue178_test \
-	issue200_test \
-	log_test \
-	memenv_test \
-	recovery_test \
-	skiplist_test \
-	table_test \
-	version_edit_test \
-	version_set_test \
-	write_batch_test
+STATIC_LIBOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(SOURCES:.cc=.o))
+STATIC_MEMENVOBJECTS := $(addprefix $(STATIC_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))

-PROGRAMS = db_bench leveldbutil $(TESTS)
-BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
+DEVICE_LIBOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(SOURCES:.cc=.o))
+DEVICE_MEMENVOBJECTS := $(addprefix $(DEVICE_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))

-LIBRARY = libleveldb.a
-MEMENVLIBRARY = libmemenv.a
+SIMULATOR_LIBOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(SOURCES:.cc=.o))
+SIMULATOR_MEMENVOBJECTS := $(addprefix $(SIMULATOR_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
+
+SHARED_LIBOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(SOURCES:.cc=.o))
+SHARED_MEMENVOBJECTS := $(addprefix $(SHARED_OUTDIR)/, $(MEMENV_SOURCES:.cc=.o))
+
+TESTUTIL := $(STATIC_OUTDIR)/util/testutil.o
+TESTHARNESS := $(STATIC_OUTDIR)/util/testharness.o $(TESTUTIL)
+
+STATIC_TESTOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(TESTS)))
+STATIC_UTILOBJS := $(addprefix $(STATIC_OUTDIR)/, $(addsuffix .o, $(UTILS)))
+STATIC_ALLOBJS := $(STATIC_LIBOBJECTS) $(STATIC_MEMENVOBJECTS) $(STATIC_TESTOBJS) $(STATIC_UTILOBJS) $(TESTHARNESS)
+DEVICE_ALLOBJS := $(DEVICE_LIBOBJECTS) $(DEVICE_MEMENVOBJECTS)
+SIMULATOR_ALLOBJS := $(SIMULATOR_LIBOBJECTS) $(SIMULATOR_MEMENVOBJECTS)

 default: all

 # Should we build shared libraries?
 ifneq ($(PLATFORM_SHARED_EXT),)

+# Many leveldb test apps use non-exported API's. Only build a subset for testing.
+SHARED_ALLOBJS := $(SHARED_LIBOBJECTS) $(SHARED_MEMENVOBJECTS) $(TESTHARNESS)
+
 ifneq ($(PLATFORM_SHARED_VERSIONED),true)
-SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
-SHARED2 = $(SHARED1)
-SHARED3 = $(SHARED1)
-SHARED = $(SHARED1)
+SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
+SHARED_LIB2 = $(SHARED_LIB1)
+SHARED_LIB3 = $(SHARED_LIB1)
+SHARED_LIBS = $(SHARED_LIB1)
+SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
 else
 # Update db.h if you change these.
-SHARED_MAJOR = 1
-SHARED_MINOR = 18
-SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
-SHARED2 = $(SHARED1).$(SHARED_MAJOR)
-SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
-SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
-$(SHARED1): $(SHARED3)
-	ln -fs $(SHARED3) $(SHARED1)
-$(SHARED2): $(SHARED3)
-	ln -fs $(SHARED3) $(SHARED2)
+SHARED_VERSION_MAJOR = 1
+SHARED_VERSION_MINOR = 19
+SHARED_LIB1 = libleveldb.$(PLATFORM_SHARED_EXT)
+SHARED_LIB2 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR)
+SHARED_LIB3 = $(SHARED_LIB1).$(SHARED_VERSION_MAJOR).$(SHARED_VERSION_MINOR)
+SHARED_LIBS = $(SHARED_OUTDIR)/$(SHARED_LIB1) $(SHARED_OUTDIR)/$(SHARED_LIB2) $(SHARED_OUTDIR)/$(SHARED_LIB3)
+$(SHARED_OUTDIR)/$(SHARED_LIB1): $(SHARED_OUTDIR)/$(SHARED_LIB3)
+	ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB1)
+$(SHARED_OUTDIR)/$(SHARED_LIB2): $(SHARED_OUTDIR)/$(SHARED_LIB3)
+	ln -fs $(SHARED_LIB3) $(SHARED_OUTDIR)/$(SHARED_LIB2)
+SHARED_MEMENVLIB = $(SHARED_OUTDIR)/libmemenv.a
 endif

-$(SHARED3):
-	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3) $(LIBS)
+$(SHARED_OUTDIR)/$(SHARED_LIB3): $(SHARED_LIBOBJECTS)
+	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED_LIB2) $(SHARED_LIBOBJECTS) -o $(SHARED_OUTDIR)/$(SHARED_LIB3) $(LIBS)

 endif  # PLATFORM_SHARED_EXT

-all: $(SHARED) $(LIBRARY)
+all: $(SHARED_LIBS) $(SHARED_PROGRAMS) $(STATIC_OUTDIR)/libleveldb.a $(STATIC_OUTDIR)/libmemenv.a $(STATIC_PROGRAMS)

-check: all $(PROGRAMS) $(TESTS)
-	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
+check: $(STATIC_PROGRAMS)
+	for t in $(notdir $(TESTS)); do echo "***** Running $$t"; $(STATIC_OUTDIR)/$$t || exit 1; done

 clean:
-	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk
-	-rm -rf ios-x86/* ios-arm/*
+	-rm -rf out-static out-shared out-ios-x86 out-ios-arm out-ios-universal
+	-rm -f build_config.mk
+	-rm -rf ios-x86 ios-arm

-$(LIBRARY): $(LIBOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(LIBOBJECTS)
+$(STATIC_OUTDIR):
+	mkdir $@

-db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) $(LDFLAGS) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
+$(STATIC_OUTDIR)/db: | $(STATIC_OUTDIR)
+	mkdir $@

-db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) $(LDFLAGS) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
+$(STATIC_OUTDIR)/helpers/memenv: | $(STATIC_OUTDIR)
+	mkdir -p $@

-db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) $(LDFLAGS) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
+$(STATIC_OUTDIR)/port: | $(STATIC_OUTDIR)
+	mkdir $@

-leveldbutil: db/leveldb_main.o $(LIBOBJECTS)
-	$(CXX) $(LDFLAGS) db/leveldb_main.o $(LIBOBJECTS) -o $@ $(LIBS)
+$(STATIC_OUTDIR)/table: | $(STATIC_OUTDIR)
+	mkdir $@

-arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(STATIC_OUTDIR)/util: | $(STATIC_OUTDIR)
+	mkdir $@

-autocompact_test: db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/autocompact_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+.PHONY: STATIC_OBJDIRS
+STATIC_OBJDIRS: \
+	$(STATIC_OUTDIR)/db \
+	$(STATIC_OUTDIR)/port \
+	$(STATIC_OUTDIR)/table \
+	$(STATIC_OUTDIR)/util \
+	$(STATIC_OUTDIR)/helpers/memenv

-bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SHARED_OUTDIR):
+	mkdir $@

-c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SHARED_OUTDIR)/db: | $(SHARED_OUTDIR)
+	mkdir $@

-cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SHARED_OUTDIR)/helpers/memenv: | $(SHARED_OUTDIR)
+	mkdir -p $@

-coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SHARED_OUTDIR)/port: | $(SHARED_OUTDIR)
+	mkdir $@

-corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SHARED_OUTDIR)/table: | $(SHARED_OUTDIR)
+	mkdir $@

-crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SHARED_OUTDIR)/util: | $(SHARED_OUTDIR)
+	mkdir $@

-db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+.PHONY: SHARED_OBJDIRS
+SHARED_OBJDIRS: \
+	$(SHARED_OUTDIR)/db \
+	$(SHARED_OUTDIR)/port \
+	$(SHARED_OUTDIR)/table \
+	$(SHARED_OUTDIR)/util \
+	$(SHARED_OUTDIR)/helpers/memenv

-dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(DEVICE_OUTDIR):
+	mkdir $@

-env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(DEVICE_OUTDIR)/db: | $(DEVICE_OUTDIR)
+	mkdir $@

-fault_injection_test: db/fault_injection_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/fault_injection_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(DEVICE_OUTDIR)/helpers/memenv: | $(DEVICE_OUTDIR)
+	mkdir -p $@

-filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(DEVICE_OUTDIR)/port: | $(DEVICE_OUTDIR)
+	mkdir $@

-filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(DEVICE_OUTDIR)/table: | $(DEVICE_OUTDIR)
+	mkdir $@

-hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(DEVICE_OUTDIR)/util: | $(DEVICE_OUTDIR)
+	mkdir $@

-issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+.PHONY: DEVICE_OBJDIRS
+DEVICE_OBJDIRS: \
+	$(DEVICE_OUTDIR)/db \
+	$(DEVICE_OUTDIR)/port \
+	$(DEVICE_OUTDIR)/table \
+	$(DEVICE_OUTDIR)/util \
+	$(DEVICE_OUTDIR)/helpers/memenv

-issue200_test: issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SIMULATOR_OUTDIR):
+	mkdir $@

-log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SIMULATOR_OUTDIR)/db: | $(SIMULATOR_OUTDIR)
+	mkdir $@

-recovery_test: db/recovery_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/recovery_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SIMULATOR_OUTDIR)/helpers/memenv: | $(SIMULATOR_OUTDIR)
+	mkdir -p $@

-table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SIMULATOR_OUTDIR)/port: | $(SIMULATOR_OUTDIR)
+	mkdir $@

-skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SIMULATOR_OUTDIR)/table: | $(SIMULATOR_OUTDIR)
+	mkdir $@

-version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+$(SIMULATOR_OUTDIR)/util: | $(SIMULATOR_OUTDIR)
+	mkdir $@

-version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+.PHONY: SIMULATOR_OBJDIRS
+SIMULATOR_OBJDIRS: \
+	$(SIMULATOR_OUTDIR)/db \
+	$(SIMULATOR_OUTDIR)/port \
+	$(SIMULATOR_OUTDIR)/table \
+	$(SIMULATOR_OUTDIR)/util \
+	$(SIMULATOR_OUTDIR)/helpers/memenv

-write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
-
-$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
-	rm -f $@
-	$(AR) -rs $@ $(MEMENVOBJECTS)
-
-memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
-	$(CXX) $(LDFLAGS) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) -o $@ $(LIBS)
+$(STATIC_ALLOBJS): | STATIC_OBJDIRS
+$(DEVICE_ALLOBJS): | DEVICE_OBJDIRS
+$(SIMULATOR_ALLOBJS): | SIMULATOR_OBJDIRS
+$(SHARED_ALLOBJS): | SHARED_OBJDIRS

 ifeq ($(PLATFORM), IOS)
-# For iOS, create universal object files to be used on both the simulator and
+$(DEVICE_OUTDIR)/libleveldb.a: $(DEVICE_LIBOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(DEVICE_LIBOBJECTS)
+
+$(SIMULATOR_OUTDIR)/libleveldb.a: $(SIMULATOR_LIBOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(SIMULATOR_LIBOBJECTS)
+
+$(DEVICE_OUTDIR)/libmemenv.a: $(DEVICE_MEMENVOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(DEVICE_MEMENVOBJECTS)
+
+$(SIMULATOR_OUTDIR)/libmemenv.a: $(SIMULATOR_MEMENVOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(SIMULATOR_MEMENVOBJECTS)
+
+# For iOS, create universal object libraries to be used on both the simulator and
 # a device.
-SIMULATORSDK=$(shell xcrun -sdk iphonesimulator --show-sdk-path)
-DEVICESDK=$(shell xcrun -sdk iphoneos --show-sdk-path)
-IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64
-
-.cc.o:
-	mkdir -p ios-x86/$(dir $@)
-	xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64 -c $< -o ios-x86/$@
-	mkdir -p ios-arm/$(dir $@)
-	xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot "$(DEVICESDK)" $(IOSARCH) -c $< -o ios-arm/$@
-	xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@
-
-.c.o:
-	mkdir -p ios-x86/$(dir $@)
-	xcrun -sdk iphonesimulator $(CC) $(CFLAGS) -isysroot "$(SIMULATORSDK)" -arch i686 -arch x86_64 -c $< -o ios-x86/$@
-	mkdir -p ios-arm/$(dir $@)
-	xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot "$(DEVICESDK)" $(IOSARCH) -c $< -o ios-arm/$@
-	xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@
+$(STATIC_OUTDIR)/libleveldb.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a
+	lipo -create $(DEVICE_OUTDIR)/libleveldb.a $(SIMULATOR_OUTDIR)/libleveldb.a -output $@

+$(STATIC_OUTDIR)/libmemenv.a: $(STATIC_OUTDIR) $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a
+	lipo -create $(DEVICE_OUTDIR)/libmemenv.a $(SIMULATOR_OUTDIR)/libmemenv.a -output $@
 else
-.cc.o:
+$(STATIC_OUTDIR)/libleveldb.a:$(STATIC_LIBOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(STATIC_LIBOBJECTS)
+
+$(STATIC_OUTDIR)/libmemenv.a:$(STATIC_MEMENVOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(STATIC_MEMENVOBJECTS)
+endif
+
+$(SHARED_MEMENVLIB):$(SHARED_MEMENVOBJECTS)
+	rm -f $@
+	$(AR) -rs $@ $(SHARED_MEMENVOBJECTS)
+
+$(STATIC_OUTDIR)/db_bench:db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_bench.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/db_bench_sqlite3:doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_sqlite3.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lsqlite3 $(LIBS)
+
+$(STATIC_OUTDIR)/db_bench_tree_db:doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) doc/bench/db_bench_tree_db.cc $(STATIC_LIBOBJECTS) $(TESTUTIL) -o $@ -lkyotocabinet $(LIBS)
+
+$(STATIC_OUTDIR)/leveldbutil:db/leveldbutil.cc $(STATIC_LIBOBJECTS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/leveldbutil.cc $(STATIC_LIBOBJECTS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/arena_test:util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/arena_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/autocompact_test:db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/autocompact_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/bloom_test:util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/bloom_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/c_test:$(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/db/c_test.o $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/cache_test:util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/cache_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/coding_test:util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/coding_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/corruption_test:db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/corruption_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/crc32c_test:util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/crc32c_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/db_test:db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/db_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/dbformat_test:db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/dbformat_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/env_test:util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/env_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/fault_injection_test:db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/fault_injection_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/filename_test:db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/filename_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/filter_block_test:table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) table/filter_block_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/hash_test:util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) util/hash_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/issue178_test:issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue178_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/issue200_test:issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) issues/issue200_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/log_test:db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/log_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/recovery_test:db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/recovery_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/table_test:table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) table/table_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/skiplist_test:db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/skiplist_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/version_edit_test:db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_edit_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/version_set_test:db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/version_set_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/write_batch_test:db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) $(CXXFLAGS) db/write_batch_test.cc $(STATIC_LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
+$(STATIC_OUTDIR)/memenv_test:$(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS)
+	$(XCRUN) $(CXX) $(LDFLAGS) $(STATIC_OUTDIR)/helpers/memenv/memenv_test.o $(STATIC_OUTDIR)/libmemenv.a $(STATIC_OUTDIR)/libleveldb.a $(TESTHARNESS) -o $@ $(LIBS)
+
+$(SHARED_OUTDIR)/db_bench:$(SHARED_OUTDIR)/db/db_bench.o $(SHARED_LIBS) $(TESTUTIL)
+	$(XCRUN) $(CXX) $(LDFLAGS) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SHARED_OUTDIR)/db/db_bench.o $(TESTUTIL) $(SHARED_OUTDIR)/$(SHARED_LIB3) -o $@ $(LIBS)
+
+.PHONY: run-shared
+run-shared: $(SHARED_OUTDIR)/db_bench
+	LD_LIBRARY_PATH=$(SHARED_OUTDIR) $(SHARED_OUTDIR)/db_bench
+
+$(SIMULATOR_OUTDIR)/%.o: %.cc
+	xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
+
+$(DEVICE_OUTDIR)/%.o: %.cc
+	xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
+
+$(SIMULATOR_OUTDIR)/%.o: %.c
+	xcrun -sdk iphonesimulator $(CC) $(CFLAGS) $(SIMULATOR_CFLAGS) -c $< -o $@
+
+$(DEVICE_OUTDIR)/%.o: %.c
+	xcrun -sdk iphoneos $(CC) $(CFLAGS) $(DEVICE_CFLAGS) -c $< -o $@
+
+$(STATIC_OUTDIR)/%.o: %.cc
 	$(CXX) $(CXXFLAGS) -c $< -o $@

-.c.o:
+$(STATIC_OUTDIR)/%.o: %.c
 	$(CC) $(CFLAGS) -c $< -o $@
-endif
+
+$(SHARED_OUTDIR)/%.o: %.cc
+	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
+
+$(SHARED_OUTDIR)/%.o: %.c
+	$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 **LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**

+[![Build Status](https://travis-ci.org/google/leveldb.svg?branch=master)](https://travis-ci.org/google/leveldb)
+
 Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)

 # Features
@@ -10,9 +12,11 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
  * Multiple changes can be made in one atomic batch.
  * Users can create a transient snapshot to get a consistent view of data.
  * Forward and backward iteration is supported over the data.
-  * Data is automatically compressed using the [Snappy compression library](http://code.google.com/p/snappy).
+  * Data is automatically compressed using the [Snappy compression library](http://google.github.io/snappy/).
  * External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
-  * [Detailed documentation](http://htmlpreview.github.io/?https://github.com/google/leveldb/blob/master/doc/index.html) about how to use the library is included with the source code.
+
+# Documentation
+  [LevelDB library documentation](https://rawgit.com/google/leveldb/master/doc/index.html) is online and bundled with the source code.


 # Limitations
@@ -20,6 +24,37 @@ Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
  * Only a single process (possibly multi-threaded) can access a particular database at a time.
  * There is no client-server support builtin to the library.  An application that needs such support will have to wrap their own server around the library.

+# Contributing to the leveldb Project
+The leveldb project welcomes contributions. leveldb's primary goal is to be
+a reliable and fast key/value store. Changes that are in line with the
+features/limitations outlined above, and meet the requirements below,
+will be considered.
+
+Contribution requirements:
+
+1. **POSIX only**. We _generally_ will only accept changes that are both
+   compiled, and tested on a POSIX platform - usually Linux. Very small
+   changes will sometimes be accepted, but consider that more of an
+   exception than the rule.
+
+2. **Stable API**. We strive very hard to maintain a stable API. Changes that
+   require changes for projects using leveldb _might_ be rejected without
+   sufficient benefit to the project.
+
+3. **Tests**: All changes must be accompanied by a new (or changed) test, or
+   a sufficient explanation as to why a new (or changed) test is not required.
+
+## Submitting a Pull Request
+Before any pull request will be accepted the author must first sign a
+Contributor License Agreement (CLA) at https://cla.developers.google.com/.
+
+In order to keep the commit timeline linear
+[squash](https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Squashing-Commits)
+your changes down to a single commit and [rebase](https://git-scm.com/docs/git-rebase)
+on google/leveldb/master. This keeps the commit timeline linear and more easily sync'ed
+with the internal repository at Google. More information at GitHub's
+[About Git rebase](https://help.github.com/articles/about-git-rebase/) page.
+
 # Performance

 Here is a performance report (with explanations) from the run of the
--- a/2
+++ b/2
@@ -161,7 +161,7 @@ DIRS="$PREFIX/db $PREFIX/util $PREFIX/table"
 set -f # temporarily disable globbing so that our patterns aren't expanded
 PRUNE_TEST="-name *test*.cc -prune"
 PRUNE_BENCH="-name *_bench.cc -prune"
-PRUNE_TOOL="-name leveldb_main.cc -prune"
+PRUNE_TOOL="-name leveldbutil.cc -prune"
 PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_TOOL -o -name '*.cc' -print | sort | sed "s,^$PREFIX/,," | tr "\n" " "`

 set +f # re-enable globbing
--- a/db/db_impl.cc
+++ b/db/db_impl.cc
@@ -851,8 +851,9 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact,
    delete iter;
    if (s.ok()) {
      Log(options_.info_log,
-          "Generated table #%llu: %lld keys, %lld bytes",
+          "Generated table #%llu@%d: %lld keys, %lld bytes",
          (unsigned long long) output_number,
+          compact->compaction->level(),
          (unsigned long long) current_entries,
          (unsigned long long) current_bytes);
    }
--- a/db/leveldb_main.cc
+++ b/db/leveldb_main.cc
--- a/db/log_reader.cc
+++ b/db/log_reader.cc
@@ -25,7 +25,8 @@ Reader::Reader(SequentialFile* file, Reporter* reporter, bool checksum,
      eof_(false),
      last_record_offset_(0),
      end_of_buffer_offset_(0),
-      initial_offset_(initial_offset) {
+      initial_offset_(initial_offset),
+      resyncing_(initial_offset > 0) {
 }

 Reader::~Reader() {
@@ -72,8 +73,25 @@ bool Reader::ReadRecord(Slice* record, std::string* scratch) {

  Slice fragment;
  while (true) {
-    uint64_t physical_record_offset = end_of_buffer_offset_ - buffer_.size();
    const unsigned int record_type = ReadPhysicalRecord(&fragment);
+
+    // ReadPhysicalRecord may have only had an empty trailer remaining in its
+    // internal buffer. Calculate the offset of the next physical record now
+    // that it has returned, properly accounting for its header size.
+    uint64_t physical_record_offset =
+        end_of_buffer_offset_ - buffer_.size() - kHeaderSize - fragment.size();
+
+    if (resyncing_) {
+      if (record_type == kMiddleType) {
+        continue;
+      } else if (record_type == kLastType) {
+        resyncing_ = false;
+        continue;
+      } else {
+        resyncing_ = false;
+      }
+    }
+
    switch (record_type) {
      case kFullType:
        if (in_fragmented_record) {
--- a/db/log_reader.h
+++ b/db/log_reader.h
@@ -73,6 +73,11 @@ class Reader {
  // Offset at which to start looking for the first record to return
  uint64_t const initial_offset_;

+  // True if we are resynchronizing after a seek (initial_offset_ > 0). In
+  // particular, a run of kMiddleType and kLastType records can be silently
+  // skipped in this mode
+  bool resyncing_;
+
  // Extend record types with the following special values
  enum {
    kEof = kMaxRecordType + 1,
--- a/db/log_test.cc
+++ b/db/log_test.cc
@@ -79,7 +79,7 @@ class LogTest {
    virtual Status Skip(uint64_t n) {
      if (n > contents_.size()) {
        contents_.clear();
-        return Status::NotFound("in-memory file skipepd past end");
+        return Status::NotFound("in-memory file skipped past end");
      }

      contents_.remove_prefix(n);
@@ -105,21 +105,23 @@ class LogTest {
  ReportCollector report_;
  bool reading_;
  Writer* writer_;
-  Reader reader_;
+  Reader* reader_;

  // Record metadata for testing initial offset functionality
  static size_t initial_offset_record_sizes_[];
  static uint64_t initial_offset_last_record_offsets_[];
+  static int num_initial_offset_records_;

 public:
  LogTest() : reading_(false),
              writer_(new Writer(&dest_)),
-              reader_(&source_, &report_, true/*checksum*/,
-                      0/*initial_offset*/) {
+              reader_(new Reader(&source_, &report_, true/*checksum*/,
+                      0/*initial_offset*/)) {
  }

  ~LogTest() {
    delete writer_;
+    delete reader_;
  }

  void ReopenForAppend() {
@@ -143,7 +145,7 @@ class LogTest {
    }
    std::string scratch;
    Slice record;
-    if (reader_.ReadRecord(&record, &scratch)) {
+    if (reader_->ReadRecord(&record, &scratch)) {
      return record.ToString();
    } else {
      return "EOF";
@@ -191,13 +193,18 @@ class LogTest {
  }

  void WriteInitialOffsetLog() {
-    for (int i = 0; i < 4; i++) {
+    for (int i = 0; i < num_initial_offset_records_; i++) {
      std::string record(initial_offset_record_sizes_[i],
                         static_cast<char>('a' + i));
      Write(record);
    }
  }

+  void StartReadingAt(uint64_t initial_offset) {
+    delete reader_;
+    reader_ = new Reader(&source_, &report_, true/*checksum*/, initial_offset);
+  }
+
  void CheckOffsetPastEndReturnsNoRecords(uint64_t offset_past_end) {
    WriteInitialOffsetLog();
    reading_ = true;
@@ -217,32 +224,48 @@ class LogTest {
    source_.contents_ = Slice(dest_.contents_);
    Reader* offset_reader = new Reader(&source_, &report_, true/*checksum*/,
                                       initial_offset);
-    Slice record;
-    std::string scratch;
-    ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
-    ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
-              record.size());
-    ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
-              offset_reader->LastRecordOffset());
-    ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
+
+    // Read all records from expected_record_offset through the last one.
+    ASSERT_LT(expected_record_offset, num_initial_offset_records_);
+    for (; expected_record_offset < num_initial_offset_records_;
+         ++expected_record_offset) {
+      Slice record;
+      std::string scratch;
+      ASSERT_TRUE(offset_reader->ReadRecord(&record, &scratch));
+      ASSERT_EQ(initial_offset_record_sizes_[expected_record_offset],
+                record.size());
+      ASSERT_EQ(initial_offset_last_record_offsets_[expected_record_offset],
+                offset_reader->LastRecordOffset());
+      ASSERT_EQ((char)('a' + expected_record_offset), record.data()[0]);
+    }
    delete offset_reader;
  }
-
 };

 size_t LogTest::initial_offset_record_sizes_[] =
    {10000,  // Two sizable records in first block
     10000,
     2 * log::kBlockSize - 1000,  // Span three blocks
-     1};
+     1,
+     13716,  // Consume all but two bytes of block 3.
+     log::kBlockSize - kHeaderSize, // Consume the entirety of block 4.
+    };

 uint64_t LogTest::initial_offset_last_record_offsets_[] =
    {0,
     kHeaderSize + 10000,
     2 * (kHeaderSize + 10000),
     2 * (kHeaderSize + 10000) +
-         (2 * log::kBlockSize - 1000) + 3 * kHeaderSize};
+         (2 * log::kBlockSize - 1000) + 3 * kHeaderSize,
+     2 * (kHeaderSize + 10000) +
+         (2 * log::kBlockSize - 1000) + 3 * kHeaderSize
+         + kHeaderSize + 1,
+     3 * log::kBlockSize,
+    };

+// LogTest::initial_offset_last_record_offsets_ must be defined before this.
+int LogTest::num_initial_offset_records_ =
+    sizeof(LogTest::initial_offset_last_record_offsets_)/sizeof(uint64_t);

 TEST(LogTest, Empty) {
  ASSERT_EQ("EOF", Read());
@@ -463,6 +486,22 @@ TEST(LogTest, PartialLastIsIgnored) {
  ASSERT_EQ(0, DroppedBytes());
 }

+TEST(LogTest, SkipIntoMultiRecord) {
+  // Consider a fragmented record:
+  //    first(R1), middle(R1), last(R1), first(R2)
+  // If initial_offset points to a record after first(R1) but before first(R2)
+  // incomplete fragment errors are not actual errors, and must be suppressed
+  // until a new first or full record is encountered.
+  Write(BigString("foo", 3*kBlockSize));
+  Write("correct");
+  StartReadingAt(kBlockSize);
+
+  ASSERT_EQ("correct", Read());
+  ASSERT_EQ("", ReportMessage());
+  ASSERT_EQ(0, DroppedBytes());
+  ASSERT_EQ("EOF", Read());
+}
+
 TEST(LogTest, ErrorJoinsRecords) {
  // Consider two fragmented records:
  //    first(R1) last(R1) first(R2) last(R2)
@@ -532,6 +571,10 @@ TEST(LogTest, ReadFourthStart) {
      3);
 }

+TEST(LogTest, ReadInitialOffsetIntoBlockPadding) {
+  CheckInitialOffsetRecord(3 * log::kBlockSize - 3, 5);
+}
+
 TEST(LogTest, ReadEnd) {
  CheckOffsetPastEndReturnsNoRecords(0);
 }
--- a/db/memtable.h
+++ b/db/memtable.h
@@ -36,10 +36,7 @@ class MemTable {
  }

  // Returns an estimate of the number of bytes of data in use by this
-  // data structure.
-  //
-  // REQUIRES: external synchronization to prevent simultaneous
-  // operations on the same MemTable.
+  // data structure. It is safe to call when MemTable is being modified.
  size_t ApproximateMemoryUsage();

  // Return an iterator that yields the contents of the memtable.
--- a/db/recovery_test.cc
+++ b/db/recovery_test.cc
@@ -104,10 +104,10 @@ class RecoveryTest {
    return GetFiles(kLogFile)[0];
  }

-  std::vector<std::uint64_t> GetFiles(FileType t) {
+  std::vector<uint64_t> GetFiles(FileType t) {
    std::vector<std::string> filenames;
    ASSERT_OK(env_->GetChildren(dbname_, &filenames));
-    std::vector<std::uint64_t> result;
+    std::vector<uint64_t> result;
    for (size_t i = 0; i < filenames.size(); i++) {
      uint64_t number;
      FileType type;
--- a/db/skiplist_test.cc
+++ b/db/skiplist_test.cc
@@ -250,7 +250,7 @@ class ConcurrentTest {
        // Note that generation 0 is never inserted, so it is ok if
        // <*,0,*> is missing.
        ASSERT_TRUE((gen(pos) == 0) ||
-                    (gen(pos) > initial_state.Get(key(pos)))
+                    (gen(pos) > static_cast<Key>(initial_state.Get(key(pos))))
                    ) << "key: " << key(pos)
                      << "; gen: " << gen(pos)
                      << "; initgen: "
--- a/doc/index.html
+++ b/doc/index.html
@@ -22,7 +22,7 @@ directory.  The following example shows how to open a database,
 creating it if necessary:
 <p>
 <pre>
-  #include &lt;assert&gt;
+  #include &lt;cassert&gt;
  #include "leveldb/db.h"

  leveldb::DB* db;
--- a/include/leveldb/db.h
+++ b/include/leveldb/db.h
@@ -14,7 +14,7 @@ namespace leveldb {

 // Update Makefile if you change these
 static const int kMajorVersion = 1;
-static const int kMinorVersion = 18;
+static const int kMinorVersion = 19;

 struct Options;
 struct ReadOptions;
--- a/include/leveldb/status.h
+++ b/include/leveldb/status.h
@@ -63,6 +63,9 @@ class Status {
  // Returns true iff the status indicates a NotSupportedError.
  bool IsNotSupportedError() const { return code() == kNotSupported; }

+  // Returns true iff the status indicates an InvalidArgument.
+  bool IsInvalidArgument() const { return code() == kInvalidArgument; }
+
  // Return a string representation of this status suitable for printing.
  // Returns the string "OK" for success.
  std::string ToString() const;
--- a/port/atomic_pointer.h
+++ b/port/atomic_pointer.h
@@ -39,6 +39,8 @@
 #define ARCH_CPU_ARM64_FAMILY 1
 #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__)
 #define ARCH_CPU_PPC_FAMILY 1
+#elif defined(__mips__)
+#define ARCH_CPU_MIPS_FAMILY 1
 #endif

 namespace leveldb {
@@ -110,6 +112,13 @@ inline void MemoryBarrier() {
 }
 #define LEVELDB_HAVE_MEMORY_BARRIER

+// MIPS
+#elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__)
+inline void MemoryBarrier() {
+  __asm__ __volatile__("sync" : : : "memory");
+}
+#define LEVELDB_HAVE_MEMORY_BARRIER
+
 #endif

 // AtomicPointer built using platform-specific MemoryBarrier()
--- a/port/port_posix.cc
+++ b/port/port_posix.cc
@@ -7,7 +7,6 @@
 #include <cstdlib>
 #include <stdio.h>
 #include <string.h>
-#include "util/logging.h"

 namespace leveldb {
 namespace port {
--- a/table/filter_block.cc
+++ b/table/filter_block.cc
@@ -68,7 +68,7 @@ void FilterBlockBuilder::GenerateFilter() {

  // Generate filter for current set of keys and append to result_.
  filter_offsets_.push_back(result_.size());
-  policy_->CreateFilter(&tmp_keys_[0], num_keys, &result_);
+  policy_->CreateFilter(&tmp_keys_[0], static_cast<int>(num_keys), &result_);

  tmp_keys_.clear();
  keys_.clear();
@@ -97,7 +97,7 @@ bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) {
  if (index < num_) {
    uint32_t start = DecodeFixed32(offset_ + index*4);
    uint32_t limit = DecodeFixed32(offset_ + index*4 + 4);
-    if (start <= limit && limit <= (offset_ - data_)) {
+    if (start <= limit && limit <= static_cast<size_t>(offset_ - data_)) {
      Slice filter = Slice(data_ + start, limit - start);
      return policy_->KeyMayMatch(key, filter);
    } else if (start == limit) {
--- a/table/format.cc
+++ b/table/format.cc
@@ -30,15 +30,14 @@ Status BlockHandle::DecodeFrom(Slice* input) {
 }

 void Footer::EncodeTo(std::string* dst) const {
-#ifndef NDEBUG
  const size_t original_size = dst->size();
-#endif
  metaindex_handle_.EncodeTo(dst);
  index_handle_.EncodeTo(dst);
  dst->resize(2 * BlockHandle::kMaxEncodedLength);  // Padding
  PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber & 0xffffffffu));
  PutFixed32(dst, static_cast<uint32_t>(kTableMagicNumber >> 32));
  assert(dst->size() == original_size + kEncodedLength);
+  (void)original_size;  // Disable unused variable warning.
 }

 Status Footer::DecodeFrom(Slice* input) {
--- a/table/table.cc
+++ b/table/table.cc
@@ -82,7 +82,7 @@ Status Table::Open(const Options& options,
    *table = new Table(rep);
    (*table)->ReadMeta(footer);
  } else {
-    if (index_block) delete index_block;
+    delete index_block;
  }

  return s;
--- a/table/table_test.cc
+++ b/table/table_test.cc
@@ -853,12 +853,20 @@ TEST(TableTest, ApproximateOffsetOfCompressed) {
  options.compression = kSnappyCompression;
  c.Finish(options, &keys, &kvmap);

-  ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"),       0,      0));
-  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"),       0,      0));
-  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"),       0,      0));
-  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"),    2000,   3000));
-  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"),    2000,   3000));
-  ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"),    4000,   6000));
+  // Expected upper and lower bounds of space used by compressible strings.
+  static const int kSlop = 1000;  // Compressor effectiveness varies.
+  const int expected = 2500;  // 10000 * compression ratio (0.25)
+  const int min_z = expected - kSlop;
+  const int max_z = expected + kSlop;
+
+  ASSERT_TRUE(Between(c.ApproximateOffsetOf("abc"), 0, kSlop));
+  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k01"), 0, kSlop));
+  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k02"), 0, kSlop));
+  // Have now emitted a large compressible string, so adjust expected offset.
+  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k03"), min_z, max_z));
+  ASSERT_TRUE(Between(c.ApproximateOffsetOf("k04"), min_z, max_z));
+  // Have now emitted two large compressible strings, so adjust expected offset.
+  ASSERT_TRUE(Between(c.ApproximateOffsetOf("xyz"), 2 * min_z, 2 * max_z));
 }

 }  // namespace leveldb
--- a/util/arena.cc
+++ b/util/arena.cc
@@ -9,8 +9,7 @@ namespace leveldb {

 static const int kBlockSize = 4096;

-Arena::Arena() {
-  blocks_memory_ = 0;
+Arena::Arena() : memory_usage_(0) {
  alloc_ptr_ = NULL;  // First allocation will allocate a block
  alloc_bytes_remaining_ = 0;
 }
@@ -60,8 +59,9 @@ char* Arena::AllocateAligned(size_t bytes) {

 char* Arena::AllocateNewBlock(size_t block_bytes) {
  char* result = new char[block_bytes];
-  blocks_memory_ += block_bytes;
  blocks_.push_back(result);
+  memory_usage_.NoBarrier_Store(
+      reinterpret_cast<void*>(MemoryUsage() + block_bytes + sizeof(char*)));
  return result;
 }

--- a/util/arena.h
+++ b/util/arena.h
@@ -9,6 +9,7 @@
 #include <assert.h>
 #include <stddef.h>
 #include <stdint.h>
+#include "port/port.h"

 namespace leveldb {

@@ -24,10 +25,9 @@ class Arena {
  char* AllocateAligned(size_t bytes);

  // Returns an estimate of the total memory usage of data allocated
-  // by the arena (including space allocated but not yet used for user
-  // allocations).
+  // by the arena.
  size_t MemoryUsage() const {
-    return blocks_memory_ + blocks_.capacity() * sizeof(char*);
+    return reinterpret_cast<uintptr_t>(memory_usage_.NoBarrier_Load());
  }

 private:
@@ -41,8 +41,8 @@ class Arena {
  // Array of new[] allocated memory blocks
  std::vector<char*> blocks_;

-  // Bytes of memory in blocks allocated so far
-  size_t blocks_memory_;
+  // Total memory usage of the arena.
+  port::AtomicPointer memory_usage_;

  // No copying allowed
  Arena(const Arena&);
--- a/util/bloom.cc
+++ b/util/bloom.cc
@@ -47,7 +47,7 @@ class BloomFilterPolicy : public FilterPolicy {
    dst->resize(init_size + bytes, 0);
    dst->push_back(static_cast<char>(k_));  // Remember # of probes in filter
    char* array = &(*dst)[init_size];
-    for (size_t i = 0; i < n; i++) {
+    for (int i = 0; i < n; i++) {
      // Use double-hashing to generate a sequence of hash values.
      // See analysis in [Kirsch,Mitzenmacher 2006].
      uint32_t h = BloomHash(keys[i]);
--- a/util/bloom_test.cc
+++ b/util/bloom_test.cc
@@ -46,7 +46,8 @@ class BloomTest {
      key_slices.push_back(Slice(keys_[i]));
    }
    filter_.clear();
-    policy_->CreateFilter(&key_slices[0], key_slices.size(), &filter_);
+    policy_->CreateFilter(&key_slices[0], static_cast<int>(key_slices.size()),
+                          &filter_);
    keys_.clear();
    if (kVerbose >= 2) DumpFilter();
  }
--- a/util/cache.cc
+++ b/util/cache.cc
@@ -19,6 +19,23 @@ Cache::~Cache() {
 namespace {

 // LRU cache implementation
+//
+// Cache entries have an "in_cache" boolean indicating whether the cache has a
+// reference on the entry.  The only ways that this can become false without the
+// entry being passed to its "deleter" are via Erase(), via Insert() when
+// an element with a duplicate key is inserted, or on destruction of the cache.
+//
+// The cache keeps two linked lists of items in the cache.  All items in the
+// cache are in one list or the other, and never both.  Items still referenced
+// by clients but erased from the cache are in neither list.  The lists are:
+// - in-use:  contains the items currently referenced by clients, in no
+//   particular order.  (This list is used for invariant checking.  If we
+//   removed the check, elements that would otherwise be on this list could be
+//   left as disconnected singleton lists.)
+// - LRU:  contains the items not currently referenced by clients, in LRU order
+// Elements are moved between these lists by the Ref() and Unref() methods,
+// when they detect an element in the cache acquiring or losing its only
+// external reference.

 // An entry is a variable length heap-allocated structure.  Entries
 // are kept in a circular doubly linked list ordered by access time.
@@ -30,7 +47,8 @@ struct LRUHandle {
  LRUHandle* prev;
  size_t charge;      // TODO(opt): Only allow uint32_t?
  size_t key_length;
-  uint32_t refs;
+  bool in_cache;      // Whether entry is in the cache.
+  uint32_t refs;      // References, including cache reference, if present.
  uint32_t hash;      // Hash of key(); used for fast sharding and comparisons
  char key_data[1];   // Beginning of key

@@ -155,8 +173,10 @@ class LRUCache {

 private:
  void LRU_Remove(LRUHandle* e);
-  void LRU_Append(LRUHandle* e);
+  void LRU_Append(LRUHandle*list, LRUHandle* e);
+  void Ref(LRUHandle* e);
  void Unref(LRUHandle* e);
+  bool FinishErase(LRUHandle* e);

  // Initialized before use.
  size_t capacity_;
@@ -167,34 +187,55 @@ class LRUCache {

  // Dummy head of LRU list.
  // lru.prev is newest entry, lru.next is oldest entry.
+  // Entries have refs==1 and in_cache==true.
  LRUHandle lru_;

+  // Dummy head of in-use list.
+  // Entries are in use by clients, and have refs >= 2 and in_cache==true.
+  LRUHandle in_use_;
+
  HandleTable table_;
 };

 LRUCache::LRUCache()
    : usage_(0) {
-  // Make empty circular linked list
+  // Make empty circular linked lists.
  lru_.next = &lru_;
  lru_.prev = &lru_;
+  in_use_.next = &in_use_;
+  in_use_.prev = &in_use_;
 }

 LRUCache::~LRUCache() {
+  assert(in_use_.next == &in_use_);  // Error if caller has an unreleased handle
  for (LRUHandle* e = lru_.next; e != &lru_; ) {
    LRUHandle* next = e->next;
-    assert(e->refs == 1);  // Error if caller has an unreleased handle
+    assert(e->in_cache);
+    e->in_cache = false;
+    assert(e->refs == 1);  // Invariant of lru_ list.
    Unref(e);
    e = next;
  }
 }

+void LRUCache::Ref(LRUHandle* e) {
+  if (e->refs == 1 && e->in_cache) {  // If on lru_ list, move to in_use_ list.
+    LRU_Remove(e);
+    LRU_Append(&in_use_, e);
+  }
+  e->refs++;
+}
+
 void LRUCache::Unref(LRUHandle* e) {
  assert(e->refs > 0);
  e->refs--;
-  if (e->refs <= 0) {
-    usage_ -= e->charge;
+  if (e->refs == 0) { // Deallocate.
+    assert(!e->in_cache);
    (*e->deleter)(e->key(), e->value);
    free(e);
+  } else if (e->in_cache && e->refs == 1) {  // No longer in use; move to lru_ list.
+    LRU_Remove(e);
+    LRU_Append(&lru_, e);
  }
 }

@@ -203,10 +244,10 @@ void LRUCache::LRU_Remove(LRUHandle* e) {
  e->prev->next = e->next;
 }

-void LRUCache::LRU_Append(LRUHandle* e) {
-  // Make "e" newest entry by inserting just before lru_
-  e->next = &lru_;
-  e->prev = lru_.prev;
+void LRUCache::LRU_Append(LRUHandle* list, LRUHandle* e) {
+  // Make "e" newest entry by inserting just before *list
+  e->next = list;
+  e->prev = list->prev;
  e->prev->next = e;
  e->next->prev = e;
 }
@@ -215,9 +256,7 @@ Cache::Handle* LRUCache::Lookup(const Slice& key, uint32_t hash) {
  MutexLock l(&mutex_);
  LRUHandle* e = table_.Lookup(key, hash);
  if (e != NULL) {
-    e->refs++;
-    LRU_Remove(e);
-    LRU_Append(e);
+    Ref(e);
  }
  return reinterpret_cast<Cache::Handle*>(e);
 }
@@ -239,46 +278,57 @@ Cache::Handle* LRUCache::Insert(
  e->charge = charge;
  e->key_length = key.size();
  e->hash = hash;
-  e->refs = 2;  // One from LRUCache, one for the returned handle
+  e->in_cache = false;
+  e->refs = 1;  // for the returned handle.
  memcpy(e->key_data, key.data(), key.size());
-  LRU_Append(e);
-  usage_ += charge;

-  LRUHandle* old = table_.Insert(e);
-  if (old != NULL) {
-    LRU_Remove(old);
-    Unref(old);
-  }
+  if (capacity_ > 0) {
+    e->refs++;  // for the cache's reference.
+    e->in_cache = true;
+    LRU_Append(&in_use_, e);
+    usage_ += charge;
+    FinishErase(table_.Insert(e));
+  } // else don't cache.  (Tests use capacity_==0 to turn off caching.)

  while (usage_ > capacity_ && lru_.next != &lru_) {
    LRUHandle* old = lru_.next;
-    LRU_Remove(old);
-    table_.Remove(old->key(), old->hash);
-    Unref(old);
+    assert(old->refs == 1);
+    bool erased = FinishErase(table_.Remove(old->key(), old->hash));
+    if (!erased) {  // to avoid unused variable when compiled NDEBUG
+      assert(erased);
+    }
  }

  return reinterpret_cast<Cache::Handle*>(e);
 }

-void LRUCache::Erase(const Slice& key, uint32_t hash) {
-  MutexLock l(&mutex_);
-  LRUHandle* e = table_.Remove(key, hash);
+// If e != NULL, finish removing *e from the cache; it has already been removed
+// from the hash table.  Return whether e != NULL.  Requires mutex_ held.
+bool LRUCache::FinishErase(LRUHandle* e) {
  if (e != NULL) {
+    assert(e->in_cache);
    LRU_Remove(e);
+    e->in_cache = false;
+    usage_ -= e->charge;
    Unref(e);
  }
+  return e != NULL;
+}
+
+void LRUCache::Erase(const Slice& key, uint32_t hash) {
+  MutexLock l(&mutex_);
+  FinishErase(table_.Remove(key, hash));
 }

 void LRUCache::Prune() {
  MutexLock l(&mutex_);
-  for (LRUHandle* e = lru_.next; e != &lru_; ) {
-    LRUHandle* next = e->next;
-    if (e->refs == 1) {
-      table_.Remove(e->key(), e->hash);
-      LRU_Remove(e);
-      Unref(e);
+  while (lru_.next != &lru_) {
+    LRUHandle* e = lru_.next;
+    assert(e->refs == 1);
+    bool erased = FinishErase(table_.Remove(e->key(), e->hash));
+    if (!erased) {  // to avoid unused variable when compiled NDEBUG
+      assert(erased);
    }
-    e = next;
  }
 }

--- a/util/cache_test.cc
+++ b/util/cache_test.cc
@@ -59,6 +59,11 @@ class CacheTest {
                                   &CacheTest::Deleter));
  }

+  Cache::Handle* InsertAndReturnHandle(int key, int value, int charge = 1) {
+    return cache_->Insert(EncodeKey(key), EncodeValue(value), charge,
+                          &CacheTest::Deleter);
+  }
+
  void Erase(int key) {
    cache_->Erase(EncodeKey(key));
  }
@@ -135,8 +140,11 @@ TEST(CacheTest, EntriesArePinned) {
 TEST(CacheTest, EvictionPolicy) {
  Insert(100, 101);
  Insert(200, 201);
+  Insert(300, 301);
+  Cache::Handle* h = cache_->Lookup(EncodeKey(300));

-  // Frequently used entry must be kept around
+  // Frequently used entry must be kept around,
+  // as must things that are still in use.
  for (int i = 0; i < kCacheSize + 100; i++) {
    Insert(1000+i, 2000+i);
    ASSERT_EQ(2000+i, Lookup(1000+i));
@@ -144,6 +152,25 @@ TEST(CacheTest, EvictionPolicy) {
  }
  ASSERT_EQ(101, Lookup(100));
  ASSERT_EQ(-1, Lookup(200));
+  ASSERT_EQ(301, Lookup(300));
+  cache_->Release(h);
+}
+
+TEST(CacheTest, UseExceedsCacheSize) {
+  // Overfill the cache, keeping handles on all inserted entries.
+  std::vector<Cache::Handle*> h;
+  for (int i = 0; i < kCacheSize + 100; i++) {
+    h.push_back(InsertAndReturnHandle(1000+i, 2000+i));
+  }
+
+  // Check that all the entries can be found in the cache.
+  for (int i = 0; i < h.size(); i++) {
+    ASSERT_EQ(2000+i, Lookup(1000+i));
+  }
+
+  for (int i = 0; i < h.size(); i++) {
+    cache_->Release(h[i]);
+  }
 }

 TEST(CacheTest, HeavyEntries) {
Author	SHA1	Message	Date
cmumford	3080a45b62	Increase leveldb version to 1.19. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=129930720	2016-08-11 07:33:30 -07:00
sanjay	fa6dc010a2	A zippy change broke test assumptions about the size of compressed output. Fix the tests by allowing more slop in zippy's behavior. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=123432472	2016-07-06 09:16:11 -07:00
m3b	06a191b8de	fix problems in LevelDB's caching code Background: LevelDB uses a cache (util/cache.h, util/cache.cc) of (key,value) pairs for two purposes: - a cache of (table, file handle) pairs - a cache of blocks The cache places the (key,value) pairs in a reference-counted wrapper. When it returns a value, it returns a reference to this wrapper. When the client has finished using the reference and its enclosed (key,value), it calls Release() to decrement the reference count. Each (key,value) pair has an associated resource usage. The cache maintains the sum of the usages of the elements it holds, and removes values as needed to keep the sum below a capacity threshold. It maintains an LRU list so that it will remove the least-recently used elements first. The max_open_files option to LevelDB sets the size of the cache of (table, file handle) pairs. The option is not used in any other way. The observed behaviour: If LevelDB at any time used more file handles concurrently than the cache size set via max_open_files, it attempted to reduce the number by evicting entries from the table cache. This could happen most easily during compaction, and if max_open_files was low. Because the handles were in use, their reference count did not drop to zero, and so the usage sum in the cache was not modified by the evictions. Subsequent Insert() calls returned valid handles, but their entries were immediately evicted from the cache, which though empty still acted as though full. As a result, there was effectively no caching, and the number of open file handles rose []ly until it hit system-imposed limits and the process died. If one set max_open_files lower, the cache was more likely to exhibit this beahviour, and cause the process to run out of file descriptors. That is, max_open_files acted in almost exactly the opposite manner from what was intended. The problems: 1. The cache kept all elements on its LRU list eligible for capacity eviction---even those with outstanding references from clients. This was ineffective in reducing resource consumption because there was an outstanding reference, guaranteeing that the items remained. A secondary issue was that there is no guarantee that these in-use items will be the last things reached in the LRU chain, which actually recorded "least-recently requested" rather than "least-recently used". 2. The sum of usages was decremented not when a (key,value) was evicted from the cache, but when its reference count went to zero. Thus, when things were removed from the cache, either by garbage collection or via Erase(), the usage sum was not necessarily decreased. This allowed the cache to act as though full when it was in fact not, reducing caching effectiveness, and leading to more resources being consumed---the opposite of what the evictions were intended to achieve. 3. (minor) The cache's clients insert items into it by first looking up the key, and inserting only if no value is found. Although the cache has an internal lock, the clients use no locking to ensure atomicity of the Lookup/Insert pair. (see table/table.cc: block_cache->Insert() and db/table_cache.cc: cache_->Insert()). Thus, if two threads Insert() at about the same time, they can both Lookup(), find nothing, and both Insert(). The second Insert() would evict the first value, leaving each thread with a handle on its own version of the data, and with the second version in the cache. It would be better if both threads ended up with a handle on the same (key,value) pair, which implies it must be the first item inserted. This suggests that Insert() should not replace an existing value. This can be made safe with current usage inside LeveDB itself, but this is not easy to change first because Cache is a public interface, so to change the semantics of an existing call might break things, second because Cache is an abstract virtual class, so adding a new abstract virtual method may break other implementations, and third, the new method "insert without replacing" cannot be implemented in terms of the existing methods, so cannot be implemented with a non-abstract default. But fortunately, the effects of this issue are minor, so this issue is not fixed by this change. The changes: The assumption in the fixes is that it is always better to cache entries unless removal from the cache would lead to deallocation. Cache entries now have an "in_cache" boolean indicating whether the cache has a reference on the entry. The only ways that this can become false without the entry being passed to its "deleter" are via Erase(), via Insert() when an element with a duplicate key is inserted, or on destruction of the cache. The cache now keeps two linked lists instead of one. All items in the cache are in one list or the other, and never both. Items still referenced by clients but erased from the cache are in neither list. The lists are: - in-use: contains the items currently referenced by clients, in no particular order. (This list is used for invariant checking. If we removed the check, elements that would otherwise be on this list could be left as disconnected singleton lists.) - LRU: contains the items not currently referenced by clients, in LRU order A new internal Ref() method increments the reference count. If incrementing from 1 to 2 for an item in the cache, it is moved from the LRU list to the in-use list. The Unref() call now moves things from the in-use list to the LRU list if the reference count falls to 1, and the item is in the cache. It no longer adjusts the usage sum. The usage sum now reflects only what is in the cache, rather than including still-referenced items that have been evicted. The LRU_Append() now takes a "list" parameter so that it can be used to append either to the LRU list or the in-use list. Lookup() is modified to use the new Ref() call, rather than adjusting the reference count and LRU chain directly. Insert() eviction code is also modified to adjust the usage sum and the in_cache boolean of the evicted elements. Some LevelDB tests assume that there will be no caching whatsoever if the cache size is set to zero, so this is handled as a special case. A new private method FinishErase() is factored out with the common code from where items are removed from the cache. Erase() is modified to adjust the usage sum and the in_cache boolean of the erased elements, and to use FinishErase(). Prune() is modified to use FinishErase() also, and to make use of the fact that the lru_ list now contains only items with reference count 1. - EvictionPolicy is modified to test that an entry with an outstanding handle is not evicted. This test fails with the old cache.cc. - A new test case UseExceedsCacheSize verifies that even when the cache is overfull of entries with outstanding handles, none are evicted. This test fails with the old cache.cc, and is the key issue that causes file descriptors to run out when the cache size is set too small. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=123247237	2016-07-06 09:15:53 -07:00
John Abd-El-Malek	a7bff697ba	Fix LevelDB build when asserts are enabled in release builds. (#367 ) * Fix LevelDB build when asserts are enabled in release builds. BUG=https://bugs.chromium.org/p/chromium/issues/detail?id=603166 * fix * Add comment	2016-04-15 10:58:27 -07:00
Nicholas Westlake	ea992b467b	Change std::uint64_t to uint64_t (#354 ) -This fixes compile errors with default setup on RHEL 6 systems.	2016-04-12 15:38:09 -07:00
mjwiacek	e84b5bdb5a	This CL fixes a bug encountered when reading records from leveldb files that have been split, as in a [] input task split. Detailed description: Suppose an input split is generated between two leveldb record blocks and the preceding block ends with null padding. A reader that previously read at least 1 record within the first block (before encountering the padding) upon trying to read the next record, will successfully and correctly read the next logical record from the subsequent block, but will return a last record offset pointing to the padding in the first block. When this happened in a [], it resulted in duplicate records being handled at what appeared to be different offsets that were separated by only a few bytes. This behavior is only observed when at least 1 record was read from the first block before encountering the padding. If the initial offset for a reader was within the padding, the correct record offset would be reported, namely the offset within the second block. The tests failed to catch this scenario/bug, because each read test only read a single record with an initial offset. This CL adds an explicit test case for this scenario, and modifies the test structure to read all remaining records in the test case after an initial offset is specified. Thus an initial offset that jumps to record #3, with 5 total records in the test file, will result in reading 2 records, and validating the offset of each of them in order to pass successfully. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=115338487	2016-03-31 15:53:34 -07:00
cmumford	3211343909	Deleted redundant null ptr check prior to delete. Fixes issue #338. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=113439460	2016-03-31 15:53:30 -07:00
Chris Mumford	7306ef856a	Merge pull request #348 from randomascii/master Fix signed/unsigned mismatch on VC++ builds	2016-02-24 14:39:03 -08:00
Bruce Dawson	6b18316d01	Fix signed/unsigned mismatch on VC++ builds	2016-02-19 13:59:19 -08:00
cmumford	adbe3eb073	Putting build artifacts in subdirectory. 1. Object files, libraries, and compiled executables are put into subdirectories. 2. The shared library is linked from individual object files. This provides for greater parallelism on large desktops while at the same time making for easier builds on small (i.e. embedded) systems. Fixes issue #279. 3. One program, db_bench, is compiled using the shared library. 4. The source file for "leveldbutil" was renamed from leveldb_main.cc to leveldbutil.cc. This provides for simpler makefile rules. 5. Because all targets placed the library (libleveldb.a) at the top level, the last platform built (desktop/device) always overwrote any prior artifact. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=113407013	2016-01-29 16:10:00 -08:00
Chris Mumford	2d0320a458	Merge pull request #329 from ralphtheninja/travis-badge Add travis build badge to README	2016-01-15 11:17:41 -08:00
Lars-Magnus Skog	dd1c3c3572	add travis build badge	2016-01-15 18:29:01 +01:00
Chris Mumford	43fcf23af0	Merge pull request #328 from cmumford/master Added a Travis CI build file.	2016-01-14 21:17:21 -08:00
Chris Mumford	9fcae61641	Added a Travis CI build file. This allows for continuous integration builds by travis-ci.org. More information at https://docs.travis-ci.com/user/languages/cpp	2016-01-14 17:41:48 -08:00
Chris Mumford	dac40d25f6	Merge pull request #284 from ideawu/master log compaction output file's level along with number	2016-01-12 11:30:32 -08:00
Chris Mumford	8ec241a3b0	Merge pull request #317 from falvojr/patch-1 Update README.md	2016-01-12 10:52:33 -08:00
Chris Mumford	5d36bedd1c	Merge pull request #272 from vapier/master Fix Android/MIPS build.	2016-01-12 10:47:33 -08:00
cmumford	4753c9b617	Added a contributors section to README.md In preparation for accepting GitHub pull requests this new README section outlines the general criteria that the leveldb project owners will use when accepting external (and internal) project contributions. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=111349899	2016-01-04 13:29:41 -08:00
Chris Mumford	e2446d0848	Merge pull request #275 from paulirish/patch-1 readme: improved documentation link	2015-12-09 15:04:30 -08:00
ssid	706b7f8d43	Resolve race when getting approximate-memory-usage property The write operations in the table happens without holding the mutex lock, but concurrent writes are avoided using "writers_" queue. The Arena::MemoryUsage could access the blocks when write happens. So, the memory usage is cached in atomic word and can be loaded from any thread safely. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=107573379	2015-12-09 11:27:50 -08:00
cmumford	3c9ff3c691	Only compiling TrimSpace on linux. Incorporated change by zmodem at https://github.com/google/leveldb/pull/310 to fix issue #310. This change will only build TrimSace on linux to avoid unused function warning/error. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=105323419	2015-12-09 10:35:07 -08:00
cmumford	f8d205cf89	Including atomic_pointer.h in port_posix A recent CL (104348226) created the port_posix library, but omitted: port/atomic_pointer.h. And when: [] test third_party/leveldb:all was run this error was reported: //third_party/leveldb:port_posix does not depend on a module exporting 'third_party/leveldb/port/atomic_pointer.h' ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=105243399	2015-12-09 10:35:07 -08:00
ndmatthews	889de31a5a	Let LevelDB use xcrun to determine Xcode.app path instead of using a hardcoded path. This allows build agents to select from multiple Xcode installations. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=104859097	2015-12-09 10:34:58 -08:00
ssid	528c2bc6ad	Add "approximate-memory-usage" property to leveldb::DB::GetProperty The approximate RAM usage of the database is calculated from the memory allocated for write buffers and the block cache. This is to give an estimate of memory usage to leveldb clients. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=104222307	2015-12-09 10:34:58 -08:00
tzik	359b6bcec2	Add leveldb::Cache::Prune Prune() drops on-memory read cache of the database, so that the client can relief its memory shortage. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=101335710	2015-12-09 10:34:58 -08:00
pkasting	50e77a8263	Fix size_t/int comparison/conversion issues in leveldb. The create function took \|num_keys\| as an int, but callers and implementers wanted it to function as a size_t (e.g. passing std::vector::size() in, passing it to vector constructors as a size arg, indexing containers by it, etc.). This resulted in implicit conversions between the two types as well as warnings (found with Chromium's external copy of these sources, built with MSVC) about signed vs. unsigned comparisons. The leveldb sources were already widely using size_t elsewhere, e.g. for key and filter lengths, so using size_t here is not inconsistent with the existing code. However, it does change the public C API. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=101074871	2015-12-09 10:34:58 -08:00
cmumford	5208e7952d	Added leveldb::Status::IsInvalidArgument() method. All other Status::Code enum values have an Is**() method with the one exception of InvalidArgument. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=97166441	2015-12-09 10:34:57 -08:00
Mike Wiacek	ce45404bba	Suppress error reporting after seeking but before a valid First or Full record is encountered. Fix a spelling mistake.	2015-12-09 10:34:57 -08:00
Chris Mumford	b9afa1f2e7	include <assert> -> <cassert> Fixes reported public issue #280.	2015-12-09 10:34:57 -08:00
Venilton FalvoJr	edf2939c0d	Update README.md	2015-11-23 16:24:16 -02:00
Chris Mumford	65190ac48b	Will not reuse manifest if reuse_logs options is false. Prior implementation would always try to reuse the manifest, even if reuse_logs was false (the default). This was missed because the stock Env::NewAppendableFile implementation returns false forcing the creation of a new log.	2015-08-11 14:59:48 -07:00
Sanjay Ghemawat	ac1d69da31	LevelDB now attempts to reuse the preceding MANIFEST and log file when re-opened. (Based on a suggestion by cmumford.) "open" benchmark on my workstation speeds up significantly since we can now avoid three fdatasync calls and a compaction per open: Before: ~80000 microseconds After: ~130 microseconds Details: (1) Added Options::reuse_logs (currently defaults to false) to control new behavior. The intention is to change the default to true after some baking. (2) Added Env::NewAppendableFile() whose default implementation returns a not-supported error. (3) VersionSet::Recovery attempts to reuse the MANIFEST from which it is recovering. (4) DBImpl recovery attempts to reuse the last log file and memtable. (5) db_test.cc now tests a new configuration that sets reuse_logs to true. (6) fault_injection_test also tests a reuse_logs==true config. (7) Added a new recovery_test.	2015-08-11 14:56:39 -07:00
ideawu	76bba139c0	fix indent	2015-04-20 12:41:01 +08:00
ideawu	8fcceb2a6f	log compaction output file's level along with number	2015-04-20 12:39:14 +08:00
Paul Irish	0e0f07417c	documentation. improved link	2015-02-17 09:55:40 -08:00
Paul Irish	c85addcdf3	readme: improved documentation link This replaces htmlpreview with [rawgit](http://rawgit.com/). rawgit is faster and doesnt use JS to render the page, making it SEO friendly.	2015-01-10 17:36:40 -08:00
David Turner	ceff6f1215	Fix Android/MIPS build. port/atomic_pointer.h was missing an implementation for MemoryBarrier() for this platform.	2014-12-17 14:18:54 -05:00