Add formated test

Reviewers: mferencevic

Reviewed By: mferencevic

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1750

.gitignore

@@ -29,3 +29,72 @@ ve3/
 perf.data*
 TAGS
 *.apollo_measurements
+
+# Lisp compiled object code
+*.fas
+*.fasl
+
+# Cap'n Proto generated files
+*.capnp.c++
+*.capnp.h
+
+# LCP generated C++ & Cap'n Proto files
+*.lcp.cpp
+
+# TODO delete later
+src/storage/distributed/concurrent_id_mapper_rpc_messages.capnp
+src/storage/distributed/concurrent_id_mapper_rpc_messages.hpp
+#####
+
+
+src/database/distributed/counters_rpc_messages.capnp
+src/database/distributed/counters_rpc_messages.hpp
+src/database/distributed/serialization.capnp
+src/database/distributed/serialization.hpp
+src/database/single_node_ha/serialization.capnp
+src/database/single_node_ha/serialization.hpp
+src/distributed/bfs_rpc_messages.capnp
+src/distributed/bfs_rpc_messages.hpp
+src/distributed/coordination_rpc_messages.capnp
+src/distributed/coordination_rpc_messages.hpp
+src/distributed/data_rpc_messages.capnp
+src/distributed/data_rpc_messages.hpp
+src/distributed/durability_rpc_messages.capnp
+src/distributed/durability_rpc_messages.hpp
+src/distributed/dynamic_worker_rpc_messages.capnp
+src/distributed/dynamic_worker_rpc_messages.hpp
+src/distributed/index_rpc_messages.capnp
+src/distributed/index_rpc_messages.hpp
+src/distributed/plan_rpc_messages.capnp
+src/distributed/plan_rpc_messages.hpp
+src/distributed/pull_produce_rpc_messages.capnp
+src/distributed/pull_produce_rpc_messages.hpp
+src/distributed/storage_gc_rpc_messages.capnp
+src/distributed/storage_gc_rpc_messages.hpp
+src/distributed/token_sharing_rpc_messages.capnp
+src/distributed/token_sharing_rpc_messages.hpp
+src/distributed/updates_rpc_messages.capnp
+src/distributed/updates_rpc_messages.hpp
+src/query/frontend/ast/ast.hpp
+src/query/frontend/ast/ast_serialization.capnp
+src/query/frontend/ast/ast_serialization.hpp
+src/durability/distributed/state_delta.capnp
+src/durability/distributed/state_delta.hpp
+src/durability/single_node/state_delta.hpp
+src/durability/single_node_ha/state_delta.hpp
+src/query/frontend/semantic/symbol.hpp
+src/query/frontend/semantic/symbol_serialization.capnp
+src/query/frontend/semantic/symbol_serialization.hpp
+src/query/plan/distributed_ops.capnp
+src/query/plan/distributed_ops.hpp
+src/query/plan/operator.hpp
+src/raft/log_entry.capnp
+src/raft/log_entry.hpp
+src/raft/raft_rpc_messages.capnp
+src/raft/raft_rpc_messages.hpp
+src/stats/stats_rpc_messages.capnp
+src/stats/stats_rpc_messages.hpp
+src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.capnp
+src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp
+src/transactions/distributed/engine_rpc_messages.capnp
+src/transactions/distributed/engine_rpc_messages.hpp

.ycm_extra_conf.py

@@ -32,6 +32,10 @@ BASE_FLAGS = [
     '-I./libs/gflags/include',
     '-I./experimental/distributed/src',
     '-I./libs/postgresql/include',
+    '-I./libs/bzip2',
+    '-I./libs/zlib',
+    '-I./libs/rocksdb/include',
+    '-I./libs/librdkafka/include/librdkafka',
     '-I./build/include'
 ]
 

CHANGELOG.md

@@ -1,5 +1,87 @@
 # Change Log
 
+## v0.14.0
+
+### Breaking Changes
+
+* Write-ahead log format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Reduce memory usage in distributed usage.
+* Add `DROP INDEX` feature.
+* Improve SSL error messages.
+
+### Bug Fixes and Other Changes
+
+* [Enterprise Ed.] Fix issues with reading and writing in a distributed query.
+* Correctly handle an edge case with unique constraint checks.
+* Fix a minor issue with `mg_import_csv`.
+* Fix an issue with `EXPLAIN`.
+
+## v0.13.0
+
+### Breaking Changes
+
+* Write-ahead log format changed (not backward compatible).
+* Snapshot format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Authentication and authorization support.
+* [Enterprise Ed.] Kafka integration.
+* [Enterprise Ed.] Support dynamic worker addition in distributed.
+* Reduce memory usage and improve overall performance.
+* Add `CREATE UNIQUE INDEX` clause to openCypher.
+* Add `EXPLAIN` clause to openCypher.
+* Add `inDegree` and `outDegree` functions to openCypher.
+* Improve BFS performance when both endpoints are known.
+* Add new `node-label`, `relationship-type` and `quote` options to
+  `mg_import_csv` tool.
+* Reduce memory usage of `mg_import_csv`.
+
+### Bug Fixes and Other Changes
+
+* [Enterprise Ed.] Fix an edge case in distributed index creation.
+* [Enterprise Ed.] Fix issues with Cartesian in distributed queries.
+* Correctly handle large messages in Bolt protocol.
+* Fix issues when handling explicitly started transactions in queries.
+* Allow openCypher keywords to be used as variable names.
+* Revise and make user visible error messages consistent.
+* Improve aborting time consuming execution.
+
+## v0.12.0
+
+### Breaking Changes
+
+* Snapshot format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* Improved Id Cypher function.
+* Added string functions to openCypher (`lTrim`, `left`, `rTrim`, `replace`,
+ `reverse`, `right`, `split`, `substring`, `toLower`, `toUpper`, `trim`).
+* Added `timestamp` function to openCypher.
+* Added support for dynamic property access with `[]` operator.
+
+## v0.11.0
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Improve Cartesian support in distributed queries.
+* [Enterprise Ed.] Improve distributed execution of BFS.
+* [Enterprise Ed.] Dynamic graph partitioner added.
+* Static nodes/edges id generators exposed through the Id Cypher function.
+* Properties on disk added.
+* Telemetry added.
+* SSL support added.
+* `toString` function added.
+
+### Bug Fixes and Other Changes
+
+* Document issues with Docker on OS X.
+* Add BFS and Dijkstra's algorithm examples to documentation.
+
 ## v0.10.0
 
 ### Breaking Changes
@@ -8,7 +90,7 @@
 
 ### Major Features and Improvements
 
-* [Enterprise Ed.] Distributed storage and execution
+* [Enterprise Ed.] Distributed storage and execution.
 * `reduce` and `single` functions added to openCypher.
 * `wShortest` edge expansion added to openCypher.
 * Support packaging RPM on CentOS 7.
@@ -25,6 +107,7 @@
 * Report an error if trying to compare a property value with something that
   cannot be stored in a property.
 * Fix crashes in some obscure cases.
+* Commit log automatically garbage collected.
 * Add minor performance improvements.
 
 ## v0.9.0
@@ -50,7 +133,7 @@
 
 * CASE construct (without aggregations).
 * Named path support added.
-* Maps can now be stored as vertex/edge properties.
+* Maps can now be stored as node/edge properties.
 * Map indexing supported.
 * `rand` function added.
 * `assert` function added.

CMakeLists.txt

@@ -42,7 +42,7 @@ string(STRIP ${COMMIT_HASH} COMMIT_HASH)
 
 # -----------------------------------------------------------------------------
 
-project(memgraph VERSION 0.9.0)
+project(memgraph VERSION 0.14.0)
 # -----------------------------------------------------------------------------
 
 # setup CMake module path, defines path for include() and find_package()
@@ -137,8 +137,8 @@ if (USE_READLINE)
   endif()
 endif()
 
-set(Boost_USE_STATIC_LIBS ON)
-find_package(Boost 1.62 REQUIRED COMPONENTS iostreams serialization)
+# OpenSSL
+find_package(OpenSSL REQUIRED)
 
 set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
 add_subdirectory(libs EXCLUDE_FROM_ALL)
@@ -147,19 +147,25 @@ include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
 include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
 include_directories(SYSTEM ${FMT_INCLUDE_DIR})
 include_directories(SYSTEM ${ANTLR4_INCLUDE_DIR})
+include_directories(SYSTEM ${BZIP2_INCLUDE_DIR})
+include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
+include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR})
+include_directories(SYSTEM ${CAPNP_INCLUDE_DIR})
+include_directories(SYSTEM ${LIBRDKAFKA_INCLUDE_DIR})
 # -----------------------------------------------------------------------------
 
 # openCypher parser -----------------------------------------------------------
 set(opencypher_frontend ${CMAKE_SOURCE_DIR}/src/query/frontend/opencypher)
 set(opencypher_generated ${opencypher_frontend}/generated)
-set(opencypher_grammar ${opencypher_frontend}/grammar/Cypher.g4)
+set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
+set(opencypher_parser_grammar ${opencypher_frontend}/grammar/MemgraphCypher.g4)
 
 # enumerate all files that are generated from antlr
 set(antlr_opencypher_generated_src
-   ${opencypher_generated}/CypherLexer.cpp
-   ${opencypher_generated}/CypherParser.cpp
-   ${opencypher_generated}/CypherBaseVisitor.cpp
-   ${opencypher_generated}/CypherVisitor.cpp
+   ${opencypher_generated}/MemgraphCypherLexer.cpp
+   ${opencypher_generated}/MemgraphCypher.cpp
+   ${opencypher_generated}/MemgraphCypherBaseVisitor.cpp
+   ${opencypher_generated}/MemgraphCypherVisitor.cpp
 )
 
 # Provide a command to generate sources if missing. If this were a
@@ -168,9 +174,11 @@ add_custom_command(OUTPUT ${antlr_opencypher_generated_src}
   COMMAND
   ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
   COMMAND
-  java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.6-complete.jar -Dlanguage=Cpp -visitor -o ${opencypher_generated} -package antlropencypher ${opencypher_grammar}
+  java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.6-complete.jar -Dlanguage=Cpp -visitor -o ${opencypher_generated} -package antlropencypher ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
   WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
-  DEPENDS ${opencypher_grammar})
+  DEPENDS ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+          ${opencypher_frontend}/grammar/CypherLexer.g4
+          ${opencypher_frontend}/grammar/Cypher.g4)
 
 # add custom target for generation
 add_custom_target(generate_opencypher_parser
@@ -186,6 +194,11 @@ option(EXPERIMENTAL "Build experimental binaries" OFF)
 option(CUSTOMERS "Build customer binaries" OFF)
 option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
 option(TOOLS "Build tools binaries" ON)
+option(MG_COMMUNITY "Build Memgraph Community Edition" OFF)
+option(ASAN "Build with Address Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
+option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
+option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
+option(THIN_LTO "Build with link time optimization" OFF)
 
 if (TEST_COVERAGE)
   string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -196,6 +209,66 @@ if (TEST_COVERAGE)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
 endif()
 
+if (MG_COMMUNITY)
+  add_definitions(-DMG_COMMUNITY)
+endif()
+
+if (ASAN)
+  # Enable Addres sanitizer and get nicer stack traces in error messages.
+  # NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
+  # distribution to symbolize the stack traces (note that ideally the
+  # llvm-symbolizer version must match the version of ASan runtime library).
+  # Just make sure llvm-symbolizer is in PATH before running the binary or
+  # provide it in separate ASAN_SYMBOLIZER_PATH environment variable.
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
+  # To detect Stack-use-after-return bugs set run-time flag:
+  #   ASAN_OPTIONS=detect_stack_use_after_return=1
+  # To check initialization order bugs set run-time flag:
+  #   ASAN_OPTIONS=check_initialization_order=true
+  #     This mode reports an error if initializer for a global variable accesses
+  #     dynamically initialized global from another translation unit, which is
+  #     not yet initialized
+  #   ASAN_OPTIONS=strict_init_order=true
+  #     This mode reports an error if initializer for a global variable accesses
+  #     any dynamically initialized global from another translation unit.
+endif()
+
+if (TSAN)
+  # ThreadSanitizer generally requires all code to be compiled with -fsanitize=thread.
+  # If some code (e.g. dynamic libraries) is not compiled with the flag, it can
+  # lead to false positive race reports, false negative race reports and/or
+  # missed stack frames in reports depending on the nature of non-instrumented
+  # code. To not produce false positive reports ThreadSanitizer has to see all
+  # synchronization in the program, some synchronization operations (namely,
+  # atomic operations and thread-safe static initialization) are intercepted
+  # during compilation (and can only be intercepted during compilation).
+  # ThreadSanitizer stack trace collection also relies on compiler instrumentation
+  # (unwinding stack on each memory access is too expensive).
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
+  # By default ThreadSanitizer uses addr2line utility to symbolize reports.
+  # llvm-symbolizer is faster, consumes less memory and produces much better
+  # reports. To use it set runtime flag:
+  #   TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
+  # For more runtime flags see: https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
+endif()
+
+if (UBSAN)
+  # Compile with UBSAN but disable vptr check. This is disabled because it
+  # requires linking with clang++ to make sure C++ specific parts of the
+  # runtime library and c++ standard libraries are present.
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
+  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
+  # Make sure llvm-symbolizer binary is in path
+endif()
+
+if (THIN_LTO)
+  set(CMAKE_CXX_FLAGS"${CMAKE_CXX_FLAGS} -flto=thin")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto=thin")
+endif()
+
 # Add subprojects
 include_directories(src)
 add_subdirectory(src)
@@ -251,19 +324,25 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
  Contains Memgraph, the graph database. It aims to deliver developers the
  speed, simplicity and scale required to build the next generation of
  applications driver by real-time connected data.")
+# Add `openssl` package to dependencies list. Used to generate SSL certificates.
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0)")
 
 # RPM specific
-set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION /var /var/lib /var/log /etc/logrotate.d /lib /lib/systemd /lib/systemd/system)
-set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils,systemd")
-# set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_SOURCE_DIR}/release/rpm/memgraph.spec.in")
-set(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/release/rpm/preinst")
-set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/release/rpm/postinst")
-set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_SOURCE_DIR}/release/rpm/prerm")
+set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
+set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
+  /var /var/lib /var/log /etc/logrotate.d
+  /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
+set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
+# NOTE: user specfile has a bug in cmake 3.7.2, this needs to be patched
+# manually in: ~/cmake/share/cmake-3.7/Modules/CPackRPM.cmake line 2273
+# Or newer cmake version used
+set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_SOURCE_DIR}/release/rpm/memgraph.spec.in")
 # Description formatting is important, no line must be greater than 80 characters.
 set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
-            : It aims to deliver developers the speed, simplicity and scale
-            : required to build the next generation of applications driver by
-            : real-time connected data.")
+It aims to deliver developers the speed, simplicity and scale required to build
+the next generation of applications driver by real-time connected data.")
+# Add `openssl` package to dependencies list. Used to generate SSL certificates.
+set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0")
 
 # All variables must be set before including.
 include(CPack)

apollo_archives.yaml

@@ -1,7 +1,10 @@
 - name: Binaries
   archive:
     - build_debug/memgraph
+    - build_debug/memgraph_distributed
     - build_release/memgraph
+    - build_release/memgraph_distributed
+    - build_release/tools/src/mg_client
     - build_release/tools/src/mg_import_csv
     - build_release/tools/src/mg_statsd
     - config
@@ -13,8 +16,3 @@
     - .
   filename: documentation.tar.gz
   host: true
-
-- name: Release (user docs)
-  project: ^NEVER$ # TODO (mferencevic): replace with 'release' once user documentation creation is fixed (couscous -> pandoc)
-  archive: docs/user_technical/docs.tar.gz
-  filename: release_user_docs.tar.gz

apollo_build.yaml

@@ -7,7 +7,7 @@
     cd memgraph
 
     # Initialize and create documentation.
-    TIMEOUT=600 ./init
+    TIMEOUT=1200 ./init
     doxygen Doxyfile
 
     # Remove default build directory.
@@ -17,21 +17,21 @@
     mkdir build_debug
     cd build_debug
     cmake ..
-    TIMEOUT=1000 make -j$THREADS
+    TIMEOUT=1200 make -j$THREADS
 
     # Build coverage binaries.
     cd ..
     mkdir build_coverage
     cd build_coverage
     cmake -DTEST_COVERAGE=ON ..
-    TIMEOUT=1000 make -j$THREADS memgraph__unit
+    TIMEOUT=1200 make -j$THREADS memgraph__unit
 
     # Build release binaries.
     cd ..
     mkdir build_release
     cd build_release
     cmake -DCMAKE_BUILD_TYPE=release ..
-    TIMEOUT=1000 make -j$THREADS memgraph tools memgraph__macro_benchmark memgraph__stress memgraph__manual__card_fraud_generate_snapshot
+    TIMEOUT=1200 make -j$THREADS memgraph memgraph_distributed tools memgraph__macro_benchmark memgraph__stress memgraph__manual__card_fraud_generate_snapshot memgraph__feature_benchmark__kafka__benchmark
 
     # Generate distributed card fraud dataset.
     cd ../tests/distributed/card_fraud
@@ -41,19 +41,19 @@
     # Checkout to parent commit and initialize.
     cd ../parent
     git checkout HEAD~1
-    TIMEOUT=600 ./init
+    TIMEOUT=1200 ./init
 
     # Build parent release binaries.
     mkdir build_release
     cd build_release
     cmake -DCMAKE_BUILD_TYPE=release ..
-    TIMEOUT=1000 make -j$THREADS memgraph memgraph__macro_benchmark
+    TIMEOUT=1200 make -j$THREADS memgraph memgraph__macro_benchmark
 
 
 # release build is the default one
 - name: Release build
   commands: |
-    TIMEOUT=600 ./init
+    TIMEOUT=1200 ./init
     doxygen Doxyfile
 
     # Remove default build directory.
@@ -63,32 +63,27 @@
     mkdir build_debug
     cd build_debug
     cmake ..
-    TIMEOUT=1000 make -j$THREADS
+    TIMEOUT=1200 make -j$THREADS
 
     # Build coverage binaries.
     cd ..
     mkdir build_coverage
     cd build_coverage
     cmake -DTEST_COVERAGE=ON ..
-    TIMEOUT=1000 make -j$THREADS memgraph__unit
+    TIMEOUT=1200 make -j$THREADS memgraph__unit
 
     # Build release binaries.
     cd ..
     mkdir build_release
     cd build_release
     cmake -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF ..
-    TIMEOUT=1000 make -j$THREADS
+    TIMEOUT=1200 make -j$THREADS
 
     # Create Debian package.
     mkdir output
     cd output
     cpack -G DEB --config ../CPackConfig.cmake
 
-    # Create user technical documentation for community site.
-    cd ../../docs/user_technical
-    # TODO (mferencevic): uncomment this once couscous is replaced with pandoc
-    #./bundle_community
-
     # Generate distributed card fraud dataset.
     cd ../../tests/distributed/card_fraud
     ./generate_dataset.sh

cmake/FindSeccomp.cmake

@@ -0,0 +1,90 @@
+#.rst:
+# FindSeccomp
+# -----------
+#
+# Try to locate the libseccomp library.
+# If found, this will define the following variables:
+#
+# ``Seccomp_FOUND``
+#     True if the seccomp library is available
+# ``Seccomp_INCLUDE_DIRS``
+#     The seccomp include directories
+# ``Seccomp_LIBRARIES``
+#     The seccomp libraries for linking
+#
+# If ``Seccomp_FOUND`` is TRUE, it will also define the following
+# imported target:
+#
+# ``Seccomp::Seccomp``
+#     The Seccomp library
+#
+# Since 5.44.0.
+
+#=============================================================================
+# Copyright (c) 2017 Martin Flöser <mgraesslin@kde.org>
+# Copyright (c) 2017 David Kahles <david.kahles96@gmail.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+# 3. The name of the author may not be used to endorse or promote products
+#    derived from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#=============================================================================
+
+find_package(PkgConfig QUIET)
+pkg_check_modules(PKG_Libseccomp QUIET libseccomp)
+
+find_path(Seccomp_INCLUDE_DIRS
+    NAMES
+        seccomp.h
+    HINTS
+        ${PKG_Libseccomp_INCLUDE_DIRS}
+)
+find_library(Seccomp_LIBRARIES
+    NAMES
+        seccomp
+    HINTS
+        ${PKG_Libseccomp_LIBRARY_DIRS}
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Seccomp
+    FOUND_VAR
+        Seccomp_FOUND
+    REQUIRED_VARS
+        Seccomp_LIBRARIES
+        Seccomp_INCLUDE_DIRS
+)
+
+if (Seccomp_FOUND AND NOT TARGET Seccomp::Seccomp)
+    add_library(Seccomp::Seccomp UNKNOWN IMPORTED)
+    set_target_properties(Seccomp::Seccomp PROPERTIES
+        IMPORTED_LOCATION "${Seccomp_LIBRARIES}"
+        INTERFACE_INCLUDE_DIRECTORIES "${Seccomp_INCLUDE_DIRS}"
+    )
+endif()
+
+mark_as_advanced(Seccomp_LIBRARIES Seccomp_INCLUDE_DIRS)
+
+include(FeatureSummary)
+set_package_properties(Seccomp PROPERTIES
+    URL "https://github.com/seccomp/libseccomp"
+    DESCRIPTION "The enhanced seccomp library."
+)

cmake/functions.cmake

@@ -85,3 +85,27 @@ function(get_target_cxx_flags target result)
     endif()
     set(${result} ${flags} PARENT_SCOPE)
 endfunction()
+
+# Define `add_capnp` function for registering a capnp file for generation.
+#
+# The `define_add_capnp` expects 3 arguments:
+#   * name -- name for the function, you usually want `add_capnp`
+#   * main_src_files -- variable to be updated with generated cpp files
+#   * generated_capnp_files -- variable to be updated with generated hpp and cpp files
+#
+# The `add_capnp` function expects a single argument, path to capnp file.
+# Each added file is standalone and we avoid recompiling everything.
+macro(define_add_capnp name main_src_files generated_capnp_files)
+  function(${name} capnp_src_file)
+    set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.c++)
+    set(h_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.h)
+    add_custom_command(OUTPUT ${cpp_file} ${h_file}
+      COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file} -I ${CMAKE_SOURCE_DIR}/src
+      DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file} capnproto-proj
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    # Update *global* generated_capnp_files
+    set(${generated_capnp_files} ${${generated_capnp_files}} ${cpp_file} ${h_file} PARENT_SCOPE)
+    # Update *global* main_src_files
+    set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE)
+  endfunction(${name})
+endmacro(define_add_capnp)

config/community.conf

@@ -16,6 +16,12 @@
 # Port the server should listen on.
 --port=7687
 
+# Path to a SSL certificate file that should be used.
+--cert-file=/etc/memgraph/ssl/cert.pem
+
+# Path to a SSL key file that should be used.
+--key-file=/etc/memgraph/ssl/key.pem
+
 # Number of workers used by the Memgraph server. By default, this will be the
 # number of processing units available on the machine.
 # --num-workers=8
@@ -30,6 +36,18 @@
 # it will log a warning.
 --memory-warning-threshold=1024
 
+# The telemetry collects data about the machine that is executing the database
+# (CPU, Memory, OS and Kernel Information) and data about the database runtime
+# (CPU usage, Memory usage, Vertices and Edges count). It is used to provide a
+# better product, easy to disable and does not collect any sensitive data.
+--telemetry-enabled=true
+
+# Memgraph offers an option to store a certain amount of data on a disk. More
+# precisely, the user can pass a list of properties they wish to keep stored on
+# a disk. The property names have to be separated with a comma. An example
+# would be --properties-on-disk=biography,summary.
+#--properties-on-disk=
+
 ## Query
 #
 # Various settings related to openCypher query execution.
@@ -54,8 +72,7 @@
 # are used: snapshots store the total current database state while write-ahead
 # logs store small changes incrementally. They are used in tandem to provide
 # fast and storage-efficient persistence. Some aspects of snapshot taking
-# are configurable, while write-ahead logging is pre-configured for optimal
-# performance.
+# and write-ahead logging are configurable.
 --durability-enabled=true
 
 # Path to the directory where snapshots and write-ahead log files will be stored.
@@ -75,6 +92,15 @@
 # for new ones. If set to -1, the number of kept snapshots is unlimited.
 --snapshot-max-retained=3
 
+# Specifies whether WAL updates should be written on disk immediately after a
+# transaction finishes. Setting this parameter to false does introduce risk of
+# database inconsistency because an operating system or hardware crash might
+# lead to missing transactions in the write-ahead log, but the database will
+# handle this as if those transactions never happened. Turning
+# synchronous-commit off can be a useful trade-off between exact durability and
+# performance.
+--synchronous-commit=false
+
 ## Logging
 
 # Path to where the log should be stored.

config/durability_stress.conf

@@ -0,0 +1,15 @@
+# MEMGRAPH DEFAULT DURABILITY STRESS TESTING CONFIG
+
+# NOTE: all paths are relative to the run folder
+# (where the executable is run)
+
+# enable durability
+--durability-enabled=true
+--snapshot-cycle-sec=5
+--snapshot-on-exit=false
+--snapshot-max-retained=2
+
+--db-recover-on-startup=true
+
+# increase query timeout (10 min)
+--query-execution-time-sec=600

docs/.gitignore

@@ -1,6 +1,2 @@
 doxygen/html/
 doxygen/latex/
-user_technical/.bundle_community/
-user_technical/.couscous/
-user_technical/couscous.phar
-user_technical/docs.tar.gz

docs/dev/cpp-code-conventions.md

@@ -5,15 +5,80 @@ C++ code.
 
 ## Code Style
 
-Memgraph uses the Google Style Guide for C++ in most of its code. You should
-follow them whenever writing new code. The style guide can be found
-[here](https://google.github.io/styleguide/cppguide.html).
+Memgraph uses the
+[Google Style Guide for C++](https://google.github.io/styleguide/cppguide.html)
+in most of its code.  You should follow them whenever writing new code.
+
+### Often Overlooked Style Conventions
+
+#### Pointers & References
+
+References provide a shorter syntax for accessing members and better declare
+the intent that a pointer *should* not be `nullptr`. They do not prevent
+accessing a `nullptr` and obfuscate the client/calling code because the
+reference argument is passed just like a value. Errors with such code have
+been very difficult to debug. Therefore, pointers are always used. They will
+not prevent bugs but will make some of them more obvious when reading code.
+
+The only time a reference can be used is if it is `const`. Note that this
+kind of reference is not allowed if it is stored somewhere, i.e. in a class.
+You should use a pointer to `const` then. The primary reason being is that
+references obscure the semantics of moving an object, thus making bugs with
+references pointing to invalid memory harder to track down.
+
+[Style guide reference](https://google.github.io/styleguide/cppguide.html#Reference_Arguments)
+
+#### Constructors & RAII
+
+RAII (Resource Acquisition is Initialization) is a nice mechanism for managing
+resources. It is especially useful when exceptions are used, such as in our
+code. Unfortunately, they do have 2 major downsides.
+
+  * Only exceptions can be used for to signal failure.
+  * Calls to virtual methods are not resolved as expected.
+
+For those reasons the style guide recommends minimal work that cannot fail.
+Using virtual methods or doing a lot more should be delegated to some form of
+`Init` method, possibly coupled with factory functions. Similar rules apply to
+destructors, which are not allowed to even throw exceptions.
+
+[Style guide reference](https://google.github.io/styleguide/cppguide.html#Doing_Work_in_Constructors)
 
 ### Additional Style Conventions
 
-Code style conventions which are left undefined by Google are specified here.
+Old code may have broken Google C++ Style accidentally, but the new code
+should adhere to it as close as possible. We do have some exceptions
+to Google style as well as additions for unspecified conventions.
 
-#### Template parameter naming
+#### Using C++ Exceptions
+
+Unlike Google, we do not forbid using exceptions.
+
+But, you should be very careful when using them and introducing new ones. They
+are indeed handy, but cause problems with understanding the control flow since
+exceptions are another form of `goto`. It also becomes very hard to determine
+that the program is in correct state after the stack is unwound and the thrown
+exception handled. Other than those issues, throwing exceptions in destructors
+will terminate the program. The same will happen if a thread doesn't handle an
+exception even though it is not the main thread.
+
+[Style guide reference](https://google.github.io/styleguide/cppguide.html#Exceptions)
+
+#### Assertions
+
+We use `CHECK` and `DCHECK` macros from glog library. You are encouraged to
+use them as often as possible to both document and validate various pre and
+post conditions of a function.
+
+`CHECK` remains even in release build and should be preferred over it's cousin
+`DCHECK` which only exists in debug builds. The primary reason is that you
+want to trigger assertions in release builds in case the tests didn't
+completely validate all code paths. It is better to fail fast and crash the
+program, than to leave it in undefined state and potentially corrupt end
+user's work. In cases when profiling shows that `CHECK` is causing visible
+slowdown you should switch to `DCHECK`.
+
+#### Template Parameter Naming
 
 Template parameter names should start with capital letter 'T' followed by a
 short descriptive name. For example:
@@ -63,11 +128,9 @@ containing 3 slashes (`///`). Take a look at the 2 examples below.
 ### Line Comment
 
 ```cpp
-///
 /// One sentence, brief description.
 ///
 /// Long form description.
-///
 ```
 
 If you only have a brief description, you may collapse the documentation into

docs/dev/diagram/dependencies.dot

@@ -0,0 +1,288 @@
+// dot -Tpng dependencies.dot -o /path/to/output.png
+
+// TODO (buda): Put PropertyValueStore to storage namespace
+
+digraph {
+  // At the beginning of each block there is a default style for that block
+  label="Memgraph Dependencies Diagram"; fontname="Roboto Bold"; fontcolor=black;
+    fontsize=26; labelloc=top; labeljust=right;
+  compound=true; // If true, allow edges between clusters
+  rankdir=TB; // Alternatives: LR
+  node [shape=record fontname="Roboto", fontsize=12, fontcolor=white];
+  edge [color="#B5AFB7"];
+
+  // -- Legend --
+  // dir=both arrowtail=diamond arrowhead=vee -> group ownership
+  // dir=both arrowtail=none, arrowhead=vee -> ownership; stack or uptr
+
+  subgraph cluster_tcp_end_client_communication {
+    label="TCP End Client Communication"; fontsize=14;
+    node [style=filled, color="#DD2222" fillcolor="#DD2222"];
+
+    // Owned elements
+    "communication::Server";
+    "io::network::Socket";
+
+    // Intracluster connections
+    "communication::Server" -> "io::network::Socket"
+      [label="socket_" dir=both arrowtail=none arrowhead=vee];
+  }
+
+  subgraph cluster_bolt_server {
+    label="Bolt Server"; fontsize=14;
+    node [style=filled, color="#62A2CA" fillcolor="#62A2CA"];
+
+    // Owned elements
+    "communication::bolt::SessionData";
+    "communication::bolt::Session";
+    "communication::bolt::Encoder";
+    "communication::bolt::Decoder";
+
+    // Intracluster connections
+    "communication::bolt::Session" -> "communication::bolt::Encoder"
+      [label="encoder_", dir=both arrowtail=none, arrowhead=vee];
+    "communication::bolt::Session" -> "communication::bolt::Decoder"
+      [label="decoder_", dir=both arrowtail=none, arrowhead=vee];
+  }
+
+  subgraph cluster_opencypher_engine {
+    label="openCypher Engine"; fontsize=14;
+    node [style=filled, color="#68BDF6" fillcolor="#68BDF6"];
+
+    // Owned Elements
+    "query::Interpreter";
+    "query::AstTreeStorage";
+    "query::TypedValue"
+    "query::Path";
+    "query::Simbol";
+    "query::Context";
+    "query::ExpressionEvaluator";
+    "query::Frame";
+    "query::SymbolTable";
+    "query::plan::LogicalOperator";
+    "query::plan::Cursor";
+    "query::plan::CostEstimator";
+
+    // Intracluster connections
+    "query::Interpreter" -> "query::AstTreeStorage"
+      [label="ast_cache" dir=both arrowtail=diamond arrowhead=vee];
+    "query::TypedValue" -> "query::Path";
+    "query::plan::Cursor" -> "query::Frame";
+    "query::plan::Cursor" -> "query::Context";
+    "query::plan::LogicalOperator" -> "query::Symbol";
+    "query::plan::LogicalOperator" -> "query::SymbolTable";
+    "query::plan::LogicalOperator" -> "query::plan::Cursor";
+  }
+
+
+  subgraph cluster_storage {
+    label="Storage" fontsize=14;
+    node [style=filled, color="#FB6E00" fillcolor="#FB6E00"];
+
+    // Owned Elements
+    "database::GraphDb";
+    "database::GraphDbAccessor";
+    "storage::Record";
+    "storage::Vertex";
+    "storage::Edge";
+    "storage::RecordAccessor";
+    "storage::VertexAccessor";
+    "storage::EdgeAccessor";
+    "storage::Common";
+    "storage::Label";
+    "storage::EdgeType";
+    "storage::Property";
+    "storage::compression";
+    "storage::SingleNodeConcurrentIdMapper";
+    "storage::Location";
+    "storage::StorageTypesLocation";
+    "PropertyValueStore";
+    "storage::RecordLock";
+    "mvcc::Version";
+    "mvcc::Record";
+    "mvcc::VersionList";
+
+    // Intracluster connections
+    "storage::VertexAccessor" -> "storage::RecordAccessor"
+      [arrowhead=onormal];
+    "storage::EdgeAccessor" -> "storage::RecordAccessor"
+      [arrowhead=onormal];
+    "storage::RecordAccessor" -> "database::GraphDbAccessor"
+      [style=dashed arrowhead=vee];
+    "storage::Vertex" -> "mvcc::Record"
+      [arrowhead=onormal];
+    "storage::Edge" -> "mvcc::Record"
+      [arrowhead=onormal];
+    "storage::Edge" -> "PropertyValueStore"
+      [arrowhead=vee];
+    "storage::Vertex" -> "PropertyValueStore"
+      [arrowhead=vee];
+    "storage::Edge" -> "mvcc::VersionList"
+      [label="from,to" arrowhead=vee style=dashed];
+    "storage::VertexAccessor" -> "storage::Vertex"
+      [arrowhead=vee];
+    "storage::EdgeAccessor" -> "storage::Edge"
+      [arrowhead=vee];
+    "storage::SingleNodeConcurrentIdMapper" -> "storage::StorageTypesLocation"
+      [arrowhead=vee];
+    "storage::StorageTypesLocation" -> "storage::Location"
+      [arrowhead=vee];
+    "storage::Storage" -> "storage::StorageTypesLocation"
+      [arrowhead=vee];
+    "storage::Property" -> "storage::Common"
+      [arrowhead=onormal];
+    "storage::Label" -> "storage::Common"
+      [arrowhead=onormal];
+    "storage::EdgeType" -> "storage::Common"
+      [arrowhead=onormal];
+    "storage::Property" -> "storage::Location"
+      [arrowhead=vee];
+    "PropertyValueStore" -> "storage::Property"
+      [arrowhead=vee];
+    "PropertyValueStore" -> "storage::Location"
+      [arrowhead=vee];
+    "database::GraphDbAccessor" -> "database::GraphDb"
+      [arrowhead=vee];
+    "database::GraphDbAccessor" -> "tx::TransactionId"
+      [arrowhead=vee];
+    "mvcc::VersionList" -> "storge::RecordLock"
+      [label="lock" arrowhead=vee];
+    "mvcc::VersionList" -> "mvcc::Record"
+      [label="head" arrowhead=vee];
+    "mvcc::Record" -> "mvcc::Version"
+      [arrowhead=onormal];
+
+    // Explicit positioning
+    {rank=same;
+        "database::GraphDbAccessor";
+        "storage::VertexAccessor";
+        "storage::EdgeAccessor";}
+    {rank=same;
+        "storage::Common";
+        "storage::compression";}
+  }
+
+  subgraph cluster_properties_on_disk {
+    label="Properties on Disk" fontsize=14;
+    node [style=filled, color="#102647" fillcolor="#102647"];
+
+    // Owned Elements
+    "storage::KVStore";
+    "rocksdb";
+
+    // Intracluster connections
+    "storage::KVStore" -> "rocksdb";
+  }
+
+  subgraph cluster_distributed {
+    label="Distributed" fontsize=14;
+    node [style=filled, color="#FFC500" fillcolor="#FFC500"];
+
+    // Owned Elements
+    "distributed::DataManager";
+    "distributed::DataRpcClients";
+
+    // Intracluster connections
+    "distributed::DataManager" -> "distributed::DataRpcClients"
+      [arrowhead=vee];
+    "storage::RecordAccessor" -> "distributed::DataManager"
+      [style=dashed arrowhead=vee];
+  }
+
+  subgraph cluster_dynamic_partitioning {
+    label="Dynamic Partitioning" fontsize=14;
+    node [style=filled, color="#720096" fillcolor="#720096"];
+
+    // Owned Elements
+    "DynamicPartitioner";
+  }
+
+  subgraph cluster_security {
+    label="Security" fontsize=14;
+    node [style=filled, color="#857F87" fillcolor="#857F87"];
+
+    // Owned Elements
+    "Communication Encryption";
+    "Data Encryption";
+    "Access Control";
+    "Audit Logging";
+  }
+
+  subgraph cluster_web_dashboard {
+    label="Dashaboard" fontsize=14;
+    node [style=filled, color="#FF0092" fillcolor="#FF0092"];
+
+    // Owned Elements
+    "Memgraph Ops / Memgraph Cockpit";
+  }
+
+  subgraph cluster_rpc {
+    label="RPC" fontsize=14;
+    node [style=filled, color="#857F87" fillcolor="#857F87"];
+
+    // Owned Elements
+    "communication::rpc::Server";
+    "communication::rpc::Client";
+  }
+
+  subgraph cluster_ingestion {
+    label="Ingestion" fontsize=14;
+    node [style=filled, color="#0B6D88" fillcolor="#0B6D88"];
+
+    // Owned Elements
+    "Extract";
+    "Transform";
+    "Load";
+    "Amazon S3";
+    "Kafka";
+
+    // Intracluster connections
+    "Extract" -> "Amazon S3";
+    "Extract" -> "Kafka";
+
+    // Explicit positioning
+    {rank=same;"Extract";"Transform";"Load";}
+  }
+
+  // -- Intercluster connections --
+  // cluster_tcp_end_client_communication -- cluster_bolt_server
+  "communication::Server" -> "communication::bolt::SessionData" [color=black];
+  "communication::Server" -> "communication::bolt::Session" [color=black];
+  // cluster_bolt_server -> cluster_storage
+  "communication::bolt::SessionData" -> "database::GraphDb" [color=red];
+  "communication::bolt::Session" -> "database::GraphDbAccessor" [color=red];
+  // cluster_bolt_server -> cluster_opencypher_engine
+  "communication::bolt::SessionData" -> "query::Interpreter" [color=red];
+  // cluster_opencypher_engine -- cluster_storage
+  "query::Interpreter" -> "database::GraphDbAccessor" [color=black];
+  "query::Interpreter" -> "storage::VertexAccessor" [color=black];
+  "query::Interpreter" -> "storage::EdgeAccessor" [color=black];
+  "query::TypedValue" -> "storage::VertexAccessor" [color=black];
+  "query::TypedValue" -> "storage::EdgeAccessor" [color=black];
+  "query::Path" -> "storage::VertexAccessor"
+    [label="vertices" dir=both arrowtail=diamond arrowhead=vee color=black];
+  "query::Path" -> "storage::EdgeAccessor"
+    [label="edges" dir=both arrowtail=diamond arrowhead=vee color=black];
+  "query::plan::LogicalOperator" -> "database::GraphDbAccessor"
+    [color=black arrowhead=vee];
+  // cluster_distributed -- cluster_storage
+  "distributed::DataManager" -> "database::GraphDb"
+    [arrowhead=vee style=dashed color=red];
+  "distributed::DataManager" -> "tx::TransactionId"
+    [label="ves_caches_key" dir=both arrowhead=none arrowtail=diamond
+     color=red];
+  "distributed::DataManager" -> "storage::Vertex"
+    [label="vertices_caches" dir=both arrowhead=none arrowtail=diamond
+     color=red];
+  "distributed::DataManager" -> "storage::Edge"
+    [label="edges_caches" dir=both arrowhead=none arrowtail=diamond
+     color=red];
+  // cluster_storage -- cluster_properties_on_disk
+  "PropertyValueStore" -> "storage::KVStore"
+    [label="static" arrowhead=vee color=black];
+  // cluster_dynamic_partitioning -- cluster_storage
+  "database::GraphDb" -> "DynamicPartitioner"
+    [arrowhead=vee color=red];
+  "DynamicPartitioner" -> "database::GraphDbAccessor"
+    [arrowhead=vee color=black];
+}

docs/dev/diagram/dgp/logical.dot

@@ -0,0 +1,22 @@
+digraph {
+  // label="Dynamig Graph Partitioning";
+  fontname="Roboto Bold"; fontcolor=black;
+    fontsize=26; labelloc=top; labeljust=center;
+  compound=true; // If true, allow edges between clusters
+  rankdir=TB; // Alternatives: LR
+  node [shape=record fontname="Roboto", fontsize=12, fontcolor=white
+        style=filled, color="#FB6E00" fillcolor="#FB6E00"];
+  edge [color="#B5AFB7"];
+
+  "distributed::DistributedGraphDb" -> "distributed::TokenSharingRpcServer";
+
+  "distributed::TokenSharingRpcServer" -> "communication::rpc::Server";
+  "distributed::TokenSharingRpcServer" -> "distributed::Coordination";
+  "distributed::TokenSharingRpcServer" -> "distributed::TokenSharingRpcClients";
+  "distributed::TokenSharingRpcServer" -> "distributed::dgp::Partitioner";
+
+  "distributed::dgp::Partitioner" -> "distributed::DistributedGraphDb" [style=dashed];
+
+  "distributed::dgp::Partitioner" -> "distributed::dgp::VertexMigrator";
+  "distributed::dgp::VertexMigrator" -> "database::GraphDbAccessor" [style=dashed];
+}

docs/dev/distributed/addressing.md

@@ -0,0 +1,43 @@
+# Distributed addressing
+
+In distributed Memgraph a single graph element must be owned by exactly
+one worker. It is possible that multiple workers have cached copies of
+a single graph element (which is inevitable), but there is only one
+owner.
+
+The owner of a graph element can change. This is not yet implemented,
+but is intended. Graph partitioning is intended to be dynamic.
+
+Graph elements refer to other graph elements that are possibly on some
+other worker. Even though each graph element is identified with a unique
+ID, that ID does not contain the information about where that element
+currently resides (which worker is the owner).
+
+Thus we introduce the concept of a global address. It indicates both
+which graph element is referred to (it's global ID), and where it
+resides. Semantically it's a pair of two elements, but for efficiency
+it's stored in 64 bits.
+
+The global address is efficient for usage in a cluster: it indicates
+where something can be found. However, finding a graph element based on
+it's ID is still not a free operation (in the current implementation
+it's a skiplist lookup). So, whenever possible, it's better to use local
+addresses (pointers).
+
+Succinctly, the requirements for addressing are:
+- global addressing containing location info
+- fast local addressing
+- storage of both types in the same location efficiently
+- translation between the two
+
+The `storage::Address` class handles the enumerated storage
+requirements. It stores either a local or global address in the size of
+a local pointer (typically 8 bytes).
+
+Conversion between the two is done in multiple places. The general
+approach is to use local addresses (when possible) only for local
+in-memory handling. All the communication and persistence uses global
+addresses. Also, when receiving address from another worker, attempt to
+localize addresses as soon as possible, so that least code has to worry
+about potential inefficiency of using a global address for a local graph
+element.

docs/dev/distributed/durability.md

@@ -0,0 +1,50 @@
+# Distributed durability
+
+Durability in distributed is slightly different then in single-node as
+the state itself is shared between multiple workers and none of those
+states are independent.
+
+Note that recovering from persistent storage must result in a stable
+database state. This means that across the cluster the state
+modification of every transaction that was running is either recovered
+fully or not at all. Also, if transaction A committed before transaction B,
+then if B is recovered so must A.
+
+## Snapshots
+
+It is possibly avoidable but highly desirable that the database can be
+recovered from snapshot only, without relying on WAL files. For this to
+be possible in distributed, it must be ensured that the same
+transactions are recovered on all the workers (including master) in the
+cluster. Since the snapshot does not contain information about which
+state change happened in which transaction, the only way to achieve this
+is to have synchronized snapshots. This means that the process of
+creating a snapshot, which is in itself transactional (it happens within
+a transaction and thus observes some consistent database state), must
+happen in the same transaction. This is achieved by the master starting
+a snapshot generating transaction and triggering the process on all
+workers in the cluster.
+
+## WAL
+
+Unlike the snapshot, write-ahead logs contain the information on which
+transaction made which state change. This makes it possible to include
+or exclude transactions during the recovery process. What is necessary
+however is a global consensus on which of the transactions should be
+recovered and which not, to ensure recovery into a consistent state.
+
+It would be possible to achieve this with some kind of synchronized
+recovery process, but it would impose constraints on cluster startup and
+would not be trivial.
+
+A simpler alternative is that the consensus is achieved beforehand,
+while the database (to be recovered) is still operational. What is
+necessary is to keep track of which transactions are guaranteed to
+have been flushed to the WAL files on all the workers in the cluster. It
+makes sense to keep this record on the master, so a mechanism is
+introduced which periodically pings all the workers, telling them to
+flush their WALs, and writes some sort of a log indicating that this has
+been confirmed. The downside of this is a periodic broadcast must be
+done, and that potentially slightly less data can be recovered in the
+case of a crash then if using a post-crash consensus. It is however much
+simpler to implement.

docs/dev/distributed/dynamic_graph_partitioning.md

@@ -0,0 +1,51 @@
+## Dynamic Graph Partitioning
+
+Memgraph supports dynamic graph partitioning similar to the Spinner algorithm, 
+mentioned in this paper: [https://arxiv.org/pdf/1404.3861.pdf].
+
+Dgp is useful because it tries to group `local` date on the same worker, i.e. 
+it tries to keep closely connected data on one worker. It tries to avoid jumps 
+across workers when querying/traversing the distributed graph.
+
+### Our implementation
+
+It works independently on each worker but it is always running the migration 
+on only one worker at the same time. It achieves that by sharing a token 
+between workers, and the token ownership is transferred to the next worker 
+when the current worker finishes its migration step. 
+
+The reason that we want workers to work in  disjoint time slots is it avoid 
+serialization errors caused by creating/removing edges of vertices during 
+migrations, which might cause an update of some vertex from two or more 
+different transactions.
+
+### Migrations
+
+For each vertex and workerid (label in the context of Dgp algorithm) we define  
+a score function. Score function takes into account labels of surrounding 
+endpoints of vertex edges (in/out) and the capacity of the worker with said 
+label. Score function loosely looks like this 
+```
+locality(v, l) = 
+count endpoints of edges of vertex `v` with label `l` / degree of `v`
+
+capacity(l) = 
+number of vertices on worker `l` divided by the worker capacity 
+(usually equal to the average number of vertices per worker)
+
+score(v, l) = locality(v, l) - capacity(l)
+```
+We also define two flags alongside ```dynamic_graph_partitioner_enabled```, 
+   ```dgp_improvement_threshold``` and ```dgp_max_batch_size```.
+
+These two flags are used during the migration phase.
+When deciding if we need to migrate some vertex `v` from worker `l1` to worker 
+`l2` we examine the difference in scores, i.e.
+if score(v, l1) - dgp_improvement_threshold / 100 < score(v, l2) then we 
+migrate the vertex. 
+
+Max batch size flag limits the number of vertices we can transfer in one batch 
+(one migration step). 
+Setting this value to a too large value will probably cause
+a lot of interference with client queries, and having it a small value
+will slow down convergence of the algorithm.

docs/dev/distributed/general_concepts.md

@@ -0,0 +1,54 @@
+# Memgraph distributed
+
+This chapter describes some of the concepts used in distributed
+Memgraph. By "distributed" here we mean the sharding of a single graph
+onto multiple processing units (servers).
+
+## Conceptual organization
+
+There is a single master and multiple workers. The master contains all
+the global sources of truth (transaction engine,
+[label|edge-type|property] to name mappings). Also, in the current
+organization it is the only one that contains a Bolt server (for
+communication with the end client) and an interpretation engine. Workers
+contain the data and means of subquery interpretation (query plans
+recieved from the master) and means of communication with the master and
+other workers.
+
+In many query plans the load on the master is much larger then the load
+on the workers. For that reason it might be beneficial to make the
+master contain less data (or none at all), and/or having multiple
+interpretation masters.
+
+## Logic organization
+
+Both the distributed and the single node Memgraph use the same codebase.
+In cases where the behavior in single-node differs from that in
+distributed, some kind of dynamic behavior change is implemented (either
+through inheritance or conditional logic).
+
+### GraphDb
+
+The `database::GraphDb` is an "umbrella" object for parts of the
+database such as storage, garbage collection, transaction engine etc.
+There is a class heirarchy of `GraphDb` implementations, as well as a
+base interface object. There are subclasses for single-node, master and
+worker deplotyments. Which implementation is used depends on the
+configuration processed in the `main` entry point of memgraph.
+
+The `GraphDb` interface exposes getters to base classes of
+other similar heirarchies (for example to `tx::Engine`). In that way
+much of the code that uses those objects (for example query plan
+interpretation) is agnostic to the type of deployment.
+
+### RecordAccessors
+
+The functionality of `RecordAccessors` and it's subclasses is already
+documented. It's important to note that the same implementation of
+accessors is used in all deployments, with internal changes of behavior
+depending on the locality of the graph element (vertex or edge) the
+accessor represents. For example, if the graph element is local, an
+update operation on an accessor will make the necessary MVCC ops, update
+local data, indexes, the write-ahead log etc. However, if the accessor
+represents a remote graph element, an update will trigger an RPC message
+to the owner about the update and a change in the local cache.

docs/dev/distributed/updates.md

@@ -0,0 +1,103 @@
+# Distributed updates
+
+Operations that modify the graph state are somewhat more complex in the
+distributed system, as opposed to a single-node Memgraph deployment. The
+complexity arises from two factors.
+
+First, the data being modified is not necessarily owned by the worker
+performing the modification. This situation is completely valid workers
+execute parts of the query plan and parts must be executed by the
+master.
+
+Second, there are less guarantees regarding multi-threaded access. In
+single-node Memgraph it was guaranteed that only one transaction will be
+performing database work in a single transaction. This implied that
+per-version storage could be thread-unsafe. In distributed Memgraph it
+is possible that multiple threads could be performing work in the same
+transaction as a consequence of the query being executed at the same
+time on multiple workers and those executions interacting with the
+globally partitioned database state.
+
+## Deferred state modification
+
+Making the per-version data storage thread-safe would most likely have a
+performance impact very undesirable in a transactional database intended
+for high throughput.
+
+An alternative is that state modification over unsafe structures is not
+performed immediately when requested, but postponed until it is safe to
+do (there is is a guarantee of no concurrent access).
+
+Since local query plan execution is done the same way on local data as
+it is in single-node Memgraph, it is not possible to deffer that part of
+the modification story. What can be deferred are modifications requested
+by other workers. Since local query plan execution still is
+single-threaded, this approach is safe.
+
+At the same time those workers requesting the remote update can update
+local copies (caches) of the not-owned data since that cache is only
+being used by the single, local-execution thread.
+
+### Visibility
+
+Since updates are deferred the question arises: when do the updates
+become visible? The above described process offers the following
+visibility guarantees:
+- updates done on the local state are visible to the owner
+- updates done on the local state are NOT visible to anyone else during
+  the same (transaction + command)
+- updates done on remote state are deferred on the owner and not
+  visible to the owner until applied
+- updates done on the remote state are applied immediately to the local
+  caches and thus visible locally
+
+This implies an inconsistent view of the database state. In a concurrent
+execution of a single query this can hardly be avoided and is accepted
+as such. It does not change the Cypher query execution semantic in any
+of the well-defined scenarios. It possibly changes some of the behaviors
+in which the semantic is not well defined even in single-node execution.
+
+### Synchronization, update application
+
+In many queries it is mandatory to observe the latest global graph state
+(typically when returning it to the client). That means that before that
+happens all the deferred updates need to be applied, and all the caches
+to remote data invalidated. Exactly this happens when executing queries
+that modify the graph state. At some point a global synchronization
+point is reached. First it is waited that all workers finish the
+execution of query plan parts performing state modifications. After that
+all the workers are told to apply the deferred updates they received to
+their graph state. Since there is no concurrent query plan execution,
+this is safe. Once that is done all the local caches are cleared and the
+requested data can be returned to the client.
+
+### Command advancement
+
+In complex queries where a read part follows a state modification part
+the synchronization process after the state modification part is
+followed by command advancement, like in single-node execution.
+
+## Creation
+
+Graph element creation is not deferred. This is practical because the
+response to a creation is the global ID of the newly created element. At
+the same time it is safe because no other worker (including the owner)
+will be using the newly added graph element.
+
+## Updating
+
+Updating is deferred, as described. Note that this also means that
+record locking conflicts are deferred and serialization errors
+(including lock timeouts) are postponed until the deferred update
+application phase. In certain scenarios it might be beneficial to force
+these errors to happen earlier, when the deferred update request is
+processed.
+
+## Deletion
+
+Deletion is also deferred. Deleting an edge implies a modification of
+it's endpoint vertices, which must be deferred as those data structures
+are not thread-safe. Deleting a vertex is either with detaching, in
+which case an arbitrary number of updates are implied in the vertex's
+neighborhood, or without detaching which relies on checking the current
+state of the graph which is generally impossible in distributed.

docs/dev/durability/snapshots.md

@@ -0,0 +1,22 @@
+# Snapshots
+
+A "snapshot" is a record of the current database state stored in permanent
+storage. Note that the term "snapshot" is used also in the context of
+the transaction engine to denote a set of running transactions.
+
+A snapshot is written to the file by Memgraph periodically if so
+configured. The snapshot creation  process is done within a transaction created
+specifically for that purpose. The transaction is needed to ensure that
+the stored state is internally consistent.
+
+The database state can be recovered from the snapshot during startup, if
+so configured. This recovery works in conjunction with write-ahead log
+recovery.
+
+A single snapshot contains all the data needed to recover a database. In
+that sense snapshots are independent of each other and old snapshots can
+be deleted once the new ones are safely stored, if it is not necessary
+to revert the database to some older state.
+
+The exact format of the snapshot file is defined inline in the snapshot
+creation code.

docs/dev/durability/wal.md

@@ -0,0 +1,55 @@
+# Write-ahead logging
+
+Typically WAL denotes the process of writing a "log" of database
+operations (state changes) to persistent storage before committing the
+transaction, thus ensuring that the state can be recovered (in the case
+of a crash) for all the transactions which the database committed.
+
+The WAL is a fine-grained durability format. It's purpose is to store
+database changes fast. It's primary purpose is not to provide
+space-efficient storage, nor to support fast recovery. For that reason
+it's often used in combination with a different persistence mechanism
+(in Memgraph's case the "snapshot") that has complementary
+characteristics.
+
+### Guarantees
+
+Ensuring that the log is written before the transaction is committed can
+slow down the database. For that reason this guarantee is most often
+configurable in databases.
+
+Memgraph offers two options for the WAL. The default option, where the WAL is
+flushed to the disk periodically and transactions do not wait for this to
+complete, introduces the risk of database inconsistency because an operating
+system or hardware crash might lead to missing transactions in the WAL. Memgraph
+will handle this as if those transactions never happened. The second option,
+called synchronous commit, will instruct Memgraph to wait for the WAL to be
+flushed to the disk when a transactions completes and the transaction will wait
+for this to complete. This option can be turned on with the
+`--synchronous-commit` command line flag.
+
+### Format
+
+The WAL file contains a series of DB state changes called `StateDelta`s.
+Each of them describes what the state change is and in which transaction
+it happened. Also some kinds of meta-information needed to ensure proper
+state recovery are recorded (transaction beginnings and commits/abort).
+
+The following is guaranteed w.r.t. `StateDelta` ordering in
+a single WAL file:
+- For two ops in the same transaction, if op A happened before B in the
+  database, that ordering is preserved in the log.
+- Transaction begin/commit/abort messages also appear in exactly the
+  same order as they were executed in the transactional engine.
+
+### Recovery
+
+The database can recover from the WAL on startup. This works in
+conjunction with snapshot recovery. The database attempts to recover from
+the latest snapshot and then apply as much as possible from the WAL
+files. Only those transactions that were not recovered from the snapshot
+are recovered from the WAL, for speed efficiency. It is possible (but
+inefficient) to recover the database from WAL only, provided all the WAL
+files created from DB start are available. It is not possible to recover
+partial database state (i.e. from some suffix of WAL files, without the
+preceding snapshot).

docs/dev/lcp.md

@@ -0,0 +1,775 @@
+# Lisp C++ Preprocessor (LCP)
+
+In our development process we are using Common Lisp to generate some parts of
+the C++ codebase. The idea behind this is supplementing C++ with better
+meta-programming capabilities to automate tasks and prevent bugs due to code
+duplication. Primary candidate for using more powerful meta-programming is
+generating serialization code. Such code is almost always the same: go through
+all `struct` or `class` members and invoke the serialization function on them.
+Writing such code manually is error prone when adding members, because you may
+easily forget to correctly update the serialization code. Thus, the Lisp C++
+Preprocessor was born. It is hooked in our build process as a step before
+compilation. The remainder of the document describes how to use LCP and its
+features.
+
+Contents
+
+  * [Running LCP](#running-lcp)
+  * [Writing LCP](#writing-lcp)
+    - [Inlining C++ in Common Lisp](#inlining-cpp)
+    - [C++ Namespaces](#cpp-namespaces)
+    - [C++ Enumerations](#cpp-enums)
+    - [C++ Classes & Structs](#cpp-classes)
+    - [Defining an RPC](#defining-an-rpc)
+    - [Cap'n Proto Serialization](#capnp-serial)
+
+## Running LCP
+
+You can generate C++ from an LCP file by running the following command.
+
+`./tools/lcp <path-to-file.lcp>`
+
+The LCP will produce a `path-to-file.hpp` file and potentially a
+`path-to-file.lcp.cpp` file. The `.cpp` file is generated if some parts of the
+code need to be in the implementation file. This is usually the case when
+generating serialization code. Note that the `.cpp` file has the extension
+appended to `.lcp`, so that you are free to define your own `path-to-file.cpp`
+which includes the generated `path-to-file.hpp`.
+
+One serialization format uses Cap'n Proto library, but to use it, you need to
+provide an ID. The ID is generated by invoking `capnp id`. When you want to
+generate Cap'n Proto serialization, you need to pass the generated ID to LCP.
+
+`./tools/lcp <path-to-file.lcp> $(capnp id)`
+
+Generating Cap'n Proto serialization will produce an additional file,
+`path-to-file.capnp`, which contains the serialization schema.
+
+You may wonder why the LCP doesn't invoke `capnp id` itself. Unfortunately,
+such behaviour would be wrong when running LCP on the same file multiple
+times. Each run would produce a different ID and the serialization code would
+be incompatible between versions.
+
+### CMake
+
+The LCP is run in CMake using the `add_lcp` function defined in
+`CMakeLists.txt`. You can take a look at the function documentation there for
+information on how to add your new LCP files to the build system.
+
+## Writing LCP
+
+A LCP file should have the `.lcp` extension, but the code written is
+completely valid Common Lisp code. This means that you have a complete
+language at your disposal before even the C++ is compiled. You can view this
+as similar to the C++ templates and macros, but they do not have access to
+a complete language.
+
+Besides Common Lisp, you are allowed to write C++ code verbatim. This means
+that C++ and Lisp code coexist in the file. How to do that, as well as other
+features are described below.
+
+### Inlining C++ in Common Lisp {#inlining-cpp}
+
+To insert C++ code, you need to use a `#>cpp ... cpp<#` block. This is most
+often used at the top of the file to write Doxygen documentation and put some
+includes. For example:
+
+```cpp
+#>cpp
+/// @file My Doxygen style documentation about this file
+
+#pragma once
+
+#include <vector>
+cpp<#
+```
+
+The above code will be pasted as is into the generated header file. If you
+wish to have a C++ block in the `.cpp` implementation file instead, you should
+use `lcp:in-impl` function. For example:
+
+```cpp
+(lcp:in-impl
+#>cpp
+void MyClass::Method(int awesome_number) {
+  // Do something with awesome_number
+}
+cpp<#)
+```
+
+The C++ block also supports string interpolation with a syntax akin to shell
+variable access, `${lisp-variable}`. At the moment, only variables are
+supported and they have to be pretty printable in Common Lisp (i.e. support
+the `~A` format directive). For example, we can make a precomputed sinus
+function for integers from 0 to 5:
+
+```lisp
+(let ((sin-from-0-to-5
+        (format nil "~{~A~^, ~}" (loop for i from 0 below 5 collect (sin i)))))
+  #>cpp
+  static const double kSinFrom0To5[] = {${sin-from-0-to-5}};
+  cpp<#)
+```
+
+The following will be generated.
+
+```cpp
+static const double kSinFrom0To5[] = {0.0, 0.84147096, 0.9092974, 0.14112, -0.7568025};
+```
+
+Since you have a complete language at your disposal, this is a powerful tool
+to generate tables for computations which would take a very long time during
+the execution of the C++ program.
+
+### C++ Namespaces {#cpp-namespaces}
+
+Although you can use inline C++ to open and close namespaces, it is
+recommended to use `lcp:namespace` and `lcp:pop-namespace` functions. LCP will
+report an error if you have an unclosed namespace, unlike Clang and GCC which
+most of the times give strange errors due to C++ grammar ambiguity. Additional
+benefit is that LCP will track the namespace stack and correctly wrap any C++
+code which should be put in the `.cpp` file.
+
+For example:
+
+```lisp
+;; example.lcp
+(lcp:namespace utils)
+
+;; Function declaration in header
+#>cpp
+bool StartsWith(const std::string &string, const std::string &prefix);
+cpp<#
+
+;; Function implementation in implementation file
+(lcp:in-impl
+#>cpp
+bool StartsWith(const std::string &string, const std::string &prefix) {
+  // Implementation code
+  return false;
+}
+cpp<#)
+
+(lcp:pop-namespace) ;; utils
+```
+
+The above will produce 2 files, header and implementation:
+
+```cpp
+// example.hpp
+namespace utils {
+
+bool StartsWith(const std::string &string, const std::string &prefix);
+
+}
+```
+
+```cpp
+// example.lcp.cpp
+namespace utils {
+
+bool StartsWith(const std::string &string, const std::string &prefix) {
+  // Implementation code
+  return false;
+}
+
+}
+```
+
+### C++ Enumerations {#cpp-enums}
+
+LCP provides a `lcp:define-enum` macro to define a C++ `enum class` type. This
+will make LCP aware of the type and all its possible values. This makes it
+possible to generate the serialization code. In the future, LCP may generate
+"string to enum" and "enum to string" functions.
+
+Example:
+
+```lisp
+(lcp:define-enum days-in-week
+  (monday tuesday wednesday thursday friday saturday sunday)
+  ;; Optional documentation
+  (:documentation "Enumerates days of the week")
+  ;; Optional directive to generate serialization code
+  (:serialize))
+```
+
+Produces:
+
+```cpp
+/// Enumerates days of the week
+enum class DaysInWeek {
+  MONDAY,
+  TUESDAY,
+  WEDNESDAY,
+  THURSDAY,
+  FRIDAY,
+  SATURDAY,
+  SUNDAY
+};
+
+// serialization code ...
+```
+
+### C++ Classes & Structs {#cpp-classes}
+
+For defining C++ classes, there is a `lcp:define-class` macro. Its counterpart
+for structures is `lcp:define-struct`. They are exactly the same, but
+`lcp:define-struct` will put members in public scope by default. Just like in
+C++.
+
+Defining classes is a bit more involved, because they have many customization
+options. They syntax follows the syntax of class definition in Common Lisp
+(see `defclass`).
+
+Basic example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((primitive-value :int64_t)
+   (stl-vector "std::vector<int>"))
+  ;; Optional documentation
+  (:documentation "My class documentation")
+  ;; Define explicitly public, protected or private code. All are optional.
+  (:public #>cpp // some public code, e.g. methods cpp<#)
+  (:protected #>cpp // protected cpp<#)
+  (:private #>cpp //private cpp<#))
+```
+
+The above will generate:
+
+```cpp
+/// My class documentation
+class MyClass {
+ public:
+  // some public code, e.g. methods
+
+ protected:
+  // protected
+
+ private:
+  // private
+
+  int64_t primitive_value_;
+  std::vector<int> stl_vector_;
+};
+```
+
+As you can see, members in LCP are followed by a type. For primitive types, a
+Lisp keyword is used. E.g. `:int64_t`, `:bool`, etc. Other types, like STL
+containers use a valid C++ string to specify type.
+
+C++ supports nesting types inside a class. You can do the same in LCP inside
+any of the scoped additions.
+
+For example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((member "NestedType")
+   (value "NestedEnum"))
+  (:private
+    (lcp:define-enum nested-enum (first-value second-value))
+
+    (lcp:define-class nested-type ()
+      ((member :int64_t)))
+
+    #>cpp
+    // Some other C++ code
+    cpp<#))
+```
+
+The above should produce expected results.
+
+You can add a base classes after the class name. The name should be a Lisp
+symbol for base classes defined through `lcp:define-class`, so that LCP tracks
+the inheritance. Otherwise, it should be a string.
+
+For example:
+
+```lisp
+(lcp:define-class derived (my-class "UnknownInterface")
+  ())
+```
+
+Will generate:
+
+```cpp
+class Derived : public MyClass, public UnknownInterface {
+};
+```
+
+Similarly, you can specify template parameters. Instead of giving just a name
+to `define-class`, you give a list where the first element is the name of the
+class, while others name the template parameters.
+
+```lisp
+(lcp:define-class (my-map t-key t-value) ()
+  ((underlying-map "std::unordered_map<TKey, TValue>")))
+```
+
+The above will generate:
+
+```cpp
+template <class TKey, class TValue>
+class MyMap {
+ private:
+  std::unordered_map<TKey, TValue> underlying_map_;
+};
+```
+
+Other than tweaking the class definition, you can also do additional
+configuration of members. The following options are supported.
+
+  * `:initval` -- sets the initial value of a member
+  * `:reader` -- generates a public getter
+  * `:scope` -- set the scope of a member, one of `:public`, `:private` or
+    `:protected`
+  * `:documentation` -- Doxygen documentation of a member
+  * various serialization options which are explained later
+
+For example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((member "std::vector<int>" :scope :protected :initval "1, 2, 3" :reader t
+                              :documentation "Member documentation")))
+```
+
+Will generate:
+
+```cpp
+class MyClass {
+ public:
+  const auto &member() { return member_; }
+
+ protected:
+  /// Member documentation
+  std::vector<int> member_{1, 2, 3};
+};
+```
+
+### Defining an RPC
+
+In our codebase, we have implemented remote procedure calls. These are used
+for communication between Memgraph instances in a distributed system. Each RPC
+is registered by its type and requires serializable data structures. Writing
+RPC compliant structure requires a lot of boilerplate. To ease the pain of
+defining a new RPC we have a macro, `lcp:define-rpc`.
+
+Definition consists of 2 parts: request and response. You can specify members
+of each part. Member definition is the same as in `lcp:define-class`.
+
+For example:
+
+```lisp
+(lcp:define-rpc query-result
+  (:request
+    ((tx-id "tx::TransactionId")
+     (query-id :int64_t)))
+  (:response
+    ((values "std::vector<int>"))))
+```
+
+The above will generate relatively large amount of C++ code, which is omitted
+here as the details aren't important for understanding the use. Examining the
+generated code is left as an exercise for the reader.
+
+The important detail is that in C++ you will have a `QueryResultRpc`
+structure, which is used to register the behaviour of an RPC server. You need
+to perform the registration manually. For example:
+
+```cpp
+// somewhere in code you have a server instance
+rpc_server.Register<QueryResultRpc>(
+    [](const auto &req_reader, auto *res_builder) {
+      QueryResultReq request;
+      Load(&request, req_reader);
+      // process the request and send the response
+      QueryResultRes response(values_for_response);
+      Save(response, res_builder);
+    });
+
+
+// somewhere else you have a client which sends the RPC
+tx::TransactionId tx_id = ...
+int64_t query_id = ...
+auto response = rpc_client.template Call<QueryResultRpc>(tx_id, query_id);
+if (response) {
+  const auto &values = response->getValues();
+  // do something with values
+}
+```
+
+RPC structures use Cap'n Proto for serialization. The above variables
+`req_reader` and `res_builder` are used to access Cap'n Proto structures.
+Obviously, the LCP will generate the Cap'n Proto schema alongside the C++
+code for serialization.
+
+
+### Cap'n Proto Serialization {#capnp-serial}
+
+Primary purpose of LCP was to make serialization of types easier. Our
+serialization library of choice for C++ is Cap'n Proto. LCP provides
+generation and tuning of its serialization code. Previously, LCP supported
+Boost.Serialization, but it was removed.
+
+To specify a class or structure for serialization, you may pass a
+`:serialize :capnp` option when defining such type. (Note that
+`lcp:define-enum` takes `:serialize` without any arguments).
+
+For example:
+
+```lisp
+(lcp:define-struct my-struct ()
+  ((member :int64_t))
+  (:serialize :capnp))
+```
+
+`:serialize` option will generate a Cap'n Proto schema of the class and store
+it in the `.capnp` file. C++ code will be generated for saving and loading
+members:
+
+```cpp
+// Top level functions
+void Save(const MyStruct &self, capnp::MyStruct::Builder *builder);
+void Load(MyStruct *self, const capnp::MyStruct::Reader &reader);
+```
+
+Since we use top level functions, the class needs to have some sort of public
+access to its members.
+
+The schema file will be namespaced in `capnp`. To change add a prefix
+namespace use `lcp:capnp-namespace` function. For example, if we use
+`(lcp:capnp-namespace "my_namespace")` then the reader and builder would be in
+`my_namespace::capnp`.
+
+Serializing a class hierarchy is also supported. The most basic case with
+single inheritance works out of the box. Handling other cases is explained in
+later sections.
+
+For example:
+
+```lisp
+(lcp:define-class base ()
+  ((base-member "std::vector<int64_t>" :scope :public))
+  (:serialize :capnp))
+
+(lcp:define-class derived (base)
+  ((derived-member :bool :scope :public))
+  (:serialize :capnp))
+```
+
+Note that all classes need to have the `:serialize` option set. Signatures of
+`Save` and `Load` functions are changed to accept reader and builder to the
+base class. The `Load` function now takes a `std::unique_ptr<T> *` which is
+used to take ownership of a concrete type. This approach transfers the
+responsibility of type allocation and construction from the user of `Load` to
+`Load` itself.
+
+```cpp
+void Save(const Derived &self, capnp::Base::Builder *builder);
+void Load(std::unique_ptr<Base> *self, const capnp::Base::Reader &reader);
+```
+
+#### Multiple Inheritance
+
+Cap'n Proto does not support any form of inheritance, instead we are
+handling it manually. Single inheritance was relatively easy to add to Cap'n
+Proto, we simply enumerate all derived types inside the union of a base type.
+
+Multiple inheritance is a different beast and as such is not directly
+supported.
+
+One way to use multiple inheritance is only to implement the interface of pure
+virtual classes without any members (i.e. interface classes). In such a case,
+you do not want to serialize any other base class except the primary one. To
+let LCP know that is the case, use `:ignore-other-base-classes t`. LCP will
+only try to serialize the base class that is the first (leftmost) in the list
+of super classes.
+
+```lisp
+(lcp:define-class derived (primary-base some-interface other-interface)
+  ...
+  (:serialize :capnp :ignore-other-base-classes t))
+```
+
+Another form of multiple inheritance is reusing some common code. In
+actuality, this is a very bad code practice and should be replaced with
+composition. If it would take too long to fix such code to use composition
+proper, we can tell LCP to treat such inheritance as if they are indeed
+composed. This is done via `:inherit-compose` option.
+
+For example:
+
+```lisp
+(lcp:define-class derived (first-base second-base)
+  ...
+  (:serialize :capnp :inherit-compose '(second-base)))
+```
+
+With `:inherit-compose` you can pass a list of parent classes which should be
+encoded as composition inside the Cap'n Proto schema. LCP will complain if
+there is multiple inheritance but you didn't specify `:inherit-compose`.
+
+The downside of this approach is that `Save` and `Load` will work only on
+`FirstBase`. Serializing a pointer to `SecondBase` would be incorrect.
+
+#### Inheriting C++ Class Outside of LCP
+
+Classes defined outside of `lcp:define-class` are not visible to LCP and LCP
+will not be able to generate correct serialization code.
+
+The cases so far have been only with classes that are pure interface and need
+no serialization code. This is signaled to LCP by passing the option `:base t`
+to `:serialize :capnp`. LCP will treat such classes as actually being the base
+class of a hierarchy.
+
+For example:
+
+```lisp
+(lcp:define-class my-class ("utils::TotalOrdering")
+  (...)
+  (:serialize :capnp :base t))
+
+(lcp:define-class derived (my-class)
+  (...)
+  (:serialize :capnp))
+```
+
+Only the base class for serialization has the `:base t` option set. Derived
+classes are defined as usual. This relies on the fact that we do not expect
+anyone to have a pointer to `utils::TotalOrdering` and use it for
+serialization and deserialization.
+
+#### Template Classes
+
+Currently, LCP supports the most primitive form of serializing templated
+classes. The template arguments must be provided to specify an explicit
+instantiation. Cap'n Proto does support generics, so we may want to upgrade
+LCP to use them in the future.
+
+To specify template arguments, pass a `:type-args` option. For example:
+
+```lisp
+(lcp:define-class (my-container t-value) ()
+  (...)
+  (:serialize :capnp :type-args '(my-class)))
+```
+
+The above will support serialization of `MyContainer<MyClass>` type.
+
+The syntax will work even if our templated class inherits from non-templated
+classes. All other cases of inheritance with templates are forbidden in LCP
+serialization.
+
+#### Cap'n Proto Schemas and Type Conversions
+
+You can import other serialization schemas by using `lcp:capnp-import`
+function. It expects a name for the import and the path to the schema file.
+
+For example, to import everything from `utils/serialization.capnp` under the
+name `Utils`, you can do the following:
+
+```lisp
+(lcp:capnp-import 'utils "/utils/serialization.capnp")
+```
+
+To use those types, you need to register a conversion from C++ type to schema
+type. There are two options, registering a whole file conversion with
+`lcp:capnp-type-conversion` or converting a specific class member.
+
+For example, you have a class with member of type `Bound` and there is a
+schema for it also named `Bound` inside the imported schema.
+
+You can use `lcp:capnp-type-conversion` like so:
+
+```lisp
+(lcp:capnp-type-conversion "Bound" "Utils.Bound")
+
+(lcp:define-class my-class ()
+  ((my-bound "Bound")))
+```
+
+Specifying only a member conversion can be done with `:capnp-type` member
+option:
+
+```lisp
+(lcp:define-class my-class ()
+  ((my-bound "Bound" :capnp-type "Utils.Bound")))
+```
+
+#### Custom Save and Load Hooks
+
+Sometimes the default serialization is not adequate and you may wish to
+provide your own serialization code. For those reasons, LCP provides
+`:capnp-save`, `:capnp-load` and `:capnp-init` options on each class member.
+
+The simplest is `:capnp-init` which when set to `nil` will not generate an
+`init<member>` call on a builder. Cap'n Proto requires that compound types are
+initialized before beginning to serialize its members. `:capnp-init` allows you
+to delay the initialization to your custom save code. You rarely want to set
+`:capnp-init nil`.
+
+Custom save code is added as a value of `:capnp-save`. It should be a function
+which takes 3 arguments.
+
+  1. Name of builder variable.
+  2. Name of the class (or struct) member.
+  3. Name of the member in Cap'n Proto schema.
+
+The result of the function needs to be a C++ code block.
+
+You will rarely need to use the 3rd argument, so it should be ignored in most
+cases. It is usually needed when you set `:capnp-init nil`, so that you can
+correctly initialize the builder.
+
+Similarly, `:capnp-load` expects a function taking a reader, C++ member and
+Cap'n Proto member,  then returns a C++ block.
+
+Example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((my-member "ComplexType"
+              :capnp-init nil
+              :capnp-save (lambda (builder member capnp-name)
+                            #>cpp
+                            auto data = ${member}.GetSaveData();
+                            auto my_builder = ${builder}.init${capnp-name}();
+                            my_builder.setData(data);
+                            cpp<#)
+              :capnp-load (lambda (reader member capnp-name)
+                            (declare (ignore capnp-name))
+                            #>cpp
+                            auto data = ${reader}.getData();
+                            ${member}.LoadFromData(data);
+                            cpp<#)))
+  (:serialize :capnp))
+```
+
+With custom serialization code, you may want to get additional details through
+extra arguments to `Save` and `Load` functions. This is described in the next
+section.
+
+There are also cases where you always need custom serialization code. LCP
+provides helper functions for abstracting some common details. These functions
+are listed further down in this document.
+
+#### Arguments for Save and Load
+
+Default arguments for `Save` and `Load` function are Cap'n Proto builder and
+reader, respectively. In some cases you may wish to send additional arguments.
+This is most commonly needed when tracking `shared_ptr` serialization, to
+avoid serializing the same pointer multiple times.
+
+Additional arguments are specified by passing `:save-args` and `:load-args`.
+You can specify either of them, but in most cases you want both.
+
+For example:
+
+```lisp
+;; Class for tracking details during save
+(lcp:define-class save-helper ()
+  (...))
+
+;; Class for tracking details during load
+(lcp:define-class load-helper ()
+  (...))
+
+(lcp:define-class my-class ()
+  ((member "std::shared_ptr<int>"
+           :capnp-save ;; custom save
+           :capnp-load ;; custom load
+    ))
+  (:serialize :capnp
+              :save-args '((save-helper "SaveHelper *"))
+              :load-args '((load-helper "LoadHelper *"))))
+```
+
+The custom serialization code will now have access to `save_helper` and
+`load_helper` variables in C++. You can add more arguments by expanding the
+list of pairs, e.g.
+
+```lisp
+:save-args '((first-helper "SomeType *") (second-helper "OtherType *") ...)
+```
+
+#### Custom Serialization Helper Functions
+
+##### Helper for `std::optional`
+
+When using `std::optional` with primitive C++ types or custom types known to
+LCP, you do not need to use any helper. In the example below, things should be
+serialized as expected:
+
+```lisp
+(lcp:define-class my-class-with-primitive-optional ()
+  ((primitive-optional "std::experimental::optional<int64_t>"))
+  (:serialize :capnp))
+
+(lcp:define-class my-class-with-known-type-optional ()
+  ((known-type-optional "std::experimental::optional<MyClassWithPrimitiveOptional>"))
+  (:serialize :capnp))
+```
+
+In cases when the value contained in `std::optional` needs custom
+serialization code you may use `lcp:capnp-save-optional` and
+`lcp:capnp-load-optional`.
+
+Both functions expect 3 arguments.
+
+  1. Cap'n Proto type in C++.
+  2. C++ type of the value inside `std::optional`.
+  3. Optional C++ lambda code.
+
+The lambda code is optional, because LCP will generate the default
+serialization code which invokes `Save` and `Load` function on the value
+stored inside the optional. Since most of the serialized classes follow the
+convention, you will rarely need to provide this 3rd argument.
+
+For example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((member "std::experimental::optional<SomeType>"
+           :capnp-save (lcp:capnp-save-optional
+                         "capnp::SomeType" "SomeType"
+                         "[](auto *builder, const auto &val) { ... }")
+           :capnp-load (lcp:capnp-load-optional
+                         "capnp:::SomeType" "SomeType"
+                         "[](const auto &reader) { ... return loaded_val; }"))))
+```
+
+##### Helper for `std::vector`
+
+For custom serialization of vector elements, you may use
+`lcp:capnp-save-vector` and `lcp:capnp-load-vector`. They function exactly the
+same as helpers for `std::optional`.
+
+##### Helper for enumerations
+
+If the enumeration is defined via `lcp:define-enum`, the default LCP
+serialization should generate the correct code.
+
+However, if LCP cannot infer the serialization code, you can use helper
+functions `lcp:capnp-save-enum` and `lcp:capnp-load-enum`. Both functions
+require 3 arguments.
+
+  1. C++ type of equivalent Cap'n Proto enum.
+  2. Original C++ enum type.
+  3. List of enumeration values.
+
+Example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((enum-value "SomeEnum"
+               :capnp-init nil ;; must be set to nil
+               :capnp-save (lcp:capnp-save-enum
+                             "capnp::SomeEnum" "SomeEnum"
+                             '(first-value second-value))
+               :capnp-load (lcp:capnp-load-enum
+                             "capnp::SomeEnum" "SomeEnum"
+                             '(first-value second-value)))))
+```

docs/dev/other-code-conventions.md

@@ -0,0 +1,15 @@
+# Other Code Conventions
+
+While we are mainly programming in C++, we do use other programming languages
+when appropriate. This chapter describes conventions for such code.
+
+## Python
+
+Code written in Python should adhere to
+[PEP 8](https://www.python.org/dev/peps/pep-0008/). You should run `flake8` on
+your code to automatically check compliance.
+
+## Common Lisp
+
+Code written in Common Lisp should adhere to
+[Google Common Lisp Style](https://google.github.io/styleguide/lispguide.xml).

docs/dev/quick-start.md

@@ -76,3 +76,13 @@ To make extra sure, run the unit tests:
 If you have any trouble running the above commands, contact your nearest
 developer who successfully built Memgraph. Ask for help and insist on getting
 this document updated with correct steps!
+
+## Next Steps
+
+Familiarise yourself with our code conventions:
+
+  * [C++ Code](cpp-code-conventions.md)
+  * [Other Code](other-code-conventions.md)
+
+Take a look at the list of [required reading](required-reading.md) for
+brushing up on technical skills.

docs/dev/required-reading.md

@@ -0,0 +1,129 @@
+# Required Reading
+
+This chapter lists a few books that should be read by everyone working on
+Memgraph. Since Memgraph is developed primarily with C++, Python and Common
+Lisp, books are oriented around those languages. Of course, there are plenty
+of general books which will help you improve your technical skills (such as
+"The Pragmatic Programmer", "The Mythical Man-Month", etc.), but they are not
+listed here. This way the list should be kept short and the *required* part in
+"Required Reading" more easily honored.
+
+Some of these books you may find in our office, so feel free to pick them up.
+If any are missing and you would like a physical copy, don't be afraid to
+request the book for our office shelves.
+
+Besides reading, don't get stuck in a rut and be a
+[Blub Programmer](http://www.paulgraham.com/avg.html).
+
+## Effective C++ by Scott Meyers
+
+Required for C++ developers.
+
+The book is a must-read as it explains most common gotchas of using C++. After
+reading this book, you are good to write competent C++ which will pass code
+reviews easily.
+
+## Effective Modern C++ by Scott Meyers
+
+Required for C++ developers.
+
+This is a continuation of the previous book, it covers updates to C++ which
+came with C++11 and later. The book isn't as imperative as the previous one,
+but it will make you aware of modern features we are using in our codebase.
+
+## Practical Common Lisp by Peter Siebel
+
+Required for Common Lisp developers.
+
+Free: http://www.gigamonkeys.com/book/
+
+We use Common Lisp to generate C++ code and make our lives easier.
+Unfortunately, not many developers are familiar with the language. This book
+will make you familiar very quickly as it has tons of very practical
+exercises. E.g. implementing unit testing library, serialization library and
+bundling all that to create a mp3 music server.
+
+## Effective Python by Brett Slatkin
+
+(Almost) required reading for Python developers.
+
+Why the "almost"? Well, Python is relatively easy to pick up and you will
+probably learn all the gotchas during code review from someone more
+experienced. This makes the book less necessary for a newcomer to Memgraph,
+but the book is not advanced enough to delegate it to
+[Advanced Reading](#advanced-reading). The book is written in similar vein as
+the "Effective C++" ones and will make you familiar with nifty Python features
+that make everyone's lives easier.
+
+# Advanced Reading
+
+The books listed below are not required reading, but you may want to read them
+at some point when you feel comfortable enough.
+
+## Design Patterns by Gamma et. al.
+
+Recommended for C++ developers.
+
+This book is highly divisive because it introduced a culture centered around
+design patterns. The main issues is overuse of patterns which complicates the
+code. This has made many Java programs to serve as examples of highly
+complicated, "enterprise" code.
+
+Unfortunately, design patterns are pretty much missing
+language features. This is most evident in dynamic languages such as Python
+and Lisp, as demonstrated by
+[Peter Norvig](http://www.norvig.com/design-patterns/).
+
+Or as [Paul Graham](http://www.paulgraham.com/icad.html) put it:
+
+```
+This practice is not only common, but institutionalized. For example, in the
+OO world you hear a good deal about "patterns". I wonder if these patterns are
+not sometimes evidence of case (c), the human compiler, at work. When I see
+patterns in my programs, I consider it a sign of trouble. The shape of a
+program should reflect only the problem it needs to solve. Any other
+regularity in the code is a sign, to me at least, that I'm using abstractions
+that aren't powerful enough-- often that I'm generating by hand the expansions
+of some macro that I need to write
+```
+
+After presenting the book so negatively, why you should even read it then?
+Well, it is good to be aware of those design patterns and use them when
+appropriate. They can improve modularity and reuse of the code. You will also
+find examples of such patterns in our code, primarily Strategy and Visitor
+patterns. The book is also a good stepping stone to more advanced reading
+about software design.
+
+## Modern C++ Design by Andrei Alexandrescu
+
+Recommended for C++ developers.
+
+This book can be treated as a continuation of the previous "Design Patterns"
+book. It introduced "dark arts of template meta-programming" to the world.
+Many of the patterns are converted to use C++ templates which makes them even
+better for reuse. But, like the previous book, there are downsides if used too
+much. You should approach it with a critical eye and it will help you
+understand ideas that are used in some parts of our codebase.
+
+## Large Scale C++ Software Design by John Lakos
+
+Recommended for C++ developers.
+
+An old book, but well worth the read. Lakos presents a very pragmatic view of
+writing modular software and how it affects both development time as well as
+program runtime. Some things are outdated or controversial, but it will help
+you understand how the whole C++ process of working in a large team, compiling
+and linking affects development.
+
+## On Lisp by Paul Graham
+
+Recommended for Common Lisp developers.
+
+Free: http://www.paulgraham.com/onlisp.html
+
+An excellent continuation to "Practical Common Lisp". It starts of slow, as if
+introducing the language, but very quickly picks up speed. The main meat of
+the book are macros and their uses. From using macros to define cooperative
+concurrency to including Prolog as if it's part of Common Lisp. The book will
+help you understand more advanced macros that are occasionally used in our
+Lisp C++ Preprocessor (LCP).

docs/dev/storage/accessors.md

@@ -0,0 +1,110 @@
+# DatabaseAccessor
+
+A `DatabaseAccessor` actually wraps a transactional access to database
+data, for a single transaction. In that sense the naming is bad. It
+encapsulates references to the database and the transaction object.
+
+It contains logic for working with database content (graph element
+data) in the context of a single transaction. All CRUD operations are
+performed within a single transaction (as Memgraph is a transactional
+database), and therefore iteration over data, finding a specific graph
+element etc are all functionalities of a `GraphDbAccessor`.
+
+In single-node Memgraph the database accessor also defined the lifetime
+of a transaction. Even though a `Transaction` object was owned by the
+transactional engine, it was `GraphDbAccessor`'s lifetime that object
+was bound to (the transaction was implicitly aborted in
+`GraphDbAccessor`'s destructor, if it was not explicitly ended before
+that).
+
+# RecordAccessor
+
+It is important to understand data organization and access in the
+storage layer. This discussion pertains to vertices and edges as graph
+elements that the end client works with.
+
+Memgraph uses MVCC (documented on it's own page). This means that for
+each graph element there could be different versions visible to
+different currently executing transactions. When we talk about a
+`Vertex` or `Edge` as a data structure we typically mean one of those
+versions. In code this semantic is implemented so that both those classes
+inherit `mvcc::Record`, which in turn inherits `mvcc::Version`.
+
+Handling MVCC and visibility is not in itself trivial. Next to that,
+there is other book-keeping to be performed when working with data. For
+that reason, Memgraph uses "accessors" to define an API of working with
+data in a safe way. Most of the code in Memgraph (for example the
+interpretation code) should work with accessors. There is a
+`RecordAccessor` as a base class for `VertexAccessor` and
+`EdgeAccessor`. Following is an enumeration of their purpose.
+
+### Data access
+
+The client interacts with Memgraph using the Cypher query language. That
+language has certain semantics which imply that multiple versions of the
+data need to be visible during the execution of a single query. For
+example: expansion over the graph is always done over the graph state as
+it was at the beginning of the transaction.
+
+The `RecordAccessor` exposes functions to switch between the old and the new
+versions of the same graph element (intelligently named `SwitchOld` and
+`SwitchNew`) within a single transaction. In that way the client code
+(mostly the interpreter) can avoid dealing with the underlying MVCC
+version concepts.
+
+### Updates
+
+Data updates are also done through accessors. Meaning: there are methods
+on the accessors that modify data, the client code should almost never
+interact directly with `Vertex` or `Edge` objects.
+
+The accessor layer takes care of creating version in the MVCC layer and
+performing updates on appropriate versions.
+
+Next, for many kinds of updates it is necessary to update the relevant
+indexes. There are implicit indexes for vertex labels, as
+well as user-created indexes for (label, property) pairs. The accessor
+layer takes care of updating the indexes when these values are changed.
+
+Each update also triggers a log statement in the write-ahead log. This
+is also handled by the accessor layer.
+
+### Distributed
+
+In distributed Memgraph accessors also contain a lot of the remote graph
+element handling logic. More info on that is available in the
+documentation for distributed.
+
+### Deferred MVCC data lookup for Edges
+
+Vertices and edges are versioned using MVCC. This means that for each
+transaction an MVCC lookup needs to be done to determine which version
+is visible to that transaction. This tends to slow things down due to
+cache invalidations (version lists and versions are stored in arbitrary
+locations on the heap).
+
+However, for edges, only the properties are mutable. The edge endpoints
+and type are fixed once the edge is created. For that reason both edge
+endpoints and type are available in vertex data, so that when expanding
+it is not mandatory to do MVCC lookups of versioned, mutable data. This
+logic is implemented in `RecordAccessor` and `EdgeAccessor`.
+
+### Exposure
+
+The original idea and implementation of graph element accessors was that
+they'd prevent client code from ever interacting with raw `Vertex` or
+`Edge` data. This however turned out to be impractical when implementing
+distributed Memgraph and the raw data members have since been exposed
+(through getters to old and new version pointers). However, refrain from
+working with that data directly whenever possible! Always consider the
+accessors to be the first go-to for interacting with data, especially
+when in the context of a transaction.
+
+# Skiplist accessor
+
+The term "accessor" is also used in the context of a skiplist. Every
+operation on a skiplist must be performed within on an
+accessor. The skiplist ensures that there will be no physical deletions
+of an object during the lifetime of an accessor. This mechanism is used
+to ensure deletion correctness in a highly concurrent container.
+We only mention that here to avoid confusion regarding terminology.

docs/dev/storage/indexes.md

@@ -0,0 +1,116 @@
+# Label indexes
+
+These are unsorted indexes that contain all the vertices that have the label
+the indexes are for (one index per label). These kinds of indexes get
+automatically generated for each label used in the database.
+
+### Updating the indexes
+
+Whenever something gets added to the record we update the index (add that
+record to index). We keep an index which might contain garbage (not relevant
+records, because the value got removed or something similar) but we will
+filter it out when querying the index. We do it like this because we don't
+have to do bookkeeping and deciding if we update the index on the end of the
+transaction (commit/abort phase), moreover current interpreter advances the
+command in transaction and as such assumes that the indexes now contain
+objects added in the previous command inside this transaction, so we need to
+update over the whole scope of transaction (whenever something is added to the
+record).
+
+### Index Entries Label
+
+These kinds of indexes are internally keeping track of pair (record, vlist).
+Why do we need to keep track of exactly those two things?
+
+Problems with two different approaches
+
+1) Keep track of just the record:
+
+  - We need the `VersionList` for creating an accessor (this in itself is a
+    deal-breaker).
+  - Semantically it makes sense. An edge/vertex maps bijectionally to a
+    `VersionList`.
+  - We might try to access some members of record while the record is being
+    modified from another thread.
+  - A vertex/edge could get updated, thus expiring the record in the index.
+    The newly created record should be present in the index, but it's not.
+    Without the `VersionList` we can't reach the newly created record.
+  - Probably there are even more reasons... It should be obvious by now that
+    we need the `VersionList` in the index.
+
+2) Keep track of just the version list:
+
+  - Removing from an index is a problem for two major reasons. First, if we
+    only have the `VersionList`, checking if it should be removed implies
+    checking all the reachable records, which is not thread-safe. Second,
+    there are issues with concurrent removal and insertion. The cleanup thread
+    could determine the vertex/edge should be removed from the index and
+    remove it, while in between those ops another thread attempts to insert
+    the `VersionList` into the index. The insertion does nothing because the
+    `VersionList` is already in, but it gets removed immediately after.
+
+Because of inability to keep track of just the record, or value, we need to
+keep track of both of them.  Resolution of problems mentioned above, in the
+same order, with (record, vlist) pair
+
+  - simple `vlist.find(current transaction)` will get us the newest visible
+    record
+  - we'll never try to access some record if it's still being written since we
+    will always operate on vlist.find returned record
+  - newest record will contain that label
+  - since we have (record, vlist) pair as the key in the index when we update
+    and delete in the same time we will never delete the same record, vlist
+    pair we are adding because the record, vlist pair we are deleting is
+    already superseded by a newer record and as such won't be inserted while
+    it's being deleted
+
+### Querying the index
+
+We run through the index for the given label and do `vlist.find` operation for
+the current transaction, and check if the newest return record has that
+label. If it has it then we return it. By now you are probably wondering
+aren't we sometimes returning duplicate vlist entries? And you are wondering
+correctly, we would be returning them, but we are making sure that the entires
+in the index are sorted by their `vlist*` and as such we can filter consecutive
+duplicate `vlist*` to only return one of those while still being able to create
+an iterator to index.
+
+### Cleaning the index
+
+Cleaning the index is not as straightforward as it seems as a lot of garbage
+can accumulate, but it's hard to know when exactly can we delete some (record,
+vlist) pair. First, let's assume that we are doing the cleaning process at
+some `transaction_id`, `id` such that there doesn't exist an active transaction
+with an id lower than `id`.
+
+We scan through the whole index and for each (record, vlist) pair we first
+check if it was deleted before the id (i.e. no transaction with an id >= `id`
+will ever again see that record), if it was deleted before we might naively
+say that it's safe to delete it, but, we must take into account that when some
+new record is created from this record (update operation), that record still
+contains the label but by deleting this record we won't be able to see that
+vlist because that new record won't add again to index because we didn't
+explicitly add that label again to it.
+
+Because of this we have to 'update' this index (record, vlist) pair. We have
+to update the record to now point to a newer record in vlist, the one that is
+not deleted yet. We can do that by querying the `version_list` for the last
+record inside (oldest it has — remember that `mvcc_gc` will re-link not
+visible records so the last record will be visible for the current GC id).
+When updating the record inside the index, it's not okay to just update the
+pointer and leave the index as it is, because with updating the `record*` we
+might change the relative order of entries inside the index. We first have to
+re-insert it with new `record*`, and then delete the old entry. And we need to
+do insertion before the remove operation! Otherwise it could happen that the
+vlist with a newer record with that label won't exist while some transaction
+is querying the index.
+
+Records which we added as a consequence of deleting older records will be
+eventually removed from the index if they don't contain label because if we
+see that the record is not deleted we try to check if that record still
+contains the label. We also need to be careful here because we can't check
+that while the record is being potentially updated by some transaction (race
+condition), so we need can check if records still contain label if it's
+creation id is smaller than our `id`, as that implies that the creating
+transaction either aborted or committed as our `id` is equal to the oldest
+active transaction in time of starting the GC.

docs/dev/storage/property-storage.md

@@ -0,0 +1,131 @@
+# Property storage
+
+Although the reader is probably familiar with properties in *Memgraph*, let's
+briefly recap. 
+
+Both vertices and edges can store an arbitrary number of properties. Properties
+are, in essence, ordered pairs of property names and property values. Each
+property name within a single graph element (edge/node) can store a single
+property value. Property names are represented as strings, while property values
+must be one of the following types:
+
+ Type      | Description
+-----------|------------
+ `Null`    | Denotes that the property has no value. This is the same as if the property does not exist.
+ `String`  | A character string, i.e. text.
+ `Boolean` | A boolean value, either `true` or `false`.
+ `Integer` | An integer number.
+ `Float`   | A floating-point number, i.e. a real number.
+ `List`    | A list containing any number of property values of any supported type. It can be used to store multiple values under a single property name.
+ `Map`     | A mapping of string keys to values of any supported type.
+
+Property values are modeled in a class conveniently called `PropertyValue`.
+
+## Mapping between property names and property keys.
+
+Although users think of property names in terms of descriptive strings 
+(e.g. "location" or "department"), *Memgraph* internally converts those names
+into property keys which are, essentially, unsigned 16-bit integers.
+
+Property keys are modelled by a not-so-conveniently named class called
+`Property` which can be found in `storage/types.hpp`. The actual conversion
+between property names and property keys is done within the `ConcurrentIdMapper`
+but the internals of that implementation are out of scope for understanding
+property storage.
+
+## PropertyValueStore
+
+Both `Edge` and `Vertex` objects contain an instance of `PropertyValueStore`
+object which is responsible for storing properties of a corresponding graph
+element.
+
+An interface of `PropertyValueStore` is as follows:
+
+ Method    | Description
+-----------|------------
+ `at`      | Returns the `PropertyValue` for a given `Property` (key).
+ `set`     | Stores a given `PropertyValue` under a given `Property` (key).
+ `erase`   | Deletes a given `Property` (key) alongside its corresponding `PropertyValue`.
+ `clear`   | Clears the storage.
+ `iterator`| Provides an extension of `std::input_iterator` that iterates over storage.
+
+## Storage location
+
+By default, *Memgraph* is an in-memory database and all properties are therefore
+stored in working memory unless specified otherwise by the user. User has an
+option to specify via the command line which properties they wish to be stored
+on disk.
+
+Storage location of each property is encapsulated within a `Property` object
+which is ensured by the `ConcurrentIdMapper`. More precisely, the unsigned 16-bit
+property key has the following format:
+
+```
+|---location--|------id------|
+|-Memory|Disk-|-----2^15-----|
+``` 
+
+In other words, the most significant bit determines the location where the
+property will be stored. 
+
+### In-memory storage
+
+The underlying implementation of in-memory storage for the time being is
+`std::vector<std::pair<Property, PropertyValue>>`. Implementations of`at`, `set`
+and `erase` are linear in time. This implementation is arguably more efficient
+than `std::map` or `std::unordered_map` when the average number of properties of
+a record is relatively small (up to 10) which seems to be the case.
+
+### On-disk storage
+
+#### KVStore
+
+Disk storage is modeled by an abstraction of key-value storage as implemented in
+`storage/kvstore.hpp'. An interface of this abstraction is as follows:
+
+ Method         | Description
+----------------|------------
+ `Put`          | Stores the given value under the given key.
+ `Get`          | Obtains the given value stored under the given key.
+ `Delete`       | Deletes a given (key, value) pair from storage..
+ `DeletePrefix` | Deletes all (key, value) pairs where key begins with a given prefix.
+ `Size`         | Returns the size of the storage or, optionally, the number of stored pairs that begin with a given prefix.
+ `iterator`     | Provides an extension of `std::input_iterator` that iterates over storage.
+
+Keys and values in this context are of type `std::string`.
+
+The actual underlying implementation of this abstraction uses
+[RocksDB]{https://rocksdb.org} &mdash; a persistent key-value store for fast
+storage.
+
+It is worthy to note that the custom iterator implementation allows the user
+to iterate over a given prefix. Otherwise, the implementation follows familiar
+c++ constructs and can be used as follows:
+
+```
+KVStore storage = ...;
+for (auto it = storage.begin(); it != storage.end(); ++it) {}
+for (auto kv : storage) {}
+for (auto it = storage.begin("prefix"); it != storage.end("prefix"); ++it) {}
+```
+
+Note that it is not possible to scan over multiple prefixes. For instance, one
+might assume that you can scan over all keys that fall in a certain
+lexicographical range. Unfortunately, that is not the case and running the
+following code will result in an infinite loop with a touch of undefined
+behavior.
+
+```
+KVStore storage = ...;
+for (auto it = storage.begin("alpha"); it != storage.end("omega"); ++it) {}
+```
+
+#### Data organization on disk
+
+Each `PropertyValueStore` instance can access a static `KVStore` object that can
+store `(key, value)` pairs on disk. The key of each property on disk consists of
+two parts &mdash; a unique identifier (unsigned 64-bit integer) of the current
+record version (see mvcc docummentation for further clarification) and a 
+property key as described above. The actual value of the property is serialized
+into a bytestring using bolt `BaseEncoder`. Similarly, deserialization is
+performed by bolt `Decoder`.

docs/dev/toolchain-bootstrap.md

@@ -0,0 +1,152 @@
+# Bootstrapping Compilation Toolchain for Memgraph
+
+Requirements:
+
+  * libstdc++ shipped with gcc-6.3 or gcc-6.4
+  * cmake >= 3.1, Debian Stretch uses cmake-3.7.2
+  * clang-3.9
+
+## Installing gcc-6.4
+
+gcc-6.3 has a bug, so use the 6.4 version which is just a bugfix release.
+
+Requirements on CentOS 7:
+
+  * wget
+  * make
+  * gcc (bootstrap)
+  * gcc-c++ (bootstrap)
+  * gmp-devel (bootstrap)
+  * mpfr-devel (bootstrap)
+  * libmpc-devel (bootstrap)
+  * zip
+  * perl
+  * dejagnu (testing)
+  * expect (testing)
+  * tcl (testing)
+
+```
+wget ftp://ftp.mpi-sb.mpg.de/pub/gnu/mirror/gcc.gnu.org/pub/gcc/releases/gcc-6.4.0/gcc-6.4.0.tar.gz
+tar xf gcc-6.4.0.tar.gz
+cd gcc-6.4.0
+mkdir build
+cd build
+../configure --disable-multilib --prefix=<install-dst>
+make
+# Testing
+make -k check
+make install
+```
+
+*Do not put gcc + libs on PATH* (unless you know what you are doing).
+
+## Installing cmake-3.7.2
+
+Requirements on CentOS 7:
+
+  * wget
+  * make
+  * gcc
+  * gcc-c++
+  * ncurses-devel (optional, for ccmake)
+
+```
+wget https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
+tar xf cmake-3.7.2.tar.gz
+cd cmake-3.7.2.tar.gz
+./bootstrap --prefix<install-dst>
+make
+make install
+```
+
+Put cmake on PATH (if appropriate)
+
+**Fix the bug in CpackRPM**
+
+`"<path-to-cmake>/share/cmake-3.7/Modules/CPackRPM.cmake" line 2273 of 2442`
+
+The line
+
+```
+set(RPMBUILD_FLAGS "-bb")
+```
+needs to be before
+
+```
+if(CPACK_RPM_GENERATE_USER_BINARY_SPECFILE_TEMPLATE OR NOT CPACK_RPM_USER_BINARY_SPECFILE)
+```
+
+It was probably accidentally placed after, and is fixed in later cmake
+releases.
+
+## Installing clang-3.9
+
+Requirements on CentOS 7:
+
+  * wget
+  * make
+  * cmake
+
+```
+wget http://releases.llvm.org/3.9.1/llvm-3.9.1.src.tar.xz
+tar xf llvm-3.9.1.src.tar.xz
+mv llvm-3.9.1.src llvm
+
+wget http://releases.llvm.org/3.9.1/cfe-3.9.1.src.tar.xz
+tar xf cfe-3.9.1.src.tar.xz
+mv cfe-3.9.1.src llvm/tools/clang
+
+cd llvm
+mkdir build
+cd build
+cmake -DCMAKE_BUILD_TYPE="Release" -DGCC_INSTALL_PREFIX=<gcc-dir> \
+-DCMAKE_C_COMPILER=<gcc> -DCMAKE_CXX_COMPILER=<g++> \
+-DCMAKE_CXX_LINK_FLAGS="-L<gcc-dir>/lib64 -Wl,-rpath,<gcc-dir>/lib64" \
+-DCMAKE_INSTALL_PREFIX=<install-dst> ..
+make
+# Testing
+make check-clang
+make install
+```
+
+Put clang on PATH (if appropriate)
+
+## Memgraph
+
+Requirements on CentOS 7:
+
+  * libuuid-devel (antlr4)
+  * java-1.8.0-openjdk (antlr4)
+  * boost-static (too low version --- compile manually)
+  * rpm-build (RPM)
+  * python3 (tests, ...)
+  * which (required for rocksdb)
+  * sbcl (lisp C++ preprocessing)
+
+### Boost 1.62
+
+```
+wget https://netix.dl.sourceforge.net/project/boost/boost/1.62.0/boost_1_62_0.tar.gz
+tar xf boost_1_62_0.tar.gz
+cd boost_1_62_0
+./bootstrap.sh --with-toolset=clang --with-libraries=iostreams,serialization --prefix=<install-dst>
+./b2
+# Default installs to /usr/local/
+./b2 install
+```
+
+### Building Memgraph
+
+clang is *required* to be findable by cmake, i.e. it should be on PATH.
+cmake isn't required to be on the path, since you run it manually, so can use
+the full path to executable in order to run it. Obviously, it is convenient to
+put cmake also on PATH.
+
+Building is done as explained in [Quick Start](quick-start.md), but each
+`make` invocation needs to be prepended with:
+
+`LD_RUN_PATH=<gcc-dir>/lib64 make ...`
+
+### RPM
+
+Name format: `memgraph-<version>-<pkg-version>.<arch>.rpm`

docs/feature_ref/dynamic_graph_partitioning.md

@@ -0,0 +1,20 @@
+## Dynamic Graph Partitioner
+
+Memgraph supports dynamic graph partitioning which dynamically improves
+performance on badly partitioned dataset over workers. To enable it, the user
+should use the following flag when firing up the *master* node:
+
+```plaintext
+--dynamic_graph_partitioner_enable
+```
+
+### Parameters
+
+| Name | Default Value | Description | Range |
+|------|---------------|-------------|-------|
+|--dgp_improvement_threshold | 10 | How much better should specific node score
+be to consider a migration to another worker. This represents the minimal
+difference between new score that the vertex will have when migrated and the
+old one such that it's migrated. | Min: 1, Max: 100
+|--dgp_max_batch_size | 2000 | Maximal amount of vertices which should be
+migrated in one dynamic graph partitioner step. | Min: 1, Max: MaxInt32 |

docs/feature_spec/distributed.md

@@ -0,0 +1,78 @@
+# Distributed Memgraph specs
+This document describes reasnonings behind Memgraphs distributed concepts.
+
+## Distributed state machine
+Memgraphs distributed mode introduces two states of the cluster, recovering and
+working. The change between states shouldn't happen often, but when it happens
+it can take a while to make a transition from one to another.
+
+### Recovering
+This state is the default state for Memgraph when the cluster starts with
+recovery flags. If the recovery finishes successfully, the state changes to
+working. If recovery fails, the user will be presented with a message that
+explains what happened and what are the next steps.
+
+Another way to enter this state is failure. If the cluster encounters a failure,
+the master will enter the Recovering mode. This time, it will wait for all
+workers to respond with a message saying they are alive and well, and making
+sure they all have consistent state.
+
+### Working
+This state should be the default state of Memgraph most of the time. When in
+this state, Memgraph accepts connections from Bolt clients and allows query
+execution.
+
+If distributed execution fails for a transaction, that transaction, and all
+other active transactions will be aborted and the cluster will enter the
+Recovering state.
+
+## Durability
+One of the important concepts in distributed Memgraph is durability.
+
+### Cluster configuration
+When running Memgraph in distributed mode, the master will store cluster
+metadata in a persistent store. If fore some reason the cluster shuts down,
+recovering Memgraph from durability files shouldn't require any additional
+flags.
+
+### Database ID
+Each new and clean run of Memgraph should generate a new globally unique
+database id.  This id will associate all files that have persisted with this
+run. Adding the database id to snapshots, write-ahead logs and cluster metadata
+files ties them a specific Memgraph run, and it makes recovery easier to reason
+about.
+
+When recovering, the cluster won't generate a new id, but will reuse the one
+from the snapshot/wal that it was able to recover from.
+
+### Durability files
+Memgraph uses snapshots and write-ahead logs for durability.
+
+When Memgraph recovers it has to make sure all machines in the cluster recover
+to the same recovery point. This is done by finding a common snapshot and
+finding common transactions in per-machine available write-ahead logs.
+
+Since we can not be sure that each machine persisted durability files, we need
+to be able to negotiate a common recovery point in the cluster. Possible
+durability file failures could require to start the cluster from scratch,
+purging everything from storage and recovering from existing durability files.
+
+We need to ensure that we keep wal files containing information about
+transactions between all existing snapshots. This will provide better durability
+in the case of a random machine durability file failure, where the cluster can
+find a common recovery point that all machines in the cluster have.
+
+Also, we should suggest and make clear docs that anything less than two
+snapshots isn't considered safe for recovery.
+
+### Recovery
+The recovery happens in following steps:
+* Master enables worker registration.
+* Master recovers cluster metadata from the persisted storage.
+* Master waits all required workers to register.
+* Master broadcasts a recovery request to all workers.
+* Workers respond with with a set of possible recovery points.
+* Master finds a common recovery point for the whole cluster.
+* Master broadcasts a recovery request with the common recovery point.
+* Master waits for the cluster to recover.
+* After a successful cluster recovery, master can enter Working state.

docs/feature_spec/dynamic_graph_partitioning.md

@@ -0,0 +1,75 @@
+# Dynamic Graph Partitioning (abbr. DGP)
+
+## Implementation
+
+Take a look under `dev/memgraph/distributed/dynamic_graph_partitioning.md`.
+
+### Implemented parameters
+
+--dynamic-graph-partitioner-enabled (If the dynamic graph partitioner should be
+  enabled.) type: bool default: false (start time)
+--dgp-improvement-threshold (How much better should specific node score be
+  to consider a migration to another worker. This represents the minimal
+  difference between new score that the vertex will have when migrated
+  and the old one such that it's migrated.) type: int32 default: 10
+  (start time)
+--dgp-max-batch-size (Maximal amount of vertices which should be migrated
+  in one dynamic graph partitioner step.) type: int32 default: 2000
+  (start time)
+
+## Planning
+
+### Design decisions
+
+* Each partitioning session has to be a new transaction.
+* When and how does an instance perform the moves?
+  * Periodically.
+  * Token sharing (round robin, exactly one instance at a time has an
+    opportunity to perform the moves).
+* On server-side serialization error (when DGP receives an error).
+  -> Quit partitioning and wait for the next turn.
+* On client-side serialization error (when end client receives an error).
+  -> The client should never receive an error because of any
+     internal operation.
+  -> For the first implementation, it's good enough to wait until data becomes
+     available again.
+  -> It would be nice to achieve that DGP has lower priority than end client
+     operations.
+
+### End-user parameters
+
+* --dynamic-graph-partitioner-enabled (execution time)
+* --dgp-improvement-threshold (execution time)
+* --dgp-max-batch-size (execution time)
+* --dgp-min-batch-size (execution time)
+  -> Minimum number of nodes that will be moved in each step.
+* --dgp-fitness-threshold (execution time)
+  -> Do not perform moves if partitioning is good enough.
+* --dgp-delta-turn-time (execution time)
+  -> Time between each turn.
+* --dgp-delta-step-time (execution time)
+  -> Time between each step.
+* --dgp-step-time (execution time)
+  -> Time limit per each step.
+
+### Testing
+
+The implementation has to provide good enough results in terms of:
+  * How good the partitioning is (numeric value), aka goodness.
+  * Workload execution time.
+  * Stress test correctness.
+
+Test cases:
+  * N not connected subgraphs
+    -> shuffle nodes to N instances
+    -> run partitioning
+    -> test perfect partitioning.
+  * N connected subgraph
+    -> shuffle nodes to N instance
+    -> run partitioning
+    -> test partitioning.
+  * Take realistic workload (Long Running, LDBC1, LDBC2, Card Fraud, BFS, WSP)
+    -> measure exec time
+    -> run partitioning
+    -> test partitioning
+    -> measure exec time (during and after partitioning).

docs/feature_spec/high_availability.md

@@ -0,0 +1,213 @@
+# High Availability (abbr. HA)
+
+## Introduction
+
+High availability is a characteristic of a system which aims to ensure a
+certain level of operational performance for a higher-than-normal period.
+Although there are multiple ways to design highly available systems, Memgraph
+strives to achieve HA by elimination of single points of failure. In essence,
+this implies adding redundancy to the system so that a failure of a component
+does not imply the failure of the entire system.
+
+## Theoretical Background
+
+The following chapter serves as an introduction into some theoretical aspects
+of Memgraph's high availability implementation. If the reader is solely
+interested in design decisions around HA implementation, they can skip this
+chapter.
+
+An important implication of any HA implementation stems from Eric Brewer's
+[CAP Theorem](https://fenix.tecnico.ulisboa.pt/downloadFile/1126518382178117/10.e-CAP-3.pdf)
+which states that it is impossible for a distributed system to simultaneously
+achieve:
+
+  * Consistency (C - every read receives the most recent write or an error)
+  * Availability (A - every request receives a response that is not an error)
+  * Partition tolerance (P - The system continues to operate despite an
+                             arbitrary number of messages being dropped by the
+                             network between nodes)
+
+In the context of HA, Memgraph should strive to achieve CA.
+
+### Consensus
+
+Implications of the CAP theorem naturally lead us towards introducing a
+cluster of machines which will have identical internal states. When a designated
+machine for handling client requests fails, it can simply be replaced with
+another.
+
+Well... turns out this is not as easy as it sounds :(
+
+Keeping around a cluster of machines with consistent internal state is an
+inherently difficult problem. More precisely, this problem is as hard as
+getting a cluster of machines to agree on a single value, which is a highly
+researched area in distributed systems. Our research of state of the art
+consensus algorithms lead us to Diego Ongaro's
+[Raft algorithm](https://raft.github.io/raft.pdf).
+
+#### Raft
+
+As you might have guessed, analyzing each subtle detail of Raft goes way
+beyond the scope of this document. In the remainder of the chapter we will
+outline only the most important ideas and implications, leaving all further
+analysis to the reader. Detailed explanation can be found either in Diego's
+[dissertation](https://ramcloud.stanford.edu/~ongaro/thesis.pdf) \[1\] or the
+Raft [paper](https://raft.github.io/raft.pdf) \[2\].
+
+In essence, Raft allows us to implement the previously mentioned idea of
+managing a cluster of machines with identical internal states. In other
+words, the Raft protocol allows us to manage a cluster of replicated
+state machines which is fully functional as long as the *majority* of
+the machines in the cluster operate correctly.
+
+Another important fact is that those state machines must be *deterministic*.
+In other words, the same command on two different machines with the same
+internal state must yield the same result. This is important because Memgraph,
+as a black box, is not entirely deterministic. Non-determinism can easily be
+introduced by the user (e.g. by using the `rand` function) or by algorithms
+behind query execution (e.g. introducing fuzzy logic in the planner could yield
+a different order of results). Luckily, once we enter the storage level,
+everything should be fully deterministic.
+
+To summarize, Raft is a protocol which achieves consensus in a cluster of
+deterministic state machines via log replication. The cluster is fully
+functional if the majority of the machines work correctly. The reader
+is strongly encouraged to gain a deeper understanding (at least read through
+the paper) of Raft before reading the rest of this document.
+
+## Integration with Memgraph
+
+The first thing that should be defined is a single instruction within the
+context of Raft (i.e. a single entry in a replicated log). As mentioned
+before, these instructions should be completely deterministic when applied
+to the state machine. We have therefore decided that the appropriate level
+of abstraction within Memgraph corresponds to `StateDelta`-s (data structures
+which describe a single change to the Memgraph state, used for durability
+in WAL). Moreover, a single instruction in a replicated log will consist of a
+batch of `StateDelta`s which correspond to a single **committed** transaction.
+This decision both improves performance and handles some special cases that
+present themselves otherwise by leveraging the knowledge that the transaction
+should be committed.
+
+"What happens with aborted transactions?"
+
+A great question, they are handled solely by the leader which is the only
+machine that communicates with the client. Aborted transactions do not alter
+the state of the database and there is no need to replicate it to other machines
+in the cluster. If, for instance, the leader dies before returning the result
+of some read operation in an aborted transaction, the client will notice that
+the leader has crashed. A new leader will be elected in the next term and the
+client should retry the transaction.
+
+"OK, that makes sense! But, wait a minute, this is broken by design! Merely
+generating `StateDelta`s on the leader for any transaction will taint its
+internal storage before sending the first RPC to some follower. This deviates
+from Raft and will crash the universe!"
+
+Another great observation. It is indeed true that applying `StateDelta`s makes
+changes to local storage, but only a single type of `StateDelta` makes that
+  change durable. That `StateDelta` type is called `TRANSACTION_COMMIT` and we
+will change its behaviour when working as a HA instance. More precisely, we
+must not allow the transaction engine to modify the commit log saying that
+the transaction has been committed. That action should be delayed until those
+`StateDelta`s have been applied to the majority of the cluster. At that point
+the commit log can be safely modified leaving it up to Raft to ensure the
+durability of the transaction.
+
+We should also address one subtle detail that arises in this case. Consider
+the following scenario:
+
+  * The leader starts working on a transaction which creates a new record in the
+    database. Suppose that record is stored in the leader's internal storage
+    but the transaction was not committed (i.e. no such entry in the commit log).
+  * The leader should start replicating those `StateDelta`s to its followers
+    but, suddenly, it's cut off from the rest of the cluster.
+  * Due to timeout, a new election is held and a new leader has been elected.
+  * Our old leader comes back to life and becomes a follower.
+  * The new leader receives a transaction which creates that same record, but
+    this transaction is successfully replicated and committed by the new leader.
+
+The problem lies in the fact that there is still a record within the internal
+storage of our old leader with the same transaction ID and GID as the recently
+committed record by the new leader. Obviously, this is broken. As a solution, we
+will prefix the transaction ID with the Raft term, thus ensuring that the
+transaction IDs will differ.
+
+"When will followers append that transaction to their commit logs?"
+
+When the leader deduces that the transaction is safe to commit, it will include
+the relevant information in all further heartbeats which will alert the
+followers that it is safe to commit those entries from their raft logs.
+Naturally, the followers need not to delay appending data to the commit log
+as they know that the transaction has already been committed (from the clusters
+point of view). If this sounds really messed up, seriously, read the Raft paper.
+
+"How does the raft log differ from WAL"
+
+Conceptually, it doesn't. When operating in HA, we don't really need the
+recovery mechanisms implemented in Memgraph thus far. When a dead machine
+comes back to life, it will eventually come in sync with the rest of the
+cluster and everything will be done using the machine's raft log as well
+as the messages received from the cluster leader.
+
+"Those logs will become huge, isn't that recovery going to be painfully slow?"
+
+True, but there are mechanisms for making raft logs more compact. The most
+popular method is, wait for it, making snapshots :)
+Although the process of bringing an old machine back to life is a long one,
+it doesn't really affect the performance of the cluster in a great degree.
+The cluster will work perfectly fine with that machine being way out of sync.
+
+"I don't know, everything seems to be a lot slower than before!"
+
+Absolutely true, the user should be aware that they will suffer dire
+consequences on the performance side if they choose to be highly available.
+As Frankie says, "That's life!".
+
+"Also, I didn't really care about most of the things you've said. I'm
+not a part of the storage team and couldn't care less about the issues you
+face, how does HA affect 'my part of the codebase'?"
+
+Answer for query execution: That's ok, you'll be able to use the same beloved
+API (when we implement it, he he :) towards storage and continue to
+make fun of us when you find a bug.
+
+Answer for infrastructure: We'll talk. Some changes will surely need to
+be made on the Memgraph client. There is a chapter in Diego's dissertation
+called 'Client interaction', but we'll cross that bridge when we get there.
+There will also be the whole 'integration with Jepsen tests' thing going on.
+
+Answer for analytics: I'm astonished you've read this article. Wanna join
+storage?
+
+### Subtlety Regarding Reads
+
+As we have hinted in the previous chapter, we would like to bypass log
+replication for operations which do not alter the internal state of Memgraph.
+Those operations should therefore be handled only by the leader, which is not
+as trivial as it seems. The subtlety arises from the fact that a (newly-elected)
+leader can have an entry in its log which was committed by the previous leader
+that has crashed but that entry is not yet committed in its internal storage
+by the current leader. Moreover, the rule about safely committing logs that are
+replicated on the majority of the cluster only applies for entries replicated in
+the leaders current term. Therefore, we are faced with two issues:
+
+  * We cannot simply perform read operations if the leader has a non-committed
+    entry in its log (breaks consistency).
+  * Replicating those entries onto the majority of the cluster is not enough
+    to guarantee that they can be safely committed.
+
+This can be solved by introducing a blank no-op operation which the new leader
+will try to replicate at the start of its term. Once that operation is
+replicated and committed, the leader can safely perform those non-altering
+operations on its own.
+
+For further information about these issues, you should check out section
+5.4.2 from the raft paper \[1\] which hints as to why its not safe to commit
+entries from previous terms. Also, you should check out section 6.4 from
+the thesis \[2\] which goes into more details around efficiently processing
+read-only queries.
+
+## How do we test HA
+
+[Check this out](https://jepsen.io/analyses/dgraph-1-0-2)

docs/feature_spec/kafka/opencypher.md

@@ -0,0 +1,80 @@
+# Kafka - openCypher clause
+
+One must be able to specify the following when importing data from Kafka:
+
+* Kafka URI
+* Kafka topic
+* Transform [script](transform.md) URI
+
+
+Minimum required syntax looks like:
+```opencypher
+CREATE STREAM stream_name AS LOAD DATA KAFKA 'URI'
+  WITH TOPIC 'topic'
+  WITH TRANSFORM 'URI';
+```
+
+
+The full openCypher clause for creating a stream is:
+```opencypher
+CREATE STREAM stream_name AS
+  LOAD DATA KAFKA 'URI'
+  WITH TOPIC 'topic'
+  WITH TRANSFORM 'URI'
+  [BATCH_INTERVAL milliseconds]
+  [BATCH_SIZE count]
+```
+The `CREATE STREAM` clause happens in a transaction.
+
+`WITH TOPIC` parameter specifies the Kafka topic from which we'll stream
+data.
+
+`WITH TRANSFORM` parameter should contain a URI of the transform script.
+
+`BATCH_INTERVAL` parameter defines the time interval in milliseconds
+which is the time between two successive stream importing operations.
+
+`BATCH_SIZE` parameter defines the count of Kafka messages that will be
+batched together before import.
+
+If both `BATCH_INTERVAL` and `BATCH_SIZE` parameters are given, the condition
+that is satisfied first will trigger the batched import.
+
+Default value for `BATCH_INTERVAL` is 100 milliseconds, and the default value
+for `BATCH_SIZE` is 10;
+
+The `DROP` clause deletes a stream:
+```opencypher
+DROP STREAM stream_name;
+```
+
+The `SHOW` clause enables you to see all configured streams:
+```opencypher
+SHOW STREAMS;
+```
+
+You can also start/stop streams with the `START` and `STOP` clauses:
+```opencypher
+START STREAM stream_name [LIMIT count BATCHES];
+STOP STREAM stream_name;
+```
+A stream needs to be stopped in order to start it and it needs to be started in
+order to stop it. Starting a started or stopping a stopped stream will not
+affect that stream.
+
+There are also convenience clauses to start and stop all streams:
+```opencypher
+START ALL STREAMS;
+STOP ALL STREAMS;
+```
+
+Before the actual import, you can also test the stream with the `TEST
+STREAM` clause:
+```opencypher
+TEST STREAM stream_name [LIMIT count BATCHES];
+```
+When a stream is tested, data extraction and transformation occurs, but no
+output is inserted in the graph.
+
+A stream needs to be stopped in order to test it. When the batch limit is
+omitted, `TEST STREAM` will run for only one batch by default.

docs/feature_spec/kafka/transform.md

@@ -0,0 +1,34 @@
+# Kafka - data transform
+
+The transform script is a user defined script written in Python.  The script
+should be aware of the data format in the Kafka message.
+
+Each Kafka message is byte length encoded, which means that the first eight
+bytes of each message contain the length of the message.
+
+A sample code for a streaming transform script could look like this:
+
+```python
+def create_vertex(vertex_id):
+  return ("CREATE (:Node {id: $id})", {"id": vertex_id})
+
+
+def create_edge(from_id, to_id):
+  return ("MATCH (n:Node {id: $from_id}), (m:Node {id: $to_id}) "\
+          "CREATE (n)-[:Edge]->(m)", {"from_id": from_id, "to_id": to_id})
+
+
+def stream(batch):
+    result = []
+    for item in batch:
+        message = item.decode('utf-8').strip().split()
+        if len(message) == 1:
+          result.append(create_vertex(message[0])))
+        else:
+          result.append(create_edge(message[0], message[1]))
+    return result
+
+```
+
+The script should output openCypher query strings based on the type of the
+records.

docs/feature_spec/tensorflow_op/technicalities.md

@@ -0,0 +1,61 @@
+# Tensorflow Op - Technicalities
+
+The final result should be a shared object (".so") file that can be
+dynamically loaded by the Tensorflow runtime in order to directly
+access the bolt client.
+
+## About Tensorflow
+
+Tensorflow is usually used with Python such that the Python code is used
+to define a directed acyclic computation graph. Basically no computation
+is done in Python. Instead, values from Python are copied into the graph
+structure as constants to be used by other Ops. The directed acyclic graph
+naturally ends up with two sets of border nodes, one for inputs, one for
+outputs. These are sometimes called "feeds".
+
+Following the Python definition of the graph, during training, the entire
+data processing graph/pipeline is called from Python as a single expression.
+This leads to lazy evaluation since the called result has already been
+defined for a while.
+
+Tensorflow internally works with tensors, i.e. n-dimensional arrays. That
+means all of its inputs need to be matrices as well as its outputs. While
+it is possible to feed data directly from Python's numpy matrices straight
+into Tensorflow, this is less desirable than using the Tensorflow data API
+(which defines data input and processing as a Tensorflow graph) because:
+
+  1. The data API is written in C++ and entirely avoids Python and as such
+  is faster
+  2. The data API, unlike Python is available in "Tensorflow serving". The
+  default way to serve Tensorflow models in production.
+
+Once the entire input pipeline is defined via the tf.data API, its input
+is basically a list of node IDs the model is supposed to work with. The
+model, through the data API knows how to connect to Memgraph and execute
+openCypher queries in order to get the remaining data it needs.
+(For example features of neighbouring nodes.)
+
+## The Interface
+
+I think it's best you read the official guide...  
+<https://www.tensorflow.org/extend/adding_an_op>  
+And especially the addition that specifies how data ops are special  
+<https://www.tensorflow.org/extend/new_data_formats>  
+
+## Compiling the TF Op
+
+There are two options for compiling a custom op.  
+One of them involves pulling the TF source, adding your code to it and
+compiling via bazel.  
+This is probably awkward to do for us and would
+significantly slow down compilation.  
+
+The other method involves installing Tensorflow as a Python package and
+pulling the required headers from for example:  
+`/usr/local/lib/python3.6/site-packages/tensorflow/include`  
+We can then compile our Op with our regular build system.
+
+This is practical since we can copy the required headers to our repo.
+If necessary, we can have several versions of the headers to build several
+versions of our Op for every TF version which we want to support.
+(But this is unlikely to be required as the API should be stable).

docs/feature_spec/tensorflow_op/usage_example.md

@@ -0,0 +1,142 @@
+# Example for Using the Bolt Client Tensorflow Op
+
+## Dynamic Loading
+
+``` python3
+import tensorflow as tf
+
+mg_ops = tf.load_op_library('/usr/bin/memgraph/tensorflow_ops.so')
+```
+
+## Basic Usage
+
+``` python3
+dataset = mg_ops.OpenCypherDataset(
+            # This is probably unfortunate as the username and password
+            # get hardcoded into the graph, but for the simple case it's fine
+            "hostname:7687", auth=("user", "pass"),
+
+            # Our query
+            '''
+            MATCH (n:Train) RETURN n.id, n.features
+            ''',
+
+            # Cast return values to these types
+            (tf.string, tf.float32))
+
+# Some Tensorflow data api boilerplate
+iterator = dataset.make_one_shot_iterator()
+next_element = iterator.get_next()
+
+# Up to now we have only defined our computation graph which basically
+# just connects to Memgraph
+# `next_element` is not really data but a handle to a node in the Tensorflow
+# graph, which we can and do evaluate
+# It is a Tensorflow tensor with shape=(None, 2)
+# and dtype=(tf.string, tf.float)
+# shape `None` means the shape of the tensor is unknown at definition time
+# and is dynamic and will only be known once the tensor has been evaluated
+
+with tf.Session() as sess:
+    node_ids = sess.run(next_element)
+    # `node_ids` contains IDs and features of all the nodes 
+    # in the graph with the label "Train"
+    # It is a numpy.ndarray with a shape ($n_matching_nodes, 2)
+```
+
+## Memgraph Client as a Generic Tensorflow Op
+
+Other than the Tensorflow Data Op, we'll want to support a generic Tensorflow
+Op which can be put anywhere in the Tensorflow computation Graph. It takes in
+an arbitrary tensor and produces a tensor. This would be used in the GraphSage
+algorithm to fetch the lowest level features into Tensorflow
+
+```python3
+requested_ids = np.array([1, 2, 3])
+ids_placeholder = tf.placeholder(tf.int32)
+
+model = mg_ops.OpenCypher()
+    "hostname:7687", auth=("user", "pass"),
+    """
+    UNWIND $node_ids as nid
+    MATCH (n:Train {id: nid})
+    RETURN n.features
+    """,
+
+    # What to call the input tensor as an openCypher parameter
+    parameter_name="node_ids",
+
+    # Type of our resulting tensor
+    dtype=(tf.float32)
+)
+
+features = model(ids_placeholder)
+
+with tf.Session() as sess:
+    result = sess.run(features,
+                    feed_dict={ids_placeholder: requested_ids})
+```
+
+This is probably easier to implement than the Data Op, so it might be a good
+idea to start with.
+
+## Production Usage
+
+During training, in the GraphSage algorithm at least, Memgraph is at the
+beginning and at the end of the Tensorflow computation graph.  
+At the beginning, the Data Op provides the node IDs which are fed into the
+generic Tensorflow Op to find their neighbours and their neighbours and
+their features.
+
+Production usage differs in that we don't use the Data Op. The Data Op is
+effectively cut off and the initial input is fed by Tensorflow serving,
+with the data found in the request.
+
+For example a JSON request to classify a node might look like:
+
+`POST http://host:port/v1/models/GraphSage/versions/v1:classify`
+
+With the contents:
+
+```json
+{
+    "examples": [
+        {"node_id": 1},
+        {"node_id": 2}
+    ],
+}
+```
+
+Every element of the "examples" list is an example to be computed. Each is
+represented by a dict with keys matching names of feeds in the Tensorflow
+graph and values being the values we want fed in for each example
+
+The REST API then replies in kind with the classification result in JSON
+
+Note about adding our custom Op to Tensorflow serving.  
+Our Ops .so can be added into the Bazel build to link with Tensorflow serving
+or it can be dynamically loaded by starting Tensorflow serving with a flag
+`--custom_op_paths`
+
+## Considerations
+
+There might be issues here that the url to connect to Memgraph is
+hardcoded into the op and would thus be wrong when moved to production,
+requiring some type of a hack to make work. We probably want to solve
+this by having the client op take in another tf.Variable as an input
+which would contain a connection url and username/password.  
+We have to research whether this makes it easy enough to move to
+production, as the connection string variable is still a part of the
+graph, but maybe easier to replace.  
+
+It is probably the best idea to utilize openCypher parameters to make
+our queries flexible. The exact API as to how to declare the parameters
+in Python is open to discussion.
+
+The Data Op might not even be necessary to implement as it is not
+key for production use. It can be replaced in training mode with
+feed dicts and either
+
+ 1. Getting the initial list of nodes via a Python Bolt client
+ 2. Creating a separate Tensorflow computation graph that gets all the
+ relevant node IDs into Python

docs/presentation/latex-template/README.md

@@ -0,0 +1,22 @@
+# Memgraph LaTeX Beamer Template
+
+This folder contains all of the needed files for creating a presentation with
+Memgraph styling. You should use this style for any public presentations.
+
+Feel free to improve it according to style guidelines and raise issues if you
+find any.
+
+## Usage
+
+Copy the contents of this folder (excluding this README file) to where you
+want to write your own presentation. After copying, you can start editing the
+`template.tex` with your content.
+
+To compile the presentation to a PDF, run `latexmk -pdf -xelatex`. Some
+directives require XeLaTeX, so you need to pass `-xelatex` as the final option
+of `latexmk`. You may also need to install some packages if the compilation
+complains about missing packages.
+
+To clean up the generated files, use `latexmk -C`. This will also delete the
+generated PDF. If you wish to remove generated files except the PDF, use
+`latexmk -c`.

docs/presentation/latex-template/mg-beamer.cls

@@ -0,0 +1,82 @@
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesClass{mg-beamer}[2018/03/26 Memgraph Beamer]
+
+\DeclareOption*{\PassOptionsToClass{\CurrentOption}{beamer}}
+
+\ProcessOptions \relax
+
+\LoadClass{beamer}
+
+\usetheme{Pittsburgh}
+
+% Memgraph color palette
+\definecolor{mg-purple}{HTML}{720096}
+\definecolor{mg-red}{HTML}{DD2222}
+\definecolor{mg-orange}{HTML}{FB6E00}
+\definecolor{mg-yellow}{HTML}{FFC500}
+\definecolor{mg-gray}{HTML}{857F87}
+\definecolor{mg-black}{HTML}{231F20}
+
+\RequirePackage{fontspec}
+% Title fonts
+\setbeamerfont{frametitle}{family={\fontspec[Path = ./mg-style/fonts/]{EncodeSansSemiCondensed-Regular.ttf}}}
+\setbeamerfont{title}{family={\fontspec[Path = ./mg-style/fonts/]{EncodeSansSemiCondensed-Regular.ttf}}}
+% Body font
+\RequirePackage[sfdefault,light]{roboto}
+% Roboto is pretty bad for monospace font. We will find a replacement.
+% \setmonofont{RobotoMono-Regular.ttf}[Path = ./mg-style/fonts/]
+
+% Title slide styles
+% \setbeamerfont{frametitle}{size=\huge}
+% \setbeamerfont{title}{size=\huge}
+% \setbeamerfont{date}{size=\tiny}
+
+% Other typography styles
+\setbeamertemplate{frametitle}[default][center]
+\setbeamercolor{frametitle}{fg=mg-black}
+\setbeamercolor{title}{fg=mg-black}
+\setbeamercolor{section in toc}{fg=mg-black}
+\setbeamercolor{local structure}{fg=mg-orange}
+\setbeamercolor{alert text}{fg=mg-red}
+
+% Commands
+\newcommand{\mgalert}[1]{{\usebeamercolor[fg]{alert text}#1}}
+\newcommand{\titleframe}{\frame[plain]{\titlepage}}
+\newcommand{\mgtexttt}[1]{{\textcolor{mg-gray}{\texttt{#1}}}}
+
+% Title slide background
+\RequirePackage{tikz,calc}
+% Use title-slide-169 if aspect ration is 16:9
+\pgfdeclareimage[interpolate=true,width=\paperwidth,height=\paperheight]{logo}{mg-style/title-slide-169}
+\setbeamertemplate{background}{
+  \begin{tikzpicture}
+  \useasboundingbox (0,0) rectangle (\the\paperwidth,\the\paperheight);
+    \pgftext[at=\pgfpoint{0}{0},left,base]{\pgfuseimage{logo}};
+  \ifnum\thepage>1\relax
+  \useasboundingbox (0,0) rectangle (\the\paperwidth,\the\paperheight);
+      \fill[white, opacity=1](0,\the\paperheight)--(\the\paperwidth,\the\paperheight)--(\the\paperwidth,0)--(0,0)--(0,\the\paperheight);
+  \fi
+  \end{tikzpicture}
+}
+
+% Footline content
+\setbeamertemplate{navigation symbols}{}%remove navigation symbols
+ \setbeamertemplate{footline}{
+  \begin{beamercolorbox}[ht=1.6cm,wd=\paperwidth]{footlinecolor}
+    \vspace{0.1cm}
+    \hfill
+    \begin{minipage}[c]{3cm}
+      \begin{center}
+       \includegraphics[height=0.8cm]{mg-style/memgraph-logo.png}
+      \end{center}
+    \end{minipage}
+    \begin{minipage}[c]{7cm}
+         \insertshorttitle\ --- \insertsection
+    \end{minipage}
+    \begin{minipage}[c]{2cm}
+        \tiny{\insertframenumber{} of \inserttotalframenumber}
+    \end{minipage}
+   \end{beamercolorbox}
+}
+
+\endinput

docs/presentation/latex-template/template.tex

@@ -0,0 +1,40 @@
+% Set 16:9 aspect ratio
+\documentclass[aspectratio=169]{mg-beamer}
+% Default directive sets the regular 4:3 aspect ratio
+% \documentclass{mg-beamer}
+\mode<presentation>
+
+% requires xelatex
+\usepackage{ccicons}
+
+\title{Insert Presentation Title}
+\titlegraphic{\ccbyncnd}
+\author{Insert Name}
+
+% Institute doesn't look good in our current styling class.
+% \institute[Memgraph Ltd.]{\pgfimage[height=1.5cm]{mg-logo.png}}
+
+% Date is autogenerated on compilation, so no need to set it explicitly,
+% unless you wish to override it with a different date.
+% \date{March 23, 2018}
+
+\begin{document}
+
+\titleframe
+
+\section{Intro}
+
+\begin{frame}{Contents}
+  \tableofcontents
+\end{frame}
+
+\begin{frame}{Memgraph Markup Test}
+  \begin{itemize}
+    \item \mgtexttt{Prefer \\mgtexttt for monospace}
+    \item Replace this slide with your own
+    \item Add even more slides in different sections
+    \item Make sure you spellcheck your presentation
+  \end{itemize}
+\end{frame}
+
+\end{document}

docs/user_technical/README.md

@@ -1,22 +0,0 @@
-# Technical Documentation
-
-## About Memgraph
-
-Memgraph is an ACID compliant high performance transactional in-memory graph
-database management system featuring highly concurrent
-data structures, multi-version concurrency control and asynchronous IO.
-
-[//]: # (When adding a new documentation file, please add it to the list)
-
-## Contents
-
-  * [About Memgraph](#about-memgraph)
-  * [Quick Start](quick-start.md)
-  * [Examples](examples.md)
-  * [Drivers](drivers.md)
-  * [Storable Data Types](data-types.md)
-  * [openCypher Query Language](open-cypher.md)
-  * [Import Tools](import-tools.md)
-  * [Upcoming Features](upcoming-features.md)
-
-[//]: # (Nothing should go below the contents section)

docs/user_technical/bundle_community

@@ -1,58 +0,0 @@
-#!/bin/bash
-
-working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-build_dir=.bundle_community
-
-# curl and php are required
-if ! which php >/dev/null; then
-    echo "php is required!"
-    exit 1
-fi
-
-cd ${working_dir}
-generator="couscous.phar"
-
-# download the generator if it's not already downloaded
-if [ ! -f ${generator} ]; then
-    curl -OS http://couscous.io/${generator}
-fi
-
-# create build directory
-if [ -d $build_dir ]; then
-    rm -rf $build_dir
-fi
-mkdir $build_dir
-
-# copy all files to build directory
-cp couscous.yml *.md $generator $build_dir
-
-# go to build directory
-cd $build_dir
-
-# modify config file
-cat >> couscous.yml <<EOF
-template:
-    directory: .
-EOF
-
-# create template file
-echo "{{ content|raw }}" > default.twig
-
-# generate the site
-php ${generator} generate
-
-# create contents file
-cd .couscous/generated
-tail -n +$( grep -n "Contents" index.html | cut -d ':' -f 1 ) index.html | \
-    grep href | grep .html | \
-    sed -r 's/^[^"]+"//' | sed -r 's/.html">/ /' | sed -r 's/<.+//' > .contents
-rm index.html
-
-# create archive
-target=docs.tar.gz
-if [ -f $target ]; then
-    rm $target
-fi
-tar -czf $working_dir/$target .contents *.html
-
-echo "Created archive: $working_dir/$target"

docs/user_technical/couscous.yml

@@ -1,2 +0,0 @@
-title: Memgraph
-subTitle: Technical Documentation

docs/user_technical/data-types.md

@@ -1,47 +0,0 @@
-## Storable Data Types
-
-Since *Memgraph* is a *graph* database management system, data is stored in
-the form of graph elements: nodes and edges. Each graph element can also
-contain various types of data. This chapter describes which data types are
-supported in *Memgraph*.
-
-### Node Labels & Edge Types
-
-Each node can have any number of labels. A label is a text value, which can be
-used to *label* or group nodes according to users' desires. A user can change
-labels at any time. Similarly to labels, each edge can have a type,
-represented as text. Unlike nodes, which can have multiple labels or none at
-all, edges *must* have exactly one edge type. Another difference to labels, is
-that the edge types are set upon creation and never modified again.
-
-### Properties
-
-Nodes and edges can store various properties. These are like mappings or
-tables containing property names and their accompanying values. Property names
-are represented as text, while values can be of different types. Each property
-name can store a single value, it is not possible to have multiple properties
-with the same name on a single graph element. Naturally, the same property
-names can be found across multiple graph elements. Also, there are no
-restrictions on the number of properties that can be stored in a single graph
-element. The only restriction is that the values must be of the supported
-types. Following is a table of supported data types.
-
- Type      | Description
------------|------------
- `Null`    | Denotes that the property has no value. This is the same as if the property does not exist.
- `String`  | A character string, i.e. text.
- `Boolean` | A boolean value, either `true` or `false`.
- `Integer` | An integer number.
- `Float`   | A floating-point number, i.e. a real number.
- `List`    | A list containing any number of property values of any supported type. It can be used to store multiple values under a single property name.
- `Map`     | A mapping of string keys to values of any supported type.
-
- Note that even though it's possible to store `List` and `Map` property values, it is not possible to modify them. It is however possible to replace them completely. So, the following queries are legal:
-
-    CREATE (:Node {property: [1, 2, 3]})
-    CREATE (:Node {property: {key: "value"}})
-
-However, these queries are not:
-
-    MATCH (n:Node) SET n.property[0] = 0
-    MATCH (n:Node) SET n.property.key = "other value"

docs/user_technical/drivers.md

@@ -1,205 +0,0 @@
-## Bolt Drivers
-
-Clients connect to Memgraph using the
-[Bolt protocol](https://boltprotocol.org/). Bolt was designed for efficient
-communication with graph databases. Memgraph supports
-[Version 1](https://boltprotocol.org/v1/) of the protocol. Official Bolt
-protocol drivers are provided for multiple programming languages:
-
-  * [Java](https://github.com/neo4j/neo4j-java-driver)
-  * [Python](https://github.com/neo4j/neo4j-python-driver)
-  * [Javascript](https://github.com/neo4j/neo4j-javascript-driver)
-  * [C#](https://github.com/neo4j/neo4j-dotnet-driver)
-
-### Python Driver Example
-
-Neo4j officially supports Python for interacting with an openCypher and Bolt
-compliant database. For details consult the
-[official documentation](http://neo4j.com/docs/api/python-driver) and the
-[GitHub project](https://github.com/neo4j/neo4j-python-driver).  Following is
-a basic usage example:
-
-```
-from neo4j.v1 import GraphDatabase, basic_auth
-
-# Initialize and configure the driver.
-#   * provide the correct URL where Memgraph is reachable;
-#   * use an empty user name and password, and
-#   * disable encryption (not supported).
-driver = GraphDatabase.driver("bolt://localhost:7687",
-                              auth=basic_auth("", ""),
-                              encrypted=False)
-
-# Start a session in which queries are executed.
-session = driver.session()
-
-# Execute openCypher queries.
-# After each query, call either `consume()` or `data()`
-session.run('CREATE (alice:Person {name: "Alice", age: 22})').consume()
-
-# Get all the vertices from the database (potentially multiple rows).
-vertices = session.run('MATCH (n) RETURN n').data()
-# Assuming we started with an empty database, we should have Alice
-# as the only row in the results.
-only_row = vertices.pop()
-alice = only_row["n"]
-
-# Print out what we retrieved.
-print("Found a vertex with labels '{}', name '{}' and age {}".format(
-  alice['name'], alice.labels, alice['age'])
-
-# Remove all the data from the database.
-session.run('MATCH (n) DETACH DELETE n').consume()
-
-# Close the session and the driver.
-session.close()
-driver.close()
-```
-
-### Java Driver Example
-
-The details about Java driver can be found
-[on GitHub](https://github.com/neo4j/neo4j-java-driver).
-
-The example below is equivalent to Python example. Major difference is that
-`Config` object has to be created before the driver construction.  Encryption
-has to be disabled by calling `withoutEncryption` method against the `Config`
-builder.
-
-```
-import org.neo4j.driver.v1.*;
-import org.neo4j.driver.v1.types.*;
-import static org.neo4j.driver.v1.Values.parameters;
-import java.util.*;
-
-public class JavaQuickStart {
-    public static void main(String[] args) {
-        // Initialize driver.
-        Config config = Config.build().withoutEncryption().toConfig();
-        Driver driver = GraphDatabase.driver("bolt://localhost:7687",
-                                             AuthTokens.basic("",""),
-                                             config);
-        // Execute basic queries.
-        try (Session session = driver.session()) {
-            StatementResult rs1 = session.run("MATCH (n) DETACH DELETE n");
-            StatementResult rs2 = session.run(
-                "CREATE (alice: Person {name: 'Alice', age: 22})");
-            StatementResult rs3 = session.run( "MATCH (n) RETURN n");
-            List<Record> records = rs3.list();
-            Record record = records.get(0);
-            Node node = record.get("n").asNode();
-            System.out.println(node.get("name").asString());
-        } catch (Exception e) {
-            System.out.println(e);
-            System.exit(1);
-        }
-        // Cleanup.
-        driver.close();
-    }
-}
-```
-
-### Javascript Driver Example
-
-The details about Javascript driver can be found
-[on GitHub](https://github.com/neo4j/neo4j-javascript-driver).
-
-The Javascript example below is equivalent to Python and Java examples. SSL
-can be disabled by passing `{encrypted: 'ENCRYPTION_OFF'}` during the driver
-construction.
-
-Here is an example related to `Node.js`. Memgraph doesn't have integrated
-support for `WebSocket` which is required during the execution in any web
-browser. If you want to run `openCypher` queries from a web browser,
-[websockify](https://github.com/novnc/websockify) has to be up and running.
-Requests from web browsers are wrapped into `WebSocket` messages, and a proxy
-is needed to handle the overhead. The proxy has to be configured to point out
-to Memgraph's Bolt port and web browser driver has to send requests to the
-proxy port.
-
-```
-var neo4j = require('neo4j-driver').v1;
-var driver = neo4j.driver("bolt://localhost:7687",
-                          neo4j.auth.basic("neo4j", "1234"),
-                          {encrypted: 'ENCRYPTION_OFF'});
-var session = driver.session();
-
-function die() {
-  session.close();
-  driver.close();
-}
-
-function run_query(query, callback) {
-  var run = session.run(query, {});
-  run.then(callback).catch(function (error) {
-    console.log(error);
-    die();
-  });
-}
-
-run_query("MATCH (n) DETACH DELETE n", function (result) {
-  console.log("Database cleared.");
-  run_query("CREATE (alice: Person {name: 'Alice', age: 22})", function (result) {
-    console.log("Record created.");
-    run_query("MATCH (n) RETURN n", function (result) {
-      console.log("Record matched.");
-      var alice = result.records[0].get("n");
-      console.log(alice.labels[0]);
-      console.log(alice.properties["name"]);
-      session.close();
-      driver.close();
-    });
-  });
-});
-```
-
-### C# Driver Example
-
-The C# driver is hosted
-[on GitHub](https://github.com/neo4j/neo4j-dotnet-driver). The example below
-performs the same work as all of the previous examples. Encryption is disabled
-by setting `EncryptionLevel.NONE` on the `Config`.
-
-```
-using System;
-using System.Linq;
-using Neo4j.Driver.V1;
-
-public class Basic {
-  public static void Main(string[] args) {
-    // Initialize the driver.
-    var config = Config.DefaultConfig;
-    config.EncryptionLevel = EncryptionLevel.None;
-    using(var driver = GraphDatabase.Driver("bolt://localhost:7687", AuthTokens.None, config))
-      using(var session = driver.Session())
-      {
-        // Run basic queries.
-        session.Run("MATCH (n) DETACH DELETE n").Consume();
-        session.Run("CREATE (alice:Person {name: \"Alice\", age: 22})").Consume();
-        var result = session.Run("MATCH (n) RETURN n").First();
-        var alice = (INode) result["n"];
-        Console.WriteLine(alice["name"]);
-        Console.WriteLine(string.Join(", ", alice.Labels));
-        Console.WriteLine(alice["age"]);
-      }
-    Console.WriteLine("All ok!");
-  }
-}
-```
-
-### Limitations
-
-Memgraph is currently in early stage, and has a number of limitations we plan
-to remove in future versions.
-
-#### Multiple Users & Authorization
-
-Memgraph is currently single-user only. There is no way to control user
-privileges. The default user has read and write privileges over the whole
-database.
-
-#### Secure Sockets Layer (SSL)
-
-Secure connections are not supported. For this reason each client
-driver needs to be configured not to use encryption. Consult driver-specific
-guides for details.

docs/user_technical/examples.md

@@ -1,173 +0,0 @@
-## Examples
-
-This chapter shows you how to use Memgraph on real-world data and how to get interesting
-and useful information out of it.
-
-### TED Talks Example
-
-[TED](https://www.ted.com/) is a nonprofit organization devoted to spreading ideas,
-usually in the form of short, powerful talks.
-Today, TED talks are influential videos from expert speakers on almost all
-topics — from science to business to global issues.
-Here we present a small dataset which consists of 97 talks. We'll show you how
-to model this data as a graph and demonstrate a few example queries.
-
-#### Data Model
-Each TED talk has a main speaker, so we
-identify two types of nodes — `Talk` and `Speaker`. Also, we will add
-an edge of type `Gave` pointing to a `Talk` from its main `Speaker`. Each speaker has a name
-so we can add property `name` to `Speaker` node. Likewise, we'll add properties
-`name`, `title` and `description` to node `Talk`.
-Furthermore, each talk is given in a specific TED event, so
-we can create node `Event` with property `name` and relationship `InEvent`
-between talk and event.
-
-Talks are tagged with keywords to facilitate searching, hence we
-add node `Tag` with property `name` and relationship `HasTag` between talk and
-tag. Moreover, users give ratings to each talk
-by selecting up to three predefined string values.
-Therefore we add node `Rating` with these values as property `name` and relationship
-`HasRating` with property `user_count` between talk and rating nodes.
-
-#### Example Queries
-
-We have prepared a database snapshot for this example, so you can easily import it
-when starting Memgraph using the `--durability-directory` option.
-
-```
-/usr/lib/memgraph/memgraph --durability-directory /usr/share/memgraph/examples/TEDTalk \
-  --durability-enabled=false --snapshot-on-exit=false
-```
-
-When using Memgraph installed from DEB package, you may need to stop the
-currently running Memgraph server before you can import the example. Use the
-following command:
-
-```
-systemctl stop memgraph
-```
-
-When using Docker, you can import the example with the following command:
-
-```
-docker run -p 7687:7687 \
-  -v mg_lib:/var/lib/memgraph -v mg_log:/var/log/memgraph -v mg_etc:/etc/memgraph \
-  memgraph --durability-directory /usr/share/memgraph/examples/TEDTalk \
-  --durability-enabled=false --snapshot-on-exit=false
-```
-
-Now you're ready to try out some of the following queries.
-
-NOTE: If you modify the dataset, the changes will stay only during this run of
-Memgraph.
-
-1) Find all talks given by specific speaker:
-```
-MATCH (n:Speaker {name: "Hans Rosling"})-[:Gave]->(m:Talk)
-RETURN m.title;
-```
-
-
-2) Find the top 20 speakers with most talks given:
-
-```
-MATCH (n:Speaker)-[:Gave]->(m)
-RETURN n.name, COUNT(m) as TalksGiven
-ORDER BY TalksGiven DESC LIMIT 20;
-```
-
-3) Find talks related by tag to specific talk and count them:
-```
-MATCH (n:Talk {name: "Michael Green: Why we should build wooden skyscrapers"})-[:HasTag]->(t:Tag)<-[:HasTag]-(m:Talk)
-WITH * ORDER BY m.name
-RETURN t.name, COLLECT(m.name), COUNT(m) AS TalksCount
-ORDER BY TalksCount DESC;
-```
-
-4) Find 20 most frequently used tags:
-```
-MATCH (t:Tag)<-[:HasTag]-(n:Talk)
-RETURN t.name as Tag, COUNT(n) AS TalksCount
-ORDER BY TalksCount DESC, Tag LIMIT 20;
-```
-
-5) Find 20 talks most rated as "Funny". If you want to query by other ratings,
-possible values are: Obnoxious, Jaw-dropping, OK, Persuasive, Beautiful, Confusing,
-Longwinded, Unconvincing, Fascinating, Ingenious, Courageous, Funny, Informative and
-Inspiring.
-```
-MATCH (r:Rating{name:"Funny"})<-[e:HasRating]-(m:Talk)
-RETURN m.name, e.user_count ORDER BY e.user_count DESC LIMIT 20;
-```
-
-6) Find inspiring talks and their speakers from the field of technology:
-```
-MATCH (n:Talk)-[:HasTag]->(m:Tag {name: "technology"})
-MATCH (n)-[r:HasRating]->(p:Rating {name: "Inspiring"})
-MATCH (n)<-[:Gave]-(s:Speaker)
-WHERE r.user_count > 1000
-RETURN n.title, s.name, r.user_count ORDER BY r.user_count DESC;
-```
-
-7) Now let's see one real-world example &mdash; how to make a real-time recommendation.
-If you've just watched a talk from a certain speaker(e.g. Hans Rosling) you might be
-interested in finding more talks from the same speaker on a similar topic:
-
-```
-MATCH (n:Speaker {name: "Hans Rosling"})-[:Gave]->(m:Talk)
-MATCH (t:Talk {title: "New insights on poverty"})-[:HasTag]->(tag:Tag)<-[:HasTag]-(m)
-WITH * ORDER BY tag.name
-RETURN m.title as Title, COLLECT(tag.name), COUNT(tag) as TagCount
-ORDER BY TagCount DESC, Title;
-```
-
-The following few queries are focused on extracting information about TED events.
-
-8) Find how many talks were given per event:
-```
-MATCH (n:Event)<-[:InEvent]-(t:Talk)
-RETURN n.name as Event, COUNT(t) AS TalksCount
-ORDER BY TalksCount DESC, Event
-LIMIT 20;
-```
-
-9) Find the most popular tags in the specific event:
-```
-MATCH (n:Event {name:"TED2006"})<-[:InEvent]-(t:Talk)-[:HasTag]->(tag:Tag)
-RETURN tag.name as Tag, COUNT(t) AS TalksCount
-ORDER BY TalksCount DESC, Tag
-LIMIT 20;
-```
-
-10) Discover which speakers participated in more than 2 events:
-```
-MATCH (n:Speaker)-[:Gave]->(t:Talk)-[:InEvent]->(e:Event)
-WITH n, COUNT(e) AS EventsCount WHERE EventsCount > 2
-RETURN n.name as Speaker, EventsCount
-ORDER BY EventsCount DESC, Speaker;
-```
-
-11) For each speaker search for other speakers that participated in same events:
-```
-MATCH (n:Speaker)-[:Gave]->()-[:InEvent]->(e:Event)<-[:InEvent]-()<-[:Gave]-(m:Speaker)
-WHERE n.name != m.name
-WITH DISTINCT n, m ORDER BY m.name
-RETURN n.name AS Speaker, COLLECT(m.name) AS Others
-ORDER BY Speaker;
-```
-
-Now you're ready to explore the world of graph databases with Memgraph
-by yourself and try it on many more examples and datasets.
-
-### Graph Gists Examples
-
-A nice looking set of small graph examples can be found
-[here](https://neo4j.com/graphgists/).  You can take any use-case and try to
-execute the queries against Memgraph. To clear the database between trying out
-examples, execute the query:
-
-```
-MATCH (n) DETACH DELETE n;
-```
-
-

docs/user_technical/import-tools.md

@@ -1,118 +0,0 @@
-## Import Tools
-
-Memgraph comes with tools for importing data into the database. Currently,
-only import of CSV formatted is supported. We plan to support more formats in
-the future.
-
-### CSV Import Tool
-
-CSV data should be in Neo4j CSV compatible format. Detailed format
-specification can be found
-[here](https://neo4j.com/docs/operations-manual/current/tools/import/file-header-format/).
-
-The import tool is run from the console, using the `mg_import_csv` command.
-
-If you installed Memgraph using Docker, you will need to run the importer
-using the following command:
-
-```
-docker run -v mg_lib:/var/lib/memgraph -v mg_etc:/etc/memgraph -v mg_import:/import-data \
-  --entrypoint=mg_import_csv memgraph
-```
-
-You can pass CSV files containing node data using the `--nodes` option.
-Multiple files can be specified by repeating the `--nodes` option.  At least
-one node file should be specified. Similarly, graph edges (also known as
-relationships) are passed via the `--relationships` option.  Multiple
-relationship files are imported by repeating the option. Unlike nodes,
-relationships are not required.
-
-After reading the CSV files, the tool will by default search for the installed
-Memgraph configuration. If the configuration is found, the data will be
-written in the configured durability directory. If the configuration isn't
-found, you will need to use the `--out` option to specify the output file. You
-can use the same option to override the default behaviour.
-
-Memgraph will recover the imported data on the next startup by looking in the
-durability directory.
-
-For information on other options, run:
-
-```
-mg_import_csv --help
-```
-
-When using Docker, this translates to:
-
-```
-docker run --entrypoint=mg_import_csv memgraph --help
-```
-
-#### Example
-
-Let's import a simple dataset.
-
-Store the following in `comment_nodes.csv`.
-
-```
-id:ID(COMMENT_ID),country:string,browser:string,content:string,:LABEL
-0,Croatia,Chrome,yes,Message;Comment
-1,United Kingdom,Chrome,thanks,Message;Comment
-2,Germany,,LOL,Message;Comment
-3,France,Firefox,I see,Message;Comment
-4,Italy,Internet Explorer,fine,Message;Comment
-```
-
-Now, let's add `forum_nodes.csv`.
-
-```
-id:ID(FORUM_ID),title:string,:LABEL
-0,General,Forum
-1,Support,Forum
-2,Music,Forum
-3,Film,Forum
-4,Programming,Forum
-```
-
-And finally, set relationships between comments and forums in
-`relationships.csv`.
-
-```
-:START_ID(COMMENT_ID),:END_ID(FORUM_ID),:TYPE
-0,0,POSTED_ON
-1,1,POSTED_ON
-2,2,POSTED_ON
-3,3,POSTED_ON
-4,4,POSTED_ON
-```
-
-Now, you can import the dataset in Memgraph.
-
-WARNING: Your existing recovery data will be considered obsolete, and Memgraph
-will load the new dataset.
-
-Use the following command:
-
-```
-mg_import_csv --nodes=comment_nodes.csv --nodes=forum_nodes.csv --relationships=relationships.csv
-```
-
-If using Docker, things are a bit more complicated. First you need to move the
-CSV files where the Docker image can see them:
-
-```
-mkdir -p /var/lib/docker/volumes/mg_import/_data
-cp comment_nodes.csv forum_nodes.csv relationships.csv /var/lib/docker/volumes/mg_import/_data
-```
-
-Then, run the importer with the following:
-
-```
-docker run -v mg_lib:/var/lib/memgraph -v mg_etc:/etc/memgraph -v mg_import:/import-data \
-  --entrypoint=mg_import_csv memgraph \
-  --nodes=/import-data/comment_nodes.csv --nodes=/import-data/forum_nodes.csv \
-  --relationships=/import-data/relationships.csv
-```
-
-Next time you run Memgraph, the dataset will be loaded.
-

docs/user_technical/open-cypher.md

@@ -1,733 +0,0 @@
-## openCypher Query Language
-
-[*openCypher*](http://www.opencypher.org/) is a query language for querying
-graph databases. It aims to be intuitive and easy to learn, while
-providing a powerful interface for working with graph based data.
-
-*Memgraph* supports most of the commonly used constructs of the language. This
-chapter contains the details of implemented features. Additionally,
-not yet supported features of the language are listed.
-
-  * [Reading Existing Data](#reading-existing-data)
-  * [Writing New Data](#writing-new-data)
-  * [Reading & Writing](#reading-amp-writing)
-  * [Indexing](#indexing)
-  * [Other Features](#other-features)
-
-### Reading Existing Data
-
-The simplest usage of the language is to find data stored in the
-database. For that purpose, the following clauses are offered:
-
-  * `MATCH`, which searches for patterns;
-  * `WHERE`, for filtering the matched data and
-  * `RETURN`, for defining what will be presented to the user in the result
-    set.
-  * `UNION` and `UNION ALL` for combining results from multiple queries.
-
-#### MATCH
-
-This clause is used to obtain data from Memgraph by matching it to a given
-pattern. For example, to find each node in the database, you can use the
-following query.
-
-    MATCH (node) RETURN node
-
-Finding connected nodes can be achieved by using the query:
-
-    MATCH (node1)-[connection]-(node2) RETURN node1, connection, node2
-
-In addition to general pattern matching, you can narrow the search down by
-specifying node labels and properties. Similarly, edge types and properties
-can also be specified. For example, finding each node labeled as `Person` and
-with property `age` being 42, is done with the following query.
-
-    MATCH (n :Person {age: 42}) RETURN n
-
-While their friends can be found with the following.
-
-    MATCH (n :Person {age: 42})-[:FriendOf]-(friend) RETURN friend
-
-There are cases when a user needs to find data which is connected by
-traversing a path of connections, but the user doesn't know how many
-connections need to be traversed. openCypher allows for designating patterns
-with *variable path lengths*. Matching such a path is achieved by using the
-`*` (*asterisk*) symbol inside the edge element of a pattern. For example,
-traversing from `node1` to `node2` by following any number of connections in a
-single direction can be achieved with:
-
-    MATCH (node1)-[r*]->(node2) RETURN node1, r, node2
-
-If paths are very long, finding them could take a long time. To prevent that,
-a user can provide the minimum and maximum length of the path. For example,
-paths of length between 2 and 4 can be obtained with a query like:
-
-    MATCH (node1)-[r*2..4]->(node2) RETURN node1, r, node2
-
-It is possible to name patterns in the query and return the resulting paths.
-This is especially useful when matching variable length paths:
-
-    MATCH path = ()-[r*2..4]->() RETURN path
-
-More details on how `MATCH` works can be found
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/match/).
-
-The `MATCH` clause can be modified by prepending the `OPTIONAL` keyword.
-`OPTIONAL MATCH` clause behaves the same as a regular `MATCH`, but when it
-fails to find the pattern, missing parts of the pattern will be filled with
-`null` values. Examples can be found
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/optional-match/).
-
-#### WHERE
-
-You have already seen that simple filtering can be achieved by using labels
-and properties in `MATCH` patterns. When more complex filtering is desired,
-you can use `WHERE` paired with `MATCH` or `OPTIONAL MATCH`. For example,
-finding each person older than 20 is done with the this query.
-
-    MATCH (n :Person) WHERE n.age > 20 RETURN n
-
-Additional examples can be found
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/where/).
-
-#### RETURN
-
-The `RETURN` clause defines which data should be included in the resulting
-set. Basic usage was already shown in the examples for `MATCH` and `WHERE`
-clauses. Another feature of `RETURN` is renaming the results using the `AS`
-keyword.
-
-Example.
-
-    MATCH (n :Person) RETURN n AS people
-
-That query would display all nodes under the header named `people` instead of
-`n`.
-
-When you want to get everything that was matched, you can use the `*`
-(*asterisk*) symbol.
-
-This query:
-
-    MATCH (node1)-[connection]-(node2) RETURN *
-
-is equivalent to:
-
-    MATCH (node1)-[connection]-(node2) RETURN node1, connection, node2
-
-`RETURN` can be followed by the `DISTINCT` operator, which will remove
-duplicate results. For example, getting unique names of people can be achieved
-with:
-
-    MATCH (n :Person) RETURN DISTINCT n.name
-
-Besides choosing what will be the result and how it will be named, the
-`RETURN` clause can also be used to:
-
-  * limit results with `LIMIT` sub-clause;
-  * skip results with `SKIP` sub-clause;
-  * order results with `ORDER BY` sub-clause and
-  * perform aggregations (such as `count`).
-
-More details on `RETURN` can be found
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/return/).
-
-##### SKIP & LIMIT
-
-These sub-clauses take a number of how many results to skip or limit.
-For example, to get the first 3 results you can use this query.
-
-    MATCH (n :Person) RETURN n LIMIT 3
-
-If you want to get all the results after the first 3, you can use the
-following.
-
-    MATCH (n :Person) RETURN n SKIP 3
-
-The `SKIP` and `LIMIT` can be combined. So for example, to get the 2nd result,
-you can do:
-
-    MATCH (n :Person) RETURN n SKIP 1 LIMIT 1
-
-##### ORDER BY
-
-Since the patterns which are matched can come in any order, it is very useful
-to be able to enforce some ordering among the results. In such cases, you can
-use the `ORDER BY` sub-clause.
-
-For example, the following query will get all `:Person` nodes and order them
-by their names.
-
-    MATCH (n :Person) RETURN n ORDER BY n.name
-
-By default, ordering will be in the ascending order. To change the order to be
-descending, you should append `DESC`.
-
-For example, to order people by their name descending, you can use this query.
-
-    MATCH (n :Person) RETURN n ORDER BY n.name DESC
-
-You can also order by multiple variables. The results will be sorted by the
-first variable listed. If the values are equal, the results are sorted by the
-second variable, and so on.
-
-Example. Ordering by first name descending and last name ascending.
-
-    MATCH (n :Person) RETURN n ORDER BY n.name DESC, n.lastName
-
-Note that `ORDER BY` sees only the variable names as carried over by `RETURN`.
-This means that the following will result in an error.
-
-    MATCH (n :Person) RETURN old AS new ORDER BY old.name
-
-Instead, the `new` variable must be used:
-
-    MATCH (n: Person) RETURN old AS new ORDER BY new.name
-
-The `ORDER BY` sub-clause may come in handy with `SKIP` and/or `LIMIT`
-sub-clauses. For example, to get the oldest person you can use the following.
-
-    MATCH (n :Person) RETURN n ORDER BY n.age DESC LIMIT 1
-
-##### Aggregating
-
-openCypher has functions for aggregating data. Memgraph currently supports
-the following aggregating functions.
-
-  * `avg`, for calculating the average.
-  * `collect`, for collecting multiple values into a single list or map. If given a single expression values are collected into a list. If given two expressions, values are collected into a map where the first expression denotes map keys (must be string values) and the second expression denotes map values.
-  * `count`, for counting the resulting values.
-  * `max`, for calculating the maximum result.
-  * `min`, for calculating the minimum result.
-  * `sum`, for getting the sum of numeric results.
-
-Example, calculating the average age:
-
-    MATCH (n :Person) RETURN avg(n.age) AS averageAge
-
-Collecting items into a list:
-
-    MATCH (n :Person) RETURN collect(n.name) AS list_of_names
-
-Collecting items into a map:
-
-    MATCH (n :Person) RETURN collect(n.name, n.age) AS map_name_to_age
-
-Click
-[here](https://neo4j.com/docs/developer-manual/current/cypher/functions/aggregating/)
-for additional details on how aggregations work.
-
-#### UNION and UNION ALL
-
-openCypher supports combining results from multiple queries into a single result
-set. That result will contain rows that belong to queries in the union
-respecting the union type.
-
-Using `UNION` will contain only distinct rows while `UNION ALL` will keep all
-rows from all given queries.
-
-Restrictions when using `UNION` or `UNION ALL`:
-  * The number and the names of columns returned by queries must be the same
-    for all of them.
-  * There can be only one union type between single queries, ie. a query can't
-    contain both `UNION` and `UNION ALL`.
-
-Example, get distinct names that are shared between persons and movies:
-
-    MATCH(n: Person) RETURN n.name as name UNION MATCH(n: Movie) RETURN n.name as name
-
-Example, get all names that are shared between persons and movies (including duplicates):
-
-    MATCH(n: Person) RETURN n.name as name UNION ALL MATCH(n: Movie) RETURN n.name as name
-
-### Writing New Data
-
-For adding new data, you can use the following clauses.
-
-  * `CREATE`, for creating new nodes and edges.
-  * `SET`, for adding new or updating existing labels and properties.
-  * `DELETE`, for deleting nodes and edges.
-  * `REMOVE`, for removing labels and properties.
-
-You can still use the `RETURN` clause to produce results after writing, but it
-is not mandatory.
-
-Details on which kind of data can be stored in *Memgraph* can be found in
-**Storable Data Types** chapter.
-
-#### CREATE
-
-This clause is used to add new nodes and edges to the database. The creation
-is done by providing a pattern, similarly to `MATCH` clause.
-
-For example, to create 2 new nodes connected with a new edge, use this query.
-
-    CREATE (node1)-[:edge_type]->(node2)
-
-Labels and properties can be set during creation using the same syntax as in
-[MATCH](#match) patterns. For example, creating a node with a label and a
-property:
-
-    CREATE (node :Label {property: "my property value"}
-
-Additional information on `CREATE` is
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/create/).
-
-#### SET
-
-The `SET` clause is used to update labels and properties of already existing
-data.
-
-Example. Incrementing everyone's age by 1.
-
-    MATCH (n :Person) SET n.age = n.age + 1
-
-Click
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/create/)
-for a more detailed explanation on what can be done with `SET`.
-
-#### DELETE
-
-This clause is used to delete nodes and edges from the database.
-
-Example. Removing all edges of a single type.
-
-    MATCH ()-[edge :type]-() DELETE edge
-
-When testing the database, you want to often have a clean start by deleting
-every node and edge in the database. It is reasonable that deleting each node
-should delete all edges coming into or out of that node.
-
-    MATCH (node) DELETE node
-
-But, openCypher prevents accidental deletion of edges. Therefore, the above
-query will report an error. Instead, you need to use the `DETACH` keyword,
-which will remove edges from a node you are deleting. The following should
-work and *delete everything* in the database.
-
-    MATCH (node) DETACH DELETE node
-
-More examples are
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/delete/).
-
-#### REMOVE
-
-The `REMOVE` clause is used to remove labels and properties from nodes and
-edges.
-
-Example.
-
-    MATCH (n :WrongLabel) REMOVE n :WrongLabel, n.property
-
-### Reading & Writing
-
-OpenCypher supports combining multiple reads and writes using the
-`WITH` clause. In addition to combining, the `MERGE` clause is provided which
-may create patterns if they do not exist.
-
-#### WITH
-
-The write part of the query cannot be simply followed by another read part. In
-order to combine them, `WITH` clause must be used. The names this clause
-establishes are transferred from one part to another.
-
-For example, creating a node and finding all nodes with the same property.
-
-    CREATE (node {property: 42}) WITH node.property AS propValue
-    MATCH (n {property: propValue}) RETURN n
-
-Note that the `node` is not visible after `WITH`, since only `node.property`
-was carried over.
-
-This clause behaves very much like `RETURN`, so you should refer to features
-of `RETURN`.
-
-#### MERGE
-
-The `MERGE` clause is used to ensure that a pattern you are looking for exists
-in the database. This means that if the pattern is not found, it will be
-created. In a way, this clause is like a combination of `MATCH` and `CREATE`.
-
-
-Example. Ensure that a person has at least one friend.
-
-    MATCH (n :Person) MERGE (n)-[:FriendOf]->(m)
-
-The clause also provides additional features for updating the values depending
-on whether the pattern was created or matched. This is achieved with `ON
-CREATE` and `ON MATCH` sub clauses.
-
-Example. Set a different properties depending on what `MERGE` did.
-
-    MATCH (n :Person) MERGE (n)-[:FriendOf]->(m)
-    ON CREATE SET m.prop = "created" ON MATCH SET m.prop = "existed"
-
-For more details, click [this
-link](https://neo4j.com/docs/developer-manual/current/cypher/clauses/merge/).
-
-### Indexing
-
-An index stores additional information on certain types of data, so that
-retrieving said data becomes more efficient. Downsides of indexing are:
-
-  * requiring extra storage for each index and
-  * slowing down writes to the database.
-
-Carefully choosing which data to index can tremendously improve data retrieval
-efficiency, and thus make index downsides negligible.
-
-Memgraph automatically indexes labeled data. This improves queries
-which fetch nodes by label:
-
-    MATCH (n :Label) ... RETURN n
-
-Indexing can also be applied to data with a specific combination of label and
-property. These are not automatically created, instead a user needs to create
-them explicitly. Creation is done using a special
-`CREATE INDEX ON :Label(property)` language construct.
-
-For example, to index nodes which is labeled as `:Person` and has a property
-named `age`:
-
-    CREATE INDEX ON :Person(age)
-
-After the index is created, retrieving those nodes will become more efficient.
-For example, the following query will retrieve all nodes which have an `age`
-property, instead of fetching each `:Person` node and checking whether the
-property exists.
-
-    MATCH (n :Person {age: 42}) RETURN n
-
-Using index based retrieval also works when filtering labels and properties
-with `WHERE`. For example, the same effect as in the previous example can be
-done with:
-
-    MATCH (n) WHERE n:Person AND n.age = 42 RETURN n
-
-Since the filter inside `WHERE` can contain any kind of an expression, the
-expression can be complicated enough so that the index does not get used. We
-are continuously improving the recognition of index usage opportunities from a
-`WHERE` expression. If there is any suspicion that an index may not be used,
-we recommend putting properties and labels inside the `MATCH` pattern.
-
-Currently, once an index is created it cannot be deleted. This feature will be
-implemented very soon. The expected syntax for removing an index will be `DROP
-INDEX ON :Label(property)`.
-
-### Other Features
-
-The following sections describe some of the other supported features.
-
-#### Filtering Variable Length Paths
-
-OpenCypher supports only simple filtering when matching variable length paths.
-For example:
-
-    MATCH (n)-[r:Type * {x: 42}]-(m)
-
-This will produce only those paths whose edges have the required `Type` and `x`
-property value.
-
-Memgraph extends openCypher with a syntax for arbitrary filter expressions
-during path matching. The next example filters edges which have property `x`
-between `0` and `10`.
-
-    MATCH (n)-[r * (edge, node | 0 < edge.x < 10)]-(m)
-
-Here we introduce a lambda function with parentheses, where the first two
-arguments, `edge` and `node`, correspond to each edge and node during path
-matching. `node` is the destination node we are moving to across the current
-`edge`. The last `node` value will be the same value as `m`. Following the
-pipe (`|`) character is an arbitrary expression which must produce a boolean
-value.  If `True`, matching continues, otherwise the path is discarded.
-
-The previous example can be written using the `all` function:
-
-    MATCH (n)-[r *]-(m) WHERE all(edge IN r WHERE 0 < edge.x < 10)
-
-However, filtering using a lambda function is more efficient because paths
-may be discarded earlier in the traversal. Furthermore, it provides more
-flexibility for deciding what kind of paths are matched due to more expressive
-filtering capabilities. Therefore, filtering through lambda functions should
-be preferred whenever possible.
-
-#### Breadth First Search
-
-A typical graph use-case is searching for the shortest path between nodes.
-The openCypher standard does not define this feature, so Memgraph provides
-a custom implementation, based on the edge expansion syntax.
-
-Finding the shortest path between nodes can be done using breadth-first
-expansion:
-
-    MATCH (a {id: 723})-[r:Type *bfs..10]-(b {id: 882}) RETURN *
-
-The above query will find all paths of length up to 10 between nodes `a` and `b`.
-The edge type and maximum path length are used in the same way like in variable
-length expansion.
-
-To find only the shortest path, simply append `LIMIT 1` to the `RETURN` clause.
-
-    MATCH (a {id: 723})-[r:Type *bfs..10]-(b {id: 882}) RETURN * LIMIT 1
-
-Breadth-first expansion allows an arbitrary expression filter that determines
-if an expansion is allowed. Following is an example in which expansion is
-allowed only over edges whose `x` property is greater than `12` and nodes `y`
-whose property is less than `3`:
-
-    MATCH (a {id: 723})-[*bfs..10 (e, n | e.x > 12 and n.y < 3)]-() RETURN *
-
-The filter is defined as a lambda function over `e` and `n`, which denote the edge
-and node being expanded over in the breadth first search.
-
-There are a few benefits of the breadth-first expansion approach, as opposed to
-a specialized `shortestPath` function. For one, it is possible to inject
-expressions that filter on nodes and edges along the path itself, not just the final
-destination node. Furthermore, it's possible to find multiple paths to multiple destination
-nodes regardless of their length. Also, it is possible to simply go through a node's
-neighbourhood in breadth-first manner.
-
-Currently, it isn't possible to get all shortest paths to a single node using
-Memgraph's breadth-first expansion.
-
-#### Weighted Shortest Path
-
-Another standard use-case in a graph is searching for the weighted shortest
-path between nodes. The openCypher standard does not define this feature, so
-Memgraph provides a custom implementation, based on the edge expansion syntax.
-
-Finding the weighted shortest path between nodes is done using the weighted
-shortest path expansion:
-
-    MATCH (a {id: 723})-[le *wShortest 10 (e, n | e.weight) total_weight]-(b {id: 882}) RETURN *
-
-The above query will find the shortest path of length up to 10 nodes between
-nodes `a`  and `b`.
-
-Weighted Shortest Path expansion allows an arbitrary expression that determines
-the weight for the current expansion. Total weight of a path is calculated as
-the sum of all weights on the path between two nodes. Following is an example in
-which the weight between nodes is defined as the product of edge weights
-(instead of sum), assuming all weights are greater than '1':
-
-    MATCH (a {id: 723})-[le *wShortest 10 (e, n | log(e.weight)) total_weight]-(b {id: 882}) RETURN exp(total_weight)
-
-
-Weighted Shortest Path expansions also allows an arbitrary expression filter
-that determines if an expansion is allowed. Following is an example in which
-expansion is allowed only over edges whose `x` property is greater than `12`
-and nodes `y` whose property is less than `3`:
-
-    MATCH (a {id: 723})-[le *wShortest 10 (e, n | e.weight) total_weight (e, n | e.x > 12 and n.y < 3)]-(b {id: 882}) RETURN exp(total_weight)
-
-Both weight and filter expression are defined as lambda functions over `e` and
-`n`, which denote the edge and the node being expanded over in the weighted
-shortest path search.
-
-#### UNWIND
-
-The `UNWIND` clause is used to unwind a list of values as individual rows.
-
-Example. Produce rows out of a single list.
-
-    UNWIND [1,2,3] AS listElement RETURN listElement
-
-More examples are
-[here](https://neo4j.com/docs/developer-manual/current/cypher/clauses/unwind/).
-
-#### Functions
-
-You have already been introduced to one type of functions, [aggregating
-functions](#aggregating). This section contains the list of other supported
-functions.
-
- Name            | Description
------------------|------------
- `coalesce`      | Returns the first non null argument.
- `startNode`     | Returns the starting node of an edge.
- `endNode`       | Returns the destination node of an edge.
- `degree`        | Returns the number of edges (both incoming and outgoing) of a node.
- `head`          | Returns the first element of a list.
- `last`          | Returns the last element of a list.
- `properties`    | Returns the properties of a node or an edge.
- `size`          | Returns the number of elements in a list or a map. When given a string it returns the number of characters. When given a path it returns the number of expansions (edges) in that path.
- `toBoolean`     | Converts the argument to a boolean.
- `toFloat`       | Converts the argument to a floating point number.
- `toInteger`     | Converts the argument to an integer.
- `type`          | Returns the type of an edge as a character string.
- `keys`          | Returns a list keys of properties from an edge or a node. Each key is represented as a string of characters.
- `labels`        | Returns a list of labels from a node. Each label is represented as a character string.
- `nodes`         | Returns a list of nodes from a path.
- `relationships` | Returns a list of relationships from a path.
- `range`         | Constructs a list of value in given range.
- `tail`          | Returns all elements after the first of a given list.
- `abs`           | Returns the absolute value of a number.
- `ceil`          | Returns the smallest integer greater than or equal to given number.
- `floor`         | Returns the largest integer smaller than or equal to given number.
- `round`         | Returns the number, rounded to the nearest integer. Tie-breaking is done using the *commercial rounding*,  where -1.5 produces -2 and 1.5 produces 2.
- `exp`           | Calculates `e^n` where `e` is the base of the natural logarithm, and `n` is the given number.
- `log`           | Calculates the natural logarithm of a given number.
- `log10`         | Calculates the logarithm (base 10) of a given number.
- `sqrt`          | Calculates the square root of a given number.
- `acos`          | Calculates the arccosine of a given number.
- `asin`          | Calculates the arcsine of a given number.
- `atan`          | Calculates the arctangent of a given number.
- `atan2`         | Calculates the arctangent2 of a given number.
- `cos`           | Calculates the cosine of a given number.
- `sin`           | Calculates the sine of a given number.
- `tan`           | Calculates the tangent of a given number.
- `sign`          | Applies the signum function to a given number and returns the result. The signum of positive numbers is 1, of negative -1 and for 0 returns 0.
- `e`             | Returns the base of the natural logarithm.
- `pi`            | Returns the constant *pi*.
- `rand`          | Returns a random floating point number between 0 (inclusive) and 1 (exclusive).
- `startsWith`    | Check if the first argument starts with the second.
- `endsWith`      | Check if the first argument ends with the second.
- `contains`      | Check if the first argument has an element which is equal to the second argument.
- `all`           | Check if all elements of a list satisfy a predicate.<br/>The syntax is: `all(variable IN list WHERE predicate)`.<br/> NOTE: Whenever possible, use Memgraph's lambda functions when [matching](#filtering-variable-length-paths) instead.
- `single`        | Check if only one element of a list satisfies a predicate.<br/>The syntax is: `single(variable IN list WHERE predicate)`.
- `reduce`        | Accumulate list elements into a single result by applying an expression. The syntax is:<br/>`reduce(accumulator = initial_value, variable IN list | expression)`.
- `assert`        | Raises an exception reported to the client if the given argument is not `true`.
- `counter`       | Generates integers that are guaranteed to be unique on the database level, for the given counter name.
- `counterSet`    | Sets the counter with the given name to the given value.
- `indexInfo`     | Returns a list of all the indexes available in the database. The list includes indexes that are not yet ready for use (they are concurrently being built by another transaction).
-
-#### String Operators
-
-Apart from comparison and concatenation operators openCypher provides special
-string operators for easier matching of substrings:
-
-Operator           | Description
--------------------|------------
- `a STARTS WITH b` | Returns true if prefix of string a is equal to string b.
- `a ENDS WITH b`   | Returns true if suffix of string a is equal to string b.
- `a CONTAINS b`    | Returns true if some substring of string a is equal to string b.
-
-#### Parameters
-
-When automating the queries for Memgraph, it comes in handy to change only
-some parts of the query. Usually, these parts are values which are used for
-filtering results or similar, while the rest of the query remains the same.
-
-Parameters allow reusing the same query, but with different parameter values.
-The syntax uses the `$` symbol to designate a parameter name. We don't allow
-old Cypher parameter syntax using curly braces. For example, you can parameterize
-filtering a node property:
-
-    MATCH (node1 {property: $propertyValue}) RETURN node1
-
-You can use parameters instead of any literal in the query, but not instead of
-property maps even though that is allowed in standard openCypher. Following
-example is illegal in Memgraph:
-
-    MATCH (node1 $propertyValue) RETURN node1
-
-To use parameters with Python driver use following syntax:
-
-    session.run('CREATE (alice:Person {name: $name, age: $ageValue}',
-                name='Alice', ageValue=22)).consume()
-
-To use parameters which names are integers you will need to wrap parameters in
-a dictionary and convert them to strings before running a query:
-
-    session.run('CREATE (alice:Person {name: $0, age: $1}',
-                {'0': "Alice", '1': 22})).consume()
-
-To use parameters with some other driver please consult appropriate
-documentation.
-
-#### CASE
-
-Conditional expressions can be expressed in openCypher language by simple and
-generic form of `CASE` expression. A simple form is used to compare an expression
-against multiple predicates. For the first matched predicate result of the
-expression provided after the `THEN` keyword is returned.  If no expression is
-matched value following `ELSE` is returned is provided, or `null` if `ELSE` is not
-used:
-
-    MATCH (n)
-    RETURN CASE n.currency WHEN "DOLLAR" THEN "$" WHEN "EURO" THEN "€" ELSE "UNKNOWN" END
-
-In generic form, you don't need to provide an expression whose value is compared to
-predicates, but you can list multiple predicates and the first one that evaluates
-to true is matched:
-
-    MATCH (n)
-    RETURN CASE WHEN n.height < 30 THEN "short" WHEN n.height > 300 THEN "tall" END
-
-### Differences
-
-Although we try to implement openCypher query language as closely to the
-language reference as possible, we had to make some changes to enhance the
-user experience.
-
-#### Symbolic Names
-
-We don't allow symbolic names (variables, label names...) to be openCypher
-keywords (WHERE, MATCH, COUNT, SUM...).
-
-#### Unicode Codepoints in String Literal
-
-Use `\u` followed by 4 hex digits in string literal for UTF-16 codepoint and
-`\U` with 8 hex digits for UTF-32 codepoint in Memgraph.
-
-
-### Difference from Neo4j's Cypher Implementation
-
-The openCypher initiative stems from Neo4j's Cypher query language. Following is a list
-of most important differences between Neo's Cypher and Memgraph's openCypher implementation,
-for users that are already familiar with Neo4j. There might be other differences not documented
-here (especially subtle semantic ones).
-
-#### Unsupported Constructs
-
-* Data importing. Memgraph doesn't support Cypher's CSV importing capabilities.
-* The `FOREACH` language construct for performing an operation on every list element.
-* The `CALL` construct for a standalone function call. This can be expressed using
-  `RETURN functioncall()`. For example, with Memgraph you can get information about
-  the indexes present in the database using the `RETURN indexinfo()` openCypher query.
-* Stored procedures.
-* Regular expressions for string matching.
-* `shortestPath` and `allShortestPaths` functions. `shortestPath` can be expressed using
-  Memgraph's breadth-first expansion syntax already described in this document.
-* Patterns in expressions. For example, Memgraph doesn't support `size((n)-->())`. Most of the time
-  the same functionalities can be expressed differently in Memgraph using `OPTIONAL` expansions,
-  function calls etc.
-* Map projections such as `MATCH (n) RETURN n {.property1, .property2}`.
-
-#### Unsupported Functions
-
-General purpose functions:
-* `id()` - Memgraph does not expose a public unique node identifier.
-* `exists(n.property)` - This can be expressed using `n.property IS NOT NULL`.
-* `length()` is named `size()` in Memgraph.
-
-Path functions:
-* `extract()`
-
-Aggregation functions:
-* `count(DISTINCT variable)` - This can be expressed using `WITH DISTINCT variable RETURN count(variable)`.
-
-Mathematical functions:
-* `percentileDisc()`
-* `stDev()`
-* `point()`
-* `distance()`
-* `degrees()`
-
-String functions:
-* `replace()`
-* `substring()`
-* `left()`
-* `trim()`
-* `toupper()`
-* `tolower()`
-* `split()`
-* `reverse()`
-
-List functions:
-* `any()`
-* `none()`
-* `single()`
-* `head()`
-* `last()`
-* `tail()`

docs/user_technical/quick-start.md

@@ -1,148 +0,0 @@
-## Quick Start
-
-This chapter outlines installing and running Memgraph, as well as executing
-basic queries against the database.
-
-### Installation
-
-The Memgraph binary is offered as:
-
-  * Debian package for Debian 9 (Stretch) and
-  * Docker image.
-
-After downloading the binary, proceed to the corresponding section below.
-
-#### Docker Installation
-
-Before proceeding with the installation, please install the Docker engine on
-the system. Instructions on how to install Docker can be found on the
-[official Docker website](https://docs.docker.com/engine/installation).
-Memgraph Docker image was built with Docker version `1.12` and should be
-compatible with all later versions.
-
-After installing and running Docker, download the Memgraph Docker image and
-import it with the following command.
-
-```
-docker load -i /path/to/memgraph-<version>-docker.tar.gz
-```
-
-Memgraph is then started with another docker command.
-
-```
-docker run -p 7687:7687 \
-  -v mg_lib:/var/lib/memgraph -v mg_log:/var/log/memgraph -v mg_etc:/etc/memgraph \
-  memgraph
-```
-
-On success, expect to see output similar to the following.
-
-```
-Starting 8 workers
-Server is fully armed and operational
-Listening on 0.0.0.0 at 7687
-```
-
-Memgraph is now ready to process queries, you may now proceed to
-[querying](#querying). To stop Memgraph, press `Ctrl-c`.
-
-Memgraph configuration is available in Docker's named volume `mg_etc`. On
-Linux systems it should be in
-`/var/lib/docker/volumes/mg_etc/_data/memgraph.conf`. After changing the
-configuration, Memgraph needs to be restarted.
-
-#### Debian Package Installation
-
-After downloading Memgraph as a Debian package, install it by running the
-following.
-
-```
-dpkg -i /path/to/memgraph_<version>.deb
-```
-
-If the installation was successful, Memgraph should already be running. To
-make sure that is true, start it explicitly with the command:
-
-```
-systemctl start memgraph
-```
-
-To verify that Memgraph is running, run the following command.
-
-```
-journalctl --unit memgraph
-```
-
-It is expected to see something like the following output.
-
-```
-Nov 23 13:40:13 hostname memgraph[14654]: Starting 8 workers
-Nov 23 13:40:13 hostname memgraph[14654]: Server is fully armed and operational
-Nov 23 13:40:13 hostname memgraph[14654]: Listening on 0.0.0.0 at 7687
-```
-
-Memgraph is now ready to process queries, you may now proceed to
-[querying](#querying). To shutdown Memgraph server, issue the following
-command.
-
-```
-systemctl stop memgraph
-```
-
-Memgraph configuration is available in `/etc/memgraph/memgraph.conf`. After
-changing the configuration, Memgraph needs to be restarted.
-
-### Querying
-
-Memgraph supports the openCypher query language which has been developed by
-[Neo4j](http://neo4j.com). The language is currently going through a
-vendor-independent standardization process. It's a declarative language
-developed specifically for interaction with graph databases.
-
-The easiest way to execute openCypher queries against Memgraph, is using
-Neo4j's command-line tool. The command-line `neo4j-client` can be installed as
-described [on the official website](https://neo4j-client.net).
-
-After installing `neo4j-client`, connect to the running Memgraph instance by
-issuing the following shell command.
-
-```
-neo4j-client --insecure -u "" -p ""  localhost 7687
-```
-
-After the client has started it should present a command prompt similar to:
-
-```
-neo4j-client 2.1.3
-Enter `:help` for usage hints.
-Connected to 'neo4j://@localhost:7687' (insecure)
-neo4j>
-```
-
-At this point it is possible to execute openCypher queries on Memgraph. Each
-query needs to end with the `;` (*semicolon*) character. For example:
-
-```
-CREATE (u:User {name: "Alice"})-[:Likes]->(m:Software {name: "Memgraph"});
-```
-
-The above will create 2 nodes in the database, one labeled "User" with name
-"Alice" and the other labeled "Software" with name "Memgraph". It will also
-create a relationship that "Alice" *likes* "Memgraph".
-
-To find created nodes and relationships, execute the following query:
-
-```
-MATCH (u:User)-[r]->(x) RETURN u, r, x;
-```
-
-### Where to Next
-
-To learn more about the openCypher language, visit **openCypher Query
-Language** chapter in this document. For real-world examples of how to use
-Memgraph visit **Examples** chapter. If you wish to use a programming language
-to execute queries on Memgraph, go to the **Drivers** chapter. Details on what
-can be stored in Memgraph are in **Storable Data Types** chapter.
-
-We *welcome and encourage* your feedback!
-

docs/user_technical/run

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-# curl and php are required
-if [ "$( which curl )" == "" ] || [ "$( which php )" == "" ]; then
-    echo "curl and php are required!"
-    exit 1
-fi
-
-cd ${working_dir}
-generator="couscous.phar"
-
-# download the generator if it's not already downloaded
-if [ ! -f ${generator} ]; then
-    curl -OS http://couscous.io/${generator}
-fi
-
-# run the preview
-php ${generator} preview

docs/user_technical/upcoming-features.md

@@ -1,56 +0,0 @@
-## Upcoming Features
-
-This chapter describes some of the planned features, that we at Memgraph are
-working on.
-
-### Performance Improvements
-
-Excellent database performance is one of Memgraph's long-standing goals. We
-will be continually working on improving the performance. This includes:
-
-  * query compilation;
-  * query execution;
-  * core engine performance;
-  * algorithmic improvements (i.e. bidirectional breadth-first search);
-  * memory usage and
-  * other improvements.
-
-### Label-Property Index Usage Improvements
-
-Currently, indexing combinations of labels and properties can be created, but
-cannot be deleted. We plan to add a new query language construct which will
-allow deletion of created indices.
-
-### Improving openCypher Support
-
-Although we have implemented the most common features of the openCypher query
-language, there are other useful features we are still working on.
-
-#### Functions
-
-Memgraph's openCypher implementation supports the most useful functions, but
-there are more which openCypher provides. Some are related to not yet
-implemented features like paths, while some may use the features Memgraph
-already supports. Out of the remaining functions, some are more useful than
-others and as such they will be supported sooner.
-
-#### List Comprehensions
-
-List comprehensions are similar to the supported `collect` function, which
-generates a list out of multiple values. But unlike `collect`, list
-comprehensions offer a powerful mechanism for filtering or otherwise
-manipulating values which are collected into a list.
-
-For example, getting numbers between 0 and 10 and squaring them:
-
-    RETURN [x IN range(0, 10) | x^2] AS squares
-
-Another example, to collect `:Person` nodes with `age` less than 42, without
-list comprehensions can be achieved with:
-
-    MATCH (n :Person) WHERE n.age < 42 RETURN collect(n)
-
-Using list comprehensions, the same can be done with the query:
-
-    MATCH (n :Person) RETURN [n IN collect(n) WHERE n.age < 42]
-

init

@@ -1,21 +1,22 @@
 #!/bin/bash -e
 
-# TODO: Consider putting boost library in libs/setup.sh, since the license
-# allows source modification and static compilation. Unfortunately, it is quite
-# a pain to set up the boost build process.
 required_pkgs=(git arcanist # source code control
-               cmake clang-3.8 llvm-3.8 pkg-config # build system
+               cmake clang-3.9 llvm-3.9 pkg-config # build system
                curl wget # for downloading libs
                uuid-dev default-jre-headless # required by antlr
                libreadline-dev # for memgraph console
-               libboost-iostreams-dev
-               libboost-serialization-dev
+               libssl-dev
+               libseccomp-dev
                python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
+               uuid-dev # mg-utils
+               libcurl4-openssl-dev # mg-requests
+               sbcl # for custom Lisp C++ preprocessing
               )
 
 optional_pkgs=(clang-format # source code formatting
                doxygen graphviz # source documentation generators
                php-cli # for user technical documentation generators
+               mono-runtime mono-mcs nodejs # for driver tests
               )
 
 use_sudo=0
@@ -43,7 +44,9 @@ function setup_virtualenv () {
     # create new virtualenv
     virtualenv -p python3 ve3 || exit 1
     source ve3/bin/activate
-    pip install -i http://deps.memgraph.io:3141/root/pypi \
+    # we need to increase the timeout for pip because our local cache server
+    # can sometimes be stupid, see: https://github.com/devpi/devpi/issues/208
+    pip --timeout 1000 install -i http://deps.memgraph.io:3141/root/pypi \
                 --trusted-host deps.memgraph.io -r requirements.txt || exit 1
     deactivate
 
@@ -107,6 +110,26 @@ fi
 # create a default build directory
 mkdir -p ./build
 
+# quicklisp package manager for Common Lisp
+# TODO: We should at some point cache or have a mirror of packages we use.
+quicklisp_install_dir="$HOME/quicklisp"
+if [[ -v QUICKLISP_HOME ]]; then
+  quicklisp_install_dir="${QUICKLISP_HOME}"
+fi
+
+# TODO: move the installation of LCP's dependencies into ./setup.sh
+if [[ ! -f "${quicklisp_install_dir}/setup.lisp" ]]; then
+  wget -nv https://beta.quicklisp.org/quicklisp.lisp -O quicklisp.lisp || exit 1
+  echo \
+  "
+  (load \"${DIR}/quicklisp.lisp\")
+  (quicklisp-quickstart:install :path \"${quicklisp_install_dir}\")
+  (ql:quickload '(:cl-ppcre :prove) :silent t)
+  " | sbcl --script || exit 1
+  rm -rf quicklisp.lisp || exit 1
+fi
+ln -Tfs "$DIR/src/lisp" "${quicklisp_install_dir}/local-projects/lcp"
+
 # setup libs (download)
 cd libs
 ./cleanup.sh

libs/CMakeLists.txt

@@ -11,6 +11,10 @@ function(import_header_library name include_dir)
   add_library(${name} INTERFACE IMPORTED GLOBAL)
   set_property(TARGET ${name} PROPERTY
                INTERFACE_INCLUDE_DIRECTORIES ${include_dir})
+  string(TOUPPER ${name} _upper_name)
+  set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
+      "Path to ${name} include directory" FORCE)
+  mark_as_advanced(${_upper_name}_INCLUDE_DIR)
 endfunction(import_header_library)
 
 function(import_library name type location)
@@ -39,7 +43,8 @@ endfunction(import_library)
 function(add_external_project name)
   set(options NO_C_COMPILER)
   set(one_value_kwargs SOURCE_DIR)
-  set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND)
+  set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
+      CONFIGURE_COMMAND)
   cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
   set(source_dir ${CMAKE_CURRENT_SOURCE_DIR}/${name})
   if (KW_SOURCE_DIR)
@@ -50,11 +55,13 @@ function(add_external_project name)
   endif()
   ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS}
                       PREFIX ${source_dir} SOURCE_DIR ${source_dir}
+                      CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND}
                       CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
                                  -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                                  -DCMAKE_INSTALL_PREFIX=${source_dir}
                                  ${KW_CMAKE_ARGS}
-                      INSTALL_COMMAND ${KW_INSTALL_COMMAND})
+                      INSTALL_COMMAND ${KW_INSTALL_COMMAND}
+                      BUILD_COMMAND ${KW_BUILD_COMMAND})
 endfunction(add_external_project)
 
 # Calls `add_external_project`, sets NAME_LIBRARY, NAME_INCLUDE_DIR variables
@@ -92,7 +99,8 @@ import_external_library(fmt STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/fmt/lib/libfmt.a
   ${CMAKE_CURRENT_SOURCE_DIR}/fmt/include
   # Skip testing.
-  CMAKE_ARGS -DFMT_TEST=OFF)
+  CMAKE_ARGS -DFMT_TEST=OFF
+             -DCMAKE_POSITION_INDEPENDENT_CODE=ON)
 
 
 # Setup ltalloc library
@@ -140,7 +148,8 @@ import_external_library(gflags STATIC
   # Don't register installation in ~/.cmake
   CMAKE_ARGS -DREGISTER_INSTALL_PREFIX=OFF
              -DBUILD_gflags_nothreads_LIB=OFF
-             -DGFLAGS_NO_FILENAMES=${GFLAGS_NO_FILENAMES})
+             -DGFLAGS_NO_FILENAMES=${GFLAGS_NO_FILENAMES}
+             -DCMAKE_POSITION_INDEPENDENT_CODE=ON)
 
 # Setup google logging after gflags (so that glog can use it).
 set(GLOG_DISABLE_OPTIONS "0")
@@ -164,3 +173,91 @@ import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR})
 
 # Setup json
 import_header_library(json ${CMAKE_CURRENT_SOURCE_DIR})
+
+# Setup bzip2
+import_external_library(bzip2 STATIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/libbz2.a
+    ${CMAKE_CURRENT_SOURCE_DIR}/bzip2
+    # bzip2's Makefile has -g CFLAG which is redundant
+    CONFIGURE_COMMAND sed -i "s/-Wall -Winline -O2 -g/-Wall -Winline -O2/g" ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/Makefile
+    BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/bzip2
+                       CC=${CMAKE_C_COMPILER}
+                       CXX=${CMAKE_CXX_COMPILER}
+    INSTALL_COMMAND true)
+
+# Setup zlib
+import_external_library(zlib STATIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/zlib/lib/libz.a
+    ${CMAKE_CURRENT_SOURCE_DIR}/zlib)
+
+# Setup RocksDB
+import_external_library(rocksdb STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/librocksdb.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include
+  # RocksDB's cmake on Linux doesn't generate static_lib target.
+  # That's the reason why NoOps (true) are used as configure
+  # and install commands. Build command uses RocksDB's Makefile.
+  CONFIGURE_COMMAND true
+  BUILD_COMMAND ROCKSDB_DISABLE_FALLOCATE=1
+                ROCKSDB_DISABLE_SNAPPY=1
+                ROCKSDB_DISABLE_LZ4=1
+                ROCKSDB_DISABLE_ZSTD=1
+                ROCKSDB_DISABLE_NUMA=1
+                ROCKSDB_DISABLE_TBB=1
+                ROCKSDB_DISABLE_JEMALLOC=1
+                ROCKSDB_DISABLE_TCMALLOC=1
+                make -C ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb static_lib
+                     -j${NPROC}
+                     CC=${CMAKE_C_COMPILER}
+                     CXX=${CMAKE_CXX_COMPILER}
+  INSTALL_COMMAND true)
+
+# Setup Cap'n Proto
+ExternalProject_Add(capnproto-proj
+                    PREFIX ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
+                    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
+                    BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto
+                    CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/configure
+                                      --prefix=${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local
+                                      --enable-shared=no --silent
+                                      CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER}
+                    BUILD_COMMAND make -j${NPROC} check)
+set(CAPNP_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/include
+    CACHE FILEPATH "Path to capnproto include directory" FORCE)
+set(CAPNP_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/lib/libcapnp.a
+    CACHE FILEPATH "Path to capnproto library" FORCE)
+set(KJ_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/lib/libkj.a
+    CACHE FILEPATH "Path to kj library (used by capnproto)" FORCE)
+import_library(capnp STATIC ${CAPNP_LIBRARY} capnproto-proj)
+import_library(kj STATIC ${KJ_LIBRARY} capnproto-proj)
+set(CAPNP_EXE ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/bin/capnp
+    CACHE FILEPATH "Path to capnproto executable" FORCE)
+set(CAPNP_CXX_EXE ${CMAKE_CURRENT_SOURCE_DIR}/capnproto/local/bin/capnpc-c++
+    CACHE FILEPATH "Path to capnproto c++ plugin executable" FORCE)
+mark_as_advanced(CAPNP_INCLUDE_DIR CAPNP_LIBRARY KJ_LIBRARY CAPNP_EXE CAPNP_CXX_EXE)
+
+# Setup librdkafka.
+import_external_library(librdkafka STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/include/librdkafka
+  CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON
+             -DRDKAFKA_BUILD_EXAMPLES=OFF
+             -DRDKAFKA_BUILD_TESTS=OFF
+             -DCMAKE_INSTALL_LIBDIR=lib
+             -DWITH_SSL=ON
+             # If we want SASL, we need to install it on build machines
+             -DWITH_SASL=OFF)
+
+import_library(librdkafka++ STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka++.a
+  librdkafka-proj)
+
+# Setup libbcrypt
+import_external_library(libbcrypt STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt/bcrypt.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt
+  CONFIGURE_COMMAND sed s/-Wcast-align// -i ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt/crypt_blowfish/Makefile
+  BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt
+                     CC=${CMAKE_C_COMPILER}
+                     CXX=${CMAKE_CXX_COMPILER}
+  INSTALL_COMMAND true)

libs/setup.sh

@@ -91,6 +91,11 @@ clone git://deps.memgraph.io/glog.git glog $glog_tag
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
 clone git://deps.memgraph.io/gflags.git gflags $gflags_tag
 
+# libbcrypt
+# git clone https://github.com/rg3/libbcrypt
+libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
+clone git://deps.memgraph.io/libbcrypt.git libbcrypt $libbcrypt_tag
+
 # neo4j
 wget -nv http://deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
 tar -xzf neo4j.tar.gz
@@ -111,3 +116,24 @@ cd ..
 # git clone https://github.com/r-lyeh/ltalloc.git
 ltalloc_tag="43b51c14857111f993f277c46151fdfac91525a2" # Nov 16, 2017
 clone git://deps.memgraph.io/ltalloc.git ltalloc $ltalloc_tag
+
+bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
+clone git://deps.memgraph.io/bzip2.git bzip2 $bzip2_tag
+
+zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
+clone git://deps.memgraph.io/zlib.git zlib $zlib_tag
+
+rocksdb_tag="dbd8fa09b823826dd2a30bc119dad7a6fa9a4c6d" # v5.11.3 Mar 12, 2018
+clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag
+
+# Cap'n Proto serialization (and RPC) lib
+wget -nv http://deps.memgraph.io/capnproto-c++-0.6.1.tar.gz -O capnproto.tar.gz
+tar -xzf capnproto.tar.gz
+rm -rf capnproto
+mv capnproto-c++-0.6.1 capnproto
+rm capnproto.tar.gz
+
+# kafka
+kafka_tag="c319b4e987d0bc4fe4f01cf91419d90b62061655" # Mar 8, 2018
+# git clone https://github.com/edenhill/librdkafka.git
+clone git://deps.memgraph.io/librdkafka.git librdkafka $kafka_tag

poc/bloom_map_concurrent.cpp

@@ -1,184 +0,0 @@
-#include <random>
-#include <thread>
-
-#include <benchmark/benchmark_api.h>
-#include <gflags/gflags.h>
-#include <glog/logging.h>
-
-#include "bloom_filter.hpp"
-#include "concurrent_bloom_map.hpp"
-#include "utils/hashing/fnv64.hpp"
-#include "utils/random/random_generator.hpp"
-
-/*
-  ConcurrentMap Benchmark Test:
-    - tests time of Insertion, Contain and Delete operations
-
-    - benchmarking time per operation
-
-    - test run ConcurrentMap with the following keys and values:
-        - <int,int>
-        - <int, string>
-        - <string, int>
-        - <string, string>
-*/
-
-using utils::random::NumberGenerator;
-using utils::random::PairGenerator;
-using utils::random::StringGenerator;
-using StringHashFunction = std::function<uint64_t(const std::string &)>;
-
-using IntegerGenerator = NumberGenerator<std::uniform_int_distribution<int>,
-                                         std::default_random_engine, int>;
-
-DEFINE_int32(start, 0, "Range start");
-DEFINE_int32(end, 1000000000, "Range end");
-DEFINE_int32(threads, 1, "Number of threads");
-DEFINE_int32(string_length, 128, "String length");
-
-// Global arguments
-int MAX_ELEMENTS = 1 << 18, MULTIPLIER = 2;
-int THREADS, RANGE_START, RANGE_END, STRING_LENGTH;
-
-/*
-  ConcurrentMap Insertion Benchmark Test
-*/
-template <class K, class V, class F>
-static void InsertValue(benchmark::State &state,
-                        ConcurrentBloomMap<K, V, F> *map,
-                        const std::vector<std::pair<K, V>> &elements) {
-  while (state.KeepRunning()) {
-    for (int start = 0; start < state.range(0); start++) {
-      map->insert(elements[start].first, elements[start].second);
-    }
-  }
-  state.SetComplexityN(state.range(0));
-}
-
-/*
-  ConcurrentMap Contains Benchmark Test
-*/
-template <class K, class V, class F>
-static void ContainsValue(benchmark::State &state,
-                          ConcurrentBloomMap<K, V, F> *map,
-                          const std::vector<std::pair<K, V>> elements) {
-  while (state.KeepRunning()) {
-    for (int start = 0; start < state.range(0); start++) {
-      map->contains(elements[start].first);
-    }
-  }
-  state.SetComplexityN(state.range(0));
-}
-
-auto BM_InsertValue = [](benchmark::State &state, auto *map, auto &elements) {
-  InsertValue(state, map, elements);
-};
-
-auto BM_ContainsValue = [](benchmark::State &state, auto *map, auto elements) {
-  ContainsValue(state, map, elements);
-};
-
-/*
-  Commandline Argument Parsing
-
-  Arguments:
-   * Integer Range Minimum
-      -start number
-
-   *  Integer Range Maximum
-      - end number
-
-   * Number of threads
-      - threads number
-
-   * Random String lenght
-      -string-length number
-*/
-void parse_arguments() {
-  RANGE_START = FLAGS_start;
-  RANGE_END = FLAGS_end;
-
-  THREADS = std::min(FLAGS_threads,
-                     static_cast<int>(std::thread::hardware_concurrency()));
-
-  STRING_LENGTH = FLAGS_string_length;
-}
-
-int main(int argc, char **argv) {
-  benchmark::Initialize(&argc, argv);
-  parse_arguments();
-  google::InitGoogleLogging(argv[0]);
-
-  StringGenerator sg(STRING_LENGTH);
-  IntegerGenerator ig(RANGE_START, RANGE_END);
-
-  /*
-    Creates RandomGenerators, ConcurentMaps and Random Element Vectors for the
-    following use cases:
-
-      Map elements contain keys and value for:
-        <int, int>,
-        <int, string>
-        <string, int>
-        <string, string>
-  */
-
-  // random generators for tests
-  PairGenerator<IntegerGenerator, IntegerGenerator> piig(&ig, &ig);
-  PairGenerator<StringGenerator, StringGenerator> pssg(&sg, &sg);
-  PairGenerator<StringGenerator, IntegerGenerator> psig(&sg, &ig);
-  PairGenerator<IntegerGenerator, StringGenerator> pisg(&ig, &sg);
-
-  StringHashFunction hash1 = fnv64;
-  StringHashFunction hash2 = fnv1a64;
-  std::vector<StringHashFunction> funcs = {hash1, hash2};
-
-  BloomFilter<std::string, 128> bloom_filter_(funcs);
-
-  // maps used for testing
-  // ConcurrentBloomMap<int, int> ii_map;
-  // ConcurrentBloomMap<int, std::string> is_map;
-  using Filter = BloomFilter<std::string, 128>;
-  ConcurrentBloomMap<std::string, int, Filter> si_map(bloom_filter_);
-  ConcurrentBloomMap<std::string, std::string, Filter> ss_map(bloom_filter_);
-
-  // random elements for testing
-  // auto ii_elems = utils::random::generate_vector(piig, MAX_ELEMENTS);
-  // auto is_elems = utils::random::generate_vector(pisg, MAX_ELEMENTS);
-  auto si_elems = utils::random::generate_vector(psig, MAX_ELEMENTS);
-  auto ss_elems = utils::random::generate_vector(pssg, MAX_ELEMENTS);
-
-  /* insertion Tests */
-  benchmark::RegisterBenchmark("InsertValue[String, Int]", BM_InsertValue,
-                               &si_map, si_elems)
-      ->RangeMultiplier(MULTIPLIER)
-      ->Range(1, MAX_ELEMENTS)
-      ->Complexity(benchmark::oN)
-      ->Threads(THREADS);
-
-  benchmark::RegisterBenchmark("InsertValue[String, String]", BM_InsertValue,
-                               &ss_map, ss_elems)
-      ->RangeMultiplier(MULTIPLIER)
-      ->Range(1, MAX_ELEMENTS)
-      ->Complexity(benchmark::oN)
-      ->Threads(THREADS);
-
-  // Contains Benchmark Tests
-  benchmark::RegisterBenchmark("ContainsValue[String, Int]", BM_ContainsValue,
-                               &si_map, si_elems)
-      ->RangeMultiplier(MULTIPLIER)
-      ->Range(1, MAX_ELEMENTS)
-      ->Complexity(benchmark::oN)
-      ->Threads(THREADS);
-
-  benchmark::RegisterBenchmark("ContainsValue[String, String]",
-                               BM_ContainsValue, &ss_map, ss_elems)
-      ->RangeMultiplier(MULTIPLIER)
-      ->Range(1, MAX_ELEMENTS)
-      ->Complexity(benchmark::oN)
-      ->Threads(THREADS);
-
-  benchmark::RunSpecifiedBenchmarks();
-
-  return 0;
-}

poc/concurrent_bloom_map.hpp

@@ -1,35 +0,0 @@
-#pragma once
-
-#include "data_structures/concurrent/common.hpp"
-#include "data_structures/concurrent/concurrent_map.hpp"
-#include "data_structures/concurrent/skiplist.hpp"
-
-using std::pair;
-
-template <class Key, class Value, class BloomFilter>
-class ConcurrentBloomMap {
-  using item_t = Item<Key, Value>;
-  using list_it = typename SkipList<item_t>::Iterator;
-
- private:
-  ConcurrentMap<Key, Value> map_;
-  BloomFilter filter_;
-
- public:
-  ConcurrentBloomMap(BloomFilter filter) : filter_(filter) {}
-
-  std::pair<list_it, bool> insert(const Key &key, const Value &data) {
-    filter_.insert(key);
-
-    auto accessor = std::move(map_.access());
-
-    return accessor.insert(key, data);
-  }
-
-  bool contains(const Key &key) {
-    if (!filter_.contains(key)) return false;
-
-    auto accessor = map_.access();
-    return accessor.contains(key);
-  }
-};

poc/rh_hashmap.hpp

@@ -64,7 +64,7 @@ class RhBase {
 
   // Base for all iterators. It can start from any point in map.
   template <class It>
-  class IteratorBase : public Crtp<It> {
+  class IteratorBase : public utils::Crtp<It> {
    protected:
     IteratorBase() : map(nullptr) { advanced = index = ~((size_t)0); }
     IteratorBase(const RhBase *map) {

release/arch-pkg/memgraph.install

@@ -1,11 +1,9 @@
 post_install() {
   # Add the 'memgraph' user and group and set permissions on
   # 'var/*/memgraph' directories.
-  useradd --system memgraph
-  res=$?
-  if [[ "$res" != "0" && "$res" != "9" ]]; then
-    exit 1
-  fi
+  getent group memgraph >/dev/null || groupadd -r memgraph || exit 1
+  getent passwd memgraph >/dev/null || \
+    useradd -r -g memgraph -d /var/lib/memgraph memgraph || exit 1
   chown memgraph:memgraph /var/lib/memgraph || exit 1
   chmod 750 /var/lib/memgraph || exit 1
   chown memgraph:adm /var/log/memgraph || exit 1

release/debian/postinst

@@ -21,14 +21,33 @@ case "$1" in
     configure)
         # Add the 'memgraph' user and group and set permissions on
         # 'var/*/memgraph' directories.
-        adduser --quiet --system --group --no-create-home --shell /bin/bash memgraph || exit 1
+        adduser --quiet --system --group --home /var/lib/memgraph --no-create-home --shell /bin/bash memgraph || exit 1
         echo "Don't forget to switch to the 'memgraph' user to use Memgraph" || exit 1
         chown memgraph:memgraph /var/lib/memgraph || exit 1
         chmod 750 /var/lib/memgraph || exit 1
         chown memgraph:adm /var/log/memgraph || exit 1
         chmod 750 /var/log/memgraph || exit 1
-        # Make examples directory immutable (optional)
-        chattr +i -R /usr/share/memgraph/examples || true
+
+        # Create telemetry directory in examples
+        for i in /usr/share/memgraph/examples/*; do
+            # The telemetry directory may already exist from some prior installation
+            if [ ! -d $i/telemetry ]; then
+                mkdir $i/telemetry || exit 1
+            fi
+            chown -R memgraph:memgraph $i/telemetry || exit 1
+            # Make snapshots directory immutable (optional)
+            chattr +i -R $i/snapshots || true
+        done
+
+        # Generate SSL certificates
+        if [ ! -d /etc/memgraph/ssl ]; then
+            mkdir /etc/memgraph/ssl || exit 1
+            openssl req -x509 -newkey rsa:4096 -days 3650 -nodes \
+                -keyout /etc/memgraph/ssl/key.pem -out /etc/memgraph/ssl/cert.pem \
+                -subj "/C=GB/ST=London/L=London/O=Memgraph Ltd./CN=Memgraph DB" || exit 1
+            chown memgraph:memgraph /etc/memgraph/ssl/* || exit 1
+            chmod 400 /etc/memgraph/ssl/* || exit 1
+        fi
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

release/debian/postrm

@@ -26,6 +26,16 @@ case "$1" in
         for var_file in $var_files; do
             rm -rf $var_file
         done
+        # Remove generated SSL certificates
+        if [ -d /etc/memgraph/ssl ]; then
+            rm -rf /etc/memgraph/ssl
+        fi
+        # Remove stored telemetry data
+        for i in /usr/share/memgraph/examples/*; do
+            if [ -d $i/telemetry ]; then
+                rm -rf $i/telemetry
+            fi
+        done
         # Don't remove the 'memgraph' user, since we cannot be sure whether it
         # existed before.
     ;;

release/docker/memgraph.dockerfile

@@ -2,6 +2,10 @@ FROM debian:stretch
 
 ARG deb_release
 
+RUN apt-get update && apt-get install -y \
+    openssl libcurl3 libssl1.1 \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
 COPY ${deb_release} /
 
 # Install memgraph package

release/examples/build_examples

@@ -1,21 +1,21 @@
 #!/bin/bash -e
 
 function print_help () {
-    echo "Usage: $0 MEMGRAPH_BUILD_DIR"
+    echo "Usage: $0 MEMGRAPH_EXE BOLT_CLIENT_EXE"
     echo "Build example snapshots using the compiled memgraph."
 }
 
-if [[ $# -ne 1 ]]; then
+if [[ $# -ne 2 ]]; then
   print_help
   exit 1
 fi
 
-memgraph_exe="$1/memgraph"
+memgraph_exe="$1"
 if [[ ! -x ${memgraph_exe} ]]; then
   echo "Expected memgraph executable at '${memgraph_exe}'"
   exit 1
 fi
-bolt_client_exe="$1/tests/manual/bolt_client"
+bolt_client_exe="$2"
 if [[ ! -x ${bolt_client_exe} ]]; then
   echo "Expected bolt_client executable at '${bolt_client_exe}'"
   exit 1

release/rpm/memgraph.spec.in

@@ -0,0 +1,118 @@
+# -*- rpm-spec -*-
+BuildRoot:      %_topdir/@CPACK_PACKAGE_FILE_NAME@@CPACK_RPM_PACKAGE_COMPONENT_PART_PATH@
+Summary:        @CPACK_RPM_PACKAGE_SUMMARY@
+Name:           @CPACK_RPM_PACKAGE_NAME@
+Version:        @CPACK_RPM_PACKAGE_VERSION@
+Release:        @CPACK_RPM_PACKAGE_RELEASE@
+License:        @CPACK_RPM_PACKAGE_LICENSE@
+# Group field is deprecated
+# Group:          @CPACK_RPM_PACKAGE_GROUP@
+Vendor:         @CPACK_RPM_PACKAGE_VENDOR@
+BuildRequires: systemd
+
+@TMP_RPM_URL@
+@TMP_RPM_REQUIRES@
+@TMP_RPM_REQUIRES_PRE@
+@TMP_RPM_REQUIRES_POST@
+@TMP_RPM_REQUIRES_PREUN@
+@TMP_RPM_REQUIRES_POSTUN@
+@TMP_RPM_PROVIDES@
+@TMP_RPM_OBSOLETES@
+@TMP_RPM_CONFLICTS@
+@TMP_RPM_AUTOPROV@
+@TMP_RPM_AUTOREQ@
+@TMP_RPM_AUTOREQPROV@
+@TMP_RPM_BUILDARCH@
+@TMP_RPM_PREFIXES@
+
+@TMP_RPM_DEBUGINFO@
+
+%define _rpmdir %_topdir/RPMS
+%define _srcrpmdir %_topdir/SRPMS
+@FILE_NAME_DEFINE@
+%define _unpackaged_files_terminate_build 0
+@TMP_RPM_SPEC_INSTALL_POST@
+@CPACK_RPM_SPEC_MORE_DEFINE@
+@CPACK_RPM_COMPRESSION_TYPE_TMP@
+
+%description
+@CPACK_RPM_PACKAGE_DESCRIPTION@
+
+# This is a shortcutted spec file generated by CMake RPM generator
+# we skip _install step because CPack does that for us.
+# We do only save CPack installed tree in _prepr
+# and then restore it in build.
+%prep
+# Put the systemd unit where it is expected on this system
+mkdir -p $RPM_BUILD_ROOT/%{_unitdir}
+mv $RPM_BUILD_ROOT/lib/systemd/system/memgraph.service $RPM_BUILD_ROOT/%{_unitdir}
+rm -rf $RPM_BUILD_ROOT/lib
+# Fix the incorrect directory permissions set by cpack (this is fixed in CMake 3.11)
+find $RPM_BUILD_ROOT -type d | xargs chmod 755
+# After setting up custom prep, continue with CMake's default
+mv $RPM_BUILD_ROOT %_topdir/tmpBBroot
+
+%install
+if [ -e $RPM_BUILD_ROOT ];
+then
+  rm -rf $RPM_BUILD_ROOT
+fi
+mv %_topdir/tmpBBroot $RPM_BUILD_ROOT
+
+@TMP_RPM_DEBUGINFO_INSTALL@
+
+%clean
+
+%post
+# memgraph user and group must be set in preinst
+chown memgraph:memgraph /var/lib/memgraph || exit 1
+chmod 750 /var/lib/memgraph || exit 1
+chown memgraph:adm /var/log/memgraph || exit 1
+chmod 750 /var/log/memgraph || exit 1
+# Make examples directory immutable (optional)
+chattr +i -R /usr/share/memgraph/examples || true
+
+# Generate SSL certificates
+if [ ! -d /etc/memgraph/ssl ]; then
+    mkdir /etc/memgraph/ssl || exit 1
+    openssl req -x509 -newkey rsa:4096 -days 3650 -nodes \
+        -keyout /etc/memgraph/ssl/key.pem -out /etc/memgraph/ssl/cert.pem \
+        -subj "/C=GB/ST=London/L=London/O=Memgraph Ltd./CN=Memgraph DB" || exit 1
+    chown memgraph:memgraph /etc/memgraph/ssl/* || exit 1
+    chmod 400 /etc/memgraph/ssl/* || exit 1
+fi
+@RPM_SYMLINK_POSTINSTALL@
+@CPACK_RPM_SPEC_POSTINSTALL@
+
+%postun
+@CPACK_RPM_SPEC_POSTUNINSTALL@
+
+%pre
+# Add the 'memgraph' user and group
+getent group memgraph >/dev/null || groupadd -r memgraph || exit 1
+getent passwd memgraph >/dev/null || \
+  useradd -r -g memgraph -d /var/lib/memgraph -s /bin/bash memgraph || exit 1
+echo "Don't forget to switch to the 'memgraph' user to use Memgraph" || exit 1
+@CPACK_RPM_SPEC_PREINSTALL@
+
+%preun
+# Remove optional immutability from examples directory to allow removal
+chattr -i -R /usr/share/memgraph/examples || true
+@CPACK_RPM_SPEC_PREUNINSTALL@
+
+%files
+%defattr(@TMP_DEFAULT_FILE_PERMISSIONS@,@TMP_DEFAULT_USER@,@TMP_DEFAULT_GROUP@,@TMP_DEFAULT_DIR_PERMISSIONS@)
+@CPACK_RPM_INSTALL_FILES@
+# Since we moved the memgraph.service file, declare it explicitly here.
+# NOTE: memgraph.service must not be marked as configuration file.
+%{_unitdir}/memgraph.service
+
+# Override CPACK_RPM_ABSOLUTE_INSTALL_FILES with our %config(noreplace), cpack
+# uses plain %config.
+%config(noreplace) "/etc/memgraph/memgraph.conf"
+%config(noreplace) "/etc/logrotate.d/memgraph"
+
+@CPACK_RPM_USER_INSTALL_FILES@
+
+%changelog
+@CPACK_RPM_SPEC_CHANGELOG@

release/rpm/postinst

@@ -1,10 +0,0 @@
-# Hackish way to get the sytemd unit at the right place
-mv /lib/systemd/system/memgraph.service %{_unitdir}
-# memgraph user and group must be set in preinst
-chown memgraph:memgraph /var/lib/memgraph || exit 1
-chmod 750 /var/lib/memgraph || exit 1
-chown memgraph:adm /var/log/memgraph || exit 1
-chmod 750 /var/log/memgraph || exit 1
-# Make examples directory immutable (optional)
-chattr +i -R /usr/share/memgraph/examples || true
-

release/rpm/preinst

@@ -1,6 +0,0 @@
-# Add the 'memgraph' user and group
-getent group memgraph >/dev/null || groupadd -r memgraph || exit 1
-getent passwd memgraph >/dev/null || \
-  useradd -r -g memgraph -d /var/lib/memgraph -s /bin/bash memgraph || exit 1
-echo "Don't forget to switch to the 'memgraph' user to use Memgraph" || exit 1
-

release/rpm/prerm

@@ -1,2 +0,0 @@
-# Remove optional immutability from examples directory to allow removal
-chattr -i -R /usr/share/memgraph/examples || true

src/CMakeLists.txt

@@ -1,96 +1,349 @@
 # CMake configuration for the main memgraph library and executable
 
-# all memgraph src files
-set(memgraph_src_files
-    communication/buffer.cpp
-    communication/bolt/v1/decoder/decoded_value.cpp
-    communication/rpc/buffer.cpp
-    communication/rpc/client.cpp
-    communication/rpc/protocol.cpp
-    communication/rpc/server.cpp
+# add memgraph sub libraries, ordered by dependency
+add_subdirectory(lisp)
+add_subdirectory(utils)
+add_subdirectory(requests)
+add_subdirectory(integrations)
+add_subdirectory(io)
+add_subdirectory(telemetry)
+add_subdirectory(communication)
+add_subdirectory(stats)
+add_subdirectory(auth)
+add_subdirectory(tensorflow)
+
+# ----------------------------------------------------------------------------
+# Memgraph Single Node
+# ----------------------------------------------------------------------------
+set(mg_single_node_sources
     data_structures/concurrent/skiplist_gc.cpp
-    database/config.cpp
-    database/counters.cpp
-    database/graph_db.cpp
-    database/graph_db_accessor.cpp
-    database/state_delta.cpp
-    distributed/coordination_master.cpp
-    distributed/coordination_worker.cpp
-    distributed/index_rpc_server.cpp
-    distributed/plan_consumer.cpp
-    distributed/plan_dispatcher.cpp
-    distributed/cache.cpp
-    distributed/data_manager.cpp
-    distributed/data_rpc_clients.cpp
-    distributed/data_rpc_server.cpp
-    distributed/produce_rpc_server.cpp
-    distributed/pull_rpc_clients.cpp
-    distributed/updates_rpc_clients.cpp
-    distributed/updates_rpc_server.cpp
-    durability/paths.cpp
-    durability/recovery.cpp
-    durability/snapshooter.cpp
-    durability/wal.cpp
-    io/network/addrinfo.cpp
-    io/network/endpoint.cpp
-    io/network/socket.cpp
+    database/single_node/config.cpp
+    database/single_node/graph_db.cpp
+    database/single_node/graph_db_accessor.cpp
+    durability/single_node/state_delta.cpp
+    durability/single_node/paths.cpp
+    durability/single_node/recovery.cpp
+    durability/single_node/snapshooter.cpp
+    durability/single_node/wal.cpp
+    glue/auth.cpp
+    glue/communication.cpp
     query/common.cpp
-    query/repl.cpp
     query/frontend/ast/ast.cpp
+    query/frontend/ast/pretty_print.cpp
     query/frontend/ast/cypher_main_visitor.cpp
+    query/frontend/semantic/required_privileges.cpp
     query/frontend/semantic/symbol_generator.cpp
     query/frontend/stripped.cpp
     query/interpret/awesome_memgraph_functions.cpp
     query/interpreter.cpp
-    query/plan/distributed.cpp
     query/plan/operator.cpp
     query/plan/preprocess.cpp
+    query/plan/pretty_print.cpp
     query/plan/rule_based_planner.cpp
     query/plan/variable_start_planner.cpp
+    query/repl.cpp
     query/typed_value.cpp
-    stats/metrics.cpp
-    stats/stats.cpp
-    storage/concurrent_id_mapper_master.cpp
-    storage/concurrent_id_mapper_worker.cpp
-    storage/edge_accessor.cpp
-    storage/locking/record_lock.cpp
-    storage/property_value.cpp
-    storage/record_accessor.cpp
-    storage/vertex_accessor.cpp
-    threading/sync/rwlock.cpp
-    threading/thread.cpp
-    transactions/engine_master.cpp
-    transactions/engine_single_node.cpp
-    transactions/engine_worker.cpp
-    utils/demangle.cpp
-    utils/file.cpp
-    utils/network.cpp
-    utils/watchdog.cpp
+    storage/single_node/edge_accessor.cpp
+    storage/common/locking/record_lock.cpp
+    storage/common/types/property_value.cpp
+    storage/common/types/property_value_store.cpp
+    storage/single_node/record_accessor.cpp
+    storage/single_node/vertex_accessor.cpp
+    transactions/single_node/engine.cpp
+    memgraph_init.cpp
 )
-# -----------------------------------------------------------------------------
 
-string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
+define_add_lcp(add_lcp_single_node mg_single_node_sources generated_lcp_single_node_files)
 
-# memgraph_lib depend on these libraries
-set(MEMGRAPH_ALL_LIBS stdc++fs Threads::Threads fmt cppitertools
-    antlr_opencypher_parser_lib dl glog gflags
-    ${Boost_IOSTREAMS_LIBRARY_RELEASE}
-    ${Boost_SERIALIZATION_LIBRARY_RELEASE})
+add_lcp_single_node(durability/single_node/state_delta.lcp)
+add_lcp_single_node(query/frontend/ast/ast.lcp)
+add_lcp_single_node(query/frontend/semantic/symbol.lcp)
+add_lcp_single_node(query/plan/operator.lcp)
+
+add_custom_target(generate_lcp_single_node DEPENDS ${generated_lcp_single_node_files})
+
+set(MG_SINGLE_NODE_LIBS stdc++fs Threads::Threads fmt cppitertools
+    antlr_opencypher_parser_lib dl glog gflags capnp kj
+    mg-utils mg-io mg-integrations-kafka mg-requests mg-communication mg-auth mg-stats)
 
 if (USE_LTALLOC)
-    list(APPEND MEMGRAPH_ALL_LIBS ltalloc)
-# TODO(mferencevic): Enable this when clang is updated on apollo.
-#    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto")
+  list(APPEND MG_SINGLE_NODE_LIBS ltalloc)
+  # TODO(mferencevic): Enable this when clang is updated on apollo.
+  # set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto")
 endif()
 
 if (READLINE_FOUND)
-    list(APPEND MEMGRAPH_ALL_LIBS readline)
+  list(APPEND MG_SINGLE_NODE_LIBS readline)
 endif()
 
+add_library(mg-single-node STATIC ${mg_single_node_sources})
+target_link_libraries(mg-single-node ${MG_SINGLE_NODE_LIBS})
+add_dependencies(mg-single-node generate_opencypher_parser)
+add_dependencies(mg-single-node generate_lcp_single_node)
+target_compile_definitions(mg-single-node PUBLIC MG_SINGLE_NODE)
+
+# ----------------------------------------------------------------------------
+# END Memgraph Single Node
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Memgraph Distributed
+# ----------------------------------------------------------------------------
+
+set(mg_distributed_sources
+    database/distributed/distributed_counters.cpp
+    database/distributed/distributed_graph_db.cpp
+    distributed/bfs_rpc_clients.cpp
+    distributed/bfs_subcursor.cpp
+    distributed/cluster_discovery_master.cpp
+    distributed/cluster_discovery_worker.cpp
+    distributed/coordination.cpp
+    distributed/coordination_master.cpp
+    distributed/coordination_worker.cpp
+    distributed/data_manager.cpp
+    distributed/data_rpc_clients.cpp
+    distributed/data_rpc_server.cpp
+    distributed/dgp/partitioner.cpp
+    distributed/dgp/vertex_migrator.cpp
+    distributed/durability_rpc_master.cpp
+    distributed/durability_rpc_worker.cpp
+    distributed/dynamic_worker.cpp
+    distributed/index_rpc_server.cpp
+    distributed/plan_consumer.cpp
+    distributed/plan_dispatcher.cpp
+    distributed/produce_rpc_server.cpp
+    distributed/pull_rpc_clients.cpp
+    distributed/updates_rpc_clients.cpp
+    distributed/updates_rpc_server.cpp
+    query/distributed_interpreter.cpp
+    query/plan/distributed.cpp
+    query/plan/distributed_ops.cpp
+    query/plan/distributed_pretty_print.cpp
+    storage/distributed/concurrent_id_mapper_master.cpp
+    storage/distributed/concurrent_id_mapper_worker.cpp
+    transactions/distributed/engine_master.cpp
+    transactions/distributed/engine_worker.cpp
+    data_structures/concurrent/skiplist_gc.cpp
+    database/distributed/config.cpp
+    database/distributed/graph_db_accessor.cpp
+    durability/distributed/state_delta.cpp
+    durability/distributed/paths.cpp
+    durability/distributed/recovery.cpp
+    durability/distributed/snapshooter.cpp
+    durability/distributed/wal.cpp
+    glue/auth.cpp
+    glue/communication.cpp
+    query/common.cpp
+    query/frontend/ast/ast.cpp
+    query/frontend/ast/pretty_print.cpp
+    query/frontend/ast/cypher_main_visitor.cpp
+    query/frontend/semantic/required_privileges.cpp
+    query/frontend/semantic/symbol_generator.cpp
+    query/frontend/stripped.cpp
+    query/interpret/awesome_memgraph_functions.cpp
+    query/interpreter.cpp
+    query/plan/operator.cpp
+    query/plan/preprocess.cpp
+    query/plan/pretty_print.cpp
+    query/plan/rule_based_planner.cpp
+    query/plan/variable_start_planner.cpp
+    query/repl.cpp
+    query/serialization.cpp
+    query/typed_value.cpp
+    storage/common/locking/record_lock.cpp
+    storage/common/types/property_value.cpp
+    storage/common/types/property_value_store.cpp
+    storage/distributed/edge_accessor.cpp
+    storage/distributed/record_accessor.cpp
+    storage/distributed/rpc/serialization.cpp
+    storage/distributed/vertex_accessor.cpp
+    memgraph_init.cpp
+    transactions/distributed/engine_single_node.cpp
+)
+
+# -----------------------------------------------------------------------------
+
+define_add_capnp(add_capnp_distributed mg_distributed_sources generated_capnp_files)
+
+define_add_lcp(add_lcp_distributed mg_distributed_sources generated_lcp_distributed_files)
+
+add_lcp_distributed(durability/distributed/state_delta.lcp)
+add_lcp_distributed(database/distributed/counters_rpc_messages.lcp CAPNP_SCHEMA @0x95a2c3ea3871e945)
+add_capnp_distributed(database/distributed/counters_rpc_messages.capnp)
+add_lcp_distributed(database/distributed/serialization.lcp CAPNP_SCHEMA @0xdea01657b3563887
+                    DEPENDS durability/distributed/state_delta.lcp)
+add_capnp_distributed(database/distributed/serialization.capnp)
+add_lcp_distributed(distributed/bfs_rpc_messages.lcp CAPNP_SCHEMA @0x8e508640b09b6d2a)
+add_capnp_distributed(distributed/bfs_rpc_messages.capnp)
+add_lcp_distributed(distributed/coordination_rpc_messages.lcp CAPNP_SCHEMA @0x93df0c4703cf98fb)
+add_capnp_distributed(distributed/coordination_rpc_messages.capnp)
+add_lcp_distributed(distributed/data_rpc_messages.lcp CAPNP_SCHEMA @0xc1c8a341ba37aaf5)
+add_capnp_distributed(distributed/data_rpc_messages.capnp)
+add_lcp_distributed(distributed/durability_rpc_messages.lcp CAPNP_SCHEMA @0xf5e53bc271e2163d)
+add_capnp_distributed(distributed/durability_rpc_messages.capnp)
+add_lcp_distributed(distributed/index_rpc_messages.lcp CAPNP_SCHEMA @0xa8aab46862945bd6)
+add_capnp_distributed(distributed/index_rpc_messages.capnp)
+add_lcp_distributed(distributed/plan_rpc_messages.lcp CAPNP_SCHEMA @0xfcbc48dc9f106d28)
+add_capnp_distributed(distributed/plan_rpc_messages.capnp)
+add_lcp_distributed(distributed/pull_produce_rpc_messages.lcp CAPNP_SCHEMA @0xa78a9254a73685bd
+                    DEPENDS transactions/distributed/serialization.lcp)
+add_capnp_distributed(distributed/pull_produce_rpc_messages.capnp)
+add_lcp_distributed(distributed/storage_gc_rpc_messages.lcp CAPNP_SCHEMA @0xd705663dfe36cf81)
+add_capnp_distributed(distributed/storage_gc_rpc_messages.capnp)
+add_lcp_distributed(distributed/token_sharing_rpc_messages.lcp CAPNP_SCHEMA @0x8f295db54ec4caec)
+add_capnp_distributed(distributed/token_sharing_rpc_messages.capnp)
+add_lcp_distributed(distributed/updates_rpc_messages.lcp CAPNP_SCHEMA @0x82d5f38d73c7b53a)
+add_capnp_distributed(distributed/updates_rpc_messages.capnp)
+add_lcp_distributed(distributed/dynamic_worker_rpc_messages.lcp CAPNP_SCHEMA @0x8c53f6c9a0c71b05)
+add_capnp_distributed(distributed/dynamic_worker_rpc_messages.capnp)
+
+# distributed_ops.lcp is leading the capnp code generation, so we don't need
+# to generate any capnp for operator.lcp
+add_lcp_distributed(query/frontend/ast/ast.lcp)
+add_lcp_distributed(query/frontend/ast/ast_serialization.lcp CAPNP_SCHEMA @0xb107d3d6b4b1600b
+        DEPENDS query/frontend/ast/ast.lcp)
+add_capnp_distributed(query/frontend/ast/ast_serialization.capnp)
+add_lcp_distributed(query/frontend/semantic/symbol.lcp)
+add_lcp_distributed(query/frontend/semantic/symbol_serialization.lcp CAPNP_SCHEMA @0x93c1dcee84e93b76
+        DEPENDS query/frontend/semantic/symbol.lcp)
+add_lcp_distributed(query/plan/operator.lcp)
+add_lcp_distributed(query/plan/distributed_ops.lcp CAPNP_SCHEMA @0xe5cae8d045d30c42
+        DEPENDS query/plan/operator.lcp)
+add_capnp_distributed(query/plan/distributed_ops.capnp)
+
+add_lcp_distributed(storage/distributed/rpc/concurrent_id_mapper_rpc_messages.lcp CAPNP_SCHEMA @0xa6068dae93d225dd)
+add_capnp_distributed(storage/distributed/rpc/concurrent_id_mapper_rpc_messages.capnp)
+add_lcp_distributed(transactions/distributed/engine_rpc_messages.lcp CAPNP_SCHEMA @0xde02b7c49180cad5
+        DEPENDS transactions/distributed/serialization.lcp)
+add_capnp_distributed(transactions/distributed/engine_rpc_messages.capnp)
+
+add_custom_target(generate_lcp_distributed DEPENDS ${generated_lcp_distributed_files})
+
+# Registering capnp must come after registering lcp files.
+
+add_capnp_distributed(communication/rpc/messages.capnp)
+add_capnp_distributed(durability/distributed/serialization.capnp)
+add_capnp_distributed(query/frontend/semantic/symbol_serialization.capnp)
+add_capnp_distributed(query/serialization.capnp)
+add_capnp_distributed(storage/distributed/rpc/serialization.capnp)
+
+add_custom_target(generate_capnp DEPENDS generate_lcp_distributed ${generated_capnp_files})
+
+set(MG_DISTRIBUTED_LIBS stdc++fs Threads::Threads fmt cppitertools
+    antlr_opencypher_parser_lib dl glog gflags capnp kj
+    mg-utils mg-io mg-integrations-kafka mg-requests mg-communication mg-auth mg-stats)
+
 # STATIC library used by memgraph executables
-add_library(memgraph_lib STATIC ${memgraph_src_files})
-target_link_libraries(memgraph_lib ${MEMGRAPH_ALL_LIBS})
-add_dependencies(memgraph_lib generate_opencypher_parser)
+add_library(mg-distributed STATIC ${mg_distributed_sources})
+target_link_libraries(mg-distributed ${MG_DISTRIBUTED_LIBS})
+add_dependencies(mg-distributed generate_opencypher_parser)
+add_dependencies(mg-distributed generate_lcp_distributed)
+add_dependencies(mg-distributed generate_capnp)
+target_compile_definitions(mg-distributed PUBLIC MG_DISTRIBUTED)
+
+# ----------------------------------------------------------------------------
+# END Memgraph Distributed
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Memgraph Single Node High Availability
+# ----------------------------------------------------------------------------
+set(mg_single_node_ha_sources
+    data_structures/concurrent/skiplist_gc.cpp
+    database/single_node_ha/config.cpp
+    database/single_node_ha/graph_db.cpp
+    database/single_node_ha/graph_db_accessor.cpp
+    durability/single_node_ha/state_delta.cpp
+    durability/single_node_ha/paths.cpp
+    durability/single_node_ha/snapshooter.cpp
+    glue/auth.cpp
+    glue/communication.cpp
+    raft/coordination.cpp
+    raft/raft_server.cpp
+    query/common.cpp
+    query/frontend/ast/ast.cpp
+    query/frontend/ast/cypher_main_visitor.cpp
+    query/frontend/semantic/required_privileges.cpp
+    query/frontend/semantic/symbol_generator.cpp
+    query/frontend/stripped.cpp
+    query/interpret/awesome_memgraph_functions.cpp
+    query/interpreter.cpp
+    query/plan/operator.cpp
+    query/plan/preprocess.cpp
+    query/plan/pretty_print.cpp
+    query/plan/rule_based_planner.cpp
+    query/plan/variable_start_planner.cpp
+    query/repl.cpp
+    query/typed_value.cpp
+    storage/common/types/property_value.cpp
+    storage/common/types/property_value_store.cpp
+    storage/common/locking/record_lock.cpp
+    storage/single_node_ha/edge_accessor.cpp
+    storage/single_node_ha/record_accessor.cpp
+    storage/single_node_ha/rpc/serialization.cpp
+    storage/single_node_ha/vertex_accessor.cpp
+    transactions/single_node_ha/engine.cpp
+    memgraph_init.cpp
+)
+
+define_add_capnp(add_capnp_single_node_ha mg_single_node_ha_sources generated_capnp_single_node_ha_files)
+
+define_add_lcp(add_lcp_single_node_ha mg_single_node_ha_sources generated_lcp_single_node_ha_files)
+
+add_lcp_single_node_ha(durability/single_node_ha/state_delta.lcp)
+add_lcp_single_node_ha(database/single_node_ha/serialization.lcp CAPNP_SCHEMA @0xd0f4a502575fb6f7
+        DEPENDS durability/single_node_ha/state_delta.lcp)
+add_capnp_single_node_ha(database/single_node_ha/serialization.capnp)
+add_lcp_single_node_ha(query/frontend/ast/ast.lcp)
+add_lcp_single_node_ha(query/frontend/semantic/symbol.lcp)
+add_lcp_single_node_ha(query/plan/operator.lcp)
+add_lcp_single_node_ha(raft/raft_rpc_messages.lcp CAPNP_SCHEMA @0xa6c29b4287233b66)
+add_capnp_single_node_ha(raft/raft_rpc_messages.capnp)
+add_lcp_single_node_ha(raft/log_entry.lcp CAPNP_SCHEMA @0x96c07fe13850c22a)
+add_capnp_single_node_ha(raft/log_entry.capnp)
+
+add_custom_target(generate_lcp_single_node_ha DEPENDS ${generated_lcp_single_node_ha_files})
+
+# Registering capnp must come after registering lcp files.
+
+add_capnp_single_node_ha(storage/single_node_ha/rpc/serialization.capnp)
+
+add_custom_target(generate_capnp_single_node_ha DEPENDS generate_lcp_single_node_ha ${generated_capnp_single_node_ha_files})
+
+set(MG_SINGLE_NODE_HA_LIBS stdc++fs Threads::Threads fmt cppitertools
+    antlr_opencypher_parser_lib dl glog gflags capnp kj
+    mg-utils mg-io mg-integrations-kafka mg-requests mg-communication mg-auth mg-stats)
+
+if (USE_LTALLOC)
+  list(APPEND MG_SINGLE_NODE_HA_LIBS ltalloc)
+  # TODO(mferencevic): Enable this when clang is updated on apollo.
+  # set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto")
+endif()
+
+if (READLINE_FOUND)
+  list(APPEND MG_SINGLE_NODE_HA_LIBS readline)
+endif()
+
+add_library(mg-single-node-ha STATIC ${mg_single_node_ha_sources})
+target_link_libraries(mg-single-node-ha ${MG_SINGLE_NODE_HA_LIBS})
+add_dependencies(mg-single-node-ha generate_opencypher_parser)
+add_dependencies(mg-single-node-ha generate_lcp_single_node_ha)
+add_dependencies(mg-single-node-ha generate_capnp_single_node_ha)
+target_compile_definitions(mg-single-node-ha PUBLIC MG_SINGLE_NODE_HA)
+
+# ----------------------------------------------------------------------------
+# END Memgraph Single Node High Availability
+# ----------------------------------------------------------------------------
+
+string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
+
+# STATIC library used to store key-value pairs
+add_library(kvstore_lib STATIC storage/common/kvstore/kvstore.cpp)
+target_link_libraries(kvstore_lib stdc++fs mg-utils rocksdb bzip2 zlib glog gflags)
+
+# STATIC library for dummy key-value storage
+add_library(kvstore_dummy_lib STATIC storage/common/kvstore/kvstore_dummy.cpp)
+target_link_libraries(kvstore_dummy_lib mg-utils)
 
 # Generate a version.hpp file
 set(VERSION_STRING ${memgraph_VERSION})
@@ -98,8 +351,8 @@ configure_file(version.hpp.in version.hpp @ONLY)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
 # memgraph main executable
-add_executable(memgraph memgraph_bolt.cpp)
-target_link_libraries(memgraph memgraph_lib)
+add_executable(memgraph memgraph.cpp)
+target_link_libraries(memgraph mg-single-node kvstore_lib telemetry_lib)
 set_target_properties(memgraph PROPERTIES
                       # Set the executable output name to include version information.
                       OUTPUT_NAME "memgraph-${memgraph_VERSION}-${COMMIT_HASH}_${CMAKE_BUILD_TYPE}"
@@ -121,6 +374,11 @@ endif()
 # Everything here is under "memgraph" install component.
 set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph")
 
+# TODO: Default directory permissions to 755
+# NOTE: This is added in CMake 3.11, so enable it then
+#set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
+#    OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ)
+
 # Install and rename executable to just 'memgraph' Since we have to rename,
 # we cannot use the recommended `install(TARGETS ...)`.
 install(PROGRAMS $<TARGET_FILE:memgraph>
@@ -143,6 +401,39 @@ install(FILES ${CMAKE_SOURCE_DIR}/release/memgraph.service
 
 # Install examples
 set(examples ${CMAKE_SOURCE_DIR}/release/examples)
-install(CODE "execute_process(COMMAND ${examples}/build_examples ${CMAKE_BINARY_DIR}
-                              WORKING_DIRECTORY ${examples})")
+install(
+  CODE
+  "execute_process(COMMAND ${examples}/build_examples
+                   ${CMAKE_BINARY_DIR}/memgraph
+                   ${CMAKE_BINARY_DIR}/tests/manual/bolt_client
+                   WORKING_DIRECTORY ${examples})")
 install(DIRECTORY ${examples}/build/ DESTINATION share/memgraph/examples)
+
+
+# memgraph distributed main executable
+add_executable(memgraph_distributed memgraph_distributed.cpp)
+target_link_libraries(memgraph_distributed mg-distributed kvstore_lib telemetry_lib)
+set_target_properties(memgraph_distributed PROPERTIES
+                      # Set the executable output name to include version information.
+                      OUTPUT_NAME "memgraph_distributed-${memgraph_VERSION}-${COMMIT_HASH}_${CMAKE_BUILD_TYPE}"
+                      # Output the executable in main binary dir.
+                      RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+# Create symlink to the built executable.
+add_custom_command(TARGET memgraph_distributed POST_BUILD
+                   COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph_distributed> ${CMAKE_BINARY_DIR}/memgraph_distributed
+                   BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph_distributed
+                   COMMENT Creating symlink to memgraph distributed executable)
+
+# memgraph single node high availability executable
+add_executable(memgraph_ha memgraph_ha.cpp)
+target_link_libraries(memgraph_ha mg-single-node-ha kvstore_lib telemetry_lib)
+set_target_properties(memgraph_ha PROPERTIES
+                      # Set the executable output name to include version information.
+                      OUTPUT_NAME "memgraph_ha-${memgraph_VERSION}-${COMMIT_HASH}_${CMAKE_BUILD_TYPE}"
+                      # Output the executable in main binary dir.
+                      RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+# Create symlink to the built executable.
+add_custom_command(TARGET memgraph_ha POST_BUILD
+                   COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph_ha> ${CMAKE_BINARY_DIR}/memgraph_ha
+                   BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph_ha
+                   COMMENT Creating symlink to memgraph single node high availability executable)

src/auth/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(auth_src_files
+    auth.cpp
+    crypto.cpp
+    models.cpp)
+
+add_library(mg-auth STATIC ${auth_src_files})
+target_link_libraries(mg-auth json libbcrypt)
+target_link_libraries(mg-auth mg-utils)

src/auth/auth.cpp

@@ -0,0 +1,197 @@
+#include "auth/auth.hpp"
+
+#include "auth/exceptions.hpp"
+
+namespace auth {
+
+const std::string kUserPrefix = "user:";
+const std::string kRolePrefix = "role:";
+const std::string kLinkPrefix = "link:";
+
+/**
+ * All data stored in the `Auth` storage is stored in an underlying
+ * `storage::KVStore`. Because we are using a key-value store to store the data,
+ * the data has to be encoded. The encoding used is as follows:
+ *
+ * User: key="user:<username>", value="<json_encoded_members_of_user>"
+ * Role: key="role:<rolename>", value="<json_endoded_members_of_role>"
+ *
+ * The User->Role relationship isn't stored in the `User` encoded data because
+ * we want to be able to delete/modify a Role and have it automatically be
+ * removed/modified in all linked users. Because of that we store the links to
+ * the role as a foreign-key like mapping in the KVStore. It is saved as
+ * follows:
+ *
+ * key="link:<username>", value="<rolename>"
+ */
+
+Auth::Auth(const std::string &storage_directory)
+    : storage_(storage_directory) {}
+
+std::experimental::optional<User> Auth::Authenticate(
+    const std::string &username, const std::string &password) {
+  auto user = GetUser(username);
+  if (!user) return std::experimental::nullopt;
+  if (!user->CheckPassword(password)) return std::experimental::nullopt;
+  return user;
+}
+
+std::experimental::optional<User> Auth::GetUser(const std::string &username) {
+  auto existing_user = storage_.Get(kUserPrefix + username);
+  if (!existing_user) return std::experimental::nullopt;
+
+  nlohmann::json data;
+  try {
+    data = nlohmann::json::parse(*existing_user);
+  } catch (const nlohmann::json::parse_error &e) {
+    throw AuthException("Couldn't load user data!");
+  }
+
+  auto user = User::Deserialize(data);
+  auto link = storage_.Get(kLinkPrefix + username);
+
+  if (link) {
+    auto role = GetRole(*link);
+    if (role) {
+      user.SetRole(*role);
+    }
+  }
+  return user;
+}
+
+void Auth::SaveUser(const User &user) {
+  bool success = false;
+  if (user.role()) {
+    success = storage_.PutMultiple(
+        {{kUserPrefix + user.username(), user.Serialize().dump()},
+         {kLinkPrefix + user.username(), user.role()->rolename()}});
+  } else {
+    success = storage_.PutAndDeleteMultiple(
+        {{kUserPrefix + user.username(), user.Serialize().dump()}},
+        {kLinkPrefix + user.username()});
+  }
+  if (!success) {
+    throw AuthException("Couldn't save user '{}'!", user.username());
+  }
+}
+
+std::experimental::optional<User> Auth::AddUser(
+    const std::string &username,
+    const std::experimental::optional<std::string> &password) {
+  auto existing_user = GetUser(username);
+  if (existing_user) return std::experimental::nullopt;
+  auto existing_role = GetRole(username);
+  if (existing_role) return std::experimental::nullopt;
+  auto new_user = User(username);
+  new_user.UpdatePassword(password);
+  SaveUser(new_user);
+  return new_user;
+}
+
+bool Auth::RemoveUser(const std::string &username) {
+  if (!storage_.Get(kUserPrefix + username)) return false;
+  std::vector<std::string> keys(
+      {kLinkPrefix + username, kUserPrefix + username});
+  if (!storage_.DeleteMultiple(keys)) {
+    throw AuthException("Couldn't remove user '{}'!", username);
+  }
+  return true;
+}
+
+std::vector<auth::User> Auth::AllUsers() {
+  std::vector<auth::User> ret;
+  for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix);
+       ++it) {
+    auto user = GetUser(it->first.substr(kUserPrefix.size()));
+    if (user) {
+      ret.push_back(*user);
+    }
+  }
+  return ret;
+}
+
+bool Auth::HasUsers() {
+  return storage_.begin(kUserPrefix) != storage_.end(kUserPrefix);
+}
+
+std::experimental::optional<Role> Auth::GetRole(const std::string &rolename) {
+  auto existing_role = storage_.Get(kRolePrefix + rolename);
+  if (!existing_role) return std::experimental::nullopt;
+
+  nlohmann::json data;
+  try {
+    data = nlohmann::json::parse(*existing_role);
+  } catch (const nlohmann::json::parse_error &e) {
+    throw AuthException("Couldn't load role data!");
+  }
+
+  return Role::Deserialize(data);
+}
+
+void Auth::SaveRole(const Role &role) {
+  if (!storage_.Put(kRolePrefix + role.rolename(), role.Serialize().dump())) {
+    throw AuthException("Couldn't save role '{}'!", role.rolename());
+  }
+}
+
+std::experimental::optional<Role> Auth::AddRole(const std::string &rolename) {
+  auto existing_role = GetRole(rolename);
+  if (existing_role) return std::experimental::nullopt;
+  auto existing_user = GetUser(rolename);
+  if (existing_user) return std::experimental::nullopt;
+  auto new_role = Role(rolename);
+  SaveRole(new_role);
+  return new_role;
+}
+
+bool Auth::RemoveRole(const std::string &rolename) {
+  if (!storage_.Get(kRolePrefix + rolename)) return false;
+  std::vector<std::string> keys;
+  for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix);
+       ++it) {
+    if (it->second == rolename) {
+      keys.push_back(it->first);
+    }
+  }
+  keys.push_back(kRolePrefix + rolename);
+  if (!storage_.DeleteMultiple(keys)) {
+    throw AuthException("Couldn't remove role '{}'!", rolename);
+  }
+  return true;
+}
+
+std::vector<auth::Role> Auth::AllRoles() {
+  std::vector<auth::Role> ret;
+  for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix);
+       ++it) {
+    auto rolename = it->first.substr(kRolePrefix.size());
+    auto role = GetRole(rolename);
+    if (role) {
+      ret.push_back(*role);
+    } else {
+      throw AuthException("Couldn't load role '{}'!", rolename);
+    }
+  }
+  return ret;
+}
+
+std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename) {
+  std::vector<auth::User> ret;
+  for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix);
+       ++it) {
+    auto username = it->first.substr(kLinkPrefix.size());
+    if (it->second == rolename) {
+      auto user = GetUser(username);
+      if (user) {
+        ret.push_back(*user);
+      } else {
+        throw AuthException("Couldn't load user '{}'!", username);
+      }
+    }
+  }
+  return ret;
+}
+
+std::mutex &Auth::WithLock() { return lock_; }
+
+}  // namespace auth

src/auth/auth.hpp

@@ -0,0 +1,152 @@
+#pragma once
+
+#include <experimental/optional>
+#include <mutex>
+#include <vector>
+
+#include "auth/exceptions.hpp"
+#include "auth/models.hpp"
+#include "storage/common/kvstore/kvstore.hpp"
+
+namespace auth {
+
+/**
+ * This class serves as the main Authentication/Authorization storage.
+ * It provides functions for managing Users, Roles and Permissions.
+ * NOTE: The functions in this class aren't thread safe. Use the `WithLock` lock
+ * if you want to have safe modifications of the storage.
+ */
+class Auth final {
+ public:
+  Auth(const std::string &storage_directory);
+
+  /**
+   * Authenticates a user using his username and password.
+   *
+   * @param username
+   * @param password
+   *
+   * @return a user when the username and password match, nullopt otherwise
+   */
+  std::experimental::optional<User> Authenticate(const std::string &username,
+                                                 const std::string &password);
+
+  /**
+   * Gets a user from the storage.
+   *
+   * @param username
+   *
+   * @return a user when the user exists, nullopt otherwise
+   */
+  std::experimental::optional<User> GetUser(const std::string &username);
+
+  /**
+   * Saves a user object to the storage.
+   *
+   * @param user
+   */
+  void SaveUser(const User &user);
+
+  /**
+   * Creates a user if the user doesn't exist.
+   *
+   * @param username
+   * @param password
+   *
+   * @return a user when the user is created, nullopt if the user exists
+   */
+  std::experimental::optional<User> AddUser(
+      const std::string &username,
+      const std::experimental::optional<std::string> &password =
+          std::experimental::nullopt);
+
+  /**
+   * Removes a user from the storage.
+   *
+   * @param username
+   *
+   * @return `true` if the user existed and was removed, `false` if the user
+   *         doesn't exist
+   */
+  bool RemoveUser(const std::string &username);
+
+  /**
+   * Gets all users from the storage.
+   *
+   * @return a list of users
+   */
+  std::vector<User> AllUsers();
+
+  /**
+   * Returns whether there are users in the storage.
+   *
+   * @return `true` if the storage contains any users, `false` otherwise
+   */
+  bool HasUsers();
+
+  /**
+   * Gets a role from the storage.
+   *
+   * @param rolename
+   *
+   * @return a role when the role exists, nullopt otherwise
+   */
+  std::experimental::optional<Role> GetRole(const std::string &rolename);
+
+  /**
+   * Saves a role object to the storage.
+   *
+   * @param role
+   */
+  void SaveRole(const Role &role);
+
+  /**
+   * Creates a role if the role doesn't exist.
+   *
+   * @param rolename
+   *
+   * @return a role when the role is created, nullopt if the role exists
+   */
+  std::experimental::optional<Role> AddRole(const std::string &rolename);
+
+  /**
+   * Removes a role from the storage.
+   *
+   * @param rolename
+   *
+   * @return `true` if the role existed and was removed, `false` if the role
+   *         doesn't exist
+   */
+  bool RemoveRole(const std::string &rolename);
+
+  /**
+   * Gets all roles from the storage.
+   *
+   * @return a list of roles
+   */
+  std::vector<Role> AllRoles();
+
+  /**
+   * Gets all users for a role from the storage.
+   *
+   * @param rolename
+   *
+   * @return a list of roles
+   */
+  std::vector<User> AllUsersForRole(const std::string &rolename);
+
+  /**
+   * Returns a reference to the lock that should be used for all operations that
+   * require more than one interaction with this class.
+   */
+  std::mutex &WithLock();
+
+ private:
+  storage::KVStore storage_;
+  // Even though the `storage::KVStore` class is guaranteed to be thread-safe we
+  // use a mutex to lock all operations on the `User` and `Role` storage because
+  // some operations on the users and/or roles may require more than one
+  // operation on the storage.
+  std::mutex lock_;
+};
+}  // namespace auth

src/auth/crypto.cpp

@@ -0,0 +1,35 @@
+#include "auth/crypto.hpp"
+
+#include <libbcrypt/bcrypt.h>
+
+#include "auth/exceptions.hpp"
+
+namespace auth {
+
+const std::string EncryptPassword(const std::string &password) {
+  char salt[BCRYPT_HASHSIZE];
+  char hash[BCRYPT_HASHSIZE];
+
+  // We use `-1` as the workfactor for `bcrypt_gensalt` to let it fall back to
+  // its default value of `12`. Increasing the workfactor increases the time
+  // needed to generate the salt.
+  if (bcrypt_gensalt(-1, salt) != 0) {
+    throw AuthException("Couldn't generate hashing salt!");
+  }
+
+  if (bcrypt_hashpw(password.c_str(), salt, hash) != 0) {
+    throw AuthException("Couldn't hash password!");
+  }
+
+  return std::string(hash);
+}
+
+bool VerifyPassword(const std::string &password, const std::string &hash) {
+  int ret = bcrypt_checkpw(password.c_str(), hash.c_str());
+  if (ret == -1) {
+    throw AuthException("Couldn't check password!");
+  }
+  return ret == 0;
+}
+
+}  // namespace auth

src/auth/crypto.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+namespace auth {
+
+const std::string EncryptPassword(const std::string &password);
+
+bool VerifyPassword(const std::string &password, const std::string &hash);
+
+}  // namespace auth

src/auth/exceptions.hpp

@@ -0,0 +1,15 @@
+#pragma once
+
+#include "utils/exceptions.hpp"
+
+namespace auth {
+
+/**
+ * This exception class is thrown for all exceptions that can occur when dealing
+ * with the Auth library.
+ */
+class AuthException : public utils::BasicException {
+ public:
+  using utils::BasicException::BasicException;
+};
+}  // namespace auth

src/auth/models.cpp

@@ -0,0 +1,251 @@
+#include "auth/models.hpp"
+
+#include <regex>
+
+#include <gflags/gflags.h>
+
+#include "auth/crypto.hpp"
+#include "auth/exceptions.hpp"
+#include "utils/cast.hpp"
+
+DEFINE_bool(auth_password_permit_null, true,
+            "Set to false to disable null passwords.");
+
+DEFINE_string(auth_password_strength_regex, ".+",
+              "The regular expression that should be used to match the entire "
+              "entered password to ensure its strength.");
+
+namespace auth {
+
+std::string PermissionToString(Permission permission) {
+  switch (permission) {
+    case Permission::MATCH:
+      return "MATCH";
+    case Permission::CREATE:
+      return "CREATE";
+    case Permission::MERGE:
+      return "MERGE";
+    case Permission::DELETE:
+      return "DELETE";
+    case Permission::SET:
+      return "SET";
+    case Permission::REMOVE:
+      return "REMOVE";
+    case Permission::INDEX:
+      return "INDEX";
+    case Permission::AUTH:
+      return "AUTH";
+    case Permission::STREAM:
+      return "STREAM";
+  }
+}
+
+std::string PermissionLevelToString(PermissionLevel level) {
+  switch (level) {
+    case PermissionLevel::GRANT:
+      return "GRANT";
+    case PermissionLevel::NEUTRAL:
+      return "NEUTRAL";
+    case PermissionLevel::DENY:
+      return "DENY";
+  }
+}
+
+Permissions::Permissions(uint64_t grants, uint64_t denies) {
+  // The deny bitmask has higher priority than the grant bitmask.
+  denies_ = denies;
+  // Mask out the grant bitmask to make sure that it is correct.
+  grants_ = grants & (~denies);
+}
+
+PermissionLevel Permissions::Has(Permission permission) const {
+  // Check for the deny first because it has greater priority than a grant.
+  if (denies_ & utils::UnderlyingCast(permission)) {
+    return PermissionLevel::DENY;
+  } else if (grants_ & utils::UnderlyingCast(permission)) {
+    return PermissionLevel::GRANT;
+  }
+  return PermissionLevel::NEUTRAL;
+}
+
+void Permissions::Grant(Permission permission) {
+  // Remove the possible deny.
+  denies_ &= ~utils::UnderlyingCast(permission);
+  // Now we grant the permission.
+  grants_ |= utils::UnderlyingCast(permission);
+}
+
+void Permissions::Revoke(Permission permission) {
+  // Remove the possible grant.
+  grants_ &= ~utils::UnderlyingCast(permission);
+  // Remove the possible deny.
+  denies_ &= ~utils::UnderlyingCast(permission);
+}
+
+void Permissions::Deny(Permission permission) {
+  // First deny the permission.
+  denies_ |= utils::UnderlyingCast(permission);
+  // Remove the possible grant.
+  grants_ &= ~utils::UnderlyingCast(permission);
+}
+
+std::vector<Permission> Permissions::GetGrants() const {
+  std::vector<Permission> ret;
+  for (const auto &permission : kPermissionsAll) {
+    if (Has(permission) == PermissionLevel::GRANT) {
+      ret.push_back(permission);
+    }
+  }
+  return ret;
+}
+
+std::vector<Permission> Permissions::GetDenies() const {
+  std::vector<Permission> ret;
+  for (const auto &permission : kPermissionsAll) {
+    if (Has(permission) == PermissionLevel::DENY) {
+      ret.push_back(permission);
+    }
+  }
+  return ret;
+}
+
+nlohmann::json Permissions::Serialize() const {
+  nlohmann::json data = nlohmann::json::object();
+  data["grants"] = grants_;
+  data["denies"] = denies_;
+  return data;
+}
+
+Permissions Permissions::Deserialize(const nlohmann::json &data) {
+  if (!data.is_object()) {
+    throw AuthException("Couldn't load permissions data!");
+  }
+  if (!data["grants"].is_number_unsigned() ||
+      !data["denies"].is_number_unsigned()) {
+    throw AuthException("Couldn't load permissions data!");
+  }
+  return {data["grants"], data["denies"]};
+}
+
+uint64_t Permissions::grants() const { return grants_; }
+uint64_t Permissions::denies() const { return denies_; }
+
+bool operator==(const Permissions &first, const Permissions &second) {
+  return first.grants() == second.grants() && first.denies() == second.denies();
+}
+
+bool operator!=(const Permissions &first, const Permissions &second) {
+  return !(first == second);
+}
+
+Role::Role(const std::string &rolename) : rolename_(rolename) {}
+
+Role::Role(const std::string &rolename, const Permissions &permissions)
+    : rolename_(rolename), permissions_(permissions) {}
+
+const std::string &Role::rolename() const { return rolename_; }
+const Permissions &Role::permissions() const { return permissions_; }
+Permissions &Role::permissions() { return permissions_; }
+
+nlohmann::json Role::Serialize() const {
+  nlohmann::json data = nlohmann::json::object();
+  data["rolename"] = rolename_;
+  data["permissions"] = permissions_.Serialize();
+  return data;
+}
+
+Role Role::Deserialize(const nlohmann::json &data) {
+  if (!data.is_object()) {
+    throw AuthException("Couldn't load role data!");
+  }
+  if (!data["rolename"].is_string() || !data["permissions"].is_object()) {
+    throw AuthException("Couldn't load role data!");
+  }
+  auto permissions = Permissions::Deserialize(data["permissions"]);
+  return {data["rolename"], permissions};
+}
+
+bool operator==(const Role &first, const Role &second) {
+  return first.rolename_ == second.rolename_ &&
+         first.permissions_ == second.permissions_;
+}
+
+User::User(const std::string &username) : username_(username) {}
+
+User::User(const std::string &username, const std::string &password_hash,
+           const Permissions &permissions)
+    : username_(username),
+      password_hash_(password_hash),
+      permissions_(permissions) {}
+
+bool User::CheckPassword(const std::string &password) {
+  if (password_hash_ == "") return true;
+  return VerifyPassword(password, password_hash_);
+}
+
+void User::UpdatePassword(
+    const std::experimental::optional<std::string> &password) {
+  if (password) {
+    std::regex re(FLAGS_auth_password_strength_regex);
+    if (!std::regex_match(*password, re)) {
+      throw AuthException(
+          "The user password doesn't conform to the required strength! Regex: "
+          "{}",
+          FLAGS_auth_password_strength_regex);
+    }
+    password_hash_ = EncryptPassword(*password);
+  } else {
+    if (!FLAGS_auth_password_permit_null) {
+      throw AuthException("Null passwords aren't permitted!");
+    }
+    password_hash_ = "";
+  }
+}
+
+void User::SetRole(const Role &role) { role_.emplace(role); }
+
+void User::ClearRole() { role_ = std::experimental::nullopt; }
+
+const Permissions User::GetPermissions() const {
+  if (role_) {
+    return Permissions(permissions_.grants() | role_->permissions().grants(),
+                       permissions_.denies() | role_->permissions().denies());
+  }
+  return permissions_;
+}
+
+const std::string &User::username() const { return username_; }
+
+const Permissions &User::permissions() const { return permissions_; }
+Permissions &User::permissions() { return permissions_; }
+
+std::experimental::optional<Role> User::role() const { return role_; }
+
+nlohmann::json User::Serialize() const {
+  nlohmann::json data = nlohmann::json::object();
+  data["username"] = username_;
+  data["password_hash"] = password_hash_;
+  data["permissions"] = permissions_.Serialize();
+  // The role shouldn't be serialized here, it is stored as a foreign key.
+  return data;
+}
+
+User User::Deserialize(const nlohmann::json &data) {
+  if (!data.is_object()) {
+    throw AuthException("Couldn't load user data!");
+  }
+  if (!data["username"].is_string() || !data["password_hash"].is_string() ||
+      !data["permissions"].is_object()) {
+    throw AuthException("Couldn't load user data!");
+  }
+  auto permissions = Permissions::Deserialize(data["permissions"]);
+  return {data["username"], data["password_hash"], permissions};
+}
+
+bool operator==(const User &first, const User &second) {
+  return first.username_ == second.username_ &&
+         first.password_hash_ == second.password_hash_ &&
+         first.permissions_ == second.permissions_ &&
+         first.role_ == second.role_;
+}
+}  // namespace auth

src/auth/models.hpp

@@ -0,0 +1,138 @@
+#pragma once
+
+#include <experimental/optional>
+#include <string>
+
+#include <json/json.hpp>
+
+namespace auth {
+
+// These permissions must have values that are applicable for usage in a
+// bitmask.
+enum class Permission : uint64_t {
+  MATCH = 0x00000001,
+  CREATE = 0x00000002,
+  MERGE = 0x00000004,
+  DELETE = 0x00000008,
+  SET = 0x00000010,
+  REMOVE = 0x00000020,
+  INDEX = 0x00000040,
+  AUTH = 0x00010000,
+  STREAM = 0x00020000,
+};
+
+// Constant list of all available permissions.
+const std::vector<Permission> kPermissionsAll = {
+    Permission::MATCH,  Permission::CREATE, Permission::MERGE,
+    Permission::DELETE, Permission::SET,    Permission::REMOVE,
+    Permission::INDEX,  Permission::AUTH,   Permission::STREAM};
+
+// Function that converts a permission to its string representation.
+std::string PermissionToString(Permission permission);
+
+// Class that indicates what permission level the user/role has.
+enum class PermissionLevel {
+  GRANT,
+  NEUTRAL,
+  DENY,
+};
+
+// Function that converts a permission level to its string representation.
+std::string PermissionLevelToString(PermissionLevel level);
+
+class Permissions final {
+ public:
+  Permissions(uint64_t grants = 0, uint64_t denies = 0);
+
+  PermissionLevel Has(Permission permission) const;
+
+  void Grant(Permission permission);
+
+  void Revoke(Permission permission);
+
+  void Deny(Permission permission);
+
+  std::vector<Permission> GetGrants() const;
+
+  std::vector<Permission> GetDenies() const;
+
+  nlohmann::json Serialize() const;
+
+  static Permissions Deserialize(const nlohmann::json &data);
+
+  uint64_t grants() const;
+  uint64_t denies() const;
+
+ private:
+  uint64_t grants_{0};
+  uint64_t denies_{0};
+};
+
+bool operator==(const Permissions &first, const Permissions &second);
+
+bool operator!=(const Permissions &first, const Permissions &second);
+
+class Role final {
+ public:
+  Role(const std::string &rolename);
+
+  Role(const std::string &rolename, const Permissions &permissions);
+
+  const std::string &rolename() const;
+  const Permissions &permissions() const;
+  Permissions &permissions();
+
+  nlohmann::json Serialize() const;
+
+  static Role Deserialize(const nlohmann::json &data);
+
+  friend bool operator==(const Role &first, const Role &second);
+
+ private:
+  std::string rolename_;
+  Permissions permissions_;
+};
+
+bool operator==(const Role &first, const Role &second);
+
+// TODO (mferencevic): Implement password expiry.
+class User final {
+ public:
+  User(const std::string &username);
+
+  User(const std::string &username, const std::string &password_hash,
+       const Permissions &permissions);
+
+  bool CheckPassword(const std::string &password);
+
+  void UpdatePassword(const std::experimental::optional<std::string> &password =
+                          std::experimental::nullopt);
+
+  void SetRole(const Role &role);
+
+  void ClearRole();
+
+  const Permissions GetPermissions() const;
+
+  const std::string &username() const;
+
+  const Permissions &permissions() const;
+  Permissions &permissions();
+
+  std::experimental::optional<Role> role() const;
+
+  nlohmann::json Serialize() const;
+
+  static User Deserialize(const nlohmann::json &data);
+
+  friend bool operator==(const User &first, const User &second);
+
+ private:
+  std::string username_;
+  std::string password_hash_;
+  Permissions permissions_;
+  std::experimental::optional<Role> role_;
+};
+
+bool operator==(const User &first, const User &second);
+}  // namespace auth

src/communication/CMakeLists.txt

@@ -0,0 +1,25 @@
+set(communication_src_files
+    bolt/v1/value.cpp
+    buffer.cpp
+    client.cpp
+    context.cpp
+    helpers.cpp
+    init.cpp
+    rpc/client.cpp
+    rpc/protocol.cpp
+    rpc/server.cpp)
+
+define_add_capnp(add_capnp communication_src_files communication_capnp_files)
+
+add_capnp(rpc/messages.capnp)
+
+add_custom_target(generate_communication_capnp DEPENDS ${communication_capnp_files})
+
+add_library(mg-communication STATIC ${communication_src_files})
+target_link_libraries(mg-communication Threads::Threads mg-utils mg-io fmt glog gflags)
+target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})
+target_include_directories(mg-communication SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
+#target_link_libraries(mg-communication capnp kj)
+add_dependencies(mg-communication generate_communication_capnp)
+
+target_compile_options(mg-communication PRIVATE -fPIC)

src/communication/bolt/client.hpp

@@ -6,111 +6,147 @@
 #include "communication/bolt/v1/decoder/decoder.hpp"
 #include "communication/bolt/v1/encoder/chunked_encoder_buffer.hpp"
 #include "communication/bolt/v1/encoder/client_encoder.hpp"
-
-#include "query/typed_value.hpp"
+#include "communication/client.hpp"
+#include "communication/context.hpp"
+#include "io/network/endpoint.hpp"
 #include "utils/exceptions.hpp"
 
 namespace communication::bolt {
 
-class ClientFatalException : public utils::BasicException {
- public:
-  using utils::BasicException::BasicException;
-  ClientFatalException()
-      : utils::BasicException(
-            "Something went wrong while communicating with the server!") {}
-};
-
+/// This exception is thrown whenever an error occurs during query execution
+/// that isn't fatal (eg. mistyped query or some transient error occurred).
+/// It should be handled by everyone who uses the client.
 class ClientQueryException : public utils::BasicException {
  public:
   using utils::BasicException::BasicException;
   ClientQueryException() : utils::BasicException("Couldn't execute query!") {}
 };
 
-struct QueryData {
-  std::vector<std::string> fields;
-  std::vector<std::vector<DecodedValue>> records;
-  std::map<std::string, DecodedValue> metadata;
+/// This exception is thrown whenever a fatal error occurs during query
+/// execution and/or connecting to the server.
+/// It should be handled by everyone who uses the client.
+class ClientFatalException : public utils::BasicException {
+ public:
+  using utils::BasicException::BasicException;
 };
 
-class Client {
+// Internal exception used whenever a communication error occurs. You should
+// only handle the `ClientFatalException`.
+class ServerCommunicationException : public ClientFatalException {
  public:
-  Client() {}
+  ServerCommunicationException()
+      : ClientFatalException("Couldn't communicate with the server!") {}
+};
+
+// Internal exception used whenever a malformed data error occurs. You should
+// only handle the `ClientFatalException`.
+class ServerMalformedDataException : public ClientFatalException {
+ public:
+  ServerMalformedDataException()
+      : ClientFatalException("The server sent malformed data!") {}
+};
+
+/// Structure that is used to return results from an executed query.
+struct QueryData {
+  std::vector<std::string> fields;
+  std::vector<std::vector<Value>> records;
+  std::map<std::string, Value> metadata;
+};
+
+/// Bolt client.
+/// It has methods used to connect to the server and execute queries against the
+/// server. It supports both SSL and plaintext connections.
+class Client final {
+ public:
+  explicit Client(communication::ClientContext *context) : client_(context) {}
 
   Client(const Client &) = delete;
   Client(Client &&) = delete;
   Client &operator=(const Client &) = delete;
   Client &operator=(Client &&) = delete;
 
-  bool Connect(const io::network::Endpoint &endpoint,
+  /// Method used to connect to the server. Before executing queries this method
+  /// should be called to set-up the connection to the server. After the
+  /// connection is set-up, multiple queries may be executed through a single
+  /// established connection.
+  /// @throws ClientFatalException when we couldn't connect to the server
+  void Connect(const io::network::Endpoint &endpoint,
                const std::string &username, const std::string &password,
-               const std::string &client_name = "memgraph-bolt/0.0.1") {
+               const std::string &client_name = "memgraph-bolt") {
     if (!client_.Connect(endpoint)) {
-      LOG(ERROR) << "Couldn't connect to " << endpoint;
-      return false;
+      throw ClientFatalException("Couldn't connect to {}!", endpoint);
     }
 
     if (!client_.Write(kPreamble, sizeof(kPreamble), true)) {
-      LOG(ERROR) << "Couldn't send preamble!";
-      return false;
+      DLOG(ERROR) << "Couldn't send preamble!";
+      throw ServerCommunicationException();
     }
     for (int i = 0; i < 4; ++i) {
       if (!client_.Write(kProtocol, sizeof(kProtocol), i != 3)) {
-        LOG(ERROR) << "Couldn't send protocol version!";
-        return false;
+        DLOG(ERROR) << "Couldn't send protocol version!";
+        throw ServerCommunicationException();
       }
     }
 
     if (!client_.Read(sizeof(kProtocol))) {
-      LOG(ERROR) << "Couldn't get negotiated protocol version!";
-      return false;
+      DLOG(ERROR) << "Couldn't get negotiated protocol version!";
+      throw ServerCommunicationException();
     }
     if (memcmp(kProtocol, client_.GetData(), sizeof(kProtocol)) != 0) {
-      LOG(ERROR) << "Server negotiated unsupported protocol version!";
-      return false;
+      DLOG(ERROR) << "Server negotiated unsupported protocol version!";
+      throw ClientFatalException(
+          "The server negotiated an usupported protocol version!");
     }
     client_.ShiftData(sizeof(kProtocol));
 
     if (!encoder_.MessageInit(client_name, {{"scheme", "basic"},
                                             {"principal", username},
                                             {"credentials", password}})) {
-      LOG(ERROR) << "Couldn't send init message!";
-      return false;
+      DLOG(ERROR) << "Couldn't send init message!";
+      throw ServerCommunicationException();
     }
 
     Signature signature;
-    DecodedValue metadata;
+    Value metadata;
     if (!ReadMessage(&signature, &metadata)) {
-      LOG(ERROR) << "Couldn't read init message response!";
-      return false;
+      DLOG(ERROR) << "Couldn't read init message response!";
+      throw ServerCommunicationException();
     }
     if (signature != Signature::Success) {
-      LOG(ERROR) << "Handshake failed!";
-      return false;
+      DLOG(ERROR) << "Handshake failed!";
+      throw ClientFatalException("Handshake with the server failed!");
     }
 
     DLOG(INFO) << "Metadata of init message response: " << metadata;
-
-    return true;
   }
 
+  /// Function used to execute queries against the server. Before you can
+  /// execute queries you must connect the client to the server.
+  /// @throws ClientQueryException when there is some transient error while
+  ///                              executing the query (eg. mistyped query,
+  ///                              etc.)
+  /// @throws ClientFatalException when we couldn't communicate with the server
   QueryData Execute(const std::string &query,
-                    const std::map<std::string, DecodedValue> &parameters) {
+                    const std::map<std::string, Value> &parameters) {
+    if (!client_.IsConnected()) {
+      throw ClientFatalException(
+          "You must first connect to the server before using the client!");
+    }
+
     DLOG(INFO) << "Sending run message with statement: '" << query
                << "'; parameters: " << parameters;
 
-    std::map<std::string, query::TypedValue> params_tv(parameters.begin(),
-                                                       parameters.end());
-    encoder_.MessageRun(query, params_tv, false);
+    encoder_.MessageRun(query, parameters);
     encoder_.MessagePullAll();
 
     DLOG(INFO) << "Reading run message response";
     Signature signature;
-    DecodedValue fields;
+    Value fields;
     if (!ReadMessage(&signature, &fields)) {
-      throw ClientFatalException();
+      throw ServerCommunicationException();
     }
-    if (fields.type() != DecodedValue::Type::Map) {
-      throw ClientFatalException();
+    if (fields.type() != Value::Type::Map) {
+      throw ServerMalformedDataException();
     }
 
     if (signature == Signature::Failure) {
@@ -122,35 +158,35 @@ class Client {
       }
       throw ClientQueryException();
     } else if (signature != Signature::Success) {
-      throw ClientFatalException();
+      throw ServerMalformedDataException();
     }
 
     DLOG(INFO) << "Reading pull_all message response";
     Marker marker;
-    DecodedValue metadata;
-    std::vector<std::vector<DecodedValue>> records;
+    Value metadata;
+    std::vector<std::vector<Value>> records;
     while (true) {
       if (!GetMessage()) {
-        throw ClientFatalException();
+        throw ServerCommunicationException();
       }
       if (!decoder_.ReadMessageHeader(&signature, &marker)) {
-        throw ClientFatalException();
+        throw ServerCommunicationException();
       }
       if (signature == Signature::Record) {
-        DecodedValue record;
-        if (!decoder_.ReadValue(&record, DecodedValue::Type::List)) {
-          throw ClientFatalException();
+        Value record;
+        if (!decoder_.ReadValue(&record, Value::Type::List)) {
+          throw ServerCommunicationException();
         }
-        records.push_back(record.ValueList());
+        records.emplace_back(std::move(record.ValueList()));
       } else if (signature == Signature::Success) {
         if (!decoder_.ReadValue(&metadata)) {
-          throw ClientFatalException();
+          throw ServerCommunicationException();
         }
         break;
       } else if (signature == Signature::Failure) {
-        DecodedValue data;
+        Value data;
         if (!decoder_.ReadValue(&data)) {
-          throw ClientFatalException();
+          throw ServerCommunicationException();
         }
         HandleFailure();
         auto &tmp = data.ValueMap();
@@ -160,42 +196,43 @@ class Client {
         }
         throw ClientQueryException();
       } else {
-        throw ClientFatalException();
+        throw ServerMalformedDataException();
       }
     }
 
-    if (metadata.type() != DecodedValue::Type::Map) {
-      throw ClientFatalException();
+    if (metadata.type() != Value::Type::Map) {
+      throw ServerMalformedDataException();
     }
 
-    QueryData ret{{}, records, metadata.ValueMap()};
+    QueryData ret{{}, std::move(records), std::move(metadata.ValueMap())};
 
     auto &header = fields.ValueMap();
     if (header.find("fields") == header.end()) {
-      throw ClientFatalException();
+      throw ServerMalformedDataException();
     }
-    if (header["fields"].type() != DecodedValue::Type::List) {
-      throw ClientFatalException();
+    if (header["fields"].type() != Value::Type::List) {
+      throw ServerMalformedDataException();
     }
     auto &field_vector = header["fields"].ValueList();
 
     for (auto &field_item : field_vector) {
-      if (field_item.type() != DecodedValue::Type::String) {
-        throw ClientFatalException();
+      if (field_item.type() != Value::Type::String) {
+        throw ServerMalformedDataException();
       }
-      ret.fields.push_back(field_item.ValueString());
+      ret.fields.emplace_back(std::move(field_item.ValueString()));
     }
 
     return ret;
   }
 
+  /// Close the active client connection.
   void Close() { client_.Close(); };
 
  private:
   bool GetMessage() {
     client_.ClearData();
     while (true) {
-      if (!client_.Read(CHUNK_HEADER_SIZE)) return false;
+      if (!client_.Read(kChunkHeaderSize)) return false;
 
       size_t chunk_size = client_.GetData()[0];
       chunk_size <<= 8;
@@ -209,16 +246,16 @@ class Client {
     return true;
   }
 
-  bool ReadMessage(Signature *signature, DecodedValue *ret) {
+  bool ReadMessage(Signature *signature, Value *ret) {
     Marker marker;
     if (!GetMessage()) return false;
     if (!decoder_.ReadMessageHeader(signature, &marker)) return false;
     return ReadMessageData(marker, ret);
   }
 
-  bool ReadMessageData(Marker marker, DecodedValue *ret) {
+  bool ReadMessageData(Marker marker, Value *ret) {
     if (marker == Marker::TinyStruct) {
-      *ret = DecodedValue();
+      *ret = Value();
       return true;
     } else if (marker == Marker::TinyStruct1) {
       return decoder_.ReadValue(ret);
@@ -228,18 +265,18 @@ class Client {
 
   void HandleFailure() {
     if (!encoder_.MessageAckFailure()) {
-      throw ClientFatalException();
+      throw ServerCommunicationException();
     }
     while (true) {
       Signature signature;
-      DecodedValue data;
+      Value data;
       if (!ReadMessage(&signature, &data)) {
-        throw ClientFatalException();
+        throw ServerCommunicationException();
       }
       if (signature == Signature::Success) {
         break;
       } else if (signature != Signature::Ignored) {
-        throw ClientFatalException();
+        throw ServerMalformedDataException();
       }
     }
   }

src/communication/bolt/v1/codes.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <cstdint>
-#include "utils/underlying_cast.hpp"
 
 namespace communication::bolt {
 
@@ -82,4 +81,4 @@ static constexpr Marker Marker16[3] = {Marker::String16, Marker::List16,
                                        Marker::Map16};
 static constexpr Marker Marker32[3] = {Marker::String32, Marker::List32,
                                        Marker::Map32};
-}
+}  // namespace communication::bolt

src/communication/bolt/v1/constants.hpp

@@ -7,13 +7,12 @@ namespace communication::bolt {
 /**
  * Sizes related to the chunk defined in Bolt protocol.
  */
-static constexpr size_t CHUNK_HEADER_SIZE = 2;
-static constexpr size_t MAX_CHUNK_SIZE = 65535;
-static constexpr size_t CHUNK_END_MARKER_SIZE = 2;
-static constexpr size_t WHOLE_CHUNK_SIZE = CHUNK_HEADER_SIZE + MAX_CHUNK_SIZE;
+static constexpr size_t kChunkHeaderSize = 2;
+static constexpr size_t kChunkMaxDataSize = 65535;
+static constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
 
 /**
  * Handshake size defined in the Bolt protocol.
  */
-static constexpr size_t HANDSHAKE_SIZE = 20;
-}
+static constexpr size_t kHandshakeSize = 20;
+}  // namespace communication::bolt

src/communication/bolt/v1/decoder/buffer.hpp

@@ -1,94 +0,0 @@
-#pragma once
-
-#include <algorithm>
-#include <cstring>
-#include <memory>
-#include <vector>
-
-#include "glog/logging.h"
-
-#include "communication/bolt/v1/constants.hpp"
-#include "io/network/stream_buffer.hpp"
-#include "utils/bswap.hpp"
-
-namespace communication::bolt {
-
-/**
- * @brief Buffer
- *
- * Has methods for writing and reading raw data.
- *
- * Allocating, writing and written stores data in the buffer. The stored
- * data can then be read using the pointer returned with the data function.
- * The current implementation stores data in a single fixed length buffer.
- *
- * @tparam Size the size of the internal byte array, defaults to the maximum
- *         size of a chunk in the Bolt protocol
- */
-template <size_t Size = WHOLE_CHUNK_SIZE>
-class Buffer {
- private:
-  using StreamBufferT = io::network::StreamBuffer;
-
- public:
-  Buffer() = default;
-
-  /**
-   * Allocates a new StreamBuffer from the internal buffer.
-   * This function returns a pointer to the first currently free memory
-   * location in the internal buffer. Also, it returns the size of the
-   * available memory.
-   */
-  StreamBufferT Allocate() {
-    return StreamBufferT{&data_[size_], Size - size_};
-  }
-
-  /**
-   * This method is used to notify the buffer that the data has been written.
-   * To write data to this buffer you should do this:
-   * Call Allocate(), then write to the returned data pointer.
-   * IMPORTANT: Don't write more data then the returned size, you will cause
-   * a memory overflow. Then call Written(size) with the length of data that
-   * you have written into the buffer.
-   *
-   * @param len the size of data that has been written into the buffer
-   */
-  void Written(size_t len) {
-    size_ += len;
-    DCHECK(size_ <= Size) << "Written more than storage has space!";
-  }
-
-  /**
-   * This method shifts the available data for len. It is used when you read
-   * some data from the buffer and you want to remove it from the buffer.
-   *
-   * @param len the length of data that has to be removed from the start of
-   *            the buffer
-   */
-  void Shift(size_t len) {
-    DCHECK(len <= size_) << "Tried to shift more data than the buffer has!";
-    memmove(data_, data_ + len, size_ - len);
-    size_ -= len;
-  }
-
-  /**
-   * This method clears the buffer.
-   */
-  void Clear() { size_ = 0; }
-
-  /**
-   * This function returns a pointer to the internal buffer. It is used for
-   * reading data from the buffer.
-   */
-  uint8_t *data() { return data_; }
-
-  /**
-   * This function returns the size of available data for reading.
-   */
-  size_t size() { return size_; }
-
- private:
-  uint8_t data_[Size];
-  size_t size_{0};
-};
-}  // namespace communication::bolt

src/communication/bolt/v1/decoder/chunked_decoder_buffer.hpp

@@ -42,7 +42,7 @@ template <typename TBuffer>
 class ChunkedDecoderBuffer {
  public:
   ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) {
-    data_.reserve(MAX_CHUNK_SIZE);
+    data_.reserve(kChunkMaxDataSize);
   }
 
   /**
@@ -93,7 +93,6 @@ class ChunkedDecoderBuffer {
     size_t size = buffer_.size();
 
     if (size < 2) {
-      DLOG(WARNING) << "Size < 2";
       return ChunkState::Partial;
     }
 
@@ -108,8 +107,6 @@ class ChunkedDecoderBuffer {
     }
 
     if (size < chunk_size + 2) {
-      DLOG(WARNING) << fmt::format(
-          "Chunk size is {} but only have {} data bytes.", chunk_size, size);
       return ChunkState::Partial;
     }
 

src/communication/bolt/v1/decoder/decoded_value.cpp

@@ -1,344 +0,0 @@
-#include "glog/logging.h"
-
-#include "communication/bolt/v1/decoder/decoded_value.hpp"
-
-namespace communication::bolt {
-
-#define DEF_GETTER_BY_VAL(type, value_type, field)          \
-  value_type &DecodedValue::Value##type() {                 \
-    if (type_ != Type::type) throw DecodedValueException(); \
-    return field;                                           \
-  }                                                         \
-  value_type DecodedValue::Value##type() const {            \
-    if (type_ != Type::type) throw DecodedValueException(); \
-    return field;                                           \
-  }
-
-DEF_GETTER_BY_VAL(Bool, bool, bool_v)
-DEF_GETTER_BY_VAL(Int, int64_t, int_v)
-DEF_GETTER_BY_VAL(Double, double, double_v)
-
-#undef DEF_GETTER_BY_VAL
-
-#define DEF_GETTER_BY_REF(type, value_type, field)          \
-  value_type &DecodedValue::Value##type() {                 \
-    if (type_ != Type::type) throw DecodedValueException(); \
-    return field;                                           \
-  }                                                         \
-  const value_type &DecodedValue::Value##type() const {     \
-    if (type_ != Type::type) throw DecodedValueException(); \
-    return field;                                           \
-  }
-
-DEF_GETTER_BY_REF(String, std::string, string_v)
-DEF_GETTER_BY_REF(List, std::vector<DecodedValue>, list_v)
-using map_t = std::map<std::string, DecodedValue>;
-DEF_GETTER_BY_REF(Map, map_t, map_v)
-DEF_GETTER_BY_REF(Vertex, DecodedVertex, vertex_v)
-DEF_GETTER_BY_REF(Edge, DecodedEdge, edge_v)
-DEF_GETTER_BY_REF(UnboundedEdge, DecodedUnboundedEdge, unbounded_edge_v)
-DEF_GETTER_BY_REF(Path, DecodedPath, path_v)
-
-#undef DEF_GETTER_BY_REF
-
-DecodedValue::DecodedValue(const DecodedValue &other) : type_(other.type_) {
-  switch (other.type_) {
-    case Type::Null:
-      return;
-    case Type::Bool:
-      this->bool_v = other.bool_v;
-      return;
-    case Type::Int:
-      this->int_v = other.int_v;
-      return;
-    case Type::Double:
-      this->double_v = other.double_v;
-      return;
-    case Type::String:
-      new (&string_v) std::string(other.string_v);
-      return;
-    case Type::List:
-      new (&list_v) std::vector<DecodedValue>(other.list_v);
-      return;
-    case Type::Map:
-      new (&map_v) std::map<std::string, DecodedValue>(other.map_v);
-      return;
-    case Type::Vertex:
-      new (&vertex_v) DecodedVertex(other.vertex_v);
-      return;
-    case Type::Edge:
-      new (&edge_v) DecodedEdge(other.edge_v);
-      return;
-    case Type::UnboundedEdge:
-      new (&unbounded_edge_v) DecodedUnboundedEdge(other.unbounded_edge_v);
-      return;
-    case Type::Path:
-      new (&path_v) DecodedPath(other.path_v);
-      return;
-  }
-  LOG(FATAL) << "Unsupported DecodedValue::Type";
-}
-
-DecodedValue &DecodedValue::operator=(const DecodedValue &other) {
-  if (this != &other) {
-    this->~DecodedValue();
-    // set the type of this
-    type_ = other.type_;
-
-    switch (other.type_) {
-      case Type::Null:
-        return *this;
-      case Type::Bool:
-        this->bool_v = other.bool_v;
-        return *this;
-      case Type::Int:
-        this->int_v = other.int_v;
-        return *this;
-      case Type::Double:
-        this->double_v = other.double_v;
-        return *this;
-      case Type::String:
-        new (&string_v) std::string(other.string_v);
-        return *this;
-      case Type::List:
-        new (&list_v) std::vector<DecodedValue>(other.list_v);
-        return *this;
-      case Type::Map:
-        new (&map_v) std::map<std::string, DecodedValue>(other.map_v);
-        return *this;
-      case Type::Vertex:
-        new (&vertex_v) DecodedVertex(other.vertex_v);
-        return *this;
-      case Type::Edge:
-        new (&edge_v) DecodedEdge(other.edge_v);
-        return *this;
-      case Type::UnboundedEdge:
-        new (&unbounded_edge_v) DecodedUnboundedEdge(other.unbounded_edge_v);
-        return *this;
-      case Type::Path:
-        new (&path_v) DecodedPath(other.path_v);
-        return *this;
-    }
-    LOG(FATAL) << "Unsupported DecodedValue::Type";
-  }
-  return *this;
-}
-
-DecodedValue::~DecodedValue() {
-  switch (type_) {
-    // destructor for primitive types does nothing
-    case Type::Null:
-    case Type::Bool:
-    case Type::Int:
-    case Type::Double:
-      return;
-
-    // we need to call destructors for non primitive types since we used
-    // placement new
-    case Type::String:
-      // Clang fails to compile ~std::string. It seems it is a bug in some
-      // versions of clang. using namespace std statement solves the issue.
-      using namespace std;
-      string_v.~string();
-      return;
-    case Type::List:
-      using namespace std;
-      list_v.~vector<DecodedValue>();
-      return;
-    case Type::Map:
-      using namespace std;
-      map_v.~map<std::string, DecodedValue>();
-      return;
-    case Type::Vertex:
-      vertex_v.~DecodedVertex();
-      return;
-    case Type::Edge:
-      edge_v.~DecodedEdge();
-      return;
-    case Type::UnboundedEdge:
-      unbounded_edge_v.~DecodedUnboundedEdge();
-      return;
-    case Type::Path:
-      path_v.~DecodedPath();
-      return;
-  }
-  LOG(FATAL) << "Unsupported DecodedValue::Type";
-}
-
-DecodedValue::operator query::TypedValue() const {
-  switch (type_) {
-    case Type::Null:
-      return query::TypedValue::Null;
-    case Type::Bool:
-      return query::TypedValue(bool_v);
-    case Type::Int:
-      return query::TypedValue(int_v);
-    case Type::Double:
-      return query::TypedValue(double_v);
-    case Type::String:
-      return query::TypedValue(string_v);
-    case Type::List:
-      return query::TypedValue(
-          std::vector<query::TypedValue>(list_v.begin(), list_v.end()));
-    case Type::Map:
-      return query::TypedValue(
-          std::map<std::string, query::TypedValue>(map_v.begin(), map_v.end()));
-    case Type::Vertex:
-    case Type::Edge:
-    case Type::UnboundedEdge:
-    case Type::Path:
-      throw DecodedValueException(
-          "Unsupported conversion from DecodedValue to TypedValue");
-  }
-}
-
-DecodedValue::operator PropertyValue() const {
-  switch (type_) {
-    case Type::Null:
-      return PropertyValue::Null;
-    case Type::Bool:
-      return PropertyValue(bool_v);
-    case Type::Int:
-      return PropertyValue(int_v);
-    case Type::Double:
-      return PropertyValue(double_v);
-    case Type::String:
-      return PropertyValue(string_v);
-    case Type::List: {
-      std::vector<PropertyValue> vec;
-      vec.reserve(list_v.size());
-      for (const auto &value : list_v)
-        vec.emplace_back(static_cast<PropertyValue>(value));
-      return PropertyValue(std::move(vec));
-    }
-    case Type::Map: {
-      std::map<std::string, PropertyValue> map;
-      for (const auto &kv : map_v)
-        map.emplace(kv.first, static_cast<PropertyValue>(kv.second));
-      return PropertyValue(std::move(map));
-    }
-    case Type::Vertex:
-    case Type::Edge:
-    case Type::UnboundedEdge:
-    case Type::Path:
-      throw DecodedValueException(
-          "Unsupported conversion from DecodedValue to PropertyValue");
-  }
-}
-
-std::ostream &operator<<(std::ostream &os, const DecodedVertex &vertex) {
-  os << "V(";
-  utils::PrintIterable(os, vertex.labels, ":",
-                       [&](auto &stream, auto label) { stream << label; });
-  os << " {";
-  utils::PrintIterable(os, vertex.properties, ", ",
-                       [&](auto &stream, const auto &pair) {
-                         stream << pair.first << ": " << pair.second;
-                       });
-  return os << "})";
-}
-
-std::ostream &operator<<(std::ostream &os, const DecodedEdge &edge) {
-  os << "E[" << edge.type;
-  os << " {";
-  utils::PrintIterable(os, edge.properties, ", ",
-                       [&](auto &stream, const auto &pair) {
-                         stream << pair.first << ": " << pair.second;
-                       });
-  return os << "}]";
-}
-
-std::ostream &operator<<(std::ostream &os, const DecodedUnboundedEdge &edge) {
-  os << "E[" << edge.type;
-  os << " {";
-  utils::PrintIterable(os, edge.properties, ", ",
-                       [&](auto &stream, const auto &pair) {
-                         stream << pair.first << ": " << pair.second;
-                       });
-  return os << "}]";
-}
-
-std::ostream &operator<<(std::ostream &os, const DecodedPath &path) {
-  os << path.vertices[0];
-  DCHECK(path.indices.size() % 2 == 0) << "Must have even number of indices";
-  for (auto it = path.indices.begin(); it != path.indices.end();) {
-    auto edge_ind = *it++;
-    auto vertex_ind = *it++;
-    bool arrow_to_right = true;
-    if (edge_ind < 0) {
-      arrow_to_right = false;
-      edge_ind = -edge_ind;
-    }
-
-    if (!arrow_to_right) os << "<";
-    os << "-" << path.edges[edge_ind - 1] << "-";
-    if (arrow_to_right) os << ">";
-    os << path.vertices[vertex_ind];
-  }
-
-  return os;
-}
-
-std::ostream &operator<<(std::ostream &os, const DecodedValue &value) {
-  switch (value.type_) {
-    case DecodedValue::Type::Null:
-      return os << "Null";
-    case DecodedValue::Type::Bool:
-      return os << (value.ValueBool() ? "true" : "false");
-    case DecodedValue::Type::Int:
-      return os << value.ValueInt();
-    case DecodedValue::Type::Double:
-      return os << value.ValueDouble();
-    case DecodedValue::Type::String:
-      return os << value.ValueString();
-    case DecodedValue::Type::List:
-      os << "[";
-      utils::PrintIterable(os, value.ValueList());
-      return os << "]";
-    case DecodedValue::Type::Map:
-      os << "{";
-      utils::PrintIterable(os, value.ValueMap(), ", ",
-                           [](auto &stream, const auto &pair) {
-                             stream << pair.first << ": " << pair.second;
-                           });
-      return os << "}";
-    case DecodedValue::Type::Vertex:
-      return os << value.ValueVertex();
-    case DecodedValue::Type::Edge:
-      return os << value.ValueEdge();
-    case DecodedValue::Type::UnboundedEdge:
-      return os << value.ValueUnboundedEdge();
-    case DecodedValue::Type::Path:
-      return os << value.ValuePath();
-  }
-  LOG(FATAL) << "Unsupported DecodedValue::Type";
-}
-
-std::ostream &operator<<(std::ostream &os, const DecodedValue::Type type) {
-  switch (type) {
-    case DecodedValue::Type::Null:
-      return os << "null";
-    case DecodedValue::Type::Bool:
-      return os << "bool";
-    case DecodedValue::Type::Int:
-      return os << "int";
-    case DecodedValue::Type::Double:
-      return os << "double";
-    case DecodedValue::Type::String:
-      return os << "string";
-    case DecodedValue::Type::List:
-      return os << "list";
-    case DecodedValue::Type::Map:
-      return os << "map";
-    case DecodedValue::Type::Vertex:
-      return os << "vertex";
-    case DecodedValue::Type::Edge:
-      return os << "edge";
-    case DecodedValue::Type::UnboundedEdge:
-      return os << "unbounded_edge";
-    case DecodedValue::Type::Path:
-      return os << "path";
-  }
-  LOG(FATAL) << "Unsupported DecodedValue::Type";
-}
-}

src/communication/bolt/v1/decoder/decoded_value.hpp

@@ -1,207 +0,0 @@
-#pragma once
-
-#include <map>
-#include <string>
-#include <vector>
-
-#include "query/typed_value.hpp"
-#include "storage/property_value.hpp"
-#include "utils/algorithm.hpp"
-#include "utils/exceptions.hpp"
-
-namespace communication::bolt {
-
-/** Forward declaration of DecodedValue class. */
-class DecodedValue;
-
-/**
- * Structure used when reading a Vertex with the decoder.
- * The decoder writes data into this structure.
- */
-struct DecodedVertex {
-  int64_t id;
-  std::vector<std::string> labels;
-  std::map<std::string, DecodedValue> properties;
-};
-
-/**
- * Structure used when reading an Edge with the decoder.
- * The decoder writes data into this structure.
- */
-struct DecodedEdge {
-  int64_t id;
-  int64_t from;
-  int64_t to;
-  std::string type;
-  std::map<std::string, DecodedValue> properties;
-};
-
-/**
- * Structure used when reading an UnboundEdge with the decoder.
- * The decoder writes data into this structure.
- */
-struct DecodedUnboundedEdge {
-  int64_t id;
-  std::string type;
-  std::map<std::string, DecodedValue> properties;
-};
-
-/**
- * Structure used when reading a Path with the decoder.
- * The decoder writes data into this structure.
- */
-struct DecodedPath {
-  std::vector<DecodedVertex> vertices;
-  std::vector<DecodedUnboundedEdge> edges;
-  std::vector<int64_t> indices;
-};
-
-/**
- * DecodedValue provides an encapsulation arround TypedValue, DecodedVertex
- * and DecodedEdge. This is necessary because TypedValue stores vertices and
- * edges as our internal accessors. Because of that the Bolt decoder can't
- * decode vertices and edges directly into a TypedValue so a DecodedValue is
- * used instead.
- */
-class DecodedValue {
- public:
-  /** Default constructor, makes Null */
-  DecodedValue() : type_(Type::Null) {}
-
-  /** Types that can be stored in a DecodedValue. */
-  // TODO: Path isn't supported yet!
-  enum class Type : unsigned {
-    Null,
-    Bool,
-    Int,
-    Double,
-    String,
-    List,
-    Map,
-    Vertex,
-    Edge,
-    UnboundedEdge,
-    Path
-  };
-
-  // constructors for primitive types
-  DecodedValue(bool value) : type_(Type::Bool) { bool_v = value; }
-  DecodedValue(int value) : type_(Type::Int) { int_v = value; }
-  DecodedValue(int64_t value) : type_(Type::Int) { int_v = value; }
-  DecodedValue(double value) : type_(Type::Double) { double_v = value; }
-
-  // constructors for non-primitive types
-  DecodedValue(const std::string &value) : type_(Type::String) {
-    new (&string_v) std::string(value);
-  }
-  DecodedValue(const std::vector<DecodedValue> &value) : type_(Type::List) {
-    new (&list_v) std::vector<DecodedValue>(value);
-  }
-  DecodedValue(const std::map<std::string, DecodedValue> &value)
-      : type_(Type::Map) {
-    new (&map_v) std::map<std::string, DecodedValue>(value);
-  }
-  DecodedValue(const DecodedVertex &value) : type_(Type::Vertex) {
-    new (&vertex_v) DecodedVertex(value);
-  }
-  DecodedValue(const DecodedEdge &value) : type_(Type::Edge) {
-    new (&edge_v) DecodedEdge(value);
-  }
-  DecodedValue(const DecodedUnboundedEdge &value) : type_(Type::UnboundedEdge) {
-    new (&unbounded_edge_v) DecodedUnboundedEdge(value);
-  }
-  DecodedValue(const DecodedPath &value) : type_(Type::Path) {
-    new (&path_v) DecodedPath(value);
-  }
-
-  DecodedValue &operator=(const DecodedValue &other);
-  DecodedValue(const DecodedValue &other);
-  ~DecodedValue();
-
-  Type type() const { return type_; }
-
-#define DECL_GETTER_BY_VALUE(type, value_type) \
-  value_type &Value##type();                   \
-  value_type Value##type() const;
-
-  DECL_GETTER_BY_VALUE(Bool, bool)
-  DECL_GETTER_BY_VALUE(Int, int64_t)
-  DECL_GETTER_BY_VALUE(Double, double)
-
-#undef DECL_GETTER_BY_VALUE
-
-#define DECL_GETTER_BY_REFERENCE(type, value_type) \
-  value_type &Value##type();                       \
-  const value_type &Value##type() const;
-
-  DECL_GETTER_BY_REFERENCE(String, std::string)
-  DECL_GETTER_BY_REFERENCE(List, std::vector<DecodedValue>)
-  using map_t = std::map<std::string, DecodedValue>;
-  DECL_GETTER_BY_REFERENCE(Map, map_t)
-  DECL_GETTER_BY_REFERENCE(Vertex, DecodedVertex)
-  DECL_GETTER_BY_REFERENCE(Edge, DecodedEdge)
-  DECL_GETTER_BY_REFERENCE(UnboundedEdge, DecodedUnboundedEdge)
-  DECL_GETTER_BY_REFERENCE(Path, DecodedPath)
-
-#undef DECL_GETTER_BY_REFERNCE
-
-#define TYPE_CHECKER(type) \
-  bool Is##type() const { return type_ == Type::type; }
-
-  TYPE_CHECKER(Bool)
-  TYPE_CHECKER(Int)
-  TYPE_CHECKER(Double)
-  TYPE_CHECKER(String)
-  TYPE_CHECKER(List)
-  TYPE_CHECKER(Map)
-  TYPE_CHECKER(Vertex)
-  TYPE_CHECKER(Edge)
-  TYPE_CHECKER(UnboundedEdge)
-  TYPE_CHECKER(Path)
-
-#undef TYPE_CHECKER
-
-  operator query::TypedValue() const;
-  // PropertyValue operator must be explicit to prevent ambiguity.
-  explicit operator PropertyValue() const;
-
-  friend std::ostream &operator<<(std::ostream &os, const DecodedValue &value);
-
- private:
-  Type type_;
-
-  // storage for the value of the property
-  union {
-    bool bool_v;
-    int64_t int_v;
-    double double_v;
-    std::string string_v;
-    std::vector<DecodedValue> list_v;
-    std::map<std::string, DecodedValue> map_v;
-    DecodedVertex vertex_v;
-    DecodedEdge edge_v;
-    DecodedUnboundedEdge unbounded_edge_v;
-    DecodedPath path_v;
-  };
-};
-
-/**
- * An exception raised by the DecodedValue system.
- */
-class DecodedValueException : public utils::BasicException {
- public:
-  using utils::BasicException::BasicException;
-  DecodedValueException()
-      : BasicException("Incompatible template param and type!") {}
-};
-
-/**
- * Output operators.
- */
-std::ostream &operator<<(std::ostream &os, const DecodedVertex &vertex);
-std::ostream &operator<<(std::ostream &os, const DecodedEdge &edge);
-std::ostream &operator<<(std::ostream &os, const DecodedUnboundedEdge &edge);
-std::ostream &operator<<(std::ostream &os, const DecodedPath &path);
-std::ostream &operator<<(std::ostream &os, const DecodedValue &value);
-std::ostream &operator<<(std::ostream &os, const DecodedValue::Type type);
-}  // namespace communication::bolt

src/communication/bolt/v1/decoder/decoder.hpp

@@ -5,9 +5,9 @@
 #include <glog/logging.h>
 
 #include "communication/bolt/v1/codes.hpp"
-#include "communication/bolt/v1/decoder/decoded_value.hpp"
+#include "communication/bolt/v1/value.hpp"
 #include "utils/bswap.hpp"
-#include "utils/underlying_cast.hpp"
+#include "utils/cast.hpp"
 
 namespace communication::bolt {
 
@@ -23,20 +23,17 @@ class Decoder {
   explicit Decoder(Buffer &buffer) : buffer_(buffer) {}
 
   /**
-   * Reads a DecodedValue from the available data in the buffer.
-   * This function tries to read a DecodedValue from the available data.
+   * Reads a Value from the available data in the buffer.
+   * This function tries to read a Value from the available data.
    *
-   * @param data pointer to a DecodedValue where the read data should be stored
+   * @param data pointer to a Value where the read data should be stored
    * @returns true if data has been written to the data pointer,
    *          false otherwise
    */
-  bool ReadValue(DecodedValue *data) {
+  bool ReadValue(Value *data) {
     uint8_t value;
 
-    VLOG(20) << "[ReadValue] Start";
-
     if (!buffer_.Read(&value, 1)) {
-      DLOG(WARNING) << "[ReadValue] Marker data missing!";
       return false;
     }
 
@@ -80,7 +77,6 @@ class Decoder {
         // won't perform an additional signature read.
         uint8_t signature;
         if (!buffer_.Read(&signature, 1)) {
-          DLOG(WARNING) << "[ReadVertex] Missing marker and/or signature data!";
           return false;
         }
         switch (static_cast<Signature>(signature)) {
@@ -91,9 +87,6 @@ class Decoder {
           case Signature::Path:
             return ReadPath(data);
           default:
-            DLOG(WARNING) << "[ReadValue] Expected [node | unbounded_ege | "
-                             "path] signature, received "
-                          << signature;
             return false;
         }
       }
@@ -102,11 +95,11 @@ class Decoder {
         return ReadEdge(marker, data);
 
       default:
-        if ((value & 0xF0) == underlying_cast(Marker::TinyString)) {
+        if ((value & 0xF0) == utils::UnderlyingCast(Marker::TinyString)) {
           return ReadString(marker, data);
-        } else if ((value & 0xF0) == underlying_cast(Marker::TinyList)) {
+        } else if ((value & 0xF0) == utils::UnderlyingCast(Marker::TinyList)) {
           return ReadList(marker, data);
-        } else if ((value & 0xF0) == underlying_cast(Marker::TinyMap)) {
+        } else if ((value & 0xF0) == utils::UnderlyingCast(Marker::TinyMap)) {
           return ReadMap(marker, data);
         } else {
           return ReadInt(marker, data);
@@ -116,21 +109,19 @@ class Decoder {
   }
 
   /**
-   * Reads a DecodedValue from the available data in the buffer and checks
+   * Reads a Value from the available data in the buffer and checks
    * whether the read data type matches the supplied data type.
    *
-   * @param data pointer to a DecodedValue where the read data should be stored
+   * @param data pointer to a Value where the read data should be stored
    * @param type the expected type that should be read
    * @returns true if data has been written to the data pointer and the type
    *          matches the expected type, false otherwise
    */
-  bool ReadValue(DecodedValue *data, DecodedValue::Type type) {
+  bool ReadValue(Value *data, Value::Type type) {
     if (!ReadValue(data)) {
-      DLOG(WARNING) << "[ReadValue] ReadValue call failed!";
       return false;
     }
     if (data->type() != type) {
-      DLOG(WARNING) << "[ReadValue] Decoded value has wrong type!";
       return false;
     }
     return true;
@@ -148,16 +139,12 @@ class Decoder {
   bool ReadMessageHeader(Signature *signature, Marker *marker) {
     uint8_t values[2];
 
-    VLOG(20) << "[ReadMessageHeader] Start";
-
     if (!buffer_.Read(values, 2)) {
-      DLOG(WARNING) << "[ReadMessageHeader] Marker data missing!";
       return false;
     }
 
     *marker = (Marker)values[0];
     *signature = (Signature)values[1];
-    DLOG(WARNING) << "[ReadMessageHeader] Success";
     return true;
   }
 
@@ -165,400 +152,323 @@ class Decoder {
   Buffer &buffer_;
 
  private:
-  bool ReadNull(const Marker &marker, DecodedValue *data) {
-    VLOG(20) << "[ReadNull] Start";
+  bool ReadNull(const Marker &marker, Value *data) {
     DCHECK(marker == Marker::Null) << "Received invalid marker!";
-    *data = DecodedValue();
-    VLOG(20) << "[ReadNull] Success";
+    *data = Value();
     return true;
   }
 
-  bool ReadBool(const Marker &marker, DecodedValue *data) {
-    VLOG(20) << "[ReadBool] Start";
+  bool ReadBool(const Marker &marker, Value *data) {
     DCHECK(marker == Marker::False || marker == Marker::True)
         << "Received invalid marker!";
     if (marker == Marker::False) {
-      *data = DecodedValue(false);
+      *data = Value(false);
     } else {
-      *data = DecodedValue(true);
+      *data = Value(true);
     }
-    VLOG(20) << "[ReadBool] Success";
     return true;
   }
 
-  bool ReadInt(const Marker &marker, DecodedValue *data) {
-    uint8_t value = underlying_cast(marker);
+  bool ReadInt(const Marker &marker, Value *data) {
+    uint8_t value = utils::UnderlyingCast(marker);
     int64_t ret;
-    VLOG(20) << "[ReadInt] Start";
     if (value >= 240 || value <= 127) {
-      VLOG(20) << "[ReadInt] Found a TinyInt";
       ret = value;
       if (value >= 240) ret -= 256;
     } else if (marker == Marker::Int8) {
-      VLOG(20) << "[ReadInt] Found an Int8";
       int8_t tmp;
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&tmp), sizeof(tmp))) {
-        DLOG(WARNING) << "[ReadInt] Int8 missing data!";
         return false;
       }
       ret = tmp;
     } else if (marker == Marker::Int16) {
-      VLOG(20) << "[ReadInt] Found an Int16";
       int16_t tmp;
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&tmp), sizeof(tmp))) {
-        DLOG(WARNING) << "[ReadInt] Int16 missing data!";
         return false;
       }
-      ret = bswap(tmp);
+      ret = utils::Bswap(tmp);
     } else if (marker == Marker::Int32) {
-      VLOG(20) << "[ReadInt] Found an Int32";
       int32_t tmp;
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&tmp), sizeof(tmp))) {
-        DLOG(WARNING) << "[ReadInt] Int32 missing data!";
         return false;
       }
-      ret = bswap(tmp);
+      ret = utils::Bswap(tmp);
     } else if (marker == Marker::Int64) {
-      VLOG(20) << "[ReadInt] Found an Int64";
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&ret), sizeof(ret))) {
-        DLOG(WARNING) << "[ReadInt] Int64 missing data!";
         return false;
       }
-      ret = bswap(ret);
+      ret = utils::Bswap(ret);
     } else {
-      DLOG(WARNING) << "[ReadInt] Received invalid marker "
-                    << underlying_cast(marker);
       return false;
     }
-    *data = DecodedValue(ret);
-    VLOG(20) << "[ReadInt] Success";
+    *data = Value(ret);
     return true;
   }
 
-  bool ReadDouble(const Marker marker, DecodedValue *data) {
+  bool ReadDouble(const Marker marker, Value *data) {
     uint64_t value;
     double ret;
-    VLOG(20) << "[ReadDouble] Start";
     DCHECK(marker == Marker::Float64) << "Received invalid marker!";
     if (!buffer_.Read(reinterpret_cast<uint8_t *>(&value), sizeof(value))) {
-      DLOG(WARNING) << "[ReadDouble] Missing data!";
       return false;
     }
-    value = bswap(value);
+    value = utils::Bswap(value);
     // cppcheck-suppress invalidPointerCast
     ret = *reinterpret_cast<double *>(&value);
-    *data = DecodedValue(ret);
-    VLOG(20) << "[ReadDouble] Success";
+    *data = Value(ret);
     return true;
   }
 
   int64_t ReadTypeSize(const Marker &marker, const uint8_t type) {
-    uint8_t value = underlying_cast(marker);
-    if ((value & 0xF0) == underlying_cast(MarkerTiny[type])) {
-      VLOG(20) << "[ReadTypeSize] Found a TinyType";
+    uint8_t value = utils::UnderlyingCast(marker);
+    if ((value & 0xF0) == utils::UnderlyingCast(MarkerTiny[type])) {
       return value & 0x0F;
     } else if (marker == Marker8[type]) {
-      VLOG(20) << "[ReadTypeSize] Found a Type8";
       uint8_t tmp;
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&tmp), sizeof(tmp))) {
-        DLOG(WARNING) << "[ReadTypeSize] Type8 missing data!";
         return -1;
       }
       return tmp;
     } else if (marker == Marker16[type]) {
-      VLOG(20) << "[ReadTypeSize] Found a Type16";
       uint16_t tmp;
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&tmp), sizeof(tmp))) {
-        DLOG(WARNING) << "[ReadTypeSize] Type16 missing data!";
         return -1;
       }
-      tmp = bswap(tmp);
+      tmp = utils::Bswap(tmp);
       return tmp;
     } else if (marker == Marker32[type]) {
-      VLOG(20) << "[ReadTypeSize] Found a Type32";
       uint32_t tmp;
       if (!buffer_.Read(reinterpret_cast<uint8_t *>(&tmp), sizeof(tmp))) {
-        DLOG(WARNING) << "[ReadTypeSize] Type32 missing data!";
         return -1;
       }
-      tmp = bswap(tmp);
+      tmp = utils::Bswap(tmp);
       return tmp;
     } else {
-      DLOG(WARNING) << "[ReadTypeSize] Received invalid marker "
-                    << underlying_cast(marker);
       return -1;
     }
   }
 
-  bool ReadString(const Marker &marker, DecodedValue *data) {
-    VLOG(20) << "[ReadString] Start";
+  bool ReadString(const Marker &marker, Value *data) {
+    const int kMaxStackBuffer = 8192;
+    uint8_t buffer[kMaxStackBuffer];
     auto size = ReadTypeSize(marker, MarkerString);
     if (size == -1) {
-      DLOG(WARNING) << "[ReadString] Couldn't get size!";
       return false;
     }
-    std::unique_ptr<uint8_t[]> ret(new uint8_t[size]);
-    if (!buffer_.Read(ret.get(), size)) {
-      DLOG(WARNING) << "[ReadString] Missing data!";
-      return false;
+    // Here we use a temporary buffer on the stack to prevent temporary
+    // allocations. Most of strings that are decoded are small so it makes no
+    // sense to allocate a temporary buffer every time we decode a string. This
+    // way we allocate a temporary buffer only when the string is large. This
+    // wouldn't be necessary if we had full C++17 support. In C++17 we could
+    // preallocate the `buffer[size]` in the destination string `*data =
+    // Value(std::string('\0', size))` and just call
+    // `buffer_.Read(data->ValueString().data())`.
+    if (size < kMaxStackBuffer) {
+      if (!buffer_.Read(buffer, size)) {
+        DLOG(WARNING) << "[ReadString] Missing data!";
+        return false;
+      }
+      *data = Value(std::string(reinterpret_cast<char *>(buffer), size));
+    } else {
+      std::unique_ptr<uint8_t[]> ret(new uint8_t[size]);
+      if (!buffer_.Read(ret.get(), size)) {
+        DLOG(WARNING) << "[ReadString] Missing data!";
+        return false;
+      }
+      *data = Value(std::string(reinterpret_cast<char *>(ret.get()), size));
     }
-    *data =
-        DecodedValue(std::string(reinterpret_cast<char *>(ret.get()), size));
-    VLOG(20) << "[ReadString] Success";
     return true;
   }
 
-  bool ReadList(const Marker &marker, DecodedValue *data) {
-    VLOG(20) << "[ReadList] Start";
+  bool ReadList(const Marker &marker, Value *data) {
     auto size = ReadTypeSize(marker, MarkerList);
     if (size == -1) {
-      DLOG(WARNING) << "[ReadList] Couldn't get size!";
       return false;
     }
-    std::vector<DecodedValue> ret(size);
+    *data = Value(std::vector<Value>(size));
+    auto &ret = data->ValueList();
     for (int64_t i = 0; i < size; ++i) {
       if (!ReadValue(&ret[i])) {
-        DLOG(WARNING) << "[ReadList] Couldn't read element " << i;
         return false;
       }
     }
-    *data = DecodedValue(ret);
-    VLOG(20) << "[ReadList] Success";
     return true;
   }
 
-  bool ReadMap(const Marker &marker, DecodedValue *data) {
-    VLOG(20) << "[ReadMap] Start";
+  bool ReadMap(const Marker &marker, Value *data) {
     auto size = ReadTypeSize(marker, MarkerMap);
     if (size == -1) {
-      DLOG(WARNING) << "[ReadMap] Couldn't get size!";
       return false;
     }
 
-    DecodedValue dv;
-    std::string str;
-    std::map<std::string, DecodedValue> ret;
-    for (int64_t i = 0; i < size; ++i) {
-      if (!ReadValue(&dv)) {
-        DLOG(WARNING) << "[ReadMap] Couldn't read index " << i;
-        return false;
-      }
-      if (dv.type() != DecodedValue::Type::String) {
-        DLOG(WARNING) << "[ReadMap] Index " << i << " isn't a string!";
-        return false;
-      }
-      str = dv.ValueString();
+    Value dv_key, dv_val;
 
-      if (!ReadValue(&dv)) {
-        DLOG(WARNING) << "[ReadMap] Couldn't read element " << i;
+    *data = Value(std::map<std::string, Value>());
+    auto &ret = data->ValueMap();
+    for (int64_t i = 0; i < size; ++i) {
+      if (!ReadValue(&dv_key, Value::Type::String)) {
         return false;
       }
-      ret.insert(std::make_pair(str, dv));
+      if (!ReadValue(&dv_val)) {
+        return false;
+      }
+      ret.emplace(std::move(dv_key.ValueString()), std::move(dv_val));
     }
     if (ret.size() != size) {
-      DLOG(WARNING)
-          << "[ReadMap] The client sent multiple objects with same indexes!";
       return false;
     }
 
-    *data = DecodedValue(ret);
-    VLOG(20) << "[ReadMap] Success";
     return true;
   }
 
-  bool ReadVertex(DecodedValue *data) {
-    DecodedValue dv;
-    DecodedVertex vertex;
-
-    VLOG(20) << "[ReadVertex] Start";
+  bool ReadVertex(Value *data) {
+    Value dv;
+    *data = Value(Vertex());
+    auto &vertex = data->ValueVertex();
 
     // read ID
-    if (!ReadValue(&dv, DecodedValue::Type::Int)) {
-      DLOG(WARNING) << "[ReadVertex] Couldn't read ID!";
+    if (!ReadValue(&dv, Value::Type::Int)) {
       return false;
     }
-    vertex.id = dv.ValueInt();
+    vertex.id = Id::FromInt(dv.ValueInt());
 
     // read labels
-    if (!ReadValue(&dv, DecodedValue::Type::List)) {
-      DLOG(WARNING) << "[ReadVertex] Couldn't read labels!";
+    if (!ReadValue(&dv, Value::Type::List)) {
       return false;
     }
     auto &labels = dv.ValueList();
-    vertex.labels.resize(labels.size());
+    vertex.labels.reserve(labels.size());
     for (size_t i = 0; i < labels.size(); ++i) {
-      if (labels[i].type() != DecodedValue::Type::String) {
-        DLOG(WARNING) << "[ReadVertex] Label has wrong type!";
+      if (labels[i].type() != Value::Type::String) {
         return false;
       }
-      vertex.labels[i] = labels[i].ValueString();
+      vertex.labels.emplace_back(std::move(labels[i].ValueString()));
     }
 
     // read properties
-    if (!ReadValue(&dv, DecodedValue::Type::Map)) {
-      DLOG(WARNING) << "[ReadVertex] Couldn't read properties!";
+    if (!ReadValue(&dv, Value::Type::Map)) {
       return false;
     }
-    vertex.properties = dv.ValueMap();
-
-    *data = DecodedValue(vertex);
-
-    VLOG(20) << "[ReadVertex] Success";
+    vertex.properties = std::move(dv.ValueMap());
 
     return true;
   }
 
-  bool ReadEdge(const Marker &marker, DecodedValue *data) {
+  bool ReadEdge(const Marker &marker, Value *data) {
     uint8_t value;
-    DecodedValue dv;
-    DecodedEdge edge;
-
-    VLOG(20) << "[ReadEdge] Start";
+    Value dv;
+    *data = Value(Edge());
+    auto &edge = data->ValueEdge();
 
     if (!buffer_.Read(&value, 1)) {
-      DLOG(WARNING) << "[ReadEdge] Missing marker and/or signature data!";
       return false;
     }
 
     // check header
     if (marker != Marker::TinyStruct5) {
-      DLOG(WARNING) << "[ReadEdge] Received invalid marker "
-                    << (uint64_t)underlying_cast(marker);
       return false;
     }
-    if (value != underlying_cast(Signature::Relationship)) {
-      DLOG(WARNING) << "[ReadEdge] Received invalid signature " << value;
+    if (value != utils::UnderlyingCast(Signature::Relationship)) {
       return false;
     }
 
     // read ID
-    if (!ReadValue(&dv, DecodedValue::Type::Int)) {
-      DLOG(WARNING) << "[ReadEdge] Couldn't read ID!";
+    if (!ReadValue(&dv, Value::Type::Int)) {
       return false;
     }
-    edge.id = dv.ValueInt();
+    edge.id = Id::FromInt(dv.ValueInt());
 
     // read from
-    if (!ReadValue(&dv, DecodedValue::Type::Int)) {
-      DLOG(WARNING) << "[ReadEdge] Couldn't read from_id!";
+    if (!ReadValue(&dv, Value::Type::Int)) {
       return false;
     }
-    edge.from = dv.ValueInt();
+    edge.from = Id::FromInt(dv.ValueInt());
 
     // read to
-    if (!ReadValue(&dv, DecodedValue::Type::Int)) {
-      DLOG(WARNING) << "[ReadEdge] Couldn't read to_id!";
+    if (!ReadValue(&dv, Value::Type::Int)) {
       return false;
     }
-    edge.to = dv.ValueInt();
+    edge.to = Id::FromInt(dv.ValueInt());
 
     // read type
-    if (!ReadValue(&dv, DecodedValue::Type::String)) {
-      DLOG(WARNING) << "[ReadEdge] Couldn't read type!";
+    if (!ReadValue(&dv, Value::Type::String)) {
       return false;
     }
-    edge.type = dv.ValueString();
+    edge.type = std::move(dv.ValueString());
 
     // read properties
-    if (!ReadValue(&dv, DecodedValue::Type::Map)) {
-      DLOG(WARNING) << "[ReadEdge] Couldn't read properties!";
+    if (!ReadValue(&dv, Value::Type::Map)) {
       return false;
     }
-    edge.properties = dv.ValueMap();
-
-    *data = DecodedValue(edge);
-
-    VLOG(20) << "[ReadEdge] Success";
+    edge.properties = std::move(dv.ValueMap());
 
     return true;
   }
 
-  bool ReadUnboundedEdge(DecodedValue *data) {
-    DecodedValue dv;
-    DecodedUnboundedEdge edge;
-
-    VLOG(20) << "[ReadUnboundedEdge] Start";
+  bool ReadUnboundedEdge(Value *data) {
+    Value dv;
+    *data = Value(UnboundedEdge());
+    auto &edge = data->ValueUnboundedEdge();
 
     // read ID
-    if (!ReadValue(&dv, DecodedValue::Type::Int)) {
-      DLOG(WARNING) << "[ReadUnboundedEdge] Couldn't read ID!";
+    if (!ReadValue(&dv, Value::Type::Int)) {
       return false;
     }
-    edge.id = dv.ValueInt();
+    edge.id = Id::FromInt(dv.ValueInt());
 
     // read type
-    if (!ReadValue(&dv, DecodedValue::Type::String)) {
-      DLOG(WARNING) << "[ReadUnboundedEdge] Couldn't read type!";
+    if (!ReadValue(&dv, Value::Type::String)) {
       return false;
     }
-    edge.type = dv.ValueString();
+    edge.type = std::move(dv.ValueString());
 
     // read properties
-    if (!ReadValue(&dv, DecodedValue::Type::Map)) {
-      DLOG(WARNING) << "[ReadUnboundedEdge] Couldn't read properties!";
+    if (!ReadValue(&dv, Value::Type::Map)) {
       return false;
     }
-    edge.properties = dv.ValueMap();
-
-    *data = DecodedValue(edge);
-
-    VLOG(20) << "[ReadUnboundedEdge] Success";
+    edge.properties = std::move(dv.ValueMap());
 
     return true;
   }
 
-  bool ReadPath(DecodedValue *data) {
-    DecodedValue dv;
-    DecodedPath path;
-
-    VLOG(20) << "[ReadPath] Start";
+  bool ReadPath(Value *data) {
+    Value dv;
+    *data = Value(Path());
+    auto &path = data->ValuePath();
 
     // vertices
-    if (!ReadValue(&dv, DecodedValue::Type::List)) {
-      DLOG(WARNING) << "[ReadPath] Couldn't read vertices!";
+    if (!ReadValue(&dv, Value::Type::List)) {
       return false;
     }
     for (const auto &vertex : dv.ValueList()) {
-      if (vertex.type() != DecodedValue::Type::Vertex) {
-        DLOG(WARNING) << "[ReadPath] Received a '" << vertex.type()
-                      << "' element in the vertices list!";
+      if (vertex.type() != Value::Type::Vertex) {
         return false;
       }
-      path.vertices.emplace_back(vertex.ValueVertex());
+      path.vertices.emplace_back(std::move(vertex.ValueVertex()));
     }
 
     // edges
-    if (!ReadValue(&dv, DecodedValue::Type::List)) {
-      DLOG(WARNING) << "[ReadPath] Couldn't read edges!";
+    if (!ReadValue(&dv, Value::Type::List)) {
       return false;
     }
     for (const auto &edge : dv.ValueList()) {
-      if (edge.type() != DecodedValue::Type::UnboundedEdge) {
-        DLOG(WARNING) << "[ReadPath] Received a '" << edge.type()
-                      << "' element in the edges list!";
+      if (edge.type() != Value::Type::UnboundedEdge) {
         return false;
       }
-      path.edges.emplace_back(edge.ValueUnboundedEdge());
+      path.edges.emplace_back(std::move(edge.ValueUnboundedEdge()));
     }
 
     // indices
-    if (!ReadValue(&dv, DecodedValue::Type::List)) {
-      DLOG(WARNING) << "[ReadPath] Couldn't read indices!";
+    if (!ReadValue(&dv, Value::Type::List)) {
       return false;
     }
     for (const auto &index : dv.ValueList()) {
-      if (index.type() != DecodedValue::Type::Int) {
-        DLOG(WARNING) << "[ReadPath] Received a '" << index.type()
-                      << "' element in the indices list (expected an int)!";
+      if (index.type() != Value::Type::Int) {
         return false;
       }
       path.indices.emplace_back(index.ValueInt());
     }
 
-    *data = DecodedValue(path);
-
-    VLOG(20) << "[ReadPath] Success";
-
     return true;
   }
 };

src/communication/bolt/v1/encoder/base_encoder.hpp

@@ -1,166 +1,224 @@
 #pragma once
 
-#include "communication/bolt/v1/encoder/primitive_encoder.hpp"
-#include "database/graph_db_accessor.hpp"
-#include "query/typed_value.hpp"
+#include <experimental/type_traits>
+
+#include "communication/bolt/v1/codes.hpp"
+#include "communication/bolt/v1/value.hpp"
+#include "utils/bswap.hpp"
+#include "utils/cast.hpp"
+
+static_assert(std::experimental::is_same_v<std::uint8_t, char> ||
+                  std::experimental::is_same_v<std::uint8_t, unsigned char>,
+              "communication::bolt::Encoder requires uint8_t to be "
+              "implemented as char or unsigned char.");
 
 namespace communication::bolt {
 
 /**
- * Bolt BaseEncoder. Subclass of PrimitiveEncoder. Extends it with the
- * capability to encode TypedValues (as well as lists and maps of TypedValues),
- * Edges, Vertices and Paths.
+ * Bolt BaseEncoder. Has public interfaces for writing Bolt encoded data.
+ * Supported types are: Null, Bool, Int, Double, String, List, Map, Vertex, Edge
+ *
+ * The purpose of this class is to stream bolt data into the given Buffer.
  *
  * @tparam Buffer the output buffer that should be used
  */
 template <typename Buffer>
-class BaseEncoder : public PrimitiveEncoder<Buffer> {
+class BaseEncoder {
  public:
-  explicit BaseEncoder(Buffer &buffer) : PrimitiveEncoder<Buffer>(buffer) {}
+  explicit BaseEncoder(Buffer &buffer) : buffer_(buffer) {}
 
-  void WriteList(const std::vector<query::TypedValue> &value) {
-    this->WriteTypeSize(value.size(), MarkerList);
-    for (auto &x : value) WriteTypedValue(x);
+  void WriteRAW(const uint8_t *data, uint64_t len) { buffer_.Write(data, len); }
+
+  void WriteRAW(const char *data, uint64_t len) {
+    WriteRAW((const uint8_t *)data, len);
   }
 
-  /**
-   * Writes a map value.
-   *
-   * @tparam TMap - an iterable of (std::string, TypedValue) pairs.
-   */
-  template <typename TMap>
-  void WriteMap(const TMap &value) {
-    this->WriteTypeSize(value.size(), MarkerMap);
-    for (auto &x : value) {
-      this->WriteString(x.first);
-      WriteTypedValue(x.second);
+  void WriteRAW(const uint8_t data) { WriteRAW(&data, 1); }
+
+  void WriteNull() { WriteRAW(utils::UnderlyingCast(Marker::Null)); }
+
+  void WriteBool(const bool &value) {
+    if (value)
+      WriteRAW(utils::UnderlyingCast(Marker::True));
+    else
+      WriteRAW(utils::UnderlyingCast(Marker::False));
+  }
+
+  void WriteInt(const int64_t &value) {
+    if (value >= -16L && value < 128L) {
+      WriteRAW(static_cast<uint8_t>(value));
+    } else if (value >= -128L && value < -16L) {
+      WriteRAW(utils::UnderlyingCast(Marker::Int8));
+      WriteRAW(static_cast<uint8_t>(value));
+    } else if (value >= -32768L && value < 32768L) {
+      WriteRAW(utils::UnderlyingCast(Marker::Int16));
+      WritePrimitiveValue(static_cast<int16_t>(value));
+    } else if (value >= -2147483648L && value < 2147483648L) {
+      WriteRAW(utils::UnderlyingCast(Marker::Int32));
+      WritePrimitiveValue(static_cast<int32_t>(value));
+    } else {
+      WriteRAW(utils::UnderlyingCast(Marker::Int64));
+      WritePrimitiveValue(value);
     }
   }
 
-  void WriteVertex(const VertexAccessor &vertex) {
-    this->WriteRAW(underlying_cast(Marker::TinyStruct) + 3);
-    this->WriteRAW(underlying_cast(Signature::Node));
-    WriteUInt(vertex.gid());
+  void WriteDouble(const double &value) {
+    WriteRAW(utils::UnderlyingCast(Marker::Float64));
+    uint64_t tmp = utils::MemcpyCast<uint64_t>(value);
+    WritePrimitiveValue(tmp);
+  }
+
+  void WriteTypeSize(const size_t size, const uint8_t typ) {
+    if (size <= 15) {
+      uint8_t len = size;
+      len &= 0x0F;
+      WriteRAW(utils::UnderlyingCast(MarkerTiny[typ]) + len);
+    } else if (size <= 255) {
+      uint8_t len = size;
+      WriteRAW(utils::UnderlyingCast(Marker8[typ]));
+      WriteRAW(len);
+    } else if (size <= 65535) {
+      uint16_t len = size;
+      WriteRAW(utils::UnderlyingCast(Marker16[typ]));
+      WritePrimitiveValue(len);
+    } else {
+      uint32_t len = size;
+      WriteRAW(utils::UnderlyingCast(Marker32[typ]));
+      WritePrimitiveValue(len);
+    }
+  }
+
+  void WriteString(const std::string &value) {
+    WriteTypeSize(value.size(), MarkerString);
+    WriteRAW(value.c_str(), value.size());
+  }
+  void WriteList(const std::vector<Value> &value) {
+    WriteTypeSize(value.size(), MarkerList);
+    for (auto &x : value) WriteValue(x);
+  }
+
+  void WriteMap(const std::map<std::string, Value> &value) {
+    WriteTypeSize(value.size(), MarkerMap);
+    for (auto &x : value) {
+      WriteString(x.first);
+      WriteValue(x.second);
+    }
+  }
+
+  void WriteVertex(const Vertex &vertex) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
+    WriteRAW(utils::UnderlyingCast(Signature::Node));
+    WriteInt(vertex.id.AsInt());
 
     // write labels
-    const auto &labels = vertex.labels();
-    this->WriteTypeSize(labels.size(), MarkerList);
-    for (const auto &label : labels)
-      this->WriteString(vertex.db_accessor().LabelName(label));
+    const auto &labels = vertex.labels;
+    WriteTypeSize(labels.size(), MarkerList);
+    for (const auto &label : labels) WriteString(label);
 
     // write properties
-    const auto &props = vertex.Properties();
-    this->WriteTypeSize(props.size(), MarkerMap);
+    const auto &props = vertex.properties;
+    WriteTypeSize(props.size(), MarkerMap);
     for (const auto &prop : props) {
-      this->WriteString(vertex.db_accessor().PropertyName(prop.first));
-      WriteTypedValue(prop.second);
+      WriteString(prop.first);
+      WriteValue(prop.second);
     }
   }
 
-  void WriteEdge(const EdgeAccessor &edge, bool unbound = false) {
-    this->WriteRAW(underlying_cast(Marker::TinyStruct) + (unbound ? 3 : 5));
-    this->WriteRAW(underlying_cast(unbound ? Signature::UnboundRelationship
+  void WriteEdge(const Edge &edge, bool unbound = false) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + (unbound ? 3 : 5));
+    WriteRAW(utils::UnderlyingCast(unbound ? Signature::UnboundRelationship
                                            : Signature::Relationship));
 
-    WriteUInt(edge.gid());
+    WriteInt(edge.id.AsInt());
     if (!unbound) {
-      WriteUInt(edge.from().gid());
-      WriteUInt(edge.to().gid());
+      WriteInt(edge.from.AsInt());
+      WriteInt(edge.to.AsInt());
     }
 
-    // write type
-    this->WriteString(edge.db_accessor().EdgeTypeName(edge.EdgeType()));
+    WriteString(edge.type);
 
-    // write properties
-    const auto &props = edge.Properties();
-    this->WriteTypeSize(props.size(), MarkerMap);
+    const auto &props = edge.properties;
+    WriteTypeSize(props.size(), MarkerMap);
     for (const auto &prop : props) {
-      this->WriteString(edge.db_accessor().PropertyName(prop.first));
-      WriteTypedValue(prop.second);
+      WriteString(prop.first);
+      WriteValue(prop.second);
     }
   }
 
-  void WritePath(const query::Path &path) {
-    // Prepare the data structures to be written.
-    //
-    // Unique vertices in the path.
-    std::vector<VertexAccessor> vertices;
-    // Unique edges in the path.
-    std::vector<EdgeAccessor> edges;
-    // Indices that map path positions to vertices/edges elements. Positive
-    // indices for left-to-right directionality and negative for right-to-left.
-    std::vector<int> indices;
+  void WriteEdge(const UnboundedEdge &edge) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
+    WriteRAW(utils::UnderlyingCast(Signature::UnboundRelationship));
 
-    // Helper function. Looks for the given element in the collection. If found
-    // it puts it's index into `indices`. Otherwise emplaces the given element
-    // into the collection and puts that index into `indices`. A multiplier is
-    // added to switch between positive and negative indices (that define edge
-    // direction).
-    auto add_element = [&indices](auto &collection, const auto &element,
-                                  int multiplier, int offset) {
-      auto found = std::find(collection.begin(), collection.end(), element);
-      indices.emplace_back(multiplier *
-                           (std::distance(collection.begin(), found) + offset));
-      if (found == collection.end()) collection.emplace_back(element);
-    };
+    WriteInt(edge.id.AsInt());
 
-    vertices.emplace_back(path.vertices()[0]);
-    for (uint i = 0; i < path.size(); i++) {
-      const auto &e = path.edges()[i];
-      const auto &v = path.vertices()[i + 1];
-      add_element(edges, e, e.to_is(v) ? 1 : -1, 1);
-      add_element(vertices, v, 1, 0);
+    WriteString(edge.type);
+
+    const auto &props = edge.properties;
+    WriteTypeSize(props.size(), MarkerMap);
+    for (const auto &prop : props) {
+      WriteString(prop.first);
+      WriteValue(prop.second);
     }
-
-    // Write data.
-    this->WriteRAW(underlying_cast(Marker::TinyStruct) + 3);
-    this->WriteRAW(underlying_cast(Signature::Path));
-    this->WriteTypeSize(vertices.size(), MarkerList);
-    for (auto &v : vertices) WriteVertex(v);
-    this->WriteTypeSize(edges.size(), MarkerList);
-    for (auto &e : edges) WriteEdge(e, true);
-    this->WriteTypeSize(indices.size(), MarkerList);
-    for (auto &i : indices) this->WriteInt(i);
   }
 
-  void WriteTypedValue(const query::TypedValue &value) {
+  void WritePath(const Path &path) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
+    WriteRAW(utils::UnderlyingCast(Signature::Path));
+    WriteTypeSize(path.vertices.size(), MarkerList);
+    for (auto &v : path.vertices) WriteVertex(v);
+    WriteTypeSize(path.edges.size(), MarkerList);
+    for (auto &e : path.edges) WriteEdge(e);
+    WriteTypeSize(path.indices.size(), MarkerList);
+    for (auto &i : path.indices) WriteInt(i);
+  }
+
+  void WriteValue(const Value &value) {
     switch (value.type()) {
-      case query::TypedValue::Type::Null:
-        this->WriteNull();
+      case Value::Type::Null:
+        WriteNull();
         break;
-      case query::TypedValue::Type::Bool:
-        this->WriteBool(value.Value<bool>());
+      case Value::Type::Bool:
+        WriteBool(value.ValueBool());
         break;
-      case query::TypedValue::Type::Int:
-        this->WriteInt(value.Value<int64_t>());
+      case Value::Type::Int:
+        WriteInt(value.ValueInt());
         break;
-      case query::TypedValue::Type::Double:
-        this->WriteDouble(value.Value<double>());
+      case Value::Type::Double:
+        WriteDouble(value.ValueDouble());
         break;
-      case query::TypedValue::Type::String:
-        this->WriteString(value.Value<std::string>());
+      case Value::Type::String:
+        WriteString(value.ValueString());
         break;
-      case query::TypedValue::Type::List:
-        WriteList(value.Value<std::vector<query::TypedValue>>());
+      case Value::Type::List:
+        WriteList(value.ValueList());
         break;
-      case query::TypedValue::Type::Map:
-        WriteMap(value.Value<std::map<std::string, query::TypedValue>>());
+      case Value::Type::Map:
+        WriteMap(value.ValueMap());
         break;
-      case query::TypedValue::Type::Vertex:
-        WriteVertex(value.Value<VertexAccessor>());
+      case Value::Type::Vertex:
+        WriteVertex(value.ValueVertex());
         break;
-      case query::TypedValue::Type::Edge:
-        WriteEdge(value.Value<EdgeAccessor>());
+      case Value::Type::Edge:
+        WriteEdge(value.ValueEdge());
         break;
-      case query::TypedValue::Type::Path:
+      case Value::Type::UnboundedEdge:
+        WriteEdge(value.ValueUnboundedEdge());
+        break;
+      case Value::Type::Path:
         WritePath(value.ValuePath());
         break;
     }
   }
 
+ protected:
+  Buffer &buffer_;
+
  private:
-  void WriteUInt(const uint64_t &value) {
-    this->WriteInt(*reinterpret_cast<const int64_t *>(&value));
+  template <class T>
+  void WritePrimitiveValue(T value) {
+    value = utils::Bswap(value);
+    WriteRAW(reinterpret_cast<const uint8_t *>(&value), sizeof(value));
   }
 };
+
 }  // namespace communication::bolt

src/communication/bolt/v1/encoder/chunked_encoder_buffer.hpp

@@ -12,18 +12,14 @@
 
 namespace communication::bolt {
 
-// TODO: implement a better flushing strategy + optimize memory allocations!
-// TODO: see how bolt splits message over more TCP packets
-//       -> test for more TCP packets!
-
 /**
  * @brief ChunkedEncoderBuffer
  *
- * Has methods for writing and flushing data into the message buffer.
+ * Has methods for writing data into the chunk buffer.
  *
- * Writing data stores data in the internal buffer and flushing data sends
- * the currently stored data to the OutputStream. Chunking prepends data length
- * and appends chunk end marker (0x00 0x00).
+ * Writing data stores data in the internal buffer and chunking data sends
+ * the currently stored data to the OutputStream. Flushing prepends the data
+ * length to each chunk.
  *
  * | chunk header | --- chunk --- | another chunk | -- end marker -- |
  * | ------- whole chunk -------- |  whole chunk  | chunk of size 0  |
@@ -31,8 +27,13 @@ namespace communication::bolt {
  * | --------------------------- message --------------------------- |
  * | --------------------------- buffer  --------------------------- |
  *
- * The current implementation stores the whole message into a single buffer
- * which is std::vector.
+ * NOTE: To send a message end marker (chunk of size 0) it is necessary to
+ * explicitly call the `Flush` method on an empty buffer. In that way the user
+ * can control when the message is over and the whole message isn't
+ * unnecessarily buffered in memory.
+ *
+ * The current implementation stores only a single chunk into memory and sends
+ * it immediately to the output stream when new data arrives.
  *
  * @tparam TOutputStream the output stream that should be used
  */
@@ -44,176 +45,78 @@ class ChunkedEncoderBuffer {
 
   /**
    * Writes n values into the buffer. If n is bigger than whole chunk size
-   * values are automatically chunked.
+   * values are automatically chunked and sent to the output buffer.
    *
    * @param values data array of bytes
    * @param n is the number of bytes
    */
   void Write(const uint8_t *values, size_t n) {
-    int written = 0;
+    size_t written = 0;
 
     while (n > 0) {
-      // Define number of bytes  which will be copied into chunk because
-      // chunk is a fixed length array.
-      auto size = n < WHOLE_CHUNK_SIZE - pos_ ? n : WHOLE_CHUNK_SIZE - pos_;
+      // Define the number of bytes which will be copied into the chunk because
+      // the internal storage is a fixed length array.
+      size_t size =
+          n < kChunkMaxDataSize - have_ ? n : kChunkMaxDataSize - have_;
 
-      // Copy size values to chunk array.
-      std::memcpy(chunk_.data() + pos_, values + written, size);
+      // Copy `size` values to the chunk array.
+      std::memcpy(chunk_.data() + kChunkHeaderSize + have_, values + written,
+                  size);
 
-      // Update positions. Position pointer and incomming size have to be
-      // updated because all incomming values have to be processed.
+      // Update positions. The position pointer and incoming size have to be
+      // updated because all incoming values have to be processed.
       written += size;
-      pos_ += size;
+      have_ += size;
       n -= size;
 
-      // If chunk is full copy it into the message buffer and make space for
-      // other incomming values that are left in the values array.
-      if (pos_ == WHOLE_CHUNK_SIZE) Chunk(false);
+      // If the chunk is full, send it to the output stream and clear the
+      // internal storage to make space for other incoming values that are left
+      // in the values array.
+      if (have_ == kChunkMaxDataSize) Flush(true);
     }
   }
 
   /**
-   * Wrap the data from chunk array (append header and end marker) and put
-   * the whole chunk into the buffer.
+   * Wrap the data from the chunk array (append the size header) and send
+   * the whole chunk into the output stream.
    *
-   * @param message_done if set to true then chunk appends an end message
-   *                     marker to the chunk, should always be set to true
-   *                     (the default value), false is used only internally
+   * @param have_more this parameter is passed to the underlying output stream
+   *                  `Write` method to indicate wether we have more data
+   *                  waiting to be sent (in order to optimize network packets)
    */
-  void Chunk(bool message_done = true) {
-    // 1. Write the size of the chunk (CHUNK HEADER).
-    uint16_t size = pos_ - CHUNK_HEADER_SIZE;
-    // Write the higher byte.
-    chunk_[0] = size >> 8;
-    // Write the lower byte.
-    chunk_[1] = size & 0xFF;
+  bool Flush(bool have_more = false) {
+    // Write the size of the chunk.
+    chunk_[0] = have_ >> 8;
+    chunk_[1] = have_ & 0xFF;
 
-    // 2. Determine the final size for the end marker.
-    if (message_done) size_ += 2;
-
-    // 3. Copy whole chunk into the buffer.
-    size_ += pos_;
-    std::copy(chunk_.begin(), chunk_.begin() + pos_,
-              std::back_inserter(buffer_));
-
-    // 4. Insert message end marker.
-    if (message_done) {
-      buffer_.push_back(0);
-      buffer_.push_back(0);
-    }
-
-    // 5. Remember first chunk size.
-    if (first_chunk_size_ == -1) first_chunk_size_ = size_;
-
-    // 6. Cleanup.
-    //     * pos_ has to be reset to the size of chunk header (reserved
-    //       space for the chunk size)
-    pos_ = CHUNK_HEADER_SIZE;
-  }
-
-  /**
-   * Sends the whole buffer(message) to the client.
-   * @returns true if the data was successfully sent to the client
-   *          false otherwise
-   */
-  bool Flush() {
-    // Call chunk if is hasn't been called.
-    if (pos_ > CHUNK_HEADER_SIZE) Chunk();
-
-    // Early return if buffer is empty because there is nothing to write.
-    if (size_ == 0) return true;
-
-    // Flush the whole buffer.
-    if (!output_stream_.Write(buffer_.data() + offset_, size_ - offset_))
-      return false;
-    DLOG(INFO) << "Flushed << " << size_ << " bytes.";
+    // Write the data to the stream.
+    auto ret = output_stream_.Write(chunk_.data(), kChunkHeaderSize + have_,
+                                    have_more);
 
     // Cleanup.
     Clear();
-    return true;
+
+    return ret;
   }
 
-  /**
-   * Sends only the first message chunk in the buffer to the client.
-   * @returns true if the data was successfully sent to the client
-   *          false otherwise
-   */
-  bool FlushFirstChunk() {
-    // Call chunk if is hasn't been called.
-    if (pos_ > CHUNK_HEADER_SIZE) Chunk();
-
-    // Early return if buffer is empty because there is nothing to write.
-    if (size_ == 0) return false;
-
-    // Early return if there is no first chunk
-    if (first_chunk_size_ == -1) return false;
-
-    // Flush the first chunk
-    if (!output_stream_.Write(buffer_.data(), first_chunk_size_)) return false;
-    DLOG(INFO) << "Flushed << " << first_chunk_size_ << " bytes.";
-
-    // Cleanup.
-    // Here we use offset as a method of deleting from the front of the
-    // data vector. Because the first chunk will always be relatively
-    // small comparing to the rest of the data it is more optimal just to
-    // skip the first part of the data than to shift everything in the
-    // vector buffer.
-    offset_ = first_chunk_size_;
-    first_chunk_size_ = -1;
-    return true;
-  }
-
-  /**
-   * Clears the internal buffers.
-   */
-  void Clear() {
-    buffer_.clear();
-    size_ = 0;
-    first_chunk_size_ = -1;
-    offset_ = 0;
-  }
+  /** Clears the internal buffers. */
+  void Clear() { have_ = 0; }
 
   /**
    * Returns a boolean indicating whether there is data in the buffer.
    * @returns true if there is data in the buffer,
    *          false otherwise
    */
-  bool HasData() { return buffer_.size() > 0 || size_ > 0; }
+  bool HasData() { return have_ > 0; }
 
  private:
-  /**
-   * The output stream used.
-   */
+  // The output stream used.
   TOutputStream &output_stream_;
 
-  /**
-   * Buffer for a single chunk.
-   */
-  std::array<uint8_t, WHOLE_CHUNK_SIZE> chunk_;
+  // Buffer for a single chunk.
+  std::array<uint8_t, kChunkWholeSize> chunk_;
 
-  /**
-   * Buffer for the message which will be sent to a client.
-   */
-  std::vector<uint8_t> buffer_;
-
-  /**
-   * Size of the message.
-   */
-  size_t size_{0};
-
-  /**
-   * Size of first chunk in the buffer.
-   */
-  int32_t first_chunk_size_{-1};
-
-  /**
-   * Offset from the start of the buffer.
-   */
-  size_t offset_{0};
-
-  /**
-   * Current position in chunk array.
-   */
-  size_t pos_{CHUNK_HEADER_SIZE};
+  // Amount of data in chunk array.
+  size_t have_{0};
 };
 }  // namespace communication::bolt

src/communication/bolt/v1/encoder/client_encoder.hpp

@@ -39,11 +39,15 @@ class ClientEncoder : private BaseEncoder<Buffer> {
    *          when flushing, false otherwise
    */
   bool MessageInit(const std::string client_name,
-                   const std::map<std::string, query::TypedValue> &auth_token) {
-    WriteRAW(underlying_cast(Marker::TinyStruct2));
-    WriteRAW(underlying_cast(Signature::Init));
+                   const std::map<std::string, Value> &auth_token) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
+    WriteRAW(utils::UnderlyingCast(Signature::Init));
     WriteString(client_name);
     WriteMap(auth_token);
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 
@@ -61,20 +65,20 @@ class ClientEncoder : private BaseEncoder<Buffer> {
    * @returns true if the data was successfully sent to the client
    *          when flushing, false otherwise
    */
-  bool MessageRun(const std::string statement,
-                  const std::map<std::string, query::TypedValue> &parameters,
-                  bool flush = true) {
-    WriteRAW(underlying_cast(Marker::TinyStruct2));
-    WriteRAW(underlying_cast(Signature::Run));
+  bool MessageRun(const std::string &statement,
+                  const std::map<std::string, Value> &parameters,
+                  bool have_more = true) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
+    WriteRAW(utils::UnderlyingCast(Signature::Run));
     WriteString(statement);
     WriteMap(parameters);
-    if (flush) {
-      return buffer_.Flush();
-    } else {
-      buffer_.Chunk();
-      // Chunk always succeeds, so return true
-      return true;
-    }
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done. Here we
+    // forward the `have_more` flag to indicate if there is more data that will
+    // be sent.
+    return buffer_.Flush(have_more);
   }
 
   /**
@@ -88,8 +92,12 @@ class ClientEncoder : private BaseEncoder<Buffer> {
    *          when flushing, false otherwise
    */
   bool MessageDiscardAll() {
-    WriteRAW(underlying_cast(Marker::TinyStruct));
-    WriteRAW(underlying_cast(Signature::DiscardAll));
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
+    WriteRAW(utils::UnderlyingCast(Signature::DiscardAll));
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 
@@ -104,8 +112,12 @@ class ClientEncoder : private BaseEncoder<Buffer> {
    *          when flushing, false otherwise
    */
   bool MessagePullAll() {
-    WriteRAW(underlying_cast(Marker::TinyStruct));
-    WriteRAW(underlying_cast(Signature::PullAll));
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
+    WriteRAW(utils::UnderlyingCast(Signature::PullAll));
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 
@@ -120,8 +132,12 @@ class ClientEncoder : private BaseEncoder<Buffer> {
    *          when flushing, false otherwise
    */
   bool MessageAckFailure() {
-    WriteRAW(underlying_cast(Marker::TinyStruct));
-    WriteRAW(underlying_cast(Signature::AckFailure));
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
+    WriteRAW(utils::UnderlyingCast(Signature::AckFailure));
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 
@@ -136,9 +152,13 @@ class ClientEncoder : private BaseEncoder<Buffer> {
    *          when flushing, false otherwise
    */
   bool MessageReset() {
-    WriteRAW(underlying_cast(Marker::TinyStruct));
-    WriteRAW(underlying_cast(Signature::Reset));
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
+    WriteRAW(utils::UnderlyingCast(Signature::Reset));
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 };
-}
+}  // namespace communication::bolt

src/communication/bolt/v1/encoder/encoder.hpp

@@ -24,10 +24,7 @@ class Encoder : private BaseEncoder<Buffer> {
   Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
 
   /**
-   * Writes a Record message. This method only stores data in the Buffer.
-   * It doesn't send the values out to the Buffer (Chunk is called at the
-   * end of this method). To send the values Flush method has to be called
-   * after this method.
+   * Sends a Record message.
    *
    * From the Bolt v1 documentation:
    *   RecordMessage (signature=0x71) {
@@ -36,11 +33,18 @@ class Encoder : private BaseEncoder<Buffer> {
    *
    * @param values the fields list object that should be sent
    */
-  void MessageRecord(const std::vector<query::TypedValue> &values) {
-    WriteRAW(underlying_cast(Marker::TinyStruct1));
-    WriteRAW(underlying_cast(Signature::Record));
+  bool MessageRecord(const std::vector<Value> &values) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
+    WriteRAW(utils::UnderlyingCast(Signature::Record));
     WriteList(values);
-    buffer_.Chunk();
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done. Here we tell
+    // the buffer that there will be more data because this is a Record message
+    // and it will surely be followed by either a Record, Success or Failure
+    // message.
+    return buffer_.Flush(true);
   }
 
   /**
@@ -52,22 +56,18 @@ class Encoder : private BaseEncoder<Buffer> {
    *   }
    *
    * @param metadata the metadata map object that should be sent
-   * @param flush should method flush the socket
    * @returns true if the data was successfully sent to the client
    *          when flushing, false otherwise
    */
-  bool MessageSuccess(const std::map<std::string, query::TypedValue> &metadata,
-                      bool flush = true) {
-    WriteRAW(underlying_cast(Marker::TinyStruct1));
-    WriteRAW(underlying_cast(Signature::Success));
+  bool MessageSuccess(const std::map<std::string, Value> &metadata) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
+    WriteRAW(utils::UnderlyingCast(Signature::Success));
     WriteMap(metadata);
-    if (flush) {
-      return buffer_.Flush();
-    } else {
-      buffer_.Chunk();
-      // Chunk always succeeds, so return true
-      return true;
-    }
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
+    return buffer_.Flush();
   }
 
   /**
@@ -79,7 +79,7 @@ class Encoder : private BaseEncoder<Buffer> {
    *          false otherwise
    */
   bool MessageSuccess() {
-    std::map<std::string, query::TypedValue> metadata;
+    std::map<std::string, Value> metadata;
     return MessageSuccess(metadata);
   }
 
@@ -95,11 +95,14 @@ class Encoder : private BaseEncoder<Buffer> {
    * @returns true if the data was successfully sent to the client,
    *          false otherwise
    */
-  bool MessageFailure(
-      const std::map<std::string, query::TypedValue> &metadata) {
-    WriteRAW(underlying_cast(Marker::TinyStruct1));
-    WriteRAW(underlying_cast(Signature::Failure));
+  bool MessageFailure(const std::map<std::string, Value> &metadata) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
+    WriteRAW(utils::UnderlyingCast(Signature::Failure));
     WriteMap(metadata);
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 
@@ -115,11 +118,14 @@ class Encoder : private BaseEncoder<Buffer> {
    * @returns true if the data was successfully sent to the client,
    *          false otherwise
    */
-  bool MessageIgnored(
-      const std::map<std::string, query::TypedValue> &metadata) {
-    WriteRAW(underlying_cast(Marker::TinyStruct1));
-    WriteRAW(underlying_cast(Signature::Ignored));
+  bool MessageIgnored(const std::map<std::string, Value> &metadata) {
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
+    WriteRAW(utils::UnderlyingCast(Signature::Ignored));
     WriteMap(metadata);
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 
@@ -132,8 +138,12 @@ class Encoder : private BaseEncoder<Buffer> {
    *          false otherwise
    */
   bool MessageIgnored() {
-    WriteRAW(underlying_cast(Marker::TinyStruct));
-    WriteRAW(underlying_cast(Signature::Ignored));
+    WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
+    WriteRAW(utils::UnderlyingCast(Signature::Ignored));
+    // Try to flush all remaining data in the buffer, but tell it that we will
+    // send more data (the end of message chunk).
+    if (!buffer_.Flush(true)) return false;
+    // Flush an empty chunk to indicate that the message is done.
     return buffer_.Flush();
   }
 };

src/communication/bolt/v1/encoder/primitive_encoder.hpp

@@ -1,137 +0,0 @@
-#pragma once
-
-#include <string>
-
-#include "communication/bolt/v1/codes.hpp"
-#include "storage/property_value.hpp"
-#include "utils/bswap.hpp"
-
-namespace communication::bolt {
-
-/**
- * Bolt PrimitiveEncoder. Has public interfaces for writing Bolt encoded data.
- * Supported types are: Null, Bool, Int, Double, String and PropertyValue.
- *
- * Bolt encoding is used both for streaming data to network clients and for
- * database durability.
- *
- * @tparam Buffer the output buffer that should be used
- */
-template <typename Buffer>
-class PrimitiveEncoder {
- public:
-  PrimitiveEncoder(Buffer &buffer) : buffer_(buffer) {}
-
-  void WriteRAW(const uint8_t *data, uint64_t len) { buffer_.Write(data, len); }
-
-  void WriteRAW(const char *data, uint64_t len) {
-    WriteRAW((const uint8_t *)data, len);
-  }
-
-  void WriteRAW(const uint8_t data) { WriteRAW(&data, 1); }
-
-  template <class T>
-  void WriteValue(T value) {
-    value = bswap(value);
-    WriteRAW(reinterpret_cast<const uint8_t *>(&value), sizeof(value));
-  }
-
-  void WriteNull() { WriteRAW(underlying_cast(Marker::Null)); }
-
-  void WriteBool(const bool &value) {
-    if (value)
-      WriteRAW(underlying_cast(Marker::True));
-    else
-      WriteRAW(underlying_cast(Marker::False));
-  }
-
-  void WriteInt(const int64_t &value) {
-    if (value >= -16L && value < 128L) {
-      WriteRAW(static_cast<uint8_t>(value));
-    } else if (value >= -128L && value < -16L) {
-      WriteRAW(underlying_cast(Marker::Int8));
-      WriteRAW(static_cast<uint8_t>(value));
-    } else if (value >= -32768L && value < 32768L) {
-      WriteRAW(underlying_cast(Marker::Int16));
-      WriteValue(static_cast<int16_t>(value));
-    } else if (value >= -2147483648L && value < 2147483648L) {
-      WriteRAW(underlying_cast(Marker::Int32));
-      WriteValue(static_cast<int32_t>(value));
-    } else {
-      WriteRAW(underlying_cast(Marker::Int64));
-      WriteValue(value);
-    }
-  }
-
-  void WriteDouble(const double &value) {
-    WriteRAW(underlying_cast(Marker::Float64));
-    WriteValue(*reinterpret_cast<const int64_t *>(&value));
-  }
-
-  void WriteTypeSize(const size_t size, const uint8_t typ) {
-    if (size <= 15) {
-      uint8_t len = size;
-      len &= 0x0F;
-      WriteRAW(underlying_cast(MarkerTiny[typ]) + len);
-    } else if (size <= 255) {
-      uint8_t len = size;
-      WriteRAW(underlying_cast(Marker8[typ]));
-      WriteRAW(len);
-    } else if (size <= 65535) {
-      uint16_t len = size;
-      WriteRAW(underlying_cast(Marker16[typ]));
-      WriteValue(len);
-    } else {
-      uint32_t len = size;
-      WriteRAW(underlying_cast(Marker32[typ]));
-      WriteValue(len);
-    }
-  }
-
-  void WriteString(const std::string &value) {
-    WriteTypeSize(value.size(), MarkerString);
-    WriteRAW(value.c_str(), value.size());
-  }
-
-  void WritePropertyValue(const PropertyValue &value) {
-    auto write_list = [this](const std::vector<PropertyValue> &value) {
-      WriteTypeSize(value.size(), MarkerList);
-      for (auto &x : value) WritePropertyValue(x);
-    };
-
-    auto write_map = [this](const std::map<std::string, PropertyValue> &value) {
-      WriteTypeSize(value.size(), MarkerMap);
-      for (auto &x : value) {
-        WriteString(x.first);
-        WritePropertyValue(x.second);
-      }
-    };
-    switch (value.type()) {
-      case PropertyValue::Type::Null:
-        WriteNull();
-        break;
-      case PropertyValue::Type::Bool:
-        WriteBool(value.Value<bool>());
-        break;
-      case PropertyValue::Type::Int:
-        WriteInt(value.Value<int64_t>());
-        break;
-      case PropertyValue::Type::Double:
-        WriteDouble(value.Value<double>());
-        break;
-      case PropertyValue::Type::String:
-        WriteString(value.Value<std::string>());
-        break;
-      case PropertyValue::Type::List:
-        write_list(value.Value<std::vector<PropertyValue>>());
-        break;
-      case PropertyValue::Type::Map:
-        write_map(value.Value<std::map<std::string, PropertyValue>>());
-        break;
-    }
-  }
-
- protected:
-  Buffer &buffer_;
-};
-}  // namespace communication::bolt

src/communication/bolt/v1/encoder/result_stream.hpp

@@ -1,75 +0,0 @@
-#pragma once
-
-#include "communication/bolt/v1/encoder/chunked_encoder_buffer.hpp"
-#include "communication/bolt/v1/encoder/encoder.hpp"
-#include "query/typed_value.hpp"
-
-namespace communication::bolt {
-
-/**
- * A high level API for streaming a Bolt response. Exposes
- * functionalities used by the compiler and query plans (which
- * should not use any lower level API).
- *
- * @tparam Encoder Encoder used.
- */
-template <typename Encoder>
-class ResultStream {
- public:
-  ResultStream(Encoder &encoder) : encoder_(encoder) {}
-
-  /**
-   * Writes a header. Typically a header is something like:
-   * [ "Header1", "Header2", "Header3" ]
-   *
-   * @param fields the header fields that should be sent.
-   */
-  void Header(const std::vector<std::string> &fields) {
-    std::vector<query::TypedValue> vec;
-    std::map<std::string, query::TypedValue> data;
-    for (auto &i : fields) vec.push_back(query::TypedValue(i));
-    data.insert(std::make_pair(std::string("fields"), query::TypedValue(vec)));
-    // this message shouldn't send directly to the client because if an error
-    // happened the client will receive two messages (success and failure)
-    // instead of only one
-    encoder_.MessageSuccess(data, false);
-  }
-
-  /**
-   * Writes a result. Typically a result is something like:
-   * [
-   *     Value1,
-   *     Value2,
-   *     Value3
-   * ]
-   * NOTE: The result fields should be in the same ordering that the header
-   *       fields were sent in.
-   *
-   * @param values the values that should be sent
-   */
-  void Result(std::vector<query::TypedValue> &values) {
-    encoder_.MessageRecord(values);
-  }
-
-  /**
-   * Writes a summary. Typically a summary is something like:
-   * {
-   *    "type" : "r" | "rw" | ...,
-   *    "stats": {
-   *        "nodes_created": 12,
-   *        "nodes_deleted": 0
-   *     }
-   * }
-   *
-   * @param summary the summary map object that should be sent
-   */
-  void Summary(const std::map<std::string, query::TypedValue> &summary) {
-    // at this point message should not flush the socket so
-    // here is false because chunk has to be called instead of flush
-    encoder_.MessageSuccess(summary, false);
-  }
-
- private:
-  Encoder &encoder_;
-};
-}

src/communication/bolt/v1/session.hpp

@@ -7,29 +7,17 @@
 #include "communication/bolt/v1/constants.hpp"
 #include "communication/bolt/v1/decoder/chunked_decoder_buffer.hpp"
 #include "communication/bolt/v1/decoder/decoder.hpp"
+#include "communication/bolt/v1/encoder/chunked_encoder_buffer.hpp"
 #include "communication/bolt/v1/encoder/encoder.hpp"
-#include "communication/bolt/v1/encoder/result_stream.hpp"
 #include "communication/bolt/v1/state.hpp"
 #include "communication/bolt/v1/states/error.hpp"
 #include "communication/bolt/v1/states/executing.hpp"
 #include "communication/bolt/v1/states/handshake.hpp"
 #include "communication/bolt/v1/states/init.hpp"
-#include "communication/buffer.hpp"
-#include "database/graph_db.hpp"
-#include "query/interpreter.hpp"
-#include "threading/sync/spinlock.hpp"
-#include "transactions/transaction.hpp"
 #include "utils/exceptions.hpp"
 
 namespace communication::bolt {
 
-/** Encapsulates Dbms and Interpreter that are passed through the network server
- * and worker to the session. */
-struct SessionData {
-  database::MasterBase &db;
-  query::Interpreter interpreter{db};
-};
-
 /**
  * Bolt Session Exception
  *
@@ -51,21 +39,30 @@ class SessionException : public utils::BasicException {
 template <typename TInputStream, typename TOutputStream>
 class Session {
  public:
-  using ResultStreamT =
-      ResultStream<Encoder<ChunkedEncoderBuffer<TOutputStream>>>;
+  using TEncoder = Encoder<ChunkedEncoderBuffer<TOutputStream>>;
 
-  Session(SessionData &data, TInputStream &input_stream,
-          TOutputStream &output_stream)
-      : db_(data.db),
-        interpreter_(data.interpreter),
-        input_stream_(input_stream),
-        output_stream_(output_stream) {}
+  Session(TInputStream *input_stream, TOutputStream *output_stream)
+      : input_stream_(*input_stream), output_stream_(*output_stream) {}
 
-  ~Session() {
-    if (db_accessor_) {
-      Abort();
-    }
-  }
+  virtual ~Session() {}
+
+  /**
+   * Process the given `query` with `params`.
+   */
+  virtual std::vector<std::string> Interpret(
+      const std::string &query, const std::map<std::string, Value> &params) = 0;
+
+  /**
+   * Put results of the processed query in the `encoder`.
+   */
+  virtual std::map<std::string, Value> PullAll(TEncoder *encoder) = 0;
+
+  /** Aborts currently running query. */
+  virtual void Abort() = 0;
+
+  /** Return `true` if the user was successfully authenticated. */
+  virtual bool Authenticate(const std::string &username,
+                            const std::string &password) = 0;
 
   /**
    * Executes the session after data has been read into the buffer.
@@ -73,13 +70,16 @@ class Session {
    */
   void Execute() {
     if (UNLIKELY(!handshake_done_)) {
-      if (input_stream_.size() < HANDSHAKE_SIZE) {
+      // Resize the input buffer to ensure that a whole chunk can fit into it.
+      // This can be done only once because the buffer holds its size.
+      input_stream_.Resize(kChunkWholeSize);
+
+      // Receive the handshake.
+      if (input_stream_.size() < kHandshakeSize) {
         DLOG(WARNING) << fmt::format("Received partial handshake of size {}",
                                      input_stream_.size());
         return;
       }
-      DLOG(WARNING) << fmt::format("Decoding handshake of size {}",
-                                   input_stream_.size());
       state_ = StateHandshakeRun(*this);
       if (UNLIKELY(state_ == State::Close)) {
         ClientFailureInvalidData();
@@ -102,11 +102,9 @@ class Session {
           break;
         case State::Idle:
         case State::Result:
-        case State::WaitForRollback:
           state_ = StateExecutingRun(*this, state_);
           break;
-        case State::ErrorIdle:
-        case State::ErrorWaitForRollback:
+        case State::Error:
           state_ = StateErrorRun(*this, state_);
           break;
         default:
@@ -122,50 +120,22 @@ class Session {
         ClientFailureInvalidData();
         return;
       }
-
-      DLOG(INFO) << fmt::format("Input stream size: {}", input_stream_.size());
-      DLOG(INFO) << fmt::format("Decoder buffer size: {}",
-                                decoder_buffer_.Size());
     }
   }
 
-  /**
-   * Commits associated transaction.
-   */
-  void Commit() {
-    DCHECK(db_accessor_) << "Commit called and there is no transaction";
-    db_accessor_->Commit();
-    db_accessor_ = nullptr;
-  }
-
-  /**
-   * Aborts associated transaction.
-   */
-  void Abort() {
-    DCHECK(db_accessor_) << "Abort called and there is no transaction";
-    db_accessor_->Abort();
-    db_accessor_ = nullptr;
-  }
-
   // TODO: Rethink if there is a way to hide some members. At the momement all
   // of them are public.
-  database::MasterBase &db_;
-  query::Interpreter &interpreter_;
   TInputStream &input_stream_;
   TOutputStream &output_stream_;
 
   ChunkedEncoderBuffer<TOutputStream> encoder_buffer_{output_stream_};
-  Encoder<ChunkedEncoderBuffer<TOutputStream>> encoder_{encoder_buffer_};
-  ResultStreamT result_stream_{encoder_};
+  TEncoder encoder_{encoder_buffer_};
 
   ChunkedDecoderBuffer<TInputStream> decoder_buffer_{input_stream_};
   Decoder<ChunkedDecoderBuffer<TInputStream>> decoder_{decoder_buffer_};
 
   bool handshake_done_{false};
   State state_{State::Handshake};
-  // GraphDbAccessor of active transaction in the session, can be null if
-  // there is no associated transaction.
-  std::unique_ptr<database::GraphDbAccessor> db_accessor_;
 
  private:
   void ClientFailureInvalidData() {
@@ -183,4 +153,5 @@ class Session {
     throw SessionException("Something went wrong during session execution!");
   }
 };
+
 }  // namespace communication::bolt

src/communication/bolt/v1/state.hpp

@@ -31,23 +31,11 @@ enum class State : uint8_t {
    */
   Result,
 
-  /**
-   * There was an acked error in explicitly started transaction, now we are
-   * waiting for "ROLLBACK" in RUN command.
-   */
-  WaitForRollback,
-
   /**
    * This state handles errors, if client handles error response correctly next
    * state is Idle.
    */
-  ErrorIdle,
-
-  /**
-   * This state handles errors, if client handles error response correctly next
-   * state is WaitForRollback.
-   */
-  ErrorWaitForRollback,
+  Error,
 
   /**
    * This is a 'virtual' state (it doesn't have a run function) which tells