Add formated test

.arcconfig

@@ -0,0 +1,7 @@
+{
+  "project_id" : "memgraph",
+  "conduit_uri" : "https://phabricator.memgraph.io",
+  "phabricator_uri" : "https://phabricator.memgraph.io",
+  "git.default-relative-commit": "origin/master",
+  "arc.land.onto.default": "master"
+}

.clang-format

@@ -1,10 +1,9 @@
 ---
 Language:     Cpp
 BasedOnStyle: Google
-Standard:     "c++20"
+Standard:     "C++11"
 UseTab:       Never
 DerivePointerAlignment: false
 PointerAlignment: Right
-ColumnLimit : 120
-IncludeBlocks: Preserve
+ColumnLimit : 80
 ...

.clang-tidy

@@ -1,92 +0,0 @@
----
-Checks: '*,
-         -abseil-string-find-str-contains,
-         -altera-id-dependent-backward-branch,
-         -altera-struct-pack-align,
-         -altera-unroll-loops,
-         -android-*,
-         -cert-err58-cpp,
-         -cppcoreguidelines-avoid-c-arrays,
-         -cppcoreguidelines-avoid-goto,
-         -cppcoreguidelines-avoid-magic-numbers,
-         -cppcoreguidelines-no-malloc,
-         -cppcoreguidelines-non-private-member-variables-in-classes,
-         -cppcoreguidelines-owning-memory,
-         -cppcoreguidelines-pro-bounds-array-to-pointer-decay,
-         -cppcoreguidelines-pro-bounds-constant-array-index,
-         -cppcoreguidelines-pro-bounds-pointer-arithmetic,
-         -cppcoreguidelines-pro-type-member-init,
-         -cppcoreguidelines-pro-type-reinterpret-cast,
-         -cppcoreguidelines-pro-type-static-cast-downcast,
-         -cppcoreguidelines-pro-type-union-access,
-         -cppcoreguidelines-pro-type-vararg,
-         -cppcoreguidelines-special-member-functions,
-         -fuchsia-default-arguments,
-         -fuchsia-default-arguments-calls,
-         -fuchsia-default-arguments-declarations,
-         -fuchsia-overloaded-operator,
-         -fuchsia-statically-constructed-objects,
-         -fuchsia-trailing-return,
-         -fuchsia-virtual-inheritance,
-         -google-explicit-constructor,
-         -google-readability-*,
-         -google-runtime-references,
-         -hicpp-avoid-c-arrays,
-         -hicpp-avoid-goto,
-         -hicpp-braces-around-statements,
-         -hicpp-member-init,
-         -hicpp-no-array-decay,
-         -hicpp-no-assembler,
-         -hicpp-no-malloc,
-         -hicpp-use-equals-default,
-         -hicpp-use-nullptr,
-         -hicpp-vararg,
-         -llvm-header-guard,
-         -llvm-include-order,
-         -llvmlibc-callee-namespace,
-         -llvmlibc-implementation-in-namespace,
-         -llvmlibc-restrict-system-libc-headers,
-         -misc-non-private-member-variables-in-classes,
-         -modernize-avoid-c-arrays,
-         -modernize-concat-nested-namespaces,
-         -modernize-pass-by-value,
-         -modernize-use-equals-default,
-         -modernize-use-nodiscard,
-         -modernize-use-trailing-return-type,
-         -performance-unnecessary-value-param,
-         -readability-braces-around-statements,
-         -readability-else-after-return,
-         -readability-function-cognitive-complexity,
-         -readability-implicit-bool-conversion,
-         -readability-magic-numbers,
-         -readability-named-parameter,
-         -misc-no-recursion,
-         -concurrency-mt-unsafe,
-         -bugprone-easily-swappable-parameters'
-
-WarningsAsErrors: ''
-HeaderFilterRegex: 'src/.*'
-AnalyzeTemporaryDtors: false
-FormatStyle:     none
-CheckOptions:
-  - key:             google-readability-braces-around-statements.ShortStatementLines
-    value:           '1'
-  - key:             google-readability-function-size.StatementThreshold
-    value:           '800'
-  - key:             google-readability-namespace-comments.ShortNamespaceLines
-    value:           '10'
-  - key:             google-readability-namespace-comments.SpacesBeforeComments
-    value:           '2'
-  - key:             modernize-loop-convert.MaxCopySize
-    value:           '16'
-  - key:             modernize-loop-convert.MinConfidence
-    value:           reasonable
-  - key:             modernize-loop-convert.NamingStyle
-    value:           CamelCase
-  - key:             modernize-pass-by-value.IncludeStyle
-    value:           llvm
-  - key:             modernize-replace-auto-ptr.IncludeStyle
-    value:           llvm
-  - key:             modernize-use-nullptr.NullMacros
-    value:           'NULL'
-...

.githooks/pre-commit

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-project_folder=$(git rev-parse --show-toplevel)
-if git rev-parse --verify HEAD >/dev/null 2>&1
-then
-	against=HEAD
-else
-	# Initial commit: diff against an empty tree object
-	against=$(git hash-object -t tree /dev/null)
-fi
-
-# Redirect output to stderr.
-exec 1>&2
-
-tmpdir=$(mktemp -d repo-XXXXXXXX)
-trap "rm -rf $tmpdir" EXIT INT
-
-modified_files=$(git diff --cached --name-only --diff-filter=AM $against | sed -nE "/.*\.(cpp|cc|cxx|c|h|hpp)$/p")
-FAIL=0
-for file in $modified_files; do
-  echo "Checking $file..."
-
-  cp $project_folder/.clang-format $project_folder/.clang-tidy $tmpdir
-
-  git checkout-index --prefix="$tmpdir/" -- $file
-
-  # Do not break header checker
-  echo "Running header checker..."
-  $project_folder/tools/header-checker.py $tmpdir/$file $file --amend-year
-  CODE=$?
-  if [ $CODE -ne 0 ]; then
-    FAIL=1
-  fi
-done;
-
-return ${FAIL}

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,34 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: "[BUG] "
-labels: bug
-assignees: gitbuda, antonio2368
-
----
-
-
-**Memgraph version**
-Which version did you use?
-
-**Environment**
-Some information about the environment you are using Memgraph on: operating
-system, how do you connect, with or without docker, which driver etc.
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Run the following query '...'
-2. Click on '....'
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Logs**
-If applicable, add logs of Memgraph, CLI output or screenshots to help explain
-your problem.
-
-**Additional context**
-Add any other context about the problem here.

.github/pull_request_template.md

@@ -1,14 +0,0 @@
-[master < Epic] PR
-- [ ] Check, and update documentation if necessary
-- [ ] Write E2E tests
-- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
-- [ ] Provide the full content or a guide for the final git message
-
-[master < Task] PR
-- [ ] Check, and update documentation if necessary
-- [ ] Provide the full content or a guide for the final git message
-
-
-To keep docs changelog up to date, one more thing to do:
-- [ ] Write a release note here
-- [ ] Tag someone from docs team in the comments

.github/workflows/daily_benchmark.yaml

@@ -1,82 +0,0 @@
-name: Daily Benchmark
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 22 * * *"
-
-jobs:
-  release_benchmarks:
-    name: "Release benchmarks"
-    runs-on: [self-hosted, Linux, X64, Diff, Gen7]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build only memgraph release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Run macro benchmarks
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Get branch name (merge)
-        if: github.event_name != 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
-
-      - name: Get branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
-
-      - name: Upload macro benchmark results
-        run: |
-          cd tools/bench-graph-client
-          virtualenv -p python3 ve3
-          source ve3/bin/activate
-          pip install -r requirements.txt
-          ./main.py --benchmark-name "macro_benchmark" \
-                    --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
-                    --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}" \
-                    --head-branch-name "${{ env.BRANCH_NAME }}"
-
-      - name: Run mgbench
-        run: |
-          cd tests/mgbench
-          ./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
-
-      - name: Upload mgbench results
-        run: |
-          cd tools/bench-graph-client
-          virtualenv -p python3 ve3
-          source ve3/bin/activate
-          pip install -r requirements.txt
-          ./main.py --benchmark-name "mgbench" \
-                    --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
-                    --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}" \
-                    --head-branch-name "${{ env.BRANCH_NAME }}"

.github/workflows/diff.yaml

@@ -1,452 +0,0 @@
-name: Diff
-concurrency:
-  group: ${{ github.head_ref || github.sha }}
-  cancel-in-progress: true
-
-on:
-  push:
-    branches:
-      - master
-  workflow_dispatch:
-  pull_request:
-    paths-ignore:
-      - "docs/**"
-      - "**/*.md"
-      - ".clang-format"
-      - "CODEOWNERS"
-      - "licenses/*"
-
-jobs:
-  community_build:
-    name: "Community build"
-    runs-on: [self-hosted, Linux, X64, Diff]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build community binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build community binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
-          make -j$THREADS
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure -j$THREADS
-
-  code_analysis:
-    name: "Code analysis"
-    runs-on: [self-hosted, Linux, X64, Diff]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-        # This is also needed if we want do to comparison against other branches
-        # See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
-      - name: Fetch all history for all tags and branches
-        run: git fetch
-
-      - name: Initialize deps
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-      - name: Set base branch
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
-
-      - name: Set base branch # if we manually dispatch or push to master
-        if: ${{ github.event_name != 'pull_request' }}
-        run: |
-          echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
-
-      - name: Python code analysis
-        run: |
-          CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
-          for file in ${CHANGED_FILES}; do
-            echo ${file}
-            if [[ ${file} == *.py ]]; then
-              python3 -m black --check --diff ${file}
-              python3 -m isort --check-only --diff ${file}
-            fi
-          done
-
-      - name: Build combined ASAN, UBSAN and coverage binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-          cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
-          make -j$THREADS memgraph__unit
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
-          cd build
-          LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
-
-      - name: Compute code coverage
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Compute code coverage.
-          cd tools/github
-          ./coverage_convert
-
-          # Package code coverage.
-          cd generated
-          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
-
-      - name: Save code coverage
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/generated/code_coverage.tar.gz
-
-      - name: Run clang-tidy
-        run: |
-          source /opt/toolchain-v4/activate
-
-          # Restrict clang-tidy results only to the modified parts
-          git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build  -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
-
-          # Fail if any warning is reported
-          ! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
-
-  debug_build:
-    name: "Debug build"
-    runs-on: [self-hosted, Linux, X64, Diff]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build debug binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build debug binaries.
-          cd build
-          cmake ..
-          make -j$THREADS
-
-      - name: Run leftover CTest tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run leftover CTest tests (all except unit and benchmark tests).
-          cd build
-          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
-
-      - name: Run drivers tests
-        run: |
-          ./tests/drivers/run.sh
-
-      - name: Run integration tests
-        run: |
-          cd tests/integration
-          for name in *; do
-            if [ ! -d $name ]; then continue; fi
-            pushd $name >/dev/null
-            echo "Running: $name"
-            if [ -x prepare.sh ]; then
-              ./prepare.sh
-            fi
-            if [ -x runner.py ]; then
-              ./runner.py
-            elif [ -x runner.sh ]; then
-              ./runner.sh
-            fi
-            echo
-            popd >/dev/null
-          done
-
-      - name: Run cppcheck and clang-format
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run cppcheck and clang-format.
-          cd tools/github
-          ./cppcheck_and_clang_format diff
-
-      - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/cppcheck_and_clang_format.txt
-
-  release_build:
-    name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Diff]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Run GQL Behave tests
-        run: |
-          cd tests/gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure -j$THREADS
-
-      - name: Run e2e tests
-        run: |
-          # TODO(gitbuda): Setup mgclient and pymgclient properly.
-          cd tests
-          ./setup.sh
-          source ve3/bin/activate
-          cd e2e
-          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
-
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
-      - name: Run durability test
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Create enterprise DEB package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise DEB package.
-          mkdir output && cd output
-          cpack -G DEB --config ../CPackConfig.cmake
-
-      - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Enterprise DEB package"
-          path: build/output/memgraph*.deb
-
-      - name: Save test data
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: "Test data"
-          path: |
-            # multiple paths could be defined
-            build/logs
-
-  release_jepsen_test:
-    name: "Release Jepsen Test"
-    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
-    #continue-on-error: true
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build only memgraph release binarie.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS memgraph
-
-      - name: Run Jepsen tests
-        run: |
-          cd tests/jepsen
-          ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
-
-      - name: Save Jepsen report
-        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
-        with:
-          name: "Jepsen Report"
-          path: tests/jepsen/Jepsen.tar.gz
-
-  release_benchmarks:
-    name: "Release benchmarks"
-    runs-on: [self-hosted, Linux, X64, Diff, Gen7]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build only memgraph release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Run macro benchmarks
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Get branch name (merge)
-        if: github.event_name != 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
-
-      - name: Get branch name (pull request)
-        if: github.event_name == 'pull_request'
-        shell: bash
-        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
-
-      - name: Upload macro benchmark results
-        run: |
-          cd tools/bench-graph-client
-          virtualenv -p python3 ve3
-          source ve3/bin/activate
-          pip install -r requirements.txt
-          ./main.py --benchmark-name "macro_benchmark" \
-                    --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
-                    --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}" \
-                    --head-branch-name "${{ env.BRANCH_NAME }}"
-
-      - name: Run mgbench
-        run: |
-          cd tests/mgbench
-          ./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
-
-      - name: Upload mgbench results
-        run: |
-          cd tools/bench-graph-client
-          virtualenv -p python3 ve3
-          source ve3/bin/activate
-          pip install -r requirements.txt
-          ./main.py --benchmark-name "mgbench" \
-                    --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
-                    --github-run-id "${{ github.run_id }}" \
-                    --github-run-number "${{ github.run_number }}" \
-                    --head-branch-name "${{ env.BRANCH_NAME }}"

.github/workflows/full_clang_tidy.yaml

@@ -1,46 +0,0 @@
-name: Run clang-tidy on the full codebase
-
-on:
-  workflow_dispatch:
-
-jobs:
-  clang_tidy_check:
-    name: "Clang-tidy check"
-    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build debug binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build debug binaries.
-
-          cd build
-          cmake ..
-          make -j$THREADS
-
-      - name: Run clang-tidy
-        run: |
-          source /opt/toolchain-v4/activate
-
-          # The results are also written to standard output in order to retain them in the logs
-          ./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
-            tee ./build/full_clang_tidy_output.txt
-
-      - name: Summarize clang-tidy results
-        run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

.github/workflows/package_all.yaml

@@ -1,229 +0,0 @@
-name: Package All
-
-# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
-
-on: workflow_dispatch
-
-jobs:
-  centos-7:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package centos-7
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: centos-7
-          path: build/output/centos-7/memgraph*.rpm
-
-  centos-9:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package centos-9
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: centos-9
-          path: build/output/centos-9/memgraph*.rpm
-
-  debian-10:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package debian-10
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: debian-10
-          path: build/output/debian-10/memgraph*.deb
-
-  debian-11:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package debian-11
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: debian-11
-          path: build/output/debian-11/memgraph*.deb
-
-  docker:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          cd release/package
-          ./run.sh package debian-11 --for-docker
-          ./run.sh docker
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: docker
-          path: build/output/docker/memgraph*.tar.gz
-
-  ubuntu-1804:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package ubuntu-18.04
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: ubuntu-1804
-          path: build/output/ubuntu-18.04/memgraph*.deb
-
-  ubuntu-2004:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package ubuntu-20.04
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: ubuntu-2004
-          path: build/output/ubuntu-20.04/memgraph*.deb
-
-  ubuntu-2204:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package ubuntu-22.04
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: ubuntu-2204
-          path: build/output/ubuntu-22.04/memgraph*.deb
-
-  debian-11-platform:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package debian-11 --for-platform
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: debian-11-platform
-          path: build/output/debian-11/memgraph*.deb
-
-  fedora-36:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package fedora-36
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: fedora-36
-          path: build/output/fedora-36/memgraph*.rpm
-
-  amzn-2:
-    runs-on: [self-hosted, DockerMgBuild, X64]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package amzn-2
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: amzn-2
-          path: build/output/amzn-2/memgraph*.rpm
-
-  debian-11-arm:
-    runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package debian-11-arm
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: debian-11-arm
-          path: build/output/debian-11-arm/memgraph*.deb
-
-  ubuntu-2204-arm:
-    runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
-    timeout-minutes: 60
-    steps:
-      - name: "Set up repository"
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # Required because of release/get_version.py
-      - name: "Build package"
-        run: |
-          ./release/package/run.sh package ubuntu-22.04-arm
-      - name: "Upload package"
-        uses: actions/upload-artifact@v3
-        with:
-          name: ubuntu-22.04-arm
-          path: build/output/ubuntu-22.04-arm/memgraph*.deb

.github/workflows/release_centos8.yaml

@@ -1,315 +0,0 @@
-name: Release CentOS 8
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 22 * * *"
-
-jobs:
-  community_build:
-    name: "Community build"
-    runs-on: [self-hosted, Linux, X64, CentOS8]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 960
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build community binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build community binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
-          make -j$THREADS
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-  coverage_build:
-    name: "Coverage build"
-    runs-on: [self-hosted, Linux, X64, CentOS8]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build coverage binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build coverage binaries.
-          cd build
-          cmake -DTEST_COVERAGE=ON ..
-          make -j$THREADS memgraph__unit
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Compute code coverage
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Compute code coverage.
-          cd tools/github
-          ./coverage_convert
-
-          # Package code coverage.
-          cd generated
-          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
-
-      - name: Save code coverage
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/generated/code_coverage.tar.gz
-
-  debug_build:
-    name: "Debug build"
-    runs-on: [self-hosted, Linux, X64, CentOS8]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build debug binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build debug binaries.
-          cd build
-          cmake ..
-          make -j$THREADS
-
-      - name: Run leftover CTest tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run leftover CTest tests (all except unit and benchmark tests).
-          cd build
-          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
-
-      - name: Run drivers tests
-        run: |
-          ./tests/drivers/run.sh
-
-      - name: Run integration tests
-        run: |
-          cd tests/integration
-          for name in *; do
-            if [ ! -d $name ]; then continue; fi
-            pushd $name >/dev/null
-            echo "Running: $name"
-            if [ -x prepare.sh ]; then
-              ./prepare.sh
-            fi
-            if [ -x runner.py ]; then
-              ./runner.py
-            elif [ -x runner.sh ]; then
-              ./runner.sh
-            fi
-            echo
-            popd >/dev/null
-          done
-
-      - name: Run cppcheck and clang-format
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run cppcheck and clang-format.
-          cd tools/github
-          ./cppcheck_and_clang_format diff
-
-      - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/cppcheck_and_clang_format.txt
-
-  release_build:
-    name: "Release build"
-    runs-on: [self-hosted, Linux, X64, CentOS8]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 960
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Create enterprise RPM package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise RPM package.
-          mkdir output && cd output
-          cpack -G RPM --config ../CPackConfig.cmake
-          rpmlint memgraph*.rpm
-
-      - name: Save enterprise RPM package
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Enterprise RPM package"
-          path: build/output/memgraph*.rpm
-
-      - name: Run micro benchmark tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run micro benchmark tests.
-          cd build
-          # The `eval` benchmark needs a large stack limit.
-          ulimit -s 262144
-          ctest -R memgraph__benchmark -V
-
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Run parallel macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QueryParallelSuite MemgraphRunner \
-            --groups aggregation_parallel create_parallel bfs_parallel \
-            --num-database-workers 9 --num-clients-workers 30 \
-            --no-strict
-
-      - name: Run GQL Behave tests
-        run: |
-          cd tests/gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Run e2e tests
-        run: |
-          # TODO(gitbuda): Setup mgclient and pymgclient properly.
-          cd tests
-          ./setup.sh
-          source ve3/bin/activate
-          cd e2e
-          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
-
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
-      - name: Run stress test (large)
-        run: |
-          cd tests/stress
-          ./continuous_integration --large-dataset
-
-      - name: Run durability test (plain)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Run durability test (large)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 20

.github/workflows/release_debian10.yaml

@@ -1,356 +0,0 @@
-name: Release Debian 10
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 22 * * *"
-
-jobs:
-  community_build:
-    name: "Community build"
-    runs-on: [self-hosted, Linux, X64, Debian10]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 960
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build community binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build community binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
-          make -j$THREADS
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-  coverage_build:
-    name: "Coverage build"
-    runs-on: [self-hosted, Linux, X64, Debian10]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build coverage binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build coverage binaries.
-          cd build
-          cmake -DTEST_COVERAGE=ON ..
-          make -j$THREADS memgraph__unit
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Compute code coverage
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Compute code coverage.
-          cd tools/github
-          ./coverage_convert
-
-          # Package code coverage.
-          cd generated
-          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
-
-      - name: Save code coverage
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/generated/code_coverage.tar.gz
-
-  debug_build:
-    name: "Debug build"
-    runs-on: [self-hosted, Linux, X64, Debian10]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build debug binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build debug binaries.
-          cd build
-          cmake ..
-          make -j$THREADS
-
-      - name: Run leftover CTest tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run leftover CTest tests (all except unit and benchmark tests).
-          cd build
-          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
-
-      - name: Run drivers tests
-        run: |
-          ./tests/drivers/run.sh
-
-      - name: Run integration tests
-        run: |
-          cd tests/integration
-          for name in *; do
-            if [ ! -d $name ]; then continue; fi
-            pushd $name >/dev/null
-            echo "Running: $name"
-            if [ -x prepare.sh ]; then
-              ./prepare.sh
-            fi
-            if [ -x runner.py ]; then
-              ./runner.py
-            elif [ -x runner.sh ]; then
-              ./runner.sh
-            fi
-            echo
-            popd >/dev/null
-          done
-
-      - name: Run cppcheck and clang-format
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run cppcheck and clang-format.
-          cd tools/github
-          ./cppcheck_and_clang_format diff
-
-      - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/cppcheck_and_clang_format.txt
-
-  release_build:
-    name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Debian10]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 960
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Create enterprise DEB package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise DEB package.
-          mkdir output && cd output
-          cpack -G DEB --config ../CPackConfig.cmake
-
-      - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Enterprise DEB package"
-          path: build/output/memgraph*.deb
-
-      - name: Run micro benchmark tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run micro benchmark tests.
-          cd build
-          # The `eval` benchmark needs a large stack limit.
-          ulimit -s 262144
-          ctest -R memgraph__benchmark -V
-
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Run parallel macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QueryParallelSuite MemgraphRunner \
-            --groups aggregation_parallel create_parallel bfs_parallel \
-            --num-database-workers 9 --num-clients-workers 30 \
-            --no-strict
-
-      - name: Run GQL Behave tests
-        run: |
-          cd tests/gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Run e2e tests
-        run: |
-          # TODO(gitbuda): Setup mgclient and pymgclient properly.
-          cd tests
-          ./setup.sh
-          source ve3/bin/activate
-          cd e2e
-          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
-
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
-      - name: Run stress test (large)
-        run: |
-          cd tests/stress
-          ./continuous_integration --large-dataset
-
-      - name: Run durability test (plain)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Run durability test (large)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 20
-
-  release_jepsen_test:
-    name: "Release Jepsen Test"
-    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 60
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build only memgraph release binary.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS memgraph
-
-      - name: Run Jepsen tests
-        run: |
-          cd tests/jepsen
-          ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
-
-      - name: Save Jepsen report
-        uses: actions/upload-artifact@v3
-        if: ${{ always() }}
-        with:
-          name: "Jepsen Report"
-          path: tests/jepsen/Jepsen.tar.gz

.github/workflows/release_docker.yaml

@@ -1,69 +0,0 @@
-name: Publish Docker images
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: "Memgraph binary version to publish on DockerHub."
-        required: true
-      force_release:
-        type: boolean
-        required: false
-        default: false
-
-jobs:
-  docker_publish:
-    runs-on: ubuntu-latest
-    env:
-      DOCKER_ORGANIZATION_NAME: memgraph
-      DOCKER_REPOSITORY_NAME: memgraph
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Download memgraph binary
-        run: |
-          cd release/docker
-          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
-          curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
-
-      - name: Check if specified version is already pushed
-        run: |
-          EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
-          echo $EXISTS
-          if [[ ${EXISTS} -eq 0 ]]; then
-              echo 'The specified version has been already released to DockerHub.'
-              if [[ ${{ github.event.inputs.force_release }} = true ]]; then
-                echo 'Forcing the release!'
-              else
-                echo 'Stopping the release!'
-                exit 1
-              fi
-          else
-              echo 'All good the specified version has not been release to DockerHub.'
-          fi
-
-      - name: Build & push docker images
-        run: |
-          cd release/docker
-          docker buildx build \
-          --build-arg BINARY_NAME="memgraph-" \
-          --build-arg EXTENSION="deb" \
-          --platform linux/amd64,linux/arm64 \
-          --tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
-          --tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
-          --file memgraph_deb.dockerfile \
-          --push .

.github/workflows/release_ubuntu2004.yaml

@@ -1,314 +0,0 @@
-name: Release Ubuntu 20.04
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 22 * * *"
-
-jobs:
-  community_build:
-    name: "Community build"
-    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 960
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build community binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build community binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
-          make -j$THREADS
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-  coverage_build:
-    name: "Coverage build"
-    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build coverage binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build coverage binaries.
-          cd build
-          cmake -DTEST_COVERAGE=ON ..
-          make -j$THREADS memgraph__unit
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Compute code coverage
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Compute code coverage.
-          cd tools/github
-          ./coverage_convert
-
-          # Package code coverage.
-          cd generated
-          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
-
-      - name: Save code coverage
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/generated/code_coverage.tar.gz
-
-  debug_build:
-    name: "Debug build"
-    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build debug binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build debug binaries.
-          cd build
-          cmake ..
-          make -j$THREADS
-
-      - name: Run leftover CTest tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run leftover CTest tests (all except unit and benchmark tests).
-          cd build
-          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
-
-      - name: Run drivers tests
-        run: |
-          ./tests/drivers/run.sh
-
-      - name: Run integration tests
-        run: |
-          cd tests/integration
-          for name in *; do
-            if [ ! -d $name ]; then continue; fi
-            pushd $name >/dev/null
-            echo "Running: $name"
-            if [ -x prepare.sh ]; then
-              ./prepare.sh
-            fi
-            if [ -x runner.py ]; then
-              ./runner.py
-            elif [ -x runner.sh ]; then
-              ./runner.sh
-            fi
-            echo
-            popd >/dev/null
-          done
-
-      - name: Run cppcheck and clang-format
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run cppcheck and clang-format.
-          cd tools/github
-          ./cppcheck_and_clang_format diff
-
-      - name: Save cppcheck and clang-format errors
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Code coverage"
-          path: tools/github/cppcheck_and_clang_format.txt
-
-  release_build:
-    name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
-    env:
-      THREADS: 24
-      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
-      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
-    timeout-minutes: 960
-
-    steps:
-      - name: Set up repository
-        uses: actions/checkout@v3
-        with:
-          # Number of commits to fetch. `0` indicates all history for all
-          # branches and tags. (default: 1)
-          fetch-depth: 0
-
-      - name: Build release binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Initialize dependencies.
-          ./init
-
-          # Build release binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS
-
-      - name: Create enterprise DEB package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise DEB package.
-          mkdir output && cd output
-          cpack -G DEB --config ../CPackConfig.cmake
-
-      - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Enterprise DEB package"
-          path: build/output/memgraph*.deb
-
-      - name: Run micro benchmark tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run micro benchmark tests.
-          cd build
-          # The `eval` benchmark needs a large stack limit.
-          ulimit -s 262144
-          ctest -R memgraph__benchmark -V
-
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Run parallel macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QueryParallelSuite MemgraphRunner \
-            --groups aggregation_parallel create_parallel bfs_parallel \
-            --num-database-workers 9 --num-clients-workers 30 \
-            --no-strict
-
-      - name: Run GQL Behave tests
-        run: |
-          cd tests/gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Run e2e tests
-        run: |
-          # TODO(gitbuda): Setup mgclient and pymgclient properly.
-          cd tests
-          ./setup.sh
-          source ve3/bin/activate
-          cd e2e
-          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
-
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
-      - name: Run stress test (large)
-        run: |
-          cd tests/stress
-          ./continuous_integration --large-dataset
-
-      - name: Run durability test (plain)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Run durability test (large)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 20

.gitignore

@@ -34,28 +34,67 @@ TAGS
 *.fas
 *.fasl
 
+# Cap'n Proto generated files
+*.capnp.c++
+*.capnp.h
+
+# LCP generated C++ & Cap'n Proto files
+*.lcp.cpp
+
+# TODO delete later
+src/storage/distributed/concurrent_id_mapper_rpc_messages.capnp
+src/storage/distributed/concurrent_id_mapper_rpc_messages.hpp
+#####
+
+
+src/database/distributed/counters_rpc_messages.capnp
+src/database/distributed/counters_rpc_messages.hpp
+src/database/distributed/serialization.capnp
 src/database/distributed/serialization.hpp
+src/database/single_node_ha/serialization.capnp
 src/database/single_node_ha/serialization.hpp
+src/distributed/bfs_rpc_messages.capnp
 src/distributed/bfs_rpc_messages.hpp
+src/distributed/coordination_rpc_messages.capnp
 src/distributed/coordination_rpc_messages.hpp
+src/distributed/data_rpc_messages.capnp
 src/distributed/data_rpc_messages.hpp
+src/distributed/durability_rpc_messages.capnp
 src/distributed/durability_rpc_messages.hpp
+src/distributed/dynamic_worker_rpc_messages.capnp
 src/distributed/dynamic_worker_rpc_messages.hpp
+src/distributed/index_rpc_messages.capnp
 src/distributed/index_rpc_messages.hpp
+src/distributed/plan_rpc_messages.capnp
 src/distributed/plan_rpc_messages.hpp
+src/distributed/pull_produce_rpc_messages.capnp
 src/distributed/pull_produce_rpc_messages.hpp
+src/distributed/storage_gc_rpc_messages.capnp
 src/distributed/storage_gc_rpc_messages.hpp
+src/distributed/token_sharing_rpc_messages.capnp
 src/distributed/token_sharing_rpc_messages.hpp
+src/distributed/updates_rpc_messages.capnp
 src/distributed/updates_rpc_messages.hpp
+src/query/frontend/ast/ast.hpp
+src/query/frontend/ast/ast_serialization.capnp
+src/query/frontend/ast/ast_serialization.hpp
+src/durability/distributed/state_delta.capnp
 src/durability/distributed/state_delta.hpp
 src/durability/single_node/state_delta.hpp
 src/durability/single_node_ha/state_delta.hpp
-src/query/distributed/frontend/semantic/symbol_serialization.hpp
-src/query/distributed/plan/ops.hpp
+src/query/frontend/semantic/symbol.hpp
+src/query/frontend/semantic/symbol_serialization.capnp
+src/query/frontend/semantic/symbol_serialization.hpp
+src/query/plan/distributed_ops.capnp
+src/query/plan/distributed_ops.hpp
+src/query/plan/operator.hpp
+src/raft/log_entry.capnp
 src/raft/log_entry.hpp
+src/raft/raft_rpc_messages.capnp
 src/raft/raft_rpc_messages.hpp
-src/raft/snapshot_metadata.hpp
-src/raft/storage_info_rpc_messages.hpp
+src/stats/stats_rpc_messages.capnp
 src/stats/stats_rpc_messages.hpp
+src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.capnp
 src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp
+src/transactions/distributed/engine_rpc_messages.capnp
 src/transactions/distributed/engine_rpc_messages.hpp

.pre-commit-config.yaml

@@ -1,20 +0,0 @@
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
-    hooks:
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 23.1.0
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-        name: isort (python)
-  - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v13.0.0
-    hooks:
-      - id: clang-format

.ycm_extra_conf.py

@@ -0,0 +1,172 @@
+import os
+import os.path
+import fnmatch
+import logging
+import ycm_core
+
+BASE_FLAGS = [
+    '-Wall',
+    '-Wextra',
+    '-Werror',
+    '-Wno-long-long',
+    '-Wno-variadic-macros',
+    '-fexceptions',
+    '-ferror-limit=10000',
+    '-std=c++1z',
+    '-xc++',
+    '-I/usr/lib/',
+    '-I/usr/include/',
+    '-I./src',
+    '-I./include',
+    '-I./libs/fmt',
+    '-I./libs/yaml-cpp',
+    '-I./libs/glog/include',
+    '-I./libs/googletest/googletest/include',
+    '-I./libs/googletest/googlemock/include',
+    '-I./libs/benchmark/include',
+    '-I./libs/cereal/include',
+    # We include cppitertools headers directly from libs directory.
+    '-I./libs',
+    '-I./libs/rapidcheck/include',
+    '-I./libs/antlr4/runtime/Cpp/runtime/src',
+    '-I./libs/gflags/include',
+    '-I./experimental/distributed/src',
+    '-I./libs/postgresql/include',
+    '-I./libs/bzip2',
+    '-I./libs/zlib',
+    '-I./libs/rocksdb/include',
+    '-I./libs/librdkafka/include/librdkafka',
+    '-I./build/include'
+]
+
+SOURCE_EXTENSIONS = [
+    '.cpp',
+    '.cxx',
+    '.cc',
+    '.c',
+    '.m',
+    '.mm'
+]
+
+HEADER_EXTENSIONS = [
+    '.h',
+    '.hxx',
+    '.hpp',
+    '.hh'
+]
+
+# set the working directory of YCMD to be this file
+os.chdir(os.path.dirname(os.path.realpath(__file__)))
+
+def IsHeaderFile(filename):
+    extension = os.path.splitext(filename)[1]
+    return extension in HEADER_EXTENSIONS
+
+def GetCompilationInfoForFile(database, filename):
+    if IsHeaderFile(filename):
+        basename = os.path.splitext(filename)[0]
+        for extension in SOURCE_EXTENSIONS:
+            replacement_file = basename + extension
+            if os.path.exists(replacement_file):
+                compilation_info = database.GetCompilationInfoForFile(replacement_file)
+                if compilation_info.compiler_flags_:
+                    return compilation_info
+        return None
+    return database.GetCompilationInfoForFile(filename)
+
+def FindNearest(path, target):
+    candidate = os.path.join(path, target)
+    if(os.path.isfile(candidate) or os.path.isdir(candidate)):
+        logging.info("Found nearest " + target + " at " + candidate)
+        return candidate;
+    else:
+        parent = os.path.dirname(os.path.abspath(path));
+        if(parent == path):
+            raise RuntimeError("Could not find " + target);
+        return FindNearest(parent, target)
+
+def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
+    if not working_directory:
+        return list(flags)
+    new_flags = []
+    make_next_absolute = False
+    path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ]
+    for flag in flags:
+        new_flag = flag
+
+        if make_next_absolute:
+            make_next_absolute = False
+            if not flag.startswith('/'):
+                new_flag = os.path.join(working_directory, flag)
+
+        for path_flag in path_flags:
+            if flag == path_flag:
+                make_next_absolute = True
+                break
+
+            if flag.startswith(path_flag):
+                path = flag[ len(path_flag): ]
+                new_flag = path_flag + os.path.join(working_directory, path)
+                break
+
+        if new_flag:
+            new_flags.append(new_flag)
+    return new_flags
+
+
+def FlagsForClangComplete(root):
+    try:
+        clang_complete_path = FindNearest(root, '.clang_complete')
+        clang_complete_flags = open(clang_complete_path, 'r').read().splitlines()
+        return clang_complete_flags
+    except:
+        return None
+
+def FlagsForInclude(root):
+    try:
+        include_path = FindNearest(root, 'include')
+        flags = []
+        for dirroot, dirnames, filenames in os.walk(include_path):
+            for dir_path in dirnames:
+                real_path = os.path.join(dirroot, dir_path)
+                flags = flags + ["-I" + real_path]
+        return flags
+    except:
+        return None
+
+def FlagsForCompilationDatabase(root, filename):
+    try:
+        compilation_db_path = FindNearest(root, 'compile_commands.json')
+        compilation_db_dir = os.path.dirname(compilation_db_path)
+        logging.info("Set compilation database directory to " + compilation_db_dir)
+        compilation_db =  ycm_core.CompilationDatabase(compilation_db_dir)
+        if not compilation_db:
+            logging.info("Compilation database file found but unable to load")
+            return None
+        compilation_info = GetCompilationInfoForFile(compilation_db, filename)
+        if not compilation_info:
+            logging.info("No compilation info for " + filename + " in compilation database")
+            return None
+        return MakeRelativePathsInFlagsAbsolute(
+            compilation_info.compiler_flags_,
+            compilation_info.compiler_working_dir_)
+    except:
+        return None
+
+def FlagsForFile(filename):
+    root = os.path.realpath(filename);
+    compilation_db_flags = FlagsForCompilationDatabase(root, filename)
+    if compilation_db_flags:
+        final_flags = compilation_db_flags
+    else:
+        final_flags = BASE_FLAGS
+        clang_flags = FlagsForClangComplete(root)
+        if clang_flags:
+            final_flags = final_flags + clang_flags
+        include_flags = FlagsForInclude(root)
+        if include_flags:
+            final_flags = final_flags + include_flags
+    return {
+        'flags': final_flags,
+        'do_cache': True
+    }

CHANGELOG.md

@@ -1,5 +1,231 @@
-Change Log for all versions of Memgraph can be found on-line at
-https://docs.memgraph.com/memgraph/changelog
+# Change Log
 
-All the updates to the Change Log can be made in the following repository:
-https://github.com/memgraph/docs
+## v0.14.0
+
+### Breaking Changes
+
+* Write-ahead log format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Reduce memory usage in distributed usage.
+* Add `DROP INDEX` feature.
+* Improve SSL error messages.
+
+### Bug Fixes and Other Changes
+
+* [Enterprise Ed.] Fix issues with reading and writing in a distributed query.
+* Correctly handle an edge case with unique constraint checks.
+* Fix a minor issue with `mg_import_csv`.
+* Fix an issue with `EXPLAIN`.
+
+## v0.13.0
+
+### Breaking Changes
+
+* Write-ahead log format changed (not backward compatible).
+* Snapshot format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Authentication and authorization support.
+* [Enterprise Ed.] Kafka integration.
+* [Enterprise Ed.] Support dynamic worker addition in distributed.
+* Reduce memory usage and improve overall performance.
+* Add `CREATE UNIQUE INDEX` clause to openCypher.
+* Add `EXPLAIN` clause to openCypher.
+* Add `inDegree` and `outDegree` functions to openCypher.
+* Improve BFS performance when both endpoints are known.
+* Add new `node-label`, `relationship-type` and `quote` options to
+  `mg_import_csv` tool.
+* Reduce memory usage of `mg_import_csv`.
+
+### Bug Fixes and Other Changes
+
+* [Enterprise Ed.] Fix an edge case in distributed index creation.
+* [Enterprise Ed.] Fix issues with Cartesian in distributed queries.
+* Correctly handle large messages in Bolt protocol.
+* Fix issues when handling explicitly started transactions in queries.
+* Allow openCypher keywords to be used as variable names.
+* Revise and make user visible error messages consistent.
+* Improve aborting time consuming execution.
+
+## v0.12.0
+
+### Breaking Changes
+
+* Snapshot format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* Improved Id Cypher function.
+* Added string functions to openCypher (`lTrim`, `left`, `rTrim`, `replace`,
+ `reverse`, `right`, `split`, `substring`, `toLower`, `toUpper`, `trim`).
+* Added `timestamp` function to openCypher.
+* Added support for dynamic property access with `[]` operator.
+
+## v0.11.0
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Improve Cartesian support in distributed queries.
+* [Enterprise Ed.] Improve distributed execution of BFS.
+* [Enterprise Ed.] Dynamic graph partitioner added.
+* Static nodes/edges id generators exposed through the Id Cypher function.
+* Properties on disk added.
+* Telemetry added.
+* SSL support added.
+* `toString` function added.
+
+### Bug Fixes and Other Changes
+
+* Document issues with Docker on OS X.
+* Add BFS and Dijkstra's algorithm examples to documentation.
+
+## v0.10.0
+
+### Breaking Changes
+
+* Snapshot format changed (not backward compatible).
+
+### Major Features and Improvements
+
+* [Enterprise Ed.] Distributed storage and execution.
+* `reduce` and `single` functions added to openCypher.
+* `wShortest` edge expansion added to openCypher.
+* Support packaging RPM on CentOS 7.
+
+### Bug Fixes and Other Changes
+
+* Report an error if updating a deleted element.
+* Log an error if reading info on available memory fails.
+* Fix a bug when `MATCH` would stop matching if a result was empty, but later
+  results still contain data to be matched. The simplest case of this was the
+  query: `UNWIND [1,2,3] AS x MATCH (n :Label {prop: x}) RETURN n`. If there
+  was no node `(:Label {prop: 1})`, then the `MATCH` wouldn't even try to find
+  for `x` being 2 or 3.
+* Report an error if trying to compare a property value with something that
+  cannot be stored in a property.
+* Fix crashes in some obscure cases.
+* Commit log automatically garbage collected.
+* Add minor performance improvements.
+
+## v0.9.0
+
+### Breaking Changes
+
+* Snapshot format changed (not backward compatible).
+* Snapshot configuration flags changed, general durability flags added.
+
+### Major Features and Improvements
+
+* Write-ahead log added.
+* `nodes` and `relationships` functions added.
+* `UNION` and `UNION ALL` is implemented.
+* Concurrent index creation is now enabled.
+
+### Bug Fixes and Other Changes
+
+
+## v0.8.0
+
+### Major Features and Improvements
+
+* CASE construct (without aggregations).
+* Named path support added.
+* Maps can now be stored as node/edge properties.
+* Map indexing supported.
+* `rand` function added.
+* `assert` function added.
+* `counter` and `counterSet` functions added.
+* `indexInfo` function added.
+* `collect` aggregation now supports Map collection.
+* Changed the BFS syntax.
+
+### Bug Fixes and Other Changes
+
+* Use \u to specify 4 digit codepoint and \U for 8 digit
+* Keywords appearing in header (named expressions) keep original case.
+* Our Bolt protocol implementation is now completely compatible with the protocol version 1 specification. (https://boltprotocol.org/v1/)
+* Added a log warning when running out of memory and the `memory_warning_threshold` flag
+* Edges are no longer additionally filtered after expansion.
+
+## v0.7.0
+
+### Major Features and Improvements
+
+* Variable length path `MATCH`.
+* Explicitly started transactions (multi-query transactions).
+* Map literal.
+* Query parameters (except for parameters in place of property maps).
+* `all` function in openCypher.
+* `degree` function in openCypher.
+* User specified transaction execution timeout.
+
+### Bug Fixes and Other Changes
+
+* Concurrent `BUILD INDEX` deadlock now returns an error to the client.
+* A `MATCH` preceeded by `OPTIONAL MATCH` expansion inconsistencies.
+* High concurrency Antlr parsing bug.
+* Indexing improvements.
+* Query stripping and caching speedups.
+
+## v0.6.0
+
+### Major Features and Improvements
+
+* AST caching.
+* Label + property index support.
+* Different logging setup & format.
+
+## v0.5.0
+
+### Major Features and Improvements
+
+* Use label indexes to speed up querying.
+* Generate multiple query plans and use the cost estimator to select the best.
+* Snapshots & Recovery.
+* Abandon old yaml configuration and migrate to gflags.
+* Query stripping & AST caching support.
+
+### Bug Fixes and Other Changes
+
+* Fixed race condition in MVCC. Hints exp+aborted race condition prevented.
+* Fixed conceptual bug in MVCC GC. Evaluate old records w.r.t. the oldest.
+  transaction's id AND snapshot.
+* User friendly error messages thrown from the query engine.
+
+## Build 837
+
+### Bug Fixes and Other Changes
+
+* List indexing supported with preceeding IN (for example in query `RETURN 1 IN [[1,2]][0]`).
+
+## Build 825
+
+### Major Features and Improvements
+
+* RETURN *, count(*), OPTIONAL MATCH, UNWIND, DISTINCT (except DISTINCT in aggregate functions), list indexing and slicing, escaped labels, IN LIST operator, range function.
+
+### Bug Fixes and Other Changes
+
+* TCP_NODELAY -> import should be faster.
+* Clear hint bits.
+
+## Build 783
+
+### Major Features and Improvements
+
+* SKIP, LIMIT, ORDER BY.
+* Math functions.
+* Initial support for MERGE clause.
+
+### Bug Fixes and Other Changes
+
+* Unhandled Lock Timeout Exception.
+
+## Build 755
+
+### Major Features and Improvements
+
+* MATCH, CREATE, WHERE, SET, REMOVE, DELETE.

CMakeLists.txt

@@ -1,6 +1,6 @@
 # MemGraph CMake configuration
 
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.1)
 
 # !! IMPORTANT !! run ./project_root/init.sh before cmake command
 # to download dependencies
@@ -31,129 +31,18 @@ find_program(CLANGXX_FOUND clang++)
 if (CLANG_FOUND AND CLANGXX_FOUND)
   set(CMAKE_C_COMPILER ${CLANG_FOUND})
   set(CMAKE_CXX_COMPILER ${CLANGXX_FOUND})
-else()
-  message(FATAL_ERROR "Couldn't find clang and/or clang++!")
 endif()
 
+# Get current commit hash.
+execute_process(
+    OUTPUT_VARIABLE COMMIT_HASH
+    COMMAND git rev-parse --short HEAD
+)
+string(STRIP ${COMMIT_HASH} COMMIT_HASH)
+
 # -----------------------------------------------------------------------------
 
-project(memgraph)
-
-# Install licenses.
-install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
-        DESTINATION share/doc/memgraph)
-
-# For more information about how to release a new version of Memgraph, see
-# `release/README.md`.
-
-# Option that is used to specify which version of Memgraph should be built. The
-# default is `ON` which causes the build system to build Memgraph Enterprise.
-# Memgraph Community is built if explicitly set to `OFF`.
-option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON)
-
-# Set the current version here to override the automatic version detection. The
-# version must be specified as `X.Y.Z`. Primarily used when building new patch
-# versions.
-set(MEMGRAPH_OVERRIDE_VERSION "")
-
-# Custom suffix that this version should have. The suffix can be any arbitrary
-# string. Primarily used when building a version for a specific customer.
-set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "")
-
-# Variables used to generate the versions.
-if (MG_ENTERPRISE)
-  set(get_version_offering "")
-else()
-  set(get_version_offering "--open-source")
-endif()
-set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
-
-# Get version that should be used in the binary.
-execute_process(
-    OUTPUT_VARIABLE MEMGRAPH_VERSION
-    RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
-    COMMAND "${get_version_script}" ${get_version_offering}
-                "${MEMGRAPH_OVERRIDE_VERSION}"
-                "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
-                "--memgraph-root-dir"
-                "${CMAKE_CURRENT_SOURCE_DIR}"
-)
-if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
-    message(FATAL_ERROR "Unable to get Memgraph version.")
-else()
-    MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
-endif()
-
-# Get version that should be used in the DEB package.
-execute_process(
-    OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
-    RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
-    COMMAND "${get_version_script}" ${get_version_offering}
-                --variant deb
-                "${MEMGRAPH_OVERRIDE_VERSION}"
-                "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
-                "--memgraph-root-dir"
-                "${CMAKE_CURRENT_SOURCE_DIR}"
-)
-if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
-    message(FATAL_ERROR "Unable to get Memgraph DEB version.")
-else()
-    MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
-endif()
-
-# Get version that should be used in the RPM package.
-execute_process(
-    OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
-    RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
-    COMMAND "${get_version_script}" ${get_version_offering}
-                --variant rpm
-                "${MEMGRAPH_OVERRIDE_VERSION}"
-                "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
-                "--memgraph-root-dir"
-                "${CMAKE_CURRENT_SOURCE_DIR}"
-)
-if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
-    message(FATAL_ERROR "Unable to get Memgraph RPM version.")
-else()
-    MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
-endif()
-
-# We want the above variables to be updated each time something is committed to
-# the repository. That is why we include a dependency on the current git HEAD
-# to trigger a new CMake run when the git repository state changes. This is a
-# hack, as CMake doesn't have a mechanism to regenerate variables when
-# something changes (only files can be regenerated).
-# https://cmake.org/pipermail/cmake/2018-October/068389.html
-#
-# The hack in the above link is nearly correct but it has a fatal flaw. The
-# `CMAKE_CONFIGURE_DEPENDS` isn't a `GLOBAL` property, it is instead a
-# `DIRECTORY` property and as such must be set in the `DIRECTORY` scope.
-# https://cmake.org/cmake/help/v3.14/manual/cmake-properties.7.html
-#
-# Unlike the above mentioned hack, we don't use the `.git/index` file. That
-# file changes on every `git add` (even on `git status`) so it triggers
-# unnecessary recalculations of the release version. The release version only
-# changes on every `git commit` or `git checkout`. That is why we watch the
-# following files for changes:
-#   - `.git/HEAD` -> changes each time a `git checkout` is issued
-#   - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
-#     you are on a branch) and you have to monitor the file of the specific
-#     branch to detect when a `git commit` was issued
-# More details about the contents of the `.git` directory and the specific
-# files used can be seen here:
-# https://git-scm.com/book/en/v2/Git-Internals-Git-References
-set(git_directory "${CMAKE_SOURCE_DIR}/.git")
-if (EXISTS "${git_directory}")
-  set_property(DIRECTORY APPEND PROPERTY
-      CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
-  file(STRINGS "${git_directory}/HEAD" git_head_data)
-  if (git_head_data MATCHES "^ref: ")
-    string(SUBSTRING "${git_head_data}" 5 -1 git_head_ref)
-    set_property(DIRECTORY APPEND PROPERTY
-        CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
-  endif()
-endif()
-
+project(memgraph VERSION 0.14.0)
 # -----------------------------------------------------------------------------
 
 # setup CMake module path, defines path for include() and find_package()
@@ -173,28 +62,21 @@ add_custom_target(clean_all
 
 # build flags -----------------------------------------------------------------
 
-# Export the compile commands so that we can use clang-tidy. Additional benefit
-# is easier debugging of compilation and linker flags.
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-# c99-designator is disabled because of required mixture of designated and
-# non-designated initializers in Python Query Module code (`py_module.cpp`).
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-    -Werror=switch -Werror=switch-bool -Werror=return-type \
-    -Werror=return-stack-address \
-    -Wno-c99-designator \
-    -DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
+# TODO: set here 17 once it will be available in the cmake version (3.8)
+# set(CMAKE_CXX_STANDARD 17)
+# set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# For now, explicitly set -std= flag for C++17.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z -Wall \
+    -Werror=switch -Werror=switch-bool -Werror=return-type")
 
 # Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
     "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
 
 # Statically link libgcc and libstdc++, the GCC allows this according to:
-# https://gcc.gnu.org/onlinedocs/gcc-10.2.0/libstdc++/manual/manual/license.html
 # https://www.gnu.org/licenses/gcc-exception-faq.html
-# Last checked for gcc-10.2 which we are using on the build machines.
+# Last checked for gcc-7.3, we are using gcc-6.3 on build machines (license is
+# valid there also).
 # ** If we change versions, recheck this! **
 # ** Static linking is allowed only for executables! **
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
@@ -205,8 +87,6 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
 # release flags
 set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
 
-SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
-
 #debug flags
 set(PREFERRED_DEBUGGER "gdb" CACHE STRING
     "Tunes the debug output for your preferred debugger (gdb or lldb).")
@@ -222,6 +102,13 @@ else()
     set(CMAKE_CXX_FLAGS_DEBUG "-g")
 endif()
 
+# ndebug
+option(NDEBUG "No debug" OFF)
+message(STATUS "NDEBUG: ${NDEBUG} (be careful CMAKE_BUILD_TYPE can also \
+append this flag)")
+if(NDEBUG)
+    add_definitions( -DNDEBUG )
+endif()
 # -----------------------------------------------------------------------------
 
 # default build type is debug
@@ -231,20 +118,15 @@ endif()
 message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
 # -----------------------------------------------------------------------------
 
-if (NOT MG_ARCH)
-    set(MG_ARCH_DESCR "Host architecture to build Memgraph on. Supported values are x86_64, ARM64.")
-    if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")
-        set(MG_ARCH "ARM64" CACHE STRING ${MG_ARCH_DESCR})
-    else()
-        set(MG_ARCH "x86_64" CACHE STRING ${MG_ARCH_DESCR})
-    endif()
-endif()
-message(STATUS "MG_ARCH: ${MG_ARCH}")
-
 # setup external dependencies -------------------------------------------------
 
 # threading
 find_package(Threads REQUIRED)
+
+# optional Ltalloc
+option(USE_LTALLOC "Use Ltalloc instead of default allocator (default OFF). \
+Set this to ON to link with Ltalloc." OFF)
+
 # optional readline
 option(USE_READLINE "Use GNU Readline library if available (default ON). \
 Set this to OFF to prevent linking with Readline even if it is available." ON)
@@ -255,16 +137,68 @@ if (USE_READLINE)
   endif()
 endif()
 
+# OpenSSL
+find_package(OpenSSL REQUIRED)
+
 set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
 add_subdirectory(libs EXCLUDE_FROM_ALL)
 
+include_directories(SYSTEM ${GFLAGS_INCLUDE_DIR})
+include_directories(SYSTEM ${GLOG_INCLUDE_DIR})
+include_directories(SYSTEM ${FMT_INCLUDE_DIR})
+include_directories(SYSTEM ${ANTLR4_INCLUDE_DIR})
+include_directories(SYSTEM ${BZIP2_INCLUDE_DIR})
+include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
+include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIR})
+include_directories(SYSTEM ${CAPNP_INCLUDE_DIR})
+include_directories(SYSTEM ${LIBRDKAFKA_INCLUDE_DIR})
+# -----------------------------------------------------------------------------
+
+# openCypher parser -----------------------------------------------------------
+set(opencypher_frontend ${CMAKE_SOURCE_DIR}/src/query/frontend/opencypher)
+set(opencypher_generated ${opencypher_frontend}/generated)
+set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
+set(opencypher_parser_grammar ${opencypher_frontend}/grammar/MemgraphCypher.g4)
+
+# enumerate all files that are generated from antlr
+set(antlr_opencypher_generated_src
+   ${opencypher_generated}/MemgraphCypherLexer.cpp
+   ${opencypher_generated}/MemgraphCypher.cpp
+   ${opencypher_generated}/MemgraphCypherBaseVisitor.cpp
+   ${opencypher_generated}/MemgraphCypherVisitor.cpp
+)
+
+# Provide a command to generate sources if missing. If this were a
+# custom_target, it would always run and we don't want that.
+add_custom_command(OUTPUT ${antlr_opencypher_generated_src}
+  COMMAND
+  ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
+  COMMAND
+  java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.6-complete.jar -Dlanguage=Cpp -visitor -o ${opencypher_generated} -package antlropencypher ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+  WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+  DEPENDS ${opencypher_lexer_grammar} ${opencypher_parser_grammar}
+          ${opencypher_frontend}/grammar/CypherLexer.g4
+          ${opencypher_frontend}/grammar/Cypher.g4)
+
+# add custom target for generation
+add_custom_target(generate_opencypher_parser
+  DEPENDS ${antlr_opencypher_generated_src})
+
+add_library(antlr_opencypher_parser_lib STATIC ${antlr_opencypher_generated_src})
+target_link_libraries(antlr_opencypher_parser_lib antlr4)
+# -----------------------------------------------------------------------------
+
 # Optional subproject configuration -------------------------------------------
+option(POC "Build proof of concept binaries" OFF)
+option(EXPERIMENTAL "Build experimental binaries" OFF)
+option(CUSTOMERS "Build customer binaries" OFF)
 option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
 option(TOOLS "Build tools binaries" ON)
-option(QUERY_MODULES "Build query modules containing custom procedures" ON)
+option(MG_COMMUNITY "Build Memgraph Community Edition" OFF)
 option(ASAN "Build with Address Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
 option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
 option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
+option(THIN_LTO "Build with link time optimization" OFF)
 
 if (TEST_COVERAGE)
   string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -275,15 +209,11 @@ if (TEST_COVERAGE)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
 endif()
 
-if (MG_ENTERPRISE)
-  add_definitions(-DMG_ENTERPRISE)
+if (MG_COMMUNITY)
+  add_definitions(-DMG_COMMUNITY)
 endif()
 
-set(ENABLE_JEMALLOC ON)
-
 if (ASAN)
-  message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
-  set(ENABLE_JEMALLOC OFF)
   # Enable Addres sanitizer and get nicer stack traces in error messages.
   # NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
   # distribution to symbolize the stack traces (note that ideally the
@@ -330,37 +260,90 @@ if (UBSAN)
   # runtime library and c++ standard libraries are present.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
-  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
-  # Make sure llvm-symbolizer binary is in path.
-  # To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
+  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
+  # Make sure llvm-symbolizer binary is in path
 endif()
 
-set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact Python version used by the query modules")
-set(MG_PYTHON_PATH "" CACHE STRING "Specify the exact Python path used by the query modules")
+if (THIN_LTO)
+  set(CMAKE_CXX_FLAGS"${CMAKE_CXX_FLAGS} -flto=thin")
+  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto=thin")
+endif()
 
 # Add subprojects
 include_directories(src)
 add_subdirectory(src)
 
-# Release configuration
-add_subdirectory(release)
-
-option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
-message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
-
-if (MG_ENABLE_TESTING)
-  enable_testing()
-  add_subdirectory(tests)
+if(POC)
+  add_subdirectory(poc)
 endif()
 
+if(EXPERIMENTAL)
+  add_subdirectory(experimental)
+endif()
+
+if(CUSTOMERS)
+  add_subdirectory(customers)
+endif()
+
+enable_testing()
+add_subdirectory(tests)
+
 if(TOOLS)
   add_subdirectory(tools)
 endif()
 
-if(QUERY_MODULES)
-  add_subdirectory(query_modules)
-endif()
+# -----------------------------------------------------------------------------
 
-install(FILES ${CMAKE_BINARY_DIR}/bin/mgconsole
-  PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
-  TYPE BIN)
+# ---- Setup CPack --------
+# General setup
+set(CPACK_PACKAGE_NAME memgraph)
+set(CPACK_PACKAGE_VENDOR "Memgraph Ltd.")
+set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
+    "High performance, in-memory, transactional graph database")
+set(CPACK_PACKAGE_VERSION_MAJOR ${memgraph_VERSION_MAJOR})
+set(CPACK_PACKAGE_VERSION_MINOR ${memgraph_VERSION_MINOR})
+set(CPACK_PACKAGE_VERSION_PATCH ${memgraph_VERSION_PATCH})
+set(CPACK_PACKAGE_VERSION_TWEAK ${memgraph_VERSION_TWEAK})
+set(CPACK_PACKAGE_FILE_NAME ${CPACK_PACKAGE_NAME}-${memgraph_VERSION}-${COMMIT_HASH}${CPACK_SYSTEM_NAME})
+
+# DEB specific
+# Instead of using "name <email>" format, we use "email (name)" to prevent
+# errors due to full stop, '.' at the end of "Ltd". (See: RFC 822)
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "tech@memgraph.com (Memgraph Ltd.)")
+set(CPACK_DEBIAN_PACKAGE_SECTION non-free/database)
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE https://memgraph.com)
+set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
+    "${CMAKE_SOURCE_DIR}/release/debian/conffiles;"
+    "${CMAKE_SOURCE_DIR}/release/debian/copyright;"
+    "${CMAKE_SOURCE_DIR}/release/debian/prerm;"
+    "${CMAKE_SOURCE_DIR}/release/debian/postrm;"
+    "${CMAKE_SOURCE_DIR}/release/debian/postinst;")
+set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+# Description formatting is important, summary must be followed with a newline and 1 space.
+set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
+ Contains Memgraph, the graph database. It aims to deliver developers the
+ speed, simplicity and scale required to build the next generation of
+ applications driver by real-time connected data.")
+# Add `openssl` package to dependencies list. Used to generate SSL certificates.
+set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0)")
+
+# RPM specific
+set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
+set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
+  /var /var/lib /var/log /etc/logrotate.d
+  /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
+set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
+# NOTE: user specfile has a bug in cmake 3.7.2, this needs to be patched
+# manually in: ~/cmake/share/cmake-3.7/Modules/CPackRPM.cmake line 2273
+# Or newer cmake version used
+set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_SOURCE_DIR}/release/rpm/memgraph.spec.in")
+# Description formatting is important, no line must be greater than 80 characters.
+set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
+It aims to deliver developers the speed, simplicity and scale required to build
+the next generation of applications driver by real-time connected data.")
+# Add `openssl` package to dependencies list. Used to generate SSL certificates.
+set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0")
+
+# All variables must be set before including.
+include(CPack)
+# ---- End Setup CPack ----

CODE_OF_CONDUCT.md

@@ -1,127 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-We as members, contributors, and leaders pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
-nationality, personal appearance, race, caste, color, religion, or sexual
-identity and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-- Demonstrating empathy and kindness toward other people
-- Being respectful of differing opinions, viewpoints, and experiences
-- Giving and gracefully accepting constructive feedback
-- Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-- Focusing on what is best not just for us as individuals, but for the overall
-  community
-
-Examples of unacceptable behavior include:
-
-- The use of sexualized language or imagery, and sexual attention or advances of
-  any kind
-- Trolling, insulting or derogatory comments, and personal or political attacks
-- Public or private harassment
-- Publishing other's private information, such as a physical or email address,
-  without their explicit permission
-- Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Community leaders are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Community leaders have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the community leaders responsible for enforcement at
-[contact@memgraph.com](contact@memgraph.com). All complaints will be reviewed
-and investigated promptly and fairly.
-
-All community leaders are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Community leaders will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from community leaders, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series of
-actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or permanent
-ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior, harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within the
-community.
-
-## Attribution
-
-This Code of Conduct is adapted from the Contributor Covenant, version 2.1,
-available at
-[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html](https://www.contributor-covenant.org/version/2/1/code_of_conduct.html).
-
-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder][mozilla coc].
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq).
-Translations are available at
-[https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations).

CONTRIBUTING.md

@@ -1,121 +0,0 @@
-# How to contribute?
-
-This is a general purpose guide for contributing to Memgraph. We're still
-working out the kinks to make contributing to this project as easy and
-transparent as possible, but we're not quite there yet. Hopefully, this document
-makes the process for contributing clear and answers some questions that you may
-have.
-
-- [How to contribute?](#how-to-contribute)
-  - [Open development](#open-development)
-  - [Branch organization](#branch-organization)
-  - [Bugs & changes](#bugs--changes)
-    - [Where to find known issues?](#where-to-find-known-issues)
-    - [Proposing a change](#proposing-a-change)
-    - [Your first pull request](#your-first-pull-request)
-    - [Sending a pull request](#sending-a-pull-request)
-    - [Style guide](#style-guide)
-  - [How to get in touch?](#how-to-get-in-touch)
-  - [Code of Conduct](#code-of-conduct)
-  - [License](#license)
-  - [Attribution](#attribution)
-
-## Open development
-
-All work on Memgraph is done via [GitHub](https://github.com/memgraph/memgraph).
-Both core team members and external contributors send pull requests which go
-through the same review process.
-
-## Branch organization
-
-Most pull requests should target the [`master
-branch`](https://github.com/memgraph/memgraph/tree/master). We only use separate
-branches for developing new features and fixing bugs before they are merged with
-`master`. We do our best to keep `master` in good shape, with all tests passing.
-
-Code that lands in `master` must be compatible with the latest stable release.
-It may contain additional features but no breaking changes if it's not
-absolutely necessary. We should be able to release a new minor version from the
-tip of `master` at any time.
-
-## Bugs & changes
-
-### Where to find known issues?
-
-We are using [GitHub Issues](https://github.com/memgraph/memgraph/issues) for
-our public bugs. We keep a close eye on this and try to make it clear when we
-have an internal fix in progress. Before filing a new task, try to make sure
-your problem doesn't already exist.
-
-### Proposing a change
-
-If you intend to change the public API, or make any non-trivial changes to the
-implementation, we recommend [filing an
-issue](https://github.com/memgraph/memgraph/issues/new). This lets us reach an
-agreement on your proposal before you put significant effort into it.
-
-If you're only fixing a bug, it's fine to submit a pull request right away but
-we still recommend to file an issue detailing what you're fixing. This is
-helpful in case we don't accept that specific fix but want to keep track of the
-issue.
-
-### Your first pull request
-
-Working on your first Pull Request? You can learn how from this free video
-series:
-
-**[How to Contribute to an Open Source Project on
-GitHub](https://app.egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github)**
-
-If you decide to fix an issue, please be sure to check the comment thread in
-case somebody is already working on a fix. If nobody is working on it at the
-moment, please leave a comment stating that you intend to work on it so other
-people don't accidentally duplicate your effort.
-
-If somebody claims an issue but doesn't follow up for more than two weeks, it's
-fine to take it over but you should still leave a comment.
-
-### Sending a pull request
-
-The core team is monitoring for pull requests. We will review your pull request
-and either merge it, request changes to it, or close it with an explanation.
-**Before submitting a pull request,** please make sure the following is done:
-
-1. Fork [the repository](https://github.com/memgraph/memgraph) and create your
-   branch from `master`.
-2. If you've fixed a bug or added code that should be tested, add tests!
-3. Use the formatter `clang-format` for C/C++ code and `flake8` for Python code.
-   `clang-format` will automatically detect the `.clang-format` file in the root
-   directory while `flake8` can be used with the default configuration.
-
-### Style guide
-
-Memgraph uses the [Google Style
-Guide](https://google.github.io/styleguide/cppguide.html) for C++ in most of its
-code. You should follow them whenever writing new code.
-
-## How to get in touch?
-
-Aside from communicating directly via Pull Requests and Issues, the Memgraph
-Community [Discord Server](https://discord.gg/memgraph) is the best place for
-conversing with project maintainers and other community members.
-
-## [Code of Conduct](https://github.com/memgraph/memgraph/blob/master/CODE_OF_CONDUCT.md)
-
-Memgraph has adopted the [Contributor
-Covenant](https://www.contributor-covenant.org/) as its Code of Conduct, and we
-expect project participants to adhere to it. Please read [the full
-text](https://github.com/memgraph/memgraph/blob/master/CODE_OF_CONDUCT.md) so
-that you can understand what actions will and will not be tolerated.
-
-## License
-
-By contributing to Memgraph, you agree that your contributions will be licensed
-under the [Memgraph licensing
-scheme](https://github.com/memgraph/memgraph/blob/master/LICENSE).
-
-## Attribution
-
-This Contributing guide is adapted from the **React.js Contributing guide**
-available at
-[https://reactjs.org/docs/how-to-contribute.html](https://reactjs.org/docs/how-to-contribute.html).

Doxyfile

@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "The World's Most Powerful Graph Database"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           = docs/doxygen/memgraph_logo.png
+PROJECT_LOGO           = Doxylogo.png
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is
@@ -839,6 +839,7 @@ EXCLUDE_PATTERNS      += */Testing/*
 EXCLUDE_PATTERNS      += */tests/*
 EXCLUDE_PATTERNS      += */dist/*
 EXCLUDE_PATTERNS      += */tools/*
+EXCLUDE_PATTERNS      += */customers/*
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the

LICENSE

@@ -1,5 +0,0 @@
-Source code in this repository is variously licensed under the Business Source
-License 1.1 (BSL), the Memgraph Enterprise License (MEL). A copy of each licence
-can be found in the licences directory. Source code in a given file is licensed
-under the BSL and the copyright belongs to The Memgraph Authors unless
-otherwise noted at th beginning of the file.

README.md

@@ -1,187 +1,24 @@
-<p align="center">
-<img width="400px" src="https://uploads-ssl.webflow.com/5e7ceb09657a69bdab054b3a/5e7ceb09657a6937ab054bba_Black_Original%20_Logo.png">
-</p>
+# memgraph
 
----
+Memgraph is an ACID compliant high performance transactional distributed
+in-memory graph database featuring runtime native query compiling, lock free
+data structures, multi-version concurrency control and asynchronous IO.
 
-<p align="center">
-  <a href="https://github.com/memgraph/memgraph/blob/master/licenses/APL.txt">
-    <img src="https://img.shields.io/badge/license-APL-green" alt="license" title="license"/>
-  </a>
-  <a href="https://github.com/memgraph/memgraph/blob/master/licenses/BSL.txt">
-    <img src="https://img.shields.io/badge/license-BSL-yellowgreen" alt="license" title="license"/>
-  </a>
-  <a href="https://github.com/memgraph/memgraph/blob/master/licenses/MEL.txt" alt="Documentation">
-    <img src="https://img.shields.io/badge/license-MEL-yellow" alt="license" title="license"/>
-  </a>
-</p>
+## dependencies
 
-<p align="center">
-  <a href="https://github.com/memgraph/memgraph">
-     <img src="https://img.shields.io/github/actions/workflow/status/memgraph/memgraph/release_debian10.yaml?branch=master&label=build%20and%20test&logo=github"/>
-  </a>
-  <a href="https://memgraph.com/docs/" alt="Documentation">
-    <img src="https://img.shields.io/badge/documentation-Memgraph-orange" />
-  </a>
-</p>
+Memgraph can be compiled using any modern c++ compiler. It mostly relies on
+the standard template library, however, some things do require external
+libraries.
 
-<p align="center">
-  <a href="https://memgr.ph/join-discord">
-    <img src="https://img.shields.io/badge/Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white" alt="Discord"/>
-  </a>
-</p>
+Some code contains linux-specific libraries and the build is only supported
+on a 64 bit linux kernel.
 
-## :clipboard: Description
-
-Memgraph is an open source graph database built for real-time streaming and
-compatible with Neo4j. Whether you're a developer or a data scientist with
-interconnected data, Memgraph will get you the immediate actionable insights
-fast.
-
-Memgraph directly connects to your streaming infrastructure. You can ingest data
-from sources like Kafka, SQL, or plain CSV files. Memgraph provides a standard
-interface to query your data with Cypher, a widely-used and declarative query
-language that is easy to write, understand and optimize for performance. This is
-achieved by using the property graph data model, which stores data in terms of
-objects, their attributes, and the relationships that connect them. This is a
-natural and effective way to model many real-world problems without relying on
-complex SQL schemas.
-
-Memgraph is implemented in C/C++ and leverages an in-memory first architecture
-to ensure that you’re getting the [best possible
-performance](http://memgraph.com/benchgraph) consistently and without surprises.
-It’s also ACID-compliant and highly available.
-
-## :zap: Features
-
-- Run Python, Rust, and C/C++ code natively, check out the
-  [MAGE](https://github.com/memgraph/mage) graph algorithm library
-- Native support for machine learning
-- Streaming support
-- Replication
-- Authentication and authorization
-- ACID compliance
-
-
-## :video_game: Memgraph Playground
-
-You don't need to install anything to try out Memgraph. Check out 
-our **[Memgraph Playground](https://playground.memgraph.com/)** sandboxes in 
-your browser.
-
-<p align="left">
-  <a href="https://playground.memgraph.com/">
-    <img width="450px" alt="Memgraph Playground" src="https://download.memgraph.com/asset/github/memgraph/memgraph-playground.png">
-  </a>
-</p>
-
-## :floppy_disk: Download & Install
-
-### Windows
-
-[![Windows](https://img.shields.io/badge/Windows-Docker-0078D6?style=for-the-badge&logo=windows&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-windows-docker)
-[![Windows](https://img.shields.io/badge/Windows-WSL-0078D6?style=for-the-badge&logo=windows&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-windows-wsl)
-
-### macOS
-
-[![macOS](https://img.shields.io/badge/macOS-Docker-000000?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-macos-docker)
-[![macOS](https://img.shields.io/badge/lima-AACF41?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu)
-
-### Linux
-
-[![Linux](https://img.shields.io/badge/Linux-Docker-FCC624?style=for-the-badge&logo=linux&logoColor=black)](https://memgraph.com/docs/memgraph/install-memgraph-on-linux-docker)
-[![Debian](https://img.shields.io/badge/Debian-D70A53?style=for-the-badge&logo=debian&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-debian)
-[![Ubuntu](https://img.shields.io/badge/Ubuntu-E95420?style=for-the-badge&logo=ubuntu&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu)
-[![Cent OS](https://img.shields.io/badge/cent%20os-002260?style=for-the-badge&logo=centos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
-[![Fedora](https://img.shields.io/badge/fedora-0B57A4?style=for-the-badge&logo=fedora&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
-[![RedHat](https://img.shields.io/badge/redhat-EE0000?style=for-the-badge&logo=redhat&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
-
-You can find the binaries and Docker images on the [Download
-Hub](https://memgraph.com/download) and the installation instructions in the
-[official documentation](https://memgraph.com/docs/memgraph/installation).
-
-
-## :cloud: Memgraph Cloud
-
-Check out [Memgraph Cloud](https://memgraph.com/docs/memgraph-cloud) - a cloud service fully managed on AWS and available in 6 geographic regions around the world. Memgraph Cloud allows you to create projects with Enterprise instances of MemgraphDB from your browser.
-
-<p align="left">
-  <a href="https://memgraph.com/docs/memgraph-cloud">
-    <img width="450px" alt="Memgraph Cloud" src="https://public-assets.memgraph.com/memgraph-gifs%2Fcloud.gif">
-  </a>
-</p>
-
-## :link: Connect to Memgraph
-
-[Connect to the database](https://memgraph.com/docs/memgraph/connect-to-memgraph) using Memgraph Lab, mgconsole, various drivers (Python, C/C++ and others) and WebSocket. 
-
-### :microscope: Memgraph Lab
-
-Visualize graphs and play with queries to understand your data. [Memgraph Lab](https://memgraph.com/docs/memgraph-lab) is a user interface that helps you explore and manipulate the data stored in Memgraph. Visualize graphs, execute ad hoc queries, and optimize their performance.
-
-<p align="left">
-  <a href="https://memgraph.com/docs/memgraph-lab">
-    <img width="450px" alt="Memgraph Cloud" src="https://public-assets.memgraph.com/memgraph-gifs%2Flab.gif">
-  </a>
-</p>
-
-## :file_folder: Import data
-
-[Import data](https://memgraph.com/docs/memgraph/import-data) into Memgraph using Kafka, RedPanda or Pulsar streams, CSV and JSON files, or Cypher commands.
-
-## :bookmark_tabs: Documentation
-
-The Memgraph documentation is available at
-[memgraph.com/docs](https://memgraph.com/docs).
-
-## :question: Configuration
-
-Command line options that Memgraph accepts are available in the [reference
-guide](https://memgraph.com/docs/memgraph/reference-guide/configuration).
-
-## :trophy: Contributing
-
-The main purpose of this repository is to continue evolving Memgraph, making it
-faster and easier to use. Development of Memgraph happens in the open on GitHub,
-and we are grateful to the community for contributing bug fixes and
-improvements. Read below to learn how you can take part in improving Memgraph.
-
-### Code of Conduct
-
-Memgraph has adopted a Code of Conduct that we expect project participants to
-adhere to. Please read [the full text](CODE_OF_CONDUCT.md) so that you can
-understand what actions will and will not be tolerated.
-
-### Contributing Guide
-
-Read our [contributing guide](CONTRIBUTING.md) to learn about our development
-process and how to propose bug fixes and improvements.
-
-### Internals
-
-Read our
-[internal](https://memgraph.notion.site/Memgraph-Internals-12b69132d67a417898972927d6870bd2)
-docs to learn more about Memgraph's architecture, how to build the project from
-source and how to start contributing. All information related to the database,
-can be found in the aforementioned docs.
-
-### :scroll: License
-
-Memgraph Community is available under the [BSL
-license](./licenses/BSL.txt).</br> Memgraph Enterprise is available under the
-[MEL license](./licenses/MEL.txt).
-
-## :busts_in_silhouette: Community
-
-- :purple_heart: [**Discord**](https://discord.gg/memgraph)
-- :ocean: [**Stack Overflow**](https://stackoverflow.com/questions/tagged/memgraphdb)
-- :busts_in_silhouette: [**Discourse forum**](https://discourse.memgraph.com/)
-- :bird: [**Twitter**](https://twitter.com/memgraphdb)
-- :movie_camera:
-  [**YouTube**](https://www.youtube.com/channel/UCZ3HOJvHGxtQ_JHxOselBYg)
-
-<p align="center">
-  <a href="#">
-    <img src="https://img.shields.io/badge/⬆️ back_to_top_⬆️-white" alt="Back to top" title="Back to top"/>
-  </a>
-</p>
+* linux
+* clang 3.8 (good c++11 support, especially lock free atomics)
+* antlr (compiler frontend)
+* cppitertools
+* fmt format
+* google benchmark
+* google test
+* glog
+* gflags

apollo_archives.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+import json
+import os
+import re
+import subprocess
+import sys
+
+# paths
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+BUILD_OUTPUT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "build_release", "output"))
+
+# helpers
+def run_cmd(cmd, cwd):
+    return subprocess.run(cmd, cwd=cwd, check=True,
+        stdout=subprocess.PIPE).stdout.decode("utf-8")
+
+# check project
+if re.search(r"release", os.environ.get("PROJECT", "")) is None:
+    print(json.dumps([]))
+    sys.exit(0)
+
+# generate archive
+deb_name = run_cmd(["find", ".", "-maxdepth", "1", "-type", "f",
+        "-name", "memgraph*.deb"], BUILD_OUTPUT_DIR).split("\n")[0][2:]
+arch = run_cmd(["dpkg", "--print-architecture"], BUILD_OUTPUT_DIR).split("\n")[0]
+version = deb_name.split("-")[1]
+# generate Debian package file name as expected by Debian Policy
+standard_deb_name = "memgraph_{}-1_{}.deb".format(version, arch)
+
+archive_path = os.path.relpath(os.path.join(BUILD_OUTPUT_DIR,
+        deb_name), SCRIPT_DIR)
+
+archives = [{
+    "name": "Release (deb package)",
+    "archive": archive_path,
+    "filename": standard_deb_name,
+}]
+
+print(json.dumps(archives, indent=4, sort_keys=True))

apollo_archives.yaml

@@ -0,0 +1,18 @@
+- name: Binaries
+  archive:
+    - build_debug/memgraph
+    - build_debug/memgraph_distributed
+    - build_release/memgraph
+    - build_release/memgraph_distributed
+    - build_release/tools/src/mg_client
+    - build_release/tools/src/mg_import_csv
+    - build_release/tools/src/mg_statsd
+    - config
+  filename: binaries.tar.gz
+
+- name: Doxygen documentation
+  cd: docs/doxygen/html
+  archive:
+    - .
+  filename: documentation.tar.gz
+  host: true

apollo_build.yaml

@@ -0,0 +1,89 @@
+- name: Diff build
+  project: ^mg-master-diff$
+  commands: |
+    # Copy untouched repository to parent folder.
+    cd ..
+    cp -r memgraph parent
+    cd memgraph
+
+    # Initialize and create documentation.
+    TIMEOUT=1200 ./init
+    doxygen Doxyfile
+
+    # Remove default build directory.
+    rm -r build
+
+    # Build debug binaries.
+    mkdir build_debug
+    cd build_debug
+    cmake ..
+    TIMEOUT=1200 make -j$THREADS
+
+    # Build coverage binaries.
+    cd ..
+    mkdir build_coverage
+    cd build_coverage
+    cmake -DTEST_COVERAGE=ON ..
+    TIMEOUT=1200 make -j$THREADS memgraph__unit
+
+    # Build release binaries.
+    cd ..
+    mkdir build_release
+    cd build_release
+    cmake -DCMAKE_BUILD_TYPE=release ..
+    TIMEOUT=1200 make -j$THREADS memgraph memgraph_distributed tools memgraph__macro_benchmark memgraph__stress memgraph__manual__card_fraud_generate_snapshot memgraph__feature_benchmark__kafka__benchmark
+
+    # Generate distributed card fraud dataset.
+    cd ../tests/distributed/card_fraud
+    ./generate_dataset.sh
+    cd ../../..
+
+    # Checkout to parent commit and initialize.
+    cd ../parent
+    git checkout HEAD~1
+    TIMEOUT=1200 ./init
+
+    # Build parent release binaries.
+    mkdir build_release
+    cd build_release
+    cmake -DCMAKE_BUILD_TYPE=release ..
+    TIMEOUT=1200 make -j$THREADS memgraph memgraph__macro_benchmark
+
+
+# release build is the default one
+- name: Release build
+  commands: |
+    TIMEOUT=1200 ./init
+    doxygen Doxyfile
+
+    # Remove default build directory.
+    rm -r build
+
+    # Build debug binaries.
+    mkdir build_debug
+    cd build_debug
+    cmake ..
+    TIMEOUT=1200 make -j$THREADS
+
+    # Build coverage binaries.
+    cd ..
+    mkdir build_coverage
+    cd build_coverage
+    cmake -DTEST_COVERAGE=ON ..
+    TIMEOUT=1200 make -j$THREADS memgraph__unit
+
+    # Build release binaries.
+    cd ..
+    mkdir build_release
+    cd build_release
+    cmake -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF ..
+    TIMEOUT=1200 make -j$THREADS
+
+    # Create Debian package.
+    mkdir output
+    cd output
+    cpack -G DEB --config ../CPackConfig.cmake
+
+    # Generate distributed card fraud dataset.
+    cd ../../tests/distributed/card_fraud
+    ./generate_dataset.sh

cmake/FindJemalloc.cmake

@@ -1,55 +0,0 @@
-# Try to find jemalloc library
-#
-# Use this module as:
-#   find_package(Jemalloc)
-#
-# or:
-#   find_package(Jemalloc REQUIRED)
-#
-# This will define the following variables:
-#
-#   Jemalloc_FOUND           True if the system has the jemalloc library.
-#   Jemalloc_INCLUDE_DIRS    Include directories needed to use jemalloc.
-#   Jemalloc_LIBRARIES       Libraries needed to link to jemalloc.
-#
-# The following cache variables may also be set:
-#
-# Jemalloc_INCLUDE_DIR     The directory containing jemalloc/jemalloc.h.
-# Jemalloc_LIBRARY         The path to the jemalloc static library.
-
-find_path(Jemalloc_INCLUDE_DIR NAMES jemalloc/jemalloc.h PATH_SUFFIXES include)
-
-find_library(Jemalloc_LIBRARY NAMES libjemalloc.a PATH_SUFFIXES lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Jemalloc
-  FOUND_VAR Jemalloc_FOUND
-  REQUIRED_VARS
-    Jemalloc_LIBRARY
-    Jemalloc_INCLUDE_DIR
-)
-
-if(Jemalloc_FOUND)
-  set(Jemalloc_LIBRARIES ${Jemalloc_LIBRARY})
-  set(Jemalloc_INCLUDE_DIRS ${Jemalloc_INCLUDE_DIR})
-else()
-  if(Jemalloc_FIND_REQUIRED)
-    message(FATAL_ERROR "Cannot find jemalloc!")
-  else()
-    message(WARNING "jemalloc is not found!")
-  endif()
-endif()
-
-if(Jemalloc_FOUND AND NOT TARGET Jemalloc::Jemalloc)
-  add_library(Jemalloc::Jemalloc UNKNOWN IMPORTED)
-  set_target_properties(Jemalloc::Jemalloc
-    PROPERTIES
-      IMPORTED_LOCATION "${Jemalloc_LIBRARY}"
-      INTERFACE_INCLUDE_DIRECTORIES "${Jemalloc_INCLUDE_DIR}"
-  )
-endif()
-
-mark_as_advanced(
-  Jemalloc_INCLUDE_DIR
-  Jemalloc_LIBRARY
-)

cmake/functions.cmake

@@ -85,3 +85,27 @@ function(get_target_cxx_flags target result)
     endif()
     set(${result} ${flags} PARENT_SCOPE)
 endfunction()
+
+# Define `add_capnp` function for registering a capnp file for generation.
+#
+# The `define_add_capnp` expects 3 arguments:
+#   * name -- name for the function, you usually want `add_capnp`
+#   * main_src_files -- variable to be updated with generated cpp files
+#   * generated_capnp_files -- variable to be updated with generated hpp and cpp files
+#
+# The `add_capnp` function expects a single argument, path to capnp file.
+# Each added file is standalone and we avoid recompiling everything.
+macro(define_add_capnp name main_src_files generated_capnp_files)
+  function(${name} capnp_src_file)
+    set(cpp_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.c++)
+    set(h_file ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file}.h)
+    add_custom_command(OUTPUT ${cpp_file} ${h_file}
+      COMMAND ${CAPNP_EXE} compile -o${CAPNP_CXX_EXE} ${capnp_src_file} -I ${CMAKE_SOURCE_DIR}/src
+      DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_src_file} capnproto-proj
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+    # Update *global* generated_capnp_files
+    set(${generated_capnp_files} ${${generated_capnp_files}} ${cpp_file} ${h_file} PARENT_SCOPE)
+    # Update *global* main_src_files
+    set(${main_src_files} ${${main_src_files}} ${cpp_file} PARENT_SCOPE)
+  endfunction(${name})
+endmacro(define_add_capnp)

config/benchmarking.conf

@@ -0,0 +1,19 @@
+# MEMGRAPH DEFAULT BENCHMARKING CONFIG
+
+# NOTE: all paths are relative to the run folder
+# (where the executable is run)
+
+# no durability
+--durability-enabled=false
+--snapshot-on-exit=false
+--db-recover-on-startup=false
+
+# no GC
+--gc-cycle-sec=-1
+--skiplist_gc_interval=-1
+
+# no query execution time limit
+--query_execution_time_sec=-1
+
+# number of workers
+--num-workers=1

config/community.conf

@@ -0,0 +1,119 @@
+# Default Memgraph Configuration
+#
+# This is the default configuration for memgraph. Settings from this file will
+# be overridden by a configuration file in '$HOME/.memgraph/config', so you can
+# keep this file intact. Additional configuration can be specified in a file
+# pointed to by 'MEMGRAPH_CONFIG' environment variable or by passing arguments
+# on the command line.
+#
+# Each configuration setting is of the form: '--setting-name=value'.
+
+## Database
+
+# IP address the server should listen on.
+--interface=0.0.0.0
+
+# Port the server should listen on.
+--port=7687
+
+# Path to a SSL certificate file that should be used.
+--cert-file=/etc/memgraph/ssl/cert.pem
+
+# Path to a SSL key file that should be used.
+--key-file=/etc/memgraph/ssl/key.pem
+
+# Number of workers used by the Memgraph server. By default, this will be the
+# number of processing units available on the machine.
+# --num-workers=8
+
+# Interval, in seconds, when the garbage collection (GC) should run. GC is used
+# for releasing memory that is no longer needed. For example, deleted graph
+# elements which cannot be seen by any running or new transactions. If set to
+# -1 the GC will never run (use with caution, memory will never get released).
+--gc-cycle-sec=30
+
+# If Memgraph detects there is less available RAM than the given number in MB,
+# it will log a warning.
+--memory-warning-threshold=1024
+
+# The telemetry collects data about the machine that is executing the database
+# (CPU, Memory, OS and Kernel Information) and data about the database runtime
+# (CPU usage, Memory usage, Vertices and Edges count). It is used to provide a
+# better product, easy to disable and does not collect any sensitive data.
+--telemetry-enabled=true
+
+# Memgraph offers an option to store a certain amount of data on a disk. More
+# precisely, the user can pass a list of properties they wish to keep stored on
+# a disk. The property names have to be separated with a comma. An example
+# would be --properties-on-disk=biography,summary.
+#--properties-on-disk=
+
+## Query
+#
+# Various settings related to openCypher query execution.
+
+# Maximum allowed query execution time, in seconds. Any queries exceeding this
+# limit will be aborted. Setting to -1 removes the limit.
+--query-execution-time-sec=30
+
+# Cache generated query execution plans. This speeds up planning repeated
+# queries which produce multiple complex execution plans. The downside is that
+# some executions may use inferior plans if the database state changed. To
+# disable caching, set to false.
+#--query-plan-cache=false
+
+# Time to live for cached query plans, in seconds. This tries to minimize the
+# downside of caching by evicting old plans after the given time.
+#--query-plan-cache-ttl=60
+
+## Durability
+#
+# Memgraph can store database state to persistent storage. Two mechanisms
+# are used: snapshots store the total current database state while write-ahead
+# logs store small changes incrementally. They are used in tandem to provide
+# fast and storage-efficient persistence. Some aspects of snapshot taking
+# and write-ahead logging are configurable.
+--durability-enabled=true
+
+# Path to the directory where snapshots and write-ahead log files will be stored.
+--durability-directory=/var/lib/memgraph/durability
+
+# Recover the database on startup.
+--db-recover-on-startup=true
+
+# Interval of taking snapshots, in seconds. If set to -1, the snapshot feature
+# will be turned off.
+--snapshot-cycle-sec=300
+
+# Create a snapshot when closing Memgraph.
+--snapshot-on-exit=true
+
+# Maximum number of kept snapshots. Old snapshots will be deleted to make room
+# for new ones. If set to -1, the number of kept snapshots is unlimited.
+--snapshot-max-retained=3
+
+# Specifies whether WAL updates should be written on disk immediately after a
+# transaction finishes. Setting this parameter to false does introduce risk of
+# database inconsistency because an operating system or hardware crash might
+# lead to missing transactions in the write-ahead log, but the database will
+# handle this as if those transactions never happened. Turning
+# synchronous-commit off can be a useful trade-off between exact durability and
+# performance.
+--synchronous-commit=false
+
+## Logging
+
+# Path to where the log should be stored.
+--log-file=/var/log/memgraph/memgraph.log
+
+# If true, log messages will go to stderr in addition to logfiles.
+#--also-log-to-stderr=true
+
+## Additional Configuration Inclusion
+
+# Include additional configuration from this file. Settings with the same name
+# will override previously read values. Note, that reading the configuration,
+# which called '--flag-file' will continue after inclusion. Therefore, settings
+# after '--flag-file' may override the included ones.
+#--flag-file=another.conf
+

config/durability_stress.conf

@@ -0,0 +1,15 @@
+# MEMGRAPH DEFAULT DURABILITY STRESS TESTING CONFIG
+
+# NOTE: all paths are relative to the run folder
+# (where the executable is run)
+
+# enable durability
+--durability-enabled=true
+--snapshot-cycle-sec=5
+--snapshot-on-exit=false
+--snapshot-max-retained=2
+
+--db-recover-on-startup=true
+
+# increase query timeout (10 min)
+--query-execution-time-sec=600

config/flags.yaml

@@ -1,113 +0,0 @@
-header: >-
-  Memgraph Configuration
-
-  This is the main configuration file for Memgraph. You can modify this file to
-  suit your specific needs. Additional configuration can be specified by
-  including another configuration file, in a file pointed to by the
-  'MEMGRAPH_CONFIG' environment variable or by passing arguments on the command
-  line.
-
-  Each configuration setting is in the form: '--setting-name=value'.
-
-footer: >-
-  Additional Configuration Inclusion
-
-  You can include additional configuration files from this file. Additional
-  files are processed after this file. Settings that are set in the additional
-  files will override previously set values. Additional configuration files are
-  specified with the '--flag-file' flag.
-
-  Example:
-
-  --flag-file=another.conf
-
-modifications:
-
-  # Each modification should consist of the following parameters:
-  #   * name: the name of the flag that should be modified (with underscores)
-  #           [string]
-  #   * value: the value that should be set instead of the binary provided
-  #            default value [string]
-  #   * override: set to `true` to uncomment the config option by default
-  #               [boolean]
-
-  - name: "data_directory"
-    value: "/var/lib/memgraph"
-    override: true
-
-  - name: "log_file"
-    value: "/var/log/memgraph/memgraph.log"
-    override: true
-
-  - name: "log_level"
-    value: "WARNING"
-    override: true
-
-  - name: "bolt_num_workers"
-    value: ""
-    override: false
-
-  - name: "bolt_cert_file"
-    value: "/etc/memgraph/ssl/cert.pem"
-    override: false
-
-  - name: "bolt_key_file"
-    value: "/etc/memgraph/ssl/key.pem"
-    override: false
-
-  - name: "storage_properties_on_edges"
-    value: "true"
-    override: true
-
-  - name: "storage_recover_on_startup"
-    value: "true"
-    override: true
-
-  - name: "storage_snapshot_interval_sec"
-    value: "300"
-    override: true
-
-  - name: "storage_snapshot_on_exit"
-    value: "true"
-    override: true
-
-  - name: "storage_snapshot_retention_count"
-    value: "3"
-    override: true
-
-  - name: "storage_wal_enabled"
-    value: "true"
-    override: true
-
-  - name: "telemetry_enabled"
-    value: "true"
-    override: true
-
-  - name: "query_modules_directory"
-    value: "/usr/lib/memgraph/query_modules"
-    override: true
-
-  - name: "auth_module_executable"
-    value: "/usr/lib/memgraph/auth_module/example.py"
-    override: false
-
-  - name: "memory_limit"
-    value: "0"
-    override: true
-
-  - name: "isolation_level"
-    value: "SNAPSHOT_ISOLATION"
-    override: true
-
-  - name: "allow_load_csv"
-    value: "true"
-    override: false
-
-undocumented:
-  - "flag_file"
-  - "also_log_to_stderr"
-  - "help"
-  - "help_xml"
-  - "version"
-  - "organization_name"
-  - "license_key"

config/generate.py

@@ -1,116 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import copy
-import os
-import subprocess
-import sys
-import textwrap
-import xml.etree.ElementTree as ET
-
-import yaml
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-CONFIG_FILE = os.path.join(SCRIPT_DIR, "flags.yaml")
-WIDTH = 80
-
-
-def wrap_text(s, initial_indent="# "):
-    return "\n#\n".join(
-        map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "), s.split("\n"))
-    )
-
-
-def extract_flags(binary_path):
-    ret = {}
-    data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
-    root = ET.fromstring(data)
-    for child in root:
-        if child.tag == "usage" and child.text.lower().count("warning"):
-            raise Exception("You should set the usage message!")
-        if child.tag == "flag":
-            flag = {}
-            for elem in child:
-                flag[elem.tag] = elem.text if elem.text is not None else ""
-            flag["override"] = False
-            ret[flag["name"]] = flag
-    return ret
-
-
-def apply_config_to_flags(config, flags):
-    flags = copy.deepcopy(flags)
-    for name in config["undocumented"]:
-        flags.pop(name)
-    for modification in config["modifications"]:
-        name = modification["name"]
-        if name not in flags:
-            print("WARNING: Flag '" + name + "' missing from binary!", file=sys.stderr)
-            continue
-        flags[name]["default"] = modification["value"]
-        flags[name]["override"] = modification["override"]
-    return flags
-
-
-def extract_sections(flags):
-    sections = []
-    other = []
-    current_section = ""
-    current_flags = []
-    for name in sorted(flags.keys()):
-        section = name.split("_")[0]
-        if section == current_section:
-            current_flags.append(name)
-        else:
-            if len(current_flags) < 2:
-                other.extend(current_flags)
-            else:
-                sections.append((current_section, current_flags))
-            current_section = section
-            current_flags = [name]
-    if len(current_flags) < 2:
-        other.extend(current_flags)
-    else:
-        sections.append((current_section, current_flags))
-    sections.append(("other", other))
-    assert set(sum(map(lambda x: x[1], sections), [])) == set(
-        flags.keys()
-    ), "The section extraction algorithm lost some flags!"
-    return sections
-
-
-def generate_config_file(sections, flags):
-    ret = wrap_text(config["header"]) + "\n\n\n"
-    for section, section_flags in sections:
-        ret += wrap_text(section.capitalize(), initial_indent="## ") + "\n\n"
-        for name in section_flags:
-            flag = flags[name]
-            helpstr = flag["meaning"] + " [" + flag["type"] + "]"
-            ret += wrap_text(helpstr) + "\n"
-            prefix = "# " if not flag["override"] else ""
-            ret += prefix + "--" + flag["name"].replace("_", "-") + "=" + flag["default"] + "\n\n"
-        ret += "\n"
-    ret += wrap_text(config["footer"])
-    return ret.strip() + "\n"
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("memgraph_binary", help="path to Memgraph binary")
-    parser.add_argument("output_file", help="path where to store the generated Memgraph " "configuration file")
-    parser.add_argument("--config-file", default=CONFIG_FILE, help="path to generator configuration file")
-
-    args = parser.parse_args()
-    flags = extract_flags(args.memgraph_binary)
-
-    with open(args.config_file) as f:
-        config = yaml.safe_load(f)
-
-    flags = apply_config_to_flags(config, flags)
-    sections = extract_sections(flags)
-    data = generate_config_file(sections, flags)
-
-    dirname = os.path.dirname(args.output_file)
-    if dirname and not os.path.exists(dirname):
-        os.makedirs(dirname)
-
-    with open(args.output_file, "w") as f:
-        f.write(data)

config/stress.conf

@@ -0,0 +1,15 @@
+# MEMGRAPH DEFAULT STRESS TESTING CONFIG
+
+# NOTE: all paths are relative to the run folder
+# (where the executable is run)
+
+# enable durability
+--durability-enabled=true
+--snapshot-cycle-sec=600
+--snapshot-on-exit=true
+--snapshot-max-retained=1
+
+--db-recover-on-startup=false
+
+# increase query timeout (10 min)
+--query-execution-time-sec=600

config/testing.conf

@@ -0,0 +1,9 @@
+# MEMGRAPH DEFAULT TESTING CONFIG
+
+# NOTE: all paths are relative to the run folder
+# (where the executable is run)
+
+# no durability
+--durability-enabled=false
+--snapshot-on-exit=false
+--db-recover-on-startup=false

customers/CMakeLists.txt

@@ -0,0 +1,2 @@
+project(mg_customers)
+add_subdirectory(otto)

customers/elliott/README_internal.md

@@ -0,0 +1,70 @@
+DISCLAIMER: this is just an initial test, graph might not resemble
+the graph in the use case at all and the data might be completely
+irrelevant.
+
+We tried generating a few sample graphs from the vague description
+given in the use case doc. Then we tried writing queries that would
+solve the problem of updating nodes when a leaf value changes,
+assuming all the internal nodes compute only the sum function.
+
+We start by creating an index on `id` property to improve initial lookup
+performance:
+
+    CREATE INDEX ON :Leaf(id)
+
+Set values of all leafs to 1:
+
+    MATCH (u:Leaf) SET u.value = 1
+
+Now we initialize the values of all other nodes in the graph:
+
+    MATCH (u) WHERE NOT u:Leaf SET u.value = 0
+
+    MATCH (u) WITH u
+    ORDER BY u.topological_index DESC
+    MATCH (u)-->(v) SET u.value = u.value + v.value
+
+Change the value of a leaf:
+
+    MATCH (u:Leaf {id: "18"}) SET u.value = 10
+
+We have to reset all the updated nodes to a neutral element:
+
+    MATCH (u:Leaf {id: "18"})<-[* bfs]-(v)
+    WHERE NOT v:Leaf SET v.value = 0
+
+Finally, we recalculate their values in topological order:
+
+    MATCH (u:Leaf {id: "18"})<-[* bfs]-(v)
+    WITH v ORDER BY v.topological_index DESC
+    MATCH (v)-->(w) SET v.value = v.value + w.value
+
+There are a few assumptions made worth pointing out.
+
+* We are able to efficiently maintain topological order
+  of vertices in the graph.
+
+* It is possible to accumulate the value of the function.  Formally: 
+  $$f(x_1, x_2, ..., x_n) = g(...(g(g(x_1, x_2), x_3), ...), x_n)$$
+
+* There is a neutral element for the operation. However, this
+  assumption can be dropped by introducing an artificial neutral element.
+
+Number of operations required is proportional to the sum of degrees of affected
+nodes.
+
+We generated graph with $10^5$ nodes ($20\ 000$ nodes in each layer),  varied the
+degree distribution in node layers and measured time for the query to execute:
+
+| # |  Root-Category-Group degree | Group-CustomGroup-Leaf degree |   Time    |
+|:-:|:---------------------------:|:-----------------------------:|:---------:|
+| 1 |            [1, 10]          |           [20, 40]            |   ~1.1s   |
+| 2 |            [1, 10]          |          [50, 100]            |   ~2.5s   |
+| 3 |           [10, 50]          |          [50, 100]            |   ~3.3s   |
+
+Due to the structure of the graph, update of a leaf required update of almost
+all the nodes in the graph so we don't show times required for initial graph
+update and update after leaf change separately.
+
+However, there is not enough info on the use case to make the test more
+sophisticated.

customers/elliott/README_public.md

@@ -0,0 +1,71 @@
+---
+title: "Elliott Management"
+subtitle: "Proof of Concept Report"
+header-title: "Elliott Management POC"
+date: 2017-10-28
+copyright: "©2017 Memgraph Ltd. All rights reserved."
+titlepage: true
+titlepage-color: FFFFFF
+titlepage-text-color: 101010
+titlepage-rule-color: 101010
+titlepage-rule-height: 1
+...
+
+# Introduction
+
+We tried generating a few sample graphs from the description given at
+the in-person meetings.  Then, we tried writing queries that would solve
+the problem of updating nodes when a leaf value changes, assuming all the
+internal nodes compute only the sum function.
+
+# Technical details
+
+We started by creating an index on `id` property to improve initial lookup
+performance:
+
+    CREATE INDEX ON :Leaf(id)
+
+Afther that, we set values of all leafs to 1:
+
+    MATCH (u:Leaf) SET u.value = 1
+
+We then initialized the values of all other nodes in the graph:
+
+    MATCH (u) WHERE NOT u:Leaf SET u.value = 0
+
+    MATCH (u) WITH u
+    ORDER BY u.topological_index DESC
+    MATCH (u)-->(v) SET u.value = u.value + v.value
+
+Leaf value change and update of affected values in the graph can
+be done using three queries. To change the value of a leaf:
+
+    MATCH (u:Leaf {id: "18"}) SET u.value = 10
+
+Then we had to reset all the affected nodes to the neutral element:
+
+    MATCH (u:Leaf {id: "18"})<-[* bfs]-(v)
+    WHERE NOT v:Leaf SET v.value = 0
+
+Finally, we recalculated their values in topological order:
+
+    MATCH (u:Leaf {id: "18"})<-[* bfs]-(v)
+    WITH v ORDER BY v.topological_index DESC
+    MATCH (v)-->(w) SET v.value = v.value + w.value
+
+There are a few assumptions necessary for the approach above to work.
+
+* We are able to maintain topological order of vertices during graph
+  structure changes.  
+
+* It is possible to accumulate the value of the function.  Formally: 
+  $$f(x_1, x_2, ..., x_n) = g(...(g(g(x_1, x_2), x_3), ...), x_n)$$
+
+* There is a neutral element for the operation. However, this
+  assumption can be dropped by introducing an artificial neutral element.
+
+Above assumptions could be changed, relaxed or dropped, depending on the
+specifics of the use case.
+
+Number of operations required is proportional to the sum of degrees of affected
+nodes. 

customers/elliott/commands.txt

@@ -0,0 +1,12 @@
+CREATE INDEX ON :Leaf(id);
+MATCH (u:Leaf) SET u.value = 1;
+MATCH (u) WHERE NOT u:Leaf SET u.value = 0;
+MATCH (u) WITH u 
+ORDER BY u.topological_index DESC
+MATCH (u)-->(v) SET u.value = u.value + v.value;
+MATCH (u:Leaf {id: "85000"}) SET u.value = 10;
+MATCH (u:Leaf {id: "85000"})<-[* bfs]-(v) 
+WHERE NOT v:Leaf SET v.value = 0;
+MATCH (u:Leaf {id: "85000"})<-[* bfs]-(v)
+WITH v ORDER BY v.topological_index DESC
+MATCH (v)-->(w) SET v.value = v.value + w.value;

customers/elliott/generate_dag

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""Generates a DAG from JSON spec in [config] and outputs nodes to
+[filename]_nodes, and edges to [filename]_edges in format convertible
+to Memgraph snapshot.
+
+Here's an example JSON spec:
+
+    {
+        "layers": [
+            {
+                "name": "A",
+                "sublayers": 1,
+                "degree_lo": 1,
+                "degree_hi": 3,
+                "nodes": 4
+            },
+            {
+                "name": "B",
+                "sublayers": 3,
+                "degree_lo": 2,
+                "degree_hi": 3,
+                "nodes": 10
+            },
+            {
+                "name": "C",
+                "sublayers": 1,
+                "degree_lo": 1,
+                "degree_hi": 1,
+                "nodes": 5
+            }
+        ]
+    }
+
+Nodes from each layer will be randomly divided into sublayers.  A node can
+only have edges pointing to nodes in lower sublayers of the same layer, or
+to nodes from the layer directly below it. Out-degree is chosen uniformly
+random from [degree_lo, degree_hi] interval."""
+
+import argparse
+from itertools import accumulate
+import json
+import random
+
+
+def _split_into_sum(n, k):
+    assert 1 <= n, "n should be at least 1"
+    assert k <= n, "k shouldn't be greater than n"
+    xs = [0] + sorted(random.sample(range(1, n), k-1)) + [n]
+    return [b - a for a, b in zip(xs, xs[1:])]
+
+
+def generate_dag(graph_config, seed=None):
+    random.seed(seed)
+
+    nodes = []
+    edges = []
+
+    layer_lo = 1
+    for layer in graph_config:
+        sublayers = _split_into_sum(layer['nodes'], layer['sublayers'])
+        sub_range = accumulate([layer_lo] + sublayers)
+        layer['sublayer_range'] = list(sub_range)
+        nodes.extend([
+            (u, layer['name'])
+            for u in range(layer_lo, layer_lo + layer['nodes'])
+        ])
+        layer_lo += layer['nodes']
+
+    edges = []
+
+    for layer, next_layer in zip(graph_config, graph_config[1:]):
+        degree_lo = layer['degree_lo']
+        degree_hi = layer['degree_hi']
+
+        sub_range = layer['sublayer_range']
+        sub_range_next = next_layer['sublayer_range']
+
+        layer_lo = sub_range[0]
+        next_layer_hi = sub_range_next[-1]
+
+        for sub_lo, sub_hi in zip(sub_range, sub_range[1:]):
+            for u in range(sub_lo, sub_hi):
+                num_edges = random.randint(degree_lo, degree_hi)
+                for _ in range(num_edges):
+                    v = random.randint(sub_hi, next_layer_hi - 1)
+                    edges.append((u, v))
+
+        for sub_lo, sub_hi in zip(sub_range_next, sub_range_next[1:]):
+            for u in range(sub_lo, sub_hi):
+                v = random.randint(layer_lo, sub_lo - 1)
+                edges.append((v, u))
+
+    return nodes, edges
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description=__doc__)
+    parser.add_argument('config', type=str, help='graph config JSON file')
+    parser.add_argument('filename', type=str,
+                        help='nodes will be stored to filename_nodes, '
+                        'edges to filename_edges')
+    parser.add_argument('--seed', type=int,
+                        help='seed for the random generator (default = '
+                        'current system time)')
+    args = parser.parse_args()
+
+    with open(args.config, 'r') as f:
+        graph_config = json.loads(f.read())['layers']
+
+    nodes, edges = generate_dag(graph_config, seed=args.seed)
+
+    #  print nodes into CSV file
+    with open('{}_nodes'.format(args.filename), 'w') as out:
+        out.write('nodeId:ID(Node),name,topological_index:Int,:LABEL\n')
+        for node_id, layer in nodes:
+            out.write('{0},{1}{0},{0},{1}\n'.format(node_id, layer))
+
+    # print edges into CSV file
+    with open('{}_edges'.format(args.filename), 'w') as out:
+        out.write(':START_ID(Node),:END_ID(Node),:TYPE\n')
+        for u, v in edges:
+            out.write('{},{},child\n'.format(u, v))

customers/elliott/graph1.json

@@ -0,0 +1,39 @@
+{
+    "layers": [
+        { 
+            "name": "Root",
+            "sublayers": 1,
+            "degree_lo": 1,
+            "degree_hi": 10,
+            "nodes": 20000
+        },
+        { 
+            "name": "Category",
+            "sublayers": 5,
+            "degree_lo": 1,
+            "degree_hi": 10,
+            "nodes": 20000
+        },
+        {   
+            "name": "Group",
+            "sublayers": 1,
+            "degree_lo": 20,
+            "degree_hi": 40,
+            "nodes": 20000
+        },
+        {
+            "name": "CustomGroup",
+            "sublayers": 15,
+            "degree_lo": 20,
+            "degree_hi": 40,
+            "nodes": 20000
+        },
+        { 
+            "name": "Leaf",
+            "sublayers": 1,
+            "degree_lo": 1,
+            "degree_hi": 1,
+            "nodes": 20000
+        }
+    ]
+}

customers/elliott/graph2.json

@@ -0,0 +1,39 @@
+{
+    "layers": [
+        { 
+            "name": "Root",
+            "sublayers": 1,
+            "degree_lo": 1,
+            "degree_hi": 10,
+            "nodes": 20000
+        },
+        { 
+            "name": "Category",
+            "sublayers": 5,
+            "degree_lo": 1,
+            "degree_hi": 10,
+            "nodes": 20000
+        },
+        {   
+            "name": "Group",
+            "sublayers": 1,
+            "degree_lo": 50,
+            "degree_hi": 100,
+            "nodes": 20000
+        },
+        {
+            "name": "CustomGroup",
+            "sublayers": 15,
+            "degree_lo": 50,
+            "degree_hi": 100,
+            "nodes": 20000
+        },
+        { 
+            "name": "Leaf",
+            "sublayers": 1,
+            "degree_lo": 50,
+            "degree_hi": 100,
+            "nodes": 20000
+        }
+    ]
+}

customers/elliott/graph3.json

@@ -0,0 +1,39 @@
+{
+    "layers": [
+        { 
+            "name": "Root",
+            "sublayers": 1,
+            "degree_lo": 10,
+            "degree_hi": 50,
+            "nodes": 20000
+        },
+        { 
+            "name": "Category",
+            "sublayers": 5,
+            "degree_lo": 10,
+            "degree_hi": 50,
+            "nodes": 20000
+        },
+        {   
+            "name": "Group",
+            "sublayers": 1,
+            "degree_lo": 50,
+            "degree_hi": 100,
+            "nodes": 20000
+        },
+        {
+            "name": "CustomGroup",
+            "sublayers": 15,
+            "degree_lo": 50,
+            "degree_hi": 100,
+            "nodes": 20000
+        },
+        { 
+            "name": "Leaf",
+            "sublayers": 1,
+            "degree_lo": 50,
+            "degree_hi": 100,
+            "nodes": 20000
+        }
+    ]
+}

customers/otto/CMakeLists.txt

@@ -0,0 +1,3 @@
+set(exec_name customers_otto_parallel_connected_components)
+add_executable(${exec_name} parallel_connected_components.cpp)
+target_link_libraries(${exec_name} memgraph_lib)

customers/otto/connected_components_test.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+This script attempts to evaluate the feasibility of using Memgraph for
+Otto group's usecase. The usecase is finding connected componentes in
+a large, very sparse graph (cca 220M nodes, 250M edges), based on a dynamic
+inclusion / exclusion of edges (w.r.t variable parameters and the source node
+type).
+
+This implementation defines a random graph with the given number of nodes
+and edges and looks for connected components using breadth-first expansion.
+Edges are included / excluded based on a simple expression, only demonstrating
+possible usage.
+"""
+
+from argparse import ArgumentParser
+import logging
+from time import time
+from collections import defaultdict
+from math import log2
+from random import randint
+
+from neo4j.v1 import GraphDatabase
+
+log = logging.getLogger(__name__)
+
+
+def generate_graph(sess, node_count, edge_count):
+    # An index that will speed-up edge creation.
+    sess.run("CREATE INDEX ON :Node(id)").consume()
+
+    # Create the given number of nodes with a randomly selected type from:
+    # [0.5, 1.5, 2.5].
+    sess.run(("UNWIND range(0, {} - 1) AS id CREATE "
+              "(:Node {{id: id, type: 0.5 + tointeger(rand() * 3)}})").format(
+                 node_count)).consume()
+
+    # Create the given number of edges, each with a 'value' property of
+    # a random [0, 3.0) float. Each edge connects two random nodes, so the
+    # expected node degree is (edge_count * 2 / node_count). Generate edges
+    # so the connectivity is non-uniform (to produce connected components of
+    # various sizes).
+    sess.run(("UNWIND range(0, {0} - 1) AS id WITH id "
+              "MATCH (from:Node {{id: tointeger(rand() * {1})}}), "
+              "(to:Node {{id: tointeger(rand() * {1} * id / {0})}}) "
+              "CREATE (from)-[:Edge {{value: 3 * rand()}}]->(to)").format(
+                 edge_count, node_count)).consume()
+
+
+def get_connected_ids(sess, node_id):
+    # Matches a node with the given ID and returns the IDs of all the nodes
+    # it is connected to. Note that within the BFS lambda expression there
+    # is an expression used to filter out edges expanded over.
+    return sess.run((
+        "MATCH (from:Node {{id: {}}})-"
+        "[*bfs (e, n | abs(from.type - e.value) < 0.80)]-(d) "
+        "RETURN count(*) AS c").format(node_id)).data()[0]['c']
+
+
+def parse_args():
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument('--endpoint', type=str, default='localhost:7687',
+                        help='Memgraph instance endpoint. ')
+    parser.add_argument('--node-count', type=int, default=1000,
+                        help='The number of nodes in the graph')
+    parser.add_argument('--edge-count', type=int, default=1000,
+                        help='The number of edges in the graph')
+    parser.add_argument('--sample-count', type=int, default=None,
+                        help='The number of samples to take')
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    logging.basicConfig(level=logging.INFO)
+    log.info("Memgraph - Otto test database generator")
+    logging.getLogger("neo4j").setLevel(logging.WARNING)
+
+    driver = GraphDatabase.driver(
+        'bolt://' + args.endpoint,
+        auth=("ignored", "ignored"),
+        encrypted=False)
+    sess = driver.session()
+
+    sess.run("MATCH (n) DETACH DELETE n").consume()
+
+    log.info("Generating graph with %s nodes and %s edges...",
+             args.node_count, args.edge_count)
+    generate_graph(sess, args.node_count, args.edge_count)
+
+    # Track which vertices have been found as part of a component.
+    start_time = time()
+    max_query_time = 0
+    log.info("Looking for connected components...")
+    # Histogram of log2 sizes of connected components found.
+    histogram = defaultdict(int)
+    sample_count = args.sample_count if args.sample_count else args.node_count
+    for i in range(sample_count):
+        node_id = randint(0, args.node_count - 1)
+        query_start_time = time()
+        log2_size = int(log2(1 + get_connected_ids(sess, node_id)))
+        max_query_time = max(max_query_time, time() - query_start_time)
+        histogram[log2_size] += 1
+    elapsed = time() - start_time
+    log.info("Connected components found in %.2f sec (avg %.2fms, max %.2fms)",
+             elapsed, elapsed / sample_count * 1000, max_query_time * 1000)
+    log.info("Component size histogram (count | range)")
+    for log2_size, count in sorted(histogram.items()):
+        log.info("\t%5d | %d - %d", count, 2 ** log2_size,
+                 2 ** (log2_size + 1) - 1)
+
+    sess.close()
+    driver.close()
+
+
+if __name__ == '__main__':
+    main()

customers/otto/parallel_connected_components.cpp

@@ -0,0 +1,207 @@
+#include <algorithm>
+#include <limits>
+#include <mutex>
+#include <random>
+#include <set>
+#include <stack>
+#include <thread>
+
+#include "gflags/gflags.h"
+#include "glog/logging.h"
+
+#include "data_structures/union_find.hpp"
+#include "database/graph_db.hpp"
+#include "database/graph_db_accessor.hpp"
+#include "storage/property_value.hpp"
+#include "threading/sync/spinlock.hpp"
+#include "utils/bound.hpp"
+#include "utils/timer.hpp"
+
+DEFINE_int32(thread_count, 1, "Number of threads");
+DEFINE_int32(vertex_count, 1000, "Number of vertices");
+DEFINE_int32(edge_count, 1000, "Number of edges");
+DECLARE_int32(gc_cycle_sec);
+
+static const std::string kLabel{"kLabel"};
+static const std::string kProperty{"kProperty"};
+
+void GenerateGraph(database::GraphDb &db) {
+  {
+    database::GraphDbAccessor dba{db};
+    dba.BuildIndex(dba.Label(kLabel), dba.Property(kProperty));
+    dba.Commit();
+  }
+
+  // Randomize the sequence of IDs of created vertices and edges to simulate
+  // real-world lack of locality.
+  auto make_id_vector = [](size_t size) {
+    gid::Generator generator{0};
+    std::vector<gid::Gid> ids(size);
+    for (size_t i = 0; i < size; ++i)
+      ids[i] = generator.Next(std::experimental::nullopt);
+    std::random_shuffle(ids.begin(), ids.end());
+    return ids;
+  };
+
+  std::vector<VertexAccessor> vertices;
+  vertices.reserve(FLAGS_vertex_count);
+  {
+    CHECK(FLAGS_vertex_count % FLAGS_thread_count == 0)
+        << "Thread count must be a factor of vertex count";
+    LOG(INFO) << "Generating " << FLAGS_vertex_count << " vertices...";
+    utils::Timer timer;
+    auto vertex_ids = make_id_vector(FLAGS_vertex_count);
+
+    std::vector<std::thread> threads;
+    SpinLock vertices_lock;
+    for (int i = 0; i < FLAGS_thread_count; ++i) {
+      threads.emplace_back([&db, &vertex_ids, &vertices, &vertices_lock, i]() {
+        database::GraphDbAccessor dba{db};
+        auto label = dba.Label(kLabel);
+        auto property = dba.Property(kProperty);
+        auto batch_size = FLAGS_vertex_count / FLAGS_thread_count;
+        for (int j = i * batch_size; j < (i + 1) * batch_size; ++j) {
+          auto vertex = dba.InsertVertex(vertex_ids[j]);
+          vertex.add_label(label);
+          vertex.PropsSet(property, static_cast<int64_t>(vertex_ids[j]));
+          vertices_lock.lock();
+          vertices.emplace_back(vertex);
+          vertices_lock.unlock();
+        }
+        dba.Commit();
+      });
+    }
+    for (auto &t : threads) t.join();
+    LOG(INFO) << "Generated " << FLAGS_vertex_count << " vertices in "
+              << timer.Elapsed().count() << " seconds.";
+  }
+  {
+    database::GraphDbAccessor dba{db};
+    for (int i = 0; i < FLAGS_vertex_count; ++i)
+      vertices[i] = *dba.Transfer(vertices[i]);
+
+    LOG(INFO) << "Generating " << FLAGS_edge_count << " edges...";
+    auto edge_ids = make_id_vector(FLAGS_edge_count);
+    std::mt19937 pseudo_rand_gen{std::random_device{}()};
+    std::uniform_int_distribution<> rand_dist{0, FLAGS_vertex_count - 1};
+    auto edge_type = dba.EdgeType("edge");
+    utils::Timer timer;
+    for (int i = 0; i < FLAGS_edge_count; ++i)
+      dba.InsertEdge(vertices[rand_dist(pseudo_rand_gen)],
+                     vertices[rand_dist(pseudo_rand_gen)], edge_type,
+                     edge_ids[i]);
+    dba.Commit();
+    LOG(INFO) << "Generated " << FLAGS_edge_count << " edges in "
+              << timer.Elapsed().count() << " seconds.";
+  }
+}
+
+auto EdgeIteration(database::GraphDb &db) {
+  database::GraphDbAccessor dba{db};
+  int64_t sum{0};
+  for (auto edge : dba.Edges(false)) sum += edge.from().gid() + edge.to().gid();
+  return sum;
+}
+
+auto VertexIteration(database::GraphDb &db) {
+  database::GraphDbAccessor dba{db};
+  int64_t sum{0};
+  for (auto v : dba.Vertices(false))
+    for (auto e : v.out()) sum += e.gid() + e.to().gid();
+  return sum;
+}
+
+auto ConnectedComponentsEdges(database::GraphDb &db) {
+  UnionFind<int64_t> connectivity{FLAGS_vertex_count};
+  database::GraphDbAccessor dba{db};
+  for (auto edge : dba.Edges(false))
+    connectivity.Connect(edge.from().gid(), edge.to().gid());
+  return connectivity.Size();
+}
+
+auto ConnectedComponentsVertices(database::GraphDb &db) {
+  UnionFind<int64_t> connectivity{FLAGS_vertex_count};
+  database::GraphDbAccessor dba{db};
+  for (auto from : dba.Vertices(false)) {
+    for (auto out_edge : from.out())
+      connectivity.Connect(from.gid(), out_edge.to().gid());
+  }
+  return connectivity.Size();
+}
+
+auto ConnectedComponentsVerticesParallel(database::GraphDb &db) {
+  UnionFind<int64_t> connectivity{FLAGS_vertex_count};
+  SpinLock connectivity_lock;
+
+  // Define bounds of vertex IDs for each thread to use.
+  std::vector<PropertyValue> bounds;
+  for (int64_t i = 0; i < FLAGS_thread_count; ++i)
+    bounds.emplace_back(i * FLAGS_vertex_count / FLAGS_thread_count);
+  bounds.emplace_back(std::numeric_limits<int64_t>::max());
+
+  std::vector<std::thread> threads;
+  for (int i = 0; i < FLAGS_thread_count; ++i) {
+    threads.emplace_back(
+        [&connectivity, &connectivity_lock, &bounds, &db, i]() {
+          database::GraphDbAccessor dba{db};
+          for (auto from :
+               dba.Vertices(dba.Label(kLabel), dba.Property(kProperty),
+                            utils::MakeBoundInclusive(bounds[i]),
+                            utils::MakeBoundExclusive(bounds[i + 1]), false)) {
+            for (auto out_edge : from.out()) {
+              std::lock_guard<SpinLock> lock{connectivity_lock};
+              connectivity.Connect(from.gid(), out_edge.to().gid());
+            }
+          }
+        });
+  }
+  for (auto &t : threads) t.join();
+  return connectivity.Size();
+}
+
+auto Expansion(database::GraphDb &db) {
+  std::vector<int> component_ids(FLAGS_vertex_count, -1);
+  int next_component_id{0};
+  std::stack<VertexAccessor> expansion_stack;
+  database::GraphDbAccessor dba{db};
+  for (auto v : dba.Vertices(false)) {
+    if (component_ids[v.gid()] != -1) continue;
+    auto component_id = next_component_id++;
+    expansion_stack.push(v);
+    while (!expansion_stack.empty()) {
+      auto next_v = expansion_stack.top();
+      expansion_stack.pop();
+      if (component_ids[next_v.gid()] != -1) continue;
+      component_ids[next_v.gid()] = component_id;
+      for (auto e : next_v.out()) expansion_stack.push(e.to());
+      for (auto e : next_v.in()) expansion_stack.push(e.from());
+    }
+  }
+
+  return next_component_id;
+}
+
+int main(int argc, char **argv) {
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  google::InitGoogleLogging(argv[0]);
+  FLAGS_gc_cycle_sec = -1;
+
+  database::SingleNode db;
+  GenerateGraph(db);
+  auto timed_call = [&db](auto callable, const std::string &descr) {
+    LOG(INFO) << "Running " << descr << "...";
+    utils::Timer timer;
+    auto result = callable(db);
+    LOG(INFO) << "\tDone in " << timer.Elapsed().count()
+              << " seconds, result: " << result;
+  };
+  timed_call(EdgeIteration, "Edge iteration");
+  timed_call(VertexIteration, "Vertex iteration");
+  timed_call(ConnectedComponentsEdges, "Connected components - Edges");
+  timed_call(ConnectedComponentsVertices, "Connected components - Vertices");
+  timed_call(ConnectedComponentsVerticesParallel,
+             "Parallel connected components - Vertices");
+  timed_call(Expansion, "Expansion");
+
+  return 0;
+}

customers/ravelin/fraud_detection.oc

@@ -0,0 +1,5 @@
+WITH tointeger(rand() * 40000000) AS from_id
+MATCH (from:Node {id : from_id}) WITH from
+MATCH path = (from)-[*bfs..50 (e, n | degree(n) < 50)]->(to) WITH path LIMIT 10000 WHERE to.fraudulent
+RETURN path, size(path)
+

customers/ravelin/snapshot_gen_config.json

@@ -0,0 +1,31 @@
+{
+   "indexes":[
+      "Node.id"
+   ],
+   "nodes":[
+      {
+         "count":40000000,
+         "labels":[
+            "Node"
+         ],
+         "properties":{
+            "id":{
+               "type":"counter",
+               "param":"Node.id"
+            },
+            "fraudulent":{
+               "type":"bernoulli",
+               "param":0.0005
+            }
+         }
+      }
+   ],
+   "edges":[
+      {
+         "count":80000000,
+         "from":"Node",
+         "to":"Node",
+         "type":"Edge"
+      }
+   ]
+}

customers/strata2018/card_fraud_config.json

@@ -0,0 +1,20 @@
+{
+  "indexes" : ["Card.id", "Pos.id", "Transaction.fraud_reported"],
+  "nodes" : [
+    {
+      "count_per_worker" : 1250000,
+      "label" : "Card"
+    },
+    {
+      "count_per_worker" : 1250000,
+      "label" : "Pos"
+    },
+    {
+      "count_per_worker" : 2500000,
+      "label" : "Transaction"
+    }
+  ],
+  "compromised_pos_probability" : 0.2,
+  "fraud_reported_probability" : 0.1,
+  "hop_probability" : 0.1
+}

docs/csv-import-tool/README.md

@@ -1,204 +0,0 @@
-# CSV Import Tool Documentation
-
-CSV is a universal and very versatile data format used to store large quantities
-of data.  Each Memgraph database instance has a CSV import tool installed called
-`mg_import_csv`.  The CSV import tool should be used for initial bulk ingestion
-of data into the database.  Upon ingestion, the CSV importer creates a snapshot
-that will be used by the database to recover its state on its next startup.
-
-If you are already familiar with the Neo4j bulk import tool, then using the
-`mg_import_csv` tool should be easy.  The CSV import tool is fully compatible
-with the [Neo4j CSV
-format](https://neo4j.com/docs/operations-manual/current/tools/import/). If you
-already have a pipeline set-up for Neo4j, you should only replace `neo4j-admin
-import` with `mg_import_csv`.
-
-## CSV File Format
-
-Each row of a CSV file represents a single entry that should be imported into
-the database.  Both nodes and relationships can be imported into the database
-using CSV files.
-
-Each set of CSV files must have a header that describes the data that is stored
-in the CSV files. Each field in the CSV header is in the format
-`<name>[:<type>]` which identifies the name that should be used for that column
-and the type that should be used for that column. The type is optional and
-defaults to `string` (see the following chapter).
-
-Each CSV field must be divided using the delimiter and each CSV field can either
-be quoted or unquoted. When the field is quoted, the first and last character in
-the field *must* be the quote character. If the field isn't quoted, and a quote
-character appears in it, it is treated as a regular character. If a quote
-character appears inside a quoted string then the quote character must be
-doubled in order to escape it. Line feeds and carriage returns are ignored in
-the CSV file, also, the file can't contain a NULL character.
-
-## Properties
-
-Both nodes and relationships can have properties added to them.  When importing
-properties, the CSV importer uses the name specified in the header of the
-corresponding CSV column for the name of the property.  A property is designated
-by specifying one of the following types in the header:
- - `integer`, `int`, `long`, `byte`, `short`: creates an integer property
- - `float`, `double`: creates a float property
- - `boolean`, `bool`: creates a boolean property
- - `string`, `char`: creates a string property
-
-When importing a boolean value, the CSV field should contain exactly the text
-`true` to import a `True` boolean value.  All other text values are treated as a
-boolean value `False`.
-
-If you want to import an array of values, you can do so by appending `[]` to any
-of the above types.  The values of the array are then determined by splitting
-the raw CSV value using the array delimiter character.
-
-Assuming that the array delimiter is `;`, the following example:
-```plaintext
-first_name,last_name:string,number:integer,aliases:string[]
-John,Doe,1,Johnny;Jo;J-man
-Melissa,Doe,2,Mel
-```
-
-Will yield these results:
-```plaintext
-CREATE ({first_name: "John", last_name: "Doe", number: 1, aliases: ["Johnny", "Jo", "J-man"]});
-CREATE ({first_name: "Melissa", last_name: "Doe", number: 2, aliases: ["Mel"]});
-```
-### Nodes
-
-When importing nodes, several more types can be specified in the header of the
-CSV file (along with all property types):
- - `ID`: id of the node that should be used as the node ID when importing
-   relationships
- - `LABEL`: designates that the field contains additional labels for the node
- - `IGNORE`: designates that the field should be ignored
-
-The `ID` field type sets the internal ID that will be used for the node when
-creating relationships.  It is optional and nodes that don't have an ID value
-specified will be imported, but can't be connected to any relationships.  If you
-want to save the ID value as a property in the database, just specify a name for
-the ID (`user_id:ID`).  If you just want to use the ID during the import, leave
-out the name of the field (`:ID`).  The `ID` field also supports creating
-separate ID spaces.  The ID space is specified with the ID space name appended
-to the `ID` type in parentheses (`ID(user)`).  That allows you to have the same
-IDs (by value) for multiple different node files (for example, numbers from 1 to
-N).  The IDs in each ID space will be treated as an independent set of IDs that
-don't interfere with IDs in another ID space.
-
-The `LABEL` field type adds additional labels to the node.  The value is treated
-as an array type so that multiple additional labels can be specified for each
-node.  The value is split using the array delimiter (`--array-delimiter` flag).
-
-### Relationships
-
-In order to be able to import relationships, you must import the nodes in the
-same invocation of `mg_import_csv` that is used to import the relationships.
-
-When importing relationships, several more types can be specified in the header
-of the CSV file (along with all property types):
- - `START_ID`: id of the start node that should be connected with the
-   relationship
- - `END_ID`: id of the end node that should be connected with the relationship
- - `TYPE`: designates the type of the relationship
- - `IGNORE`: designates that the field should be ignored
-
-The `START_ID` field type sets the start node that should be connected with the
-relationship to the end node.  The field *must* be specified and the node ID
-must be one of the node IDs that were specified in the node CSV files.  The name
-of this field is ignored.  If the node ID is in an ID space, you can specify the
-ID space for the in the same way as for the node ID (`START_ID(user)`).
-
-The `END_ID` field type sets the end node that should be connected with the
-relationship to the start node.  The field *must* be specified and the node ID
-must be one of the node IDs that were specified in the node CSV files.  The name
-of this field is ignored.  If the node ID is in an ID space, you can specify the
-ID space for the in the same way as for the node ID (`END_ID(user)`).
-
-The `TYPE` field type sets the type of the relationship.  Each relationship
-*must* have a relationship type, but it doesn't necessarily need to be specified
-in the CSV file, it can also be set externally for the whole CSV file.  The name
-of this field is ignored.
-
-## CSV Importer Flags
-
-The importer has many command line options that allow you to customize the way
-the importer loads your data.
-
-The two main flags that are used to specify the input CSV files are `--nodes`
-and `--relationships`. Basic description of these flags is provided in the table
-and more detailed explainion can be found further down bellow.
-
-
-| Flag                  | Description  |
-|-----------------------| -------------- |
-|`--nodes`                | Used to specify CSV files that contain the nodes to the importer. |
-|`--relationships`        |  Used to specify CSV files that contain the relationships to the importer.|
-|`--delimiter`            | Sets the delimiter that should be used when splitting the CSV fields (default `,`)|
-|`--quote`                | Sets the quote character that should be used to quote a CSV field (default `"`)|
-|`--array-delimiter`      | Sets the delimiter that should be used when splitting array values (default `;`)|
-|`--id-type`              | Specifies which data type should be used to store the supplied <br /> node IDs when storing them as properties (if the field name is supplied). <br /> The supported values are either `STRING` or `INTEGER`. (default `STRING`)|
-|`--ignore-empty-strings` | Instructs the importer to treat all empty strings as `Null` values  <br /> instead of an empty string value (default `false`)|
-|`--ignore-extra-columns` | Instructs the importer to ignore all columns (instead of raising an error) <br /> that aren't specified after the last specified column in the CSV header. (default `false`) |
-| `--skip-bad-relationships`| Instructs the importer to ignore all relationships (instead of raising an error) <br /> that refer to nodes that don't exist in the node files. (default `false`) |
-|`--skip-duplicate-nodes`  | Instructs the importer to ignore all duplicate nodes (instead of raising an error).  <br /> Duplicate nodes are nodes that have an ID that is the same as another node that was already imported. (default `false`) |
-| `--trim-strings`| Instructs the importer to trim all of the loaded CSV field values before processing them further. <br /> Trimming the fields removes all leading and trailing whitespace from them. (default `false`) |
-
-The `--nodes` and  `--relationships` flags are used to specify CSV files that
-contain the nodes and relationships to the importer.  Multiple files can be
-specified in each supplied `--nodes` or `--relationships` flag. Files that are
-supplied in one `--nodes` or `--relationships` flag are treated by the CSV
-parser as one big CSV file.  Only the first line of the first file is parsed for
-the CSV header, all other files (and rows) are treated as data.  This is useful
-when you have a very large CSV file and don't want to edit its first line just
-to add a CSV header.  Instead, you can specify the header in a separate file
-(e.g. `users_header.csv` or `friendships_header.csv`) and have the data intact
-in the large file (e.g. `users.csv` or `friendships.csv`).  Also, you can supply
-additional labels for each set of node files.
-
-The format of `--nodes` flag is:
-`[<label>[:<label>]...=]<file>[,<file>][,<file>]...`.  Take note that only the
-first `<file>` part is mandatory, all other parts of the flag value are
-optional. Multiple `--nodes` flags can be supplied to describe multiple sets of
-different node files.  For the importer to work, at least one `--nodes` flag
-*must* be supplied.
-
-The format of `--relationships` flag is: `[<type>=]<file>[,<file>][,<file>]...`.
-Take note that only the first `<file>` part is mandatory, all other parts of the
-flag value are optional.  Multiple `--relationships` flags can be supplied to
-describe multiple sets of different relationship files.  The `--relationships`
-flag isn't mandatory.
-
-## CSV Parser Logic
-
-The CSV parser uses the same logic as the standard Python CSV parser.  The data
-is parsed in the same way as the following snippet:
-
-```python
-import csv
-for row in csv.reader(stream, strict=True):
-    # process 'row'
-```
-
-Python uses 'excel' as the default dialect when parsing CSV files and the
-default settings for the CSV parser are:
- - delimiter: `','`
- - doublequote: `True`
- - escapechar: `None`
- - lineterminator: `'\r\n'`
- - quotechar: `'"'`
- - skipinitialspace: `False`
-
-The above snippet can be expanded to:
-
-```python
-import csv
-for row in csv.reader(stream, delimiter=',', doublequote=True,
-                      escapechar=None, lineterminator='\r\n',
-                      quotechar='"', skipinitialspace=False,
-                      strict=True):
-    # process 'row'
-```
-
-For more information about the meaning of the above values, see:
-https://docs.python.org/3/library/csv.html#csv.Dialect
-

docs/dev/cpp-code-conventions.md

@@ -0,0 +1,256 @@
+# C++ Code Conventions
+
+This chapter describes code conventions which should be followed when writing
+C++ code.
+
+## Code Style
+
+Memgraph uses the
+[Google Style Guide for C++](https://google.github.io/styleguide/cppguide.html)
+in most of its code.  You should follow them whenever writing new code.
+
+### Often Overlooked Style Conventions
+
+#### Pointers & References
+
+References provide a shorter syntax for accessing members and better declare
+the intent that a pointer *should* not be `nullptr`. They do not prevent
+accessing a `nullptr` and obfuscate the client/calling code because the
+reference argument is passed just like a value. Errors with such code have
+been very difficult to debug. Therefore, pointers are always used. They will
+not prevent bugs but will make some of them more obvious when reading code.
+
+The only time a reference can be used is if it is `const`. Note that this
+kind of reference is not allowed if it is stored somewhere, i.e. in a class.
+You should use a pointer to `const` then. The primary reason being is that
+references obscure the semantics of moving an object, thus making bugs with
+references pointing to invalid memory harder to track down.
+
+[Style guide reference](https://google.github.io/styleguide/cppguide.html#Reference_Arguments)
+
+#### Constructors & RAII
+
+RAII (Resource Acquisition is Initialization) is a nice mechanism for managing
+resources. It is especially useful when exceptions are used, such as in our
+code. Unfortunately, they do have 2 major downsides.
+
+  * Only exceptions can be used for to signal failure.
+  * Calls to virtual methods are not resolved as expected.
+
+For those reasons the style guide recommends minimal work that cannot fail.
+Using virtual methods or doing a lot more should be delegated to some form of
+`Init` method, possibly coupled with factory functions. Similar rules apply to
+destructors, which are not allowed to even throw exceptions.
+
+[Style guide reference](https://google.github.io/styleguide/cppguide.html#Doing_Work_in_Constructors)
+
+### Additional Style Conventions
+
+Old code may have broken Google C++ Style accidentally, but the new code
+should adhere to it as close as possible. We do have some exceptions
+to Google style as well as additions for unspecified conventions.
+
+#### Using C++ Exceptions
+
+Unlike Google, we do not forbid using exceptions.
+
+But, you should be very careful when using them and introducing new ones. They
+are indeed handy, but cause problems with understanding the control flow since
+exceptions are another form of `goto`. It also becomes very hard to determine
+that the program is in correct state after the stack is unwound and the thrown
+exception handled. Other than those issues, throwing exceptions in destructors
+will terminate the program. The same will happen if a thread doesn't handle an
+exception even though it is not the main thread.
+
+[Style guide reference](https://google.github.io/styleguide/cppguide.html#Exceptions)
+
+#### Assertions
+
+We use `CHECK` and `DCHECK` macros from glog library. You are encouraged to
+use them as often as possible to both document and validate various pre and
+post conditions of a function.
+
+`CHECK` remains even in release build and should be preferred over it's cousin
+`DCHECK` which only exists in debug builds. The primary reason is that you
+want to trigger assertions in release builds in case the tests didn't
+completely validate all code paths. It is better to fail fast and crash the
+program, than to leave it in undefined state and potentially corrupt end
+user's work. In cases when profiling shows that `CHECK` is causing visible
+slowdown you should switch to `DCHECK`.
+
+#### Template Parameter Naming
+
+Template parameter names should start with capital letter 'T' followed by a
+short descriptive name. For example:
+
+```cpp
+template <typename TKey, typename TValue>
+class KeyValueStore
+```
+
+## Code Formatting
+
+You should install `clang-format` and run it on code you change or add. The
+root of Memgraph's project contains the `.clang-format` file, which specifies
+how formatting should behave. Running `clang-format -style=file` in the
+project's root will read the file and behave as expected. For ease of use, you
+should integrate formatting with your favourite editor.
+
+The code formatting isn't enforced, because sometimes manual formatting may
+produce better results. Though, running `clang-format` is strongly encouraged.
+
+## Documentation
+
+Besides following the comment guidelines from [Google Style
+Guide](https://google.github.io/styleguide/cppguide.html#Comments), your
+documentation of the public API should be
+[Doxygen](https://github.com/doxygen/doxygen) compatible. For private parts of
+the code or for comments accompanying the implementation, you are free to
+break doxygen compatibility. In both cases, you should write your
+documentation as full sentences, correctly written in English.
+
+## Doxygen
+
+To start a Doxygen compatible documentation string, you should open your
+comment with either a JavaDoc style block comment (`/**`) or a line comment
+containing 3 slashes (`///`). Take a look at the 2 examples below.
+
+### Block Comment
+
+```cpp
+/**
+ * One sentence, brief description.
+ *
+ * Long form description.
+ */
+```
+
+### Line Comment
+
+```cpp
+/// One sentence, brief description.
+///
+/// Long form description.
+```
+
+If you only have a brief description, you may collapse the documentation into
+a single line.
+
+### Block Comment
+
+```cpp
+/** Brief description. */
+```
+
+### Line Comment
+
+```cpp
+/// Brief description.
+```
+
+Whichever style you choose, keep it consistent across the whole file.
+
+Doxygen supports various commands in comments, such as `@file` and `@param`.
+These help Doxygen to render specified things differently or to track them for
+cross referencing. If you want to learn more, take a look at these two links:
+
+  * http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html
+  * http://www.stack.nl/~dimitri/doxygen/manual/commands.html
+
+## Examples
+
+Below are a few examples of documentation from the codebase.
+
+### Function
+
+```cpp
+/**
+ * Removes whitespace characters from the start and from the end of a string.
+ *
+ * @param s String that is going to be trimmed.
+ *
+ * @return Trimmed string.
+ */
+inline std::string Trim(const std::string &s);
+```
+
+### Class
+
+```cpp
+/** Base class for logical operators.
+ *
+ *  Each operator describes an operation, which is to be performed on the
+ *  database. Operators are iterated over using a @c Cursor. Various operators
+ *  can serve as inputs to others and thus a sequence of operations is formed.
+ */
+class LogicalOperator
+    : public ::utils::Visitable<HierarchicalLogicalOperatorVisitor> {
+ public:
+  /** Constructs a @c Cursor which is used to run this operator.
+   *
+   *  @param GraphDbAccessor Used to perform operations on the database.
+   */
+  virtual std::unique_ptr<Cursor> MakeCursor(GraphDbAccessor &db) const = 0;
+
+  /** Return @c Symbol vector where the results will be stored.
+   *
+   *  Currently, outputs symbols are only generated in @c Produce operator.
+   *  @c Skip, @c Limit and @c OrderBy propagate the symbols from @c Produce (if
+   *  it exists as input operator). In the future, we may want this method to
+   *  return the symbols that will be set in this operator.
+   *
+   *  @param SymbolTable used to find symbols for expressions.
+   *  @return std::vector<Symbol> used for results.
+   */
+  virtual std::vector<Symbol> OutputSymbols(const SymbolTable &) const {
+    return std::vector<Symbol>();
+  }
+
+  virtual ~LogicalOperator() {}
+};
+```
+
+### File Header
+
+```cpp
+/// @file visitor.hpp
+///
+/// This file contains the generic implementation of visitor pattern.
+///
+/// There are 2 approaches to the pattern:
+///
+///   * classic visitor pattern using @c Accept and @c Visit methods, and
+///   * hierarchical visitor which also uses @c PreVisit and @c PostVisit
+///   methods.
+///
+/// Classic Visitor
+/// ===============
+///
+/// Explanation on the classic visitor pattern can be found from many
+/// sources, but here is the link to hopefully most easily accessible
+/// information: https://en.wikipedia.org/wiki/Visitor_pattern
+///
+/// The idea behind the generic implementation of classic visitor pattern is to
+/// allow returning any type via @c Accept and @c Visit methods. Traversing the
+/// class hierarchy is relegated to the visitor classes. Therefore, visitor
+/// should call @c Accept on children when visiting their parents. To implement
+/// such a visitor refer to @c Visitor and @c Visitable classes.
+///
+/// Hierarchical Visitor
+/// ====================
+///
+/// Unlike the classic visitor, the intent of this design is to allow the
+/// visited structure itself to control the traversal. This way the internal
+/// children structure of classes can remain private. On the other hand,
+/// visitors may want to differentiate visiting composite types from leaf types.
+/// Composite types are those which contain visitable children, unlike the leaf
+/// nodes. Differentiation is accomplished by providing @c PreVisit and @c
+/// PostVisit methods, which should be called inside @c Accept of composite
+/// types. Regular @c Visit is only called inside @c Accept of leaf types.
+/// To implement such a visitor refer to @c CompositeVisitor, @c LeafVisitor and
+/// @c Visitable classes.
+///
+/// Implementation of hierarchical visiting is modelled after:
+/// http://wiki.c2.com/?HierarchicalVisitorPattern
+```
+

docs/dev/diagram/dependencies.dot

@@ -0,0 +1,288 @@
+// dot -Tpng dependencies.dot -o /path/to/output.png
+
+// TODO (buda): Put PropertyValueStore to storage namespace
+
+digraph {
+  // At the beginning of each block there is a default style for that block
+  label="Memgraph Dependencies Diagram"; fontname="Roboto Bold"; fontcolor=black;
+    fontsize=26; labelloc=top; labeljust=right;
+  compound=true; // If true, allow edges between clusters
+  rankdir=TB; // Alternatives: LR
+  node [shape=record fontname="Roboto", fontsize=12, fontcolor=white];
+  edge [color="#B5AFB7"];
+
+  // -- Legend --
+  // dir=both arrowtail=diamond arrowhead=vee -> group ownership
+  // dir=both arrowtail=none, arrowhead=vee -> ownership; stack or uptr
+
+  subgraph cluster_tcp_end_client_communication {
+    label="TCP End Client Communication"; fontsize=14;
+    node [style=filled, color="#DD2222" fillcolor="#DD2222"];
+
+    // Owned elements
+    "communication::Server";
+    "io::network::Socket";
+
+    // Intracluster connections
+    "communication::Server" -> "io::network::Socket"
+      [label="socket_" dir=both arrowtail=none arrowhead=vee];
+  }
+
+  subgraph cluster_bolt_server {
+    label="Bolt Server"; fontsize=14;
+    node [style=filled, color="#62A2CA" fillcolor="#62A2CA"];
+
+    // Owned elements
+    "communication::bolt::SessionData";
+    "communication::bolt::Session";
+    "communication::bolt::Encoder";
+    "communication::bolt::Decoder";
+
+    // Intracluster connections
+    "communication::bolt::Session" -> "communication::bolt::Encoder"
+      [label="encoder_", dir=both arrowtail=none, arrowhead=vee];
+    "communication::bolt::Session" -> "communication::bolt::Decoder"
+      [label="decoder_", dir=both arrowtail=none, arrowhead=vee];
+  }
+
+  subgraph cluster_opencypher_engine {
+    label="openCypher Engine"; fontsize=14;
+    node [style=filled, color="#68BDF6" fillcolor="#68BDF6"];
+
+    // Owned Elements
+    "query::Interpreter";
+    "query::AstTreeStorage";
+    "query::TypedValue"
+    "query::Path";
+    "query::Simbol";
+    "query::Context";
+    "query::ExpressionEvaluator";
+    "query::Frame";
+    "query::SymbolTable";
+    "query::plan::LogicalOperator";
+    "query::plan::Cursor";
+    "query::plan::CostEstimator";
+
+    // Intracluster connections
+    "query::Interpreter" -> "query::AstTreeStorage"
+      [label="ast_cache" dir=both arrowtail=diamond arrowhead=vee];
+    "query::TypedValue" -> "query::Path";
+    "query::plan::Cursor" -> "query::Frame";
+    "query::plan::Cursor" -> "query::Context";
+    "query::plan::LogicalOperator" -> "query::Symbol";
+    "query::plan::LogicalOperator" -> "query::SymbolTable";
+    "query::plan::LogicalOperator" -> "query::plan::Cursor";
+  }
+
+
+  subgraph cluster_storage {
+    label="Storage" fontsize=14;
+    node [style=filled, color="#FB6E00" fillcolor="#FB6E00"];
+
+    // Owned Elements
+    "database::GraphDb";
+    "database::GraphDbAccessor";
+    "storage::Record";
+    "storage::Vertex";
+    "storage::Edge";
+    "storage::RecordAccessor";
+    "storage::VertexAccessor";
+    "storage::EdgeAccessor";
+    "storage::Common";
+    "storage::Label";
+    "storage::EdgeType";
+    "storage::Property";
+    "storage::compression";
+    "storage::SingleNodeConcurrentIdMapper";
+    "storage::Location";
+    "storage::StorageTypesLocation";
+    "PropertyValueStore";
+    "storage::RecordLock";
+    "mvcc::Version";
+    "mvcc::Record";
+    "mvcc::VersionList";
+
+    // Intracluster connections
+    "storage::VertexAccessor" -> "storage::RecordAccessor"
+      [arrowhead=onormal];
+    "storage::EdgeAccessor" -> "storage::RecordAccessor"
+      [arrowhead=onormal];
+    "storage::RecordAccessor" -> "database::GraphDbAccessor"
+      [style=dashed arrowhead=vee];
+    "storage::Vertex" -> "mvcc::Record"
+      [arrowhead=onormal];
+    "storage::Edge" -> "mvcc::Record"
+      [arrowhead=onormal];
+    "storage::Edge" -> "PropertyValueStore"
+      [arrowhead=vee];
+    "storage::Vertex" -> "PropertyValueStore"
+      [arrowhead=vee];
+    "storage::Edge" -> "mvcc::VersionList"
+      [label="from,to" arrowhead=vee style=dashed];
+    "storage::VertexAccessor" -> "storage::Vertex"
+      [arrowhead=vee];
+    "storage::EdgeAccessor" -> "storage::Edge"
+      [arrowhead=vee];
+    "storage::SingleNodeConcurrentIdMapper" -> "storage::StorageTypesLocation"
+      [arrowhead=vee];
+    "storage::StorageTypesLocation" -> "storage::Location"
+      [arrowhead=vee];
+    "storage::Storage" -> "storage::StorageTypesLocation"
+      [arrowhead=vee];
+    "storage::Property" -> "storage::Common"
+      [arrowhead=onormal];
+    "storage::Label" -> "storage::Common"
+      [arrowhead=onormal];
+    "storage::EdgeType" -> "storage::Common"
+      [arrowhead=onormal];
+    "storage::Property" -> "storage::Location"
+      [arrowhead=vee];
+    "PropertyValueStore" -> "storage::Property"
+      [arrowhead=vee];
+    "PropertyValueStore" -> "storage::Location"
+      [arrowhead=vee];
+    "database::GraphDbAccessor" -> "database::GraphDb"
+      [arrowhead=vee];
+    "database::GraphDbAccessor" -> "tx::TransactionId"
+      [arrowhead=vee];
+    "mvcc::VersionList" -> "storge::RecordLock"
+      [label="lock" arrowhead=vee];
+    "mvcc::VersionList" -> "mvcc::Record"
+      [label="head" arrowhead=vee];
+    "mvcc::Record" -> "mvcc::Version"
+      [arrowhead=onormal];
+
+    // Explicit positioning
+    {rank=same;
+        "database::GraphDbAccessor";
+        "storage::VertexAccessor";
+        "storage::EdgeAccessor";}
+    {rank=same;
+        "storage::Common";
+        "storage::compression";}
+  }
+
+  subgraph cluster_properties_on_disk {
+    label="Properties on Disk" fontsize=14;
+    node [style=filled, color="#102647" fillcolor="#102647"];
+
+    // Owned Elements
+    "storage::KVStore";
+    "rocksdb";
+
+    // Intracluster connections
+    "storage::KVStore" -> "rocksdb";
+  }
+
+  subgraph cluster_distributed {
+    label="Distributed" fontsize=14;
+    node [style=filled, color="#FFC500" fillcolor="#FFC500"];
+
+    // Owned Elements
+    "distributed::DataManager";
+    "distributed::DataRpcClients";
+
+    // Intracluster connections
+    "distributed::DataManager" -> "distributed::DataRpcClients"
+      [arrowhead=vee];
+    "storage::RecordAccessor" -> "distributed::DataManager"
+      [style=dashed arrowhead=vee];
+  }
+
+  subgraph cluster_dynamic_partitioning {
+    label="Dynamic Partitioning" fontsize=14;
+    node [style=filled, color="#720096" fillcolor="#720096"];
+
+    // Owned Elements
+    "DynamicPartitioner";
+  }
+
+  subgraph cluster_security {
+    label="Security" fontsize=14;
+    node [style=filled, color="#857F87" fillcolor="#857F87"];
+
+    // Owned Elements
+    "Communication Encryption";
+    "Data Encryption";
+    "Access Control";
+    "Audit Logging";
+  }
+
+  subgraph cluster_web_dashboard {
+    label="Dashaboard" fontsize=14;
+    node [style=filled, color="#FF0092" fillcolor="#FF0092"];
+
+    // Owned Elements
+    "Memgraph Ops / Memgraph Cockpit";
+  }
+
+  subgraph cluster_rpc {
+    label="RPC" fontsize=14;
+    node [style=filled, color="#857F87" fillcolor="#857F87"];
+
+    // Owned Elements
+    "communication::rpc::Server";
+    "communication::rpc::Client";
+  }
+
+  subgraph cluster_ingestion {
+    label="Ingestion" fontsize=14;
+    node [style=filled, color="#0B6D88" fillcolor="#0B6D88"];
+
+    // Owned Elements
+    "Extract";
+    "Transform";
+    "Load";
+    "Amazon S3";
+    "Kafka";
+
+    // Intracluster connections
+    "Extract" -> "Amazon S3";
+    "Extract" -> "Kafka";
+
+    // Explicit positioning
+    {rank=same;"Extract";"Transform";"Load";}
+  }
+
+  // -- Intercluster connections --
+  // cluster_tcp_end_client_communication -- cluster_bolt_server
+  "communication::Server" -> "communication::bolt::SessionData" [color=black];
+  "communication::Server" -> "communication::bolt::Session" [color=black];
+  // cluster_bolt_server -> cluster_storage
+  "communication::bolt::SessionData" -> "database::GraphDb" [color=red];
+  "communication::bolt::Session" -> "database::GraphDbAccessor" [color=red];
+  // cluster_bolt_server -> cluster_opencypher_engine
+  "communication::bolt::SessionData" -> "query::Interpreter" [color=red];
+  // cluster_opencypher_engine -- cluster_storage
+  "query::Interpreter" -> "database::GraphDbAccessor" [color=black];
+  "query::Interpreter" -> "storage::VertexAccessor" [color=black];
+  "query::Interpreter" -> "storage::EdgeAccessor" [color=black];
+  "query::TypedValue" -> "storage::VertexAccessor" [color=black];
+  "query::TypedValue" -> "storage::EdgeAccessor" [color=black];
+  "query::Path" -> "storage::VertexAccessor"
+    [label="vertices" dir=both arrowtail=diamond arrowhead=vee color=black];
+  "query::Path" -> "storage::EdgeAccessor"
+    [label="edges" dir=both arrowtail=diamond arrowhead=vee color=black];
+  "query::plan::LogicalOperator" -> "database::GraphDbAccessor"
+    [color=black arrowhead=vee];
+  // cluster_distributed -- cluster_storage
+  "distributed::DataManager" -> "database::GraphDb"
+    [arrowhead=vee style=dashed color=red];
+  "distributed::DataManager" -> "tx::TransactionId"
+    [label="ves_caches_key" dir=both arrowhead=none arrowtail=diamond
+     color=red];
+  "distributed::DataManager" -> "storage::Vertex"
+    [label="vertices_caches" dir=both arrowhead=none arrowtail=diamond
+     color=red];
+  "distributed::DataManager" -> "storage::Edge"
+    [label="edges_caches" dir=both arrowhead=none arrowtail=diamond
+     color=red];
+  // cluster_storage -- cluster_properties_on_disk
+  "PropertyValueStore" -> "storage::KVStore"
+    [label="static" arrowhead=vee color=black];
+  // cluster_dynamic_partitioning -- cluster_storage
+  "database::GraphDb" -> "DynamicPartitioner"
+    [arrowhead=vee color=red];
+  "DynamicPartitioner" -> "database::GraphDbAccessor"
+    [arrowhead=vee color=black];
+}

docs/dev/diagram/dgp/logical.dot

@@ -0,0 +1,22 @@
+digraph {
+  // label="Dynamig Graph Partitioning";
+  fontname="Roboto Bold"; fontcolor=black;
+    fontsize=26; labelloc=top; labeljust=center;
+  compound=true; // If true, allow edges between clusters
+  rankdir=TB; // Alternatives: LR
+  node [shape=record fontname="Roboto", fontsize=12, fontcolor=white
+        style=filled, color="#FB6E00" fillcolor="#FB6E00"];
+  edge [color="#B5AFB7"];
+
+  "distributed::DistributedGraphDb" -> "distributed::TokenSharingRpcServer";
+
+  "distributed::TokenSharingRpcServer" -> "communication::rpc::Server";
+  "distributed::TokenSharingRpcServer" -> "distributed::Coordination";
+  "distributed::TokenSharingRpcServer" -> "distributed::TokenSharingRpcClients";
+  "distributed::TokenSharingRpcServer" -> "distributed::dgp::Partitioner";
+
+  "distributed::dgp::Partitioner" -> "distributed::DistributedGraphDb" [style=dashed];
+
+  "distributed::dgp::Partitioner" -> "distributed::dgp::VertexMigrator";
+  "distributed::dgp::VertexMigrator" -> "database::GraphDbAccessor" [style=dashed];
+}

docs/dev/distributed/addressing.md

@@ -0,0 +1,43 @@
+# Distributed addressing
+
+In distributed Memgraph a single graph element must be owned by exactly
+one worker. It is possible that multiple workers have cached copies of
+a single graph element (which is inevitable), but there is only one
+owner.
+
+The owner of a graph element can change. This is not yet implemented,
+but is intended. Graph partitioning is intended to be dynamic.
+
+Graph elements refer to other graph elements that are possibly on some
+other worker. Even though each graph element is identified with a unique
+ID, that ID does not contain the information about where that element
+currently resides (which worker is the owner).
+
+Thus we introduce the concept of a global address. It indicates both
+which graph element is referred to (it's global ID), and where it
+resides. Semantically it's a pair of two elements, but for efficiency
+it's stored in 64 bits.
+
+The global address is efficient for usage in a cluster: it indicates
+where something can be found. However, finding a graph element based on
+it's ID is still not a free operation (in the current implementation
+it's a skiplist lookup). So, whenever possible, it's better to use local
+addresses (pointers).
+
+Succinctly, the requirements for addressing are:
+- global addressing containing location info
+- fast local addressing
+- storage of both types in the same location efficiently
+- translation between the two
+
+The `storage::Address` class handles the enumerated storage
+requirements. It stores either a local or global address in the size of
+a local pointer (typically 8 bytes).
+
+Conversion between the two is done in multiple places. The general
+approach is to use local addresses (when possible) only for local
+in-memory handling. All the communication and persistence uses global
+addresses. Also, when receiving address from another worker, attempt to
+localize addresses as soon as possible, so that least code has to worry
+about potential inefficiency of using a global address for a local graph
+element.

docs/dev/distributed/durability.md

@@ -0,0 +1,50 @@
+# Distributed durability
+
+Durability in distributed is slightly different then in single-node as
+the state itself is shared between multiple workers and none of those
+states are independent.
+
+Note that recovering from persistent storage must result in a stable
+database state. This means that across the cluster the state
+modification of every transaction that was running is either recovered
+fully or not at all. Also, if transaction A committed before transaction B,
+then if B is recovered so must A.
+
+## Snapshots
+
+It is possibly avoidable but highly desirable that the database can be
+recovered from snapshot only, without relying on WAL files. For this to
+be possible in distributed, it must be ensured that the same
+transactions are recovered on all the workers (including master) in the
+cluster. Since the snapshot does not contain information about which
+state change happened in which transaction, the only way to achieve this
+is to have synchronized snapshots. This means that the process of
+creating a snapshot, which is in itself transactional (it happens within
+a transaction and thus observes some consistent database state), must
+happen in the same transaction. This is achieved by the master starting
+a snapshot generating transaction and triggering the process on all
+workers in the cluster.
+
+## WAL
+
+Unlike the snapshot, write-ahead logs contain the information on which
+transaction made which state change. This makes it possible to include
+or exclude transactions during the recovery process. What is necessary
+however is a global consensus on which of the transactions should be
+recovered and which not, to ensure recovery into a consistent state.
+
+It would be possible to achieve this with some kind of synchronized
+recovery process, but it would impose constraints on cluster startup and
+would not be trivial.
+
+A simpler alternative is that the consensus is achieved beforehand,
+while the database (to be recovered) is still operational. What is
+necessary is to keep track of which transactions are guaranteed to
+have been flushed to the WAL files on all the workers in the cluster. It
+makes sense to keep this record on the master, so a mechanism is
+introduced which periodically pings all the workers, telling them to
+flush their WALs, and writes some sort of a log indicating that this has
+been confirmed. The downside of this is a periodic broadcast must be
+done, and that potentially slightly less data can be recovered in the
+case of a crash then if using a post-crash consensus. It is however much
+simpler to implement.

docs/dev/distributed/dynamic_graph_partitioning.md

@@ -0,0 +1,51 @@
+## Dynamic Graph Partitioning
+
+Memgraph supports dynamic graph partitioning similar to the Spinner algorithm, 
+mentioned in this paper: [https://arxiv.org/pdf/1404.3861.pdf].
+
+Dgp is useful because it tries to group `local` date on the same worker, i.e. 
+it tries to keep closely connected data on one worker. It tries to avoid jumps 
+across workers when querying/traversing the distributed graph.
+
+### Our implementation
+
+It works independently on each worker but it is always running the migration 
+on only one worker at the same time. It achieves that by sharing a token 
+between workers, and the token ownership is transferred to the next worker 
+when the current worker finishes its migration step. 
+
+The reason that we want workers to work in  disjoint time slots is it avoid 
+serialization errors caused by creating/removing edges of vertices during 
+migrations, which might cause an update of some vertex from two or more 
+different transactions.
+
+### Migrations
+
+For each vertex and workerid (label in the context of Dgp algorithm) we define  
+a score function. Score function takes into account labels of surrounding 
+endpoints of vertex edges (in/out) and the capacity of the worker with said 
+label. Score function loosely looks like this 
+```
+locality(v, l) = 
+count endpoints of edges of vertex `v` with label `l` / degree of `v`
+
+capacity(l) = 
+number of vertices on worker `l` divided by the worker capacity 
+(usually equal to the average number of vertices per worker)
+
+score(v, l) = locality(v, l) - capacity(l)
+```
+We also define two flags alongside ```dynamic_graph_partitioner_enabled```, 
+   ```dgp_improvement_threshold``` and ```dgp_max_batch_size```.
+
+These two flags are used during the migration phase.
+When deciding if we need to migrate some vertex `v` from worker `l1` to worker 
+`l2` we examine the difference in scores, i.e.
+if score(v, l1) - dgp_improvement_threshold / 100 < score(v, l2) then we 
+migrate the vertex. 
+
+Max batch size flag limits the number of vertices we can transfer in one batch 
+(one migration step). 
+Setting this value to a too large value will probably cause
+a lot of interference with client queries, and having it a small value
+will slow down convergence of the algorithm.

docs/dev/distributed/general_concepts.md

@@ -0,0 +1,54 @@
+# Memgraph distributed
+
+This chapter describes some of the concepts used in distributed
+Memgraph. By "distributed" here we mean the sharding of a single graph
+onto multiple processing units (servers).
+
+## Conceptual organization
+
+There is a single master and multiple workers. The master contains all
+the global sources of truth (transaction engine,
+[label|edge-type|property] to name mappings). Also, in the current
+organization it is the only one that contains a Bolt server (for
+communication with the end client) and an interpretation engine. Workers
+contain the data and means of subquery interpretation (query plans
+recieved from the master) and means of communication with the master and
+other workers.
+
+In many query plans the load on the master is much larger then the load
+on the workers. For that reason it might be beneficial to make the
+master contain less data (or none at all), and/or having multiple
+interpretation masters.
+
+## Logic organization
+
+Both the distributed and the single node Memgraph use the same codebase.
+In cases where the behavior in single-node differs from that in
+distributed, some kind of dynamic behavior change is implemented (either
+through inheritance or conditional logic).
+
+### GraphDb
+
+The `database::GraphDb` is an "umbrella" object for parts of the
+database such as storage, garbage collection, transaction engine etc.
+There is a class heirarchy of `GraphDb` implementations, as well as a
+base interface object. There are subclasses for single-node, master and
+worker deplotyments. Which implementation is used depends on the
+configuration processed in the `main` entry point of memgraph.
+
+The `GraphDb` interface exposes getters to base classes of
+other similar heirarchies (for example to `tx::Engine`). In that way
+much of the code that uses those objects (for example query plan
+interpretation) is agnostic to the type of deployment.
+
+### RecordAccessors
+
+The functionality of `RecordAccessors` and it's subclasses is already
+documented. It's important to note that the same implementation of
+accessors is used in all deployments, with internal changes of behavior
+depending on the locality of the graph element (vertex or edge) the
+accessor represents. For example, if the graph element is local, an
+update operation on an accessor will make the necessary MVCC ops, update
+local data, indexes, the write-ahead log etc. However, if the accessor
+represents a remote graph element, an update will trigger an RPC message
+to the owner about the update and a change in the local cache.

docs/dev/distributed/updates.md

@@ -0,0 +1,103 @@
+# Distributed updates
+
+Operations that modify the graph state are somewhat more complex in the
+distributed system, as opposed to a single-node Memgraph deployment. The
+complexity arises from two factors.
+
+First, the data being modified is not necessarily owned by the worker
+performing the modification. This situation is completely valid workers
+execute parts of the query plan and parts must be executed by the
+master.
+
+Second, there are less guarantees regarding multi-threaded access. In
+single-node Memgraph it was guaranteed that only one transaction will be
+performing database work in a single transaction. This implied that
+per-version storage could be thread-unsafe. In distributed Memgraph it
+is possible that multiple threads could be performing work in the same
+transaction as a consequence of the query being executed at the same
+time on multiple workers and those executions interacting with the
+globally partitioned database state.
+
+## Deferred state modification
+
+Making the per-version data storage thread-safe would most likely have a
+performance impact very undesirable in a transactional database intended
+for high throughput.
+
+An alternative is that state modification over unsafe structures is not
+performed immediately when requested, but postponed until it is safe to
+do (there is is a guarantee of no concurrent access).
+
+Since local query plan execution is done the same way on local data as
+it is in single-node Memgraph, it is not possible to deffer that part of
+the modification story. What can be deferred are modifications requested
+by other workers. Since local query plan execution still is
+single-threaded, this approach is safe.
+
+At the same time those workers requesting the remote update can update
+local copies (caches) of the not-owned data since that cache is only
+being used by the single, local-execution thread.
+
+### Visibility
+
+Since updates are deferred the question arises: when do the updates
+become visible? The above described process offers the following
+visibility guarantees:
+- updates done on the local state are visible to the owner
+- updates done on the local state are NOT visible to anyone else during
+  the same (transaction + command)
+- updates done on remote state are deferred on the owner and not
+  visible to the owner until applied
+- updates done on the remote state are applied immediately to the local
+  caches and thus visible locally
+
+This implies an inconsistent view of the database state. In a concurrent
+execution of a single query this can hardly be avoided and is accepted
+as such. It does not change the Cypher query execution semantic in any
+of the well-defined scenarios. It possibly changes some of the behaviors
+in which the semantic is not well defined even in single-node execution.
+
+### Synchronization, update application
+
+In many queries it is mandatory to observe the latest global graph state
+(typically when returning it to the client). That means that before that
+happens all the deferred updates need to be applied, and all the caches
+to remote data invalidated. Exactly this happens when executing queries
+that modify the graph state. At some point a global synchronization
+point is reached. First it is waited that all workers finish the
+execution of query plan parts performing state modifications. After that
+all the workers are told to apply the deferred updates they received to
+their graph state. Since there is no concurrent query plan execution,
+this is safe. Once that is done all the local caches are cleared and the
+requested data can be returned to the client.
+
+### Command advancement
+
+In complex queries where a read part follows a state modification part
+the synchronization process after the state modification part is
+followed by command advancement, like in single-node execution.
+
+## Creation
+
+Graph element creation is not deferred. This is practical because the
+response to a creation is the global ID of the newly created element. At
+the same time it is safe because no other worker (including the owner)
+will be using the newly added graph element.
+
+## Updating
+
+Updating is deferred, as described. Note that this also means that
+record locking conflicts are deferred and serialization errors
+(including lock timeouts) are postponed until the deferred update
+application phase. In certain scenarios it might be beneficial to force
+these errors to happen earlier, when the deferred update request is
+processed.
+
+## Deletion
+
+Deletion is also deferred. Deleting an edge implies a modification of
+it's endpoint vertices, which must be deferred as those data structures
+are not thread-safe. Deleting a vertex is either with detaching, in
+which case an arbitrary number of updates are implied in the vertex's
+neighborhood, or without detaching which relies on checking the current
+state of the graph which is generally impossible in distributed.

docs/dev/durability/snapshots.md

@@ -0,0 +1,22 @@
+# Snapshots
+
+A "snapshot" is a record of the current database state stored in permanent
+storage. Note that the term "snapshot" is used also in the context of
+the transaction engine to denote a set of running transactions.
+
+A snapshot is written to the file by Memgraph periodically if so
+configured. The snapshot creation  process is done within a transaction created
+specifically for that purpose. The transaction is needed to ensure that
+the stored state is internally consistent.
+
+The database state can be recovered from the snapshot during startup, if
+so configured. This recovery works in conjunction with write-ahead log
+recovery.
+
+A single snapshot contains all the data needed to recover a database. In
+that sense snapshots are independent of each other and old snapshots can
+be deleted once the new ones are safely stored, if it is not necessary
+to revert the database to some older state.
+
+The exact format of the snapshot file is defined inline in the snapshot
+creation code.

docs/dev/durability/wal.md

@@ -0,0 +1,55 @@
+# Write-ahead logging
+
+Typically WAL denotes the process of writing a "log" of database
+operations (state changes) to persistent storage before committing the
+transaction, thus ensuring that the state can be recovered (in the case
+of a crash) for all the transactions which the database committed.
+
+The WAL is a fine-grained durability format. It's purpose is to store
+database changes fast. It's primary purpose is not to provide
+space-efficient storage, nor to support fast recovery. For that reason
+it's often used in combination with a different persistence mechanism
+(in Memgraph's case the "snapshot") that has complementary
+characteristics.
+
+### Guarantees
+
+Ensuring that the log is written before the transaction is committed can
+slow down the database. For that reason this guarantee is most often
+configurable in databases.
+
+Memgraph offers two options for the WAL. The default option, where the WAL is
+flushed to the disk periodically and transactions do not wait for this to
+complete, introduces the risk of database inconsistency because an operating
+system or hardware crash might lead to missing transactions in the WAL. Memgraph
+will handle this as if those transactions never happened. The second option,
+called synchronous commit, will instruct Memgraph to wait for the WAL to be
+flushed to the disk when a transactions completes and the transaction will wait
+for this to complete. This option can be turned on with the
+`--synchronous-commit` command line flag.
+
+### Format
+
+The WAL file contains a series of DB state changes called `StateDelta`s.
+Each of them describes what the state change is and in which transaction
+it happened. Also some kinds of meta-information needed to ensure proper
+state recovery are recorded (transaction beginnings and commits/abort).
+
+The following is guaranteed w.r.t. `StateDelta` ordering in
+a single WAL file:
+- For two ops in the same transaction, if op A happened before B in the
+  database, that ordering is preserved in the log.
+- Transaction begin/commit/abort messages also appear in exactly the
+  same order as they were executed in the transactional engine.
+
+### Recovery
+
+The database can recover from the WAL on startup. This works in
+conjunction with snapshot recovery. The database attempts to recover from
+the latest snapshot and then apply as much as possible from the WAL
+files. Only those transactions that were not recovered from the snapshot
+are recovered from the WAL, for speed efficiency. It is possible (but
+inefficient) to recover the database from WAL only, provided all the WAL
+files created from DB start are available. It is not possible to recover
+partial database state (i.e. from some suffix of WAL files, without the
+preceding snapshot).

docs/dev/lcp.md

@@ -0,0 +1,775 @@
+# Lisp C++ Preprocessor (LCP)
+
+In our development process we are using Common Lisp to generate some parts of
+the C++ codebase. The idea behind this is supplementing C++ with better
+meta-programming capabilities to automate tasks and prevent bugs due to code
+duplication. Primary candidate for using more powerful meta-programming is
+generating serialization code. Such code is almost always the same: go through
+all `struct` or `class` members and invoke the serialization function on them.
+Writing such code manually is error prone when adding members, because you may
+easily forget to correctly update the serialization code. Thus, the Lisp C++
+Preprocessor was born. It is hooked in our build process as a step before
+compilation. The remainder of the document describes how to use LCP and its
+features.
+
+Contents
+
+  * [Running LCP](#running-lcp)
+  * [Writing LCP](#writing-lcp)
+    - [Inlining C++ in Common Lisp](#inlining-cpp)
+    - [C++ Namespaces](#cpp-namespaces)
+    - [C++ Enumerations](#cpp-enums)
+    - [C++ Classes & Structs](#cpp-classes)
+    - [Defining an RPC](#defining-an-rpc)
+    - [Cap'n Proto Serialization](#capnp-serial)
+
+## Running LCP
+
+You can generate C++ from an LCP file by running the following command.
+
+`./tools/lcp <path-to-file.lcp>`
+
+The LCP will produce a `path-to-file.hpp` file and potentially a
+`path-to-file.lcp.cpp` file. The `.cpp` file is generated if some parts of the
+code need to be in the implementation file. This is usually the case when
+generating serialization code. Note that the `.cpp` file has the extension
+appended to `.lcp`, so that you are free to define your own `path-to-file.cpp`
+which includes the generated `path-to-file.hpp`.
+
+One serialization format uses Cap'n Proto library, but to use it, you need to
+provide an ID. The ID is generated by invoking `capnp id`. When you want to
+generate Cap'n Proto serialization, you need to pass the generated ID to LCP.
+
+`./tools/lcp <path-to-file.lcp> $(capnp id)`
+
+Generating Cap'n Proto serialization will produce an additional file,
+`path-to-file.capnp`, which contains the serialization schema.
+
+You may wonder why the LCP doesn't invoke `capnp id` itself. Unfortunately,
+such behaviour would be wrong when running LCP on the same file multiple
+times. Each run would produce a different ID and the serialization code would
+be incompatible between versions.
+
+### CMake
+
+The LCP is run in CMake using the `add_lcp` function defined in
+`CMakeLists.txt`. You can take a look at the function documentation there for
+information on how to add your new LCP files to the build system.
+
+## Writing LCP
+
+A LCP file should have the `.lcp` extension, but the code written is
+completely valid Common Lisp code. This means that you have a complete
+language at your disposal before even the C++ is compiled. You can view this
+as similar to the C++ templates and macros, but they do not have access to
+a complete language.
+
+Besides Common Lisp, you are allowed to write C++ code verbatim. This means
+that C++ and Lisp code coexist in the file. How to do that, as well as other
+features are described below.
+
+### Inlining C++ in Common Lisp {#inlining-cpp}
+
+To insert C++ code, you need to use a `#>cpp ... cpp<#` block. This is most
+often used at the top of the file to write Doxygen documentation and put some
+includes. For example:
+
+```cpp
+#>cpp
+/// @file My Doxygen style documentation about this file
+
+#pragma once
+
+#include <vector>
+cpp<#
+```
+
+The above code will be pasted as is into the generated header file. If you
+wish to have a C++ block in the `.cpp` implementation file instead, you should
+use `lcp:in-impl` function. For example:
+
+```cpp
+(lcp:in-impl
+#>cpp
+void MyClass::Method(int awesome_number) {
+  // Do something with awesome_number
+}
+cpp<#)
+```
+
+The C++ block also supports string interpolation with a syntax akin to shell
+variable access, `${lisp-variable}`. At the moment, only variables are
+supported and they have to be pretty printable in Common Lisp (i.e. support
+the `~A` format directive). For example, we can make a precomputed sinus
+function for integers from 0 to 5:
+
+```lisp
+(let ((sin-from-0-to-5
+        (format nil "~{~A~^, ~}" (loop for i from 0 below 5 collect (sin i)))))
+  #>cpp
+  static const double kSinFrom0To5[] = {${sin-from-0-to-5}};
+  cpp<#)
+```
+
+The following will be generated.
+
+```cpp
+static const double kSinFrom0To5[] = {0.0, 0.84147096, 0.9092974, 0.14112, -0.7568025};
+```
+
+Since you have a complete language at your disposal, this is a powerful tool
+to generate tables for computations which would take a very long time during
+the execution of the C++ program.
+
+### C++ Namespaces {#cpp-namespaces}
+
+Although you can use inline C++ to open and close namespaces, it is
+recommended to use `lcp:namespace` and `lcp:pop-namespace` functions. LCP will
+report an error if you have an unclosed namespace, unlike Clang and GCC which
+most of the times give strange errors due to C++ grammar ambiguity. Additional
+benefit is that LCP will track the namespace stack and correctly wrap any C++
+code which should be put in the `.cpp` file.
+
+For example:
+
+```lisp
+;; example.lcp
+(lcp:namespace utils)
+
+;; Function declaration in header
+#>cpp
+bool StartsWith(const std::string &string, const std::string &prefix);
+cpp<#
+
+;; Function implementation in implementation file
+(lcp:in-impl
+#>cpp
+bool StartsWith(const std::string &string, const std::string &prefix) {
+  // Implementation code
+  return false;
+}
+cpp<#)
+
+(lcp:pop-namespace) ;; utils
+```
+
+The above will produce 2 files, header and implementation:
+
+```cpp
+// example.hpp
+namespace utils {
+
+bool StartsWith(const std::string &string, const std::string &prefix);
+
+}
+```
+
+```cpp
+// example.lcp.cpp
+namespace utils {
+
+bool StartsWith(const std::string &string, const std::string &prefix) {
+  // Implementation code
+  return false;
+}
+
+}
+```
+
+### C++ Enumerations {#cpp-enums}
+
+LCP provides a `lcp:define-enum` macro to define a C++ `enum class` type. This
+will make LCP aware of the type and all its possible values. This makes it
+possible to generate the serialization code. In the future, LCP may generate
+"string to enum" and "enum to string" functions.
+
+Example:
+
+```lisp
+(lcp:define-enum days-in-week
+  (monday tuesday wednesday thursday friday saturday sunday)
+  ;; Optional documentation
+  (:documentation "Enumerates days of the week")
+  ;; Optional directive to generate serialization code
+  (:serialize))
+```
+
+Produces:
+
+```cpp
+/// Enumerates days of the week
+enum class DaysInWeek {
+  MONDAY,
+  TUESDAY,
+  WEDNESDAY,
+  THURSDAY,
+  FRIDAY,
+  SATURDAY,
+  SUNDAY
+};
+
+// serialization code ...
+```
+
+### C++ Classes & Structs {#cpp-classes}
+
+For defining C++ classes, there is a `lcp:define-class` macro. Its counterpart
+for structures is `lcp:define-struct`. They are exactly the same, but
+`lcp:define-struct` will put members in public scope by default. Just like in
+C++.
+
+Defining classes is a bit more involved, because they have many customization
+options. They syntax follows the syntax of class definition in Common Lisp
+(see `defclass`).
+
+Basic example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((primitive-value :int64_t)
+   (stl-vector "std::vector<int>"))
+  ;; Optional documentation
+  (:documentation "My class documentation")
+  ;; Define explicitly public, protected or private code. All are optional.
+  (:public #>cpp // some public code, e.g. methods cpp<#)
+  (:protected #>cpp // protected cpp<#)
+  (:private #>cpp //private cpp<#))
+```
+
+The above will generate:
+
+```cpp
+/// My class documentation
+class MyClass {
+ public:
+  // some public code, e.g. methods
+
+ protected:
+  // protected
+
+ private:
+  // private
+
+  int64_t primitive_value_;
+  std::vector<int> stl_vector_;
+};
+```
+
+As you can see, members in LCP are followed by a type. For primitive types, a
+Lisp keyword is used. E.g. `:int64_t`, `:bool`, etc. Other types, like STL
+containers use a valid C++ string to specify type.
+
+C++ supports nesting types inside a class. You can do the same in LCP inside
+any of the scoped additions.
+
+For example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((member "NestedType")
+   (value "NestedEnum"))
+  (:private
+    (lcp:define-enum nested-enum (first-value second-value))
+
+    (lcp:define-class nested-type ()
+      ((member :int64_t)))
+
+    #>cpp
+    // Some other C++ code
+    cpp<#))
+```
+
+The above should produce expected results.
+
+You can add a base classes after the class name. The name should be a Lisp
+symbol for base classes defined through `lcp:define-class`, so that LCP tracks
+the inheritance. Otherwise, it should be a string.
+
+For example:
+
+```lisp
+(lcp:define-class derived (my-class "UnknownInterface")
+  ())
+```
+
+Will generate:
+
+```cpp
+class Derived : public MyClass, public UnknownInterface {
+};
+```
+
+Similarly, you can specify template parameters. Instead of giving just a name
+to `define-class`, you give a list where the first element is the name of the
+class, while others name the template parameters.
+
+```lisp
+(lcp:define-class (my-map t-key t-value) ()
+  ((underlying-map "std::unordered_map<TKey, TValue>")))
+```
+
+The above will generate:
+
+```cpp
+template <class TKey, class TValue>
+class MyMap {
+ private:
+  std::unordered_map<TKey, TValue> underlying_map_;
+};
+```
+
+Other than tweaking the class definition, you can also do additional
+configuration of members. The following options are supported.
+
+  * `:initval` -- sets the initial value of a member
+  * `:reader` -- generates a public getter
+  * `:scope` -- set the scope of a member, one of `:public`, `:private` or
+    `:protected`
+  * `:documentation` -- Doxygen documentation of a member
+  * various serialization options which are explained later
+
+For example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((member "std::vector<int>" :scope :protected :initval "1, 2, 3" :reader t
+                              :documentation "Member documentation")))
+```
+
+Will generate:
+
+```cpp
+class MyClass {
+ public:
+  const auto &member() { return member_; }
+
+ protected:
+  /// Member documentation
+  std::vector<int> member_{1, 2, 3};
+};
+```
+
+### Defining an RPC
+
+In our codebase, we have implemented remote procedure calls. These are used
+for communication between Memgraph instances in a distributed system. Each RPC
+is registered by its type and requires serializable data structures. Writing
+RPC compliant structure requires a lot of boilerplate. To ease the pain of
+defining a new RPC we have a macro, `lcp:define-rpc`.
+
+Definition consists of 2 parts: request and response. You can specify members
+of each part. Member definition is the same as in `lcp:define-class`.
+
+For example:
+
+```lisp
+(lcp:define-rpc query-result
+  (:request
+    ((tx-id "tx::TransactionId")
+     (query-id :int64_t)))
+  (:response
+    ((values "std::vector<int>"))))
+```
+
+The above will generate relatively large amount of C++ code, which is omitted
+here as the details aren't important for understanding the use. Examining the
+generated code is left as an exercise for the reader.
+
+The important detail is that in C++ you will have a `QueryResultRpc`
+structure, which is used to register the behaviour of an RPC server. You need
+to perform the registration manually. For example:
+
+```cpp
+// somewhere in code you have a server instance
+rpc_server.Register<QueryResultRpc>(
+    [](const auto &req_reader, auto *res_builder) {
+      QueryResultReq request;
+      Load(&request, req_reader);
+      // process the request and send the response
+      QueryResultRes response(values_for_response);
+      Save(response, res_builder);
+    });
+
+
+// somewhere else you have a client which sends the RPC
+tx::TransactionId tx_id = ...
+int64_t query_id = ...
+auto response = rpc_client.template Call<QueryResultRpc>(tx_id, query_id);
+if (response) {
+  const auto &values = response->getValues();
+  // do something with values
+}
+```
+
+RPC structures use Cap'n Proto for serialization. The above variables
+`req_reader` and `res_builder` are used to access Cap'n Proto structures.
+Obviously, the LCP will generate the Cap'n Proto schema alongside the C++
+code for serialization.
+
+
+### Cap'n Proto Serialization {#capnp-serial}
+
+Primary purpose of LCP was to make serialization of types easier. Our
+serialization library of choice for C++ is Cap'n Proto. LCP provides
+generation and tuning of its serialization code. Previously, LCP supported
+Boost.Serialization, but it was removed.
+
+To specify a class or structure for serialization, you may pass a
+`:serialize :capnp` option when defining such type. (Note that
+`lcp:define-enum` takes `:serialize` without any arguments).
+
+For example:
+
+```lisp
+(lcp:define-struct my-struct ()
+  ((member :int64_t))
+  (:serialize :capnp))
+```
+
+`:serialize` option will generate a Cap'n Proto schema of the class and store
+it in the `.capnp` file. C++ code will be generated for saving and loading
+members:
+
+```cpp
+// Top level functions
+void Save(const MyStruct &self, capnp::MyStruct::Builder *builder);
+void Load(MyStruct *self, const capnp::MyStruct::Reader &reader);
+```
+
+Since we use top level functions, the class needs to have some sort of public
+access to its members.
+
+The schema file will be namespaced in `capnp`. To change add a prefix
+namespace use `lcp:capnp-namespace` function. For example, if we use
+`(lcp:capnp-namespace "my_namespace")` then the reader and builder would be in
+`my_namespace::capnp`.
+
+Serializing a class hierarchy is also supported. The most basic case with
+single inheritance works out of the box. Handling other cases is explained in
+later sections.
+
+For example:
+
+```lisp
+(lcp:define-class base ()
+  ((base-member "std::vector<int64_t>" :scope :public))
+  (:serialize :capnp))
+
+(lcp:define-class derived (base)
+  ((derived-member :bool :scope :public))
+  (:serialize :capnp))
+```
+
+Note that all classes need to have the `:serialize` option set. Signatures of
+`Save` and `Load` functions are changed to accept reader and builder to the
+base class. The `Load` function now takes a `std::unique_ptr<T> *` which is
+used to take ownership of a concrete type. This approach transfers the
+responsibility of type allocation and construction from the user of `Load` to
+`Load` itself.
+
+```cpp
+void Save(const Derived &self, capnp::Base::Builder *builder);
+void Load(std::unique_ptr<Base> *self, const capnp::Base::Reader &reader);
+```
+
+#### Multiple Inheritance
+
+Cap'n Proto does not support any form of inheritance, instead we are
+handling it manually. Single inheritance was relatively easy to add to Cap'n
+Proto, we simply enumerate all derived types inside the union of a base type.
+
+Multiple inheritance is a different beast and as such is not directly
+supported.
+
+One way to use multiple inheritance is only to implement the interface of pure
+virtual classes without any members (i.e. interface classes). In such a case,
+you do not want to serialize any other base class except the primary one. To
+let LCP know that is the case, use `:ignore-other-base-classes t`. LCP will
+only try to serialize the base class that is the first (leftmost) in the list
+of super classes.
+
+```lisp
+(lcp:define-class derived (primary-base some-interface other-interface)
+  ...
+  (:serialize :capnp :ignore-other-base-classes t))
+```
+
+Another form of multiple inheritance is reusing some common code. In
+actuality, this is a very bad code practice and should be replaced with
+composition. If it would take too long to fix such code to use composition
+proper, we can tell LCP to treat such inheritance as if they are indeed
+composed. This is done via `:inherit-compose` option.
+
+For example:
+
+```lisp
+(lcp:define-class derived (first-base second-base)
+  ...
+  (:serialize :capnp :inherit-compose '(second-base)))
+```
+
+With `:inherit-compose` you can pass a list of parent classes which should be
+encoded as composition inside the Cap'n Proto schema. LCP will complain if
+there is multiple inheritance but you didn't specify `:inherit-compose`.
+
+The downside of this approach is that `Save` and `Load` will work only on
+`FirstBase`. Serializing a pointer to `SecondBase` would be incorrect.
+
+#### Inheriting C++ Class Outside of LCP
+
+Classes defined outside of `lcp:define-class` are not visible to LCP and LCP
+will not be able to generate correct serialization code.
+
+The cases so far have been only with classes that are pure interface and need
+no serialization code. This is signaled to LCP by passing the option `:base t`
+to `:serialize :capnp`. LCP will treat such classes as actually being the base
+class of a hierarchy.
+
+For example:
+
+```lisp
+(lcp:define-class my-class ("utils::TotalOrdering")
+  (...)
+  (:serialize :capnp :base t))
+
+(lcp:define-class derived (my-class)
+  (...)
+  (:serialize :capnp))
+```
+
+Only the base class for serialization has the `:base t` option set. Derived
+classes are defined as usual. This relies on the fact that we do not expect
+anyone to have a pointer to `utils::TotalOrdering` and use it for
+serialization and deserialization.
+
+#### Template Classes
+
+Currently, LCP supports the most primitive form of serializing templated
+classes. The template arguments must be provided to specify an explicit
+instantiation. Cap'n Proto does support generics, so we may want to upgrade
+LCP to use them in the future.
+
+To specify template arguments, pass a `:type-args` option. For example:
+
+```lisp
+(lcp:define-class (my-container t-value) ()
+  (...)
+  (:serialize :capnp :type-args '(my-class)))
+```
+
+The above will support serialization of `MyContainer<MyClass>` type.
+
+The syntax will work even if our templated class inherits from non-templated
+classes. All other cases of inheritance with templates are forbidden in LCP
+serialization.
+
+#### Cap'n Proto Schemas and Type Conversions
+
+You can import other serialization schemas by using `lcp:capnp-import`
+function. It expects a name for the import and the path to the schema file.
+
+For example, to import everything from `utils/serialization.capnp` under the
+name `Utils`, you can do the following:
+
+```lisp
+(lcp:capnp-import 'utils "/utils/serialization.capnp")
+```
+
+To use those types, you need to register a conversion from C++ type to schema
+type. There are two options, registering a whole file conversion with
+`lcp:capnp-type-conversion` or converting a specific class member.
+
+For example, you have a class with member of type `Bound` and there is a
+schema for it also named `Bound` inside the imported schema.
+
+You can use `lcp:capnp-type-conversion` like so:
+
+```lisp
+(lcp:capnp-type-conversion "Bound" "Utils.Bound")
+
+(lcp:define-class my-class ()
+  ((my-bound "Bound")))
+```
+
+Specifying only a member conversion can be done with `:capnp-type` member
+option:
+
+```lisp
+(lcp:define-class my-class ()
+  ((my-bound "Bound" :capnp-type "Utils.Bound")))
+```
+
+#### Custom Save and Load Hooks
+
+Sometimes the default serialization is not adequate and you may wish to
+provide your own serialization code. For those reasons, LCP provides
+`:capnp-save`, `:capnp-load` and `:capnp-init` options on each class member.
+
+The simplest is `:capnp-init` which when set to `nil` will not generate an
+`init<member>` call on a builder. Cap'n Proto requires that compound types are
+initialized before beginning to serialize its members. `:capnp-init` allows you
+to delay the initialization to your custom save code. You rarely want to set
+`:capnp-init nil`.
+
+Custom save code is added as a value of `:capnp-save`. It should be a function
+which takes 3 arguments.
+
+  1. Name of builder variable.
+  2. Name of the class (or struct) member.
+  3. Name of the member in Cap'n Proto schema.
+
+The result of the function needs to be a C++ code block.
+
+You will rarely need to use the 3rd argument, so it should be ignored in most
+cases. It is usually needed when you set `:capnp-init nil`, so that you can
+correctly initialize the builder.
+
+Similarly, `:capnp-load` expects a function taking a reader, C++ member and
+Cap'n Proto member,  then returns a C++ block.
+
+Example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((my-member "ComplexType"
+              :capnp-init nil
+              :capnp-save (lambda (builder member capnp-name)
+                            #>cpp
+                            auto data = ${member}.GetSaveData();
+                            auto my_builder = ${builder}.init${capnp-name}();
+                            my_builder.setData(data);
+                            cpp<#)
+              :capnp-load (lambda (reader member capnp-name)
+                            (declare (ignore capnp-name))
+                            #>cpp
+                            auto data = ${reader}.getData();
+                            ${member}.LoadFromData(data);
+                            cpp<#)))
+  (:serialize :capnp))
+```
+
+With custom serialization code, you may want to get additional details through
+extra arguments to `Save` and `Load` functions. This is described in the next
+section.
+
+There are also cases where you always need custom serialization code. LCP
+provides helper functions for abstracting some common details. These functions
+are listed further down in this document.
+
+#### Arguments for Save and Load
+
+Default arguments for `Save` and `Load` function are Cap'n Proto builder and
+reader, respectively. In some cases you may wish to send additional arguments.
+This is most commonly needed when tracking `shared_ptr` serialization, to
+avoid serializing the same pointer multiple times.
+
+Additional arguments are specified by passing `:save-args` and `:load-args`.
+You can specify either of them, but in most cases you want both.
+
+For example:
+
+```lisp
+;; Class for tracking details during save
+(lcp:define-class save-helper ()
+  (...))
+
+;; Class for tracking details during load
+(lcp:define-class load-helper ()
+  (...))
+
+(lcp:define-class my-class ()
+  ((member "std::shared_ptr<int>"
+           :capnp-save ;; custom save
+           :capnp-load ;; custom load
+    ))
+  (:serialize :capnp
+              :save-args '((save-helper "SaveHelper *"))
+              :load-args '((load-helper "LoadHelper *"))))
+```
+
+The custom serialization code will now have access to `save_helper` and
+`load_helper` variables in C++. You can add more arguments by expanding the
+list of pairs, e.g.
+
+```lisp
+:save-args '((first-helper "SomeType *") (second-helper "OtherType *") ...)
+```
+
+#### Custom Serialization Helper Functions
+
+##### Helper for `std::optional`
+
+When using `std::optional` with primitive C++ types or custom types known to
+LCP, you do not need to use any helper. In the example below, things should be
+serialized as expected:
+
+```lisp
+(lcp:define-class my-class-with-primitive-optional ()
+  ((primitive-optional "std::experimental::optional<int64_t>"))
+  (:serialize :capnp))
+
+(lcp:define-class my-class-with-known-type-optional ()
+  ((known-type-optional "std::experimental::optional<MyClassWithPrimitiveOptional>"))
+  (:serialize :capnp))
+```
+
+In cases when the value contained in `std::optional` needs custom
+serialization code you may use `lcp:capnp-save-optional` and
+`lcp:capnp-load-optional`.
+
+Both functions expect 3 arguments.
+
+  1. Cap'n Proto type in C++.
+  2. C++ type of the value inside `std::optional`.
+  3. Optional C++ lambda code.
+
+The lambda code is optional, because LCP will generate the default
+serialization code which invokes `Save` and `Load` function on the value
+stored inside the optional. Since most of the serialized classes follow the
+convention, you will rarely need to provide this 3rd argument.
+
+For example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((member "std::experimental::optional<SomeType>"
+           :capnp-save (lcp:capnp-save-optional
+                         "capnp::SomeType" "SomeType"
+                         "[](auto *builder, const auto &val) { ... }")
+           :capnp-load (lcp:capnp-load-optional
+                         "capnp:::SomeType" "SomeType"
+                         "[](const auto &reader) { ... return loaded_val; }"))))
+```
+
+##### Helper for `std::vector`
+
+For custom serialization of vector elements, you may use
+`lcp:capnp-save-vector` and `lcp:capnp-load-vector`. They function exactly the
+same as helpers for `std::optional`.
+
+##### Helper for enumerations
+
+If the enumeration is defined via `lcp:define-enum`, the default LCP
+serialization should generate the correct code.
+
+However, if LCP cannot infer the serialization code, you can use helper
+functions `lcp:capnp-save-enum` and `lcp:capnp-load-enum`. Both functions
+require 3 arguments.
+
+  1. C++ type of equivalent Cap'n Proto enum.
+  2. Original C++ enum type.
+  3. List of enumeration values.
+
+Example:
+
+```lisp
+(lcp:define-class my-class ()
+  ((enum-value "SomeEnum"
+               :capnp-init nil ;; must be set to nil
+               :capnp-save (lcp:capnp-save-enum
+                             "capnp::SomeEnum" "SomeEnum"
+                             '(first-value second-value))
+               :capnp-load (lcp:capnp-load-enum
+                             "capnp::SomeEnum" "SomeEnum"
+                             '(first-value second-value)))))
+```

docs/dev/other-code-conventions.md

@@ -0,0 +1,15 @@
+# Other Code Conventions
+
+While we are mainly programming in C++, we do use other programming languages
+when appropriate. This chapter describes conventions for such code.
+
+## Python
+
+Code written in Python should adhere to
+[PEP 8](https://www.python.org/dev/peps/pep-0008/). You should run `flake8` on
+your code to automatically check compliance.
+
+## Common Lisp
+
+Code written in Common Lisp should adhere to
+[Google Common Lisp Style](https://google.github.io/styleguide/lispguide.xml).

docs/dev/quick-start.md

@@ -0,0 +1,88 @@
+# Quick Start
+
+A short chapter on downloading the Memgraph source, compiling and running.
+
+## Obtaining the Source Code
+
+Memgraph uses `git` for source version control. You will need to install `git`
+on your machine before you can download the source code.
+
+On Debian systems, you can do it inside a terminal with the following
+command:
+
+    sudo apt-get install git
+
+On ArchLinux or Gentoo, you probably already know what to do.
+
+After installing `git`, you are now ready to fetch your own copy of Memgraph
+source code. Run the following command:
+
+    git clone https://phabricator.memgraph.io/diffusion/MG/memgraph.git
+
+The above will create a `memgraph` directory and put all source code there.
+
+## Compiling Memgraph
+
+With the source code, you are now ready to compile Memgraph. Well... Not
+quite. You'll need to download Memgraph's dependencies first.
+
+In your terminal, position yourself in the obtained memgraph directory.
+
+    cd memgraph
+
+### Installing Dependencies
+
+On Debian systems, all of the dependencies should be setup by running the
+`init` script:
+
+    ./init -s
+
+Currently, other systems aren't supported in the `init` script. But you can
+issue the needed steps manually. First run the `init` script.
+
+    ./init
+
+The script will output the required packages, which you should be able to
+install via your favorite package manager. For example, `pacman` on ArchLinux.
+After installing the packages, issue the following commands:
+
+    mkdir -p build
+    ./libs/setups.sh
+
+### Compiling
+
+With all of the dependencies installed, you need to configure the build
+system. To do that, execute the following:
+
+    cd build
+    cmake ..
+
+If everything went OK, you can now, finally, compile Memgraph.
+
+    make -j$(nproc)
+
+### Running
+
+After the compilation verify that Memgraph works:
+
+    ./memgraph --version
+
+To make extra sure, run the unit tests:
+
+    ctest -R unit -j$(nproc)
+
+## Problems
+
+If you have any trouble running the above commands, contact your nearest
+developer who successfully built Memgraph. Ask for help and insist on getting
+this document updated with correct steps!
+
+## Next Steps
+
+Familiarise yourself with our code conventions:
+
+  * [C++ Code](cpp-code-conventions.md)
+  * [Other Code](other-code-conventions.md)
+
+Take a look at the list of [required reading](required-reading.md) for
+brushing up on technical skills.

docs/dev/required-reading.md

@@ -0,0 +1,129 @@
+# Required Reading
+
+This chapter lists a few books that should be read by everyone working on
+Memgraph. Since Memgraph is developed primarily with C++, Python and Common
+Lisp, books are oriented around those languages. Of course, there are plenty
+of general books which will help you improve your technical skills (such as
+"The Pragmatic Programmer", "The Mythical Man-Month", etc.), but they are not
+listed here. This way the list should be kept short and the *required* part in
+"Required Reading" more easily honored.
+
+Some of these books you may find in our office, so feel free to pick them up.
+If any are missing and you would like a physical copy, don't be afraid to
+request the book for our office shelves.
+
+Besides reading, don't get stuck in a rut and be a
+[Blub Programmer](http://www.paulgraham.com/avg.html).
+
+## Effective C++ by Scott Meyers
+
+Required for C++ developers.
+
+The book is a must-read as it explains most common gotchas of using C++. After
+reading this book, you are good to write competent C++ which will pass code
+reviews easily.
+
+## Effective Modern C++ by Scott Meyers
+
+Required for C++ developers.
+
+This is a continuation of the previous book, it covers updates to C++ which
+came with C++11 and later. The book isn't as imperative as the previous one,
+but it will make you aware of modern features we are using in our codebase.
+
+## Practical Common Lisp by Peter Siebel
+
+Required for Common Lisp developers.
+
+Free: http://www.gigamonkeys.com/book/
+
+We use Common Lisp to generate C++ code and make our lives easier.
+Unfortunately, not many developers are familiar with the language. This book
+will make you familiar very quickly as it has tons of very practical
+exercises. E.g. implementing unit testing library, serialization library and
+bundling all that to create a mp3 music server.
+
+## Effective Python by Brett Slatkin
+
+(Almost) required reading for Python developers.
+
+Why the "almost"? Well, Python is relatively easy to pick up and you will
+probably learn all the gotchas during code review from someone more
+experienced. This makes the book less necessary for a newcomer to Memgraph,
+but the book is not advanced enough to delegate it to
+[Advanced Reading](#advanced-reading). The book is written in similar vein as
+the "Effective C++" ones and will make you familiar with nifty Python features
+that make everyone's lives easier.
+
+# Advanced Reading
+
+The books listed below are not required reading, but you may want to read them
+at some point when you feel comfortable enough.
+
+## Design Patterns by Gamma et. al.
+
+Recommended for C++ developers.
+
+This book is highly divisive because it introduced a culture centered around
+design patterns. The main issues is overuse of patterns which complicates the
+code. This has made many Java programs to serve as examples of highly
+complicated, "enterprise" code.
+
+Unfortunately, design patterns are pretty much missing
+language features. This is most evident in dynamic languages such as Python
+and Lisp, as demonstrated by
+[Peter Norvig](http://www.norvig.com/design-patterns/).
+
+Or as [Paul Graham](http://www.paulgraham.com/icad.html) put it:
+
+```
+This practice is not only common, but institutionalized. For example, in the
+OO world you hear a good deal about "patterns". I wonder if these patterns are
+not sometimes evidence of case (c), the human compiler, at work. When I see
+patterns in my programs, I consider it a sign of trouble. The shape of a
+program should reflect only the problem it needs to solve. Any other
+regularity in the code is a sign, to me at least, that I'm using abstractions
+that aren't powerful enough-- often that I'm generating by hand the expansions
+of some macro that I need to write
+```
+
+After presenting the book so negatively, why you should even read it then?
+Well, it is good to be aware of those design patterns and use them when
+appropriate. They can improve modularity and reuse of the code. You will also
+find examples of such patterns in our code, primarily Strategy and Visitor
+patterns. The book is also a good stepping stone to more advanced reading
+about software design.
+
+## Modern C++ Design by Andrei Alexandrescu
+
+Recommended for C++ developers.
+
+This book can be treated as a continuation of the previous "Design Patterns"
+book. It introduced "dark arts of template meta-programming" to the world.
+Many of the patterns are converted to use C++ templates which makes them even
+better for reuse. But, like the previous book, there are downsides if used too
+much. You should approach it with a critical eye and it will help you
+understand ideas that are used in some parts of our codebase.
+
+## Large Scale C++ Software Design by John Lakos
+
+Recommended for C++ developers.
+
+An old book, but well worth the read. Lakos presents a very pragmatic view of
+writing modular software and how it affects both development time as well as
+program runtime. Some things are outdated or controversial, but it will help
+you understand how the whole C++ process of working in a large team, compiling
+and linking affects development.
+
+## On Lisp by Paul Graham
+
+Recommended for Common Lisp developers.
+
+Free: http://www.paulgraham.com/onlisp.html
+
+An excellent continuation to "Practical Common Lisp". It starts of slow, as if
+introducing the language, but very quickly picks up speed. The main meat of
+the book are macros and their uses. From using macros to define cooperative
+concurrency to including Prolog as if it's part of Common Lisp. The book will
+help you understand more advanced macros that are occasionally used in our
+Lisp C++ Preprocessor (LCP).

docs/dev/storage/accessors.md

@@ -0,0 +1,110 @@
+# DatabaseAccessor
+
+A `DatabaseAccessor` actually wraps a transactional access to database
+data, for a single transaction. In that sense the naming is bad. It
+encapsulates references to the database and the transaction object.
+
+It contains logic for working with database content (graph element
+data) in the context of a single transaction. All CRUD operations are
+performed within a single transaction (as Memgraph is a transactional
+database), and therefore iteration over data, finding a specific graph
+element etc are all functionalities of a `GraphDbAccessor`.
+
+In single-node Memgraph the database accessor also defined the lifetime
+of a transaction. Even though a `Transaction` object was owned by the
+transactional engine, it was `GraphDbAccessor`'s lifetime that object
+was bound to (the transaction was implicitly aborted in
+`GraphDbAccessor`'s destructor, if it was not explicitly ended before
+that).
+
+# RecordAccessor
+
+It is important to understand data organization and access in the
+storage layer. This discussion pertains to vertices and edges as graph
+elements that the end client works with.
+
+Memgraph uses MVCC (documented on it's own page). This means that for
+each graph element there could be different versions visible to
+different currently executing transactions. When we talk about a
+`Vertex` or `Edge` as a data structure we typically mean one of those
+versions. In code this semantic is implemented so that both those classes
+inherit `mvcc::Record`, which in turn inherits `mvcc::Version`.
+
+Handling MVCC and visibility is not in itself trivial. Next to that,
+there is other book-keeping to be performed when working with data. For
+that reason, Memgraph uses "accessors" to define an API of working with
+data in a safe way. Most of the code in Memgraph (for example the
+interpretation code) should work with accessors. There is a
+`RecordAccessor` as a base class for `VertexAccessor` and
+`EdgeAccessor`. Following is an enumeration of their purpose.
+
+### Data access
+
+The client interacts with Memgraph using the Cypher query language. That
+language has certain semantics which imply that multiple versions of the
+data need to be visible during the execution of a single query. For
+example: expansion over the graph is always done over the graph state as
+it was at the beginning of the transaction.
+
+The `RecordAccessor` exposes functions to switch between the old and the new
+versions of the same graph element (intelligently named `SwitchOld` and
+`SwitchNew`) within a single transaction. In that way the client code
+(mostly the interpreter) can avoid dealing with the underlying MVCC
+version concepts.
+
+### Updates
+
+Data updates are also done through accessors. Meaning: there are methods
+on the accessors that modify data, the client code should almost never
+interact directly with `Vertex` or `Edge` objects.
+
+The accessor layer takes care of creating version in the MVCC layer and
+performing updates on appropriate versions.
+
+Next, for many kinds of updates it is necessary to update the relevant
+indexes. There are implicit indexes for vertex labels, as
+well as user-created indexes for (label, property) pairs. The accessor
+layer takes care of updating the indexes when these values are changed.
+
+Each update also triggers a log statement in the write-ahead log. This
+is also handled by the accessor layer.
+
+### Distributed
+
+In distributed Memgraph accessors also contain a lot of the remote graph
+element handling logic. More info on that is available in the
+documentation for distributed.
+
+### Deferred MVCC data lookup for Edges
+
+Vertices and edges are versioned using MVCC. This means that for each
+transaction an MVCC lookup needs to be done to determine which version
+is visible to that transaction. This tends to slow things down due to
+cache invalidations (version lists and versions are stored in arbitrary
+locations on the heap).
+
+However, for edges, only the properties are mutable. The edge endpoints
+and type are fixed once the edge is created. For that reason both edge
+endpoints and type are available in vertex data, so that when expanding
+it is not mandatory to do MVCC lookups of versioned, mutable data. This
+logic is implemented in `RecordAccessor` and `EdgeAccessor`.
+
+### Exposure
+
+The original idea and implementation of graph element accessors was that
+they'd prevent client code from ever interacting with raw `Vertex` or
+`Edge` data. This however turned out to be impractical when implementing
+distributed Memgraph and the raw data members have since been exposed
+(through getters to old and new version pointers). However, refrain from
+working with that data directly whenever possible! Always consider the
+accessors to be the first go-to for interacting with data, especially
+when in the context of a transaction.
+
+# Skiplist accessor
+
+The term "accessor" is also used in the context of a skiplist. Every
+operation on a skiplist must be performed within on an
+accessor. The skiplist ensures that there will be no physical deletions
+of an object during the lifetime of an accessor. This mechanism is used
+to ensure deletion correctness in a highly concurrent container.
+We only mention that here to avoid confusion regarding terminology.

docs/dev/storage/indexes.md

@@ -0,0 +1,116 @@
+# Label indexes
+
+These are unsorted indexes that contain all the vertices that have the label
+the indexes are for (one index per label). These kinds of indexes get
+automatically generated for each label used in the database.
+
+### Updating the indexes
+
+Whenever something gets added to the record we update the index (add that
+record to index). We keep an index which might contain garbage (not relevant
+records, because the value got removed or something similar) but we will
+filter it out when querying the index. We do it like this because we don't
+have to do bookkeeping and deciding if we update the index on the end of the
+transaction (commit/abort phase), moreover current interpreter advances the
+command in transaction and as such assumes that the indexes now contain
+objects added in the previous command inside this transaction, so we need to
+update over the whole scope of transaction (whenever something is added to the
+record).
+
+### Index Entries Label
+
+These kinds of indexes are internally keeping track of pair (record, vlist).
+Why do we need to keep track of exactly those two things?
+
+Problems with two different approaches
+
+1) Keep track of just the record:
+
+  - We need the `VersionList` for creating an accessor (this in itself is a
+    deal-breaker).
+  - Semantically it makes sense. An edge/vertex maps bijectionally to a
+    `VersionList`.
+  - We might try to access some members of record while the record is being
+    modified from another thread.
+  - A vertex/edge could get updated, thus expiring the record in the index.
+    The newly created record should be present in the index, but it's not.
+    Without the `VersionList` we can't reach the newly created record.
+  - Probably there are even more reasons... It should be obvious by now that
+    we need the `VersionList` in the index.
+
+2) Keep track of just the version list:
+
+  - Removing from an index is a problem for two major reasons. First, if we
+    only have the `VersionList`, checking if it should be removed implies
+    checking all the reachable records, which is not thread-safe. Second,
+    there are issues with concurrent removal and insertion. The cleanup thread
+    could determine the vertex/edge should be removed from the index and
+    remove it, while in between those ops another thread attempts to insert
+    the `VersionList` into the index. The insertion does nothing because the
+    `VersionList` is already in, but it gets removed immediately after.
+
+Because of inability to keep track of just the record, or value, we need to
+keep track of both of them.  Resolution of problems mentioned above, in the
+same order, with (record, vlist) pair
+
+  - simple `vlist.find(current transaction)` will get us the newest visible
+    record
+  - we'll never try to access some record if it's still being written since we
+    will always operate on vlist.find returned record
+  - newest record will contain that label
+  - since we have (record, vlist) pair as the key in the index when we update
+    and delete in the same time we will never delete the same record, vlist
+    pair we are adding because the record, vlist pair we are deleting is
+    already superseded by a newer record and as such won't be inserted while
+    it's being deleted
+
+### Querying the index
+
+We run through the index for the given label and do `vlist.find` operation for
+the current transaction, and check if the newest return record has that
+label. If it has it then we return it. By now you are probably wondering
+aren't we sometimes returning duplicate vlist entries? And you are wondering
+correctly, we would be returning them, but we are making sure that the entires
+in the index are sorted by their `vlist*` and as such we can filter consecutive
+duplicate `vlist*` to only return one of those while still being able to create
+an iterator to index.
+
+### Cleaning the index
+
+Cleaning the index is not as straightforward as it seems as a lot of garbage
+can accumulate, but it's hard to know when exactly can we delete some (record,
+vlist) pair. First, let's assume that we are doing the cleaning process at
+some `transaction_id`, `id` such that there doesn't exist an active transaction
+with an id lower than `id`.
+
+We scan through the whole index and for each (record, vlist) pair we first
+check if it was deleted before the id (i.e. no transaction with an id >= `id`
+will ever again see that record), if it was deleted before we might naively
+say that it's safe to delete it, but, we must take into account that when some
+new record is created from this record (update operation), that record still
+contains the label but by deleting this record we won't be able to see that
+vlist because that new record won't add again to index because we didn't
+explicitly add that label again to it.
+
+Because of this we have to 'update' this index (record, vlist) pair. We have
+to update the record to now point to a newer record in vlist, the one that is
+not deleted yet. We can do that by querying the `version_list` for the last
+record inside (oldest it has — remember that `mvcc_gc` will re-link not
+visible records so the last record will be visible for the current GC id).
+When updating the record inside the index, it's not okay to just update the
+pointer and leave the index as it is, because with updating the `record*` we
+might change the relative order of entries inside the index. We first have to
+re-insert it with new `record*`, and then delete the old entry. And we need to
+do insertion before the remove operation! Otherwise it could happen that the
+vlist with a newer record with that label won't exist while some transaction
+is querying the index.
+
+Records which we added as a consequence of deleting older records will be
+eventually removed from the index if they don't contain label because if we
+see that the record is not deleted we try to check if that record still
+contains the label. We also need to be careful here because we can't check
+that while the record is being potentially updated by some transaction (race
+condition), so we need can check if records still contain label if it's
+creation id is smaller than our `id`, as that implies that the creating
+transaction either aborted or committed as our `id` is equal to the oldest
+active transaction in time of starting the GC.

docs/dev/storage/property-storage.md

@@ -0,0 +1,131 @@
+# Property storage
+
+Although the reader is probably familiar with properties in *Memgraph*, let's
+briefly recap. 
+
+Both vertices and edges can store an arbitrary number of properties. Properties
+are, in essence, ordered pairs of property names and property values. Each
+property name within a single graph element (edge/node) can store a single
+property value. Property names are represented as strings, while property values
+must be one of the following types:
+
+ Type      | Description
+-----------|------------
+ `Null`    | Denotes that the property has no value. This is the same as if the property does not exist.
+ `String`  | A character string, i.e. text.
+ `Boolean` | A boolean value, either `true` or `false`.
+ `Integer` | An integer number.
+ `Float`   | A floating-point number, i.e. a real number.
+ `List`    | A list containing any number of property values of any supported type. It can be used to store multiple values under a single property name.
+ `Map`     | A mapping of string keys to values of any supported type.
+
+Property values are modeled in a class conveniently called `PropertyValue`.
+
+## Mapping between property names and property keys.
+
+Although users think of property names in terms of descriptive strings 
+(e.g. "location" or "department"), *Memgraph* internally converts those names
+into property keys which are, essentially, unsigned 16-bit integers.
+
+Property keys are modelled by a not-so-conveniently named class called
+`Property` which can be found in `storage/types.hpp`. The actual conversion
+between property names and property keys is done within the `ConcurrentIdMapper`
+but the internals of that implementation are out of scope for understanding
+property storage.
+
+## PropertyValueStore
+
+Both `Edge` and `Vertex` objects contain an instance of `PropertyValueStore`
+object which is responsible for storing properties of a corresponding graph
+element.
+
+An interface of `PropertyValueStore` is as follows:
+
+ Method    | Description
+-----------|------------
+ `at`      | Returns the `PropertyValue` for a given `Property` (key).
+ `set`     | Stores a given `PropertyValue` under a given `Property` (key).
+ `erase`   | Deletes a given `Property` (key) alongside its corresponding `PropertyValue`.
+ `clear`   | Clears the storage.
+ `iterator`| Provides an extension of `std::input_iterator` that iterates over storage.
+
+## Storage location
+
+By default, *Memgraph* is an in-memory database and all properties are therefore
+stored in working memory unless specified otherwise by the user. User has an
+option to specify via the command line which properties they wish to be stored
+on disk.
+
+Storage location of each property is encapsulated within a `Property` object
+which is ensured by the `ConcurrentIdMapper`. More precisely, the unsigned 16-bit
+property key has the following format:
+
+```
+|---location--|------id------|
+|-Memory|Disk-|-----2^15-----|
+``` 
+
+In other words, the most significant bit determines the location where the
+property will be stored. 
+
+### In-memory storage
+
+The underlying implementation of in-memory storage for the time being is
+`std::vector<std::pair<Property, PropertyValue>>`. Implementations of`at`, `set`
+and `erase` are linear in time. This implementation is arguably more efficient
+than `std::map` or `std::unordered_map` when the average number of properties of
+a record is relatively small (up to 10) which seems to be the case.
+
+### On-disk storage
+
+#### KVStore
+
+Disk storage is modeled by an abstraction of key-value storage as implemented in
+`storage/kvstore.hpp'. An interface of this abstraction is as follows:
+
+ Method         | Description
+----------------|------------
+ `Put`          | Stores the given value under the given key.
+ `Get`          | Obtains the given value stored under the given key.
+ `Delete`       | Deletes a given (key, value) pair from storage..
+ `DeletePrefix` | Deletes all (key, value) pairs where key begins with a given prefix.
+ `Size`         | Returns the size of the storage or, optionally, the number of stored pairs that begin with a given prefix.
+ `iterator`     | Provides an extension of `std::input_iterator` that iterates over storage.
+
+Keys and values in this context are of type `std::string`.
+
+The actual underlying implementation of this abstraction uses
+[RocksDB]{https://rocksdb.org} &mdash; a persistent key-value store for fast
+storage.
+
+It is worthy to note that the custom iterator implementation allows the user
+to iterate over a given prefix. Otherwise, the implementation follows familiar
+c++ constructs and can be used as follows:
+
+```
+KVStore storage = ...;
+for (auto it = storage.begin(); it != storage.end(); ++it) {}
+for (auto kv : storage) {}
+for (auto it = storage.begin("prefix"); it != storage.end("prefix"); ++it) {}
+```
+
+Note that it is not possible to scan over multiple prefixes. For instance, one
+might assume that you can scan over all keys that fall in a certain
+lexicographical range. Unfortunately, that is not the case and running the
+following code will result in an infinite loop with a touch of undefined
+behavior.
+
+```
+KVStore storage = ...;
+for (auto it = storage.begin("alpha"); it != storage.end("omega"); ++it) {}
+```
+
+#### Data organization on disk
+
+Each `PropertyValueStore` instance can access a static `KVStore` object that can
+store `(key, value)` pairs on disk. The key of each property on disk consists of
+two parts &mdash; a unique identifier (unsigned 64-bit integer) of the current
+record version (see mvcc docummentation for further clarification) and a 
+property key as described above. The actual value of the property is serialized
+into a bytestring using bolt `BaseEncoder`. Similarly, deserialization is
+performed by bolt `Decoder`.

docs/dev/toolchain-bootstrap.md

@@ -0,0 +1,152 @@
+# Bootstrapping Compilation Toolchain for Memgraph
+
+Requirements:
+
+  * libstdc++ shipped with gcc-6.3 or gcc-6.4
+  * cmake >= 3.1, Debian Stretch uses cmake-3.7.2
+  * clang-3.9
+
+## Installing gcc-6.4
+
+gcc-6.3 has a bug, so use the 6.4 version which is just a bugfix release.
+
+Requirements on CentOS 7:
+
+  * wget
+  * make
+  * gcc (bootstrap)
+  * gcc-c++ (bootstrap)
+  * gmp-devel (bootstrap)
+  * mpfr-devel (bootstrap)
+  * libmpc-devel (bootstrap)
+  * zip
+  * perl
+  * dejagnu (testing)
+  * expect (testing)
+  * tcl (testing)
+
+```
+wget ftp://ftp.mpi-sb.mpg.de/pub/gnu/mirror/gcc.gnu.org/pub/gcc/releases/gcc-6.4.0/gcc-6.4.0.tar.gz
+tar xf gcc-6.4.0.tar.gz
+cd gcc-6.4.0
+mkdir build
+cd build
+../configure --disable-multilib --prefix=<install-dst>
+make
+# Testing
+make -k check
+make install
+```
+
+*Do not put gcc + libs on PATH* (unless you know what you are doing).
+
+## Installing cmake-3.7.2
+
+Requirements on CentOS 7:
+
+  * wget
+  * make
+  * gcc
+  * gcc-c++
+  * ncurses-devel (optional, for ccmake)
+
+```
+wget https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
+tar xf cmake-3.7.2.tar.gz
+cd cmake-3.7.2.tar.gz
+./bootstrap --prefix<install-dst>
+make
+make install
+```
+
+Put cmake on PATH (if appropriate)
+
+**Fix the bug in CpackRPM**
+
+`"<path-to-cmake>/share/cmake-3.7/Modules/CPackRPM.cmake" line 2273 of 2442`
+
+The line
+
+```
+set(RPMBUILD_FLAGS "-bb")
+```
+needs to be before
+
+```
+if(CPACK_RPM_GENERATE_USER_BINARY_SPECFILE_TEMPLATE OR NOT CPACK_RPM_USER_BINARY_SPECFILE)
+```
+
+It was probably accidentally placed after, and is fixed in later cmake
+releases.
+
+## Installing clang-3.9
+
+Requirements on CentOS 7:
+
+  * wget
+  * make
+  * cmake
+
+```
+wget http://releases.llvm.org/3.9.1/llvm-3.9.1.src.tar.xz
+tar xf llvm-3.9.1.src.tar.xz
+mv llvm-3.9.1.src llvm
+
+wget http://releases.llvm.org/3.9.1/cfe-3.9.1.src.tar.xz
+tar xf cfe-3.9.1.src.tar.xz
+mv cfe-3.9.1.src llvm/tools/clang
+
+cd llvm
+mkdir build
+cd build
+cmake -DCMAKE_BUILD_TYPE="Release" -DGCC_INSTALL_PREFIX=<gcc-dir> \
+-DCMAKE_C_COMPILER=<gcc> -DCMAKE_CXX_COMPILER=<g++> \
+-DCMAKE_CXX_LINK_FLAGS="-L<gcc-dir>/lib64 -Wl,-rpath,<gcc-dir>/lib64" \
+-DCMAKE_INSTALL_PREFIX=<install-dst> ..
+make
+# Testing
+make check-clang
+make install
+```
+
+Put clang on PATH (if appropriate)
+
+## Memgraph
+
+Requirements on CentOS 7:
+
+  * libuuid-devel (antlr4)
+  * java-1.8.0-openjdk (antlr4)
+  * boost-static (too low version --- compile manually)
+  * rpm-build (RPM)
+  * python3 (tests, ...)
+  * which (required for rocksdb)
+  * sbcl (lisp C++ preprocessing)
+
+### Boost 1.62
+
+```
+wget https://netix.dl.sourceforge.net/project/boost/boost/1.62.0/boost_1_62_0.tar.gz
+tar xf boost_1_62_0.tar.gz
+cd boost_1_62_0
+./bootstrap.sh --with-toolset=clang --with-libraries=iostreams,serialization --prefix=<install-dst>
+./b2
+# Default installs to /usr/local/
+./b2 install
+```
+
+### Building Memgraph
+
+clang is *required* to be findable by cmake, i.e. it should be on PATH.
+cmake isn't required to be on the path, since you run it manually, so can use
+the full path to executable in order to run it. Obviously, it is convenient to
+put cmake also on PATH.
+
+Building is done as explained in [Quick Start](quick-start.md), but each
+`make` invocation needs to be prepended with:
+
+`LD_RUN_PATH=<gcc-dir>/lib64 make ...`
+
+### RPM
+
+Name format: `memgraph-<version>-<pkg-version>.<arch>.rpm`

docs/dev/workflow.md

@@ -0,0 +1,177 @@
+# Memgraph Workflow
+
+This chapter describes the usual workflow for working on Memgraph.
+
+## Git
+
+Memgraph uses [git](https://git-scm.com/) for source version control. If you
+obtained the source, you probably already have it installed. Before you can
+track new changes, you need to setup some basic information.
+
+First, tell git your name:
+
+    git config --global user.name "FirstName LastName"
+
+Then, set your Memgraph email:
+
+    git config --global user.email "my.email@memgraph.com"
+
+Finally, make git aware of your favourite editor:
+
+    git config --global core.editor "vim"
+
+## Phabricator
+
+All of the code in Memgraph needs to go through code review before it can be
+accepted in the codebase. This is done through
+[Phabricator](https://phacility.com/phabricator/). The command line tool for
+interfacing with Phabricator is
+[arcanist](https://phacility.com/phabricator/arcanist/). You should already
+have it installed if you followed the steps in [Quick Start](quick-start.md).
+
+The only required setup is to go in the root of Memgraph's project and run:
+
+    arc install-certificate
+
+## Working on Your Feature Branch
+
+Git has a concept of source code *branches*. The `master` branch contains all
+of the changes which were reviewed and accepted in Memgraph's code base. The
+`master` branch is selected by default.
+
+### Creating a Branch
+
+When working on a new feature or fixing a bug, you should create a new branch
+out of the `master` branch. For example, let's say you are adding static type
+checking to the query language compiler. You would create a branch called
+`mg_query_static_typing` with the following command:
+
+    git branch mg_query_static_typing
+
+To switch to that branch, type:
+
+    git checkout mg_query_static_typing
+
+Since doing these two steps will happen often, you can use a shortcut command:
+
+    git checkout -b mg_query_static_typing
+
+Note that a branch is created from the currently selected branch. So, if you
+wish to create another branch from `master` you need to switch to `master`
+first.
+
+The usual convention for naming your branches is `mg_<feature_name>`, you may
+switch underscores ('\_') for hyphens ('-').
+
+Do take care not to mix the case of your branch names! Certain operating
+systems (like Windows) don't distinguish the casing in git branches. This may
+cause hard to track down issues when trying to switch branches. Therefore, you
+should always name your branches with lowercase letters.
+
+### Making and Committing Changes
+
+When you have a branch for your new addition, you can now actually start
+implementing it. After some amount of time, you may have created new files,
+modified others and maybe even deleted unused files. You need to tell git to
+track those changes. This is accomplished with `git add` and `git rm`
+commands.
+
+    git add path-to-new-file path-to-modified-file
+    git rm path-to-deleted-file
+
+To check that everything is correctly tracked, you may use the `git status`
+command. It will also print the name of the currently selected branch.
+
+If everything seems OK, you should commit these changes to git.
+
+    git commit
+
+You will be presented with an editor where you need to type the commit
+message. Writing a good commit message is an art in itself. You should take a
+look at the links below. We try to follow these conventions as much as
+possible.
+
+  * [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/)
+  * [A Note About Git Commit Messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
+  * [stopwritingramblingcommitmessages](http://stopwritingramblingcommitmessages.com/)
+
+### Sending Changes on a Review
+
+After finishing your work on your feature branch, you will want to send it on
+code review. This is done through Arcanist. To do that, run the following
+command:
+
+    arc diff
+
+You will, once again, be presented with an editor where you need to describe
+your whole work. `arc` will by default fill that description with your commit
+messages. The title and summary of your work should also follow the
+conventions of git messages as described above. If you followed the
+guidelines, the message filled by `arc` should be fine.
+
+In addition to the message, you need to fill the `Reviewers:` line with
+usernames of people who should do the code review.
+
+You changes will be visible on Phabricator as a so called "diff". You can find
+the default view of active diffs
+[here](https://phabricator.memgraph.io/differential/)
+
+### Updating Changes Based on Review
+
+When you get comments in the code review, you will want to make additional
+modifications to your work. The same workflow as before applies: [Making and
+Committing Changes](#making-and-committing-changes)
+
+After making those changes, send them back on code review:
+
+    arc diff
+
+
+### Updating From New Master
+
+Let's say that, while you were working, someone else added some new features
+to the codebase that you would like to use in your current work. To obtain
+those changes you should update your `master` branch:
+
+    git checkout master
+    git pull origin master
+
+Now, these changes are on `master`, but you want them in your local branch. To
+do that, use `git rebase`:
+
+    git checkout mg_query_static_typing
+    git rebase master
+
+During `git rebase`, you may get reports that some files have conflicting
+changes. If you need help resolving them, don't be afraid to ask around! After
+you've resolved them, mark them as done with `git add` command. You may
+then continue with `git rebase --continue`.
+
+After the `git rebase` is done, you will now have new changes from `master` on
+your feature branch as if you just created and started working on that branch.
+You may continue with the usual workflow of [Making and Committing
+Changes](#making-and-committing-changes) and [Sending Changes on a
+Review](#sending-changes-on-a-review).
+
+### Sending Your Changes on Master Branch
+
+When your changes pass the code review, you are ready to integrate them in the
+`master` branch. To do that, run the following command:
+
+    arc land
+
+Arcanist will take care of obtaining the latest changes from `master` and
+merging your changes on top. If the `land` was successful, Arcanist will
+delete your local branch and you will be back on `master`. Continuing from the
+examples above, the deleted branch would be `mg_query_static_typing`.
+
+This marks the completion of your changes, and you are ready to work on
+something else.
+
+### Note For People Familiar With Git
+
+Since Arcanist takes care of merging your git commits and pushing them on
+`master`, you should *never* have to call `git merge` and `git push`. If you
+find yourself typing those commands, check that you are doing the right thing.
+The most common mistake is to use `git merge` instead of `git rebase` for the
+case described in [Updating From New Master](#updating-from-new-master).

docs/doxygen/README.md

@@ -1,6 +1,7 @@
 # Memgraph Code Documentation
 
-IMPORTANT: Auto-generated (run doxygen Doxyfile in the project root).
+IMPORTANT: auto-generated (run doxygen Doxyfile in the project root)
 
-* HTML - Open docs/doxygen/html/index.html.
-* Latex - Run make inside docs/doxygen/latex.
+* HTML - just open docs/doxygen/html/index.html
+
+* Latex - run make inside docs/doxygen/latex

docs/feature_ref/dynamic_graph_partitioning.md

@@ -0,0 +1,20 @@
+## Dynamic Graph Partitioner
+
+Memgraph supports dynamic graph partitioning which dynamically improves
+performance on badly partitioned dataset over workers. To enable it, the user
+should use the following flag when firing up the *master* node:
+
+```plaintext
+--dynamic_graph_partitioner_enable
+```
+
+### Parameters
+
+| Name | Default Value | Description | Range |
+|------|---------------|-------------|-------|
+|--dgp_improvement_threshold | 10 | How much better should specific node score
+be to consider a migration to another worker. This represents the minimal
+difference between new score that the vertex will have when migrated and the
+old one such that it's migrated. | Min: 1, Max: 100
+|--dgp_max_batch_size | 2000 | Maximal amount of vertices which should be
+migrated in one dynamic graph partitioner step. | Min: 1, Max: MaxInt32 |

docs/feature_spec/distributed.md

@@ -0,0 +1,78 @@
+# Distributed Memgraph specs
+This document describes reasnonings behind Memgraphs distributed concepts.
+
+## Distributed state machine
+Memgraphs distributed mode introduces two states of the cluster, recovering and
+working. The change between states shouldn't happen often, but when it happens
+it can take a while to make a transition from one to another.
+
+### Recovering
+This state is the default state for Memgraph when the cluster starts with
+recovery flags. If the recovery finishes successfully, the state changes to
+working. If recovery fails, the user will be presented with a message that
+explains what happened and what are the next steps.
+
+Another way to enter this state is failure. If the cluster encounters a failure,
+the master will enter the Recovering mode. This time, it will wait for all
+workers to respond with a message saying they are alive and well, and making
+sure they all have consistent state.
+
+### Working
+This state should be the default state of Memgraph most of the time. When in
+this state, Memgraph accepts connections from Bolt clients and allows query
+execution.
+
+If distributed execution fails for a transaction, that transaction, and all
+other active transactions will be aborted and the cluster will enter the
+Recovering state.
+
+## Durability
+One of the important concepts in distributed Memgraph is durability.
+
+### Cluster configuration
+When running Memgraph in distributed mode, the master will store cluster
+metadata in a persistent store. If fore some reason the cluster shuts down,
+recovering Memgraph from durability files shouldn't require any additional
+flags.
+
+### Database ID
+Each new and clean run of Memgraph should generate a new globally unique
+database id.  This id will associate all files that have persisted with this
+run. Adding the database id to snapshots, write-ahead logs and cluster metadata
+files ties them a specific Memgraph run, and it makes recovery easier to reason
+about.
+
+When recovering, the cluster won't generate a new id, but will reuse the one
+from the snapshot/wal that it was able to recover from.
+
+### Durability files
+Memgraph uses snapshots and write-ahead logs for durability.
+
+When Memgraph recovers it has to make sure all machines in the cluster recover
+to the same recovery point. This is done by finding a common snapshot and
+finding common transactions in per-machine available write-ahead logs.
+
+Since we can not be sure that each machine persisted durability files, we need
+to be able to negotiate a common recovery point in the cluster. Possible
+durability file failures could require to start the cluster from scratch,
+purging everything from storage and recovering from existing durability files.
+
+We need to ensure that we keep wal files containing information about
+transactions between all existing snapshots. This will provide better durability
+in the case of a random machine durability file failure, where the cluster can
+find a common recovery point that all machines in the cluster have.
+
+Also, we should suggest and make clear docs that anything less than two
+snapshots isn't considered safe for recovery.
+
+### Recovery
+The recovery happens in following steps:
+* Master enables worker registration.
+* Master recovers cluster metadata from the persisted storage.
+* Master waits all required workers to register.
+* Master broadcasts a recovery request to all workers.
+* Workers respond with with a set of possible recovery points.
+* Master finds a common recovery point for the whole cluster.
+* Master broadcasts a recovery request with the common recovery point.
+* Master waits for the cluster to recover.
+* After a successful cluster recovery, master can enter Working state.

docs/feature_spec/dynamic_graph_partitioning.md

@@ -0,0 +1,75 @@
+# Dynamic Graph Partitioning (abbr. DGP)
+
+## Implementation
+
+Take a look under `dev/memgraph/distributed/dynamic_graph_partitioning.md`.
+
+### Implemented parameters
+
+--dynamic-graph-partitioner-enabled (If the dynamic graph partitioner should be
+  enabled.) type: bool default: false (start time)
+--dgp-improvement-threshold (How much better should specific node score be
+  to consider a migration to another worker. This represents the minimal
+  difference between new score that the vertex will have when migrated
+  and the old one such that it's migrated.) type: int32 default: 10
+  (start time)
+--dgp-max-batch-size (Maximal amount of vertices which should be migrated
+  in one dynamic graph partitioner step.) type: int32 default: 2000
+  (start time)
+
+## Planning
+
+### Design decisions
+
+* Each partitioning session has to be a new transaction.
+* When and how does an instance perform the moves?
+  * Periodically.
+  * Token sharing (round robin, exactly one instance at a time has an
+    opportunity to perform the moves).
+* On server-side serialization error (when DGP receives an error).
+  -> Quit partitioning and wait for the next turn.
+* On client-side serialization error (when end client receives an error).
+  -> The client should never receive an error because of any
+     internal operation.
+  -> For the first implementation, it's good enough to wait until data becomes
+     available again.
+  -> It would be nice to achieve that DGP has lower priority than end client
+     operations.
+
+### End-user parameters
+
+* --dynamic-graph-partitioner-enabled (execution time)
+* --dgp-improvement-threshold (execution time)
+* --dgp-max-batch-size (execution time)
+* --dgp-min-batch-size (execution time)
+  -> Minimum number of nodes that will be moved in each step.
+* --dgp-fitness-threshold (execution time)
+  -> Do not perform moves if partitioning is good enough.
+* --dgp-delta-turn-time (execution time)
+  -> Time between each turn.
+* --dgp-delta-step-time (execution time)
+  -> Time between each step.
+* --dgp-step-time (execution time)
+  -> Time limit per each step.
+
+### Testing
+
+The implementation has to provide good enough results in terms of:
+  * How good the partitioning is (numeric value), aka goodness.
+  * Workload execution time.
+  * Stress test correctness.
+
+Test cases:
+  * N not connected subgraphs
+    -> shuffle nodes to N instances
+    -> run partitioning
+    -> test perfect partitioning.
+  * N connected subgraph
+    -> shuffle nodes to N instance
+    -> run partitioning
+    -> test partitioning.
+  * Take realistic workload (Long Running, LDBC1, LDBC2, Card Fraud, BFS, WSP)
+    -> measure exec time
+    -> run partitioning
+    -> test partitioning
+    -> measure exec time (during and after partitioning).

docs/feature_spec/high_availability.md

@@ -0,0 +1,213 @@
+# High Availability (abbr. HA)
+
+## Introduction
+
+High availability is a characteristic of a system which aims to ensure a
+certain level of operational performance for a higher-than-normal period.
+Although there are multiple ways to design highly available systems, Memgraph
+strives to achieve HA by elimination of single points of failure. In essence,
+this implies adding redundancy to the system so that a failure of a component
+does not imply the failure of the entire system.
+
+## Theoretical Background
+
+The following chapter serves as an introduction into some theoretical aspects
+of Memgraph's high availability implementation. If the reader is solely
+interested in design decisions around HA implementation, they can skip this
+chapter.
+
+An important implication of any HA implementation stems from Eric Brewer's
+[CAP Theorem](https://fenix.tecnico.ulisboa.pt/downloadFile/1126518382178117/10.e-CAP-3.pdf)
+which states that it is impossible for a distributed system to simultaneously
+achieve:
+
+  * Consistency (C - every read receives the most recent write or an error)
+  * Availability (A - every request receives a response that is not an error)
+  * Partition tolerance (P - The system continues to operate despite an
+                             arbitrary number of messages being dropped by the
+                             network between nodes)
+
+In the context of HA, Memgraph should strive to achieve CA.
+
+### Consensus
+
+Implications of the CAP theorem naturally lead us towards introducing a
+cluster of machines which will have identical internal states. When a designated
+machine for handling client requests fails, it can simply be replaced with
+another.
+
+Well... turns out this is not as easy as it sounds :(
+
+Keeping around a cluster of machines with consistent internal state is an
+inherently difficult problem. More precisely, this problem is as hard as
+getting a cluster of machines to agree on a single value, which is a highly
+researched area in distributed systems. Our research of state of the art
+consensus algorithms lead us to Diego Ongaro's
+[Raft algorithm](https://raft.github.io/raft.pdf).
+
+#### Raft
+
+As you might have guessed, analyzing each subtle detail of Raft goes way
+beyond the scope of this document. In the remainder of the chapter we will
+outline only the most important ideas and implications, leaving all further
+analysis to the reader. Detailed explanation can be found either in Diego's
+[dissertation](https://ramcloud.stanford.edu/~ongaro/thesis.pdf) \[1\] or the
+Raft [paper](https://raft.github.io/raft.pdf) \[2\].
+
+In essence, Raft allows us to implement the previously mentioned idea of
+managing a cluster of machines with identical internal states. In other
+words, the Raft protocol allows us to manage a cluster of replicated
+state machines which is fully functional as long as the *majority* of
+the machines in the cluster operate correctly.
+
+Another important fact is that those state machines must be *deterministic*.
+In other words, the same command on two different machines with the same
+internal state must yield the same result. This is important because Memgraph,
+as a black box, is not entirely deterministic. Non-determinism can easily be
+introduced by the user (e.g. by using the `rand` function) or by algorithms
+behind query execution (e.g. introducing fuzzy logic in the planner could yield
+a different order of results). Luckily, once we enter the storage level,
+everything should be fully deterministic.
+
+To summarize, Raft is a protocol which achieves consensus in a cluster of
+deterministic state machines via log replication. The cluster is fully
+functional if the majority of the machines work correctly. The reader
+is strongly encouraged to gain a deeper understanding (at least read through
+the paper) of Raft before reading the rest of this document.
+
+## Integration with Memgraph
+
+The first thing that should be defined is a single instruction within the
+context of Raft (i.e. a single entry in a replicated log). As mentioned
+before, these instructions should be completely deterministic when applied
+to the state machine. We have therefore decided that the appropriate level
+of abstraction within Memgraph corresponds to `StateDelta`-s (data structures
+which describe a single change to the Memgraph state, used for durability
+in WAL). Moreover, a single instruction in a replicated log will consist of a
+batch of `StateDelta`s which correspond to a single **committed** transaction.
+This decision both improves performance and handles some special cases that
+present themselves otherwise by leveraging the knowledge that the transaction
+should be committed.
+
+"What happens with aborted transactions?"
+
+A great question, they are handled solely by the leader which is the only
+machine that communicates with the client. Aborted transactions do not alter
+the state of the database and there is no need to replicate it to other machines
+in the cluster. If, for instance, the leader dies before returning the result
+of some read operation in an aborted transaction, the client will notice that
+the leader has crashed. A new leader will be elected in the next term and the
+client should retry the transaction.
+
+"OK, that makes sense! But, wait a minute, this is broken by design! Merely
+generating `StateDelta`s on the leader for any transaction will taint its
+internal storage before sending the first RPC to some follower. This deviates
+from Raft and will crash the universe!"
+
+Another great observation. It is indeed true that applying `StateDelta`s makes
+changes to local storage, but only a single type of `StateDelta` makes that
+  change durable. That `StateDelta` type is called `TRANSACTION_COMMIT` and we
+will change its behaviour when working as a HA instance. More precisely, we
+must not allow the transaction engine to modify the commit log saying that
+the transaction has been committed. That action should be delayed until those
+`StateDelta`s have been applied to the majority of the cluster. At that point
+the commit log can be safely modified leaving it up to Raft to ensure the
+durability of the transaction.
+
+We should also address one subtle detail that arises in this case. Consider
+the following scenario:
+
+  * The leader starts working on a transaction which creates a new record in the
+    database. Suppose that record is stored in the leader's internal storage
+    but the transaction was not committed (i.e. no such entry in the commit log).
+  * The leader should start replicating those `StateDelta`s to its followers
+    but, suddenly, it's cut off from the rest of the cluster.
+  * Due to timeout, a new election is held and a new leader has been elected.
+  * Our old leader comes back to life and becomes a follower.
+  * The new leader receives a transaction which creates that same record, but
+    this transaction is successfully replicated and committed by the new leader.
+
+The problem lies in the fact that there is still a record within the internal
+storage of our old leader with the same transaction ID and GID as the recently
+committed record by the new leader. Obviously, this is broken. As a solution, we
+will prefix the transaction ID with the Raft term, thus ensuring that the
+transaction IDs will differ.
+
+"When will followers append that transaction to their commit logs?"
+
+When the leader deduces that the transaction is safe to commit, it will include
+the relevant information in all further heartbeats which will alert the
+followers that it is safe to commit those entries from their raft logs.
+Naturally, the followers need not to delay appending data to the commit log
+as they know that the transaction has already been committed (from the clusters
+point of view). If this sounds really messed up, seriously, read the Raft paper.
+
+"How does the raft log differ from WAL"
+
+Conceptually, it doesn't. When operating in HA, we don't really need the
+recovery mechanisms implemented in Memgraph thus far. When a dead machine
+comes back to life, it will eventually come in sync with the rest of the
+cluster and everything will be done using the machine's raft log as well
+as the messages received from the cluster leader.
+
+"Those logs will become huge, isn't that recovery going to be painfully slow?"
+
+True, but there are mechanisms for making raft logs more compact. The most
+popular method is, wait for it, making snapshots :)
+Although the process of bringing an old machine back to life is a long one,
+it doesn't really affect the performance of the cluster in a great degree.
+The cluster will work perfectly fine with that machine being way out of sync.
+
+"I don't know, everything seems to be a lot slower than before!"
+
+Absolutely true, the user should be aware that they will suffer dire
+consequences on the performance side if they choose to be highly available.
+As Frankie says, "That's life!".
+
+"Also, I didn't really care about most of the things you've said. I'm
+not a part of the storage team and couldn't care less about the issues you
+face, how does HA affect 'my part of the codebase'?"
+
+Answer for query execution: That's ok, you'll be able to use the same beloved
+API (when we implement it, he he :) towards storage and continue to
+make fun of us when you find a bug.
+
+Answer for infrastructure: We'll talk. Some changes will surely need to
+be made on the Memgraph client. There is a chapter in Diego's dissertation
+called 'Client interaction', but we'll cross that bridge when we get there.
+There will also be the whole 'integration with Jepsen tests' thing going on.
+
+Answer for analytics: I'm astonished you've read this article. Wanna join
+storage?
+
+### Subtlety Regarding Reads
+
+As we have hinted in the previous chapter, we would like to bypass log
+replication for operations which do not alter the internal state of Memgraph.
+Those operations should therefore be handled only by the leader, which is not
+as trivial as it seems. The subtlety arises from the fact that a (newly-elected)
+leader can have an entry in its log which was committed by the previous leader
+that has crashed but that entry is not yet committed in its internal storage
+by the current leader. Moreover, the rule about safely committing logs that are
+replicated on the majority of the cluster only applies for entries replicated in
+the leaders current term. Therefore, we are faced with two issues:
+
+  * We cannot simply perform read operations if the leader has a non-committed
+    entry in its log (breaks consistency).
+  * Replicating those entries onto the majority of the cluster is not enough
+    to guarantee that they can be safely committed.
+
+This can be solved by introducing a blank no-op operation which the new leader
+will try to replicate at the start of its term. Once that operation is
+replicated and committed, the leader can safely perform those non-altering
+operations on its own.
+
+For further information about these issues, you should check out section
+5.4.2 from the raft paper \[1\] which hints as to why its not safe to commit
+entries from previous terms. Also, you should check out section 6.4 from
+the thesis \[2\] which goes into more details around efficiently processing
+read-only queries.
+
+## How do we test HA
+
+[Check this out](https://jepsen.io/analyses/dgraph-1-0-2)

docs/feature_spec/kafka/opencypher.md

@@ -0,0 +1,80 @@
+# Kafka - openCypher clause
+
+One must be able to specify the following when importing data from Kafka:
+
+* Kafka URI
+* Kafka topic
+* Transform [script](transform.md) URI
+
+
+Minimum required syntax looks like:
+```opencypher
+CREATE STREAM stream_name AS LOAD DATA KAFKA 'URI'
+  WITH TOPIC 'topic'
+  WITH TRANSFORM 'URI';
+```
+
+
+The full openCypher clause for creating a stream is:
+```opencypher
+CREATE STREAM stream_name AS
+  LOAD DATA KAFKA 'URI'
+  WITH TOPIC 'topic'
+  WITH TRANSFORM 'URI'
+  [BATCH_INTERVAL milliseconds]
+  [BATCH_SIZE count]
+```
+The `CREATE STREAM` clause happens in a transaction.
+
+`WITH TOPIC` parameter specifies the Kafka topic from which we'll stream
+data.
+
+`WITH TRANSFORM` parameter should contain a URI of the transform script.
+
+`BATCH_INTERVAL` parameter defines the time interval in milliseconds
+which is the time between two successive stream importing operations.
+
+`BATCH_SIZE` parameter defines the count of Kafka messages that will be
+batched together before import.
+
+If both `BATCH_INTERVAL` and `BATCH_SIZE` parameters are given, the condition
+that is satisfied first will trigger the batched import.
+
+Default value for `BATCH_INTERVAL` is 100 milliseconds, and the default value
+for `BATCH_SIZE` is 10;
+
+The `DROP` clause deletes a stream:
+```opencypher
+DROP STREAM stream_name;
+```
+
+The `SHOW` clause enables you to see all configured streams:
+```opencypher
+SHOW STREAMS;
+```
+
+You can also start/stop streams with the `START` and `STOP` clauses:
+```opencypher
+START STREAM stream_name [LIMIT count BATCHES];
+STOP STREAM stream_name;
+```
+A stream needs to be stopped in order to start it and it needs to be started in
+order to stop it. Starting a started or stopping a stopped stream will not
+affect that stream.
+
+There are also convenience clauses to start and stop all streams:
+```opencypher
+START ALL STREAMS;
+STOP ALL STREAMS;
+```
+
+Before the actual import, you can also test the stream with the `TEST
+STREAM` clause:
+```opencypher
+TEST STREAM stream_name [LIMIT count BATCHES];
+```
+When a stream is tested, data extraction and transformation occurs, but no
+output is inserted in the graph.
+
+A stream needs to be stopped in order to test it. When the batch limit is
+omitted, `TEST STREAM` will run for only one batch by default.

docs/feature_spec/kafka/transform.md

@@ -0,0 +1,34 @@
+# Kafka - data transform
+
+The transform script is a user defined script written in Python.  The script
+should be aware of the data format in the Kafka message.
+
+Each Kafka message is byte length encoded, which means that the first eight
+bytes of each message contain the length of the message.
+
+A sample code for a streaming transform script could look like this:
+
+```python
+def create_vertex(vertex_id):
+  return ("CREATE (:Node {id: $id})", {"id": vertex_id})
+
+
+def create_edge(from_id, to_id):
+  return ("MATCH (n:Node {id: $from_id}), (m:Node {id: $to_id}) "\
+          "CREATE (n)-[:Edge]->(m)", {"from_id": from_id, "to_id": to_id})
+
+
+def stream(batch):
+    result = []
+    for item in batch:
+        message = item.decode('utf-8').strip().split()
+        if len(message) == 1:
+          result.append(create_vertex(message[0])))
+        else:
+          result.append(create_edge(message[0], message[1]))
+    return result
+
+```
+
+The script should output openCypher query strings based on the type of the
+records.

docs/feature_spec/tensorflow_op/technicalities.md

@@ -0,0 +1,61 @@
+# Tensorflow Op - Technicalities
+
+The final result should be a shared object (".so") file that can be
+dynamically loaded by the Tensorflow runtime in order to directly
+access the bolt client.
+
+## About Tensorflow
+
+Tensorflow is usually used with Python such that the Python code is used
+to define a directed acyclic computation graph. Basically no computation
+is done in Python. Instead, values from Python are copied into the graph
+structure as constants to be used by other Ops. The directed acyclic graph
+naturally ends up with two sets of border nodes, one for inputs, one for
+outputs. These are sometimes called "feeds".
+
+Following the Python definition of the graph, during training, the entire
+data processing graph/pipeline is called from Python as a single expression.
+This leads to lazy evaluation since the called result has already been
+defined for a while.
+
+Tensorflow internally works with tensors, i.e. n-dimensional arrays. That
+means all of its inputs need to be matrices as well as its outputs. While
+it is possible to feed data directly from Python's numpy matrices straight
+into Tensorflow, this is less desirable than using the Tensorflow data API
+(which defines data input and processing as a Tensorflow graph) because:
+
+  1. The data API is written in C++ and entirely avoids Python and as such
+  is faster
+  2. The data API, unlike Python is available in "Tensorflow serving". The
+  default way to serve Tensorflow models in production.
+
+Once the entire input pipeline is defined via the tf.data API, its input
+is basically a list of node IDs the model is supposed to work with. The
+model, through the data API knows how to connect to Memgraph and execute
+openCypher queries in order to get the remaining data it needs.
+(For example features of neighbouring nodes.)
+
+## The Interface
+
+I think it's best you read the official guide...  
+<https://www.tensorflow.org/extend/adding_an_op>  
+And especially the addition that specifies how data ops are special  
+<https://www.tensorflow.org/extend/new_data_formats>  
+
+## Compiling the TF Op
+
+There are two options for compiling a custom op.  
+One of them involves pulling the TF source, adding your code to it and
+compiling via bazel.  
+This is probably awkward to do for us and would
+significantly slow down compilation.  
+
+The other method involves installing Tensorflow as a Python package and
+pulling the required headers from for example:  
+`/usr/local/lib/python3.6/site-packages/tensorflow/include`  
+We can then compile our Op with our regular build system.
+
+This is practical since we can copy the required headers to our repo.
+If necessary, we can have several versions of the headers to build several
+versions of our Op for every TF version which we want to support.
+(But this is unlikely to be required as the API should be stable).

docs/feature_spec/tensorflow_op/usage_example.md

@@ -0,0 +1,142 @@
+# Example for Using the Bolt Client Tensorflow Op
+
+## Dynamic Loading
+
+``` python3
+import tensorflow as tf
+
+mg_ops = tf.load_op_library('/usr/bin/memgraph/tensorflow_ops.so')
+```
+
+## Basic Usage
+
+``` python3
+dataset = mg_ops.OpenCypherDataset(
+            # This is probably unfortunate as the username and password
+            # get hardcoded into the graph, but for the simple case it's fine
+            "hostname:7687", auth=("user", "pass"),
+
+            # Our query
+            '''
+            MATCH (n:Train) RETURN n.id, n.features
+            ''',
+
+            # Cast return values to these types
+            (tf.string, tf.float32))
+
+# Some Tensorflow data api boilerplate
+iterator = dataset.make_one_shot_iterator()
+next_element = iterator.get_next()
+
+# Up to now we have only defined our computation graph which basically
+# just connects to Memgraph
+# `next_element` is not really data but a handle to a node in the Tensorflow
+# graph, which we can and do evaluate
+# It is a Tensorflow tensor with shape=(None, 2)
+# and dtype=(tf.string, tf.float)
+# shape `None` means the shape of the tensor is unknown at definition time
+# and is dynamic and will only be known once the tensor has been evaluated
+
+with tf.Session() as sess:
+    node_ids = sess.run(next_element)
+    # `node_ids` contains IDs and features of all the nodes 
+    # in the graph with the label "Train"
+    # It is a numpy.ndarray with a shape ($n_matching_nodes, 2)
+```
+
+## Memgraph Client as a Generic Tensorflow Op
+
+Other than the Tensorflow Data Op, we'll want to support a generic Tensorflow
+Op which can be put anywhere in the Tensorflow computation Graph. It takes in
+an arbitrary tensor and produces a tensor. This would be used in the GraphSage
+algorithm to fetch the lowest level features into Tensorflow
+
+```python3
+requested_ids = np.array([1, 2, 3])
+ids_placeholder = tf.placeholder(tf.int32)
+
+model = mg_ops.OpenCypher()
+    "hostname:7687", auth=("user", "pass"),
+    """
+    UNWIND $node_ids as nid
+    MATCH (n:Train {id: nid})
+    RETURN n.features
+    """,
+
+    # What to call the input tensor as an openCypher parameter
+    parameter_name="node_ids",
+
+    # Type of our resulting tensor
+    dtype=(tf.float32)
+)
+
+features = model(ids_placeholder)
+
+with tf.Session() as sess:
+    result = sess.run(features,
+                    feed_dict={ids_placeholder: requested_ids})
+```
+
+This is probably easier to implement than the Data Op, so it might be a good
+idea to start with.
+
+## Production Usage
+
+During training, in the GraphSage algorithm at least, Memgraph is at the
+beginning and at the end of the Tensorflow computation graph.  
+At the beginning, the Data Op provides the node IDs which are fed into the
+generic Tensorflow Op to find their neighbours and their neighbours and
+their features.
+
+Production usage differs in that we don't use the Data Op. The Data Op is
+effectively cut off and the initial input is fed by Tensorflow serving,
+with the data found in the request.
+
+For example a JSON request to classify a node might look like:
+
+`POST http://host:port/v1/models/GraphSage/versions/v1:classify`
+
+With the contents:
+
+```json
+{
+    "examples": [
+        {"node_id": 1},
+        {"node_id": 2}
+    ],
+}
+```
+
+Every element of the "examples" list is an example to be computed. Each is
+represented by a dict with keys matching names of feeds in the Tensorflow
+graph and values being the values we want fed in for each example
+
+The REST API then replies in kind with the classification result in JSON
+
+Note about adding our custom Op to Tensorflow serving.  
+Our Ops .so can be added into the Bazel build to link with Tensorflow serving
+or it can be dynamically loaded by starting Tensorflow serving with a flag
+`--custom_op_paths`
+
+## Considerations
+
+There might be issues here that the url to connect to Memgraph is
+hardcoded into the op and would thus be wrong when moved to production,
+requiring some type of a hack to make work. We probably want to solve
+this by having the client op take in another tf.Variable as an input
+which would contain a connection url and username/password.  
+We have to research whether this makes it easy enough to move to
+production, as the connection string variable is still a part of the
+graph, but maybe easier to replace.  
+
+It is probably the best idea to utilize openCypher parameters to make
+our queries flexible. The exact API as to how to declare the parameters
+in Python is open to discussion.
+
+The Data Op might not even be necessary to implement as it is not
+key for production use. It can be replaced in training mode with
+feed dicts and either
+
+ 1. Getting the initial list of nodes via a Python Bolt client
+ 2. Creating a separate Tensorflow computation graph that gets all the
+ relevant node IDs into Python

docs/presentation/latex-template/README.md

@@ -0,0 +1,22 @@
+# Memgraph LaTeX Beamer Template
+
+This folder contains all of the needed files for creating a presentation with
+Memgraph styling. You should use this style for any public presentations.
+
+Feel free to improve it according to style guidelines and raise issues if you
+find any.
+
+## Usage
+
+Copy the contents of this folder (excluding this README file) to where you
+want to write your own presentation. After copying, you can start editing the
+`template.tex` with your content.
+
+To compile the presentation to a PDF, run `latexmk -pdf -xelatex`. Some
+directives require XeLaTeX, so you need to pass `-xelatex` as the final option
+of `latexmk`. You may also need to install some packages if the compilation
+complains about missing packages.
+
+To clean up the generated files, use `latexmk -C`. This will also delete the
+generated PDF. If you wish to remove generated files except the PDF, use
+`latexmk -c`.

docs/presentation/latex-template/mg-beamer.cls

@@ -0,0 +1,82 @@
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesClass{mg-beamer}[2018/03/26 Memgraph Beamer]
+
+\DeclareOption*{\PassOptionsToClass{\CurrentOption}{beamer}}
+
+\ProcessOptions \relax
+
+\LoadClass{beamer}
+
+\usetheme{Pittsburgh}
+
+% Memgraph color palette
+\definecolor{mg-purple}{HTML}{720096}
+\definecolor{mg-red}{HTML}{DD2222}
+\definecolor{mg-orange}{HTML}{FB6E00}
+\definecolor{mg-yellow}{HTML}{FFC500}
+\definecolor{mg-gray}{HTML}{857F87}
+\definecolor{mg-black}{HTML}{231F20}
+
+\RequirePackage{fontspec}
+% Title fonts
+\setbeamerfont{frametitle}{family={\fontspec[Path = ./mg-style/fonts/]{EncodeSansSemiCondensed-Regular.ttf}}}
+\setbeamerfont{title}{family={\fontspec[Path = ./mg-style/fonts/]{EncodeSansSemiCondensed-Regular.ttf}}}
+% Body font
+\RequirePackage[sfdefault,light]{roboto}
+% Roboto is pretty bad for monospace font. We will find a replacement.
+% \setmonofont{RobotoMono-Regular.ttf}[Path = ./mg-style/fonts/]
+
+% Title slide styles
+% \setbeamerfont{frametitle}{size=\huge}
+% \setbeamerfont{title}{size=\huge}
+% \setbeamerfont{date}{size=\tiny}
+
+% Other typography styles
+\setbeamertemplate{frametitle}[default][center]
+\setbeamercolor{frametitle}{fg=mg-black}
+\setbeamercolor{title}{fg=mg-black}
+\setbeamercolor{section in toc}{fg=mg-black}
+\setbeamercolor{local structure}{fg=mg-orange}
+\setbeamercolor{alert text}{fg=mg-red}
+
+% Commands
+\newcommand{\mgalert}[1]{{\usebeamercolor[fg]{alert text}#1}}
+\newcommand{\titleframe}{\frame[plain]{\titlepage}}
+\newcommand{\mgtexttt}[1]{{\textcolor{mg-gray}{\texttt{#1}}}}
+
+% Title slide background
+\RequirePackage{tikz,calc}
+% Use title-slide-169 if aspect ration is 16:9
+\pgfdeclareimage[interpolate=true,width=\paperwidth,height=\paperheight]{logo}{mg-style/title-slide-169}
+\setbeamertemplate{background}{
+  \begin{tikzpicture}
+  \useasboundingbox (0,0) rectangle (\the\paperwidth,\the\paperheight);
+    \pgftext[at=\pgfpoint{0}{0},left,base]{\pgfuseimage{logo}};
+  \ifnum\thepage>1\relax
+  \useasboundingbox (0,0) rectangle (\the\paperwidth,\the\paperheight);
+      \fill[white, opacity=1](0,\the\paperheight)--(\the\paperwidth,\the\paperheight)--(\the\paperwidth,0)--(0,0)--(0,\the\paperheight);
+  \fi
+  \end{tikzpicture}
+}
+
+% Footline content
+\setbeamertemplate{navigation symbols}{}%remove navigation symbols
+ \setbeamertemplate{footline}{
+  \begin{beamercolorbox}[ht=1.6cm,wd=\paperwidth]{footlinecolor}
+    \vspace{0.1cm}
+    \hfill
+    \begin{minipage}[c]{3cm}
+      \begin{center}
+       \includegraphics[height=0.8cm]{mg-style/memgraph-logo.png}
+      \end{center}
+    \end{minipage}
+    \begin{minipage}[c]{7cm}
+         \insertshorttitle\ --- \insertsection
+    \end{minipage}
+    \begin{minipage}[c]{2cm}
+        \tiny{\insertframenumber{} of \inserttotalframenumber}
+    \end{minipage}
+   \end{beamercolorbox}
+}
+
+\endinput

docs/presentation/latex-template/template.tex

@@ -0,0 +1,40 @@
+% Set 16:9 aspect ratio
+\documentclass[aspectratio=169]{mg-beamer}
+% Default directive sets the regular 4:3 aspect ratio
+% \documentclass{mg-beamer}
+\mode<presentation>
+
+% requires xelatex
+\usepackage{ccicons}
+
+\title{Insert Presentation Title}
+\titlegraphic{\ccbyncnd}
+\author{Insert Name}
+
+% Institute doesn't look good in our current styling class.
+% \institute[Memgraph Ltd.]{\pgfimage[height=1.5cm]{mg-logo.png}}
+
+% Date is autogenerated on compilation, so no need to set it explicitly,
+% unless you wish to override it with a different date.
+% \date{March 23, 2018}
+
+\begin{document}
+
+\titleframe
+
+\section{Intro}
+
+\begin{frame}{Contents}
+  \tableofcontents
+\end{frame}
+
+\begin{frame}{Memgraph Markup Test}
+  \begin{itemize}
+    \item \mgtexttt{Prefer \\mgtexttt for monospace}
+    \item Replace this slide with your own
+    \item Add even more slides in different sections
+    \item Make sure you spellcheck your presentation
+  \end{itemize}
+\end{frame}
+
+\end{document}

environment/.gitignore

@@ -1,2 +0,0 @@
-archives
-build

environment/os/.gitignore

@@ -1,3 +0,0 @@
-*.deb
-*.rpm
-*.tar.gz

environment/os/amzn-2.sh

@@ -1,156 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "amzn-2"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    gcc gcc-c++ make # generic build tools
-    wget # used for archive download
-    gnupg2 # used for archive signature verification
-    tar gzip bzip2 xz unzip # used for archive unpacking
-    zlib-devel # zlib library used for all builds
-    expat-devel xz-devel python3-devel texinfo
-    curl libcurl-devel # for cmake
-    readline-devel # for cmake and llvm
-    libffi-devel libxml2-devel # for llvm
-    libedit-devel pcre-devel automake bison # for swig
-    file
-    openssl-devel
-    gmp-devel
-    gperf
-    diffutils
-    patch
-    libipt libipt-devel # intel
-    perl # for openssl
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz # used for archive unpacking
-    zlib # zlib library used for all builds
-    expat xz-libs python3 # for gdb
-    readline # for cmake and llvm
-    libffi libxml2 # for llvm
-    openssl-devel
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make # build system
-    wget # for downloading libs
-    libuuid-devel java-11-openjdk # required by antlr
-    readline-devel # for memgraph console
-    python3-devel # for query modules
-    openssl-devel
-    libseccomp-devel
-    python3 python3-pip nmap-ncat # for tests
-    #
-    # IMPORTANT: python3-yaml does NOT exist on CentOS
-    # Install it using `pip3 install PyYAML`
-    #
-    PyYAML # Package name here does not correspond to the yum package!
-    libcurl-devel # mg-requests
-    rpm-build rpmlint # for RPM package building
-    doxygen graphviz # source documentation generators
-    which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    local missing=""
-    # On Fedora yum/dnf and python10 use newer glibc which is not compatible
-    # with ours, so we need to momentarely disable env
-    local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""}
-    LD_LIBRARY_PATH=""
-    for pkg in $1; do
-        if [ "$pkg" == "PyYAML" ]; then
-            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
-                missing="$pkg $missing"
-            fi
-            continue
-        fi
-        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
-            missing="$pkg $missing"
-        fi
-    done
-    if [ "$missing" != "" ]; then
-        echo "MISSING PACKAGES: $missing"
-        exit 1
-    fi
-    LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
-}
-
-install() {
-    cd "$DIR"
-    if [ "$EUID" -ne 0 ]; then
-        echo "Please run as root."
-        exit 1
-    fi
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests don't work without the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    yum update -y
-    for pkg in $1; do
-        if [ "$pkg" == libipt ]; then
-            if ! yum list installed libipt >/dev/null 2>/dev/null; then
-                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == libipt-devel ]; then
-            if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
-                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == nodejs ]; then
-            curl -sL https://rpm.nodesource.com/setup_16.x | bash -
-            if ! yum list installed nodejs >/dev/null 2>/dev/null; then
-                yum install -y nodejs
-            fi
-            continue
-        fi
-        if [ "$pkg" == PyYAML ]; then
-            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
-                pip3 install --user PyYAML
-            else # Running using sudo.
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
-            fi
-            continue
-        fi
-        if [ "$pkg" == nodejs ]; then
-            curl -sL https://rpm.nodesource.com/setup_16.x | bash -
-            if ! yum list installed nodejs >/dev/null 2>/dev/null; then
-                yum install -y nodejs
-            fi
-            continue
-        fi
-        if [ "$pkg" == java-11-openjdk ]; then
-            amazon-linux-extras install -y java-openjdk11
-            continue
-        fi
-        if [ "$pkg" == java-11-openjdk-devel ]; then
-            amazon-linux-extras install -y java-openjdk11
-            yum install -y java-11-openjdk-devel
-            continue
-        fi
-        yum install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/centos-7.sh

@@ -1,148 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "centos-7"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc gcc-c++ make # generic build tools
-    wget # used for archive download
-    gnupg2 # used for archive signature verification
-    tar gzip bzip2 xz unzip # used for archive unpacking
-    zlib-devel # zlib library used for all builds
-    expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python3-devel # gdb
-    texinfo # gdb
-    libcurl-devel # cmake
-    curl # snappy
-    readline-devel # cmake and llvm
-    libffi-devel libxml2-devel perl-Digest-MD5 # llvm
-    libedit-devel pcre-devel automake bison # swig
-    file
-    openssl-devel
-    gmp-devel
-    gperf
-    patch
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz # used for archive unpacking
-    zlib # zlib library used for all builds
-    expat libipt libbabeltrace xz-libs python3 # for gdb
-    readline # for cmake and llvm
-    libffi libxml2 # for llvm
-    openssl-devel
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    make pkgconfig # build system
-    curl wget # for downloading libs
-    libuuid-devel java-11-openjdk # required by antlr
-    readline-devel # for memgraph console
-    python3-devel # for query modules
-    openssl-devel
-    libseccomp-devel
-    python3 python-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
-    #
-    # IMPORTANT: python3-yaml does NOT exist on CentOS
-    # Install it using `pip3 install PyYAML`
-    #
-    PyYAML # Package name here does not correspond to the yum package!
-    libcurl-devel # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    rpm-build rpmlint # for RPM package building
-    doxygen graphviz # source documentation generators
-    which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    local missing=""
-    for pkg in $1; do
-        if [ "$pkg" == git ]; then
-            if ! which "git" >/dev/null; then
-                missing="git $missing"
-            fi
-            continue
-        fi
-        if [ "$pkg" == "PyYAML" ]; then
-            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
-                missing="$pkg $missing"
-            fi
-            continue
-        fi
-        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
-            missing="$pkg $missing"
-        fi
-    done
-    if [ "$missing" != "" ]; then
-        echo "MISSING PACKAGES: $missing"
-        exit 1
-    fi
-}
-
-install() {
-    cd "$DIR"
-    if [ "$EUID" -ne 0 ]; then
-        echo "Please run as root."
-        exit 1
-    fi
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    yum install -y epel-release
-    yum remove -y ius-release
-    yum install -y \
-      https://repo.ius.io/ius-release-el7.rpm
-    yum update -y
-    yum install -y wget python3 python3-pip
-    yum install -y git
-    for pkg in $1; do
-        if [ "$pkg" == libipt ]; then
-            if ! yum list installed libipt >/dev/null 2>/dev/null; then
-                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == libipt-devel ]; then
-            if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
-                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == dotnet-sdk-3.1 ]; then
-            if ! yum list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
-                wget -nv https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
-                rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
-                yum update -y
-                yum install -y dotnet-sdk-3.1
-            fi
-            continue
-        fi
-        if [ "$pkg" == PyYAML ]; then
-            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
-                pip3 install --user PyYAML
-            else # Running using sudo.
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
-            fi
-            continue
-        fi
-        yum install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/centos-9.sh

@@ -1,164 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "centos-9"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils-common gcc gcc-c++ make # generic build tools
-    wget # used for archive download
-    gnupg2 # used for archive signature verification
-    tar gzip bzip2 xz unzip # used for archive unpacking
-    zlib-devel # zlib library used for all builds
-    expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
-    readline-devel # for cmake and llvm
-    libffi-devel libxml2-devel # for llvm
-    libedit-devel pcre-devel automake bison # for swig
-    file
-    openssl-devel
-    gmp-devel
-    gperf
-    diffutils
-    libipt libipt-devel # intel
-    patch
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz # used for archive unpacking
-    zlib # zlib library used for all builds
-    expat xz-libs python3 # for gdb
-    readline # for cmake and llvm
-    libffi libxml2 # for llvm
-    openssl-devel
-    perl # for openssl
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkgconf-pkg-config # build system
-    wget # for downloading libs
-    libuuid-devel java-11-openjdk # required by antlr
-    readline-devel # for memgraph console
-    python3-devel # for query modules
-    openssl-devel
-    libseccomp-devel
-    python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
-    #
-    # IMPORTANT: python3-yaml does NOT exist on CentOS
-    # Install it manually using `pip3 install PyYAML`
-    #
-    PyYAML # Package name here does not correspond to the yum package!
-    libcurl-devel # mg-requests
-    rpm-build rpmlint # for RPM package building
-    doxygen graphviz # source documentation generators
-    which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
-    sbcl # for custom Lisp C++ preprocessing
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    local missing=""
-    for pkg in $1; do
-        if [ "$pkg" == "PyYAML" ]; then
-            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
-                missing="$pkg $missing"
-            fi
-            continue
-        fi
-        if [ "$pkg" == "python3-virtualenv" ]; then
-            continue
-        fi
-        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
-            missing="$pkg $missing"
-        fi
-    done
-    if [ "$missing" != "" ]; then
-        echo "MISSING PACKAGES: $missing"
-        exit 1
-    fi
-}
-
-install() {
-    cd "$DIR"
-    if [ "$EUID" -ne 0 ]; then
-        echo "Please run as root."
-        exit 1
-    fi
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    yum update -y
-    yum install -y wget git python3 python3-pip
-    for pkg in $1; do
-        # Since there is no support for libipt-devel for CentOS 9 we install
-        # Fedoras version of same libs, they are the same version but released
-        # for different OS
-        # TODO Update when libipt-devel releases for CentOS 9
-        if [ "$pkg" == libipt ]; then
-            if ! dnf list installed libipt >/dev/null 2>/dev/null; then
-                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == libipt-devel ]; then
-            if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
-                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == libbabeltrace-devel ]; then
-            if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
-                dnf install -y http://mirror.stream.centos.org/9-stream/CRB/x86_64/os/Packages/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == sbcl ]; then
-            if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
-                dnf install -y 	https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
-            fi
-            if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
-                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
-            fi
-            if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
-                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
-            fi
-            continue
-        fi
-        if [ "$pkg" == PyYAML ]; then
-            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
-                pip3 install --user PyYAML
-            else # Running using sudo.
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
-            fi
-            continue
-        fi
-        if [ "$pkg" == python3-virtualenv ]; then
-            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
-                pip3 install virtualenv
-                pip3 install virtualenvwrapper
-            else # Running using sudo.
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
-                sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
-            fi
-            continue
-        fi
-        yum install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/debian-10.sh

@@ -1,104 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "debian-10"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # used for archive download
-    gnupg # used for archive signature verification
-    tar gzip bzip2 xz-utils unzip # used for archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
-    libcurl4-openssl-dev # for cmake
-    libreadline-dev # for cmake and llvm
-    libffi-dev libxml2-dev # for llvm
-    curl # snappy
-    file # for libunwind
-    libssl-dev # for libevent
-    libgmp-dev # for gdb
-    gperf # for proxygen
-    git # for fbthrift
-    libedit-dev libpcre3-dev automake bison # for swig
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    libreadline7 # for cmake and llvm
-    libffi6 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # for downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # for memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    netcat # tests are using nc to wait for memgraph
-    python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-    python3-yaml # for the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
-    dotnet-sdk-3.1 golang nodejs npm
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    cat >/etc/apt/sources.list <<EOF
-deb http://deb.debian.org/debian/ buster main non-free contrib
-deb-src http://deb.debian.org/debian/ buster main non-free contrib
-deb http://deb.debian.org/debian/ buster-updates main contrib non-free
-deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free
-deb http://security.debian.org/debian-security buster/updates main contrib non-free
-deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
-EOF
-    cd "$DIR"
-    apt --allow-releaseinfo-change update
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    apt install -y wget
-    for pkg in $1; do
-        if [ "$pkg" == dotnet-sdk-3.1 ]; then
-            if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
-                wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
-                dpkg -i packages-microsoft-prod.deb
-                apt-get update
-                apt-get install -y apt-transport-https dotnet-sdk-3.1
-            fi
-            continue
-        fi
-        apt install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/debian-11-arm.sh

@@ -1,98 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "debian-11"
-check_architecture "arm64" "aarch64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # used for archive download
-    gnupg # used for archive signature verification
-    tar gzip bzip2 xz-utils unzip # used for archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev liblzma-dev python3-dev texinfo # for gdb
-    libcurl4-openssl-dev # for cmake
-    libreadline-dev # for cmake and llvm
-    libffi-dev libxml2-dev # for llvm
-    libedit-dev libpcre3-dev automake bison # for swig
-    curl # snappy
-    file # for libunwind
-    libssl-dev # for libevent
-    libgmp-dev
-    gperf # for proxygen
-    git # for fbthrift
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    file # for CPack
-    libreadline8 # for cmake and llvm
-    libffi7 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # for downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # for memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    netcat # tests are using nc to wait for memgraph
-    python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-    python3-yaml # for the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
-    golang nodejs npm
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    cat >/etc/apt/sources.list <<EOF
-deb http://deb.debian.org/debian bullseye main
-deb-src http://deb.debian.org/debian bullseye main
-
-deb http://deb.debian.org/debian-security/ bullseye-security main
-deb-src http://deb.debian.org/debian-security/ bullseye-security main
-
-deb http://deb.debian.org/debian bullseye-updates main
-deb-src http://deb.debian.org/debian bullseye-updates main
-EOF
-    cd "$DIR"
-    apt update
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    apt install -y wget
-    for pkg in $1; do
-        apt install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/debian-11.sh

@@ -1,107 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "debian-11"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # used for archive download
-    gnupg # used for archive signature verification
-    tar gzip bzip2 xz-utils unzip # used for archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
-    libcurl4-openssl-dev # for cmake
-    libreadline-dev # for cmake and llvm
-    libffi-dev libxml2-dev # for llvm
-    libedit-dev libpcre3-dev automake bison # for swig
-    curl # snappy
-    file # for libunwind
-    libssl-dev # for libevent
-    libgmp-dev
-    gperf # for proxygen
-    git # for fbthrift
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    file # for CPack
-    libreadline8 # for cmake and llvm
-    libffi7 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # for downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # for memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    netcat # tests are using nc to wait for memgraph
-    python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-    python3-yaml # for the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
-    dotnet-sdk-3.1 golang nodejs npm
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    cat >/etc/apt/sources.list <<EOF
-deb http://deb.debian.org/debian bullseye main
-deb-src http://deb.debian.org/debian bullseye main
-
-deb http://deb.debian.org/debian-security/ bullseye-security main
-deb-src http://deb.debian.org/debian-security/ bullseye-security main
-
-deb http://deb.debian.org/debian bullseye-updates main
-deb-src http://deb.debian.org/debian bullseye-updates main
-EOF
-    cd "$DIR"
-    apt update
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    apt install -y wget
-    for pkg in $1; do
-        if [ "$pkg" == dotnet-sdk-3.1  ]; then
-            if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
-                wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
-                dpkg -i packages-microsoft-prod.deb
-                apt-get update
-                apt-get install -y apt-transport-https dotnet-sdk-3.1
-            fi
-            continue
-        fi
-        apt install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/fedora-36.sh

@@ -1,103 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "fedora-36"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils-common gcc gcc-c++ make # generic build tools
-    wget # used for archive download
-    gnupg2 # used for archive signature verification
-    tar gzip bzip2 xz unzip # used for archive unpacking
-    zlib-devel # zlib library used for all builds
-    expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
-    curl libcurl-devel # for cmake
-    readline-devel # for cmake and llvm
-    libffi-devel libxml2-devel # for llvm
-    libedit-devel pcre-devel automake bison # for swig
-    file
-    openssl-devel
-    gmp-devel
-    gperf
-    diffutils
-    libipt libipt-devel # intel
-    patch
-    perl # for openssl
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz # used for archive unpacking
-    zlib # zlib library used for all builds
-    expat xz-libs python3 # for gdb
-    readline # for cmake and llvm
-    libffi libxml2 # for llvm
-    openssl-devel
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkgconf-pkg-config # build system
-    wget # for downloading libs
-    libuuid-devel java-11-openjdk # required by antlr
-    readline-devel # for memgraph console
-    python3-devel # for query modules
-    openssl-devel
-    libseccomp-devel
-    python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
-    libcurl-devel # mg-requests
-    rpm-build rpmlint # for RPM package building
-    doxygen graphviz # source documentation generators
-    which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
-    sbcl # for custom Lisp C++ preprocessing
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    local missing=""
-    # On Fedora yum/dnf and python10 use newer glibc which is not compatible
-    # with ours, so we need to momentarely disable env
-    local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
-    LD_LIBRARY_PATH=""
-    for pkg in $1; do
-        if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
-            missing="$pkg $missing"
-        fi
-    done
-    if [ "$missing" != "" ]; then
-        echo "MISSING PACKAGES: $missing"
-        exit 1
-    fi
-    LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
-}
-
-install() {
-    cd "$DIR"
-    if [ "$EUID" -ne 0 ]; then
-        echo "Please run as root."
-        exit 1
-    fi
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests don't work without the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    dnf update -y
-    for pkg in $1; do
-        dnf install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/template.sh

@@ -1,38 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "todo-os-name"
-
-TOOLCHAIN_BUILD_DEPS=(
-    pkg
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    pkg
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    pkg
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    echo "TODO: Implement ${FUNCNAME[0]}."
-    exit 1
-}
-
-install() {
-    echo "TODO: Implement ${FUNCNAME[0]}."
-    exit 1
-}
-
-# http://ahmed.amayem.com/bash-indirect-expansion-exploration
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/ubuntu-18.04.sh

@@ -1,74 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "ubuntu-18.04"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # archive download
-    gnupg # archive signature verification
-    tar gzip bzip2 xz-utils unzip # archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev # gdb
-    texinfo # gdb
-    libcurl4-openssl-dev # cmake
-    libreadline-dev # cmake and llvm
-    libffi-dev libxml2-dev # llvm
-    curl # snappy
-    file
-    git # for thrift
-    libgmp-dev # for gdb
-    gperf # for proxygen
-    libssl-dev
-    libedit-dev libpcre3-dev automake bison # swig
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 libipt1 libbabeltrace1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    libreadline7 # for cmake and llvm
-    libffi6 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    python3 virtualenv python3-virtualenv python3-pip # qa, macro bench and stress tests
-    python3-yaml # the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    apt install -y $1
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/ubuntu-20.04.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "ubuntu-20.04"
-check_architecture "x86_64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # used for archive download
-    gnupg # used for archive signature verification
-    tar gzip bzip2 xz-utils unzip # used for archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
-    libcurl4-openssl-dev # for cmake
-    libreadline-dev # for cmake and llvm
-    libffi-dev libxml2-dev # for llvm
-    curl # snappy
-    file
-    git # for thrift
-    libgmp-dev # for gdb
-    gperf # for proxygen
-    libssl-dev
-    libedit-dev libpcre3-dev automake bison # for swig
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    libreadline8 # for cmake and llvm
-    libffi7 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # for downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # for memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    netcat # tests are using nc to wait for memgraph
-    python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-    python3-yaml # for the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
-    dotnet-sdk-3.1 golang nodejs npm
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    cd "$DIR"
-    apt update
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    apt install -y wget
-    for pkg in $1; do
-        if [ "$pkg" == dotnet-sdk-3.1 ]; then
-            if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then
-                wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
-                dpkg -i packages-microsoft-prod.deb
-                apt-get update
-                apt-get install -y apt-transport-https dotnet-sdk-3.1
-            fi
-            continue
-        fi
-        apt install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"

environment/os/ubuntu-22.04-arm.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-
-set -Eeuo pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-source "$DIR/../util.sh"
-
-check_operating_system "ubuntu-22.04"
-check_architecture "arm64" "aarch64"
-
-TOOLCHAIN_BUILD_DEPS=(
-    coreutils gcc g++ build-essential make # generic build tools
-    wget # used for archive download
-    gnupg # used for archive signature verification
-    tar gzip bzip2 xz-utils unzip # used for archive unpacking
-    zlib1g-dev # zlib library used for all builds
-    libexpat1-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
-    libcurl4-openssl-dev # for cmake
-    libreadline-dev # for cmake and llvm
-    libffi-dev libxml2-dev # for llvm
-    curl # snappy
-    file
-    git # for thrift
-    libgmp-dev # for gdb
-    gperf # for proxygen
-    libssl-dev
-    libedit-dev libpcre3-dev automake bison # for swig
-)
-
-TOOLCHAIN_RUN_DEPS=(
-    make # generic build tools
-    tar gzip bzip2 xz-utils # used for archive unpacking
-    zlib1g # zlib library used for all builds
-    libexpat1 libbabeltrace1 liblzma5 python3 # for gdb
-    libcurl4 # for cmake
-    libreadline8 # for cmake and llvm
-    libffi7 libxml2 # for llvm
-    libssl-dev # for libevent
-)
-
-MEMGRAPH_BUILD_DEPS=(
-    git # source code control
-    make pkg-config # build system
-    curl wget # for downloading libs
-    uuid-dev default-jre-headless # required by antlr
-    libreadline-dev # for memgraph console
-    libpython3-dev python3-dev # for query modules
-    libssl-dev
-    libseccomp-dev
-    netcat # tests are using nc to wait for memgraph
-    python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-    python3-yaml # for the configuration generator
-    libcurl4-openssl-dev # mg-requests
-    sbcl # for custom Lisp C++ preprocessing
-    doxygen graphviz # source documentation generators
-    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
-    dotnet-sdk-6.0 golang nodejs npm
-    autoconf # for jemalloc code generation
-    libtool  # for protobuf code generation
-)
-
-list() {
-    echo "$1"
-}
-
-check() {
-    check_all_dpkg "$1"
-}
-
-install() {
-    cd "$DIR"
-    apt update
-    # If GitHub Actions runner is installed, append LANG to the environment.
-    # Python related tests doesn't work the LANG export.
-    if [ -d "/home/gh/actions-runner" ]; then
-        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
-    else
-        echo "NOTE: export LANG=en_US.utf8"
-    fi
-    apt install -y wget
-    for pkg in $1; do
-        if [ "$pkg" == dotnet-sdk-6.0 ]; then
-            if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
-                wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
-                dpkg -i packages-microsoft-prod.deb
-                apt-get update
-                apt-get install -y apt-transport-https dotnet-sdk-6.0
-            fi
-            continue
-        fi
-        apt install -y "$pkg"
-    done
-}
-
-deps=$2"[*]"
-"$1" "${!deps}"