Fix memory tracking issues (#150)

* Fix parsing of types for mgp.List

.arcconfig

@@ -1,7 +0,0 @@
-{
-  "project_id" : "memgraph",
-  "conduit_uri" : "https://phabricator.memgraph.io",
-  "phabricator_uri" : "https://phabricator.memgraph.io",
-  "git.default-relative-commit": "origin/master",
-  "arc.land.onto.default": "master"
-}

.arclint

@@ -1,16 +0,0 @@
-{
-  "linters": {
-    "clang-tidy": {
-      "type": "script-and-regex",
-      "include": "(\\.(cpp|cc|cxx|c|h|hpp|lcp)$)",
-      "script-and-regex.script": "./tools/arc-clang-tidy",
-      "script-and-regex.regex": "/^(?P<file>.*):(?P<line>\\d+):(?P<char>\\d+): (?P<severity>warning|error): (?P<message>.*)$/m"
-    },
-    "clang-format": {
-      "type": "script-and-regex",
-      "include": "(\\.(cpp|cc|cxx|c|h|hpp)$)",
-      "script-and-regex.script": "./tools/arc-clang-format",
-      "script-and-regex.regex": "/^(?P<severity>warning):(?P<offset>\\d+):(?P<message>.*)$/m"
-    }
-  }
-}

.clang-format

@@ -1,9 +1,10 @@
 ---
 Language:     Cpp
 BasedOnStyle: Google
-Standard:     "C++11"
+Standard:     "c++20"
 UseTab:       Never
 DerivePointerAlignment: false
 PointerAlignment: Right
-ColumnLimit : 80
+ColumnLimit : 120
+IncludeBlocks: Preserve
 ...

.clang-tidy

@@ -36,6 +36,9 @@ Checks: '*,
          -hicpp-use-equals-default,
          -hicpp-vararg,
          -llvm-header-guard,
+         -llvmlibc-callee-namespace,
+         -llvmlibc-implementation-in-namespace,
+         -llvmlibc-restrict-system-libc-headers,
          -misc-non-private-member-variables-in-classes,
          -misc-unused-parameters,
          -modernize-avoid-c-arrays,
@@ -51,7 +54,7 @@ Checks: '*,
          -readability-magic-numbers,
          -readability-named-parameter'
 WarningsAsErrors: ''
-HeaderFilterRegex: ''
+HeaderFilterRegex: 'src/.*'
 AnalyzeTemporaryDtors: false
 FormatStyle:     none
 CheckOptions:

.githooks/pre-commit

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+project_folder=$(git rev-parse --show-toplevel)
+if git rev-parse --verify HEAD >/dev/null 2>&1
+then
+	against=HEAD
+else
+	# Initial commit: diff against an empty tree object
+	against=$(git hash-object -t tree /dev/null)
+fi
+
+# Redirect output to stderr.
+exec 1>&2
+
+tmpdir=$(mktemp -d repo-XXXXXXXX)
+trap "rm -rf $tmpdir" EXIT INT
+
+modified_files=$(git diff --cached --name-only --diff-filter=AM $against | sed -nE "/.*\.(cpp|cc|cxx|c|h|hpp)$/p")
+for file in $modified_files; do
+  echo "Checking $file..."
+
+  cp $project_folder/.clang-format $project_folder/.clang-tidy $tmpdir
+
+  git checkout-index --prefix="$tmpdir/" -- $file
+
+  echo "Running clang-format..."
+  $project_folder/tools/git-clang-format $tmpdir/$file
+  code=$?
+
+  if [ $code -ne 0 ]; then
+    break
+  fi
+done;
+
+return $code

.github/workflows/diff.yaml

@@ -0,0 +1,403 @@
+name: Diff
+
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+      - '.clang-*'
+
+jobs:
+  community_build:
+    name: "Community build"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build community binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build community binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+          make -j$THREADS
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure -j$THREADS
+
+      - name: Run stress test
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Create community DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create community DEB package.
+          cd build
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save community DEB package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Community DEB package"
+          path: build/output/memgraph*.deb
+
+  code_analysis:
+    name: "Code analysis"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build combined ASAN, UBSAN and coverage binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          cd build
+          cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
+          make -j$THREADS memgraph__unit
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
+          cd build
+          LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
+
+      - name: Compute code coverage
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Compute code coverage.
+          cd tools/github
+          ./coverage_convert
+
+          # Package code coverage.
+          cd generated
+          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+      - name: Save code coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/generated/code_coverage.tar.gz
+
+      - name: Run clang-tidy
+        run: |
+          source /opt/toolchain-v2/activate
+
+          # Restrict clang-tidy results only to the modified parts
+          git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
+
+          # Fail if any warning is reported
+          ! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
+
+  debug_build:
+    name: "Debug build"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run leftover CTest tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run leftover CTest tests (all except unit and benchmark tests).
+          cd build
+          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+      - name: Run drivers tests
+        run: |
+          ./tests/drivers/run.sh
+
+      - name: Run integration tests
+        run: |
+          cd tests/integration
+          for name in *; do
+            if [ ! -d $name ]; then continue; fi
+            pushd $name >/dev/null
+            echo "Running: $name"
+            if [ -x prepare.sh ]; then
+              ./prepare.sh
+            fi
+            if [ -x runner.py ]; then
+              ./runner.py
+            elif [ -x runner.sh ]; then
+              ./runner.sh
+            fi
+            echo
+            popd >/dev/null
+          done
+
+      - name: Run cppcheck and clang-format
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run cppcheck and clang-format.
+          cd tools/github
+          ./cppcheck_and_clang_format diff
+
+      - name: Save cppcheck and clang-format errors
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/cppcheck_and_clang_format.txt
+
+  release_build:
+    name: "Release build"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build release binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Run GQL Behave tests
+        run: |
+          cd tests/gql_behave
+          ./continuous_integration
+
+      - name: Save quality assurance status
+        uses: actions/upload-artifact@v2
+        with:
+          name: "GQL Behave Status"
+          path: |
+            tests/gql_behave/gql_behave_status.csv
+            tests/gql_behave/gql_behave_status.html
+
+      - name: Run e2e replication tests
+        run: |
+          # TODO(gitbuda): Setup mgclient and pymgclient properly.
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
+
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (SSL)
+        run: |
+          cd tests/stress
+          ./continuous_integration --use-ssl
+
+      - name: Run durability test
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 5
+
+      - name: Create enterprise DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create enterprise DEB package.
+          cd build
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save enterprise DEB package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Enterprise DEB package"
+          path: build/output/memgraph*.deb
+
+      - name: Save test data
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: "Test data"
+          path: |
+            # multiple paths could be defined
+            build/logs
+
+  release_jepsen_test:
+    name: "Release Jepsen Test"
+    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
+    #continue-on-error: true
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binarie.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS memgraph
+
+      - name: Run Jepsen tests
+        run: |
+          cd tests/jepsen
+          ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
+
+      - name: Save Jepsen report
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: "Jepsen Report"
+          path: tests/jepsen/Jepsen.tar.gz
+
+  release_benchmarks:
+    name: "Release benchmarks"
+    runs-on: [self-hosted, Linux, X64, Diff, Gen7]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binarie.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Run macro benchmarks
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Upload macro benchmark results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "macro_benchmark" \
+                    --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}"
+
+      - name: Run mgbench
+        run: |
+          cd tests/mgbench
+          ./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
+
+      - name: Upload mgbench results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "mgbench" \
+                    --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}"

.github/workflows/full_clang_tidy.yaml

@@ -0,0 +1,44 @@
+name: Run clang-tidy on the full codebase
+
+on:
+  workflow_dispatch:
+
+jobs:
+  clang_tidy_check:
+    name: "Clang-tidy check"
+    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run clang-tidy
+        run: |
+          source /opt/toolchain-v2/activate
+
+          # The results are also written to standard output in order to retain them in the logs
+          ./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
+            tee ./build/full_clang_tidy_output.txt
+
+      - name: Summarize clang-tidy results
+        run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

.github/workflows/package_all.yaml

@@ -0,0 +1,248 @@
+name: Package All
+
+# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
+
+on: workflow_dispatch
+
+jobs:
+  centos-7_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community centos-7
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-7_community
+          path: build/output/centos-7/memgraph*.rpm
+
+  centos-8_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community centos-8
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-8_community
+          path: build/output/centos-8/memgraph*.rpm
+
+  debian-9_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community debian-9
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-9_community
+          path: build/output/debian-9/memgraph*.deb
+
+  debian-10_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community debian-10
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-10_community
+          path: build/output/debian-10/memgraph*.deb
+
+  docker_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          cd release/package
+          ./run.sh package community debian-10 --for-docker
+          ./run.sh docker
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: docker_community
+          path: build/output/docker/memgraph*.tar.gz
+
+  ubuntu-1804_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community ubuntu-18.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-1804_community
+          path: build/output/ubuntu-18.04/memgraph*.deb
+
+  ubuntu-2004_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community ubuntu-20.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-2004_community
+          path: build/output/ubuntu-20.04/memgraph*.deb
+
+  centos-7_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise centos-7
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-7_enterprise
+          path: build/output/centos-7/memgraph*.rpm
+
+  centos-8_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise centos-8
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-8_enterprise
+          path: build/output/centos-8/memgraph*.rpm
+
+  debian-9_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise debian-9
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-9_enterprise
+          path: build/output/debian-9/memgraph*.deb
+
+  debian-10_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise debian-10
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-10_enterprise
+          path: build/output/debian-10/memgraph*.deb
+
+  docker_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          cd release/package
+          ./run.sh package enterprise debian-10 --for-docker
+          ./run.sh docker
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: docker_enterprise
+          path: build/output/docker/memgraph*.tar.gz
+
+  ubuntu-1804_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise ubuntu-18.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-1804_enterprise
+          path: build/output/ubuntu-18.04/memgraph*.deb
+
+  ubuntu-2004_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise ubuntu-20.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-2004_enterprise
+          path: build/output/ubuntu-20.04/memgraph*.deb

.github/workflows/release_centos8.yaml

@@ -0,0 +1,328 @@
+name: Release CentOS 8
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  community_build:
+    name: "Community build"
+    runs-on: [self-hosted, Linux, X64, CentOS8]
+    env:
+      THREADS: 24
+    timeout-minutes: 960
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build community binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build community binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+          make -j$THREADS
+
+      - name: Create community RPM package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create community RPM package.
+          cd build
+          mkdir output && cd output
+          cpack -G RPM --config ../CPackConfig.cmake
+          rpmlint memgraph*.rpm
+
+      - name: Save community RPM package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Community RPM package"
+          path: build/output/memgraph*.rpm
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (large)
+        run: |
+          cd tests/stress
+          ./continuous_integration --large-dataset
+
+  coverage_build:
+    name: "Coverage build"
+    runs-on: [self-hosted, Linux, X64, CentOS8]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build coverage binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build coverage binaries.
+          cd build
+          cmake -DTEST_COVERAGE=ON ..
+          make -j$THREADS memgraph__unit
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Compute code coverage
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Compute code coverage.
+          cd tools/github
+          ./coverage_convert
+
+          # Package code coverage.
+          cd generated
+          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+      - name: Save code coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/generated/code_coverage.tar.gz
+
+  debug_build:
+    name: "Debug build"
+    runs-on: [self-hosted, Linux, X64, CentOS8]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run leftover CTest tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run leftover CTest tests (all except unit and benchmark tests).
+          cd build
+          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+      - name: Run drivers tests
+        run: |
+          ./tests/drivers/run.sh
+
+      - name: Run integration tests
+        run: |
+          cd tests/integration
+          for name in *; do
+            if [ ! -d $name ]; then continue; fi
+            pushd $name >/dev/null
+            echo "Running: $name"
+            if [ -x prepare.sh ]; then
+              ./prepare.sh
+            fi
+            if [ -x runner.py ]; then
+              ./runner.py
+            elif [ -x runner.sh ]; then
+              ./runner.sh
+            fi
+            echo
+            popd >/dev/null
+          done
+
+      - name: Run cppcheck and clang-format
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run cppcheck and clang-format.
+          cd tools/github
+          ./cppcheck_and_clang_format diff
+
+      - name: Save cppcheck and clang-format errors
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/cppcheck_and_clang_format.txt
+
+  release_build:
+    name: "Release build"
+    runs-on: [self-hosted, Linux, X64, CentOS8]
+    env:
+      THREADS: 24
+    timeout-minutes: 960
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build release binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Create enterprise RPM package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create enterprise RPM package.
+          cd build
+          mkdir output && cd output
+          cpack -G RPM --config ../CPackConfig.cmake
+          rpmlint memgraph*.rpm
+
+      - name: Save enterprise RPM package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Enterprise RPM package"
+          path: build/output/memgraph*.rpm
+
+      - name: Run micro benchmark tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run micro benchmark tests.
+          cd build
+          # The `eval` benchmark needs a large stack limit.
+          ulimit -s 262144
+          ctest -R memgraph__benchmark -V
+
+      - name: Run macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Run parallel macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QueryParallelSuite MemgraphRunner \
+            --groups aggregation_parallel create_parallel bfs_parallel \
+            --num-database-workers 9 --num-clients-workers 30 \
+            --no-strict
+
+      - name: Run GQL Behave tests
+        run: |
+          cd tests/gql_behave
+          ./continuous_integration
+
+      - name: Save quality assurance status
+        uses: actions/upload-artifact@v2
+        with:
+          name: "GQL Behave Status"
+          path: |
+            tests/gql_behave/gql_behave_status.csv
+            tests/gql_behave/gql_behave_status.html
+
+      - name: Run e2e replication tests
+        run: |
+          # TODO(gitbuda): Setup mgclient and pymgclient properly.
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
+
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (SSL)
+        run: |
+          cd tests/stress
+          ./continuous_integration --use-ssl
+
+      - name: Run stress test (large)
+        run: |
+          cd tests/stress
+          ./continuous_integration --large-dataset
+
+      - name: Run durability test (plain)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 5
+
+      - name: Run durability test (large)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 20

.github/workflows/release_debian10.yaml

@@ -0,0 +1,366 @@
+name: Release Debian 10
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  community_build:
+    name: "Community build"
+    runs-on: [self-hosted, Linux, X64, Debian10]
+    env:
+      THREADS: 24
+    timeout-minutes: 960
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build community binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build community binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+          make -j$THREADS
+
+      - name: Create community DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create community DEB package.
+          cd build
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save community DEB package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Community DEB package"
+          path: build/output/memgraph*.deb
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (large)
+        run: |
+          cd tests/stress
+          ./continuous_integration --large-dataset
+
+  coverage_build:
+    name: "Coverage build"
+    runs-on: [self-hosted, Linux, X64, Debian10]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build coverage binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build coverage binaries.
+          cd build
+          cmake -DTEST_COVERAGE=ON ..
+          make -j$THREADS memgraph__unit
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Compute code coverage
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Compute code coverage.
+          cd tools/github
+          ./coverage_convert
+
+          # Package code coverage.
+          cd generated
+          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+      - name: Save code coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/generated/code_coverage.tar.gz
+
+  debug_build:
+    name: "Debug build"
+    runs-on: [self-hosted, Linux, X64, Debian10]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run leftover CTest tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run leftover CTest tests (all except unit and benchmark tests).
+          cd build
+          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+      - name: Run drivers tests
+        run: |
+          ./tests/drivers/run.sh
+
+      - name: Run integration tests
+        run: |
+          cd tests/integration
+          for name in *; do
+            if [ ! -d $name ]; then continue; fi
+            pushd $name >/dev/null
+            echo "Running: $name"
+            if [ -x prepare.sh ]; then
+              ./prepare.sh
+            fi
+            if [ -x runner.py ]; then
+              ./runner.py
+            elif [ -x runner.sh ]; then
+              ./runner.sh
+            fi
+            echo
+            popd >/dev/null
+          done
+
+      - name: Run cppcheck and clang-format
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run cppcheck and clang-format.
+          cd tools/github
+          ./cppcheck_and_clang_format diff
+
+      - name: Save cppcheck and clang-format errors
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/cppcheck_and_clang_format.txt
+
+  release_build:
+    name: "Release build"
+    runs-on: [self-hosted, Linux, X64, Debian10]
+    env:
+      THREADS: 24
+    timeout-minutes: 960
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build release binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Create enterprise DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create enterprise DEB package.
+          cd build
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save enterprise DEB package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Enterprise DEB package"
+          path: build/output/memgraph*.deb
+
+      - name: Run micro benchmark tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run micro benchmark tests.
+          cd build
+          # The `eval` benchmark needs a large stack limit.
+          ulimit -s 262144
+          ctest -R memgraph__benchmark -V
+
+      - name: Run macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Run parallel macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QueryParallelSuite MemgraphRunner \
+            --groups aggregation_parallel create_parallel bfs_parallel \
+            --num-database-workers 9 --num-clients-workers 30 \
+            --no-strict
+
+      - name: Run GQL Behave tests
+        run: |
+          cd tests/gql_behave
+          ./continuous_integration
+
+      - name: Save quality assurance status
+        uses: actions/upload-artifact@v2
+        with:
+          name: "GQL Behave Status"
+          path: |
+            tests/gql_behave/gql_behave_status.csv
+            tests/gql_behave/gql_behave_status.html
+
+      - name: Run e2e replication tests
+        run: |
+          # TODO(gitbuda): Setup mgclient and pymgclient properly.
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
+
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (SSL)
+        run: |
+          cd tests/stress
+          ./continuous_integration --use-ssl
+
+      - name: Run stress test (large)
+        run: |
+          cd tests/stress
+          ./continuous_integration --large-dataset
+
+      - name: Run durability test (plain)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 5
+
+      - name: Run durability test (large)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 20
+
+  release_jepsen_test:
+    name: "Release Jepsen Test"
+    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
+    env:
+      THREADS: 24
+    timeout-minutes: 60
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binary.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS memgraph
+
+      - name: Run Jepsen tests
+        run: |
+          cd tests/jepsen
+          ./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
+
+      - name: Save Jepsen report
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: "Jepsen Report"
+          path: tests/jepsen/Jepsen.tar.gz

.github/workflows/release_ubuntu2004.yaml

@@ -0,0 +1,326 @@
+name: Release Ubuntu 20.04
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 1 * * *"
+
+jobs:
+  community_build:
+    name: "Community build"
+    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+    env:
+      THREADS: 24
+    timeout-minutes: 960
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build community binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build community binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
+          make -j$THREADS
+
+      - name: Create community DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create community DEB package.
+          cd build
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save community DEB package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Community DEB package"
+          path: build/output/memgraph*.deb
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (large)
+        run: |
+          cd tests/stress
+          ./continuous_integration --large-dataset
+
+  coverage_build:
+    name: "Coverage build"
+    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build coverage binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build coverage binaries.
+          cd build
+          cmake -DTEST_COVERAGE=ON ..
+          make -j$THREADS memgraph__unit
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Compute code coverage
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Compute code coverage.
+          cd tools/github
+          ./coverage_convert
+
+          # Package code coverage.
+          cd generated
+          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+      - name: Save code coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/generated/code_coverage.tar.gz
+
+  debug_build:
+    name: "Debug build"
+    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run leftover CTest tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run leftover CTest tests (all except unit and benchmark tests).
+          cd build
+          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+      - name: Run drivers tests
+        run: |
+          ./tests/drivers/run.sh
+
+      - name: Run integration tests
+        run: |
+          cd tests/integration
+          for name in *; do
+            if [ ! -d $name ]; then continue; fi
+            pushd $name >/dev/null
+            echo "Running: $name"
+            if [ -x prepare.sh ]; then
+              ./prepare.sh
+            fi
+            if [ -x runner.py ]; then
+              ./runner.py
+            elif [ -x runner.sh ]; then
+              ./runner.sh
+            fi
+            echo
+            popd >/dev/null
+          done
+
+      - name: Run cppcheck and clang-format
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run cppcheck and clang-format.
+          cd tools/github
+          ./cppcheck_and_clang_format diff
+
+      - name: Save cppcheck and clang-format errors
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Code coverage"
+          path: tools/github/cppcheck_and_clang_format.txt
+
+  release_build:
+    name: "Release build"
+    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+    env:
+      THREADS: 24
+    timeout-minutes: 960
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build release binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Create enterprise DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Create enterprise DEB package.
+          cd build
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save enterprise DEB package
+        uses: actions/upload-artifact@v2
+        with:
+          name: "Enterprise DEB package"
+          path: build/output/memgraph*.deb
+
+      - name: Run micro benchmark tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Run micro benchmark tests.
+          cd build
+          # The `eval` benchmark needs a large stack limit.
+          ulimit -s 262144
+          ctest -R memgraph__benchmark -V
+
+      - name: Run macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Run parallel macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QueryParallelSuite MemgraphRunner \
+            --groups aggregation_parallel create_parallel bfs_parallel \
+            --num-database-workers 9 --num-clients-workers 30 \
+            --no-strict
+
+      - name: Run GQL Behave tests
+        run: |
+          cd tests/gql_behave
+          ./continuous_integration
+
+      - name: Save quality assurance status
+        uses: actions/upload-artifact@v2
+        with:
+          name: "GQL Behave Status"
+          path: |
+            tests/gql_behave/gql_behave_status.csv
+            tests/gql_behave/gql_behave_status.html
+
+      - name: Run e2e replication tests
+        run: |
+          # TODO(gitbuda): Setup mgclient and pymgclient properly.
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
+
+      - name: Run e2e memory control tests
+        run: |
+          cd tests
+          ./setup.sh
+          source ve3/bin/activate
+          cd e2e
+          LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (SSL)
+        run: |
+          cd tests/stress
+          ./continuous_integration --use-ssl
+
+      - name: Run stress test (large)
+        run: |
+          cd tests/stress
+          ./continuous_integration --large-dataset
+
+      - name: Run durability test (plain)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 5
+
+      - name: Run durability test (large)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 20

.ycm_extra_conf.py

@@ -1,172 +0,0 @@
-import os
-import os.path
-import fnmatch
-import logging
-import ycm_core
-
-BASE_FLAGS = [
-    '-Wall',
-    '-Wextra',
-    '-Werror',
-    '-Wno-long-long',
-    '-Wno-variadic-macros',
-    '-fexceptions',
-    '-ferror-limit=10000',
-    '-std=c++1z',
-    '-xc++',
-    '-I/usr/lib/',
-    '-I/usr/include/',
-    '-I./src',
-    '-I./include',
-    '-I./libs/fmt',
-    '-I./libs/yaml-cpp',
-    '-I./libs/glog/include',
-    '-I./libs/googletest/googletest/include',
-    '-I./libs/googletest/googlemock/include',
-    '-I./libs/benchmark/include',
-    '-I./libs/cereal/include',
-    # We include cppitertools headers directly from libs directory.
-    '-I./libs',
-    '-I./libs/rapidcheck/include',
-    '-I./libs/antlr4/runtime/Cpp/runtime/src',
-    '-I./libs/gflags/include',
-    '-I./experimental/distributed/src',
-    '-I./libs/postgresql/include',
-    '-I./libs/bzip2',
-    '-I./libs/zlib',
-    '-I./libs/rocksdb/include',
-    '-I./libs/librdkafka/include/librdkafka',
-    '-I./build/include'
-]
-
-SOURCE_EXTENSIONS = [
-    '.cpp',
-    '.cxx',
-    '.cc',
-    '.c',
-    '.m',
-    '.mm'
-]
-
-HEADER_EXTENSIONS = [
-    '.h',
-    '.hxx',
-    '.hpp',
-    '.hh'
-]
-
-# set the working directory of YCMD to be this file
-os.chdir(os.path.dirname(os.path.realpath(__file__)))
-
-def IsHeaderFile(filename):
-    extension = os.path.splitext(filename)[1]
-    return extension in HEADER_EXTENSIONS
-
-def GetCompilationInfoForFile(database, filename):
-    if IsHeaderFile(filename):
-        basename = os.path.splitext(filename)[0]
-        for extension in SOURCE_EXTENSIONS:
-            replacement_file = basename + extension
-            if os.path.exists(replacement_file):
-                compilation_info = database.GetCompilationInfoForFile(replacement_file)
-                if compilation_info.compiler_flags_:
-                    return compilation_info
-        return None
-    return database.GetCompilationInfoForFile(filename)
-
-def FindNearest(path, target):
-    candidate = os.path.join(path, target)
-    if(os.path.isfile(candidate) or os.path.isdir(candidate)):
-        logging.info("Found nearest " + target + " at " + candidate)
-        return candidate;
-    else:
-        parent = os.path.dirname(os.path.abspath(path));
-        if(parent == path):
-            raise RuntimeError("Could not find " + target);
-        return FindNearest(parent, target)
-
-def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
-    if not working_directory:
-        return list(flags)
-    new_flags = []
-    make_next_absolute = False
-    path_flags = [ '-isystem', '-I', '-iquote', '--sysroot=' ]
-    for flag in flags:
-        new_flag = flag
-
-        if make_next_absolute:
-            make_next_absolute = False
-            if not flag.startswith('/'):
-                new_flag = os.path.join(working_directory, flag)
-
-        for path_flag in path_flags:
-            if flag == path_flag:
-                make_next_absolute = True
-                break
-
-            if flag.startswith(path_flag):
-                path = flag[ len(path_flag): ]
-                new_flag = path_flag + os.path.join(working_directory, path)
-                break
-
-        if new_flag:
-            new_flags.append(new_flag)
-    return new_flags
-
-
-def FlagsForClangComplete(root):
-    try:
-        clang_complete_path = FindNearest(root, '.clang_complete')
-        clang_complete_flags = open(clang_complete_path, 'r').read().splitlines()
-        return clang_complete_flags
-    except:
-        return None
-
-def FlagsForInclude(root):
-    try:
-        include_path = FindNearest(root, 'include')
-        flags = []
-        for dirroot, dirnames, filenames in os.walk(include_path):
-            for dir_path in dirnames:
-                real_path = os.path.join(dirroot, dir_path)
-                flags = flags + ["-I" + real_path]
-        return flags
-    except:
-        return None
-
-def FlagsForCompilationDatabase(root, filename):
-    try:
-        compilation_db_path = FindNearest(root, 'compile_commands.json')
-        compilation_db_dir = os.path.dirname(compilation_db_path)
-        logging.info("Set compilation database directory to " + compilation_db_dir)
-        compilation_db =  ycm_core.CompilationDatabase(compilation_db_dir)
-        if not compilation_db:
-            logging.info("Compilation database file found but unable to load")
-            return None
-        compilation_info = GetCompilationInfoForFile(compilation_db, filename)
-        if not compilation_info:
-            logging.info("No compilation info for " + filename + " in compilation database")
-            return None
-        return MakeRelativePathsInFlagsAbsolute(
-            compilation_info.compiler_flags_,
-            compilation_info.compiler_working_dir_)
-    except:
-        return None
-
-def FlagsForFile(filename):
-    root = os.path.realpath(filename);
-    compilation_db_flags = FlagsForCompilationDatabase(root, filename)
-    if compilation_db_flags:
-        final_flags = compilation_db_flags
-    else:
-        final_flags = BASE_FLAGS
-        clang_flags = FlagsForClangComplete(root)
-        if clang_flags:
-            final_flags = final_flags + clang_flags
-        include_flags = FlagsForInclude(root)
-        if include_flags:
-            final_flags = final_flags + include_flags
-    return {
-        'flags': final_flags,
-        'do_cache': True
-    }

CHANGELOG.md

@@ -1,5 +1,150 @@
 # Change Log
 
+## Future
+
+### Bug Fixes
+
+* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
+  For example, parsing of `mgp.List[mgp.Map]` works now.
+* Fixed memory tracking issues. Some of the allocation and deallocation weren't
+  tracked during the query execution.
+* Fixed reading CSV files that are using CRLF as the newline symbol.
+
+## v1.4.0
+
+### Breaking Changes
+
+* Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.
+  The functionality is still the same.
+
+### Major Feature and Improvements
+
+* Added replication to community version.
+* Added support for multiple query modules directories at the same time.
+  You can now define multiple, comma-separated paths to directories from
+  which the modules will be loaded using the `--query-modules-directory` flag.
+* Added support for programatically reading in data from CSV files through the
+  `LOAD CSV` clause. We support CSV files with and without a header, the
+  supported dialect being Excel.
+* Added a new flag `--memory-limit` which enables the user to set the maximum total amount of memory
+  memgraph can allocate during its runtime.
+* Added `FREE MEMORY` query which tries to free unusued memory chunks in different parts of storage.
+* Added the memory limit and amount of currently allocated bytes in the result of `SHOW STORAGE INFO` query.
+* Added `QUERY MEMORY LIMIT num (KB|MB)` to Cypher queries which allows you to limit memory allocation for
+  the entire query. It can be added only at the end of the entire Cypher query.
+* Added logs for the different parts of the recovery process. `INFO`, `DEBUG` and `TRACE` level all contain
+  additional information that is printed out while the recovery is in progress.
+
+### Bug Fixes
+
+* Fixed garbage collector by correctly marking the oldest current timestamp
+  after the database was recovered using the durability files.
+* Fixed reloading of the modules with changed result names.
+* Fixed profile query to show the correct name of the ScanAll operator variant.
+
+## v1.3.0
+
+### Breaking Changes
+
+* Added extra information in durability files to support replication, making it
+  incompatible with the durability files generated by older versions of
+  Memgraph. Even though the replication is an Enterprise feature, the files are
+  compatible with the Community version.
+
+### Major Features and Improvements
+
+* Added support for data replication across a cluster of Memgraph instances.
+  Supported instance types are MAIN and REPLICA. Supported replication modes
+  are SYNC (all SYNC REPLICAS have to receive data before the MAIN can commit
+  the transaction), ASYNC (MAIN doesn't care if data is replicated), SYNC WITH
+  TIMEOUT (MAIN will wait for REPLICAS within the given timeout period, after
+  timout, replication isn't aborted but the replication demotes the REPLICA to
+  the ASYNC mode).
+* Added support for query type deduction. Possible query types are `r` (read),
+  `w` (write), `rw` (read-write). The query type is returned as a part of the
+  summary.
+* Improved logging capabilities by introducing granular logging levels. Added
+  new flag, `--log-level`, which specifies the minimum log level that will be
+  printed. E.g., it's possible to print incoming queries or Bolt server states.
+* Added ability to lock the storage data directory by executing the `LOCK DATA
+  DIRECTORY;` query which delays the deletion of the files contained in the
+  data directory. The data directory can be unlocked again by executing the
+  `UNLOCK DATA DIRECTORY;` query.
+
+### Bug Fixes and Other Changes
+
+* Added cleanup of query executions if not in an explicit transaction.
+* Fix RPC dangling reference.
+
+## v1.2.0
+
+### Breaking Changes
+
+* SSL is disabled by default (`--bolt-cert-file` and `--bolt-key-file` are
+  empty). This change might only affect the client connection configuration.
+
+### Major Features and Improvements
+
+* Added support for Bolt v4.0 and v4.1.
+* Added `mgp_networkx.py` as an alternative implementation of NetworkX graph
+  objects, which is useful to use Memgraph data from NetworkX algorithms
+  optimally.
+* Added `nxalg.py` query module as a proxy to NetworkX algorithms.
+* Added plan optimization to use a label-property index where the property is
+  not null. As a result, the query engine, instead of scanning all elements and
+  applying the filter, performs a label-property index lookup when possible.
+
+### Bug Fixes and Other Changes
+
+* Fixed Cypher `ID` function `Null` handling. When the `ID` function receives
+  `Null`, it will also return `Null`.
+* Fixed bug that caused random crashes in SSL communication on platforms
+  that use older versions of OpenSSL (< 1.1) by adding proper multi-threading
+  handling.
+* Fix `DISCARD` message handling. The query is now executed before discarding
+  the results.
+
+## v1.1.0
+
+### Major Features and Improvements
+
+* Properties in nodes and edges are now stored encoded and compressed. This
+  change significantly reduces memory usage. Depending on the specific dataset,
+  total memory usage can be reduced up to 50%.
+* Added support for rescanning query modules. Previously, the query modules
+  directory was scanned only upon startup. Now it is scanned each time the user
+  requests to load a query module. The functions used to load the query modules
+  were renamed to `mg.load()` and `mg.load_all()` (from `mg.reload()` and
+  `mg.reload_all()`).
+* Improved execution performance of queries that have an IN list filter by
+  using label+property indices.
+  Example: `MATCH (n:Label) WHERE n.property IN [] ...`
+* Added support for `ANY` and `NONE` openCypher functions. Previously, only
+  `ALL` and `SINGLE` functions were implemented.
+
+### Bug Fixes and Other Changes
+
+* Fixed invalid paths returned by variable expansion when the starting node and
+  destination node used the same symbol.
+  Example: `MATCH path = (n:Person {name: "John"})-[:KNOWS*]->(n) RETURN path`
+* Improved semantics of `ALL` and `SINGLE` functions to be consistent with
+  openCypher when handling lists with `Null`s.
+* `SHOW CONSTRAINT INFO` now returns property names as a list for unique
+  constraints.
+* Escaped label/property/edgetype names in `DUMP DATABASE` to support names
+  with spaces in them.
+* Fixed handling of `DUMP DATABASE` queries in multi-command transactions
+  (`BEGIN`, ..., `COMMIT`).
+* Fixed handling of various query types in explicit transactions. For example,
+  constraints were allowed to be created in multi-command transactions
+  (`BEGIN`, ..., `COMMIT`) but that isn't a transactional operation and as such
+  can't be allowed in multi-command transactions.
+* Fixed integer overflow bugs in `COUNT`, `LIMIT` and `SKIP`.
+* Fixed integer overflow bugs in weighted shortest path expansions.
+* Fixed various other integer overflow bugs in query execution.
+* Added Marvel Comic Universe tutorial.
+* Added FootballTransfers tutorial.
+
 ## v1.0.0
 
 ### Major Features and Improvements

CMakeLists.txt

@@ -35,6 +35,9 @@ else()
   message(FATAL_ERROR "Couldn't find clang and/or clang++!")
 endif()
 
+option(BUILD_FOR_DOCKER "Build Memgraph binary for docker." OFF)
+message(STATUS "BUILD_FOR_DOCKER: ${BUILD_FOR_DOCKER}")
+
 # -----------------------------------------------------------------------------
 
 project(memgraph)
@@ -50,7 +53,7 @@ option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON)
 # Set the current version here to override the automatic version detection. The
 # version must be specified as `X.Y.Z`. Primarily used when building new patch
 # versions.
-set(MEMGRAPH_OVERRIDE_VERSION "1.0.0")
+set(MEMGRAPH_OVERRIDE_VERSION "")
 
 # Custom suffix that this version should have. The suffix can be any arbitrary
 # string. Primarily used when building a version for a specific customer.
@@ -67,28 +70,52 @@ set(get_version_script "${CMAKE_SOURCE_DIR}/release/get_version.py")
 # Get version that should be used in the binary.
 execute_process(
     OUTPUT_VARIABLE MEMGRAPH_VERSION
+    RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
     COMMAND "${get_version_script}" ${get_version_enterprise}
                 "${MEMGRAPH_OVERRIDE_VERSION}"
                 "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
+                "--memgraph-root-dir"
+                "${CMAKE_SOURCE_DIR}"
 )
+if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
+    message(FATAL_ERROR "Unable to get Memgraph version.")
+else()
+    MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
+endif()
 
 # Get version that should be used in the DEB package.
 execute_process(
     OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
+    RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
     COMMAND "${get_version_script}" ${get_version_enterprise}
                 --variant deb
                 "${MEMGRAPH_OVERRIDE_VERSION}"
                 "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
+                "--memgraph-root-dir"
+                "${CMAKE_SOURCE_DIR}"
 )
+if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
+    message(FATAL_ERROR "Unable to get Memgraph DEB version.")
+else()
+    MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
+endif()
 
 # Get version that should be used in the RPM package.
 execute_process(
     OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
+    RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
     COMMAND "${get_version_script}" ${get_version_enterprise}
                 --variant rpm
                 "${MEMGRAPH_OVERRIDE_VERSION}"
                 "${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
+                "--memgraph-root-dir"
+                "${CMAKE_SOURCE_DIR}"
 )
+if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
+    message(FATAL_ERROR "Unable to get Memgraph RPM version.")
+else()
+    MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
+endif()
 
 # We want the above variables to be updated each time something is committed to
 # the repository. That is why we include a dependency on the current git HEAD
@@ -149,20 +176,23 @@ add_custom_target(clean_all
 # is easier debugging of compilation and linker flags.
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
+# c99-designator is disabled because of required mixture of designated and
+# non-designated initializers in Python Query Module code (`py_module.cpp`).
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
     -Werror=switch -Werror=switch-bool -Werror=return-type \
-    -Werror=return-stack-address")
+    -Werror=return-stack-address \
+    -Wno-c99-designator")
 
 # Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
     "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
 
 # Statically link libgcc and libstdc++, the GCC allows this according to:
-# https://gcc.gnu.org/onlinedocs/gcc-8.3.0/libstdc++/manual/manual/license.html
+# https://gcc.gnu.org/onlinedocs/gcc-10.2.0/libstdc++/manual/manual/license.html
 # https://www.gnu.org/licenses/gcc-exception-faq.html
-# Last checked for gcc-8.3 which we are using on the build machines.
+# Last checked for gcc-10.2 which we are using on the build machines.
 # ** If we change versions, recheck this! **
 # ** Static linking is allowed only for executables! **
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
@@ -282,10 +312,13 @@ if (UBSAN)
   # runtime library and c++ standard libraries are present.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
-  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
-  # Make sure llvm-symbolizer binary is in path
+  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
+  # Make sure llvm-symbolizer binary is in path.
+  # To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
 endif()
 
+set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
+
 # Add subprojects
 include_directories(src)
 add_subdirectory(src)

CODEOWNERS

@@ -0,0 +1,4 @@
+/docs/ @gitbuda
+/src/communication/ @antonio2368
+/src/query/ @the-joksim
+/src/storage/ @antonio2368

Doxyfile

@@ -51,7 +51,7 @@ PROJECT_BRIEF          = "The World's Most Powerful Graph Database"
 # pixels and the maximum width should not exceed 200 pixels. Doxygen will copy
 # the logo to the output directory.
 
-PROJECT_LOGO           = Doxylogo.png
+PROJECT_LOGO           = docs/doxygen/memgraph_logo.png
 
 # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path
 # into which the generated documentation will be written. If a relative path is

README.md

@@ -1,24 +1,21 @@
-# memgraph
+# Memgraph
 
 Memgraph is an ACID compliant high performance transactional distributed
 in-memory graph database featuring runtime native query compiling, lock free
 data structures, multi-version concurrency control and asynchronous IO.
 
-## dependencies
+## Development Documentation
 
-Memgraph can be compiled using any modern c++ compiler. It mostly relies on
-the standard template library, however, some things do require external
-libraries.
+Please continue in
+[Notion](https://www.notion.so/memgraph/memgraph-0428591638604c8385550e214ea9f3e6).
 
-Some code contains linux-specific libraries and the build is only supported
-on a 64 bit linux kernel.
+## User Documentation
 
-* linux
-* clang 3.8 (good c++11 support, especially lock free atomics)
-* antlr (compiler frontend)
-* cppitertools
-* fmt format
-* google benchmark
-* google test
-* glog
-* gflags
+Memgraph user documentation is maintained within
+[docs](https://github.com/memgraph/docs) repository. The documentation is also
+available on [GitBook](https://docs.memgraph.com).
+
+## Licences
+
+* [Community](release/LICENSE_COMMUNITY.md)
+* [Enterprise](release/LICENSE_ENTERPRISE.md)

apollo_archives.py

@@ -1,29 +0,0 @@
-#!/usr/bin/env python3
-import json
-import os
-
-SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
-
-
-def find_packages(build_output_dir):
-    ret = []
-    output_dir = os.path.join(SCRIPT_DIR, build_output_dir)
-    if os.path.exists(output_dir):
-        for fname in os.listdir(output_dir):
-            if fname.startswith("memgraph") and fname.endswith(".deb"):
-                path = os.path.join(build_output_dir, fname)
-                ret.append({
-                    "name": "Release " + fname.split("_")[1] +
-                            " (deb package)",
-                    "archive": path,
-                })
-    return ret
-
-
-archives = []
-# Find enterprise package(s).
-archives += find_packages(os.path.join("build_release", "output"))
-# Find community package(s).
-archives += find_packages(os.path.join("build_community", "output"))
-
-print(json.dumps(archives, indent=4, sort_keys=True))

apollo_archives.yaml

@@ -1,17 +0,0 @@
-- name: Binaries
-  archive:
-    - build_debug/memgraph
-    - build_debug/config/memgraph.conf
-    - build_release/memgraph
-    - build_release/config/memgraph.conf
-    - build_release/tools/src/mg_client
-    - build_community/memgraph
-    - build_community/config/memgraph.conf
-  filename: binaries.tar.gz
-
-- name: Doxygen documentation
-  cd: docs/doxygen/html
-  archive:
-    - .
-  filename: documentation.tar.gz
-  host: true

apollo_build.yaml

@@ -1,110 +0,0 @@
-- name: Diff build
-  project: ^mg-master-diff$
-  commands: |
-    # Activate toolchain
-    export PATH=/opt/toolchain-v1/bin:$PATH
-    export LD_LIBRARY_PATH=/opt/toolchain-v1/lib:/opt/toolchain-v1/lib64
-
-    # Copy untouched repository to parent folder.
-    cd ..
-    cp -r memgraph parent
-    cd memgraph
-
-    # Initialize and create documentation.
-    TIMEOUT=1200 ./init
-    doxygen Doxyfile
-
-    # Remove default build directory.
-    rm -r build
-
-    # Build debug binaries.
-    mkdir build_debug
-    cd build_debug
-    cmake ..
-    TIMEOUT=1200 make -j$THREADS
-
-    # Build coverage binaries.
-    cd ..
-    mkdir build_coverage
-    cd build_coverage
-    cmake -DTEST_COVERAGE=ON ..
-    TIMEOUT=1200 make -j$THREADS memgraph__unit
-
-    # Build release binaries.
-    cd ..
-    mkdir build_release
-    cd build_release
-    cmake -DCMAKE_BUILD_TYPE=release ..
-    TIMEOUT=1200 make -j$THREADS
-
-    # Build community binaries.
-    cd ..
-    mkdir build_community
-    cd build_community
-    cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
-    TIMEOUT=1200 make -j$THREADS
-    cd ..
-
-    # Checkout to parent commit and initialize.
-    cd ../parent
-    git checkout HEAD~1
-    TIMEOUT=1200 ./init
-
-    # Build parent release binaries.
-    mkdir build_release
-    cd build_release
-    cmake -DCMAKE_BUILD_TYPE=release ..
-    TIMEOUT=1200 make -j$THREADS memgraph memgraph__macro_benchmark
-
-
-# release build is the default one
-- name: Release build
-  commands: |
-    # Activate toolchain
-    export PATH=/opt/toolchain-v1/bin:$PATH
-    export LD_LIBRARY_PATH=/opt/toolchain-v1/lib:/opt/toolchain-v1/lib64
-
-    # Initialize and create documentation.
-    TIMEOUT=1200 ./init
-    doxygen Doxyfile
-
-    # Remove default build directory.
-    rm -r build
-
-    # Build debug binaries.
-    mkdir build_debug
-    cd build_debug
-    cmake ..
-    TIMEOUT=1200 make -j$THREADS
-
-    # Build coverage binaries.
-    cd ..
-    mkdir build_coverage
-    cd build_coverage
-    cmake -DTEST_COVERAGE=ON ..
-    TIMEOUT=1200 make -j$THREADS memgraph__unit
-
-    # Build release binaries.
-    cd ..
-    mkdir build_release
-    cd build_release
-    cmake -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF ..
-    TIMEOUT=1200 make -j$THREADS
-
-    # Create Debian package.
-    mkdir output
-    cd output
-    cpack -G DEB --config ../CPackConfig.cmake
-    cd ..
-
-    # Build community binaries.
-    cd ..
-    mkdir build_community
-    cd build_community
-    cmake -DCMAKE_BUILD_TYPE=Release -DUSE_READLINE=OFF -DMG_ENTERPRISE=OFF ..
-    TIMEOUT=1200 make -j$THREADS
-
-    # Create Debian package.
-    mkdir output
-    cd output
-    cpack -G DEB --config ../CPackConfig.cmake

config/flags.yaml

@@ -39,12 +39,8 @@ modifications:
     value: "/var/log/memgraph/memgraph.log"
     override: true
 
-  - name: "bolt_cert_file"
-    value: "/etc/memgraph/ssl/cert.pem"
-    override: true
-
-  - name: "bolt_key_file"
-    value: "/etc/memgraph/ssl/key.pem"
+  - name: "log_level"
+    value: "WARNING"
     override: true
 
   - name: "bolt_num_workers"
@@ -87,15 +83,13 @@ modifications:
     value: "/usr/lib/memgraph/auth_module/example.py"
     override: false
 
+  - name: "memory_limit"
+    value: "0"
+    override: true 
+
 undocumented:
   - "flag_file"
-  - "log_file_mode"
-  - "log_link_basename"
-  - "log_prefix"
-  - "max_log_size"
-  - "min_log_level"
+  - "also_log_to_stderr"
   - "help"
   - "help_xml"
-  - "stderr_threshold"
-  - "stop_logging_if_full_disk"
   - "version"

docs/dev/code-review.md

@@ -1,269 +0,0 @@
-# Code Review Guidelines
-
-This chapter describes some of the things you should be on the lookout when
-reviewing someone else's code.
-
-## Exceptions
-
-Although the Google C++ Style Guide forbids exceptions, we do allow them in
-our codebase. As a reviewer you should watch out for the following.
-
-The documentation of throwing functions needs to be in-sync with the
-implementation. This must be enforced recursively. I.e. if a function A now
-throws a new exception, and the function B uses A, then B needs to handle that
-exception or have its documentation updated and so on. Naturally, the same
-applies when an exception is removed.
-
-Transitive callers of the function which throws a new exception must be OK
-with that. This ties into the previous point. You need to check that all users
-of the new exception either handle it correctly or propagate it.
-
-Exceptions should not escape out of class destructors, because that will
-terminate the program. The code should be changed so that such cases are not
-possible.
-
-Exceptions being thrown in class constructors. Although this is well defined
-in C++, it usually implies that a constructor is doing too much work and the
-class construction & initialization should be redesigned. Usual approaches are
-using the (Static) Factory Method pattern or having some sort of an
-initialization method that needs to be called after the construction is done.
-Prefer the Factory Method.
-
-Don't forget that STL functions may also throw!
-
-## Pointers & References
-
-In cases when some code passes a pointer or reference, or if a code stores a
-pointer or reference you should take a careful look at the following.
-
-  * Lifetime of the pointed to value (this includes both ownership and
-    multithreaded access).
-  * In case of a class, check validity of destructor and move/copy
-    constructors.
-  * Is the pointed to value mutated, if not it should be `const` (`const Type
-    *` or `const Type &`).
-
-## Allocators & Memory Resources
-
-With the introduction of polymorphic allocators (C++17 `<memory_resource>` and
-our `utils/memory.hpp`) we get a more convenient type signatures for
-containers so as to keep the outward facing API nice. This convenience comes
-at a cost of less static checks on the type level due to type erasure.
-
-For example:
-
-    std::pmr::vector<int> first_vec(std::pmr::null_memory_resource());
-    std::pmr::vector<int> second_vec(std::pmr::new_delete_resource());
-
-    second_vec = first_vec  // What happens here?
-
-    // Or with our implementation
-    utils::MonotonicBufferResource monotonic_memory(1024);
-    std::vector<int, utils::Allocator<int>> first_vec(&monotonic_memory);
-    std::vector<int, utils::Allocator<int>> second_vec(utils::NewDeleteResource());
-
-    second_vec = first_vec  // What happens here?
-
-In the above, both `first_vec` and `second_vec` have the same type, but have
-*different* allocators! This can lead to ambiguity when moving or copying
-elements between them.
-
-You need to watch out for the following.
-
-  * Swapping can lead to undefined behaviour if the allocators are not equal.
-  * Is the move construction done with the right allocator.
-  * Is the move assignment done correctly, also it may throw an exception.
-  * Is the copy construction done with the right allocator.
-  * Is the copy assignment done correctly.
-  * Using `auto` makes allocator propagation rules rather ambiguous.
-
-## Classes & Object Oriented Programming
-
-A common mistake is to use classes, inheritance and "OOP" when it's not
-needed. This sections shows examples of encountered cases.
-
-### Classes without (Meaningful) Members
-
-    class MyCoolClass {
-     public:
-      int BeCool(int a, int b) { return a + b; }
-
-      void SaySomethingCool() { std::cout << "Hello!"; }
-    };
-
-The above class has no members (i.e. state) which affect the behaviour of
-methods. This class should need not exist, it can be easily replaced with a
-more modular (and shorter) design -- top level functions.
-
-    int BeCool(int a, int b) { return a + b; }
-
-    void SaySomethingCool() { std::cout << "Hello!"; }
-
-### Classes with a Single Public Method
-
-    clas MyAwesomeClass {
-     public:
-      MyAwesomeClass(int state) : state_(state) {}
-
-      int GetAwesome() { return GetAwesomeImpl() + 1; }
-
-     private:
-      int state_;
-
-      int GetAwesomeImpl() { return state_; }
-    };
-
-The above class has a `state_` and even a private method, but there's only one
-public method -- `GetAwesome`.
-
-You should check "Does the stored state have any meaningful influence on the
-public method?", similarly to the previous point.
-
-In the above case it doesn't, and the class should be replaced with a public
-function in `.hpp` while the private method should become a private function
-in `.cpp` (static or in anonymous namespace).
-
-    // hpp
-    int GetAwesome(int state);
-
-    // cpp
-    namespace {
-      int GetAwesomeImpl(int state) { return state; }
-    }
-    int GetAwesome(int state) { return GetAwesomeImpl(state) + 1; }
-
-A counterexample is when the state is meaningful.
-
-    class Counter {
-     public:
-      Counter(int state) : state_(state) {}
-
-      int Get() { return state_++; }
-
-     private:
-      int state_;
-    };
-
-But even that could be replaced with a closure.
-
-    auto MakeCounter(int state) {
-      return [state]() mutable { return state++; };
-    }
-
-### Private Methods
-
-Instead of private methods, top level functions should be preferred. The
-reasoning is completely explained in "Effective C++" Item 23 by Scott Meyers.
-In our codebase, even improvements to compilation times can be noticed if
-private methods in interface (`.hpp`) files are replaced with top level
-functions in implementation (`.cpp`) files.
-
-### Inheritance
-
-The rule is simple -- if there are no virtual methods (but maybe destructor),
-then the class should be marked as `final` and never inherited.
-
-If there are virtual methods (i.e. class is meant to be inherited), make sure
-that either a public virtual destructor or a protected non-virtual destructor
-exist. See "Effective C++" Item 7 by Scott Meyers. Also take a look at
-"Effective C++" Items 32---39 by Scott Meyers.
-
-An example of how inheritance with no virtual methods is replaced with
-composition.
-
-    class MyBase {
-     public:
-      virtual ~MyBase() {}
-
-      void DoSomethingBase() { ... }
-    };
-
-    class MyDerived final : public MyBase {
-     public:
-      void DoSomethingNew() { ... DoSomethingBase(); ... }
-    };
-
-With composition, the above becomes.
-
-    class MyBase final {
-     public:
-      void DoSomethingBase() { ... }
-    };
-
-    class MyDerived final {
-      MyBase base_;
-
-     public:
-      void DoSomethingNew() { ... base_.DoSomethingBase(); ... }
-    };
-
-The composition approach is preferred as it encapsulates the fact that
-`MyBase` is used for the implementation and users only interact with the
-public interface of `MyDerived`. Additionally, you can easily replace `MyBase
-base_;` with a C++ PIMPL idiom (`std::unique_ptr<MyBase> base_;`) to make the
-code more modular with regards to compilation.
-
-More advanced C++ users will recognize that the encapsulation feature of the
-non-PIMPL composition can be replaced with private inheritance.
-
-    class MyDerived final : private MyBase {
-     public:
-      void DoSomethingNew() { ... MyBase::DoSomethingBase(); ... }
-    };
-
-One of the common "counterexample" is the ability to store objects of
-different type in a container or pass them to a function. Unfortunately, this
-is not that good of a design. For example.
-
-    class MyBase {
-      ... // No virtual methods (but the destructor)
-    };
-
-    class MyFirstClass final : public MyBase { ...  };
-
-    class MySecondClass final : public MyBase { ... };
-
-    std::vector<std::unique_ptr<MyBase>> first_and_second_classes;
-    first_and_second_classes.push_back(std::make_unique<MyFirstClass>());
-    first_and_second_classes.push_back(std::make_unique<MySecondClass>());
-
-    void FunctionOnFirstOrSecond(const MyBase &first_or_second, ...) { ... }
-
-With C++17, the containers for different types should be implemented with
-`std::variant`, and as before the functions can be templated.
-
-    class MyFirstClass final { ... };
-
-    class MySecondClass final { ... };
-
-    std::vector<std::variant<MyFirstClass, MySecondClass>> first_and_second_classes;
-    // Notice no heap allocation, since we don't store a pointer
-    first_and_second_classes.emplace_back(MyFirstClass());
-    first_and_second_classes.emplace_back(MySecondClass());
-
-    // You can also use `std::variant` here instead of template
-    template <class TFirstOrSecond>
-    void FunctionOnFirstOrSecond(const TFirstOrSecond &first_or_second, ...) { ... }
-
-Naturally, if the base class has meaningful virtual methods (i.e. other than
-destructor) it maybe is OK to use inheritance but also consider alternatives.
-See "Effective C++" Items 32---39 by Scott Meyers.
-
-### Multiple Inheritance
-
-Multiple inheritance should not be used unless all base classes are pure
-interface classes. This decision is inherited from [Google C++ Style
-Guide](https://google.github.io/styleguide/cppguide.html#Inheritance). For
-example on how to design with and around multiple inheritance refer to
-"Effective C++" Item 40 by Scott Meyers.
-
-Naturally, if there *really* is no better design, then multiple inheritance is
-allowed. An example of this can be found in our codebase when inheriting
-Visitor classes (though even that could be replaced with `std::variant` for
-example).
-
-## Code Format & Style
-
-If something doesn't conform to our code formatting and style, just refer the
-author to either [C++ Style](cpp-code-conventions.md) or [Other Code
-Conventions](other-code-conventions.md).

docs/dev/cpp-code-conventions.md

@@ -1,263 +0,0 @@
-# C++ Code Conventions
-
-This chapter describes code conventions which should be followed when writing
-C++ code.
-
-## Code Style
-
-Memgraph uses the
-[Google Style Guide for C++](https://google.github.io/styleguide/cppguide.html)
-in most of its code.  You should follow them whenever writing new code.
-Besides following the style guide, take a look at
-[Code Review Guidelines](code-review.md) for common design issues and pitfalls
-with C++ as well as [Required Reading](required-reading.md).
-
-### Often Overlooked Style Conventions
-
-#### Pointers & References
-
-References provide a shorter syntax for accessing members and better declare
-the intent that a pointer *should* not be `nullptr`. They do not prevent
-accessing a `nullptr` and obfuscate the client/calling code because the
-reference argument is passed just like a value. Errors with such code have
-been very difficult to debug. Therefore, pointers are always used. They will
-not prevent bugs but will make some of them more obvious when reading code.
-
-The only time a reference can be used is if it is `const`. Note that this
-kind of reference is not allowed if it is stored somewhere, i.e. in a class.
-You should use a pointer to `const` then. The primary reason being is that
-references obscure the semantics of moving an object, thus making bugs with
-references pointing to invalid memory harder to track down.
-
-[Style guide reference](https://google.github.io/styleguide/cppguide.html#Reference_Arguments)
-
-#### Constructors & RAII
-
-RAII (Resource Acquisition is Initialization) is a nice mechanism for managing
-resources. It is especially useful when exceptions are used, such as in our
-code. Unfortunately, they do have 2 major downsides.
-
-  * Only exceptions can be used for to signal failure.
-  * Calls to virtual methods are not resolved as expected.
-
-For those reasons the style guide recommends minimal work that cannot fail.
-Using virtual methods or doing a lot more should be delegated to some form of
-`Init` method, possibly coupled with static factory methods. Similar rules
-apply to destructors, which are not allowed to even throw exceptions.
-
-[Style guide reference](https://google.github.io/styleguide/cppguide.html#Doing_Work_in_Constructors)
-
-### Additional Style Conventions
-
-Old code may have broken Google C++ Style accidentally, but the new code
-should adhere to it as close as possible. We do have some exceptions
-to Google style as well as additions for unspecified conventions.
-
-#### Using C++ Exceptions
-
-Unlike Google, we do not forbid using exceptions.
-
-But, you should be very careful when using them and introducing new ones. They
-are indeed handy, but cause problems with understanding the control flow since
-exceptions are another form of `goto`. It also becomes very hard to determine
-that the program is in correct state after the stack is unwound and the thrown
-exception handled. Other than those issues, throwing exceptions in destructors
-will terminate the program. The same will happen if a thread doesn't handle an
-exception even though it is not the main thread.
-
-[Style guide reference](https://google.github.io/styleguide/cppguide.html#Exceptions)
-
-In general, when introducing a new exception, either via `throw` statement or
-calling a function which throws, you must examine all transitive callers and
-update their implementation and/or documentation.
-
-#### Assertions
-
-We use `CHECK` and `DCHECK` macros from glog library. You are encouraged to
-use them as often as possible to both document and validate various pre and
-post conditions of a function.
-
-`CHECK` remains even in release build and should be preferred over it's cousin
-`DCHECK` which only exists in debug builds. The primary reason is that you
-want to trigger assertions in release builds in case the tests didn't
-completely validate all code paths. It is better to fail fast and crash the
-program, than to leave it in undefined state and potentially corrupt end
-user's work. In cases when profiling shows that `CHECK` is causing visible
-slowdown you should switch to `DCHECK`.
-
-#### Template Parameter Naming
-
-Template parameter names should start with capital letter 'T' followed by a
-short descriptive name. For example:
-
-```cpp
-template <typename TKey, typename TValue>
-class KeyValueStore
-```
-
-## Code Formatting
-
-You should install `clang-format` and run it on code you change or add. The
-root of Memgraph's project contains the `.clang-format` file, which specifies
-how formatting should behave. Running `clang-format -style=file` in the
-project's root will read the file and behave as expected. For ease of use, you
-should integrate formatting with your favourite editor.
-
-The code formatting isn't enforced, because sometimes manual formatting may
-produce better results. Though, running `clang-format` is strongly encouraged.
-
-## Documentation
-
-Besides following the comment guidelines from [Google Style
-Guide](https://google.github.io/styleguide/cppguide.html#Comments), your
-documentation of the public API should be
-[Doxygen](https://github.com/doxygen/doxygen) compatible. For private parts of
-the code or for comments accompanying the implementation, you are free to
-break doxygen compatibility. In both cases, you should write your
-documentation as full sentences, correctly written in English.
-
-## Doxygen
-
-To start a Doxygen compatible documentation string, you should open your
-comment with either a JavaDoc style block comment (`/**`) or a line comment
-containing 3 slashes (`///`). Take a look at the 2 examples below.
-
-### Block Comment
-
-```cpp
-/**
- * One sentence, brief description.
- *
- * Long form description.
- */
-```
-
-### Line Comment
-
-```cpp
-/// One sentence, brief description.
-///
-/// Long form description.
-```
-
-If you only have a brief description, you may collapse the documentation into
-a single line.
-
-### Block Comment
-
-```cpp
-/** Brief description. */
-```
-
-### Line Comment
-
-```cpp
-/// Brief description.
-```
-
-Whichever style you choose, keep it consistent across the whole file.
-
-Doxygen supports various commands in comments, such as `@file` and `@param`.
-These help Doxygen to render specified things differently or to track them for
-cross referencing. If you want to learn more, take a look at these two links:
-
-  * http://www.stack.nl/~dimitri/doxygen/manual/docblocks.html
-  * http://www.stack.nl/~dimitri/doxygen/manual/commands.html
-
-## Examples
-
-Below are a few examples of documentation from the codebase.
-
-### Function
-
-```cpp
-/**
- * Removes whitespace characters from the start and from the end of a string.
- *
- * @param s String that is going to be trimmed.
- *
- * @return Trimmed string.
- */
-inline std::string Trim(const std::string &s);
-```
-
-### Class
-
-```cpp
-/** Base class for logical operators.
- *
- *  Each operator describes an operation, which is to be performed on the
- *  database. Operators are iterated over using a @c Cursor. Various operators
- *  can serve as inputs to others and thus a sequence of operations is formed.
- */
-class LogicalOperator
-    : public ::utils::Visitable<HierarchicalLogicalOperatorVisitor> {
- public:
-  /** Constructs a @c Cursor which is used to run this operator.
-   *
-   *  @param GraphDbAccessor Used to perform operations on the database.
-   */
-  virtual std::unique_ptr<Cursor> MakeCursor(GraphDbAccessor &db) const = 0;
-
-  /** Return @c Symbol vector where the results will be stored.
-   *
-   *  Currently, outputs symbols are only generated in @c Produce operator.
-   *  @c Skip, @c Limit and @c OrderBy propagate the symbols from @c Produce (if
-   *  it exists as input operator). In the future, we may want this method to
-   *  return the symbols that will be set in this operator.
-   *
-   *  @param SymbolTable used to find symbols for expressions.
-   *  @return std::vector<Symbol> used for results.
-   */
-  virtual std::vector<Symbol> OutputSymbols(const SymbolTable &) const {
-    return std::vector<Symbol>();
-  }
-
-  virtual ~LogicalOperator() {}
-};
-```
-
-### File Header
-
-```cpp
-/// @file visitor.hpp
-///
-/// This file contains the generic implementation of visitor pattern.
-///
-/// There are 2 approaches to the pattern:
-///
-///   * classic visitor pattern using @c Accept and @c Visit methods, and
-///   * hierarchical visitor which also uses @c PreVisit and @c PostVisit
-///   methods.
-///
-/// Classic Visitor
-/// ===============
-///
-/// Explanation on the classic visitor pattern can be found from many
-/// sources, but here is the link to hopefully most easily accessible
-/// information: https://en.wikipedia.org/wiki/Visitor_pattern
-///
-/// The idea behind the generic implementation of classic visitor pattern is to
-/// allow returning any type via @c Accept and @c Visit methods. Traversing the
-/// class hierarchy is relegated to the visitor classes. Therefore, visitor
-/// should call @c Accept on children when visiting their parents. To implement
-/// such a visitor refer to @c Visitor and @c Visitable classes.
-///
-/// Hierarchical Visitor
-/// ====================
-///
-/// Unlike the classic visitor, the intent of this design is to allow the
-/// visited structure itself to control the traversal. This way the internal
-/// children structure of classes can remain private. On the other hand,
-/// visitors may want to differentiate visiting composite types from leaf types.
-/// Composite types are those which contain visitable children, unlike the leaf
-/// nodes. Differentiation is accomplished by providing @c PreVisit and @c
-/// PostVisit methods, which should be called inside @c Accept of composite
-/// types. Regular @c Visit is only called inside @c Accept of leaf types.
-/// To implement such a visitor refer to @c CompositeVisitor, @c LeafVisitor and
-/// @c Visitable classes.
-///
-/// Implementation of hierarchical visiting is modelled after:
-/// http://wiki.c2.com/?HierarchicalVisitorPattern
-```
-

docs/dev/durability/snapshots.md

@@ -1,22 +0,0 @@
-# Snapshots
-
-A "snapshot" is a record of the current database state stored in permanent
-storage. Note that the term "snapshot" is used also in the context of
-the transaction engine to denote a set of running transactions.
-
-A snapshot is written to the file by Memgraph periodically if so
-configured. The snapshot creation  process is done within a transaction created
-specifically for that purpose. The transaction is needed to ensure that
-the stored state is internally consistent.
-
-The database state can be recovered from the snapshot during startup, if
-so configured. This recovery works in conjunction with write-ahead log
-recovery.
-
-A single snapshot contains all the data needed to recover a database. In
-that sense snapshots are independent of each other and old snapshots can
-be deleted once the new ones are safely stored, if it is not necessary
-to revert the database to some older state.
-
-The exact format of the snapshot file is defined inline in the snapshot
-creation code.

docs/dev/durability/wal.md

@@ -1,55 +0,0 @@
-# Write-ahead logging
-
-Typically WAL denotes the process of writing a "log" of database
-operations (state changes) to persistent storage before committing the
-transaction, thus ensuring that the state can be recovered (in the case
-of a crash) for all the transactions which the database committed.
-
-The WAL is a fine-grained durability format. It's purpose is to store
-database changes fast. It's primary purpose is not to provide
-space-efficient storage, nor to support fast recovery. For that reason
-it's often used in combination with a different persistence mechanism
-(in Memgraph's case the "snapshot") that has complementary
-characteristics.
-
-### Guarantees
-
-Ensuring that the log is written before the transaction is committed can
-slow down the database. For that reason this guarantee is most often
-configurable in databases.
-
-Memgraph offers two options for the WAL. The default option, where the WAL is
-flushed to the disk periodically and transactions do not wait for this to
-complete, introduces the risk of database inconsistency because an operating
-system or hardware crash might lead to missing transactions in the WAL. Memgraph
-will handle this as if those transactions never happened. The second option,
-called synchronous commit, will instruct Memgraph to wait for the WAL to be
-flushed to the disk when a transactions completes and the transaction will wait
-for this to complete. This option can be turned on with the
-`--synchronous-commit` command line flag.
-
-### Format
-
-The WAL file contains a series of DB state changes called `StateDelta`s.
-Each of them describes what the state change is and in which transaction
-it happened. Also some kinds of meta-information needed to ensure proper
-state recovery are recorded (transaction beginnings and commits/abort).
-
-The following is guaranteed w.r.t. `StateDelta` ordering in
-a single WAL file:
-- For two ops in the same transaction, if op A happened before B in the
-  database, that ordering is preserved in the log.
-- Transaction begin/commit/abort messages also appear in exactly the
-  same order as they were executed in the transactional engine.
-
-### Recovery
-
-The database can recover from the WAL on startup. This works in
-conjunction with snapshot recovery. The database attempts to recover from
-the latest snapshot and then apply as much as possible from the WAL
-files. Only those transactions that were not recovered from the snapshot
-are recovered from the WAL, for speed efficiency. It is possible (but
-inefficient) to recover the database from WAL only, provided all the WAL
-files created from DB start are available. It is not possible to recover
-partial database state (i.e. from some suffix of WAL files, without the
-preceding snapshot).

docs/dev/other-code-conventions.md

@@ -1,15 +0,0 @@
-# Other Code Conventions
-
-While we are mainly programming in C++, we do use other programming languages
-when appropriate. This chapter describes conventions for such code.
-
-## Python
-
-Code written in Python should adhere to
-[PEP 8](https://www.python.org/dev/peps/pep-0008/). You should run `flake8` on
-your code to automatically check compliance.
-
-## Common Lisp
-
-Code written in Common Lisp should adhere to
-[Google Common Lisp Style](https://google.github.io/styleguide/lispguide.xml).

docs/dev/query/.gitignore

@@ -1 +0,0 @@
-html/

docs/dev/query/build-html

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-mkdir -p $script_dir/html
-
-for markdown_file in $(find $script_dir -name '*.md'); do
-  name=$(basename -s .md $markdown_file)
-  sed -e 's/.md/.html/' $markdown_file | \
-    pandoc -s -f markdown -t html -o $script_dir/html/$name.html
-done
-
-for dot_file in $(find $script_dir -name '*.dot'); do
-  name=$(basename -s .dot $dot_file)
-  dot -Tpng $dot_file -o $script_dir/html/$name.png
-done

docs/dev/query/contents.md

@@ -1,34 +0,0 @@
-# Query Parsing, Planning and Execution
-
-This part of the documentation deals with query execution.
-
-Memgraph currently supports only query interpretation. Each new query is
-parsed, analysed and translated into a sequence of operations which are then
-executed on the main database storage. Query execution is organized into the
-following phases:
-
-  1.  [Lexical Analysis (Tokenization)](parsing.md)
-  2.  [Syntactic Analysis (Parsing)](parsing.md)
-  3.  [Semantic Analysis and Symbol Generation](semantic.md)
-  4.  [Logical Planning](planning.md)
-  5.  [Logical Plan Execution](execution.md)
-
-The main entry point is `Interpreter::operator()`, which takes a query text
-string and produces a `Results` object. To instantiate the object,
-`Interpreter` needs to perform the above steps from 1 to 4. If any of the
-steps fail, a `QueryException` is thrown. The complete `LogicalPlan` is
-wrapped into a `CachedPlan` and stored for reuse. This way we can skip the
-whole process of analysing a query if it appears to be the same as before.
-
-When we have valid plan, the client code can invoke `Results::PullAll` with a
-stream object. The `Results` instance will then execute the plan and fill the
-stream with the obtained results.
-
-Since we want to optionally run Memgraph as a distributed database, we have
-hooks for creating a different plan of logical operators.
-`DistributedInterpreter` inherits from `Interpreter` and overrides
-`MakeLogicalPlan` method. This method needs to return a concrete instance of
-`LogicalPlan`, and in case of distributed database that will be
-`DistributedLogicalPlan`.
-
-![Intepreter Class Diagram](interpreter-class.png)

docs/dev/query/execution.md

@@ -1,373 +0,0 @@
-# Logical Plan Execution
-
-We implement classical iterator style operators. Logical operators define
-operations on database. They encapsulate the following info: what the input is
-(another `LogicalOperator`), what to do with the data, and how to do it.
-
-Currently logical operators can have zero or more input operations, and thus a
-`LogicalOperator` tree is formed. Most `LogicalOperator` types have only one
-input, so we are mostly working with chains instead of full fledged trees.
-You can find information on each operator in `src/query/plan/operator.lcp`.
-
-## Cursor
-
-Logical operators do not perform database work themselves. Instead they create
-`Cursor` objects that do the actual work, based on the info in the operator.
-Cursors expose a `Pull` method that gets called by the cursor's consumer. The
-consumer keeps pulling as long as the `Pull` returns `true` (indicating it
-successfully performed some work and might be eligible for another `Pull`).
-Most cursors will call the `Pull` function of their input provided cursor, so
-typically a cursor chain is created that is analogue to the logical operator
-chain it's created from.
-
-## Frame
-
-The `Frame` object contains all the data of the current `Pull` chain. It
-serves for communicating data between cursors.
-
-For example, in a `MATCH (n) RETURN n` query the `ScanAllCursor` places a
-vertex on the `Frame` for each `Pull`. It places it on the place reserved for
-the `n` symbol. Then the `ProduceCursor` can take that same value from the
-`Frame` because it knows the appropriate symbol. `Frame` positions are indexed
-by `Symbol` objects.
-
-## ExpressionEvaluator
-
-Expressions results are not placed on the `Frame` since they do not need to be
-communicated between different `Cursors`. Instead, expressions are evaluated
-using an instance of `ExpressionEvaluator`. Since generally speaking an
-expression can be defined by a tree of subexpressions, the
-`ExpressionEvaluator` is implemented as a tree visitor. There is a performance
-sub-optimality here because a stack is used to communicate intermediary
-expression results between elements of the tree. This is one of the reasons
-why it's planned to use `Frame` for intermediary expression results as well.
-The other reason is that it might facilitate compilation later on.
-
-## Cypher Execution Semantics
-
-Cypher query execution has *mostly* well-defined semantics. Some are
-explicitly defined by openCypher and its TCK, while others are implicitly
-defined by Neo4j's implementation of Cypher that we want to be generally
-compatible with.
-
-These semantics can in short be described as follows: a Cypher query consists
-of multiple clauses some of which modify it. Generally, every clause in the
-query, when reading it left to right, operates on a consistent state of the
-property graph, untouched by subsequent clauses. This means that a `MATCH`
-clause in the beginning operates on a graph-state in which modifications by
-the subsequent `SET` are not visible.
-
-The stated semantics feel very natural to the end-user, and Neo seems to
-implement them well. For Memgraph the situation is complex because
-`LogicalOperator` execution (through a `Cursor`) happens one `Pull` at a time
-(generally meaning all the query clauses get executed for every top-level
-`Pull`). This is not inherently consistent with Cypher semantics because a
-`SET` clause can modify data, and the `MATCH` clause that precedes it might
-see the modification in a subsequent `Pull`. Also, the `RETURN` clause might
-want to stream results to the user before all `SET` clauses have been
-executed, so the user might see some intermediate graph state. There are many
-edge-cases that Memgraph does its best to avoid to stay true to Cypher
-semantics, while at the same time using a high-performance streaming approach.
-The edge-cases are enumerated in this document along with the implementation
-details they imply.
-
-## Implementation Peculiarities
-
-### Once
-
-An operator that does nothing but whose `Cursor::Pull` returns `true` on the
-first `Pull` and `false` on subsequent ones. This operator is used when
-another operator has an optional input, because in Cypher a clause will
-typically execute once for every input from the preceding clauses, or just
-once if there was no preceding input. For example, consider the `CREATE`
-clause. In the query `CREATE (n)` only one node is created, while in the query
-`MATCH (n) CREATE (m)` a node is created for each existing node. Thus in our
-`CreateNode` logical operator the input is either a `ScanAll` operator, or a
-`Once` operator.
-
-### storage::View
-
-In the previous section, [Cypher Execution
-Semantics](#cypher-execution-semantics), we mentioned how the preceding
-clauses should not see changes made in subsequent ones. For that reason, some
-operators take a `storage::View` enum value. This value determines which state of
-the graph an operator sees.
-
-Consider the query `MATCH (n)--(m) WHERE n.x = 0 SET m.x = 1`. Naive streaming
-could match a vertex `n` on the given criteria, expand to `m`, update it's
-property, and in the next iteration consider the vertex previously matched to
-`m` and skip it because it's newly set property value does not qualify. This
-is not how Cypher works. To handle this issue properly, Memgraph designed the
-`VertexAccessor` class that tracks two versions of data: one that was visible
-before the current transaction+command, and the optional other that was
-created in the current transaction+command. The `MATCH` clause will be planned
-as `ScanAll` and `Expand` operations using `storage::View::OLD` value. This
-will ensure modifications performed in the same query do not affect it. The
-same applies to edges and the `EdgeAccessor` class.
-
-### Existing Record Detection
-
-It's possible that a pattern element has already been declared in the same
-pattern, or a preceding pattern. For example `MATCH (n)--(m), (n)--(l)` or a
-cycle-detection match `MATCH (n)-->(n) RETURN n`. Implementation-wise,
-existing record detection just checks that the expanded record is equal to the
-one already on the frame.
-
-### Why Not Use Separate Expansion Ops for Edges and Vertices?
-
-Expanding an edge and a vertex in separate ops is not feasible when matching a
-cycle in bi-directional expansions. Consider the query `MATCH (n)--(n) RETURN
-n`. Let's try to expand first the edge in one op, and vertex in the next. The
-vertex expansion consumes the edge expansion input. It takes the expanded edge
-from the frame. It needs to detect a cycle by comparing the vertex existing on
-the frame with one of the edge vertices (`from` or `to`). But which one? It
-doesn't know, and can't ensure correct cycle detection.
-
-### Data Visibility During and After SET
-
-In Cypher, setting values always works on the latest version of data (from
-preceding or current clause). That means that within a `SET` clause all the
-changes from previous clauses must be visible, as well as changes done by the
-current `SET` clause. Also, if there is a clause after `SET` it must see *all*
-the changes performed by the preceding `SET`. Both these things are best
-illustrated with the following queries executed on an empty database:
-
-    CREATE (n:A {x:0})-[:EdgeType]->(m:B {x:0})
-    MATCH (n)--(m) SET m.x = n.x + 1 RETURN labels(n), n.x, labels(m), m.x
-
-This returns:
-
-+---------+---+---------+---+
-|labels(n)|n.x|labels(m)|m.x|
-+:=======:+:=:+:=======:+:=:+
-|[A]      |2  |[B]      |1  |
-+---------+---+---------+---+
-|[B]      |1  |[A]      |2  |
-+---------+---+---------+---+
-
-The obtained result implies the following operations:
-
-  1. In the first iteration set the value of the `B.x` to 1
-  2. In the second iteration the we observe `B.x` with the value of 1 and set
-     `A.x` to 2
-  3. In `RETURN` we see all the changes made in both iterations
-
-To implement the desired behavior Memgraph utilizes two techniques. First is
-the already mentioned tracking of two versions of data in vertex accessors.
-Using this approach ensures that the second iteration in the example query
-sees the data modification performed by the preceding iteration. The second
-technique is the `Accumulate` operation that accumulates all the iterations
-from the preceding logical op before passing them to the next logical op. In
-the example query, `Accumulate` ensures that the results returned to the user
-reflect changes performed in all iterations of the query (naive streaming
-could stream results at the end of first iteration producing inconsistent
-results). Note that `Accumulate` is demanding regarding memory and slows down
-query execution. For that reason it should be used only when necessary, for
-example it does not have to be used in a query that has `MATCH` and `SET` but
-no `RETURN`.
-
-### Neo4j Inconsistency on Multiple SET Clauses
-
-Considering the preceding example it could be expected that when a query has
-multiple `SET` clauses all the changes from those preceding one are visible.
-This is not the case in Neo4j's implementation. Consider the following queries
-executed on an empty database:
-
-    CREATE (n:A {x:0})-[:EdgeType]->(m:B {x:0})
-    MATCH (n)--(m) SET n.x = n.x + 1 SET m.x = m.x * 2
-    RETURN labels(n), n.x, labels(m), m.x
-
-This returns:
-
-+---------+---+---------+---+
-|labels(n)|n.x|labels(m)|m.x|
-+:=======:+:=:+:=======:+:=:+
-|[A]      |2  |[B]      |1  |
-+---------+---+---------+---+
-|[B]      |1  |[A]      |2  |
-+---------+---+---------+---+
-
-If all the iterations of the first `SET` clause were executed before executing
-the second, all the resulting values would be 2. This not being the case, we
-conclude that Neo4j does not use a barrier-like mechanism between `SET`
-clauses.  It is Memgraph's current vision that this is inconsistent and we
-plan to reduce Neo4j compliance in favour of operation consistency.
-
-### Double Deletion
-
-It's possible to match the same graph element multiple times in a single query
-and delete it. Neo supports this, and so do we. The relevant implementation
-detail is in the `GraphDbAccessor` class, where the record deletion functions
-reside, and not in the logical plan execution. It comes down to checking if a
-record has already been deleted in the current transaction+command and not
-attempting to do it again (results in a crash).
-
-### Set + Delete Edge-case
-
-It's legal for a query to combine `SET` and `DELETE` clauses. Consider the
-following queries executed on an empty database:
-
-
-    CREATE ()-[:T]->()
-    MATCH (n)--(m) SET n.x = 42 DETACH DELETE m
-
-Due to the `MATCH` being undirected the second pull will attempt to set data
-on a deleted vertex. This is not a legal operation in Memgraph storage
-implementation. For that reason the logical operator for `SET` must check if
-the record it's trying to set something on has been deleted by the current
-transaction+command. If so, the modification is not executed.
-
-### Deletion Accumulation
-
-Sometimes it's necessary to accumulate deletions of all the matches before
-attempting to execute them. Consider this the following. Start with an empty
-database and execute queries:
-
-    CREATE ()-[:T]->()-[:T]->()
-    MATCH (a)-[r1]-(b)-[r2]-(c) DELETE r1, b, c
-
-Note that the `DELETE` clause attempts to delete node `c`, but it does not
-detach it by deleting edge `r2`. However, due to undirected edge in the
-`MATCH`, both edges get pulled and deleted.
-
-Currently Memgraph does not support this behavior, Neo does. There are a few
-ways that we could do this.
-
- * Accumulate on deletion (that sucks because we have to keep track of
-   everything that gets returned after the deletion).
- * Maybe we could stream through the deletion op, but defer actual deletion
-   until plan-execution end.
- * Ignore this because it's very edgy (this is the currently selected option).
-
-### Aggregation Without Input
-
-It is necessary to define what aggregation ops return when they receive no
-input. Following is a table that shows what Neo4j's Cypher implementation and
-SQL produce.
-
-
-+-------------+------------------------+---------------------+---------------------+------------------+
-|    \<OP\>   | 1. Cypher, no group-by | 2. Cypher, group-by | 3. SQL, no group-by | 4. SQL, group-by |
-+=============+:======================:+:===================:+:===================:+:================:+
-| Count(\*)   | 0                      | \<NO\_ROWS>         | 0                   | \<NO\_ROWS>      |
-+-------------+------------------------+---------------------+---------------------+------------------+
-| Count(prop) | 0                      | \<NO\_ROWS>         | 0                   | \<NO\_ROWS>      |
-+-------------+------------------------+---------------------+---------------------+------------------+
-| Sum         | 0                      | \<NO\_ROWS>         | NULL                | \<NO\_ROWS>      |
-+-------------+------------------------+---------------------+---------------------+------------------+
-| Avg         | NULL                   | \<NO\_ROWS>         | NULL                | \<NO\_ROWS>      |
-+-------------+------------------------+---------------------+---------------------+------------------+
-| Min         | NULL                   | \<NO\_ROWS>         | NULL                | \<NO\_ROWS>      |
-+-------------+------------------------+---------------------+---------------------+------------------+
-| Max         | NULL                   | \<NO\_ROWS>         | NULL                | \<NO\_ROWS>      |
-+-------------+------------------------+---------------------+---------------------+------------------+
-| Collect     | []                     | \<NO\_ROWS>         | N/A                 | N/A              |
-+-------------+------------------------+---------------------+---------------------+------------------+
-
-Where:
-
-    1. `MATCH (n) RETURN <OP>(n.prop)`
-    2. `MATCH (n) RETURN <OP>(n.prop), (n.prop2)`
-    3. `SELECT <OP>(prop) FROM Table`
-    4. `SELECT <OP>(prop), prop2 FROM Table GROUP BY prop2`
-
-Neo's Cypher implementation diverges from SQL only when performing `SUM`.
-Memgraph implements SQL-like behavior. It is considered that `SUM` of
-arbitrary elements should not be implicitly 0, especially in a property graph
-without a strict schema (the property in question can contain values of
-arbitrary types, or no values at all).
-
-### OrderBy
-
-The `OrderBy` logical operator sorts the results in the desired order. It
-occurs in Cypher as part of a `WITH` or `RETURN` clause. Both the concept and
-the implementation are straightforward. It's necessary for the logical op to
-`Pull` everything from its input so it can be sorted. It's not necessary to
-keep the whole `Frame` state of each input, it is sufficient to keep a list of
-`TypedValues` on which the results will be sorted, and another list of values
-that need to be remembered and recreated on the `Frame` when yielding.
-
-The sorting itself is made to reflect that of Neo's implementation which comes
-down to these points.
-
-  * `Null` comes last (as if it's greater than anything).
-  * Primitive types compare naturally, with no implicit casting except from
-    `int` to `double`.
-  * Complex types are not comparable.
-  * Every unsupported comparison results in an exception that gets propagated
-    to the end user.
-
-### Limit in Write Queries
-
-`Limit` can be used as part of a write query, in which case it will *not*
-reduce the amount of performed updates. For example, consider a database that
-has 10 vertices. The query `MATCH (n) SET n.x = 1 RETURN n LIMIT 3` will
-result in all vertices having their property value changed, while returning
-only the first to the client. This makes sense from the implementation
-standpoint, because `Accumulate` is planned after `SetProperty` but before
-`Produce` and `Limit` operations. Note that this behavior can be
-non-deterministic in some queries, since it relies on the order of iteration
-over nodes which is undefined when not explicitly specified.
-
-### Merge
-
-`MERGE` in Cypher attempts to match a pattern. If it already exists, it does
-nothing and subsequent clauses like `RETURN` can use the matched pattern
-elements. If the pattern can't match to any data, it creates it. For detailed
-information see Neo4j's [merge
-documentation.](https://neo4j.com/docs/developer-manual/current/cypher/clauses/merge/)
-
-An important thing about `MERGE` is visibility of modified data. `MERGE` takes
-an input (typically a `MATCH`) and has two additional *phases*: the merging
-part, and the subsequent set parts (`ON MATCH SET` and `ON CREATE SET`).
-Analysis of Neo4j's behavior indicates that each of these three phases (input,
-merge, set) does not see changes to the graph state done by subsequent phase.
-The input phase does not see data created by the merge phase, nor the set
-phase. This is consistent with what seems like the general Cypher philosophy
-that query clause effects aren't visible in the preceding clauses.
-
-We define the `Merge` logical operator as a *routing* operator that uses three
-logical operator branches.
-
-  1. The input from a preceding clause.
-
-     For example in `MATCH (n), (m) MERGE (n)-[:T]-(m)`. This input is
-     optional because `MERGE` is allowed to be the first clause in a query.
-
-  2. The `merge_match` branch.
-
-     This logical operator branch is `Pull`-ed from until exhausted for each
-     successful `Pull` from the input branch.
-
-  3. The `merge_create` branch.
-
-     This branch is `Pull`ed when the `merge_match` branch does not match
-     anything (no successful `Pull`s) for an input `Pull`. It is `Pull`ed only
-     once in such a situation, since only one creation needs to occur for a
-     failed match.
-
-The `ON MATCH SET` and `ON CREATE SET` parts of the `MERGE` clause are
-included in the `merge_match` and `merge_create` branches respectively. They
-are placed on the end of their branches so that they execute only when those
-branches succeed.
-
-Memgraph strives to be consistent with Neo in its `MERGE` implementation,
-while at the same time keeping performance as good as possible. Consistency
-with Neo w.r.t. graph state visibility is not trivial. Documentation for
-`Expand` and `Set` describe how Memgraph keeps track of both the updated
-version of an edge/vertex and the old one, as it was before the current
-transaction+command. This technique is also used in `Merge`. The input
-phase/branch of `Merge` always looks at the old data. The merge phase needs to
-see the new data so it doesn't create more data then necessary.
-
-For example, consider the query.
-
-    MATCH (p:Person) MERGE (c:City {name: p.lives_in})
-
-This query needs to create a city node only once for each unique `p.lives_in`.
-Finally the set phase of a `MERGE` clause should not affect the merge phase.
-To achieve this the `merge_match` branch of the `Merge` operator should see
-the latest created nodes, but filter them on their old state (if those nodes
-were not created by the `create_branch`).  Implementation-wise that means that
-`ScanAll` and `Expand` operators in the `merge_branch` need to look at the new
-graph state, while `Filter` operators the old, if available.

docs/dev/query/interpreter-class.dot

@@ -1,23 +0,0 @@
-digraph interpreter {
-  node [fontname="dejavusansmono"]
-  edge [fontname="dejavusansmono"]
-  node [shape=record]
-  edge [dir=back,arrowtail=empty,arrowsize=1.5]
-  Interpreter [label="{\N|+ operator(query : string, ...) : Results\l|
-                      # MakeLogicalPlan(...) : LogicalPlan\l|
-                      - plan_cache_ : Map(QueryHash, CachedPlan)\l}"]
-  Interpreter -> DistributedInterpreter
-  Results [label="{\N|+ PullAll(stream) : void\l|- plan_ : CachedPlan\l}"]
-  Interpreter -> Results
-    [dir=forward,style=dashed,arrowhead=open,label="<<create>>"]
-  CachedPlan -> Results
-    [dir=forward,arrowhead=odiamond,taillabel="1",headlabel="*"]
-  Interpreter -> CachedPlan [arrowtail=diamond,taillabel="1",headlabel="*"]
-  CachedPlan -> LogicalPlan [arrowtail=diamond]
-  LogicalPlan [label="{\N|+ GetRoot() : LogicalOperator
-                      \l+ GetCost() : double\l}"]
-  LogicalPlan -> SingleNodeLogicalPlan [style=dashed]
-  LogicalPlan -> DistributedLogicalPlan [style=dashed]
-  DistributedInterpreter -> DistributedLogicalPlan
-    [dir=forward,style=dashed,arrowhead=open,label="<<create>>"]
-}

docs/dev/query/parsing.md

@@ -1,62 +0,0 @@
-# Lexical and Syntactic Analysis
-
-## Antlr
-
-We use Antlr for lexical and syntax analysis of Cypher queries. Antrl uses
-grammar file `Cypher.g4` downloaded from http://www.opencypher.org to generate
-the parser and the visitor for the Cypher parse tree. Even though the provided
-grammar is not very pleasant to work with we decided not to do any drastic
-changes to it so that our transition to newly published versions of
-`Cypher.g4` would be easier. Nevertheless, we had to fix some bugs and add
-features, so our version is not completely the same.
-
-In addition to using `Cypher.g4`, we have `MemgraphCypher.g4`. This grammar
-file defines Memgraph specific extensions to the original grammar. Most
-notable example is the inclusion of syntax for handling authorization. At the
-moment, some extensions are also found in `Cypher.g4`. For example, the syntax
-for using a lambda function in relationship patterns. These extensions should
-be moved out of `Cypher.g4`, so that it remains as close to the original
-grammar as possible. Additionally, having `MemgraphCypher.g4` may not be
-enough if we wish to split the functionality for community and enterprise
-editions of Memgraph.
-
-## Abstract Syntax Tree (AST)
-
-Since Antlr generated visitor and the official openCypher grammar are not very
-practical to use, we translate the Antlr's AST to our own AST. Currently there
-are ~40 types of nodes in our AST. Their definitions can be found in
-`src/query/frontend/ast/ast.lcp`.
-
-Major groups of types can be found under the following base types.
-
-  * `Expression` --- types corresponding to Cypher expressions.
-  * `Clause` --- types corresponding to Cypher clauses.
-  * `PatternAtom` --- node or edge related information.
-  * `Query` --- different kinds of queries, allows extending the language with
-    Memgraph specific query syntax.
-
-Memory management of created AST nodes is done with `AstStorage`. Each type
-must be created by invoking `AstStorage::Create` method. This way all of the
-pointers to nodes and their children are raw pointers. The only owner of
-allocated memory is the `AstStorage`. When the storage goes out of scope, the
-pointers become invalid. It may be more natural to handle tree ownership via
-`unique_ptr`, i.e. each node owns its children. But there are some benefits to
-having a custom storage and allocation scheme.
-
-The primary reason we opted for not using `unique_ptr` is the requirement of
-Antlr's base visitor class that the resulting values must by copyable. The
-result is wrapped in `antlr::Any` so that the derived visitor classes may
-return any type they wish when visiting Antlr's AST. Unfortunately,
-`antlr::Any` does not work with non-copyable types.
-
-Another benefit of having `AstStorage` is that we can easily add a different
-allocation scheme for AST nodes. The interface of node creation would not
-change.
-
-### AST Translation
-
-The translation process is done via `CypherMainVisitor` class, which is
-derived from Antlr generated visitor. Besides instancing our AST types, a
-minimal number of syntactic checks are done on a query. These checks handle
-the cases which were valid in original openCypher grammar, but may be invalid
-when combined with other syntax elements.

docs/dev/query/planning.md

@@ -1,526 +0,0 @@
-# Logical Planning
-
-After the semantic analysis and symbol generation, the AST is converted to a
-tree of logical operators. This conversion is called *planning* and the tree
-of logical operators is called a *plan*. The whole planning process is done in
-the following steps.
-
-  1. [AST Preprocessing](#ast-preprocessing)
-
-     The first step is to preprocess the AST by collecting
-     information on filters, divide the query into parts, normalize patterns
-     in `MATCH` clauses, etc.
-
-  2. [Logical Operator Planning](#logical-operator-planning)
-
-     After the preprocess step, the planning can be done via 2 planners:
-     `VariableStartPlanner` and `RuleBasedPlanner`. The first planner will
-     generate multiple plans where each plan has different starting points for
-     searching the patterns in `MATCH` clauses. The second planner produces a
-     single plan by mapping the query parts as they are to logical operators.
-
-  3. [Logical Plan Postprocessing](#logical-plan-postprocessing)
-
-     In this stage, we perform various transformations on the generated logical
-     plan. Here we want to optimize the operations in order to improve
-     performance during the execution. Naturally, transformations need to
-     preserve the semantic behaviour of the original plan.
-
-  4. [Cost Estimation](#cost-estimation)
-
-     After the generation, the execution cost of each plan is estimated. This
-     estimation is used to select the best plan which will be executed.
-
-The implementation can be found in the `query/plan` directory, with the public
-entry point being `query/plan/planner.hpp`.
-
-## AST Preprocessing
-
-Each openCypher query consists of at least 1 **single query**. Multiple single
-queries are chained together using a **query combinator**. Currently, there is
-only one combinator, `UNION`. The preprocessing step starts in the
-`CollectQueryParts` function. This function will take a look at each single
-query and divide it into parts. Each part is separated with `RETURN` and
-`WITH` clauses. For example:
-
-    MATCH (n) CREATE (m) WITH m MATCH (l)-[]-(m) RETURN l
-    |                          |                        |
-    |------- part 1 -----------+-------- part 2 --------|
-    |                                                   |
-    |-------------------- single query -----------------|
-
-Each part is created by collecting all `MATCH` clauses and *normalizing* their
-patterns. Pattern normalization is the process of converting an arbitrarily
-long pattern chain of nodes and edges into a list of triplets `(start node,
-edge, end node)`. The triplets should preserve the semantics of the match. For
-example:
-
-    MATCH (a)-[p]-(b)-[q]-(c)-[r]-(d)
-
-is equivalent to:
-
-    MATCH (a)-[p]-(b), (b)-[q]-(c), (c)-[r]-(d)
-
-With this representation, it becomes easier to reorder the triplets and choose
-different strategies for pattern matching.
-
-In addition to normalizing patterns, all of the filter expressions in patterns
-and inside of the `WHERE` clause (of the accompanying `MATCH`) are extracted
-and stored separately. During the extraction, symbols used in the filter
-expression are collected. This allows for planning filters in a valid order,
-as the matching for triplets is being done. Another important benefit of
-having extra information on filters, is to recognize when a database index
-could be used.
-
-After each `MATCH` is processed, they are all grouped, so that even the whole
-`MATCH` clauses may be reordered. The important thing is to remember which
-symbols were used to name edges in each `MATCH`. With those symbols we can
-plan for *cyphermorphism*, i.e. ensure different edges in the search pattern
-of a single `MATCH` map to different edges in the graph. This preserves the
-semantic of the query, even though we may have reordered the matching. The
-same steps are done for `OPTIONAL MATCH`.
-
-Another clause which needs processing is `MERGE`. Here we normalize the
-pattern, since the `MERGE` is a bit like `MATCH` and `CREATE` in one.
-
-All the other clauses are left as is.
-
-In the end, each query part consists of:
-
-  * processed and grouped `MATCH` clauses;
-  * processed and grouped `OPTIONAL MATCH` clauses;
-  * processed `MERGE` matching pattern and
-  * unchanged remaining clauses.
-
-The last stored clause is guaranteed to be either `WITH` or `RETURN`.
-
-## Logical Operator Planning
-
-### Variable Start Planner
-
-The `VariableStartPlanner` generates multiple plans for a single query. Each
-plan is generated by selecting a different starting point for pattern
-matching.
-
-The algorithm works as follows.
-
-  1. For each query part:
-     1. For each node in triplets of collected `MATCH` clauses:
-        i. Add the node to a set of `expanded` nodes
-        ii. Select a triplet `(start node, edge, end node)` whose `start node` is
-            in the `expanded` set
-        iii. If no triplet was selected, choose a new starting node that isn't in
-             `expanded` and continue expanding
-        iv. Repeat steps ii. -- iii. until all triplets have been selected
-            and store that as a variation of the `MATCH` clauses
-     2. Do step 1.1. for `OPTIONAL MATCH` and `MERGE` clauses
-     3. Take all combinations of the generated `MATCH`, `OPTIONAL MATCH` and
-        `MERGE` and store them as variations of the query part.
-  2. For each combination of query part variations:
-     1. Generate a plan using the rule based planner
-
-###  Rule Based Planner
-
-The `RuleBasedPlanner` generates a single plan for a single query. A plan is
-generated by following hardcoded rules for producing logical operators. The
-following sections are an overview on how each openCypher clause is converted
-to a `LogicalOperator`.
-
-####  MATCH
-
-`MATCH` clause is used to specify which patterns need to be searched for in
-the database. These patterns are normalized in the preprocess step to be
-represented as triplets `(start node, edge, end node)`. When there is no edge,
-then the triplet is reduced only to the `start node`. Generating the operators
-is done by looping over these triplets.
-
-##### Searching for Nodes
-
-The simplest search is finding standalone nodes. For example, `MATCH (n)`
-will find all the nodes in the graph. This is accomplished by generating a
-`ScanAll` operator and forwarding the node symbol which should store the
-results. In this case, all the nodes will be referenced by `n`.
-
-Multiple nodes can be specified in a single match, e.g. `MATCH (n), (m)`.
-Planning is done by repeating the same steps for each sub pattern (separated
-by a comma). In this case, we would get 2 `ScanAll` operators chained one
-after the other. An optimization can be obtained if the node in the pattern is
-already searched for. In `MATCH (n), (n)` we can drop the second `ScanAll`
-operator since we have already generated it for the first node.
-
-##### Searching for Relationships
-
-A more advanced search includes finding nodes with relationships. For example,
-`MATCH (n)-[r]-(m)` should find every pair of connected nodes in the database.
-This means, that if a single node has multiple connections, it will be
-repeated for each combination of pairs. The generation of operators starts
-from the first node in the pattern. If we are referencing a new starting node,
-we need to generate a `ScanAll` which finds all the nodes and stores them
-into `n`. Then, we generate an `Expand` operator which reads the `n` and
-traverses all the edges of that node. The edge is stored into `r`, while the
-destination node is stored in `m`.
-
-Matching multiple relationships proceeds similarly, by repeating the same
-steps. The only difference is that we need to ensure different edges in the
-search pattern, map to different edges in the graph. This means that after each
-`Expand` operator, we need to generate an `EdgeUniquenessFilter`. We provide
-this operator with a list of symbols for the previously matched edges and the
-symbol for the current edge.
-
-For example.
-
-    MATCH (n)-[r1]-(m)-[r2]-(l)
-
-The above is preprocessed into
-
-    MATCH (n)-[r1]-(m), (m)-[r2]-(l)
-
-Then we look at each triplet in order and perform the described steps. This
-way, we would generate:
-
-    ScanAll (n) > Expand (n, r1, m) > Expand (m, r2, l) >
-        EdgeUniquenessFilter ([r1], r2)
-
-Note that we don't need to make `EdgeUniquenessFilter` after the first
-`Expand`, since there are no edges to compare to. This filtering needs to work
-across multiple pattern, but inside a *single* `MATCH` clause.
-
-Let's take a look at the following.
-
-    MATCH (n)-[r1]-(m), (m)-[r2]-(l)
-
-We would also generate the exact same operators.
-
-    ScanAll (n) > Expand (n, r1, m) > Expand (m, r2, l) >
-        EdgeUniquenessFilter ([r1], r2)
-
-On the other hand,
-
-    MATCH (n)-[r1]-(m) MATCH (m)-[r2]-(l)-[r3]-(i)
-
-would reset the uniqueness filtering at the start of the second match. This
-would mean that we output the following:
-
-    ScanAll (n) > Expand (n, r1, m) > Expand (m, r2, l) > Expand (l, r3, i) >
-        EdgeUniquenessFilter ([r2], r3)
-
-There is a difference in how we handle edge uniqueness compared to Neo4j.
-Neo4j does not allow searching for a single edge multiple times, but we've
-decided to support that.
-
-For example, the user can say the following.
-
-    MATCH (n)-[r]-(m)-[r]-l
-
-We would ensure that both `r` variables match to the same edge. In our
-terminology, we call this the *edge cycle*. For the above example, we would
-generate this plan.
-
-    ScanAll (n) > Expand (n, r, m) > Expand (m, r, l)
-
-We do not put an `EdgeUniquenessFilter` operator between 2 `Expand`
-operators and we tell the 2nd `Expand` that it is an edge cycle. This, 2nd
-`Expand` will ensure we have matched both the same edges.
-
-##### Filtering
-
-To narrow the search down, the patterns in `MATCH` can have filtered labels
-and properties. A more general filtering is done using the accompanying
-`WHERE` clause. During the preprocess step, all filters are collected and
-extracted into expressions. Additional information on which symbols are used
-is also stored. This way, each time we generate a `ScanAll` or `Expand`, we
-look at all the filters to see if any of them can be used. I.e. if the symbols
-they use have been bound by a newly produced operator. If a filter expression
-can be used, we immediately add a `Filter` operator with that expression.
-
-For example.
-
-    MATCH (n)-[r]-(m :label) WHERE n.prop = 42
-
-We would produce:
-
-    ScanAll (n) > Filter (n.prop) > Expand (n, r, m) > Filter (m :label)
-
-This means that the same plan is generated for the query:
-
-    MATCH (n {prop: 42})-[r]-(m :label)
-
-#### OPTIONAL
-
-If a `MATCH` clause is preceded by `OPTIONAL`, then we need to generate a plan
-such that we produce results even if we fail to match anything. This is
-accomplished by generating an `Optional` operator, which takes 2 operator
-trees:
-
-  * input operation and
-  * optional operation.
-
-The input is the operation we generated for the part of the query before
-`OPTIONAL MATCH`. For the optional operation, we simply generate the `OPTIONAL
-MATCH` part just like we would for regular `MATCH`. In addition to operations,
-we need to send the symbols which are set during optional matching to the
-`Optional` operator. The operator will reset values of those symbols to
-`null`, when the optional part fails to match.
-
-#### RETURN & WITH
-
-`RETURN` and `WITH` clauses are very similar to each other. The only
-difference is that `WITH` separates parts of the query and can be paired with
-`WHERE` clause.
-
-The common part is generating operators for the body of the clause. Separation
-of query parts is mostly done in semantic analysis, which checks that only the
-symbols exposed through `WITH` are visible in the query parts after the
-clause. The minor part is done in planning.
-
-##### Named Results
-
-Both clauses contain multiple named expressions (`expr AS name`) which are
-used to generate `Produce` operator.
-
-##### Aggregations
-
-If an expression contains an aggregation operator (`sum`, `avg`, ...) we need
-to plan the `Aggregate` operator as input to `Produce`. This case is more
-complex, because aggregation in openCypher can perform implicit grouping of
-results used for aggregation.
-
-For example, `WITH/RETURN sum(n.x) AS s, n.y AS group` will implicitly group
-by `n.y` expression.
-
-Another, obscure grouping can be achieved with `RETURN sum(n.a) + n.b AS s`.
-Here, the `n.b` will be used for grouping, even though both the `sum` and
-`n.b` are in the same named expression.
-
-Therefore, we need to collect all expressions which do not contain
-aggregations and use them for grouping. You may have noticed that in the last
-example `sum` is actually a sub-expression of `+`. `Aggregate` operator does
-not see that (nor it should), so the responsibility of evaluating that falls
-on `Produce`. One way is for `Aggregate` to store results of grouping
-expressions on the frame in addition to aggregation results. Unfortunately,
-this would require rewiring named expressions in `Produce` to reference
-already evaluated expressions. In the current implementation, we opted for
-`Aggregate` to store only aggregation results on the frame, while `Produce`
-will re-evaluate all the other (grouping) expressions. To handle that, symbols
-which are used in expressions are passed to `Aggregate`, so that they can be
-remembered. `Produce` will read those symbols from the frame and use it to
-re-evaluate the needed expressions.
-
-##### Accumulation
-
-After we have `Produce` and potentially `Aggregate`, we need to handle a
-special case when the part of the query before `RETURN` or `WITH` performs
-updates. For that, we want to run that part of the query fully, so that we get
-the latest results. This is accomplished by adding `Accumulate` operator as
-input to `Aggregate` or `Produce` (if there is no aggregation). Accumulation
-will store all the values for all the used symbols inside `RETURN` and `WITH`,
-so that they can be used in the operator which follows. This way, only parts
-of the frame are copied, instead of the whole frame. Here is a minor
-difference between planning `WITH`, compared to `RETURN`. Since `WITH` can
-separate writing from reading, we need to advance the transaction command.
-This enables the later, read parts of the query to obtain the newest changes.
-This is supported by passing `advance_command` flag to `Accumulate` operator.
-
-In the simplest case, common to both clauses, we have `Accumulate > Aggregate
-> Produce` operators, where `Accumulate` and `Aggregate` may be left out.
-
-##### Ordering
-
-Planning `ORDER BY` is simple enough. Since it may see new symbols (filled in
-`Produce`), we add the `OrderBy` operator at the end. The operator will change
-the order of produced results, so we pass it the ordering expressions and the
-output symbols of named expressions.
-
-##### Filtering
-
-A final difference in `WITH`, is when it contains a `WHERE` clause. For that,
-we simply generate the `Filter` operator, appended after `Produce` or
-`OrderBy` (depending which operator is last).
-
-##### Skipping and Limiting
-
-If we have `SKIP` or `LIMIT`, we generate `Skip` or `Limit` operators,
-respectively. These operators are put at the end of the clause.
-
-This placement may have some unexpected behaviour when combined with
-operations that update the graph. For example.
-
-    MATCH (n) SET n.x = n.x + 1 RETURN n LIMIT 1
-
-The above query may be interpreted as if the `SET` will be done only once.
-Since this is a write query, we need to accumulate results, so the part before
-`RETURN` will execute completely. The accumulated results will be yielded up
-to the given limit, and the user would get only the first `n` that was
-updated.  This may confuse the user because in reality, every node in the
-database had been updated.
-
-Note that `Skip` always comes before `Limit`. In the current implementation,
-they are generated directly one after the other.
-
-#### CREATE
-
-`CREATE` clause is used to create nodes and edges (relationships).
-
-For multiple `CREATE` clauses or multiple creation patterns in a single
-clause, we perform the same, following steps.
-
-##### Creating a Single Node
-
-A node is created by simply specifying a node pattern.
-
-For example `CREATE (n :label {property: "value"}), ()` would create 2 nodes.
-The 1st one would be created with a label and a property. This node could be
-referenced later in the query, by using the variable `n`. The 2nd node cannot
-be referenced and it would be created without any labels nor properties. For
-node creation, we generate a `CreateNode` operator and pass it all the details
-of node creation: variable symbol, labels and properties. In the mentioned
-example, we would have `CreateNode > CreateNode`.
-
-##### Creating a Relationship
-
-To create a relationship, the `CREATE` clause must contain a pattern with a
-directed edge. Compared to creating a single node, this case is a bit more
-complicated, because either side of the edge may not exist. By exist, we mean
-that the endpoint is a variable which already references a node.
-
-For example, `MATCH (n) CREATE (n)-[r]->(m)` would create an edge `r` and a
-node `m` for each matched node `n`. If we focus on the `CREATE` part, we
-generate `CreateExpand (n, r, m)` where `n` already exists (refers to matched
-node) and `m` would be newly created along with edge `r`. If we had only
-`CREATE (n)-[r]->(m)`, then we would need to create both nodes of the edge
-`r`. This is done by generating `CreateNode (n) > CreateExpand(n, r, m)`. The
-final case is when both endpoints refer to an existing node. For example, when
-adding a node with a cyclical connection `CREATE (n)-[r]->(n)`. In this case,
-we would generate `CreateNode (n) > CreateExpand (n, r, n)`. We would tell
-`CreateExpand` to only create the edge `r` between the already created `n`.
-
-#### MERGE
-
-Although the merge operation is complex, planning turns out to be relatively
-simple. The pattern inside the `MERGE` clause is used for both matching and
-creating. Therefore, we create 2 operator trees, one for each action.
-
-For example.
-
-    MERGE (n)-[r:r]-(m)
-
-We would generate a single `Merge` operator which has the following.
-
-  * No input operation (since it is not preceded by any other clause).
-
-  * On match operation
-
-    `ScanAll (n) > Expand (n, r, m) > Filter (r)`
-
-  * On create operation
-
-    `CreateNode (n) > CreateExpand (n, r, m)`
-
-In cases when `MERGE` contains `ON MATCH` and `ON CREATE` parts, we simply
-append their operations to the respective operator trees.
-
-Observe the following example.
-
-    MERGE (n)-[r:r]-(m) ON MATCH SET n.x = 42 ON CREATE SET m :label
-
-The `Merge` would be generated with the following.
-
-  * No input operation (again, since there is no clause preceding it).
-
-  * On match operation
-
-    `ScanAll (n) > Expand (n, r, m) > Filter (r) > SetProperty (n.x, 42)`
-
-  * On create operation
-
-    `CreateNode (n) > CreateExpand (n, r, m) > SetLabels (n, :label)`
-
-When we have preceding clauses, we simply put their operator as input to
-`Merge`.
-
-    MATCH (n) MERGE (n)-[r:r]-(m)
-
-The above would be generated as
-
-    ScanAll (n) > Merge (on_match_operation, on_create_operation)
-
-Here we need to be careful to recognize which symbols are already declared.
-But, since the `on_match_operation` uses the same algorithm for generating a
-`Match`, that problem is handled there. The same should hold for
-`on_create_operation`, which uses the process of generating a `Create`. So,
-finally for this example, the `Merge` would have:
-
-  * Input operation
-
-    `ScanAll (n)`
-
-  * On match operation
-
-    `Expand (n, r, m) > Filter (r)`
-
-    Note that `ScanAll` is not needed since we get the nodes from input.
-
-  * On create operation
-
-    `CreateExpand (n, r, m)`
-
-    Note that `CreateNode` is dropped, since we want to expand the existing one.
-
-## Logical Plan Postprocessing
-
-Postprocessing of a logical plan is done by rewriting the original plan into
-a more efficient one while preserving the original semantic of operations.
-The rewriters are found in `query/plan/rewrite` directory, and currently we
-only have one -- `IndexLookupRewriter`.
-
-### IndexLookupRewriter
-
-The job of this rewriter is to merge `Filter` and `ScanAll` operations into
-equivalent `ScanAllBy<Index>` operations. In almost all cases using indexed
-lookup will be faster than regular lookup, so `IndexLookupRewriter` simply
-does the transformations whenever possible. The simplest case being the
-following, assuming we have an index over `id`.
-
-  * Original Plan
-
-    `ScanAll (n) > Filter (id(n) == 42) > Produce (n)`
-
-  * Rewritten Plan
-
-    `ScanAllById (n, id=42) > Produce (n)`
-
-Naturally, there are some cases we need to be careful about.
-
-  1. Operators with Multiple Branches
-
-     Here we may not carry `Filter` operations outside of the operator into
-     its branches, so the branches are rewritten as stand alone plans with a
-     branch new `IndexLookupRewriter`. Some of the operators with multiple
-     branches are `Merge`, `Optional`, `Cartesian` and `Union`.
-
-  2. Expand Operators
-
-     Expand operations aren't that tricky to handle, but they have a special
-     case where we want to use an indexed lookup of the destination so that the
-     expansion is performed between known nodes. This decision may depend on
-     various parameters which may need further tweaking as we encounter more
-     use-cases of Cypher queries.
-
-## Cost Estimation
-
-Cost estimation is the final step of processing a logical plan. The
-implementation can be found in `query/plan/cost_estimator.hpp`. We give each
-operator a cost based on the estimated cardinality of results of that operator
-and on the preset coefficient of the runtime performance of that operator.
-
-This scheme is rather simple and works quite well, but there are couple of
-improvements we may want to do at some point.
-
-  * Track more information about the stored graph and use that to improve the
-    estimates.
-  * Do a quick, partial run of the plan and tweak the estimation based on how
-    much each operator produced results. This may require us having some kind
-    of representative subset of the stored graph.
-  * Write micro benchmarks for each operator and based on the results create
-    sensible preset coefficients. This would replace the current coefficients
-    which are just assumptions on how each operator implementation performs.

docs/dev/query/semantic.md

@@ -1,134 +0,0 @@
-# Semantic Analysis and Symbol Generation
-
-In this phase, various semantic and variable type checks are performed.
-Additionally, we generate symbols which map AST nodes to stored values
-computed from evaluated expressions.
-
-## Symbol Generation
-
-Implementation can be found in `query/frontend/semantic/symbol_generator.cpp`.
-
-Symbols are generated for each AST node that represents data that needs to
-have storage. Currently, these are:
-
-  * `NamedExpression`
-  * `CypherUnion`
-  * `Identifier`
-  * `Aggregation`
-
-You may notice that the above AST nodes may not correspond to something named
-by a user. For example, `Aggregation` can be a part of larger expression and
-thus remain unnamed. The reason we still generate symbols is to have a uniform
-behaviour when executing a query as well as allow for caching the results of
-expression evaluation.
-
-AST nodes do not actually store a `Symbol` instance, instead they have a
-`int32_t` index identifying the symbol in the `SymbolTable` class. This is
-done to minimize the size of AST types as well as allow easier sharing of same
-symbols with multiple instances of AST nodes.
-
-The storage for evaluated data is represented by the `Frame` class. Each
-symbol determines a unique position in the frame. During interpretation,
-evaluation of expressions which have a symbol will either read or store values
-in the frame. For example, instance of an `Identifier` will use the symbol to
-find and read the value from `Frame`. On the other hand, `NamedExpression`
-will take the result of evaluating its own expression and store it in the
-`Frame`.
-
-When a symbol is created, context of creation is used to assign a type to that
-symbol. This type is used for simple type checking operations. For example,
-`MATCH (n)` will create a symbol for variable `n`. Since the `MATCH (n)`
-represents finding a vertex in the graph, we can set `Symbol::Type::Vertex`
-for that symbol. Later, for example in `MATCH ()-[n]-()` we see that variable
-`n` is used as an edge. Since we already have a symbol for that variable, we
-detect this type mismatch and raise a `SemanticException`.
-
-Basic rule of symbol generation, is that variables inside `MATCH`, `CREATE`,
-`MERGE`, `WITH ... AS` and `RETURN ... AS` clauses establish new symbols.
-
-### Symbols in Patterns
-
-Inside `MATCH`, symbols are created only if they didn't exist before. For
-example, patterns in `MATCH (n {a: 5})--(m {b: 5}) RETURN n, m` will create 2
-symbols: one for `n` and one for `m`. `RETURN` clause will, in turn, reference
-those symbols. Symbols established in a part of pattern are immediately bound
-and visible in later parts. For example, `MATCH (n)--(n)` will create a symbol
-for variable `n` for 1st `(n)`. That symbol is referenced in 2nd `(n)`. Note
-that the symbol is not bound inside 1st `(n)` itself. What this means is that,
-for example, `MATCH (n {a: n.b})` should raise an error, because `n` is not
-yet bound when encountering `n.b`. On the other hand,
-`MATCH (n)--(n {a: n.b})` is fine.
-
-The `CREATE` is similar to `MATCH`, but it *always* establishes symbols for
-variables which create graph elements. What this means is that, for example
-`MATCH (n) CREATE (n)` is not allowed. `CREATE` wants to create a new node,
-for which we already have a symbol. In such a case, we need to throw an error
-that the variable `n` is being redeclared. On the other hand `MATCH (n) CREATE
-(n)-[r :r]->(n)` is fine, because `CREATE` will only create the edge `r`,
-connecting the already existing node `n`. Remaining behaviour is the same as
-in `MATCH`. This means that we can simplify `CREATE` to be like `MATCH` with 2
-special cases.
-
-  1. Are we creating a node, i.e. `CREATE (n)`? If yes, then the symbol for
-     `n` must not have been created before. Otherwise, we reference the
-     existing symbol.
-  2. Are we creating an edge, i.e. we encounter a variable for an edge inside
-     `CREATE`? If yes, then that variable must not reference a symbol.
-
-The `MERGE` clause is treated the same as `CREATE` with regards to symbol
-generation. The only difference is that we allow bidirectional edges in the
-pattern. When creating such a pattern, the direction of the created edge is
-arbitrarily determined.
-
-### Symbols in WITH and RETURN
-
-In addition to patterns, new symbols are established in the `WITH` clause.
-This clause makes the new symbols visible *only* to the rest of the query.
-For example, `MATCH (old) WITH old AS new RETURN new, old` should raise an
-error that `old` is unbound inside `RETURN`.
-
-There is a special case with symbol visibility in `WHERE` and `ORDER BY`. They
-need to see both the old and the new symbols. Therefore `MATCH (old) RETURN
-old AS new ORDER BY old.prop` needs to work. On the other hand, if we perform
-aggregations inside `WITH` or `RETURN`, then the old symbols should not be
-visible neither in `WHERE` nor in `ORDER BY`. Since the aggregation has to go
-through all the results in order to generate the final value, it makes no
-sense to store old symbols and their values. A query like `MATCH (old) WITH
-SUM(old.prop) AS sum WHERE old.prop = 42 RETURN sum` needs to raise an error
-that `old` is unbound inside `WHERE`.
-
-For cases when `SKIP` and `LIMIT` appear, we disallow any identifiers from
-appearing in their expressions. Basically, `SKIP` and `LIMIT` can only be
-constant expressions[^1]. For example, `MATCH (old) RETURN old AS new SKIP
-new.prop` needs to raise that variables are not allowed in `SKIP`. It makes no
-sense to allow variables, since their values may vary on each iteration. On
-the other hand, we could support variables to constant expressions, but for
-simplicity we do not. For example, `MATCH (old) RETURN old, 2 AS limit_var
-LIMIT limit_var` would still throw an error.
-
-Finally, we generate symbols for names created in `RETURN` clause. These
-symbols are used for the final results of a query.
-
-NOTE: New symbols in `WITH` and `RETURN` should be unique. This means that
-`WITH a AS same, b AS same` is not allowed, neither is a construct like
-`RETURN 2, 2`
-
-### Symbols in Functions which Establish New Scope
-
-Symbols can also be created in some functions. These functions usually take an
-expression, bind a single variable and run the expression inside the newly
-established scope.
-
-The `all` function takes a list, creates a variable for list element and runs
-the predicate expression. For example:
-
-    MATCH (n) RETURN n, all(n IN n.prop_list WHERE n < 42)
-
-We create a new symbol for use inside `all`, this means that the `WHERE n <
-42` uses the `n` which takes values from a `n.prop_list` elements. The
-original `n` bound by `MATCH` is not visible inside the `all` function, but it
-is visible outside. Therefore, the `RETURN n` and `n.prop_list` reference the
-`n` from `MATCH`.
-
-[^1]: Constant expressions are expressions for which the result can be
-  computed at compile time.

docs/dev/quick-start.md

@@ -1,108 +0,0 @@
-# Quick Start
-
-A short chapter on downloading the Memgraph source, compiling and running.
-
-## Obtaining the Source Code
-
-Memgraph uses `git` for source version control. You will need to install `git`
-on your machine before you can download the source code.
-
-On Debian systems, you can do it inside a terminal with the following
-command:
-
-    sudo apt-get install git
-
-On ArchLinux or Gentoo, you probably already know what to do.
-
-After installing `git`, you are now ready to fetch your own copy of Memgraph
-source code. Run the following command:
-
-    git clone https://phabricator.memgraph.io/diffusion/MG/memgraph.git
-
-The above will create a `memgraph` directory and put all source code there.
-
-## Compiling Memgraph
-
-With the source code, you are now ready to compile Memgraph. Well... Not
-quite. You'll need to download Memgraph's dependencies first.
-
-In your terminal, position yourself in the obtained memgraph directory.
-
-    cd memgraph
-
-### Installing Dependencies
-
-On Debian systems, dependencies that are required by the codebase should be
-setup by running the `init` script:
-
-    ./init -s
-
-Currently, other systems aren't supported in the `init` script. But you can
-issue the needed steps manually. First run the `init` script.
-
-    ./init
-
-The script will output the required packages, which you should be able to
-install via your favorite package manager. For example, `pacman` on ArchLinux.
-After installing the packages, issue the following commands:
-
-    mkdir -p build
-    ./libs/setup.sh
-
-### Compiling
-
-Memgraph is compiled using our own custom toolchain that can be obtained from
-[Toolchain repository](https://deps.memgraph.io/toolchain).  You should read
-the `README.txt` file in the repository and install the apropriate toolchain
-for your distribution.  After you have installed the toolchain you should read
-the instructions for the toolchain in the toolchain install directory
-(`/opt/toolchain-vXYZ/README.md`) and install dependencies that are necessary
-to run the toolchain.
-
-When you want to compile Memgraph you should activate the toolchain using the
-prepared toolchain activation script that is also described in the toolchain
-`README`.
-
-NOTE: You *must* activate the toolchain every time you want to compile
-Memgraph!
-
-You should now activate the toolchain in your console.
-
-    source /opt/toolchain-vXYZ/activate
-
-With all of the dependencies installed and the build environment set-up, you
-need to configure the build system. To do that, execute the following:
-
-    cd build
-    cmake ..
-
-If everything went OK, you can now, finally, compile Memgraph.
-
-    make -j$(nproc)
-
-### Running
-
-After the compilation verify that Memgraph works:
-
-    ./memgraph --version
-
-To make extra sure, run the unit tests:
-
-    ctest -R unit -j$(nproc)
-
-## Problems
-
-If you have any trouble running the above commands, contact your nearest
-developer who successfully built Memgraph. Ask for help and insist on getting
-this document updated with correct steps!
-
-## Next Steps
-
-Familiarise yourself with our code conventions and guidelines:
-
-  * [C++ Code](cpp-code-conventions.md)
-  * [Other Code](other-code-conventions.md)
-  * [Code Review Guidelines](code-review.md)
-
-Take a look at the list of [required reading](required-reading.md) for
-brushing up on technical skills.

docs/dev/required-reading.md

@@ -1,129 +0,0 @@
-# Required Reading
-
-This chapter lists a few books that should be read by everyone working on
-Memgraph. Since Memgraph is developed primarily with C++, Python and Common
-Lisp, books are oriented around those languages. Of course, there are plenty
-of general books which will help you improve your technical skills (such as
-"The Pragmatic Programmer", "The Mythical Man-Month", etc.), but they are not
-listed here. This way the list should be kept short and the *required* part in
-"Required Reading" more easily honored.
-
-Some of these books you may find in our office, so feel free to pick them up.
-If any are missing and you would like a physical copy, don't be afraid to
-request the book for our office shelves.
-
-Besides reading, don't get stuck in a rut and be a
-[Blub Programmer](http://www.paulgraham.com/avg.html).
-
-## Effective C++ by Scott Meyers
-
-Required for C++ developers.
-
-The book is a must-read as it explains most common gotchas of using C++. After
-reading this book, you are good to write competent C++ which will pass code
-reviews easily.
-
-## Effective Modern C++ by Scott Meyers
-
-Required for C++ developers.
-
-This is a continuation of the previous book, it covers updates to C++ which
-came with C++11 and later. The book isn't as imperative as the previous one,
-but it will make you aware of modern features we are using in our codebase.
-
-## Practical Common Lisp by Peter Siebel
-
-Required for Common Lisp developers.
-
-Free: http://www.gigamonkeys.com/book/
-
-We use Common Lisp to generate C++ code and make our lives easier.
-Unfortunately, not many developers are familiar with the language. This book
-will make you familiar very quickly as it has tons of very practical
-exercises. E.g. implementing unit testing library, serialization library and
-bundling all that to create a mp3 music server.
-
-## Effective Python by Brett Slatkin
-
-(Almost) required reading for Python developers.
-
-Why the "almost"? Well, Python is relatively easy to pick up and you will
-probably learn all the gotchas during code review from someone more
-experienced. This makes the book less necessary for a newcomer to Memgraph,
-but the book is not advanced enough to delegate it to
-[Advanced Reading](#advanced-reading). The book is written in similar vein as
-the "Effective C++" ones and will make you familiar with nifty Python features
-that make everyone's lives easier.
-
-# Advanced Reading
-
-The books listed below are not required reading, but you may want to read them
-at some point when you feel comfortable enough.
-
-## Design Patterns by Gamma et. al.
-
-Recommended for C++ developers.
-
-This book is highly divisive because it introduced a culture centered around
-design patterns. The main issues is overuse of patterns which complicates the
-code. This has made many Java programs to serve as examples of highly
-complicated, "enterprise" code.
-
-Unfortunately, design patterns are pretty much missing
-language features. This is most evident in dynamic languages such as Python
-and Lisp, as demonstrated by
-[Peter Norvig](http://www.norvig.com/design-patterns/).
-
-Or as [Paul Graham](http://www.paulgraham.com/icad.html) put it:
-
-```
-This practice is not only common, but institutionalized. For example, in the
-OO world you hear a good deal about "patterns". I wonder if these patterns are
-not sometimes evidence of case (c), the human compiler, at work. When I see
-patterns in my programs, I consider it a sign of trouble. The shape of a
-program should reflect only the problem it needs to solve. Any other
-regularity in the code is a sign, to me at least, that I'm using abstractions
-that aren't powerful enough-- often that I'm generating by hand the expansions
-of some macro that I need to write
-```
-
-After presenting the book so negatively, why you should even read it then?
-Well, it is good to be aware of those design patterns and use them when
-appropriate. They can improve modularity and reuse of the code. You will also
-find examples of such patterns in our code, primarily Strategy and Visitor
-patterns. The book is also a good stepping stone to more advanced reading
-about software design.
-
-## Modern C++ Design by Andrei Alexandrescu
-
-Recommended for C++ developers.
-
-This book can be treated as a continuation of the previous "Design Patterns"
-book. It introduced "dark arts of template meta-programming" to the world.
-Many of the patterns are converted to use C++ templates which makes them even
-better for reuse. But, like the previous book, there are downsides if used too
-much. You should approach it with a critical eye and it will help you
-understand ideas that are used in some parts of our codebase.
-
-## Large Scale C++ Software Design by John Lakos
-
-Recommended for C++ developers.
-
-An old book, but well worth the read. Lakos presents a very pragmatic view of
-writing modular software and how it affects both development time as well as
-program runtime. Some things are outdated or controversial, but it will help
-you understand how the whole C++ process of working in a large team, compiling
-and linking affects development.
-
-## On Lisp by Paul Graham
-
-Recommended for Common Lisp developers.
-
-Free: http://www.paulgraham.com/onlisp.html
-
-An excellent continuation to "Practical Common Lisp". It starts of slow, as if
-introducing the language, but very quickly picks up speed. The main meat of
-the book are macros and their uses. From using macros to define cooperative
-concurrency to including Prolog as if it's part of Common Lisp. The book will
-help you understand more advanced macros that are occasionally used in our
-Lisp C++ Preprocessor (LCP).

docs/dev/storage/accessors.md

@@ -1,110 +0,0 @@
-# DatabaseAccessor
-
-A `DatabaseAccessor` actually wraps a transactional access to database
-data, for a single transaction. In that sense the naming is bad. It
-encapsulates references to the database and the transaction object.
-
-It contains logic for working with database content (graph element
-data) in the context of a single transaction. All CRUD operations are
-performed within a single transaction (as Memgraph is a transactional
-database), and therefore iteration over data, finding a specific graph
-element etc are all functionalities of a `GraphDbAccessor`.
-
-In single-node Memgraph the database accessor also defined the lifetime
-of a transaction. Even though a `Transaction` object was owned by the
-transactional engine, it was `GraphDbAccessor`'s lifetime that object
-was bound to (the transaction was implicitly aborted in
-`GraphDbAccessor`'s destructor, if it was not explicitly ended before
-that).
-
-# RecordAccessor
-
-It is important to understand data organization and access in the
-storage layer. This discussion pertains to vertices and edges as graph
-elements that the end client works with.
-
-Memgraph uses MVCC (documented on it's own page). This means that for
-each graph element there could be different versions visible to
-different currently executing transactions. When we talk about a
-`Vertex` or `Edge` as a data structure we typically mean one of those
-versions. In code this semantic is implemented so that both those classes
-inherit `mvcc::Record`, which in turn inherits `mvcc::Version`.
-
-Handling MVCC and visibility is not in itself trivial. Next to that,
-there is other book-keeping to be performed when working with data. For
-that reason, Memgraph uses "accessors" to define an API of working with
-data in a safe way. Most of the code in Memgraph (for example the
-interpretation code) should work with accessors. There is a
-`RecordAccessor` as a base class for `VertexAccessor` and
-`EdgeAccessor`. Following is an enumeration of their purpose.
-
-### Data access
-
-The client interacts with Memgraph using the Cypher query language. That
-language has certain semantics which imply that multiple versions of the
-data need to be visible during the execution of a single query. For
-example: expansion over the graph is always done over the graph state as
-it was at the beginning of the transaction.
-
-The `RecordAccessor` exposes functions to switch between the old and the new
-versions of the same graph element (intelligently named `SwitchOld` and
-`SwitchNew`) within a single transaction. In that way the client code
-(mostly the interpreter) can avoid dealing with the underlying MVCC
-version concepts.
-
-### Updates
-
-Data updates are also done through accessors. Meaning: there are methods
-on the accessors that modify data, the client code should almost never
-interact directly with `Vertex` or `Edge` objects.
-
-The accessor layer takes care of creating version in the MVCC layer and
-performing updates on appropriate versions.
-
-Next, for many kinds of updates it is necessary to update the relevant
-indexes. There are implicit indexes for vertex labels, as
-well as user-created indexes for (label, property) pairs. The accessor
-layer takes care of updating the indexes when these values are changed.
-
-Each update also triggers a log statement in the write-ahead log. This
-is also handled by the accessor layer.
-
-### Distributed
-
-In distributed Memgraph accessors also contain a lot of the remote graph
-element handling logic. More info on that is available in the
-documentation for distributed.
-
-### Deferred MVCC data lookup for Edges
-
-Vertices and edges are versioned using MVCC. This means that for each
-transaction an MVCC lookup needs to be done to determine which version
-is visible to that transaction. This tends to slow things down due to
-cache invalidations (version lists and versions are stored in arbitrary
-locations on the heap).
-
-However, for edges, only the properties are mutable. The edge endpoints
-and type are fixed once the edge is created. For that reason both edge
-endpoints and type are available in vertex data, so that when expanding
-it is not mandatory to do MVCC lookups of versioned, mutable data. This
-logic is implemented in `RecordAccessor` and `EdgeAccessor`.
-
-### Exposure
-
-The original idea and implementation of graph element accessors was that
-they'd prevent client code from ever interacting with raw `Vertex` or
-`Edge` data. This however turned out to be impractical when implementing
-distributed Memgraph and the raw data members have since been exposed
-(through getters to old and new version pointers). However, refrain from
-working with that data directly whenever possible! Always consider the
-accessors to be the first go-to for interacting with data, especially
-when in the context of a transaction.
-
-# Skiplist accessor
-
-The term "accessor" is also used in the context of a skiplist. Every
-operation on a skiplist must be performed within on an
-accessor. The skiplist ensures that there will be no physical deletions
-of an object during the lifetime of an accessor. This mechanism is used
-to ensure deletion correctness in a highly concurrent container.
-We only mention that here to avoid confusion regarding terminology.

docs/dev/storage/indexes.md

@@ -1,116 +0,0 @@
-# Label indexes
-
-These are unsorted indexes that contain all the vertices that have the label
-the indexes are for (one index per label). These kinds of indexes get
-automatically generated for each label used in the database.
-
-### Updating the indexes
-
-Whenever something gets added to the record we update the index (add that
-record to index). We keep an index which might contain garbage (not relevant
-records, because the value got removed or something similar) but we will
-filter it out when querying the index. We do it like this because we don't
-have to do bookkeeping and deciding if we update the index on the end of the
-transaction (commit/abort phase), moreover current interpreter advances the
-command in transaction and as such assumes that the indexes now contain
-objects added in the previous command inside this transaction, so we need to
-update over the whole scope of transaction (whenever something is added to the
-record).
-
-### Index Entries Label
-
-These kinds of indexes are internally keeping track of pair (record, vlist).
-Why do we need to keep track of exactly those two things?
-
-Problems with two different approaches
-
-1) Keep track of just the record:
-
-  - We need the `VersionList` for creating an accessor (this in itself is a
-    deal-breaker).
-  - Semantically it makes sense. An edge/vertex maps bijectionally to a
-    `VersionList`.
-  - We might try to access some members of record while the record is being
-    modified from another thread.
-  - A vertex/edge could get updated, thus expiring the record in the index.
-    The newly created record should be present in the index, but it's not.
-    Without the `VersionList` we can't reach the newly created record.
-  - Probably there are even more reasons... It should be obvious by now that
-    we need the `VersionList` in the index.
-
-2) Keep track of just the version list:
-
-  - Removing from an index is a problem for two major reasons. First, if we
-    only have the `VersionList`, checking if it should be removed implies
-    checking all the reachable records, which is not thread-safe. Second,
-    there are issues with concurrent removal and insertion. The cleanup thread
-    could determine the vertex/edge should be removed from the index and
-    remove it, while in between those ops another thread attempts to insert
-    the `VersionList` into the index. The insertion does nothing because the
-    `VersionList` is already in, but it gets removed immediately after.
-
-Because of inability to keep track of just the record, or value, we need to
-keep track of both of them.  Resolution of problems mentioned above, in the
-same order, with (record, vlist) pair
-
-  - simple `vlist.find(current transaction)` will get us the newest visible
-    record
-  - we'll never try to access some record if it's still being written since we
-    will always operate on vlist.find returned record
-  - newest record will contain that label
-  - since we have (record, vlist) pair as the key in the index when we update
-    and delete in the same time we will never delete the same record, vlist
-    pair we are adding because the record, vlist pair we are deleting is
-    already superseded by a newer record and as such won't be inserted while
-    it's being deleted
-
-### Querying the index
-
-We run through the index for the given label and do `vlist.find` operation for
-the current transaction, and check if the newest return record has that
-label. If it has it then we return it. By now you are probably wondering
-aren't we sometimes returning duplicate vlist entries? And you are wondering
-correctly, we would be returning them, but we are making sure that the entires
-in the index are sorted by their `vlist*` and as such we can filter consecutive
-duplicate `vlist*` to only return one of those while still being able to create
-an iterator to index.
-
-### Cleaning the index
-
-Cleaning the index is not as straightforward as it seems as a lot of garbage
-can accumulate, but it's hard to know when exactly can we delete some (record,
-vlist) pair. First, let's assume that we are doing the cleaning process at
-some `transaction_id`, `id` such that there doesn't exist an active transaction
-with an id lower than `id`.
-
-We scan through the whole index and for each (record, vlist) pair we first
-check if it was deleted before the id (i.e. no transaction with an id >= `id`
-will ever again see that record), if it was deleted before we might naively
-say that it's safe to delete it, but, we must take into account that when some
-new record is created from this record (update operation), that record still
-contains the label but by deleting this record we won't be able to see that
-vlist because that new record won't add again to index because we didn't
-explicitly add that label again to it.
-
-Because of this we have to 'update' this index (record, vlist) pair. We have
-to update the record to now point to a newer record in vlist, the one that is
-not deleted yet. We can do that by querying the `version_list` for the last
-record inside (oldest it has — remember that `mvcc_gc` will re-link not
-visible records so the last record will be visible for the current GC id).
-When updating the record inside the index, it's not okay to just update the
-pointer and leave the index as it is, because with updating the `record*` we
-might change the relative order of entries inside the index. We first have to
-re-insert it with new `record*`, and then delete the old entry. And we need to
-do insertion before the remove operation! Otherwise it could happen that the
-vlist with a newer record with that label won't exist while some transaction
-is querying the index.
-
-Records which we added as a consequence of deleting older records will be
-eventually removed from the index if they don't contain label because if we
-see that the record is not deleted we try to check if that record still
-contains the label. We also need to be careful here because we can't check
-that while the record is being potentially updated by some transaction (race
-condition), so we need can check if records still contain label if it's
-creation id is smaller than our `id`, as that implies that the creating
-transaction either aborted or committed as our `id` is equal to the oldest
-active transaction in time of starting the GC.

docs/dev/storage/property-storage.md

@@ -1,131 +0,0 @@
-# Property storage
-
-Although the reader is probably familiar with properties in *Memgraph*, let's
-briefly recap. 
-
-Both vertices and edges can store an arbitrary number of properties. Properties
-are, in essence, ordered pairs of property names and property values. Each
-property name within a single graph element (edge/node) can store a single
-property value. Property names are represented as strings, while property values
-must be one of the following types:
-
- Type      | Description
------------|------------
- `Null`    | Denotes that the property has no value. This is the same as if the property does not exist.
- `String`  | A character string, i.e. text.
- `Boolean` | A boolean value, either `true` or `false`.
- `Integer` | An integer number.
- `Float`   | A floating-point number, i.e. a real number.
- `List`    | A list containing any number of property values of any supported type. It can be used to store multiple values under a single property name.
- `Map`     | A mapping of string keys to values of any supported type.
-
-Property values are modeled in a class conveniently called `PropertyValue`.
-
-## Mapping between property names and property keys.
-
-Although users think of property names in terms of descriptive strings 
-(e.g. "location" or "department"), *Memgraph* internally converts those names
-into property keys which are, essentially, unsigned 16-bit integers.
-
-Property keys are modelled by a not-so-conveniently named class called
-`Property` which can be found in `storage/types.hpp`. The actual conversion
-between property names and property keys is done within the `ConcurrentIdMapper`
-but the internals of that implementation are out of scope for understanding
-property storage.
-
-## PropertyValueStore
-
-Both `Edge` and `Vertex` objects contain an instance of `PropertyValueStore`
-object which is responsible for storing properties of a corresponding graph
-element.
-
-An interface of `PropertyValueStore` is as follows:
-
- Method    | Description
------------|------------
- `at`      | Returns the `PropertyValue` for a given `Property` (key).
- `set`     | Stores a given `PropertyValue` under a given `Property` (key).
- `erase`   | Deletes a given `Property` (key) alongside its corresponding `PropertyValue`.
- `clear`   | Clears the storage.
- `iterator`| Provides an extension of `std::input_iterator` that iterates over storage.
-
-## Storage location
-
-By default, *Memgraph* is an in-memory database and all properties are therefore
-stored in working memory unless specified otherwise by the user. User has an
-option to specify via the command line which properties they wish to be stored
-on disk.
-
-Storage location of each property is encapsulated within a `Property` object
-which is ensured by the `ConcurrentIdMapper`. More precisely, the unsigned 16-bit
-property key has the following format:
-
-```
-|---location--|------id------|
-|-Memory|Disk-|-----2^15-----|
-``` 
-
-In other words, the most significant bit determines the location where the
-property will be stored. 
-
-### In-memory storage
-
-The underlying implementation of in-memory storage for the time being is
-`std::vector<std::pair<Property, PropertyValue>>`. Implementations of`at`, `set`
-and `erase` are linear in time. This implementation is arguably more efficient
-than `std::map` or `std::unordered_map` when the average number of properties of
-a record is relatively small (up to 10) which seems to be the case.
-
-### On-disk storage
-
-#### KVStore
-
-Disk storage is modeled by an abstraction of key-value storage as implemented in
-`storage/kvstore.hpp'. An interface of this abstraction is as follows:
-
- Method         | Description
-----------------|------------
- `Put`          | Stores the given value under the given key.
- `Get`          | Obtains the given value stored under the given key.
- `Delete`       | Deletes a given (key, value) pair from storage..
- `DeletePrefix` | Deletes all (key, value) pairs where key begins with a given prefix.
- `Size`         | Returns the size of the storage or, optionally, the number of stored pairs that begin with a given prefix.
- `iterator`     | Provides an extension of `std::input_iterator` that iterates over storage.
-
-Keys and values in this context are of type `std::string`.
-
-The actual underlying implementation of this abstraction uses
-[RocksDB]{https://rocksdb.org} &mdash; a persistent key-value store for fast
-storage.
-
-It is worthy to note that the custom iterator implementation allows the user
-to iterate over a given prefix. Otherwise, the implementation follows familiar
-c++ constructs and can be used as follows:
-
-```
-KVStore storage = ...;
-for (auto it = storage.begin(); it != storage.end(); ++it) {}
-for (auto kv : storage) {}
-for (auto it = storage.begin("prefix"); it != storage.end("prefix"); ++it) {}
-```
-
-Note that it is not possible to scan over multiple prefixes. For instance, one
-might assume that you can scan over all keys that fall in a certain
-lexicographical range. Unfortunately, that is not the case and running the
-following code will result in an infinite loop with a touch of undefined
-behavior.
-
-```
-KVStore storage = ...;
-for (auto it = storage.begin("alpha"); it != storage.end("omega"); ++it) {}
-```
-
-#### Data organization on disk
-
-Each `PropertyValueStore` instance can access a static `KVStore` object that can
-store `(key, value)` pairs on disk. The key of each property on disk consists of
-two parts &mdash; a unique identifier (unsigned 64-bit integer) of the current
-record version (see mvcc docummentation for further clarification) and a 
-property key as described above. The actual value of the property is serialized
-into a bytestring using bolt `BaseEncoder`. Similarly, deserialization is
-performed by bolt `Decoder`.

docs/dev/toolchain-bootstrap.md

@@ -1,152 +0,0 @@
-# Bootstrapping Compilation Toolchain for Memgraph
-
-Requirements:
-
-  * libstdc++ shipped with gcc-6.3 or gcc-6.4
-  * cmake >= 3.1, Debian Stretch uses cmake-3.7.2
-  * clang-3.9
-
-## Installing gcc-6.4
-
-gcc-6.3 has a bug, so use the 6.4 version which is just a bugfix release.
-
-Requirements on CentOS 7:
-
-  * wget
-  * make
-  * gcc (bootstrap)
-  * gcc-c++ (bootstrap)
-  * gmp-devel (bootstrap)
-  * mpfr-devel (bootstrap)
-  * libmpc-devel (bootstrap)
-  * zip
-  * perl
-  * dejagnu (testing)
-  * expect (testing)
-  * tcl (testing)
-
-```
-wget ftp://ftp.mpi-sb.mpg.de/pub/gnu/mirror/gcc.gnu.org/pub/gcc/releases/gcc-6.4.0/gcc-6.4.0.tar.gz
-tar xf gcc-6.4.0.tar.gz
-cd gcc-6.4.0
-mkdir build
-cd build
-../configure --disable-multilib --prefix=<install-dst>
-make
-# Testing
-make -k check
-make install
-```
-
-*Do not put gcc + libs on PATH* (unless you know what you are doing).
-
-## Installing cmake-3.7.2
-
-Requirements on CentOS 7:
-
-  * wget
-  * make
-  * gcc
-  * gcc-c++
-  * ncurses-devel (optional, for ccmake)
-
-```
-wget https://cmake.org/files/v3.7/cmake-3.7.2.tar.gz
-tar xf cmake-3.7.2.tar.gz
-cd cmake-3.7.2.tar.gz
-./bootstrap --prefix<install-dst>
-make
-make install
-```
-
-Put cmake on PATH (if appropriate)
-
-**Fix the bug in CpackRPM**
-
-`"<path-to-cmake>/share/cmake-3.7/Modules/CPackRPM.cmake" line 2273 of 2442`
-
-The line
-
-```
-set(RPMBUILD_FLAGS "-bb")
-```
-needs to be before
-
-```
-if(CPACK_RPM_GENERATE_USER_BINARY_SPECFILE_TEMPLATE OR NOT CPACK_RPM_USER_BINARY_SPECFILE)
-```
-
-It was probably accidentally placed after, and is fixed in later cmake
-releases.
-
-## Installing clang-3.9
-
-Requirements on CentOS 7:
-
-  * wget
-  * make
-  * cmake
-
-```
-wget http://releases.llvm.org/3.9.1/llvm-3.9.1.src.tar.xz
-tar xf llvm-3.9.1.src.tar.xz
-mv llvm-3.9.1.src llvm
-
-wget http://releases.llvm.org/3.9.1/cfe-3.9.1.src.tar.xz
-tar xf cfe-3.9.1.src.tar.xz
-mv cfe-3.9.1.src llvm/tools/clang
-
-cd llvm
-mkdir build
-cd build
-cmake -DCMAKE_BUILD_TYPE="Release" -DGCC_INSTALL_PREFIX=<gcc-dir> \
--DCMAKE_C_COMPILER=<gcc> -DCMAKE_CXX_COMPILER=<g++> \
--DCMAKE_CXX_LINK_FLAGS="-L<gcc-dir>/lib64 -Wl,-rpath,<gcc-dir>/lib64" \
--DCMAKE_INSTALL_PREFIX=<install-dst> ..
-make
-# Testing
-make check-clang
-make install
-```
-
-Put clang on PATH (if appropriate)
-
-## Memgraph
-
-Requirements on CentOS 7:
-
-  * libuuid-devel (antlr4)
-  * java-1.8.0-openjdk (antlr4)
-  * boost-static (too low version --- compile manually)
-  * rpm-build (RPM)
-  * python3 (tests, ...)
-  * which (required for rocksdb)
-  * sbcl (lisp C++ preprocessing)
-
-### Boost 1.62
-
-```
-wget https://netix.dl.sourceforge.net/project/boost/boost/1.62.0/boost_1_62_0.tar.gz
-tar xf boost_1_62_0.tar.gz
-cd boost_1_62_0
-./bootstrap.sh --with-toolset=clang --with-libraries=iostreams,serialization --prefix=<install-dst>
-./b2
-# Default installs to /usr/local/
-./b2 install
-```
-
-### Building Memgraph
-
-clang is *required* to be findable by cmake, i.e. it should be on PATH.
-cmake isn't required to be on the path, since you run it manually, so can use
-the full path to executable in order to run it. Obviously, it is convenient to
-put cmake also on PATH.
-
-Building is done as explained in [Quick Start](quick-start.md), but each
-`make` invocation needs to be prepended with:
-
-`LD_RUN_PATH=<gcc-dir>/lib64 make ...`
-
-### RPM
-
-Name format: `memgraph-<version>-<pkg-version>.<arch>.rpm`

docs/dev/workflow.md

@@ -1,177 +0,0 @@
-# Memgraph Workflow
-
-This chapter describes the usual workflow for working on Memgraph.
-
-## Git
-
-Memgraph uses [git](https://git-scm.com/) for source version control. If you
-obtained the source, you probably already have it installed. Before you can
-track new changes, you need to setup some basic information.
-
-First, tell git your name:
-
-    git config --global user.name "FirstName LastName"
-
-Then, set your Memgraph email:
-
-    git config --global user.email "my.email@memgraph.com"
-
-Finally, make git aware of your favourite editor:
-
-    git config --global core.editor "vim"
-
-## Phabricator
-
-All of the code in Memgraph needs to go through code review before it can be
-accepted in the codebase. This is done through
-[Phabricator](https://phacility.com/phabricator/). The command line tool for
-interfacing with Phabricator is
-[arcanist](https://phacility.com/phabricator/arcanist/). You should already
-have it installed if you followed the steps in [Quick Start](quick-start.md).
-
-The only required setup is to go in the root of Memgraph's project and run:
-
-    arc install-certificate
-
-## Working on Your Feature Branch
-
-Git has a concept of source code *branches*. The `master` branch contains all
-of the changes which were reviewed and accepted in Memgraph's code base. The
-`master` branch is selected by default.
-
-### Creating a Branch
-
-When working on a new feature or fixing a bug, you should create a new branch
-out of the `master` branch. For example, let's say you are adding static type
-checking to the query language compiler. You would create a branch called
-`mg_query_static_typing` with the following command:
-
-    git branch mg_query_static_typing
-
-To switch to that branch, type:
-
-    git checkout mg_query_static_typing
-
-Since doing these two steps will happen often, you can use a shortcut command:
-
-    git checkout -b mg_query_static_typing
-
-Note that a branch is created from the currently selected branch. So, if you
-wish to create another branch from `master` you need to switch to `master`
-first.
-
-The usual convention for naming your branches is `mg_<feature_name>`, you may
-switch underscores ('\_') for hyphens ('-').
-
-Do take care not to mix the case of your branch names! Certain operating
-systems (like Windows) don't distinguish the casing in git branches. This may
-cause hard to track down issues when trying to switch branches. Therefore, you
-should always name your branches with lowercase letters.
-
-### Making and Committing Changes
-
-When you have a branch for your new addition, you can now actually start
-implementing it. After some amount of time, you may have created new files,
-modified others and maybe even deleted unused files. You need to tell git to
-track those changes. This is accomplished with `git add` and `git rm`
-commands.
-
-    git add path-to-new-file path-to-modified-file
-    git rm path-to-deleted-file
-
-To check that everything is correctly tracked, you may use the `git status`
-command. It will also print the name of the currently selected branch.
-
-If everything seems OK, you should commit these changes to git.
-
-    git commit
-
-You will be presented with an editor where you need to type the commit
-message. Writing a good commit message is an art in itself. You should take a
-look at the links below. We try to follow these conventions as much as
-possible.
-
-  * [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/)
-  * [A Note About Git Commit Messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html)
-  * [stopwritingramblingcommitmessages](http://stopwritingramblingcommitmessages.com/)
-
-### Sending Changes on a Review
-
-After finishing your work on your feature branch, you will want to send it on
-code review. This is done through Arcanist. To do that, run the following
-command:
-
-    arc diff
-
-You will, once again, be presented with an editor where you need to describe
-your whole work. `arc` will by default fill that description with your commit
-messages. The title and summary of your work should also follow the
-conventions of git messages as described above. If you followed the
-guidelines, the message filled by `arc` should be fine.
-
-In addition to the message, you need to fill the `Reviewers:` line with
-usernames of people who should do the code review.
-
-You changes will be visible on Phabricator as a so called "diff". You can find
-the default view of active diffs
-[here](https://phabricator.memgraph.io/differential/)
-
-### Updating Changes Based on Review
-
-When you get comments in the code review, you will want to make additional
-modifications to your work. The same workflow as before applies: [Making and
-Committing Changes](#making-and-committing-changes)
-
-After making those changes, send them back on code review:
-
-    arc diff
-
-
-### Updating From New Master
-
-Let's say that, while you were working, someone else added some new features
-to the codebase that you would like to use in your current work. To obtain
-those changes you should update your `master` branch:
-
-    git checkout master
-    git pull origin master
-
-Now, these changes are on `master`, but you want them in your local branch. To
-do that, use `git rebase`:
-
-    git checkout mg_query_static_typing
-    git rebase master
-
-During `git rebase`, you may get reports that some files have conflicting
-changes. If you need help resolving them, don't be afraid to ask around! After
-you've resolved them, mark them as done with `git add` command. You may
-then continue with `git rebase --continue`.
-
-After the `git rebase` is done, you will now have new changes from `master` on
-your feature branch as if you just created and started working on that branch.
-You may continue with the usual workflow of [Making and Committing
-Changes](#making-and-committing-changes) and [Sending Changes on a
-Review](#sending-changes-on-a-review).
-
-### Sending Your Changes on Master Branch
-
-When your changes pass the code review, you are ready to integrate them in the
-`master` branch. To do that, run the following command:
-
-    arc land
-
-Arcanist will take care of obtaining the latest changes from `master` and
-merging your changes on top. If the `land` was successful, Arcanist will
-delete your local branch and you will be back on `master`. Continuing from the
-examples above, the deleted branch would be `mg_query_static_typing`.
-
-This marks the completion of your changes, and you are ready to work on
-something else.
-
-### Note For People Familiar With Git
-
-Since Arcanist takes care of merging your git commits and pushing them on
-`master`, you should *never* have to call `git merge` and `git push`. If you
-find yourself typing those commands, check that you are doing the right thing.
-The most common mistake is to use `git merge` instead of `git rebase` for the
-case described in [Updating From New Master](#updating-from-new-master).

docs/doxygen/README.md

@@ -1,7 +1,6 @@
 # Memgraph Code Documentation
 
-IMPORTANT: auto-generated (run doxygen Doxyfile in the project root)
+IMPORTANT: Auto-generated (run doxygen Doxyfile in the project root).
 
-* HTML - just open docs/doxygen/html/index.html
-
-* Latex - run make inside docs/doxygen/latex
+* HTML - Open docs/doxygen/html/index.html.
+* Latex - Run make inside docs/doxygen/latex.

docs/feature_spec/distributed.md

@@ -1,78 +0,0 @@
-# Distributed Memgraph specs
-This document describes reasnonings behind Memgraphs distributed concepts.
-
-## Distributed state machine
-Memgraphs distributed mode introduces two states of the cluster, recovering and
-working. The change between states shouldn't happen often, but when it happens
-it can take a while to make a transition from one to another.
-
-### Recovering
-This state is the default state for Memgraph when the cluster starts with
-recovery flags. If the recovery finishes successfully, the state changes to
-working. If recovery fails, the user will be presented with a message that
-explains what happened and what are the next steps.
-
-Another way to enter this state is failure. If the cluster encounters a failure,
-the master will enter the Recovering mode. This time, it will wait for all
-workers to respond with a message saying they are alive and well, and making
-sure they all have consistent state.
-
-### Working
-This state should be the default state of Memgraph most of the time. When in
-this state, Memgraph accepts connections from Bolt clients and allows query
-execution.
-
-If distributed execution fails for a transaction, that transaction, and all
-other active transactions will be aborted and the cluster will enter the
-Recovering state.
-
-## Durability
-One of the important concepts in distributed Memgraph is durability.
-
-### Cluster configuration
-When running Memgraph in distributed mode, the master will store cluster
-metadata in a persistent store. If fore some reason the cluster shuts down,
-recovering Memgraph from durability files shouldn't require any additional
-flags.
-
-### Database ID
-Each new and clean run of Memgraph should generate a new globally unique
-database id.  This id will associate all files that have persisted with this
-run. Adding the database id to snapshots, write-ahead logs and cluster metadata
-files ties them a specific Memgraph run, and it makes recovery easier to reason
-about.
-
-When recovering, the cluster won't generate a new id, but will reuse the one
-from the snapshot/wal that it was able to recover from.
-
-### Durability files
-Memgraph uses snapshots and write-ahead logs for durability.
-
-When Memgraph recovers it has to make sure all machines in the cluster recover
-to the same recovery point. This is done by finding a common snapshot and
-finding common transactions in per-machine available write-ahead logs.
-
-Since we can not be sure that each machine persisted durability files, we need
-to be able to negotiate a common recovery point in the cluster. Possible
-durability file failures could require to start the cluster from scratch,
-purging everything from storage and recovering from existing durability files.
-
-We need to ensure that we keep wal files containing information about
-transactions between all existing snapshots. This will provide better durability
-in the case of a random machine durability file failure, where the cluster can
-find a common recovery point that all machines in the cluster have.
-
-Also, we should suggest and make clear docs that anything less than two
-snapshots isn't considered safe for recovery.
-
-### Recovery
-The recovery happens in following steps:
-* Master enables worker registration.
-* Master recovers cluster metadata from the persisted storage.
-* Master waits all required workers to register.
-* Master broadcasts a recovery request to all workers.
-* Workers respond with with a set of possible recovery points.
-* Master finds a common recovery point for the whole cluster.
-* Master broadcasts a recovery request with the common recovery point.
-* Master waits for the cluster to recover.
-* After a successful cluster recovery, master can enter Working state.

docs/feature_spec/dynamic_graph_partitioning.md

@@ -1,75 +0,0 @@
-# Dynamic Graph Partitioning (abbr. DGP)
-
-## Implementation
-
-Take a look under `dev/memgraph/distributed/dynamic_graph_partitioning.md`.
-
-### Implemented parameters
-
---dynamic-graph-partitioner-enabled (If the dynamic graph partitioner should be
-  enabled.) type: bool default: false (start time)
---dgp-improvement-threshold (How much better should specific node score be
-  to consider a migration to another worker. This represents the minimal
-  difference between new score that the vertex will have when migrated
-  and the old one such that it's migrated.) type: int32 default: 10
-  (start time)
---dgp-max-batch-size (Maximal amount of vertices which should be migrated
-  in one dynamic graph partitioner step.) type: int32 default: 2000
-  (start time)
-
-## Planning
-
-### Design decisions
-
-* Each partitioning session has to be a new transaction.
-* When and how does an instance perform the moves?
-  * Periodically.
-  * Token sharing (round robin, exactly one instance at a time has an
-    opportunity to perform the moves).
-* On server-side serialization error (when DGP receives an error).
-  -> Quit partitioning and wait for the next turn.
-* On client-side serialization error (when end client receives an error).
-  -> The client should never receive an error because of any
-     internal operation.
-  -> For the first implementation, it's good enough to wait until data becomes
-     available again.
-  -> It would be nice to achieve that DGP has lower priority than end client
-     operations.
-
-### End-user parameters
-
-* --dynamic-graph-partitioner-enabled (execution time)
-* --dgp-improvement-threshold (execution time)
-* --dgp-max-batch-size (execution time)
-* --dgp-min-batch-size (execution time)
-  -> Minimum number of nodes that will be moved in each step.
-* --dgp-fitness-threshold (execution time)
-  -> Do not perform moves if partitioning is good enough.
-* --dgp-delta-turn-time (execution time)
-  -> Time between each turn.
-* --dgp-delta-step-time (execution time)
-  -> Time between each step.
-* --dgp-step-time (execution time)
-  -> Time limit per each step.
-
-### Testing
-
-The implementation has to provide good enough results in terms of:
-  * How good the partitioning is (numeric value), aka goodness.
-  * Workload execution time.
-  * Stress test correctness.
-
-Test cases:
-  * N not connected subgraphs
-    -> shuffle nodes to N instances
-    -> run partitioning
-    -> test perfect partitioning.
-  * N connected subgraph
-    -> shuffle nodes to N instance
-    -> run partitioning
-    -> test partitioning.
-  * Take realistic workload (Long Running, LDBC1, LDBC2, Card Fraud, BFS, WSP)
-    -> measure exec time
-    -> run partitioning
-    -> test partitioning
-    -> measure exec time (during and after partitioning).

docs/feature_spec/high_availability.md

@@ -1,275 +0,0 @@
-# High Availability (abbr. HA)
-
-## High Level Context
-
-High availability is a characteristic of a system which aims to ensure a
-certain level of operational performance for a higher-than-normal period.
-Although there are multiple ways to design highly available systems, Memgraph
-strives to achieve HA by elimination of single points of failure. In essence,
-this implies adding redundancy to the system so that a failure of a component
-does not imply the failure of the entire system. To ensure this, HA Memgraph
-implements the [Raft consensus algorithm](https://raft.github.io/).
-
-Correct implementation of the algorithm guarantees that the cluster will be
-fully functional (available) as long as any strong majority of the servers are
-operational and can communicate with each other and with clients. For example,
-clusters of three or four machines can tolerate the failure of a single server,
-clusters of five and six machines can tolerate the failure of any two servers,
-and so on. Therefore, we strongly recommend a setup of an odd-sized cluster.
-
-### Performance Implications
-
-Internally, Raft achieves high availability by keeping a consistent replicated
-log on each server within the cluster. Therefore, we must successfully replicate
-a transaction on the majority of servers within the cluster before we actually
-commit it and report the result back to the client. This operation represents
-a significant performance hit when compared with single node version of
-Memgraph.
-
-Luckily, the algorithm can be tweaked in a way which allows read-only
-transactions to perform significantly better than those which modify the
-database state. That being said, the performance of read-only operations
-is still not going to be on par with single node Memgraph.
-
-This section will be updated with exact numbers once we integrate HA with
-new storage.
-
-With the old storage, write throughput was almost five times lower than read
-throughput (~30000 reads per second vs ~6000 writes per second).
-
-## User Facing Setup
-
-### How to Setup HA Memgraph Cluster?
-
-First, the user needs to install `memgraph_ha` package on each machine
-in their cluster. HA Memgraph should be available as a Debian package,
-so its installation on each machine should be as simple as:
-
-```plaintext
-dpkg -i /path/to/memgraph_ha_<version>.deb
-```
-
-After successful installation of the `memgraph_ha` package, the user should
-finish its configuration before attempting to start the cluster.
-
-There are two main things that need to be configured on every node in order for
-the cluster to be able to run:
-
-1. The user has to edit the main configuration file and specify the unique node
-   ID to each server in the cluster
-2. The user has to create a file that describes all IP addresses of all servers
-   that will be used in the cluster
-
-The `memgraph_ha` binary loads all main configuration parameters from
-`/etc/memgraph/memgraph_ha.conf`. On each node of the cluster, the user should
-uncomment the `--server-id=0` parameter and change its value to the `server_id`
-of that node.
-
-The last step before starting the server is to create a `coordination`
-configuration file. That file is already present as an example in
-`/etc/memgraph/coordination.json.example` and you have to copy it to
-`/etc/memgraph/coordination.json` and edit it according to your cluster
-configuration. The file contains coordination info consisting of a list of
-`server_id`, `ip_address` and `rpc_port` lists. The assumed contents of the
-`coordination.json` file are:
-
-```plaintext
-[
-  [1, "192.168.0.1", 10000],
-  [2, "192.168.0.2", 10000],
-  [3, "192.168.0.3", 10000]
-]
-```
-Here, each line corresponds to coordination of one server. The first entry is
-that server's ID, the second is its IP address and the third is the RPC port it
-listens to. This port should not be confused with the port used for client
-interaction via the Bolt protocol.
-
-The `ip_address` entered for each `server_id` *must* match the exact IP address
-that belongs to that server and that will be used to communicate to other nodes
-in the cluster. The coordination configuration file *must* be identical on all
-nodes in the cluster.
-
-After the user has set the `server_id` on each node in
-`/etc/memgraph/memgraph_ha.conf` and provided the same
-`/etc/memgraph/coordination.json` file to each node in the cluster, they can
-start the Memgraph HA service by issuing the following command on each node in
-the cluster:
-
-```plaintext
-systemctl start memgraph_ha
-```
-
-### How to Configure Raft Parameters?
-
-All Raft configuration parameters can be controlled by modifying
-`/etc/memgraph/raft.json`.  The assumed contents of the `raft.json` file are:
-
-```plaintext
-{
-  "election_timeout_min": 750,
-  "election_timeout_max": 1000,
-  "heartbeat_interval": 100,
-  "replication_timeout": 20000,
-  "log_size_snapshot_threshold": 50000
-}
-```
-
-The meaning behind each entry is demystified in the following table:
-
-Flag                          | Description
-------------------------------|------------
-`election_timeout_min`        | Lower bound for the randomly sampled reelection timer given in milliseconds
-`election_timeout_max`        | Upper bound for the randomly sampled reelection timer given in milliseconds
-`heartbeat_interval`          | Time interval between consecutive heartbeats given in milliseconds
-`replication_timeout`         | Time interval allowed for data replication given in milliseconds
-`log_size_snapshot_threshold` | Allowed number of entries in Raft log before its compaction
-
-### How to Query HA Memgraph via Proxy?
-
-This chapter describes how to query HA Memgraph using our proxy server.
-Note that this is not intended to be a long-term solution. Instead, we will
-implement a proper Memgraph HA client which is capable of communicating with
-the HA cluster. Once our own client is implemented, it will no longer be
-possible to query HA Memgraph using other clients (such as neo4j client).
-
-The Bolt protocol that is exposed by each Memgraph HA node is an extended
-version of the standard Bolt protocol. In order to be able to communicate with
-the highly available cluster of Memgraph HA nodes, the client must have some
-logic implemented in itself so that it can communicate correctly with all nodes
-in the cluster. To facilitate a faster start with the HA cluster we will build
-the Memgraph HA proxy binary that communicates with all nodes in the HA cluster
-using the extended Bolt protocol and itself exposes a standard Bolt protocol to
-the user. All standard Bolt clients (libraries and custom systems) can
-communicate with the Memgraph HA proxy without any code modifications.
-
-The HA proxy should be deployed on each client machine that is used to
-communicate with the cluster. It can't be deployed on the Memgraph HA nodes!
-
-When using the Memgraph HA proxy, the communication flow is described in the
-following diagram:
-
-```plaintext
-Memgraph HA node 1 -----+
-                        |
-Memgraph HA node 2 -----+ Memgraph HA proxy <---> any standard Bolt client (C, Java, PHP, Python, etc.)
-                        |
-Memgraph HA node 3 -----+
-```
-
-To setup the Memgraph HA proxy the user should install the `memgraph_ha_proxy`
-package.
-
-After its successful installation, the user should enter all endpoints of the
-HA Memgraph cluster servers into the configuration before attempting to start
-the HA Memgraph proxy server.
-
-The HA Memgraph proxy server loads all of its configuration from
-`/etc/memgraph/memgraph_ha_proxy.conf`. Assuming that the cluster is set up
-like in the previous examples, the user should uncomment and enter the following
-value into the `--endpoints` parameter:
-
-```plaintext
---endpoints=192.168.0.1:7687,192.168.0.2:7687,192.168.0.3:7687
-```
-
-Note that the IP addresses used in the example match the individual cluster
-nodes IP addresses, but the ports used are the Bolt server ports exposed by
-each node (currently the default value of `7687`).
-
-The user can now start the proxy by using the following command:
-
-```plaintext
-systemctl start memgraph_ha_proxy
-```
-
-After the proxy has been started, the user can query the HA cluster by
-connecting to the HA Memgraph proxy IP address using their favorite Bolt
-client.
-
-## Integration with Memgraph
-
-The first thing that should be defined is a single instruction within the
-context of Raft (i.e. a single entry in a replicated log).
-These instructions should be completely deterministic when applied
-to the state machine. We have therefore decided that the appropriate level
-of abstraction within Memgraph corresponds to `Delta`s (data structures
-which describe a single change to the Memgraph state, used for durability
-in WAL). Moreover, a single instruction in a replicated log will consist of a
-batch of `Delta`s which correspond to a single transaction that's about
-to be **committed**.
-
-Apart from `Delta`s, there are certain operations within the storage called
-`StorageGlobalOperations` which do not conform to usual transactional workflow
-(e.g. Creating indices).  Since our storage engine implementation guarantees
-that at the moment of their execution no other transactions are active, we can
-safely replicate them as well. In other words, no additional logic needs to be
-implemented because of them.
-
-Therefore, we will introduce a new `RaftDelta` object which can be constructed
-both from storage `Delta` and `StorageGlobalOperation`. Instead of appending
-these to WAL (as we do in single node), we will start to replicate them across
-our cluster. Once we have replicated the corresponding Raft log entry on
-majority of the cluster, we are able to safely commit the transaction or execute
-a global operation. If for any reason the replication fails (leadership change,
-worker failures, etc.) the transaction will be aborted.
-
-In the follower mode, we need to be able to apply `RaftDelta`s we got from
-the leader when the protocol allows us to do so. In that case, we will use the
-same concepts from durability in storage v2, i.e., applying deltas maps
-completely to recovery from WAL in storage v2.
-
-## Test and Benchmark Strategy
-
-We have already implemented some integration and stress tests. These are:
-
-1. leader election -- Tests whether leader election works properly.
-2. basic test -- Tests basic leader election and log replication.
-3. term updates test -- Tests a specific corner case (which used to fail)
-                        regarding term updates.
-4. log compaction test -- Tests whether log compaction works properly.
-5. large log entries -- Tests whether we can successfully replicate relatively
-                        large log entries.
-6. index test -- Tests whether index creation works in HA.
-7. normal operation stress test -- Long running concurrent stress test under
-                                   normal conditions (no failures).
-8. read benchmark -- Measures read throughput in HA.
-9. write benchmark -- Measures write throughput in HA.
-
-At the moment, our main goal is to pass existing tests and have a stable version
-on our stress test. We should also implement a stress test which occasionally
-introduces different types of failures in our cluster (we did this kind of
-testing manually thus far). Passing these tests should convince us that we have
-a "stable enough" version which we can start pushing to our customers.
-
-Additional (proper) testing should probably involve some ideas from
-[here](https://jepsen.io/analyses/dgraph-1-0-2)
-
-## Possible Future Changes/Improvements/Extensions
-
-There are two general directions in which we can alter HA Memgraph. The first
-direction assumes we are going to stick with the Raft protocol. In that case
-there are a few known ways to extend the basic algorithm in order to gain
-better performance or achieve extra functionality. In no particular order,
-these are:
-
-1. Improving read performance using leader leases [Section 6.4 from Raft thesis]
-2. Introducing cluster membership changes [Chapter 4 from Raft thesis]
-3. Introducing a [learner mode](https://etcd.io/docs/v3.3.12/learning/learner/).
-4. Consider different log compaction strategies [Chapter 5 from Raft thesis]
-5. Removing HA proxy and implementing our own HA Memgraph client.
-
-On the other hand, we might decide in the future to base our HA implementation
-on a completely different protocol which might even offer different guarantees.
-In that case we probably need to do a bit more of market research and weigh the
-trade-offs of different solutions.
-[This](https://www.postgresql.org/docs/9.5/different-replication-solutions.html)
-might be a good starting point.
-
-## Reading materials
-
-1. [Raft paper](https://raft.github.io/raft.pdf)
-2. [Raft thesis](https://github.com/ongardie/dissertation) (book.pdf)
-3. [Raft playground](https://raft.github.io/)
-4. [Leader Leases](https://blog.yugabyte.com/low-latency-reads-in-geo-distributed-sql-with-raft-leader-leases/)
-5. [Improving Raft ETH](https://pub.tik.ee.ethz.ch/students/2017-FS/SA-2017-80.pdf)

docs/feature_spec/kafka/opencypher.md

@@ -1,80 +0,0 @@
-# Kafka - openCypher clause
-
-One must be able to specify the following when importing data from Kafka:
-
-* Kafka URI
-* Kafka topic
-* Transform [script](transform.md) URI
-
-
-Minimum required syntax looks like:
-```opencypher
-CREATE STREAM stream_name AS LOAD DATA KAFKA 'URI'
-  WITH TOPIC 'topic'
-  WITH TRANSFORM 'URI';
-```
-
-
-The full openCypher clause for creating a stream is:
-```opencypher
-CREATE STREAM stream_name AS
-  LOAD DATA KAFKA 'URI'
-  WITH TOPIC 'topic'
-  WITH TRANSFORM 'URI'
-  [BATCH_INTERVAL milliseconds]
-  [BATCH_SIZE count]
-```
-The `CREATE STREAM` clause happens in a transaction.
-
-`WITH TOPIC` parameter specifies the Kafka topic from which we'll stream
-data.
-
-`WITH TRANSFORM` parameter should contain a URI of the transform script.
-
-`BATCH_INTERVAL` parameter defines the time interval in milliseconds
-which is the time between two successive stream importing operations.
-
-`BATCH_SIZE` parameter defines the count of Kafka messages that will be
-batched together before import.
-
-If both `BATCH_INTERVAL` and `BATCH_SIZE` parameters are given, the condition
-that is satisfied first will trigger the batched import.
-
-Default value for `BATCH_INTERVAL` is 100 milliseconds, and the default value
-for `BATCH_SIZE` is 10;
-
-The `DROP` clause deletes a stream:
-```opencypher
-DROP STREAM stream_name;
-```
-
-The `SHOW` clause enables you to see all configured streams:
-```opencypher
-SHOW STREAMS;
-```
-
-You can also start/stop streams with the `START` and `STOP` clauses:
-```opencypher
-START STREAM stream_name [LIMIT count BATCHES];
-STOP STREAM stream_name;
-```
-A stream needs to be stopped in order to start it and it needs to be started in
-order to stop it. Starting a started or stopping a stopped stream will not
-affect that stream.
-
-There are also convenience clauses to start and stop all streams:
-```opencypher
-START ALL STREAMS;
-STOP ALL STREAMS;
-```
-
-Before the actual import, you can also test the stream with the `TEST
-STREAM` clause:
-```opencypher
-TEST STREAM stream_name [LIMIT count BATCHES];
-```
-When a stream is tested, data extraction and transformation occurs, but no
-output is inserted in the graph.
-
-A stream needs to be stopped in order to test it. When the batch limit is
-omitted, `TEST STREAM` will run for only one batch by default.

docs/feature_spec/kafka/transform.md

@@ -1,34 +0,0 @@
-# Kafka - data transform
-
-The transform script is a user defined script written in Python.  The script
-should be aware of the data format in the Kafka message.
-
-Each Kafka message is byte length encoded, which means that the first eight
-bytes of each message contain the length of the message.
-
-A sample code for a streaming transform script could look like this:
-
-```python
-def create_vertex(vertex_id):
-  return ("CREATE (:Node {id: $id})", {"id": vertex_id})
-
-
-def create_edge(from_id, to_id):
-  return ("MATCH (n:Node {id: $from_id}), (m:Node {id: $to_id}) "\
-          "CREATE (n)-[:Edge]->(m)", {"from_id": from_id, "to_id": to_id})
-
-
-def stream(batch):
-    result = []
-    for item in batch:
-        message = item.decode('utf-8').strip().split()
-        if len(message) == 1:
-          result.append(create_vertex(message[0])))
-        else:
-          result.append(create_edge(message[0], message[1]))
-    return result
-
-```
-
-The script should output openCypher query strings based on the type of the
-records.

docs/feature_spec/python-query-modules.md

@@ -1,185 +0,0 @@
-# Python 3 Query Modules
-
-## Introduction
-
-Memgraph exposes a C API for writing the so called Query Modules. These
-modules contain definitions of procedures which can be invoked through the
-query language using the `CALL ... YIELD ...` syntax. This mechanism allows
-database users to extend Memgraph with their own algorithms and
-functionalities.
-
-Using a low level language like C can be quite cumbersome for writing modules,
-so it seems natural to add support for a higher level language on top of the
-existing C API.
-
-There are languages written exactly for this purpose of extending C with high
-level constructs, for example Lua and Guile. Instead of those, we have chosen
-Python 3 to be the first high level language we will support. The primary reason
-being that it's very popular, so more people should be able to write modules.
-Another benefit of Python which comes out of its popularity is the large
-ecosystem of libraries, especially graph algorithm related ones like NetworkX.
-Python does have significant performance and implementation downsides compared
-to Lua and Guile, but these are described in more detail later in this
-document.
-
-## Python 3 API Overview
-
-The Python 3 API should be as user friendly as possible as well as look
-Pythonic. This implies that some functions from the C API will not map to the
-exact same functions. The most obvious case for a Pythonic approach is
-registering procedures of a query module. Let's take a look at the C example
-and its transformation to Python.
-
-```c
-static void procedure(const struct mgp_list *args,
-                      const struct mgp_graph *graph, struct mgp_result *result,
-                      struct mgp_memory *memory);
-
-int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
-  struct mgp_proc *proc =
-      mgp_module_add_read_procedure(module, "procedure", procedure);
-  if (!proc) return 1;
-  if (!mgp_proc_add_arg(proc, "required_arg",
-                        mgp_type_nullable(mgp_type_any())))
-    return 1;
-  struct mgp_value *null_value = mgp_value_make_null(memory);
-  if (!mgp_proc_add_opt_arg(proc, "optional_arg",
-                            mgp_type_nullable(mgp_type_any()), null_value)) {
-    mgp_value_destroy(null_value);
-    return 1;
-  }
-  mgp_value_destroy(null_value);
-  if (!mgp_proc_add_result(proc, "result", mgp_type_string())) return 1;
-  if (!mgp_proc_add_result(proc, "args",
-                           mgp_type_list(mgp_type_nullable(mgp_type_any()))))
-    return 1;
-  return 0;
-}
-```
-
-In Python things should be a lot simpler.
-
-```Python
-# mgp.read_proc obtains the procedure name via __name__ attribute of a function.
-@mgp.read_proc(# Arguments passed to multiple mgp_proc_add_arg calls
-               (('required_arg', mgp.Nullable(mgp.Any)), ('optional_arg', mgp.Nullable(mgp.Any), None)),
-               # Result fields passed to multiple mgp_proc_add_result calls
-               (('result', str), ('args', mgp.List(mgp.Nullable(mgp.Any)))))
-def procedure(args, graph, result, memory):
-    pass
-```
-
-Here we have replaced `mgp_module_*` and `mgp_proc_*` C API with a much
-simpler decorator function in Python -- `mgp.read_proc`. The types of
-arguments and result fields can both be our types as well as Python builtin
-types which can map to supported `mgp_value` types. The expected builtin types
-we ought to support are: `bool`, `str`, `int`, `float` and `map`. While the
-rest of the types are provided via our Python API. Optionally, we can add
-convenience support for `object` type which would map to
-`mgp.Nullable(mgp.Any)` and `list` which would map to
-`mgp.List(mgp.Nullable(mgp.Any))`. Also, it makes sense to take a look if we
-can leverage Python's `typing` module here.
-
-Another Pythonic change is to remove `mgp_value` C API from Python altogether.
-This means that the arguments a Python procedure receives are not `mgp_value`
-instances but rather `PyObject` instances. In other words, our implementation
-would immediately marshal `mgp_value` to corresponding type in Python.
-Obviously we would need to provide our own Python types for non-builtin
-things like `mgp.Vertex` (equivalent to `mgp_vertex`) and other.
-
-Continuing from our example above, let's say the procedure was invoked through
-Cypher using the following query.
-
-    MATCH (n) CALL py_module.procedure(42, n) YIELD *;
-
-The Python procedure could then do the following and complete without throwing
-neither the AssertionError nor the ValueError.
-
-```Python
-def procedure(args, graph, result, memory):
-    assert isinstance(args, list)
-    # Unpacking throws ValueError if args does not contain exactly 2 values.
-    required_arg, optional_arg = args
-    assert isintance(required_arg, int)
-    assert isinstance(optional_arg, mgp.Vertex)
-```
-
-The rest of the C API should naturally map to either top level functions or
-class methods as appropriate.
-
-## Loading Python Query Modules
-
-Our current mechanism for loading the modules is to look for `.so` files in
-the directory specified by `--query-modules` flag. This is done when Memgraph
-is started. We can extend this mechanism to look for `.py` files in addition
-to `.so` files in the same directory and import them in the embedded Python
-interpreter. The only issue is embedding the interpreter in Memgraph.  There
-are multiple choices:
-
-  1. Building Memgraph and statically linking to Python.
-  2. Building Memgraph and dynamically linking to Python, and distributing
-     Python with Memgraph's installation.
-  3. Building Memgraph and dynamically linking to Python, but without
-     distributing the Python library.
-  4. Building Memgraph and optionally loading Python library by trying to
-     `dlopen` it.
-
-The first two options are only viable if the Python license allows, and this
-will need further investigation.
-
-The third option adds Python as an installation dependency for Memgraph, and
-without it Memgraph will not run. This is problematic for users which cannot
-or do not want to install Python 3.
-
-The fourth option avoids all of the issues present in the first 3 options, but
-comes at a higher implementation cost. We would need to try to `dlopen` the
-Python library and setup function pointers. If we succeed we would import
-`.py` files from the `--query-modules` directory. On the other hand, if the
-user does not have Python, `dlopen` would fail and Memgraph would run without
-Python support.
-
-After live discussion, we've decided to go with option 3. This way we don't
-have to worry about mismatching Python versions we support and what the users
-expect. Also, we should target Python 3.5 as that should be common between
-Debian and CentOS for which we ship installation packages.
-
-## Performance and Implementation Problems
-
-As previously mentioned, embedding Python introduces usability issues compared
-to other embeddable languages.
-
-The first, major issue is Global Interpreter Lock (GIL). Initializing Python
-will start a single global interpreter and running multiple threads will
-require acquiring GIL. In practice, this means that when multiple users run a
-procedure written in Python in parallel the execution will not actually be
-parallel. Python's interpreter will jump between executing one user's
-procedure and the other's. This can be quite an issue for long running
-procedures when multiple users are querying Memgraph. The solution for this
-issue is Python's API for sub-interpreters. Unfortunately, the support for
-them is rather poor and the API contains a lot of critical bugs when we tried
-to use them. For the time being, we will have to accept GIL and its downsides.
-Perhaps in the future we will gain more knowledge on how we could reduce the
-acquire rate of GIL or the sub-interpreter API will get improved.
-
-Another major issue is memory allocation. Python's C API does not have support
-for setting up a temporary allocator during execution of a single function.
-It only has support for setting up a global heap allocator. This obviously
-impacts our control of memory during a query procedure invocation. Besides
-potential performance penalty, a procedure could allocate much more memory
-than we would actually allow for execution of a single query. This means that
-options controlling the memory limit during query execution are useless. On
-the bright side, Python does use block style allocators and reference
-counting, so the performance penalty and global memory usage should not be
-that terrible.
-
-The final issue that isn't as major as the ones above is the global state of
-the interpreter. In practice this means that any registered procedure and
-imported module has access to any other procedure and module. This may pollute
-the namespace for other users, but it should not be much of a problem because
-Python always has things under a module scope. The other, slightly bigger
-downside is that a malicious user could use this knowledge to modify other
-modules and procedures. This seems like a major issue, but if we take the
-bigger picture into consideration, we already have a security issue in general
-by invoking `dlopen` on `.so` and potentially running arbitrary code. This was
-the trade off we chose to allow users to extend Memgraph. It's up to the users
-to write sane extensions and protect their servers from access.

docs/feature_spec/tensorflow_op/technicalities.md

@@ -1,61 +0,0 @@
-# Tensorflow Op - Technicalities
-
-The final result should be a shared object (".so") file that can be
-dynamically loaded by the Tensorflow runtime in order to directly
-access the bolt client.
-
-## About Tensorflow
-
-Tensorflow is usually used with Python such that the Python code is used
-to define a directed acyclic computation graph. Basically no computation
-is done in Python. Instead, values from Python are copied into the graph
-structure as constants to be used by other Ops. The directed acyclic graph
-naturally ends up with two sets of border nodes, one for inputs, one for
-outputs. These are sometimes called "feeds".
-
-Following the Python definition of the graph, during training, the entire
-data processing graph/pipeline is called from Python as a single expression.
-This leads to lazy evaluation since the called result has already been
-defined for a while.
-
-Tensorflow internally works with tensors, i.e. n-dimensional arrays. That
-means all of its inputs need to be matrices as well as its outputs. While
-it is possible to feed data directly from Python's numpy matrices straight
-into Tensorflow, this is less desirable than using the Tensorflow data API
-(which defines data input and processing as a Tensorflow graph) because:
-
-  1. The data API is written in C++ and entirely avoids Python and as such
-  is faster
-  2. The data API, unlike Python is available in "Tensorflow serving". The
-  default way to serve Tensorflow models in production.
-
-Once the entire input pipeline is defined via the tf.data API, its input
-is basically a list of node IDs the model is supposed to work with. The
-model, through the data API knows how to connect to Memgraph and execute
-openCypher queries in order to get the remaining data it needs.
-(For example features of neighbouring nodes.)
-
-## The Interface
-
-I think it's best you read the official guide...  
-<https://www.tensorflow.org/extend/adding_an_op>  
-And especially the addition that specifies how data ops are special  
-<https://www.tensorflow.org/extend/new_data_formats>  
-
-## Compiling the TF Op
-
-There are two options for compiling a custom op.  
-One of them involves pulling the TF source, adding your code to it and
-compiling via bazel.  
-This is probably awkward to do for us and would
-significantly slow down compilation.  
-
-The other method involves installing Tensorflow as a Python package and
-pulling the required headers from for example:  
-`/usr/local/lib/python3.6/site-packages/tensorflow/include`  
-We can then compile our Op with our regular build system.
-
-This is practical since we can copy the required headers to our repo.
-If necessary, we can have several versions of the headers to build several
-versions of our Op for every TF version which we want to support.
-(But this is unlikely to be required as the API should be stable).

docs/feature_spec/tensorflow_op/usage_example.md

@@ -1,142 +0,0 @@
-# Example for Using the Bolt Client Tensorflow Op
-
-## Dynamic Loading
-
-``` python3
-import tensorflow as tf
-
-mg_ops = tf.load_op_library('/usr/bin/memgraph/tensorflow_ops.so')
-```
-
-## Basic Usage
-
-``` python3
-dataset = mg_ops.OpenCypherDataset(
-            # This is probably unfortunate as the username and password
-            # get hardcoded into the graph, but for the simple case it's fine
-            "hostname:7687", auth=("user", "pass"),
-
-            # Our query
-            '''
-            MATCH (n:Train) RETURN n.id, n.features
-            ''',
-
-            # Cast return values to these types
-            (tf.string, tf.float32))
-
-# Some Tensorflow data api boilerplate
-iterator = dataset.make_one_shot_iterator()
-next_element = iterator.get_next()
-
-# Up to now we have only defined our computation graph which basically
-# just connects to Memgraph
-# `next_element` is not really data but a handle to a node in the Tensorflow
-# graph, which we can and do evaluate
-# It is a Tensorflow tensor with shape=(None, 2)
-# and dtype=(tf.string, tf.float)
-# shape `None` means the shape of the tensor is unknown at definition time
-# and is dynamic and will only be known once the tensor has been evaluated
-
-with tf.Session() as sess:
-    node_ids = sess.run(next_element)
-    # `node_ids` contains IDs and features of all the nodes 
-    # in the graph with the label "Train"
-    # It is a numpy.ndarray with a shape ($n_matching_nodes, 2)
-```
-
-## Memgraph Client as a Generic Tensorflow Op
-
-Other than the Tensorflow Data Op, we'll want to support a generic Tensorflow
-Op which can be put anywhere in the Tensorflow computation Graph. It takes in
-an arbitrary tensor and produces a tensor. This would be used in the GraphSage
-algorithm to fetch the lowest level features into Tensorflow
-
-```python3
-requested_ids = np.array([1, 2, 3])
-ids_placeholder = tf.placeholder(tf.int32)
-
-model = mg_ops.OpenCypher()
-    "hostname:7687", auth=("user", "pass"),
-    """
-    UNWIND $node_ids as nid
-    MATCH (n:Train {id: nid})
-    RETURN n.features
-    """,
-
-    # What to call the input tensor as an openCypher parameter
-    parameter_name="node_ids",
-
-    # Type of our resulting tensor
-    dtype=(tf.float32)
-)
-
-features = model(ids_placeholder)
-
-with tf.Session() as sess:
-    result = sess.run(features,
-                    feed_dict={ids_placeholder: requested_ids})
-```
-
-This is probably easier to implement than the Data Op, so it might be a good
-idea to start with.
-
-## Production Usage
-
-During training, in the GraphSage algorithm at least, Memgraph is at the
-beginning and at the end of the Tensorflow computation graph.  
-At the beginning, the Data Op provides the node IDs which are fed into the
-generic Tensorflow Op to find their neighbours and their neighbours and
-their features.
-
-Production usage differs in that we don't use the Data Op. The Data Op is
-effectively cut off and the initial input is fed by Tensorflow serving,
-with the data found in the request.
-
-For example a JSON request to classify a node might look like:
-
-`POST http://host:port/v1/models/GraphSage/versions/v1:classify`
-
-With the contents:
-
-```json
-{
-    "examples": [
-        {"node_id": 1},
-        {"node_id": 2}
-    ],
-}
-```
-
-Every element of the "examples" list is an example to be computed. Each is
-represented by a dict with keys matching names of feeds in the Tensorflow
-graph and values being the values we want fed in for each example
-
-The REST API then replies in kind with the classification result in JSON
-
-Note about adding our custom Op to Tensorflow serving.  
-Our Ops .so can be added into the Bazel build to link with Tensorflow serving
-or it can be dynamically loaded by starting Tensorflow serving with a flag
-`--custom_op_paths`
-
-## Considerations
-
-There might be issues here that the url to connect to Memgraph is
-hardcoded into the op and would thus be wrong when moved to production,
-requiring some type of a hack to make work. We probably want to solve
-this by having the client op take in another tf.Variable as an input
-which would contain a connection url and username/password.  
-We have to research whether this makes it easy enough to move to
-production, as the connection string variable is still a part of the
-graph, but maybe easier to replace.  
-
-It is probably the best idea to utilize openCypher parameters to make
-our queries flexible. The exact API as to how to declare the parameters
-in Python is open to discussion.
-
-The Data Op might not even be necessary to implement as it is not
-key for production use. It can be replaced in training mode with
-feed dicts and either
-
- 1. Getting the initial list of nodes via a Python Bolt client
- 2. Creating a separate Tensorflow computation graph that gets all the
- relevant node IDs into Python

docs/presentation/latex-template/README.md

@@ -1,22 +0,0 @@
-# Memgraph LaTeX Beamer Template
-
-This folder contains all of the needed files for creating a presentation with
-Memgraph styling. You should use this style for any public presentations.
-
-Feel free to improve it according to style guidelines and raise issues if you
-find any.
-
-## Usage
-
-Copy the contents of this folder (excluding this README file) to where you
-want to write your own presentation. After copying, you can start editing the
-`template.tex` with your content.
-
-To compile the presentation to a PDF, run `latexmk -pdf -xelatex`. Some
-directives require XeLaTeX, so you need to pass `-xelatex` as the final option
-of `latexmk`. You may also need to install some packages if the compilation
-complains about missing packages.
-
-To clean up the generated files, use `latexmk -C`. This will also delete the
-generated PDF. If you wish to remove generated files except the PDF, use
-`latexmk -c`.

docs/presentation/latex-template/mg-beamer.cls

@@ -1,82 +0,0 @@
-\NeedsTeXFormat{LaTeX2e}
-\ProvidesClass{mg-beamer}[2018/03/26 Memgraph Beamer]
-
-\DeclareOption*{\PassOptionsToClass{\CurrentOption}{beamer}}
-
-\ProcessOptions \relax
-
-\LoadClass{beamer}
-
-\usetheme{Pittsburgh}
-
-% Memgraph color palette
-\definecolor{mg-purple}{HTML}{720096}
-\definecolor{mg-red}{HTML}{DD2222}
-\definecolor{mg-orange}{HTML}{FB6E00}
-\definecolor{mg-yellow}{HTML}{FFC500}
-\definecolor{mg-gray}{HTML}{857F87}
-\definecolor{mg-black}{HTML}{231F20}
-
-\RequirePackage{fontspec}
-% Title fonts
-\setbeamerfont{frametitle}{family={\fontspec[Path = ./mg-style/fonts/]{EncodeSansSemiCondensed-Regular.ttf}}}
-\setbeamerfont{title}{family={\fontspec[Path = ./mg-style/fonts/]{EncodeSansSemiCondensed-Regular.ttf}}}
-% Body font
-\RequirePackage[sfdefault,light]{roboto}
-% Roboto is pretty bad for monospace font. We will find a replacement.
-% \setmonofont{RobotoMono-Regular.ttf}[Path = ./mg-style/fonts/]
-
-% Title slide styles
-% \setbeamerfont{frametitle}{size=\huge}
-% \setbeamerfont{title}{size=\huge}
-% \setbeamerfont{date}{size=\tiny}
-
-% Other typography styles
-\setbeamertemplate{frametitle}[default][center]
-\setbeamercolor{frametitle}{fg=mg-black}
-\setbeamercolor{title}{fg=mg-black}
-\setbeamercolor{section in toc}{fg=mg-black}
-\setbeamercolor{local structure}{fg=mg-orange}
-\setbeamercolor{alert text}{fg=mg-red}
-
-% Commands
-\newcommand{\mgalert}[1]{{\usebeamercolor[fg]{alert text}#1}}
-\newcommand{\titleframe}{\frame[plain]{\titlepage}}
-\newcommand{\mgtexttt}[1]{{\textcolor{mg-gray}{\texttt{#1}}}}
-
-% Title slide background
-\RequirePackage{tikz,calc}
-% Use title-slide-169 if aspect ration is 16:9
-\pgfdeclareimage[interpolate=true,width=\paperwidth,height=\paperheight]{logo}{mg-style/title-slide-169}
-\setbeamertemplate{background}{
-  \begin{tikzpicture}
-  \useasboundingbox (0,0) rectangle (\the\paperwidth,\the\paperheight);
-    \pgftext[at=\pgfpoint{0}{0},left,base]{\pgfuseimage{logo}};
-  \ifnum\thepage>1\relax
-  \useasboundingbox (0,0) rectangle (\the\paperwidth,\the\paperheight);
-      \fill[white, opacity=1](0,\the\paperheight)--(\the\paperwidth,\the\paperheight)--(\the\paperwidth,0)--(0,0)--(0,\the\paperheight);
-  \fi
-  \end{tikzpicture}
-}
-
-% Footline content
-\setbeamertemplate{navigation symbols}{}%remove navigation symbols
- \setbeamertemplate{footline}{
-  \begin{beamercolorbox}[ht=1.6cm,wd=\paperwidth]{footlinecolor}
-    \vspace{0.1cm}
-    \hfill
-    \begin{minipage}[c]{3cm}
-      \begin{center}
-       \includegraphics[height=0.8cm]{mg-style/memgraph-logo.png}
-      \end{center}
-    \end{minipage}
-    \begin{minipage}[c]{7cm}
-         \insertshorttitle\ --- \insertsection
-    \end{minipage}
-    \begin{minipage}[c]{2cm}
-        \tiny{\insertframenumber{} of \inserttotalframenumber}
-    \end{minipage}
-   \end{beamercolorbox}
-}
-
-\endinput

docs/presentation/latex-template/template.tex

@@ -1,40 +0,0 @@
-% Set 16:9 aspect ratio
-\documentclass[aspectratio=169]{mg-beamer}
-% Default directive sets the regular 4:3 aspect ratio
-% \documentclass{mg-beamer}
-\mode<presentation>
-
-% requires xelatex
-\usepackage{ccicons}
-
-\title{Insert Presentation Title}
-\titlegraphic{\ccbyncnd}
-\author{Insert Name}
-
-% Institute doesn't look good in our current styling class.
-% \institute[Memgraph Ltd.]{\pgfimage[height=1.5cm]{mg-logo.png}}
-
-% Date is autogenerated on compilation, so no need to set it explicitly,
-% unless you wish to override it with a different date.
-% \date{March 23, 2018}
-
-\begin{document}
-
-\titleframe
-
-\section{Intro}
-
-\begin{frame}{Contents}
-  \tableofcontents
-\end{frame}
-
-\begin{frame}{Memgraph Markup Test}
-  \begin{itemize}
-    \item \mgtexttt{Prefer \\mgtexttt for monospace}
-    \item Replace this slide with your own
-    \item Add even more slides in different sections
-    \item Make sure you spellcheck your presentation
-  \end{itemize}
-\end{frame}
-
-\end{document}

environment/README.md

@@ -0,0 +1,4 @@
+# Memgraph Build and Run Environments
+
+Please continue in
+[Notion](https://www.notion.so/memgraph/Tools-05e0baafb78a49b386e0063b4833d23d).

environment/os/.gitignore

@@ -0,0 +1,3 @@
+*.deb
+*.rpm
+*.tar.gz

environment/os/centos-7.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc gcc-c++ make # generic build tools
+    wget # used for archive download
+    gnupg2 # used for archive signature verification
+    tar gzip bzip2 xz unzip # used for archive unpacking
+    zlib-devel # zlib library used for all builds
+    expat-devel libipt-devel libbabeltrace-devel xz-devel python3-devel # gdb
+    texinfo # gdb
+    libcurl-devel # cmake
+    readline-devel # cmake and llvm
+    libffi-devel libxml2-devel perl-Digest-MD5 # llvm
+    libedit-devel pcre-devel automake bison # swig
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz # used for archive unpacking
+    zlib # zlib library used for all builds
+    expat libipt libbabeltrace xz-libs python3 # for gdb
+    readline # for cmake and llvm
+    libffi libxml2 # for llvm
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkgconfig # build system
+    curl wget # for downloading libs
+    libuuid-devel java-11-openjdk # required by antlr
+    readline-devel # for memgraph console
+    python3-devel # for query modules
+    openssl-devel
+    libseccomp-devel
+    python3 python-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
+    #
+    # IMPORTANT: python3-yaml does NOT exist on CentOS
+    # Install it using `pip3 install PyYAML`
+    #
+    PyYAML # Package name here does not correspond to the yum package!
+    libcurl-devel # mg-requests
+    sbcl # for custom Lisp C++ preprocessing
+    rpm-build rpmlint # for RPM package building
+    doxygen graphviz # source documentation generators
+    which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
+    autoconf # for jemalloc code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    local missing=""
+    for pkg in $1; do
+        if [ "$pkg" == git ]; then
+            if ! which "git" >/dev/null; then
+                missing="git $missing"
+            fi
+            continue
+        fi
+        if [ "$pkg" == "PyYAML" ]; then
+            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
+                missing="$pkg $missing"
+            fi
+            continue
+        fi
+        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+
+install() {
+    cd "$DIR"
+    if [ "$EUID" -ne 0 ]; then
+        echo "Please run as root."
+        exit 1
+    fi
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    yum install -y epel-release
+    yum update -y
+    yum install -y wget git python3 python3-pip
+    for pkg in $1; do
+        if [ "$pkg" == libipt ]; then
+            if ! yum list installed libipt >/dev/null 2>/dev/null; then
+                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == libipt-devel ]; then
+            if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
+                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! yum list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
+                wget -nv https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
+                rpm -Uvh https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm
+                yum update -y
+                yum install -y dotnet-sdk-3.1
+            fi
+            continue
+        fi
+        if [ "$pkg" == PyYAML ]; then
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
+            continue
+        fi
+        yum install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/centos-8.sh

@@ -0,0 +1,152 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc gcc-c++ make # generic build tools
+    wget # used for archive download
+    gnupg2 # used for archive signature verification
+    tar gzip bzip2 xz unzip # used for archive unpacking
+    zlib-devel # zlib library used for all builds
+    expat-devel libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
+    libcurl-devel # for cmake
+    readline-devel # for cmake and llvm
+    libffi-devel libxml2-devel # for llvm
+    libedit-devel pcre-devel automake bison # for swig
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz # used for archive unpacking
+    zlib # zlib library used for all builds
+    expat libipt libbabeltrace xz-libs python36 # for gdb
+    readline # for cmake and llvm
+    libffi libxml2 # for llvm
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkgconf-pkg-config # build system
+    curl wget # for downloading libs
+    libuuid-devel java-11-openjdk # required by antlr
+    readline-devel # for memgraph console
+    python36-devel # for query modules
+    openssl-devel
+    libseccomp-devel
+    python36 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
+    #
+    # IMPORTANT: python3-yaml does NOT exist on CentOS
+    # Install it manually using `pip3 install PyYAML`
+    #
+    PyYAML # Package name here does not correspond to the yum package!
+    libcurl-devel # mg-requests
+    rpm-build rpmlint # for RPM package building
+    doxygen graphviz # source documentation generators
+    which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
+    sbcl # for custom Lisp C++ preprocessing
+    autoconf # for jemalloc code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    local missing=""
+    for pkg in $1; do
+        if [ "$pkg" == "PyYAML" ]; then
+            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
+                missing="$pkg $missing"
+            fi
+            continue
+        fi
+        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+
+install() {
+    cd "$DIR"
+    if [ "$EUID" -ne 0 ]; then
+        echo "Please run as root."
+        exit 1
+    fi
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    dnf install -y epel-release
+    dnf install -y 'dnf-command(config-manager)'
+    dnf config-manager --set-enabled powertools # Required to install texinfo.
+    dnf update -y
+    dnf install -y wget git python36 python3-pip
+    for pkg in $1; do
+        if [ "$pkg" == libipt ]; then
+            if ! dnf list installed libipt >/dev/null 2>/dev/null; then
+                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == libipt-devel ]; then
+            if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
+                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        # Install GDB dependencies not present in the standard repos.
+        # https://bugs.centos.org/view.php?id=17068
+        # https://centos.pkgs.org
+	    # Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
+        # within GDB yet (an assumption).
+        if [ "$pkg" == libbabeltrace-devel ]; then
+            if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
+                dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libbabeltrace-devel-1.5.4-2.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == sbcl ]; then
+            if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
+                dnf install -y 	https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
+            fi
+            if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
+                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
+            fi
+            if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
+                dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! dnf list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
+                wget -nv https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
+                rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm
+                dnf update -y
+                dnf install -y dotnet-sdk-3.1
+            fi
+            continue
+        fi
+        if [ "$pkg" == PyYAML ]; then
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
+            continue
+        fi
+        dnf install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/debian-10.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc g++ build-essential make # generic build tools
+    wget # used for archive download
+    gnupg # used for archive signature verification
+    tar gzip bzip2 xz-utils unzip # used for archive unpacking
+    zlib1g-dev # zlib library used for all builds
+    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
+    libcurl4-openssl-dev # for cmake
+    libreadline-dev # for cmake and llvm
+    libffi-dev libxml2-dev # for llvm
+    libedit-dev libpcre3-dev automake bison # for swig
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz-utils # used for archive unpacking
+    zlib1g # zlib library used for all builds
+    libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
+    libcurl4 # for cmake
+    libreadline7 # for cmake and llvm
+    libffi6 libxml2 # for llvm
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkg-config # build system
+    curl wget # for downloading libs
+    uuid-dev default-jre-headless # required by antlr
+    libreadline-dev # for memgraph console
+    libpython3-dev python3-dev # for query modules
+    libssl-dev
+    libseccomp-dev
+    netcat # tests are using nc to wait for memgraph
+    python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
+    python3-yaml # for the configuration generator
+    libcurl4-openssl-dev # mg-requests
+    sbcl # for custom Lisp C++ preprocessing
+    doxygen graphviz # source documentation generators
+    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
+    dotnet-sdk-3.1 golang nodejs npm
+    autoconf # for jemalloc code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    check_all_dpkg "$1"
+}
+
+install() {
+    cat >/etc/apt/sources.list <<EOF
+deb http://deb.debian.org/debian/ buster main non-free contrib
+deb-src http://deb.debian.org/debian/ buster main non-free contrib
+deb http://deb.debian.org/debian/ buster-updates main contrib non-free
+deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free
+deb http://security.debian.org/debian-security buster/updates main contrib non-free
+deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
+EOF
+    cd "$DIR"
+    apt update
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    apt install -y wget
+    for pkg in $1; do
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
+                wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
+                dpkg -i packages-microsoft-prod.deb
+                apt-get update
+                apt-get install -y apt-transport-https dotnet-sdk-3.1
+            fi
+            continue
+        fi
+        apt install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/debian-9.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc g++ build-essential make # generic build tools
+    wget # used for archive download
+    gnupg # used for archive signature verification
+    tar gzip bzip2 xz-utils unzip # used for archive unpacking
+    zlib1g-dev # zlib library used for all builds
+    libexpat1-dev libipt-dev libbabeltrace-dev libbabeltrace-ctf-dev liblzma-dev python3-dev texinfo # for gdb
+    libcurl4-openssl-dev # for cmake
+    libreadline-dev # for cmake and llvm
+    libffi-dev libxml2-dev # for llvm
+    libedit-dev libpcre3-dev automake bison # for swig
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz-utils # used for archive unpacking
+    zlib1g # zlib library used for all builds
+    libexpat1 libipt1 libbabeltrace1 libbabeltrace-ctf1 liblzma5 python3 # for gdb
+    libcurl3 # for cmake
+    libreadline7 # for cmake and llvm
+    libffi6 libxml2 # for llvm
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkg-config # build system
+    curl wget # for downloading libs
+    uuid-dev default-jre-headless # required by antlr
+    libreadline-dev # for memgraph console
+    libpython3-dev python3-dev # for query modules
+    libssl-dev
+    libseccomp-dev
+    python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
+    python3-yaml # for the configuration generator
+    libcurl4-openssl-dev # mg-requests
+    sbcl # for custom Lisp C++ preprocessing
+    doxygen graphviz # source documentation generators
+    mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
+    autoconf # for jemalloc code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    check_all_dpkg "$1"
+}
+
+install() {
+    install_all_apt "$1"
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/template.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    pkg
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    pkg
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    pkg
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    echo "TODO: Implement ${FUNCNAME[0]}."
+    exit 1
+}
+
+install() {
+    echo "TODO: Implement ${FUNCNAME[0]}."
+    exit 1
+}
+
+# http://ahmed.amayem.com/bash-indirect-expansion-exploration
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/ubuntu-18.04.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc g++ build-essential make # generic build tools
+    wget # archive download
+    gnupg # archive signature verification
+    tar gzip bzip2 xz-utils unzip # archive unpacking
+    zlib1g-dev # zlib library used for all builds
+    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev # gdb
+    texinfo # gdb
+    libcurl4-openssl-dev # cmake
+    libreadline-dev # cmake and llvm
+    libffi-dev libxml2-dev # llvm
+    libedit-dev libpcre3-dev automake bison # swig
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz-utils # used for archive unpacking
+    zlib1g # zlib library used for all builds
+    libexpat1 libipt1 libbabeltrace1 liblzma5 python3 # for gdb
+    libcurl4 # for cmake
+    libreadline7 # for cmake and llvm
+    libffi6 libxml2 # for llvm
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkg-config # build system
+    curl wget # downloading libs
+    uuid-dev default-jre-headless # required by antlr
+    libreadline-dev # memgraph console
+    libpython3-dev python3-dev # for query modules
+    libssl-dev
+    libseccomp-dev
+    python3 virtualenv python3-virtualenv python3-pip # qa, macro bench and stress tests
+    python3-yaml # the configuration generator
+    libcurl4-openssl-dev # mg-requests
+    sbcl # custom Lisp C++ preprocessing
+    doxygen graphviz # source documentation generators
+    mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
+    autoconf # for jemalloc code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    check_all_dpkg "$1"
+}
+
+install() {
+    apt install -y $1
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/os/ubuntu-20.04.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+TOOLCHAIN_BUILD_DEPS=(
+    coreutils gcc g++ build-essential make # generic build tools
+    wget # used for archive download
+    gnupg # used for archive signature verification
+    tar gzip bzip2 xz-utils unzip # used for archive unpacking
+    zlib1g-dev # zlib library used for all builds
+    libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
+    libcurl4-openssl-dev # for cmake
+    libreadline-dev # for cmake and llvm
+    libffi-dev libxml2-dev # for llvm
+    libedit-dev libpcre3-dev automake bison # for swig
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz-utils # used for archive unpacking
+    zlib1g # zlib library used for all builds
+    libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
+    libcurl4 # for cmake
+    libreadline8 # for cmake and llvm
+    libffi7 libxml2 # for llvm
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make pkg-config # build system
+    curl wget # for downloading libs
+    uuid-dev default-jre-headless # required by antlr
+    libreadline-dev # for memgraph console
+    libpython3-dev python3-dev # for query modules
+    libssl-dev
+    libseccomp-dev
+    netcat # tests are using nc to wait for memgraph
+    python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
+    python3-yaml # for the configuration generator
+    libcurl4-openssl-dev # mg-requests
+    sbcl # for custom Lisp C++ preprocessing
+    doxygen graphviz # source documentation generators
+    mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
+    dotnet-sdk-3.1 golang nodejs npm
+    autoconf # for jemalloc code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    check_all_dpkg "$1"
+}
+
+install() {
+    cd "$DIR"
+    apt update
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests doesn't work the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    apt install -y wget
+    for pkg in $1; do
+        if [ "$pkg" == dotnet-sdk-3.1 ]; then
+            if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then
+                wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
+                dpkg -i packages-microsoft-prod.deb
+                apt-get update
+                apt-get install -y apt-transport-https dotnet-sdk-3.1
+            fi
+            continue
+        fi
+        apt install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"

environment/toolchain/v1.sh

@@ -0,0 +1,651 @@
+#!/bin/bash -e
+
+# helpers
+pushd () { command pushd "$@" > /dev/null; }
+popd () { command popd "$@" > /dev/null; }
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CPUS=$( cat /proc/cpuinfo | grep processor | wc -l )
+cd "$DIR"
+
+# toolchain version
+TOOLCHAIN_VERSION=1
+
+# package versions used
+GCC_VERSION=8.3.0
+BINUTILS_VERSION=2.32
+GDB_VERSION=8.2.1
+CMAKE_VERSION=3.14.2
+CPPCHECK_VERSION=1.87
+LLVM_VERSION=8.0.0
+SWIG_VERSION=3.0.12 # used only for LLVM compilation
+
+# check for installed dependencies
+DISTRO="$( egrep '^(VERSION_)?ID=' /etc/os-release | sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-' )"
+case "$DISTRO" in
+    debian-9)
+        DEPS_MANAGER=apt-get
+        DEPS_COMPILE=(
+            coreutils gcc g++ build-essential make # generic build tools
+            wget # used for archive download
+            gnupg # used for archive signature verification
+            tar gzip bzip2 xz-utils unzip # used for archive unpacking
+            zlib1g-dev # zlib library used for all builds
+            libexpat1-dev libipt-dev libbabeltrace-dev libbabeltrace-ctf-dev liblzma-dev python3-dev texinfo # for gdb
+            libcurl4-openssl-dev # for cmake
+            libreadline-dev # for cmake and llvm
+            libffi-dev libxml2-dev # for llvm
+            libedit-dev libpcre3-dev automake bison # for swig
+        )
+        DEPS_RUN=(
+            make # generic build tools
+            tar gzip bzip2 xz-utils # used for archive unpacking
+            zlib1g # zlib library used for all builds
+            libexpat1 libipt1 libbabeltrace1 libbabeltrace-ctf1 liblzma5 python3 # for gdb
+            libcurl3 # for cmake
+            libreadline7 # for cmake and llvm
+            libffi6 libxml2 # for llvm
+        )
+        ;;
+
+    debian-10)
+        DEPS_MANAGER=apt-get
+        DEPS_COMPILE=(
+            coreutils gcc g++ build-essential make # generic build tools
+            wget # used for archive download
+            gnupg # used for archive signature verification
+            tar gzip bzip2 xz-utils unzip # used for archive unpacking
+            zlib1g-dev # zlib library used for all builds
+            libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
+            libcurl4-openssl-dev # for cmake
+            libreadline-dev # for cmake and llvm
+            libffi-dev libxml2-dev # for llvm
+            libedit-dev libpcre3-dev automake bison # for swig
+        )
+        DEPS_RUN=(
+            make # generic build tools
+            tar gzip bzip2 xz-utils # used for archive unpacking
+            zlib1g # zlib library used for all builds
+            libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
+            libcurl4 # for cmake
+            libreadline7 # for cmake and llvm
+            libffi6 libxml2 # for llvm
+        )
+        ;;
+
+    ubuntu-18.04)
+        DEPS_MANAGER=apt-get
+        DEPS_COMPILE=(
+            coreutils gcc g++ build-essential make # generic build tools
+            wget # used for archive download
+            gnupg # used for archive signature verification
+            tar gzip bzip2 xz-utils unzip # used for archive unpacking
+            zlib1g-dev # zlib library used for all builds
+            libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
+            libcurl4-openssl-dev # for cmake
+            libreadline-dev # for cmake and llvm
+            libffi-dev libxml2-dev # for llvm
+            libedit-dev libpcre3-dev automake bison # for swig
+        )
+        DEPS_RUN=(
+            make # generic build tools
+            tar gzip bzip2 xz-utils # used for archive unpacking
+            zlib1g # zlib library used for all builds
+            libexpat1 libipt1 libbabeltrace1 liblzma5 python3 # for gdb
+            libcurl4 # for cmake
+            libreadline7 # for cmake and llvm
+            libffi6 libxml2 # for llvm
+        )
+        ;;
+
+    centos-7)
+        DEPS_MANAGER=yum
+        DEPS_COMPILE=(
+            coreutils gcc gcc-c++ make # generic build tools
+            wget # used for archive download
+            gnupg2 # used for archive signature verification
+            tar gzip bzip2 xz unzip # used for archive unpacking
+            zlib-devel # zlib library used for all builds
+            expat-devel libipt-devel libbabeltrace-devel xz-devel python3-devel texinfo # for gdb
+            libcurl-devel # for cmake
+            readline-devel # for cmake and llvm
+            libffi-devel libxml2-devel # for llvm
+            libedit-devel pcre-devel automake bison # for swig
+        )
+        DEPS_RUN=(
+            make # generic build tools
+            tar gzip bzip2 xz # used for archive unpacking
+            zlib # zlib library used for all builds
+            expat libipt libbabeltrace xz-libs python3 # for gdb
+            readline # for cmake and llvm
+            libffi libxml2 # for llvm
+        )
+        ;;
+
+    *)
+        echo "Unknown distribution: $DISTRO!"
+        exit 1
+        ;;
+esac
+find_missing_dependencies () {
+    local message="$1"; shift
+    local missing=""
+    while [ "$1" != "" ]; do
+        if [ "$DEPS_MANAGER" == "apt-get" ]; then
+            if ! dpkg -s $1 >/dev/null 2>/dev/null; then
+                missing="$1 $missing"
+            fi
+        elif [ "$DEPS_MANAGER" == "yum" ]; then
+            if ! yum list installed $1 >/dev/null 2>/dev/null; then
+                missing="$1 $missing"
+            fi
+        else
+            echo "Invalid package manager: $DEPS_MANAGER!"
+            exit 1
+        fi
+        shift
+    done
+    if [ "$missing" != "" ]; then
+        echo "$message: $missing"
+        exit 1
+    fi
+}
+find_missing_dependencies "Missing dependencies" ${DEPS_COMPILE[@]}
+find_missing_dependencies "All dependencies are installed, but the following runtime libraries were not found (they are probably invalid)" ${DEPS_RUN[@]}
+
+# check installation directory
+NAME=toolchain-v$TOOLCHAIN_VERSION
+PREFIX=/opt/$NAME
+mkdir -p $PREFIX >/dev/null 2>/dev/null || true
+if [ ! -d $PREFIX ] || [ ! -w $PREFIX ]; then
+    echo "Please make sure that the directory '$PREFIX' exists and is writable by the current user!"
+    echo
+    echo "If unsure, execute these commands as root:"
+    echo "    mkdir $PREFIX && chown $USER:$USER $PREFIX"
+    echo
+    echo "Press <return> when you have created the directory and granted permissions."
+    # wait for the directory to be created
+    while true; do
+        read
+        if [ ! -d $PREFIX ] || [ ! -w $PREFIX ]; then
+            echo
+            echo "You can't continue before you have created the directory and granted permissions!"
+            echo
+            echo "Press <return> when you have created the directory and granted permissions."
+        else
+            break
+        fi
+    done
+fi
+
+# create archives directory
+mkdir -p archives
+
+# download all archives
+pushd archives
+if [ ! -f gcc-$GCC_VERSION.tar.gz ]; then
+    wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz
+fi
+if [ ! -f binutils-$BINUTILS_VERSION.tar.gz ]; then
+    wget https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.gz
+fi
+if [ ! -f gdb-$GDB_VERSION.tar.gz ]; then
+    wget https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.gz
+fi
+if [ ! -f cmake-$CMAKE_VERSION.tar.gz ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz
+fi
+if [ ! -f swig-$SWIG_VERSION.tar.gz ]; then
+    wget https://github.com/swig/swig/archive/rel-$SWIG_VERSION.tar.gz -O swig-$SWIG_VERSION.tar.gz
+fi
+if [ ! -f cppcheck-$CPPCHECK_VERSION.tar.gz ]; then
+    wget https://github.com/danmar/cppcheck/archive/$CPPCHECK_VERSION.tar.gz -O cppcheck-$CPPCHECK_VERSION.tar.gz
+fi
+if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
+    wget https://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz
+    wget https://releases.llvm.org/$LLVM_VERSION/cfe-$LLVM_VERSION.src.tar.xz
+    wget https://releases.llvm.org/$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz
+    wget https://releases.llvm.org/$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
+    wget https://releases.llvm.org/$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
+fi
+if [ ! -f pahole-gdb-master.zip ]; then
+    wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
+fi
+
+# verify all archives
+# NOTE: Verification can fail if the archive is signed by another developer. I
+# haven't added commands to download all developer GnuPG keys because the
+# download is very slow. If the verification fails for you, figure out who has
+# signed the archive and download their public key instead.
+GPG="gpg --homedir .gnupg"
+KEYSERVER="hkp://keyserver.ubuntu.com"
+mkdir -p .gnupg
+chmod 700 .gnupg
+# verify gcc
+if [ ! -f gcc-$GCC_VERSION.tar.gz.sig ]; then
+    wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz.sig
+fi
+# list of valid gcc gnupg keys: https://gcc.gnu.org/mirrors.html
+$GPG --keyserver $KEYSERVER --recv-keys 0xA328C3A2C3C45C06
+$GPG --verify gcc-$GCC_VERSION.tar.gz.sig gcc-$GCC_VERSION.tar.gz
+# verify binutils
+if [ ! -f binutils-$BINUTILS_VERSION.tar.gz.sig ]; then
+    wget https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.gz.sig
+fi
+$GPG --keyserver $KEYSERVER --recv-keys 0xDD9E3C4F
+$GPG --verify binutils-$BINUTILS_VERSION.tar.gz.sig binutils-$BINUTILS_VERSION.tar.gz
+# verify gdb
+if [ ! -f gdb-$GDB_VERSION.tar.gz.sig ]; then
+    wget https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.gz.sig
+fi
+$GPG --keyserver $KEYSERVER --recv-keys 0xFF325CF3
+$GPG --verify gdb-$GDB_VERSION.tar.gz.sig gdb-$GDB_VERSION.tar.gz
+# verify cmake
+if [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt ] || [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt.asc ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt
+    wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt.asc
+    # Because CentOS 7 doesn't have the `--ignore-missing` flag for `sha256sum`
+    # we filter out the missing files from the sums here manually.
+    cat cmake-$CMAKE_VERSION-SHA-256.txt | grep "cmake-$CMAKE_VERSION.tar.gz" > cmake-$CMAKE_VERSION-SHA-256-filtered.txt
+fi
+$GPG --keyserver $KEYSERVER --recv-keys 0xC6C265324BBEBDC350B513D02D2CEF1034921684
+sha256sum -c cmake-$CMAKE_VERSION-SHA-256-filtered.txt
+$GPG --verify cmake-$CMAKE_VERSION-SHA-256.txt.asc cmake-$CMAKE_VERSION-SHA-256.txt
+# verify llvm, cfe, lld, clang-tools-extra
+if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
+    wget https://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz.sig
+    wget https://releases.llvm.org/$LLVM_VERSION/cfe-$LLVM_VERSION.src.tar.xz.sig
+    wget https://releases.llvm.org/$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz.sig
+    wget https://releases.llvm.org/$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
+    wget https://releases.llvm.org/$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
+fi
+# list of valid llvm gnupg keys: https://releases.llvm.org/download.html
+$GPG --keyserver $KEYSERVER --recv-keys 0x345AD05D
+$GPG --verify llvm-$LLVM_VERSION.src.tar.xz.sig llvm-$LLVM_VERSION.src.tar.xz
+$GPG --verify cfe-$LLVM_VERSION.src.tar.xz.sig cfe-$LLVM_VERSION.src.tar.xz
+$GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
+$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
+$GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
+popd
+
+# create build directory
+mkdir -p build
+pushd build
+
+# compile gcc
+if [ ! -f $PREFIX/bin/gcc ]; then
+    if [ -d gcc-$GCC_VERSION ]; then
+        rm -rf gcc-$GCC_VERSION
+    fi
+    tar -xvf ../archives/gcc-$GCC_VERSION.tar.gz
+    pushd gcc-$GCC_VERSION
+    ./contrib/download_prerequisites
+    mkdir build && pushd build
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
+    ../configure -v \
+        --build=x86_64-linux-gnu \
+        --host=x86_64-linux-gnu \
+        --target=x86_64-linux-gnu \
+        --prefix=$PREFIX \
+        --disable-multilib \
+        --with-system-zlib \
+        --enable-checking=release \
+        --enable-languages=c,c++,fortran \
+        --enable-gold=yes \
+        --enable-ld=yes \
+        --enable-lto \
+        --enable-bootstrap \
+        --disable-vtable-verify \
+        --disable-werror \
+        --without-included-gettext \
+        --enable-threads=posix \
+        --enable-nls \
+        --enable-clocale=gnu \
+        --enable-libstdcxx-debug \
+        --enable-libstdcxx-time=yes \
+        --enable-gnu-unique-object \
+        --enable-libmpx \
+        --enable-plugin \
+        --enable-default-pie \
+        --with-target-system-zlib \
+        --with-tune=generic \
+        --without-cuda-driver
+        #--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
+    make -j$CPUS
+    # make -k check # run test suite
+    make install
+    popd && popd
+fi
+
+# activate toolchain
+export PATH=$PREFIX/bin:$PATH
+export LD_LIBRARY_PATH=$PREFIX/lib64
+
+# compile binutils
+if [ ! -f $PREFIX/bin/ld.gold ]; then
+    if [ -d binutils-$BINUTILS_VERSION ]; then
+        rm -rf binutils-$BINUTILS_VERSION
+    fi
+    tar -xvf ../archives/binutils-$BINUTILS_VERSION.tar.gz
+    pushd binutils-$BINUTILS_VERSION
+    mkdir build && pushd build
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
+    env \
+        CC=gcc \
+        CXX=g++ \
+        CFLAGS="-g -O2" \
+        CXXFLAGS="-g -O2" \
+        LDFLAGS="" \
+        ../configure \
+            --build=x86_64-linux-gnu \
+            --host=x86_64-linux-gnu \
+            --prefix=$PREFIX \
+            --enable-ld=default \
+            --enable-gold \
+            --enable-lto \
+            --enable-plugins \
+            --enable-shared \
+            --enable-threads \
+            --with-system-zlib \
+            --enable-deterministic-archives \
+            --disable-compressed-debug-sections \
+            --enable-new-dtags \
+            --disable-werror
+    make -j$CPUS
+    # make -k check # run test suite
+    make install
+    popd && popd
+fi
+
+# compile gdb
+if [ ! -f $PREFIX/bin/gdb ]; then
+    if [ -d gdb-$GDB_VERSION ]; then
+        rm -rf gdb-$GDB_VERSION
+    fi
+    tar -xvf ../archives/gdb-$GDB_VERSION.tar.gz
+    pushd gdb-$GDB_VERSION
+    mkdir build && pushd build
+    # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
+    env \
+        CC=gcc \
+        CXX=g++ \
+        CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+        CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+        CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
+        LDFLAGS="-Wl,-z,relro" \
+        PYTHON="" \
+        ../configure \
+            --build=x86_64-linux-gnu \
+            --host=x86_64-linux-gnu \
+            --prefix=$PREFIX \
+            --disable-maintainer-mode \
+            --disable-dependency-tracking \
+            --disable-silent-rules \
+            --disable-gdbtk \
+            --disable-shared \
+            --without-guile \
+            --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
+            --with-system-readline \
+            --with-expat \
+            --with-system-zlib \
+            --with-lzma \
+            --with-babeltrace \
+            --with-intel-pt \
+            --enable-tui \
+            --with-python=python3
+    make -j$CPUS
+    make install
+    popd && popd
+fi
+
+# install pahole
+if [ ! -d $PREFIX/share/pahole-gdb ]; then
+    unzip ../archives/pahole-gdb-master.zip
+    mv pahole-gdb-master $PREFIX/share/pahole-gdb
+fi
+
+# setup system gdbinit
+if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
+    mkdir -p $PREFIX/etc/gdb
+    cat >$PREFIX/etc/gdb/gdbinit <<EOF
+# improve formatting
+set print pretty on
+set print object on
+set print static-members on
+set print vtbl on
+set print demangle on
+set demangle-style gnu-v3
+set print sevenbit-strings off
+
+# load libstdc++ pretty printers
+add-auto-load-scripts-directory $PREFIX/lib64
+add-auto-load-safe-path $PREFIX
+
+# load pahole
+python
+sys.path.insert(0, "$PREFIX/share/pahole-gdb")
+import offsets
+import pahole
+end
+EOF
+fi
+
+# compile cmake
+if [ ! -f $PREFIX/bin/cmake ]; then
+    if [ -d cmake-$CMAKE_VERSION ]; then
+        rm -rf cmake-$CMAKE_VERSION
+    fi
+    tar -xvf ../archives/cmake-$CMAKE_VERSION.tar.gz
+    pushd cmake-$CMAKE_VERSION
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=cmake&arch=amd64&ver=3.13.4-1&stamp=1549799837
+    echo 'set(CMAKE_SKIP_RPATH ON CACHE BOOL "Skip rpath" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_USE_RELATIVE_PATHS ON CACHE BOOL "Use relative paths" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_C_FLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" CACHE STRING "C flags" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_CXX_FLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" CACHE STRING "C++ flags" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_SKIP_BOOTSTRAP_TEST ON CACHE BOOL "Skip BootstrapTest" FORCE)' >> build-flags.cmake
+    echo 'set(BUILD_CursesDialog ON CACHE BOOL "Build curses GUI" FORCE)' >> build-flags.cmake
+    mkdir build && pushd build
+    ../bootstrap \
+        --prefix=$PREFIX \
+        --init=../build-flags.cmake \
+        --parallel=$CPUS \
+        --system-curl
+    make -j$CPUS
+    # make test # run test suite
+    make install
+    popd && popd
+fi
+
+# compile cppcheck
+if [ ! -f $PREFIX/bin/cppcheck ]; then
+    if [ -d cppcheck-$CPPCHECK_VERSION ]; then
+        rm -rf cppcheck-$CPPCHECK_VERSION
+    fi
+    tar -xvf ../archives/cppcheck-$CPPCHECK_VERSION.tar.gz
+    pushd cppcheck-$CPPCHECK_VERSION
+    env \
+        CC=gcc \
+        CXX=g++ \
+        PREFIX=$PREFIX \
+        CFGDIR=$PREFIX/share/cppcheck/cfg \
+            make -j$CPUS
+    env \
+        CC=gcc \
+        CXX=g++ \
+        PREFIX=$PREFIX \
+        CFGDIR=$PREFIX/share/cppcheck/cfg \
+            make install
+    popd
+fi
+
+# compile swig
+if [ ! -d swig-$SWIG_VERSION/install ]; then
+    if [ -d swig-$SWIG_VERSION ]; then
+        rm -rf swig-$SWIG_VERSION
+    fi
+    tar -xvf ../archives/swig-$SWIG_VERSION.tar.gz
+    mv swig-rel-$SWIG_VERSION swig-$SWIG_VERSION
+    pushd swig-$SWIG_VERSION
+    ./autogen.sh
+    mkdir build && pushd build
+    ../configure --prefix=$DIR/build/swig-$SWIG_VERSION/install
+    make -j$CPUS
+    make install
+    popd && popd
+fi
+
+# compile llvm
+if [ ! -f $PREFIX/bin/clang ]; then
+    if [ -d llvm-$LLVM_VERSION ]; then
+        rm -rf llvm-$LLVM_VERSION
+    fi
+    tar -xvf ../archives/llvm-$LLVM_VERSION.src.tar.xz
+    mv llvm-$LLVM_VERSION.src llvm-$LLVM_VERSION
+    tar -xvf ../archives/cfe-$LLVM_VERSION.src.tar.xz
+    mv cfe-$LLVM_VERSION.src llvm-$LLVM_VERSION/tools/clang
+    tar -xvf ../archives/lld-$LLVM_VERSION.src.tar.xz
+    mv lld-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/lld
+    tar -xvf ../archives/compiler-rt-$LLVM_VERSION.src.tar.xz
+    mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
+    tar -xvf ../archives/clang-tools-extra-$LLVM_VERSION.src.tar.xz
+    mv clang-tools-extra-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/clang/tools/extra
+    pushd llvm-$LLVM_VERSION
+    mkdir build && pushd build
+    # activate swig
+    export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
+    cmake .. \
+        -DGCC_INSTALL_PREFIX=$PREFIX \
+        -DCMAKE_C_COMPILER=$PREFIX/bin/gcc \
+        -DCMAKE_CXX_COMPILER=$PREFIX/bin/g++ \
+        -DCMAKE_CXX_LINK_FLAGS="-L$PREFIX/lib64 -Wl,-rpath,$PREFIX/lib64" \
+        -DCMAKE_INSTALL_PREFIX=$PREFIX \
+        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+        -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O2 -DNDEBUG" \
+        -DCMAKE_CXX_FLAGS=' -fuse-ld=gold -fPIC -Wno-unused-command-line-argument -Wno-unknown-warning-option' \
+        -DCMAKE_C_FLAGS=' -fuse-ld=gold -fPIC -Wno-unused-command-line-argument -Wno-unknown-warning-option' \
+        -DLLVM_LINK_LLVM_DYLIB=ON \
+        -DLLVM_INSTALL_UTILS=ON \
+        -DLLVM_VERSION_SUFFIX= \
+        -DLLVM_BUILD_LLVM_DYLIB=ON \
+        -DLLVM_ENABLE_RTTI=ON \
+        -DLLVM_ENABLE_FFI=ON \
+        -DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \
+        -DLLVM_USE_PERF=yes \
+        -DLIBCLANG_LIBRARY_VERSION=1 \
+        -DCLANG_ENABLE_BOOTSTRAP=ON
+    make -j$CPUS
+    make -j$CPUS check-clang # run clang test suite
+    make -j$CPUS check-lld # run lld test suite
+    make install
+    popd && popd
+fi
+
+# create README
+if [ ! -f $PREFIX/README.md ]; then
+    cat >$PREFIX/README.md <<EOF
+# Memgraph Toolchain v$TOOLCHAIN_VERSION
+
+## Included tools
+
+ - GCC $GCC_VERSION
+ - Binutils $BINUTILS_VERSION
+ - GDB $GDB_VERSION
+ - CMake $CMAKE_VERSION
+ - Cppcheck $CPPCHECK_VERSION
+ - LLVM (Clang, LLD, compiler-rt, Clang tools extra) $LLVM_VERSION
+
+## Required libraries
+
+In order to be able to run all of these tools you should install the following
+packages:
+
+\`\`\`
+$DEPS_MANAGER install ${DEPS_RUN[@]}
+\`\`\`
+
+## Usage
+
+In order to use the toolchain you just have to source the activation script:
+
+\`\`\`
+source $PREFIX/activate
+\`\`\`
+EOF
+fi
+
+# create activation script
+if [ ! -f $PREFIX/activate ]; then
+    cat >$PREFIX/activate <<EOF
+# This file must be used with "source $PREFIX/activate" *from bash*
+# You can't run it directly!
+
+# check for active virtual environments
+if [ "\$( type -t deactivate )" != "" ]; then
+    echo "You already have an active virtual environment!"
+    return 0
+fi
+
+# check that we aren't root
+if [ "\$USER" == "root" ]; then
+    echo "You shouldn't use the toolchan as root!"
+    return 0
+fi
+
+# save original environment
+export ORIG_PATH=\$PATH
+export ORIG_PS1=\$PS1
+export ORIG_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH
+
+# activate new environment
+export PATH=$PREFIX/bin:\$PATH
+export PS1="(TOOLCHAIN) \$PS1"
+export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64
+
+# disable root
+function su () {
+    echo "You don't want to use root functions while using the toolchain!"
+    return 1
+}
+function sudo () {
+    echo "You don't want to use root functions while using the toolchain!"
+    return 1
+}
+
+# create deactivation function
+function deactivate() {
+    export PATH=\$ORIG_PATH
+    export PS1=\$ORIG_PS1
+    export LD_LIBRARY_PATH=\$ORIG_LD_LIBRARY_PATH
+    unset ORIG_PATH ORIG_PS1 ORIG_LD_LIBRARY_PATH
+    unset -f su sudo deactivate
+}
+EOF
+fi
+
+# create toolchain archive
+if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
+    tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO.tar.gz -C /opt $NAME
+fi
+
+# output final instructions
+echo -e "\n\n"
+echo "All tools have been built. They are installed in '$PREFIX'."
+echo "In order to distribute the tools to someone else, an archive with the toolchain was created in the 'build' directory."
+echo "If you want to install the packed tools you should execute the following command:"
+echo
+echo "    tar -xvzf build/$NAME-binaries.tar.gz -C /opt"
+echo
+echo "Because the tools were built on this machine, you should probably change the permissions of the installation directory using:"
+echo
+echo "    OPTIONAL: chown -R root:root $PREFIX"
+echo
+echo "In order to use all of the newly compiled tools you should use the prepared activation script:"
+echo
+echo "    source $PREFIX/activate"
+echo
+echo "Or, for more advanced uses, you can add the following lines to your script:"
+echo
+echo "    export PATH=$PREFIX/bin:\$PATH"
+echo "    export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64"
+echo
+echo "Enjoy!"

environment/toolchain/v2.sh

@@ -4,88 +4,36 @@
 pushd () { command pushd "$@" > /dev/null; }
 popd () { command popd "$@" > /dev/null; }
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-CPUS=$( cat /proc/cpuinfo | grep processor | wc -l )
+CPUS=$( grep -c processor < /proc/cpuinfo )
 cd "$DIR"
 
+source "$DIR/../util.sh"
+DISTRO="$(operating_system)"
+
 # toolchain version
-TOOLCHAIN_VERSION=1
+TOOLCHAIN_VERSION=2
 
 # package versions used
-GCC_VERSION=8.3.0
-BINUTILS_VERSION=2.32
-GDB_VERSION=8.2.1
-CMAKE_VERSION=3.14.2
-CPPCHECK_VERSION=1.87
-LLVM_VERSION=8.0.0
-SWIG_VERSION=3.0.12 # used only for LLVM compilation
+GCC_VERSION=10.2.0
+BINUTILS_VERSION=2.35.1
+case "$DISTRO" in
+    centos-7) # because GDB >= 9 does NOT compile with readline6.
+        GDB_VERSION=8.3
+    ;;
+    *)
+        GDB_VERSION=10.1
+    ;;
+esac
+CMAKE_VERSION=3.18.4
+CPPCHECK_VERSION=2.2
+LLVM_VERSION=11.0.0
+SWIG_VERSION=4.0.2 # used only for LLVM compilation
 
-# check for installed dependencies
-DISTRO="$( egrep '^(VERSION_)?ID=' /etc/os-release | sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-' )"
-if [ "$DISTRO" == "debian-9" ] || [ "$DISTRO" == "ubuntu-18.04" ]; then
-    DEPS_MANAGER=apt-get
-    DEPS_COMPILE=(
-        coreutils gcc g++ build-essential make # generic build tools
-        wget # used for archive download
-        gnupg # used for archive signature verification
-        tar gzip bzip2 xz-utils # used for archive unpacking
-        zlib1g-dev # zlib library used for all builds
-        libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
-        libreadline-dev # for cmake and llvm
-        libffi-dev libxml2-dev # for llvm
-        libedit-dev libpcre3-dev automake bison # for swig
-    )
-    DEPS_RUN=(
-        make # generic build tools
-        tar gzip bzip2 xz-utils # used for archive unpacking
-        zlib1g # zlib library used for all builds
-        libexpat1 libipt1 libbabeltrace1 liblzma5 python3 # for gdb
-        libreadline7 # for cmake and llvm
-        libffi6 libxml2 # for llvm
-    )
-elif [ "$DISTRO" == "centos-7" ]; then
-    DEPS_MANAGER=yum
-    DEPS_COMPILE=(
-        coreutils gcc gcc-c++ make # generic build tools
-        wget # used for archive download
-        gnupg2 # used for archive signature verification
-        tar gzip bzip2 xz # used for archive unpacking
-        zlib-devel # zlib library used for all builds
-        expat-devel libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
-        readline-devel # for cmake and llvm
-        libffi-devel libxml2-devel # for llvm
-        libedit-devel pcre-devel automake bison # for swig
-    )
-    DEPS_RUN=(
-        make # generic build tools
-        tar gzip bzip2 xz # used for archive unpacking
-        zlib # zlib library used for all builds
-        expat libipt libbabeltrace xz-libs python36 # for gdb
-        readline # for cmake and llvm
-        libffi libxml2 # for llvm
-    )
-else
-    echo "Unknown distribution: $DISTRO!"
-    exit 1
-fi
-missing=""
-for dep in ${DEPS_COMPILE[@]}; do
-    if [ "$DEPS_MANAGER" == "apt-get" ]; then
-        if ! dpkg -s $dep >/dev/null 2>/dev/null; then
-            missing="$dep $missing"
-        fi
-    elif [ "$DEPS_MANAGER" == "yum" ]; then
-        if ! yum list installed $dep >/dev/null 2>/dev/null; then
-            missing="$dep $missing"
-        fi
-    else
-        echo "Invalid package manager: $DEPS_MANAGER!"
-        exit 1
-    fi
-done
-if [ "$missing" != "" ]; then
-    echo "Missing dependencies: $missing"
-    exit 1
-fi
+# Check for the dependencies.
+echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
+$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
+echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
+$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
 
 # check installation directory
 NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -136,10 +84,14 @@ if [ ! -f cppcheck-$CPPCHECK_VERSION.tar.gz ]; then
     wget https://github.com/danmar/cppcheck/archive/$CPPCHECK_VERSION.tar.gz -O cppcheck-$CPPCHECK_VERSION.tar.gz
 fi
 if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
-    wget http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz
-    wget http://releases.llvm.org/$LLVM_VERSION/cfe-$LLVM_VERSION.src.tar.xz
-    wget http://releases.llvm.org/$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz
-    wget http://releases.llvm.org/$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
+fi
+if [ ! -f pahole-gdb-master.zip ]; then
+    wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
 fi
 
 # verify all archives
@@ -156,7 +108,7 @@ if [ ! -f gcc-$GCC_VERSION.tar.gz.sig ]; then
     wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz.sig
 fi
 # list of valid gcc gnupg keys: https://gcc.gnu.org/mirrors.html
-$GPG --keyserver $KEYSERVER --recv-keys 0xC3C45C06
+$GPG --keyserver $KEYSERVER --recv-keys 0x3AB00996FC26A641
 $GPG --verify gcc-$GCC_VERSION.tar.gz.sig gcc-$GCC_VERSION.tar.gz
 # verify binutils
 if [ ! -f binutils-$BINUTILS_VERSION.tar.gz.sig ]; then
@@ -174,23 +126,28 @@ $GPG --verify gdb-$GDB_VERSION.tar.gz.sig gdb-$GDB_VERSION.tar.gz
 if [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt ] || [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt.asc ]; then
     wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt
     wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt.asc
+    # Because CentOS 7 doesn't have the `--ignore-missing` flag for `sha256sum`
+    # we filter out the missing files from the sums here manually.
+    cat cmake-$CMAKE_VERSION-SHA-256.txt | grep "cmake-$CMAKE_VERSION.tar.gz" > cmake-$CMAKE_VERSION-SHA-256-filtered.txt
 fi
-$GPG --keyserver $KEYSERVER --recv-keys 0x7BFB4EDA
-sha256sum --ignore-missing -c cmake-$CMAKE_VERSION-SHA-256.txt
+$GPG --keyserver $KEYSERVER --recv-keys 0xC6C265324BBEBDC350B513D02D2CEF1034921684
+sha256sum -c cmake-$CMAKE_VERSION-SHA-256-filtered.txt
 $GPG --verify cmake-$CMAKE_VERSION-SHA-256.txt.asc cmake-$CMAKE_VERSION-SHA-256.txt
 # verify llvm, cfe, lld, clang-tools-extra
 if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
-    wget http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz.sig
-    wget http://releases.llvm.org/$LLVM_VERSION/cfe-$LLVM_VERSION.src.tar.xz.sig
-    wget http://releases.llvm.org/$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz.sig
-    wget http://releases.llvm.org/$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
 fi
 # list of valid llvm gnupg keys: https://releases.llvm.org/download.html
 $GPG --keyserver $KEYSERVER --recv-keys 0x345AD05D
 $GPG --verify llvm-$LLVM_VERSION.src.tar.xz.sig llvm-$LLVM_VERSION.src.tar.xz
-$GPG --verify cfe-$LLVM_VERSION.src.tar.xz.sig cfe-$LLVM_VERSION.src.tar.xz
+$GPG --verify clang-$LLVM_VERSION.src.tar.xz.sig clang-$LLVM_VERSION.src.tar.xz
 $GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
 $GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
+$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
 popd
 
 # create build directory
@@ -309,6 +266,7 @@ if [ ! -f $PREFIX/bin/gdb ]; then
             --disable-gdbtk \
             --disable-shared \
             --without-guile \
+            --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
             --with-system-readline \
             --with-expat \
             --with-system-zlib \
@@ -322,6 +280,38 @@ if [ ! -f $PREFIX/bin/gdb ]; then
     popd && popd
 fi
 
+# install pahole
+if [ ! -d $PREFIX/share/pahole-gdb ]; then
+    unzip ../archives/pahole-gdb-master.zip
+    mv pahole-gdb-master $PREFIX/share/pahole-gdb
+fi
+
+# setup system gdbinit
+if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
+    mkdir -p $PREFIX/etc/gdb
+    cat >$PREFIX/etc/gdb/gdbinit <<EOF
+# improve formatting
+set print pretty on
+set print object on
+set print static-members on
+set print vtbl on
+set print demangle on
+set demangle-style gnu-v3
+set print sevenbit-strings off
+
+# load libstdc++ pretty printers
+add-auto-load-scripts-directory $PREFIX/lib64
+add-auto-load-safe-path $PREFIX
+
+# load pahole
+python
+sys.path.insert(0, "$PREFIX/share/pahole-gdb")
+import offsets
+import pahole
+end
+EOF
+fi
+
 # compile cmake
 if [ ! -f $PREFIX/bin/cmake ]; then
     if [ -d cmake-$CMAKE_VERSION ]; then
@@ -340,7 +330,8 @@ if [ ! -f $PREFIX/bin/cmake ]; then
     ../bootstrap \
         --prefix=$PREFIX \
         --init=../build-flags.cmake \
-        --parallel=$CPUS
+        --parallel=$CPUS \
+        --system-curl
     make -j$CPUS
     # make test # run test suite
     make install
@@ -358,12 +349,14 @@ if [ ! -f $PREFIX/bin/cppcheck ]; then
         CC=gcc \
         CXX=g++ \
         PREFIX=$PREFIX \
+        FILESDIR=$PREFIX/share/cppcheck \
         CFGDIR=$PREFIX/share/cppcheck/cfg \
             make -j$CPUS
     env \
         CC=gcc \
         CXX=g++ \
         PREFIX=$PREFIX \
+        FILESDIR=$PREFIX/share/cppcheck \
         CFGDIR=$PREFIX/share/cppcheck/cfg \
             make install
     popd
@@ -392,19 +385,20 @@ if [ ! -f $PREFIX/bin/clang ]; then
     fi
     tar -xvf ../archives/llvm-$LLVM_VERSION.src.tar.xz
     mv llvm-$LLVM_VERSION.src llvm-$LLVM_VERSION
-    tar -xvf ../archives/cfe-$LLVM_VERSION.src.tar.xz
-    mv cfe-$LLVM_VERSION.src llvm-$LLVM_VERSION/tools/clang
+    tar -xvf ../archives/clang-$LLVM_VERSION.src.tar.xz
+    mv clang-$LLVM_VERSION.src llvm-$LLVM_VERSION/tools/clang
     tar -xvf ../archives/lld-$LLVM_VERSION.src.tar.xz
     mv lld-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/lld
     tar -xvf ../archives/clang-tools-extra-$LLVM_VERSION.src.tar.xz
     mv clang-tools-extra-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/clang/tools/extra
+    tar -xvf ../archives/compiler-rt-$LLVM_VERSION.src.tar.xz
+    mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
     pushd llvm-$LLVM_VERSION
     mkdir build && pushd build
     # activate swig
     export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
     # influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
     cmake .. \
-        -DGCC_INSTALL_PREFIX=$PREFIX \
         -DCMAKE_C_COMPILER=$PREFIX/bin/gcc \
         -DCMAKE_CXX_COMPILER=$PREFIX/bin/g++ \
         -DCMAKE_CXX_LINK_FLAGS="-L$PREFIX/lib64 -Wl,-rpath,$PREFIX/lib64" \
@@ -420,9 +414,7 @@ if [ ! -f $PREFIX/bin/clang ]; then
         -DLLVM_ENABLE_RTTI=ON \
         -DLLVM_ENABLE_FFI=ON \
         -DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \
-        -DLLVM_USE_PERF=yes \
-        -DLIBCLANG_LIBRARY_VERSION=1 \
-        -DCLANG_ENABLE_BOOTSTRAP=ON
+        -DLLVM_USE_PERF=yes
     make -j$CPUS
     make -j$CPUS check-clang # run clang test suite
     make -j$CPUS check-lld # run lld test suite
@@ -442,7 +434,7 @@ if [ ! -f $PREFIX/README.md ]; then
  - GDB $GDB_VERSION
  - CMake $CMAKE_VERSION
  - Cppcheck $CPPCHECK_VERSION
- - LLVM (Clang, LLD, Clang tools extra) $LLVM_VERSION
+ - LLVM (Clang, LLD, compiler-rt, Clang tools extra) $LLVM_VERSION
 
 ## Required libraries
 
@@ -450,7 +442,7 @@ In order to be able to run all of these tools you should install the following
 packages:
 
 \`\`\`
-$DEPS_MANAGER install ${DEPS_RUN[@]}
+$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
 \`\`\`
 
 ## Usage
@@ -488,7 +480,7 @@ export ORIG_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH
 
 # activate new environment
 export PATH=$PREFIX/bin:\$PATH
-export PS1="(TOOLCHAIN) \$PS1"
+export PS1="($NAME) \$PS1"
 export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64
 
 # disable root

environment/util.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+operating_system() {
+    grep -E '^(VERSION_)?ID=' /etc/os-release | \
+    sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
+}
+
+check_all_yum() {
+    local missing=""
+    for pkg in $1; do
+        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+
+check_all_dpkg() {
+    local missing=""
+    for pkg in $1; do
+        if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+
+check_all_dnf() {
+    local missing=""
+    for pkg in $1; do
+        if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+}
+install_all_apt() {
+    for pkg in $1; do
+        apt install -y "$pkg"
+    done
+}

include/mgp.py

@@ -64,6 +64,13 @@ class Properties:
         self._len = None
         self._vertex_or_edge = vertex_or_edge
 
+    def __deepcopy__(self, memo):
+        # This is the same as the shallow copy, as the underlying C API should
+        # not support deepcopy. Besides, it doesn't make much sense to actually
+        # copy _mgp.Edge and _mgp.Vertex types as they are actually references
+        # to graph elements and not proper values.
+        return Properties(self._vertex_or_edge)
+
     def get(self, property_name: str, default=None) -> object:
         '''Get the value of a property with the given name or return default.
 
@@ -236,6 +243,8 @@ class Edge:
         '''Raise InvalidContextError.'''
         if not self.is_valid():
             raise InvalidContextError()
+        if not isinstance(other, Edge):
+            return NotImplemented
         return self._edge == other._edge
 
     def __hash__(self) -> int:
@@ -325,6 +334,8 @@ class Vertex:
         '''Raise InvalidContextError'''
         if not self.is_valid():
             raise InvalidContextError()
+        if not isinstance(other, Vertex):
+            return NotImplemented
         return self._vertex == other._vertex
 
     def __hash__(self) -> int:
@@ -372,8 +383,8 @@ class Path:
             pass
         # This is the same as the shallow copy, as the underlying C API should
         # not support deepcopy. Besides, it doesn't make much sense to actually
-        # copy Edge and Vertex types as they are actually references to graph
-        # elements and not proper values.
+        # copy _mgp.Edge and _mgp.Vertex types as they are actually references
+        # to graph elements and not proper values.
         path = self.__copy__()
         memo[id(self._path)] = path._path
         return path
@@ -471,7 +482,7 @@ class Vertices:
 
     def __contains__(self, vertex):
         try:
-            _ = self.graph.get_vertex_by_id(vertex.id)
+            _ = self._graph.get_vertex_by_id(vertex.id)
             return True
         except IndexError:
             return False
@@ -672,7 +683,15 @@ def _typing_to_cypher_type(type_):
                         return _mgp.type_nullable(simple_type)
                     return _mgp.type_nullable(parse_typing(type_arg_as_str))
             elif type_as_str.startswith('typing.List'):
-                type_arg_as_str, = parse_type_args(type_as_str)
+                type_arg_as_str = parse_type_args(type_as_str)
+
+                if len(type_arg_as_str) > 1:
+                    # Nested object could be a type consisting of a list of types (e.g. mgp.Map)
+                    # so we need to join the parts.
+                    type_arg_as_str = ', '.join(type_arg_as_str)
+                else:
+                    type_arg_as_str = type_arg_as_str[0]
+
                 simple_type = get_simple_type(type_arg_as_str)
                 if simple_type is not None:
                     return _mgp.type_list(simple_type)

init

@@ -1,35 +1,14 @@
 #!/bin/bash -e
 
-required_pkgs=(git arcanist # source code control
-               make pkg-config # build system
-               curl wget # for downloading libs
-               uuid-dev default-jre-headless # required by antlr
-               libreadline-dev # for memgraph console
-               libssl-dev
-               libseccomp-dev
-               python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
-               python3-yaml # for the configuration generator
-               uuid-dev # mg-utils
-               libcurl4-openssl-dev # mg-requests
-               sbcl # for custom Lisp C++ preprocessing
-               php-xml # for arcanist linters
-              )
-
-optional_pkgs=(doxygen graphviz # source documentation generators
-               php-cli # for user technical documentation generators
-               mono-runtime mono-mcs nodejs # for driver tests
-              )
-
-use_sudo=0
-
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$DIR"
 
+source "$DIR/environment/util.sh"
+
 function print_help () {
     echo "Usage: $0 [OPTION]"
-    echo -e "Check for missing packages and install them if possible.\n"
+    echo -e "Check for missing packages and setup the project.\n"
     echo "Optional arguments:"
-    echo -e "  -s\tuse sudo apt-get for installing packages"
     echo -e "  -h\tdisplay this help and exit"
 }
 
@@ -45,10 +24,7 @@ function setup_virtualenv () {
     # create new virtualenv
     virtualenv -p python3 ve3 || exit 1
     source ve3/bin/activate
-    # we need to increase the timeout for pip because our local cache server
-    # can sometimes be stupid, see: https://github.com/devpi/devpi/issues/208
-    pip --timeout 1000 install -i http://deps.memgraph.io:3141/root/pypi \
-                --trusted-host deps.memgraph.io -r requirements.txt || exit 1
+    pip --timeout 1000 install -r requirements.txt || exit 1
     deactivate
 
     popd > /dev/null
@@ -59,9 +35,6 @@ if [[ $# -gt 1 ]]; then
     exit 1
 elif  [[ $# -eq 1 ]]; then
     case "$1" in
-      -s)
-          use_sudo=1
-      ;;
       -h)
           print_help
           exit 0
@@ -74,39 +47,10 @@ elif  [[ $# -eq 1 ]]; then
   esac
 fi
 
-echo "Started installing dependencies for Memgraph"
-
-required_missing=0
-
-# install all dependencies on debian based operating systems
-for pkg in ${required_pkgs[@]}; do
-    if dpkg -s $pkg 2>/dev/null >/dev/null; then
-        echo "Found $pkg"
-    elif (( $use_sudo )); then
-        echo "Installing $pkg"
-        if [[ ! `sudo apt-get -y install $pkg` ]]; then
-            echo "Didn't install $pkg [required]"
-            required_missing=1
-        fi
-    else
-        echo "Missing $pkg [required]"
-        required_missing=1
-    fi
-done
-
-for pkg in ${optional_pkgs[@]}; do
-    if dpkg -s $pkg 2>/dev/null >/dev/null; then
-        echo "Found $pkg [optional]"
-    else
-        echo "Missing $pkg [optional]"
-    fi
-done
-
-if (( $required_missing )); then
-    echo "Missing required packages. EXITING!"
-    echo "Please, install required packages and rerun $0 again."
-    exit 2
-fi
+DISTRO=$(operating_system)
+echo "ALL BUILD PACKAGES: $($DIR/environment/os/$DISTRO.sh list MEMGRAPH_BUILD_DEPS)"
+$DIR/environment/os/$DISTRO.sh check MEMGRAPH_BUILD_DEPS
+echo "All packages are in-place..."
 
 # create a default build directory
 mkdir -p ./build
@@ -136,14 +80,14 @@ echo \
   (ql:quickload '(:lcp :lcp/test) :silent t)
   " | sbcl --script
 
-# setup libs (download)
+# Setup libs (download).
 cd libs
 ./cleanup.sh
 ./setup.sh
 cd ..
 
-# setup qa dependencies
-setup_virtualenv tests/qa
+# setup gql_behave dependencies
+setup_virtualenv tests/gql_behave
 
 # setup stress dependencies
 setup_virtualenv tests/stress
@@ -151,4 +95,18 @@ setup_virtualenv tests/stress
 # setup integration/ldap dependencies
 setup_virtualenv tests/integration/ldap
 
+# Setup tests dependencies.
+# cd tests
+# ./setup.sh
+# cd ..
+# TODO(gitbuda): Remove setup_virtualenv, replace it with tests/ve3. Take care
+# of the build order because tests/setup.py builds pymgclient which depends on
+# mgclient which is build after this script by calling make.
+
 echo "Done installing dependencies for Memgraph"
+
+echo "Linking git hooks"
+for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
+  ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook"
+  echo "Added $hook hook"
+done;

libs/.gitignore

@@ -4,3 +4,4 @@
 !cleanup.sh
 !CMakeLists.txt
 !__main.cpp
+!jemalloc.cmake

libs/CMakeLists.txt

@@ -1,11 +1,15 @@
 include(ExternalProject)
 
+include(GNUInstallDirs)
+
 include(ProcessorCount)
 ProcessorCount(NPROC)
 if (NPROC EQUAL 0)
   set(NPROC 1)
 endif()
 
+set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+
 # convenience functions
 function(import_header_library name include_dir)
   add_library(${name} INTERFACE IMPORTED GLOBAL)
@@ -102,7 +106,7 @@ import_external_library(benchmark STATIC
 
 # setup fmt format
 import_external_library(fmt STATIC
-  ${CMAKE_CURRENT_SOURCE_DIR}/fmt/lib/libfmt.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/fmt/${CMAKE_INSTALL_LIBDIR}/libfmt.a
   ${CMAKE_CURRENT_SOURCE_DIR}/fmt/include
   # Skip testing.
   CMAKE_ARGS -DFMT_TEST=OFF)
@@ -149,24 +153,6 @@ import_external_library(gflags STATIC
              -DBUILD_gflags_nothreads_LIB=OFF
              -DGFLAGS_NO_FILENAMES=${GFLAGS_NO_FILENAMES})
 
-# Setup google logging after gflags (so that glog can use it).
-set(GLOG_DISABLE_OPTIONS "0")
-if ("${CMAKE_BUILD_TYPE}" MATCHES "^(R|r)(E|e)(L|l).+")
-  set(GLOG_DISABLE_OPTIONS "1")
-endif()
-
-# Setup google logging after gflags (so that glog can use it).
-import_external_library(glog STATIC
-  ${CMAKE_CURRENT_SOURCE_DIR}/glog/lib/libglog.a
-  ${CMAKE_CURRENT_SOURCE_DIR}/glog/include
-  DEPENDS gflags-proj
-  CMAKE_ARGS -Dgflags_DIR=${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/cmake/gflags
-             -DBUILD_TESTING=OFF
-             -DGLOG_NO_FILENAMES=${GLOG_DISABLE_OPTIONS}
-             -DGLOG_NO_STACKTRACE=${GLOG_DISABLE_OPTIONS}
-             -DGLOG_NO_BUFFER_SETTINGS=${GLOG_DISABLE_OPTIONS}
-             -DGLOG_NO_TIME_PID_FILENAME=${GLOG_DISABLE_OPTIONS})
-
 # Setup cppitertools
 import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR})
 
@@ -195,7 +181,9 @@ import_external_library(zlib STATIC
 import_external_library(rocksdb STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/lib/librocksdb.a
   ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include
-  CMAKE_ARGS -DUSE_RTTI=ON
+  DEPENDS gflags-proj
+  CMAKE_ARGS -Dgflags_DIR=${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/cmake/gflags
+             -DUSE_RTTI=ON
              -DWITH_TESTS=OFF
              -DCMAKE_INSTALL_LIBDIR=lib
              -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
@@ -220,3 +208,11 @@ import_external_library(mgclient STATIC
              -DBUILD_TESTING=OFF)
 find_package(OpenSSL REQUIRED)
 target_link_libraries(mgclient INTERFACE ${OPENSSL_LIBRARIES})
+
+# Setup spdlog
+import_external_library(spdlog STATIC
+  ${CMAKE_CURRENT_SOURCE_DIR}/spdlog/${CMAKE_INSTALL_LIBDIR}/libspdlog.a
+  ${CMAKE_CURRENT_SOURCE_DIR}/spdlog/include
+  BUILD_COMMAND $(MAKE) spdlog)
+
+include(jemalloc.cmake)

libs/jemalloc.cmake

@@ -0,0 +1,55 @@
+set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
+
+set(JEMALLOC_SRCS
+    ${JEMALLOC_DIR}/src/arena.c
+    ${JEMALLOC_DIR}/src/background_thread.c
+    ${JEMALLOC_DIR}/src/base.c
+    ${JEMALLOC_DIR}/src/bin.c
+    ${JEMALLOC_DIR}/src/bitmap.c
+    ${JEMALLOC_DIR}/src/ckh.c
+    ${JEMALLOC_DIR}/src/ctl.c
+    ${JEMALLOC_DIR}/src/div.c
+    ${JEMALLOC_DIR}/src/extent.c
+    ${JEMALLOC_DIR}/src/extent_dss.c
+    ${JEMALLOC_DIR}/src/extent_mmap.c
+    ${JEMALLOC_DIR}/src/hash.c
+    ${JEMALLOC_DIR}/src/hook.c
+    ${JEMALLOC_DIR}/src/jemalloc.c
+    ${JEMALLOC_DIR}/src/large.c
+    ${JEMALLOC_DIR}/src/log.c
+    ${JEMALLOC_DIR}/src/malloc_io.c
+    ${JEMALLOC_DIR}/src/mutex.c
+    ${JEMALLOC_DIR}/src/mutex_pool.c
+    ${JEMALLOC_DIR}/src/nstime.c
+    ${JEMALLOC_DIR}/src/pages.c
+    ${JEMALLOC_DIR}/src/prng.c
+    ${JEMALLOC_DIR}/src/prof.c
+    ${JEMALLOC_DIR}/src/rtree.c
+    ${JEMALLOC_DIR}/src/sc.c
+    ${JEMALLOC_DIR}/src/stats.c
+    ${JEMALLOC_DIR}/src/sz.c
+    ${JEMALLOC_DIR}/src/tcache.c
+    ${JEMALLOC_DIR}/src/test_hooks.c
+    ${JEMALLOC_DIR}/src/ticker.c
+    ${JEMALLOC_DIR}/src/tsd.c
+    ${JEMALLOC_DIR}/src/witness.c
+    ${JEMALLOC_DIR}/src/safety_check.c
+)
+
+add_library(jemalloc ${JEMALLOC_SRCS})
+target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
+
+find_package(Threads REQUIRED)
+target_link_libraries(jemalloc PUBLIC Threads::Threads)
+
+target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
+
+if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
+    target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
+endif()
+
+target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
+# for RTLD_NEXT
+target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
+
+set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)

libs/setup.sh

@@ -2,8 +2,9 @@
 
 # Download external dependencies.
 
+local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
 working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd ${working_dir}
+cd "${working_dir}"
 
 # Clones a git repository and optionally cherry picks additional commits. The
 # function will try to preserve any local changes in the repo.
@@ -15,7 +16,11 @@ clone () {
   shift 3
   # Clone if there's no repo.
   if [[ ! -d "$dir_name" ]]; then
-    git clone "$git_repo" "$dir_name"
+    echo "Cloning from $git_repo"
+    # If the clone fails, it doesn't make sense to continue with the function
+    # execution but the whole script should continue executing because we might
+    # clone the same repo from a different source.
+    git clone "$git_repo" "$dir_name" || return 1
   fi
   pushd "$dir_name"
   # Just fetch new commits from remote repository. Don't merge/pull them in, so
@@ -29,12 +34,17 @@ clone () {
   # Stash regardless of local_changes, so that a user gets a message on stdout.
   git stash
   # Checkout the primary commit (there's no need to pull/merge).
-  git checkout $checkout_id
+  # The checkout fail should exit this script immediately because the target
+  # commit is not there and that will most likely create build-time errors.
+  git checkout "$checkout_id" || exit 1
   # Apply any optional cherry pick fixes.
   while [[ $# -ne 0 ]]; do
     local cherry_pick_id=$1
     shift
-    git cherry-pick -n $cherry_pick_id
+    # The cherry-pick fail should exit this script immediately because the
+    # target commit is not there and that will most likely create build-time
+    # errors.
+    git cherry-pick -n "$cherry_pick_id" || exit 1
   done
   # Reapply any local changes.
   if [[ $local_changes == true ]]; then
@@ -43,96 +53,182 @@ clone () {
   popd
 }
 
+file_get_try_double () {
+    primary_url="$1"
+    secondary_url="$2"
+    echo "Download primary from $primary_url secondary from $secondary_url"
+    if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
+    if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
+    filename="$(basename "$secondary_url")"
+    wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
+    echo ""
+}
+
+repo_clone_try_double () {
+    primary_url="$1"
+    secondary_url="$2"
+    folder_name="$3"
+    ref="$4"
+    echo "Cloning primary from $primary_url secondary from $secondary_url"
+    if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
+    if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
+    if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
+    if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
+    clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
+    echo ""
+}
+
+# List all dependencies.
+
+# The reason for introducing primary and secondary urls are:
+#   * HTTPS is hard to cache
+#   * Remote development workflow is more flexible if people don't have to connect to VPN
+#   * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
+#   * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
+
+# The goal of having primary urls is to have links to the "local" cache of
+# dependencies where these dependencies could be downloaded as fast as
+# possible. The actual cache server could be on your local machine, on a
+# dedicated machine inside the build cluster or on the actual build machine.
+# Download from primary_urls might fail because the cache is not installed.
+declare -A primary_urls=(
+  ["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
+  ["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
+  ["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
+  ["fmt"]="http://$local_cache_host/git/fmt.git"
+  ["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
+  ["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
+  ["gtest"]="http://$local_cache_host/git/googletest.git"
+  ["gflags"]="http://$local_cache_host/git/gflags.git"
+  ["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
+  ["bzip2"]="http://$local_cache_host/git/bzip2.git"
+  ["zlib"]="http://$local_cache_host/git/zlib.git"
+  ["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
+  ["mgclient"]="http://$local_cache_host/git/mgclient.git"
+  ["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
+  ["spdlog"]="http://$local_cache_host/git/spdlog"
+  ["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
+  ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
+  ["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
+)
+
+# The goal of secondary urls is to have links to the "source of truth" of
+# dependencies, e.g., Github or S3. Download from secondary urls, if happens
+# at all, should never fail. In other words, if it fails, the whole build
+# should fail.
+declare -A secondary_urls=(
+  ["antlr4-code"]="https://github.com/antlr/antlr4.git"
+  ["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
+  ["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
+  ["fmt"]="https://github.com/fmtlib/fmt.git"
+  ["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
+  ["gbenchmark"]="https://github.com/google/benchmark.git"
+  ["gtest"]="https://github.com/google/googletest.git"
+  ["gflags"]="https://github.com/memgraph/gflags.git"
+  ["libbcrypt"]="https://github.com/rg3/libbcrypt"
+  ["bzip2"]="https://github.com/VFR-maniac/bzip2"
+  ["zlib"]="https://github.com/madler/zlib.git"
+  ["rocksdb"]="https://github.com/facebook/rocksdb.git"
+  ["mgclient"]="https://github.com/memgraph/mgclient.git"
+  ["pymgclient"]="https://github.com/memgraph/pymgclient.git"
+  ["spdlog"]="https://github.com/gabime/spdlog"
+  ["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
+  ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
+  ["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
+)
+
 # antlr
-antlr_generator_filename="antlr-4.6-complete.jar"
-# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
-wget -nv -O ${antlr_generator_filename} http://deps.memgraph.io/${antlr_generator_filename}
-# git clone https://github.com/antlr/antlr4.git
+file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
+
 antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
-clone git://deps.memgraph.io/antlr4.git antlr4 $antlr4_tag
+repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
 # fix missing include
 sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
 
-# cppitertools
-# Use our fork that uses experimental/optional instead of unique_ptr in
-# DerefHolder. Once we move memgraph to c++17 we can use cpp17 branch from
-# original repo.
-# git clone https://github.com/memgraph/cppitertools.git
-cppitertools_tag="4231e0bc6fba2737b2a7a8a1576cf06186b0de6a" # experimental_optional 17 Aug 2017
-clone git://deps.memgraph.io/cppitertools.git cppitertools $cppitertools_tag
+# cppitertools v2.0 2019-12-23
+cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
+repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
 
 # fmt
-# git clone https://github.com/fmtlib/fmt.git
-fmt_tag="7fa8f8fa48b0903deab5bb42e6760477173ac485" # v3.0.1
-# Commit which fixes an issue when compiling with C++14 and higher.
-fmt_cxx14_fix="b9aaa507fc49680d037fd84c043f747a395bce04"
-clone git://deps.memgraph.io/fmt.git fmt $fmt_tag $fmt_cxx14_fix
+fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
+repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
 
 # rapidcheck
-# git clone https://github.com/emil-e/rapidcheck.git
-rapidcheck_tag="853e14f0f4313a9eb3c71e24848373e7b843dfd1" # Jun 23, 2017
-clone git://deps.memgraph.io/rapidcheck.git rapidcheck $rapidcheck_tag
+rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
+repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
 
 # google benchmark
-# git clone https://github.com/google/benchmark.git
 benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
-clone git://deps.memgraph.io/benchmark.git benchmark $benchmark_tag
+repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
 
 # google test
-# git clone https://github.com/google/googletest.git
 googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
-clone git://deps.memgraph.io/googletest.git googletest $googletest_tag
-
-# google logging
-# git clone https://github.com/memgraph/glog.git
-glog_tag="042a21657e79784226babab8b942f7bd0949635f" # custom version (v0.3.5+)
-clone git://deps.memgraph.io/glog.git glog $glog_tag
+repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
 
 # google flags
-# git clone https://github.com/memgraph/gflags.git
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
-clone git://deps.memgraph.io/gflags.git gflags $gflags_tag
+repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
 
 # libbcrypt
-# git clone https://github.com/rg3/libbcrypt
 libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
-clone git://deps.memgraph.io/libbcrypt.git libbcrypt $libbcrypt_tag
+repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
 
 # neo4j
-wget -nv http://deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
-tar -xzf neo4j.tar.gz
-rm -rf neo4j
+file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
+tar -xzf neo4j-community-3.2.3-unix.tar.gz
 mv neo4j-community-3.2.3 neo4j
-rm neo4j.tar.gz
+rm neo4j-community-3.2.3-unix.tar.gz
 
 # nlohmann json
 # We wget header instead of cloning repo since repo is huge (lots of test data).
 # We use head on Sep 1, 2017 instead of last release since it was long time ago.
 mkdir -p json
 cd json
-# wget "https://raw.githubusercontent.com/nlohmann/json/91e003285312167ad8365f387438ea371b465a7e/src/json.hpp"
-wget -nv http://deps.memgraph.io/json.hpp
+file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
 cd ..
 
 bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
-clone git://deps.memgraph.io/bzip2.git bzip2 $bzip2_tag
+repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
 
 zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
-clone git://deps.memgraph.io/zlib.git zlib $zlib_tag
+repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
 
-rocksdb_tag="641fae60f63619ed5d0c9d9e4c4ea5a0ffa3e253" # v5.18.3 Feb 11, 2019
-clone git://deps.memgraph.io/rocksdb.git rocksdb $rocksdb_tag
-# fix compilation flags to work with clang 8
-sed -i 's/-Wshadow/-Wno-defaulted-function-deleted/' rocksdb/CMakeLists.txt
+rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
+repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
 # remove shared library from install dependencies
 sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
 
 # mgclient
-mgclient_tag="fe94b3631385ef5dbe40a3d8458860dbcc33e6ea" # May 27, 2019
-# git clone https://github.com/memgraph/mgclient.git
-clone git://deps.memgraph.io/mgclient.git mgclient $mgclient_tag
+mgclient_tag="v1.2.0" # (2021-01-14)
+repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
 sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
+
+# pymgclient
+pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
+repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
+
+spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
+repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
+
+jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
+repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
+pushd jemalloc
+# ThreadPool select job randomly, and there can be some threads that had been
+# performed some memory heavy task before and will be inactive for some time,
+# but until it will became active again, the memory will not be freed since by
+# default each thread has it's own arena, but there should be not more then
+# 4*CPU arenas (see opt.nareans description).
+#
+# By enabling percpu_arena number of arenas limited to number of CPUs and hence
+# this problem should go away.
+#
+# muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
+# avoid spurious latencies and additional work associated with
+# MADV_DONTNEED. See
+# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
+./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
+popd

query_modules/CMakeLists.txt

@@ -32,7 +32,8 @@ install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_exampl
 
 # Install the Python modules
 install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
-install(FILES pagerank.py DESTINATION lib/memgraph/query_modules)
+install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
+install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)
 install(FILES wcc.py DESTINATION lib/memgraph/query_modules)
 
 if (MG_ENTERPRISE)

query_modules/graph_analyzer.py

@@ -1,10 +1,21 @@
+import sys
 import mgp
-
 from collections import OrderedDict
 from itertools import chain, repeat
 from inspect import cleandoc
-from typing import List, Tuple, Optional
-import networkx as nx
+from typing import List, Tuple
+try:
+    import networkx as nx
+except ImportError as import_error:
+    sys.stderr.write((
+        '\n'
+        'NOTE: Please install networkx to be able to use graph_analyzer '
+        'module. Using Python:\n'
+        + sys.version +
+        '\n'))
+    raise import_error
+# Imported last because it also depends on networkx.
+from mgp_networkx import MemgraphMultiDiGraph  # noqa E402
 
 
 _MAX_LIST_SIZE = 10
@@ -47,7 +58,7 @@ def analyze(context: mgp.ProcCtx,
     Example call (with parameter):
         CALL graph_analyzer.analyze(['nodes', 'edges']) YIELD *;
     '''
-    g = _convert_to_multidigraph(context)
+    g = MemgraphMultiDiGraph(ctx=context)
     recs = _analyze_graph(context, g, analyses)
     return [mgp.Record(name=name, value=value) for name, value in recs]
 
@@ -81,7 +92,11 @@ def analyze_subgraph(context: mgp.ProcCtx,
         YIELD *
         RETURN name, value;
     '''
-    g = _convert_to_subgraph_multidigraph(context, vertices, edges)
+    vertices, edges = map(set, [vertices, edges])
+    g = nx.subgraph_view(
+        MemgraphMultiDiGraph(ctx=context),
+        lambda n: n in vertices,
+        lambda n1, n2, e: e in edges)
     recs = _analyze_graph(context, g, analyses)
     return [mgp.Record(name=name, value=value) for name, value in recs]
 
@@ -263,36 +278,3 @@ def _strongly_components(g: nx.MultiDiGraph):
     '''Returns number of strongly connected components.'''
     comps = nx.algorithms.components.number_strongly_connected_components(g)
     return 'Number of strongly connected components', comps
-
-
-def _convert_to_multidigraph(context: mgp.ProcCtx,
-                             ) -> Optional[nx.MultiDiGraph]:
-    g = nx.MultiDiGraph()
-
-    for v in context.graph.vertices:
-        context.check_must_abort()
-        g.add_node(v.id)
-
-    for v in context.graph.vertices:
-        context.check_must_abort()
-        for e in v.out_edges:
-            g.add_edge(e.from_vertex.id, e.to_vertex.id)
-
-    return g
-
-
-def _convert_to_subgraph_multidigraph(context: mgp.ProcCtx,
-                                      vertices: mgp.List[mgp.Vertex],
-                                      edges: mgp.List[mgp.Edge]
-                                      ) -> Optional[nx.MultiDiGraph]:
-    g = nx.MultiDiGraph()
-
-    for v in vertices:
-        context.check_must_abort()
-        g.add_node(v.id)
-
-    for e in edges:
-        context.check_must_abort()
-        g.add_edge(e.from_vertex.id, e.to_vertex.id)
-
-    return g

query_modules/louvain/src/data_structures/graph.cpp

@@ -2,6 +2,7 @@
 
 #include <exception>
 #include <numeric>
+#include <stdexcept>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -17,9 +18,7 @@ Graph::Graph(uint32_t n_nodes) : n_nodes_(n_nodes), total_w_(0) {
   std::iota(community_.begin(), community_.end(), 0);
 }
 
-uint32_t Graph::Size() const {
-  return n_nodes_;
-}
+uint32_t Graph::Size() const { return n_nodes_; }
 
 uint32_t Graph::Community(uint32_t node) const { return community_.at(node); }
 
@@ -63,20 +62,15 @@ uint32_t Graph::Degree(uint32_t node) const {
   return static_cast<uint32_t>(adj_list_.at(node).size());
 }
 
-double Graph::IncidentWeight(uint32_t node) const {
-  return inc_w_.at(node);
-}
+double Graph::IncidentWeight(uint32_t node) const { return inc_w_.at(node); }
 
-double Graph::TotalWeight() const {
-  return total_w_;
-}
+double Graph::TotalWeight() const { return total_w_; }
 
 double Graph::Modularity() const {
   double ret = 0;
   // Since all weights should be positive, this implies that our graph has
   // no edges.
-  if (total_w_ == 0)
-    return 0;
+  if (total_w_ == 0) return 0;
 
   std::unordered_map<uint32_t, double> weight_c;
   std::unordered_map<uint32_t, double> degree_c;
@@ -102,4 +96,4 @@ const std::vector<Neighbour> &Graph::Neighbours(uint32_t node) const {
   return adj_list_.at(node);
 }
 
-} // namespace comdata
+}  // namespace comdata

query_modules/louvain/src/louvain_module.cpp

@@ -1,6 +1,7 @@
 #include "mg_procedure.h"
 
 #include <exception>
+#include <string>
 #include <unordered_map>
 
 #include "algorithms/algorithms.hpp"
@@ -109,7 +110,7 @@ std::optional<comdata::Graph> RunLouvain(
 }
 
 void communities(const mgp_list *args, const mgp_graph *graph,
-                        mgp_result *result, mgp_memory *memory) {
+                 mgp_result *result, mgp_memory *memory) {
   try {
     // Normalize vertex ids
     auto mem_to_louv_id = NormalizeVertexIds(graph, result, memory);
@@ -161,7 +162,7 @@ void communities(const mgp_list *args, const mgp_graph *graph,
 }
 
 void modularity(const mgp_list *args, const mgp_graph *graph,
-                       mgp_result *result, mgp_memory *memory) {
+                mgp_result *result, mgp_memory *memory) {
   try {
     // Normalize vertex ids
     auto mem_to_louv_id = NormalizeVertexIds(graph, result, memory);
@@ -172,7 +173,8 @@ void modularity(const mgp_list *args, const mgp_graph *graph,
     if (!louvain_graph) return;
 
     // Return graph modularity after Louvain
-    // TODO(ipaljak) - consider allowing the user to specify seed communities and
+    // TODO(ipaljak) - consider allowing the user to specify seed communities
+    // and
     //                 yield modularity values both before and after running
     //                 louvain.
     mgp_result_record *record = mgp_result_new_record(result);
@@ -203,7 +205,7 @@ void modularity(const mgp_list *args, const mgp_graph *graph,
   }
 }
 
-} // namespace
+}  // namespace
 
 extern "C" int mgp_init_module(struct mgp_module *module,
                                struct mgp_memory *memory) {
@@ -215,7 +217,7 @@ extern "C" int mgp_init_module(struct mgp_module *module,
     return 1;
 
   struct mgp_proc *modularity_proc =
-    mgp_module_add_read_procedure(module, "modularity", modularity);
+      mgp_module_add_read_procedure(module, "modularity", modularity);
   if (!modularity_proc) return 1;
   if (!mgp_proc_add_result(modularity_proc, "modularity", mgp_type_float()))
     return 1;
@@ -223,6 +225,4 @@ extern "C" int mgp_init_module(struct mgp_module *module,
   return 0;
 }
 
-extern "C" int mgp_shutdown_module() {
-  return 0;
-}
+extern "C" int mgp_shutdown_module() { return 0; }

query_modules/louvain/test/unit/CMakeLists.txt

@@ -17,7 +17,7 @@ function(add_unit_test test_cpp)
   # TODO: this is a temporary workaround the test build warnings
   target_compile_options(${target_name} PRIVATE -Wno-comment -Wno-sign-compare
                                                 -Wno-unused-variable)
-  target_link_libraries(${target_name} glog gflags gtest gtest_main Threads::Threads
+  target_link_libraries(${target_name} spdlog gflags gtest gtest_main Threads::Threads
                                        louvain-core louvain-test)
   # register test
   add_test(${target_name} ${exec_name})

query_modules/louvain/test/unit/graph.cpp

@@ -1,4 +1,3 @@
-#include <glog/logging.h>
 #include <gtest/gtest.h>
 
 #include "data_structures/graph.hpp"
@@ -10,8 +9,7 @@ bool CommunityCheck(const comdata::Graph &graph,
                     const std::vector<uint32_t> &c) {
   if (graph.Size() != c.size()) return false;
   for (uint32_t node_id = 0; node_id < graph.Size(); ++node_id)
-    if (graph.Community(node_id) != c[node_id])
-      return false;
+    if (graph.Community(node_id) != c[node_id]) return false;
   return true;
 }
 
@@ -20,8 +18,7 @@ bool DegreeCheck(const comdata::Graph &graph,
                  const std::vector<uint32_t> &deg) {
   if (graph.Size() != deg.size()) return false;
   for (uint32_t node_id = 0; node_id < graph.Size(); ++node_id)
-    if (graph.Degree(node_id) != deg[node_id])
-      return false;
+    if (graph.Degree(node_id) != deg[node_id]) return false;
   return true;
 }
 
@@ -127,10 +124,7 @@ TEST(Graph, Degrees) {
 
   // Chain
   // (0)--(1)--(2)--(3)--(4)
-  graph = BuildGraph(5, {{0, 1, 1},
-                     {1, 2, 1},
-                     {2, 3, 1},
-                     {3, 4, 1}});
+  graph = BuildGraph(5, {{0, 1, 1}, {1, 2, 1}, {2, 3, 1}, {3, 4, 1}});
   deg = {1, 2, 2, 2, 1};
   ASSERT_TRUE(DegreeCheck(graph, deg));
 
@@ -140,12 +134,8 @@ TEST(Graph, Degrees) {
   //   (1)   (2)
   //    |   /   \
   //   (4) (5)  (6)
-  graph = BuildGraph(7, {{0, 1, 1},
-                     {0, 2, 1},
-                     {0, 3, 1},
-                     {1, 4, 1},
-                     {2, 5, 1},
-                     {2, 6, 1}});
+  graph = BuildGraph(
+      7, {{0, 1, 1}, {0, 2, 1}, {0, 3, 1}, {1, 4, 1}, {2, 5, 1}, {2, 6, 1}});
   deg = {3, 2, 3, 1, 1, 1, 1};
   ASSERT_TRUE(DegreeCheck(graph, deg));
 
@@ -155,12 +145,12 @@ TEST(Graph, Degrees) {
   //  |  \ |  \
   // (2)--(3)-(4)
   graph = BuildGraph(5, {{0, 1, 1},
-                     {0, 2, 1},
-                     {0, 3, 1},
-                     {1, 3, 1},
-                     {1, 4, 1},
-                     {2, 3, 1},
-                     {3, 4, 1}});
+                         {0, 2, 1},
+                         {0, 3, 1},
+                         {1, 3, 1},
+                         {1, 4, 1},
+                         {2, 3, 1},
+                         {3, 4, 1}});
   deg = {3, 3, 2, 4, 2};
   ASSERT_TRUE(DegreeCheck(graph, deg));
 
@@ -170,15 +160,15 @@ TEST(Graph, Degrees) {
   //  |  \ |  \
   // (2*)--(3)-(4*)
   graph = BuildGraph(5, {{0, 1, 1},
-                     {0, 2, 1},
-                     {0, 3, 1},
-                     {1, 3, 1},
-                     {1, 4, 1},
-                     {2, 3, 1},
-                     {3, 4, 1},
-                     {1, 1, 1},
-                     {2, 2, 2},
-                     {4, 4, 4}});
+                         {0, 2, 1},
+                         {0, 3, 1},
+                         {1, 3, 1},
+                         {1, 4, 1},
+                         {2, 3, 1},
+                         {3, 4, 1},
+                         {1, 1, 1},
+                         {2, 2, 2},
+                         {4, 4, 4}});
   deg = {3, 4, 3, 4, 3};
   ASSERT_TRUE(DegreeCheck(graph, deg));
 
@@ -196,10 +186,7 @@ TEST(Graph, Weights) {
 
   // Chain
   // (0)--(1)--(2)--(3)--(4)
-  graph = BuildGraph(5, {{0, 1, 0.1},
-                     {1, 2, 0.5},
-                     {2, 3, 2.3},
-                     {3, 4, 4.2}});
+  graph = BuildGraph(5, {{0, 1, 0.1}, {1, 2, 0.5}, {2, 3, 2.3}, {3, 4, 4.2}});
   inc_w = {0.1, 0.6, 2.8, 6.5, 4.2};
   ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
   ASSERT_NEAR(graph.TotalWeight(), 7.1, 1e-6);
@@ -211,11 +198,11 @@ TEST(Graph, Weights) {
   //    |   /   \
   //   (4) (5)  (6)
   graph = BuildGraph(7, {{0, 1, 1.3},
-                     {0, 2, 0.2},
-                     {0, 3, 1},
-                     {1, 4, 3.2},
-                     {2, 5, 4.2},
-                     {2, 6, 0.7}});
+                         {0, 2, 0.2},
+                         {0, 3, 1},
+                         {1, 4, 3.2},
+                         {2, 5, 4.2},
+                         {2, 6, 0.7}});
   inc_w = {2.5, 4.5, 5.1, 1, 3.2, 4.2, 0.7};
   ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
   EXPECT_NEAR(graph.TotalWeight(), 10.6, 1e-6);
@@ -226,12 +213,12 @@ TEST(Graph, Weights) {
   //  |  \ |  \
   // (2)--(3)-(4)
   graph = BuildGraph(5, {{0, 1, 0.1},
-                     {0, 2, 0.2},
-                     {0, 3, 0.3},
-                     {1, 3, 0.4},
-                     {1, 4, 0.5},
-                     {2, 3, 0.6},
-                     {3, 4, 0.7}});
+                         {0, 2, 0.2},
+                         {0, 3, 0.3},
+                         {1, 3, 0.4},
+                         {1, 4, 0.5},
+                         {2, 3, 0.6},
+                         {3, 4, 0.7}});
   inc_w = {0.6, 1, 0.8, 2, 1.2};
   ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
   EXPECT_NEAR(graph.TotalWeight(), 2.8, 1e-6);
@@ -242,15 +229,15 @@ TEST(Graph, Weights) {
   //  |  \ |  \
   // (2*)--(3)-(4*)
   graph = BuildGraph(5, {{0, 1, 0.1},
-                     {0, 2, 0.2},
-                     {0, 3, 0.3},
-                     {1, 3, 0.4},
-                     {1, 4, 0.5},
-                     {2, 3, 0.6},
-                     {3, 4, 0.7},
-                     {1, 1, 0.8},
-                     {2, 2, 0.9},
-                     {4, 4, 1}});
+                         {0, 2, 0.2},
+                         {0, 3, 0.3},
+                         {1, 3, 0.4},
+                         {1, 4, 0.5},
+                         {2, 3, 0.6},
+                         {3, 4, 0.7},
+                         {1, 1, 0.8},
+                         {2, 2, 0.9},
+                         {4, 4, 1}});
   inc_w = {0.6, 1.8, 1.7, 2, 2.2};
   ASSERT_TRUE(IncidentWeightCheck(graph, inc_w));
   EXPECT_NEAR(graph.TotalWeight(), 5.5, 1e-6);
@@ -267,10 +254,7 @@ TEST(Graph, Modularity) {
 
   // Chain
   // (0)--(1)--(2)--(3)--(4)
-  graph = BuildGraph(5, {{0, 1, 0.1},
-                     {1, 2, 0.5},
-                     {2, 3, 2.3},
-                     {3, 4, 4.2}});
+  graph = BuildGraph(5, {{0, 1, 0.1}, {1, 2, 0.5}, {2, 3, 2.3}, {3, 4, 4.2}});
   std::vector<uint32_t> c = {0, 1, 1, 2, 2};
   SetCommunities(&graph, c);
   EXPECT_NEAR(graph.Modularity(), 0.036798254314620096, 1e-6);
@@ -282,11 +266,11 @@ TEST(Graph, Modularity) {
   //    |   /   \
   //   (4) (5)  (6)
   graph = BuildGraph(7, {{0, 1, 1.3},
-                     {0, 2, 0.2},
-                     {0, 3, 1},
-                     {1, 4, 3.2},
-                     {2, 5, 4.2},
-                     {2, 6, 0.7}});
+                         {0, 2, 0.2},
+                         {0, 3, 1},
+                         {1, 4, 3.2},
+                         {2, 5, 4.2},
+                         {2, 6, 0.7}});
   c = {0, 0, 1, 0, 0, 1, 2};
   SetCommunities(&graph, c);
   EXPECT_NEAR(graph.Modularity(), 0.4424617301530794, 1e-6);
@@ -297,12 +281,12 @@ TEST(Graph, Modularity) {
   //  |  \ |  \
   // (2)--(3)-(4)
   graph = BuildGraph(5, {{0, 1, 0.1},
-                     {0, 2, 0.2},
-                     {0, 3, 0.3},
-                     {1, 3, 0.4},
-                     {1, 4, 0.5},
-                     {2, 3, 0.6},
-                     {3, 4, 0.7}});
+                         {0, 2, 0.2},
+                         {0, 3, 0.3},
+                         {1, 3, 0.4},
+                         {1, 4, 0.5},
+                         {2, 3, 0.6},
+                         {3, 4, 0.7}});
   c = {0, 1, 1, 1, 1};
   SetCommunities(&graph, c);
   EXPECT_NEAR(graph.Modularity(), -0.022959183673469507, 1e-6);
@@ -313,15 +297,15 @@ TEST(Graph, Modularity) {
   //  |  \ |  \
   // (2*)--(3)-(4*)
   graph = BuildGraph(5, {{0, 1, 0.1},
-                     {0, 2, 0.2},
-                     {0, 3, 0.3},
-                     {1, 3, 0.4},
-                     {1, 4, 0.5},
-                     {2, 3, 0.6},
-                     {3, 4, 0.7},
-                     {1, 1, 0.8},
-                     {2, 2, 0.9},
-                     {4, 4, 1}});
+                         {0, 2, 0.2},
+                         {0, 3, 0.3},
+                         {1, 3, 0.4},
+                         {1, 4, 0.5},
+                         {2, 3, 0.6},
+                         {3, 4, 0.7},
+                         {1, 1, 0.8},
+                         {2, 2, 0.9},
+                         {4, 4, 1}});
   c = {0, 0, 0, 0, 1};
   SetCommunities(&graph, c);
   EXPECT_NEAR(graph.Modularity(), 0.188842975206611, 1e-6);

query_modules/mgp_networkx.py

@@ -0,0 +1,289 @@
+import sys
+import mgp
+import collections
+try:
+    import networkx as nx
+except ImportError as import_error:
+    sys.stderr.write((
+        '\n'
+        'NOTE: Please install networkx to be able to use Memgraph NetworkX '
+        'wrappers. Using Python:\n'
+        + sys.version +
+        '\n'))
+    raise import_error
+
+
+class MemgraphAdjlistOuterDict(collections.abc.Mapping):
+    __slots__ = ('_ctx', '_succ', '_multi')
+
+    def __init__(self, ctx, succ=True, multi=True):
+        self._ctx = ctx
+        self._succ = succ
+        self._multi = multi
+
+    def __getitem__(self, key):
+        if key not in self:
+            raise KeyError
+        return MemgraphAdjlistInnerDict(key, succ=self._succ,
+                                        multi=self._multi)
+
+    def __iter__(self):
+        return iter(self._ctx.graph.vertices)
+
+    def __len__(self):
+        return len(self._ctx.graph.vertices)
+
+    def __contains__(self, key):
+        if not isinstance(key, mgp.Vertex):
+            raise TypeError
+        return key in self._ctx.graph.vertices
+
+
+class MemgraphAdjlistInnerDict(collections.abc.Mapping):
+    __slots__ = ('_node', '_succ', '_multi', '_neighbors')
+
+    def __init__(self, node, succ=True, multi=True):
+        self._node = node
+        self._succ = succ
+        self._multi = multi
+        self._neighbors = None
+
+    def __getitem__(self, key):
+        # NOTE: NetworkX 2.4, classes/coreviews.py:143. UnionAtlas expects a
+        # KeyError when indexing with a vertex that is not a neighbor.
+        if key not in self:
+            raise KeyError
+        if not self._multi:
+            return UnhashableProperties(self._get_edge(key).properties)
+        return MemgraphEdgeKeyDict(self._node, key, self._succ)
+
+    def __iter__(self):
+        yield from self._get_neighbors()
+
+    def __len__(self):
+        return len(self._get_neighbors())
+
+    def __contains__(self, key):
+        if not isinstance(key, mgp.Vertex):
+            raise TypeError
+        return key in self._get_neighbors()
+
+    def _get_neighbors(self):
+        if not self._neighbors:
+            if self._succ:
+                self._neighbors = set(
+                    e.to_vertex for e in self._node.out_edges)
+            else:
+                self._neighbors = set(
+                    e.from_vertex for e in self._node.in_edges)
+        return self._neighbors
+
+    def _get_edge(self, neighbor):
+        if self._succ:
+            edge = list(filter(lambda e: e.to_vertex == neighbor,
+                               self._node.out_edges))
+        else:
+            edge = list(filter(lambda e: e.from_vertex == neighbor,
+                               self._node.in_edges))
+
+        assert len(edge) >= 1
+        if len(edge) > 1:
+            raise RuntimeError('Graph contains multiedges but '
+                               'is of non-multigraph type: {}'.format(edge))
+
+        return edge[0]
+
+
+class MemgraphEdgeKeyDict(collections.abc.Mapping):
+    __slots__ = ('_node', '_neighbor', '_succ', '_edges')
+
+    def __init__(self, node, neighbor, succ=True):
+        self._node = node
+        self._neighbor = neighbor
+        self._succ = succ
+        self._edges = None
+
+    def __getitem__(self, key):
+        if key not in self:
+            raise KeyError
+        return UnhashableProperties(key.properties)
+
+    def __iter__(self):
+        yield from self._get_edges()
+
+    def __len__(self):
+        return len(self._get_edges())
+
+    def __contains__(self, key):
+        if not isinstance(key, mgp.Edge):
+            raise TypeError
+        return key in self._get_edges()
+
+    def _get_edges(self):
+        if not self._edges:
+            if self._succ:
+                self._edges = list(filter(
+                    lambda e: e.to_vertex == self._neighbor,
+                    self._node.out_edges))
+            else:
+                self._edges = list(filter(
+                    lambda e: e.from_vertex == self._neighbor,
+                    self._node.in_edges))
+        return self._edges
+
+
+class UnhashableProperties(collections.abc.Mapping):
+    __slots__ = ('_properties')
+
+    def __init__(self, properties):
+        self._properties = properties
+
+    def __getitem__(self, key):
+        return self._properties[key]
+
+    def __iter__(self):
+        yield from self._properties
+
+    def __len__(self):
+        return len(self._properties)
+
+    def __contains__(self, key):
+        return key in self._properties
+
+    # NOTE: Explicitly disable hashing. See the comment in MemgraphNodeDict.
+    __hash__ = None
+
+
+class MemgraphNodeDict(collections.abc.Mapping):
+    __slots__ = ('_ctx',)
+
+    def __init__(self, ctx):
+        self._ctx = ctx
+
+    def __getitem__(self, key):
+        if key not in self:
+            raise KeyError
+        # NOTE: NetworkX 2.4, classes/digraph.py:484. NetworkX expects the
+        # tuples provided to add_nodes_from to be unhashable and cause a
+        # TypeError when trying to index the node dictionary. This happens
+        # because the data dictionary element of the tuple is unhashable. We do
+        # the same thing by returning an unhashable data dictionary.
+        return UnhashableProperties(key.properties)
+
+    def __iter__(self):
+        return iter(self._ctx.graph.vertices)
+
+    def __len__(self):
+        return len(self._ctx.graph.vertices)
+
+    def __contains__(self, key):
+        # NOTE: NetworkX 2.4, graph.py:425. Graph.__contains__ relies on
+        # self._node's (i.e. the dictionary produced by node_dict_factory)
+        # __contains__ to raise a TypeError when indexing with something weird,
+        # e.g. with sets. This is the behavior of dict.
+        if not isinstance(key, mgp.Vertex):
+            raise TypeError
+        return key in self._ctx.graph.vertices
+
+
+class MemgraphDiGraphBase:
+    def __init__(self, incoming_graph_data=None, ctx=None, multi=True,
+                 **kwargs):
+        # NOTE: We assume that our graph will never be given any initial data
+        # because we already pull our data from the Memgraph database. This
+        # assert is triggered by certain NetworkX procedures because they
+        # create a new instance of our class and try to populate it with their
+        # own data.
+        assert incoming_graph_data is None
+
+        # NOTE: We allow for ctx to be None in order to allow certain NetworkX
+        # procedures (such as `subgraph`) to work. Such procedures directly
+        # modify the graph's internal attributes and don't try to populate it
+        # with initial data or modify it.
+
+        self.node_dict_factory = lambda: MemgraphNodeDict(ctx) \
+            if ctx else self._error
+        self.node_attr_dict_factory = self._error
+
+        self.adjlist_outer_dict_factory = \
+            lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) \
+            if ctx else self._error
+        self.adjlist_inner_dict_factory = self._error
+        self.edge_key_dict_factory = self._error
+        self.edge_attr_dict_factory = self._error
+
+        # NOTE: We forbid any mutating operations because our graph is
+        # immutable and pulls its data from the Memgraph database.
+        for f in ['add_node', 'add_nodes_from', 'remove_node',
+                  'remove_nodes_from', 'add_edge', 'add_edges_from',
+                  'add_weighted_edges_from', 'new_edge_key', 'remove_edge',
+                  'remove_edges_from', 'update', 'clear']:
+            setattr(self, f, lambda *args, **kwargs: self._error())
+
+        super().__init__(None, **kwargs)
+
+        # NOTE: This is a necessary hack because NetworkX assumes that the
+        # customizable factory functions will only ever return *empty*
+        # dictionaries. In our case, the factory functions return our custom,
+        # already populated, dictionaries. Because self._pred and self._end are
+        # initialized by the same factory function, they end up storing the
+        # same adjacency lists which is not good. We correct that here.
+        self._pred = MemgraphAdjlistOuterDict(ctx, succ=False, multi=multi)
+
+    def _error(self):
+        raise RuntimeError('Modification operations are not supported')
+
+
+class MemgraphMultiDiGraph(MemgraphDiGraphBase, nx.MultiDiGraph):
+    def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
+        super().__init__(incoming_graph_data=incoming_graph_data,
+                         ctx=ctx, multi=True, **kwargs)
+
+
+def MemgraphMultiGraph(incoming_graph_data=None, ctx=None, **kwargs):
+    return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data,
+                                ctx=ctx, **kwargs).to_undirected(as_view=True)
+
+
+class MemgraphDiGraph(MemgraphDiGraphBase, nx.DiGraph):
+    def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
+        super().__init__(incoming_graph_data=incoming_graph_data,
+                         ctx=ctx, multi=False, **kwargs)
+
+
+def MemgraphGraph(incoming_graph_data=None, ctx=None, **kwargs):
+    return MemgraphDiGraph(incoming_graph_data=incoming_graph_data,
+                           ctx=ctx, **kwargs).to_undirected(as_view=True)
+
+
+class PropertiesDictionary(collections.abc.Mapping):
+    __slots__ = ('_ctx', '_prop', '_len')
+
+    def __init__(self, ctx, prop):
+        self._ctx = ctx
+        self._prop = prop
+        self._len = None
+
+    def __getitem__(self, vertex):
+        if vertex not in self:
+            raise KeyError
+        try:
+            return vertex.properties[self._prop]
+        except KeyError:
+            raise KeyError(("{} doesn\t have the required " +
+                            "property '{}'").format(vertex, self._prop))
+
+    def __iter__(self):
+        for v in self._ctx.graph.vertices:
+            if self._prop in v.properties:
+                yield v
+
+    def __len__(self):
+        if not self._len:
+            self._len = sum(1 for _ in self)
+        return self._len
+
+    def __contains__(self, vertex):
+        if not isinstance(vertex, mgp.Vertex):
+            raise TypeError
+        return self._prop in vertex.properties

query_modules/nxalg.py

@@ -0,0 +1,825 @@
+import sys
+import mgp
+try:
+    import networkx as nx
+    import numpy  # noqa E401
+    import scipy  # noqa E401
+except ImportError as import_error:
+    sys.stderr.write((
+        '\n'
+        'NOTE: Please install networkx, numpy, scipy to be able to '
+        'use proxied NetworkX algorithms. E.g., CALL nxalg.pagerank(...).\n'
+        'Using Python:\n'
+        + sys.version +
+        '\n'))
+    raise import_error
+# Imported last because it also depends on networkx.
+from mgp_networkx import (MemgraphMultiDiGraph, MemgraphDiGraph,  # noqa: E402
+                          MemgraphMultiGraph, MemgraphGraph,
+                          PropertiesDictionary)
+
+
+# networkx.algorithms.approximation.connectivity.node_connectivity
+@mgp.read_proc
+def node_connectivity(ctx: mgp.ProcCtx,
+                      source: mgp.Nullable[mgp.Vertex] = None,
+                      target: mgp.Nullable[mgp.Vertex] = None
+                      ) -> mgp.Record(connectivity=int):
+    return mgp.Record(connectivity=nx.node_connectivity(
+        MemgraphMultiDiGraph(ctx=ctx), source, target))
+
+
+# networkx.algorithms.assortativity.degree_assortativity_coefficient
+@mgp.read_proc
+def degree_assortativity_coefficient(
+        ctx: mgp.ProcCtx,
+        x: str = 'out',
+        y: str = 'in',
+        weight: mgp.Nullable[str] = None,
+        nodes: mgp.Nullable[mgp.List[mgp.Vertex]] = None
+) -> mgp.Record(assortativity=float):
+    return mgp.Record(assortativity=nx.degree_assortativity_coefficient(
+        MemgraphMultiDiGraph(ctx=ctx), x, y, weight, nodes))
+
+
+# networkx.algorithms.asteroidal.is_at_free
+@mgp.read_proc
+def is_at_free(ctx: mgp.ProcCtx) -> mgp.Record(is_at_free=bool):
+    return mgp.Record(is_at_free=nx.is_at_free(MemgraphGraph(ctx=ctx)))
+
+
+# networkx.algorithms.bipartite.basic.is_bipartite
+@mgp.read_proc
+def is_bipartite(ctx: mgp.ProcCtx) -> mgp.Record(is_bipartite=bool):
+    return mgp.Record(is_bipartite=nx.is_bipartite(
+        MemgraphMultiDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.boundary.node_boundary
+@mgp.read_proc
+def node_boundary(ctx: mgp.ProcCtx,
+                  nbunch1: mgp.List[mgp.Vertex],
+                  nbunch2: mgp.Nullable[mgp.List[mgp.Vertex]] = None
+                  ) -> mgp.Record(boundary=mgp.List[mgp.Vertex]):
+    return mgp.Record(boundary=list(nx.node_boundary(
+        MemgraphMultiDiGraph(ctx=ctx), nbunch1, nbunch2)))
+
+
+# networkx.algorithms.bridges.bridges
+@mgp.read_proc
+def bridges(ctx: mgp.ProcCtx,
+            root: mgp.Nullable[mgp.Vertex] = None
+            ) -> mgp.Record(bridges=mgp.List[mgp.Edge]):
+    g = MemgraphMultiGraph(ctx=ctx)
+    return mgp.Record(
+        bridges=[next(iter(g[u][v]))
+                 for u, v in nx.bridges(MemgraphGraph(ctx=ctx),
+                                        root=root)])
+
+
+# networkx.algorithms.centrality.betweenness_centrality
+@mgp.read_proc
+def betweenness_centrality(ctx: mgp.ProcCtx,
+                           k: mgp.Nullable[int] = None,
+                           normalized: bool = True,
+                           weight: mgp.Nullable[str] = None,
+                           endpoints: bool = False,
+                           seed: mgp.Nullable[int] = None
+                           ) -> mgp.Record(node=mgp.Vertex,
+                                           betweenness=mgp.Number):
+    return [mgp.Record(node=n, betweenness=b)
+            for n, b in nx.betweenness_centrality(
+                    MemgraphDiGraph(ctx=ctx), k=k, normalized=normalized,
+                    weight=weight, endpoints=endpoints, seed=seed).items()]
+
+
+# networkx.algorithms.chains.chain_decomposition
+@mgp.read_proc
+def chain_decomposition(ctx: mgp.ProcCtx,
+                        root: mgp.Nullable[mgp.Vertex] = None
+                        ) -> mgp.Record(chains=mgp.List[mgp.List[mgp.Edge]]):
+    g = MemgraphMultiGraph(ctx=ctx)
+    return mgp.Record(
+        chains=[[next(iter(g[u][v])) for u, v in d]
+                for d in nx.chain_decomposition(MemgraphGraph(ctx=ctx),
+                                                root=root)])
+
+
+# networkx.algorithms.chordal.is_chordal
+@mgp.read_proc
+def is_chordal(ctx: mgp.ProcCtx) -> mgp.Record(is_chordal=bool):
+    return mgp.Record(is_chordal=nx.is_chordal(MemgraphGraph(ctx=ctx)))
+
+
+# networkx.algorithms.clique.find_cliques
+@mgp.read_proc
+def find_cliques(ctx: mgp.ProcCtx
+                 ) -> mgp.Record(cliques=mgp.List[mgp.List[mgp.Vertex]]):
+    return mgp.Record(cliques=list(nx.find_cliques(
+        MemgraphMultiGraph(ctx=ctx))))
+
+
+# networkx.algorithms.cluster.clustering
+@mgp.read_proc
+def clustering(ctx: mgp.ProcCtx,
+               nodes: mgp.Nullable[mgp.List[mgp.Vertex]] = None,
+               weight: mgp.Nullable[str] = None
+               ) -> mgp.Record(node=mgp.Vertex, clustering=mgp.Number):
+    return [mgp.Record(node=n, clustering=c)
+            for n, c in nx.clustering(
+                    MemgraphDiGraph(ctx=ctx), nodes=nodes,
+                    weight=weight).items()]
+
+
+# networkx.algorithms.coloring.greedy_color
+@mgp.read_proc
+def greedy_color(ctx: mgp.ProcCtx,
+                 strategy: str = 'largest_first',
+                 interchange: bool = False
+                 ) -> mgp.Record(node=mgp.Vertex, color=int):
+    return [mgp.Record(node=n, color=c) for n, c in nx.greedy_color(
+        MemgraphMultiDiGraph(ctx=ctx), strategy, interchange).items()]
+
+
+# networkx.algorithms.communicability_alg.communicability
+@mgp.read_proc
+def communicability(ctx: mgp.ProcCtx
+                    ) -> mgp.Record(node1=mgp.Vertex, node2=mgp.Vertex,
+                                    communicability=mgp.Number):
+    return [mgp.Record(node1=n1, node2=n2, communicability=v)
+            for n1, d in nx.communicability(MemgraphGraph(ctx=ctx)).items()
+            for n2, v in d.items()]
+
+
+# networkx.algorithms.community.kclique.k_clique_communities
+@mgp.read_proc
+def k_clique_communities(
+        ctx: mgp.ProcCtx,
+        k: int,
+        cliques: mgp.Nullable[mgp.List[mgp.List[mgp.Vertex]]] = None
+) -> mgp.Record(communities=mgp.List[mgp.List[mgp.Vertex]]):
+    return mgp.Record(communities=[
+        list(s) for s in nx.community.k_clique_communities(
+            MemgraphMultiGraph(ctx=ctx), k, cliques)])
+
+
+# networkx.algorithms.approximation.kcomponents.k_components
+@mgp.read_proc
+def k_components(ctx: mgp.ProcCtx,
+                 density: mgp.Number = 0.95
+                 ) -> mgp.Record(k=int,
+                                 components=mgp.List[mgp.List[mgp.Vertex]]):
+    kcomps = nx.k_components(MemgraphMultiGraph(ctx=ctx), density)
+
+    return [mgp.Record(k=k, components=[list(s) for s in comps])
+            for k, comps in kcomps.items()]
+
+
+# networkx.algorithms.components.biconnected_components
+@mgp.read_proc
+def biconnected_components(
+        ctx: mgp.ProcCtx
+) -> mgp.Record(components=mgp.List[mgp.List[mgp.Vertex]]):
+    comps = nx.biconnected_components(MemgraphMultiGraph(ctx=ctx))
+    return mgp.Record(components=[list(s) for s in comps])
+
+
+# networkx.algorithms.components.strongly_connected_components
+@mgp.read_proc
+def strongly_connected_components(
+        ctx: mgp.ProcCtx
+) -> mgp.Record(components=mgp.List[mgp.List[mgp.Vertex]]):
+    comps = nx.strongly_connected_components(MemgraphMultiDiGraph(ctx=ctx))
+    return mgp.Record(components=[list(s) for s in comps])
+
+
+# networkx.algorithms.connectivity.edge_kcomponents.k_edge_components
+#
+# NOTE: NetworkX 2.4, algorithms/connectivity/edge_kcompnents.py:367. We create
+# a *copy* of the graph because the algorithm copies the graph using
+# __class__() and tries to modify it.
+@mgp.read_proc
+def k_edge_components(
+        ctx: mgp.ProcCtx,
+        k: int
+) -> mgp.Record(components=mgp.List[mgp.List[mgp.Vertex]]):
+    return mgp.Record(components=[list(s) for s in nx.k_edge_components(
+        nx.DiGraph(MemgraphDiGraph(ctx=ctx)), k)])
+
+
+# networkx.algorithms.core.core_number
+@mgp.read_proc
+def core_number(ctx: mgp.ProcCtx
+                ) -> mgp.Record(node=mgp.Vertex, core=mgp.Number):
+    return [mgp.Record(node=n, core=c)
+            for n, c in nx.core_number(MemgraphDiGraph(ctx=ctx)).items()]
+
+
+# networkx.algorithms.covering.is_edge_cover
+@mgp.read_proc
+def is_edge_cover(ctx: mgp.ProcCtx, cover: mgp.List[mgp.Edge]
+                  ) -> mgp.Record(is_edge_cover=bool):
+    cover = set([(e.from_vertex, e.to_vertex) for e in cover])
+    return mgp.Record(is_edge_cover=nx.is_edge_cover(
+        MemgraphMultiGraph(ctx=ctx), cover))
+
+
+# networkx.algorithms.cycles.find_cycle
+@mgp.read_proc
+def find_cycle(ctx: mgp.ProcCtx,
+               source: mgp.Nullable[mgp.List[mgp.Vertex]] = None,
+               orientation: mgp.Nullable[str] = None
+               ) -> mgp.Record(cycle=mgp.Nullable[mgp.List[mgp.Edge]]):
+    try:
+        return mgp.Record(cycle=[e for _, _, e in nx.find_cycle(
+            MemgraphMultiDiGraph(ctx=ctx), source, orientation)])
+    except nx.NetworkXNoCycle:
+        return mgp.Record(cycle=None)
+
+
+# networkx.algorithms.cycles.simple_cycles
+#
+# NOTE: NetworkX 2.4, algorithms/cycles.py:183. We create a *copy* of the graph
+# because the algorithm copies the graph using type() and tries to pass initial
+# data.
+@mgp.read_proc
+def simple_cycles(ctx: mgp.ProcCtx
+                  ) -> mgp.Record(cycles=mgp.List[mgp.List[mgp.Vertex]]):
+    return mgp.Record(cycles=list(nx.simple_cycles(
+        nx.MultiDiGraph(MemgraphMultiDiGraph(ctx=ctx)).copy())))
+
+
+# networkx.algorithms.cuts.node_expansion
+@mgp.read_proc
+def node_expansion(ctx: mgp.ProcCtx, s: mgp.List[mgp.Vertex]
+                   ) -> mgp.Record(node_expansion=mgp.Number):
+    return mgp.Record(node_expansion=nx.node_expansion(
+        MemgraphMultiDiGraph(ctx=ctx), set(s)))
+
+
+# networkx.algorithms.dag.topological_sort
+@mgp.read_proc
+def topological_sort(ctx: mgp.ProcCtx
+                     ) -> mgp.Record(nodes=mgp.Nullable[mgp.List[mgp.Vertex]]):
+    return mgp.Record(nodes=list(nx.topological_sort(
+        MemgraphMultiDiGraph(ctx=ctx))))
+
+
+# networkx.algorithms.dag.ancestors
+@mgp.read_proc
+def ancestors(ctx: mgp.ProcCtx,
+              source: mgp.Vertex
+              ) -> mgp.Record(ancestors=mgp.List[mgp.Vertex]):
+    return mgp.Record(ancestors=list(nx.ancestors(
+        MemgraphMultiDiGraph(ctx=ctx), source)))
+
+
+# networkx.algorithms.dag.descendants
+@mgp.read_proc
+def descendants(ctx: mgp.ProcCtx,
+                source: mgp.Vertex
+                ) -> mgp.Record(descendants=mgp.List[mgp.Vertex]):
+    return mgp.Record(descendants=list(nx.descendants(
+        MemgraphMultiDiGraph(ctx=ctx), source)))
+
+
+# networkx.algorithms.distance_measures.center
+#
+# NOTE: Takes more parameters.
+@mgp.read_proc
+def center(ctx: mgp.ProcCtx) -> mgp.Record(center=mgp.List[mgp.Vertex]):
+    return mgp.Record(center=list(nx.center(MemgraphMultiDiGraph(ctx=ctx))))
+
+
+# networkx.algorithms.distance_measures.diameter
+#
+# NOTE: Takes more parameters.
+@mgp.read_proc
+def diameter(ctx: mgp.ProcCtx) -> mgp.Record(diameter=int):
+    return mgp.Record(diameter=nx.diameter(MemgraphMultiDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.distance_regular.is_distance_regular
+@mgp.read_proc
+def is_distance_regular(ctx: mgp.ProcCtx
+                        ) -> mgp.Record(is_distance_regular=bool):
+    return mgp.Record(is_distance_regular=nx.is_distance_regular(
+        MemgraphMultiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.strongly_regular.is_strongly_regular
+@mgp.read_proc
+def is_strongly_regular(ctx: mgp.ProcCtx
+                        ) -> mgp.Record(is_strongly_regular=bool):
+    return mgp.Record(is_strongly_regular=nx.is_strongly_regular(
+        MemgraphMultiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.dominance.dominance_frontiers
+@mgp.read_proc
+def dominance_frontiers(ctx: mgp.ProcCtx, start: mgp.Vertex,
+                        ) -> mgp.Record(node=mgp.Vertex,
+                                        frontier=mgp.List[mgp.Vertex]):
+    return [mgp.Record(node=n, frontier=list(f))
+            for n, f in nx.dominance_frontiers(
+                    MemgraphMultiDiGraph(ctx=ctx), start).items()]
+
+
+# networkx.algorithms.dominance.immediate_dominatorss
+@mgp.read_proc
+def immediate_dominators(ctx: mgp.ProcCtx, start: mgp.Vertex,
+                         ) -> mgp.Record(node=mgp.Vertex,
+                                         dominator=mgp.Vertex):
+    return [mgp.Record(node=n, dominator=d)
+            for n, d in nx.immediate_dominators(
+                    MemgraphMultiDiGraph(ctx=ctx), start).items()]
+
+
+# networkx.algorithms.dominating.dominating_set
+@mgp.read_proc
+def dominating_set(ctx: mgp.ProcCtx, start: mgp.Vertex,
+                   ) -> mgp.Record(dominating_set=mgp.List[mgp.Vertex]):
+    return mgp.Record(dominating_set=list(nx.dominating_set(
+        MemgraphMultiDiGraph(ctx=ctx), start)))
+
+
+# networkx.algorithms.efficiency_measures.local_efficiency
+@mgp.read_proc
+def local_efficiency(ctx: mgp.ProcCtx) -> mgp.Record(local_efficiency=float):
+    return mgp.Record(local_efficiency=nx.local_efficiency(
+        MemgraphMultiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.efficiency_measures.global_efficiency
+@mgp.read_proc
+def global_efficiency(ctx: mgp.ProcCtx) -> mgp.Record(global_efficiency=float):
+    return mgp.Record(global_efficiency=nx.global_efficiency(
+        MemgraphMultiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.euler.is_eulerian
+@mgp.read_proc
+def is_eulerian(ctx: mgp.ProcCtx) -> mgp.Record(is_eulerian=bool):
+    return mgp.Record(is_eulerian=nx.is_eulerian(
+        MemgraphMultiDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.euler.is_semieulerian
+@mgp.read_proc
+def is_semieulerian(ctx: mgp.ProcCtx) -> mgp.Record(is_semieulerian=bool):
+    return mgp.Record(is_semieulerian=nx.is_semieulerian(
+        MemgraphMultiDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.euler.has_eulerian_path
+@mgp.read_proc
+def has_eulerian_path(ctx: mgp.ProcCtx) -> mgp.Record(has_eulerian_path=bool):
+    return mgp.Record(has_eulerian_path=nx.has_eulerian_path(
+        MemgraphMultiDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.hierarchy.flow_hierarchy
+@mgp.read_proc
+def flow_hierarchy(ctx: mgp.ProcCtx,
+                   weight: mgp.Nullable[str] = None
+                   ) -> mgp.Record(flow_hierarchy=float):
+    return mgp.Record(flow_hierarchy=nx.flow_hierarchy(
+        MemgraphMultiDiGraph(ctx=ctx), weight=weight))
+
+
+# networkx.algorithms.isolate.isolates
+@mgp.read_proc
+def isolates(ctx: mgp.ProcCtx) -> mgp.Record(isolates=mgp.List[mgp.Vertex]):
+    return mgp.Record(isolates=list(nx.isolates(
+        MemgraphMultiDiGraph(ctx=ctx))))
+
+
+# networkx.algorithms.isolate.is_isolate
+@mgp.read_proc
+def is_isolate(ctx: mgp.ProcCtx, n: mgp.Vertex
+               ) -> mgp.Record(is_isolate=bool):
+    return mgp.Record(is_isolate=nx.is_isolate(
+        MemgraphMultiDiGraph(ctx=ctx), n))
+
+
+# networkx.algorithms.isomorphism.is_isomorphic
+@mgp.read_proc
+def is_isomorphic(ctx: mgp.ProcCtx,
+                  nodes1: mgp.List[mgp.Vertex],
+                  edges1: mgp.List[mgp.Edge],
+                  nodes2: mgp.List[mgp.Vertex],
+                  edges2: mgp.List[mgp.Edge]
+                  ) -> mgp.Record(is_isomorphic=bool):
+    nodes1, edges1, nodes2, edges2 = map(set, [nodes1, edges1, nodes2, edges2])
+    g = MemgraphMultiDiGraph(ctx=ctx)
+    g1 = nx.subgraph_view(
+        g, lambda n: n in nodes1, lambda n1, n2, e: e in edges1)
+    g2 = nx.subgraph_view(
+        g, lambda n: n in nodes2, lambda n1, n2, e: e in edges2)
+    return mgp.Record(is_isomorphic=nx.is_isomorphic(g1, g2))
+
+
+# networkx.algorithms.link_analysis.pagerank_alg.pagerank
+@mgp.read_proc
+def pagerank(ctx: mgp.ProcCtx,
+             alpha: mgp.Number = 0.85,
+             personalization: mgp.Nullable[str] = None,
+             max_iter: int = 100,
+             tol: mgp.Number = 1e-06,
+             nstart: mgp.Nullable[str] = None,
+             weight: mgp.Nullable[str] = 'weight',
+             dangling: mgp.Nullable[str] = None,
+             ) -> mgp.Record(node=mgp.Vertex, rank=float):
+    def to_properties_dictionary(prop):
+        return None if prop is None else PropertiesDictionary(ctx, prop)
+
+    pg = nx.pagerank(MemgraphDiGraph(ctx=ctx), alpha=alpha,
+                     personalization=to_properties_dictionary(personalization),
+                     max_iter=max_iter, tol=tol,
+                     nstart=to_properties_dictionary(nstart), weight=weight,
+                     dangling=to_properties_dictionary(dangling))
+
+    return [mgp.Record(node=k, rank=v) for k, v in pg.items()]
+
+
+# networkx.algorithms.link_prediction.jaccard_coefficient
+@mgp.read_proc
+def jaccard_coefficient(
+        ctx: mgp.ProcCtx,
+        ebunch: mgp.Nullable[mgp.List[mgp.List[mgp.Vertex]]] = None
+) -> mgp.Record(u=mgp.Vertex, v=mgp.Vertex,
+                coef=float):
+    return [mgp.Record(u=u, v=v, coef=c) for u, v, c
+            in nx.jaccard_coefficient(MemgraphGraph(ctx=ctx), ebunch)]
+
+
+# networkx.algorithms.lowest_common_ancestors.lowest_common_ancestor
+@mgp.read_proc
+def lowest_common_ancestor(ctx: mgp.ProcCtx, node1: mgp.Vertex,
+                           node2: mgp.Vertex
+                           ) -> mgp.Record(ancestor=mgp.Nullable[mgp.Vertex]):
+    return mgp.Record(ancestor=nx.lowest_common_ancestor(
+        MemgraphDiGraph(ctx=ctx), node1, node2))
+
+
+# networkx.algorithms.matching.maximal_matching
+@mgp.read_proc
+def maximal_matching(ctx: mgp.ProcCtx) -> mgp.Record(edges=mgp.List[mgp.Edge]):
+    g = MemgraphMultiDiGraph(ctx=ctx)
+    return mgp.Record(edges=list(
+        next(iter(g[u][v])) for u, v in nx.maximal_matching(g)))
+
+
+# networkx.algorithms.planarity.check_planarity
+#
+# NOTE: Returns a graph.
+@mgp.read_proc
+def check_planarity(ctx: mgp.ProcCtx) -> mgp.Record(is_planar=bool):
+    return mgp.Record(is_planar=nx.check_planarity(
+        MemgraphMultiDiGraph(ctx=ctx))[0])
+
+
+# networkx.algorithms.non_randomness.non_randomness
+@mgp.read_proc
+def non_randomness(ctx: mgp.ProcCtx,
+                   k: mgp.Nullable[int] = None
+                   ) -> mgp.Record(non_randomness=float,
+                                   relative_non_randomness=float):
+    nn, rnn = nx.non_randomness(
+        MemgraphGraph(ctx=ctx), k=k)
+    return mgp.Record(non_randomness=nn, relative_non_randomness=rnn)
+
+
+# networkx.algorithms.reciprocity.reciprocity
+@mgp.read_proc
+def reciprocity(ctx: mgp.ProcCtx,
+                nodes: mgp.Nullable[mgp.List[mgp.Vertex]] = None
+                ) -> mgp.Record(node=mgp.Nullable[mgp.Vertex],
+                                reciprocity=mgp.Nullable[float]):
+    rp = nx.reciprocity(MemgraphMultiDiGraph(ctx=ctx), nodes=nodes)
+    if nodes is None:
+        return mgp.Record(node=None, reciprocity=rp)
+    else:
+        return [mgp.Record(node=n, reciprocity=r) for n, r in rp.items()]
+
+
+# networkx.algorithms.shortest_paths.generic.shortest_path
+@mgp.read_proc
+def shortest_path(ctx: mgp.ProcCtx,
+                  source: mgp.Nullable[mgp.Vertex] = None,
+                  target: mgp.Nullable[mgp.Vertex] = None,
+                  weight: mgp.Nullable[str] = None,
+                  method: str = 'dijkstra'
+                  ) -> mgp.Record(source=mgp.Vertex, target=mgp.Vertex,
+                                  path=mgp.List[mgp.Vertex]):
+    sp = nx.shortest_path(MemgraphMultiDiGraph(ctx=ctx), source=source,
+                          target=target, weight=weight, method=method)
+
+    if source and target:
+        sp = {source: {target: sp}}
+    elif source and not target:
+        sp = {source: sp}
+    elif not source and target:
+        sp = {source: {target: p} for source, p in sp.items()}
+
+    return [mgp.Record(source=s, target=t, path=p)
+            for s, d in sp.items()
+            for t, p in d.items()]
+
+
+# networkx.algorithms.shortest_paths.generic.shortest_path_length
+@mgp.read_proc
+def shortest_path_length(ctx: mgp.ProcCtx,
+                         source: mgp.Nullable[mgp.Vertex] = None,
+                         target: mgp.Nullable[mgp.Vertex] = None,
+                         weight: mgp.Nullable[str] = None,
+                         method: str = 'dijkstra'
+                         ) -> mgp.Record(source=mgp.Vertex, target=mgp.Vertex,
+                                         length=mgp.Number):
+    sp = nx.shortest_path_length(MemgraphMultiDiGraph(ctx=ctx), source=source,
+                                 target=target, weight=weight, method=method)
+
+    if source and target:
+        sp = {source: {target: sp}}
+    elif source and not target:
+        sp = {source: sp}
+    elif not source and target:
+        sp = {source: {target: l} for source, l in sp.items()}
+    else:
+        sp = dict(sp)
+
+    return [mgp.Record(source=s, target=t, length=l)
+            for s, d in sp.items()
+            for t, l in d.items()]
+
+
+# networkx.algorithms.shortest_paths.generic.all_shortest_paths
+@mgp.read_proc
+def all_shortest_paths(ctx: mgp.ProcCtx,
+                       source: mgp.Vertex,
+                       target: mgp.Vertex,
+                       weight: mgp.Nullable[str] = None,
+                       method: str = 'dijkstra'
+                       ) -> mgp.Record(paths=mgp.List[mgp.List[mgp.Vertex]]):
+    return mgp.Record(paths=list(nx.all_shortest_paths(
+        MemgraphMultiDiGraph(ctx=ctx), source=source, target=target,
+        weight=weight, method=method)))
+
+
+# networkx.algorithms.shortest_paths.generic.has_path
+@mgp.read_proc
+def has_path(ctx: mgp.ProcCtx,
+             source: mgp.Vertex,
+             target: mgp.Vertex) -> mgp.Record(has_path=bool):
+    return mgp.Record(has_path=nx.has_path(MemgraphMultiDiGraph(ctx=ctx),
+                                           source, target))
+
+
+# networkx.algorithms.shortest_paths.weighted.multi_source_dijkstra_path
+@mgp.read_proc
+def multi_source_dijkstra_path(ctx: mgp.ProcCtx,
+                               sources: mgp.List[mgp.Vertex],
+                               cutoff: mgp.Nullable[int] = None,
+                               weight: str = 'weight'
+                               ) -> mgp.Record(target=mgp.Vertex,
+                                               path=mgp.List[mgp.Vertex]):
+    return [mgp.Record(target=t, path=p)
+            for t, p in nx.multi_source_dijkstra_path(
+                    MemgraphMultiDiGraph(ctx=ctx), sources, cutoff=cutoff,
+                    weight=weight).items()]
+
+
+# networkx.algorithms.shortest_paths.weighted.multi_source_dijkstra_path_length
+@mgp.read_proc
+def multi_source_dijkstra_path_length(ctx: mgp.ProcCtx,
+                                      sources: mgp.List[mgp.Vertex],
+                                      cutoff: mgp.Nullable[int] = None,
+                                      weight: str = 'weight'
+                                      ) -> mgp.Record(target=mgp.Vertex,
+                                                      length=mgp.Number):
+    return [mgp.Record(target=t, length=l)
+            for t, l in nx.multi_source_dijkstra_path_length(
+                    MemgraphMultiDiGraph(ctx=ctx), sources, cutoff=cutoff,
+                    weight=weight).items()]
+
+
+# networkx.algorithms.simple_paths.is_simple_path
+@mgp.read_proc
+def is_simple_path(ctx: mgp.ProcCtx,
+                   nodes: mgp.List[mgp.Vertex]
+                   ) -> mgp.Record(is_simple_path=bool):
+    return mgp.Record(is_simple_path=nx.is_simple_path(
+        MemgraphMultiDiGraph(ctx=ctx), nodes))
+
+
+# networkx.algorithms.simple_paths.all_simple_paths
+@mgp.read_proc
+def all_simple_paths(ctx: mgp.ProcCtx,
+                     source: mgp.Vertex,
+                     target: mgp.Vertex,
+                     cutoff: mgp.Nullable[int] = None
+                     ) -> mgp.Record(paths=mgp.List[mgp.List[mgp.Vertex]]):
+    return mgp.Record(paths=list(nx.all_simple_paths(
+        MemgraphMultiDiGraph(ctx=ctx), source, target, cutoff=cutoff)))
+
+
+# networkx.algorithms.tournament.is_tournament
+@mgp.read_proc
+def is_tournament(ctx: mgp.ProcCtx) -> mgp.Record(is_tournament=bool):
+    return mgp.Record(is_tournament=nx.tournament.is_tournament(
+        MemgraphDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.traversal.breadth_first_search.bfs_edges
+@mgp.read_proc
+def bfs_edges(ctx: mgp.ProcCtx,
+              source: mgp.Vertex,
+              reverse: bool = False,
+              depth_limit: mgp.Nullable[int] = None
+              ) -> mgp.Record(edges=mgp.List[mgp.Edge]):
+    return mgp.Record(edges=list(nx.bfs_edges(
+        MemgraphMultiDiGraph(ctx=ctx), source, reverse=reverse,
+        depth_limit=depth_limit)))
+
+
+# networkx.algorithms.traversal.breadth_first_search.bfs_tree
+@mgp.read_proc
+def bfs_tree(ctx: mgp.ProcCtx,
+             source: mgp.Vertex,
+             reverse: bool = False,
+             depth_limit: mgp.Nullable[int] = None
+             ) -> mgp.Record(tree=mgp.List[mgp.Vertex]):
+    return mgp.Record(tree=list(nx.bfs_tree(
+        MemgraphMultiDiGraph(ctx=ctx), source, reverse=reverse,
+        depth_limit=depth_limit)))
+
+
+# networkx.algorithms.traversal.breadth_first_search.bfs_predecessors
+@mgp.read_proc
+def bfs_predecessors(ctx: mgp.ProcCtx,
+                     source: mgp.Vertex,
+                     depth_limit: mgp.Nullable[int] = None
+                     ) -> mgp.Record(node=mgp.Vertex,
+                                     predecessor=mgp.Vertex):
+    return [mgp.Record(node=n, predecessor=p)
+            for n, p in nx.bfs_predecessors(
+                    MemgraphMultiDiGraph(ctx=ctx), source,
+                    depth_limit=depth_limit)]
+
+
+# networkx.algorithms.traversal.breadth_first_search.bfs_successors
+@mgp.read_proc
+def bfs_successors(ctx: mgp.ProcCtx,
+                   source: mgp.Vertex,
+                   depth_limit: mgp.Nullable[int] = None
+                   ) -> mgp.Record(node=mgp.Vertex,
+                                   successors=mgp.List[mgp.Vertex]):
+    return [mgp.Record(node=n, successors=s)
+            for n, s in nx.bfs_successors(
+                    MemgraphMultiDiGraph(ctx=ctx), source,
+                    depth_limit=depth_limit)]
+
+
+# networkx.algorithms.traversal.depth_first_search.dfs_tree
+@mgp.read_proc
+def dfs_tree(ctx: mgp.ProcCtx,
+             source: mgp.Vertex,
+             depth_limit: mgp.Nullable[int] = None
+             ) -> mgp.Record(tree=mgp.List[mgp.Vertex]):
+    return mgp.Record(tree=list(nx.dfs_tree(
+        MemgraphMultiDiGraph(ctx=ctx), source, depth_limit=depth_limit)))
+
+
+# networkx.algorithms.traversal.depth_first_search.dfs_predecessors
+@mgp.read_proc
+def dfs_predecessors(ctx: mgp.ProcCtx,
+                     source: mgp.Vertex,
+                     depth_limit: mgp.Nullable[int] = None
+                     ) -> mgp.Record(node=mgp.Vertex,
+                                     predecessor=mgp.Vertex):
+    return [mgp.Record(node=n, predecessor=p)
+            for n, p in nx.dfs_predecessors(
+                    MemgraphMultiDiGraph(ctx=ctx), source,
+                    depth_limit=depth_limit).items()]
+
+
+# networkx.algorithms.traversal.depth_first_search.dfs_successors
+@mgp.read_proc
+def dfs_successors(ctx: mgp.ProcCtx,
+                   source: mgp.Vertex,
+                   depth_limit: mgp.Nullable[int] = None
+                   ) -> mgp.Record(node=mgp.Vertex,
+                                   successors=mgp.List[mgp.Vertex]):
+    return [mgp.Record(node=n, successors=s)
+            for n, s in nx.dfs_successors(
+                    MemgraphMultiDiGraph(ctx=ctx), source,
+                    depth_limit=depth_limit).items()]
+
+
+# networkx.algorithms.traversal.depth_first_search.dfs_preorder_nodes
+@mgp.read_proc
+def dfs_preorder_nodes(ctx: mgp.ProcCtx,
+                       source: mgp.Vertex,
+                       depth_limit: mgp.Nullable[int] = None
+                       ) -> mgp.Record(nodes=mgp.List[mgp.Vertex]):
+    return mgp.Record(nodes=list(nx.dfs_preorder_nodes(
+        MemgraphMultiDiGraph(ctx=ctx), source, depth_limit=depth_limit)))
+
+
+# networkx.algorithms.traversal.depth_first_search.dfs_postorder_nodes
+@mgp.read_proc
+def dfs_postorder_nodes(ctx: mgp.ProcCtx,
+                        source: mgp.Vertex,
+                        depth_limit: mgp.Nullable[int] = None
+                        ) -> mgp.Record(nodes=mgp.List[mgp.Vertex]):
+    return mgp.Record(nodes=list(nx.dfs_postorder_nodes(
+        MemgraphMultiDiGraph(ctx=ctx), source, depth_limit=depth_limit)))
+
+
+# networkx.algorithms.traversal.edgebfs.edge_bfs
+@mgp.read_proc
+def edge_bfs(ctx: mgp.ProcCtx,
+             source: mgp.Nullable[mgp.Vertex] = None,
+             orientation: mgp.Nullable[str] = None
+             ) -> mgp.Record(edges=mgp.List[mgp.Edge]):
+    return mgp.Record(edges=list(e for _, _, e in nx.edge_bfs(
+        MemgraphMultiDiGraph(ctx=ctx), source=source,
+        orientation=orientation)))
+
+
+# networkx.algorithms.traversal.edgedfs.edge_dfs
+@mgp.read_proc
+def edge_dfs(ctx: mgp.ProcCtx,
+             source: mgp.Nullable[mgp.Vertex] = None,
+             orientation: mgp.Nullable[str] = None
+             ) -> mgp.Record(edges=mgp.List[mgp.Edge]):
+    return mgp.Record(edges=list(e for _, _, e in nx.edge_dfs(
+        MemgraphMultiDiGraph(ctx=ctx), source=source,
+        orientation=orientation)))
+
+
+# networkx.algorithms.tree.recognition.is_tree
+@mgp.read_proc
+def is_tree(ctx: mgp.ProcCtx) -> mgp.Record(is_tree=bool):
+    return mgp.Record(is_tree=nx.is_tree(MemgraphDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.tree.recognition.is_forest
+@mgp.read_proc
+def is_forest(ctx: mgp.ProcCtx) -> mgp.Record(is_forest=bool):
+    return mgp.Record(is_forest=nx.is_forest(MemgraphDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.tree.recognition.is_arborescence
+@mgp.read_proc
+def is_arborescence(ctx: mgp.ProcCtx) -> mgp.Record(is_arborescence=bool):
+    return mgp.Record(is_arborescence=nx.is_arborescence(
+        MemgraphDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.tree.recognition.is_branching
+@mgp.read_proc
+def is_branching(ctx: mgp.ProcCtx) -> mgp.Record(is_branching=bool):
+    return mgp.Record(is_branching=nx.is_branching(MemgraphDiGraph(ctx=ctx)))
+
+
+# networkx.algorithms.tree.mst.minimum_spanning_tree
+@mgp.read_proc
+def minimum_spanning_tree(ctx: mgp.ProcCtx,
+                          weight: str = 'weight',
+                          algorithm: str = 'kruskal',
+                          ignore_nan: bool = False
+                          ) -> mgp.Record(nodes=mgp.List[mgp.Vertex],
+                                          edges=mgp.List[mgp.Edge]):
+    gres = nx.minimum_spanning_tree(MemgraphMultiGraph(ctx=ctx),
+                                    weight, algorithm, ignore_nan)
+    return mgp.Record(nodes=list(gres.nodes()),
+                      edges=[e for _, _, e in gres.edges(keys=True)])
+
+
+# networkx.algorithms.triads.triadic_census
+@mgp.read_proc
+def triadic_census(ctx: mgp.ProcCtx) -> mgp.Record(triad=str, count=int):
+    return [mgp.Record(triad=t, count=c)
+            for t, c in nx.triadic_census(MemgraphDiGraph(ctx=ctx)).items()]
+
+
+# networkx.algorithms.voronoi.voronoi_cells
+@mgp.read_proc
+def voronoi_cells(ctx: mgp.ProcCtx,
+                  center_nodes: mgp.List[mgp.Vertex],
+                  weight: str = 'weight'
+                  ) -> mgp.Record(center=mgp.Vertex,
+                                  cell=mgp.List[mgp.Vertex]):
+    return [mgp.Record(center=c1, cell=list(c2))
+            for c1, c2 in nx.voronoi_cells(
+                    MemgraphMultiDiGraph(ctx=ctx), center_nodes,
+                    weight=weight).items()]
+
+
+# networkx.algorithms.wiener.wiener_index
+@mgp.read_proc
+def wiener_index(ctx: mgp.ProcCtx, weight: mgp.Nullable[str] = None
+                 ) -> mgp.Record(wiener_index=mgp.Number):
+    return mgp.Record(wiener_index=nx.wiener_index(
+        MemgraphMultiDiGraph(ctx=ctx), weight=weight))

query_modules/pagerank.py

@@ -1,134 +0,0 @@
-import mgp
-import networkx as nx
-from itertools import chain
-
-
-class VertexDictionary:
-    def __init__(self, graph, prop):
-        self.graph = graph
-        self.prop = prop
-        self.len = None
-
-    def get(self, vertex, default=None):
-        return vertex.properties.get(self.prop, default=default)
-
-    def items(self):
-        for v in self.graph.vertices:
-            if self.prop in v.properties:
-                yield v, v.properties[self.prop]
-
-    def keys(self):
-        for k, v in self.items():
-            yield k
-
-    def values(self):
-        for k, v in self.items():
-            yield v
-
-    def __len__(self):
-        if not self.len:
-            self.len = sum(1 for _ in self.items())
-        return self.len
-
-    def __iter__(self):
-        for k, v in self.items():
-            yield k
-
-    def __getitem__(self, vertex):
-        try:
-            return vertex.properties[self.prop]
-        except KeyError:
-            raise KeyError(("Vertex {} doesn\t have the required " +
-                            "property '{}'").format(vertex.id, self.prop))
-
-    def __contains__(self, vertex):
-        return vertex in self.graph.vertices and self.prop in vertex.properties
-
-
-@mgp.read_proc
-def pagerank(ctx: mgp.ProcCtx,
-             alpha: mgp.Number = 0.85,
-             personalization: mgp.Nullable[str] = None,
-             max_iter: int = 100,
-             tol: mgp.Number = 1e-06,
-             nstart: mgp.Nullable[str] = None,
-             weight: mgp.Nullable[str] = 'weight',
-             dangling: mgp.Nullable[str] = None
-             ) -> mgp.Record(node=mgp.Vertex, rank=float):
-    '''Run the PageRank algorithm on the whole graph.
-
-    The available parameters are:
-
-    - `alpha` -- Damping parameter.
-
-    - `personalization` -- The "personalization vector". A string specifying
-      the property that will be looked up for every node to give the
-      personalization value. If a node doesn't have the specified property, its
-      personalization value will be zero. By default, a uniform distribution is
-      used.
-
-    - `max_iter` -- Maximum number of iterations in the power method eigenvalue
-      solver.
-
-    - `tol` -- Error tolerance used to check for convergence in the power
-      eigenvalue method solver.
-
-    - `nstart` -- A string specifying the property that will be looked up for
-      every node to give the starting value for the iteration.
-
-    - `weight` -- A string specifying the property that will be looked up for
-      every edge to give the weight. If None or if the property doesn't exist,
-      weights are set to 1.
-
-    - `dangling` -- The outedges to be assigned to any "dangling" nodes, i.e.,
-      nodes without any outedges. A string specifying the property that will be
-      looked up for every node to give the weight of the outedge that points to
-      that node. By default, dangling nodes are given outedges according to the
-      personalization vector. This must be selected to result in an irreducible
-      transition matrix. It may be common to have the dangling dictionary be
-      the same as the personalization dictionary.
-
-    Return a single record for every node. Each record has two fields, `node`
-    and `rank`, which together give the calculated rank for the node.
-
-    As an example, the following openCypher query calculates the ranks of all
-    the nodes in the graph using the PageRank algorithm. The personalization
-    value for every node is taken from its property named 'personalization',
-    while the `alpha` and `max_iter` parameters are set to 0.85 and 150
-    respectively:
-
-    CALL pagerank.pagerank(0.85, 'personalization', 150) YIELD *;
-
-    '''
-    def to_vertex_dictionary(prop):
-        return None if prop is None else VertexDictionary(ctx.graph, prop)
-
-    def make_and_check_vertex(v):
-        for prop in (personalization, nstart, dangling):
-            if prop is None:
-                continue
-            if not isinstance(v.properties.get(prop, default=1), (int, float)):
-                raise TypeError(("Property '{}' of vertex '{}' needs to " +
-                                "be a number").format(prop, v.id))
-        return v
-
-    def make_and_check_edge(e):
-        if (weight is not None and
-            not isinstance(e.properties.get(weight, default=1), (int, float))):
-            raise TypeError("Property '{}' of edge '{}' needs to be a number"
-                            .format(weight, e.id))
-        return e.from_vertex, e.to_vertex, e.properties
-
-    g = nx.DiGraph()
-    g.add_nodes_from(make_and_check_vertex(v) for v in ctx.graph.vertices)
-    g.add_edges_from(make_and_check_edge(e)
-                     for v in ctx.graph.vertices
-                     for e in chain(v.in_edges, v.out_edges))
-
-    pg = nx.pagerank(g, alpha=alpha,
-                     personalization=to_vertex_dictionary(personalization),
-                     max_iter=max_iter, tol=tol,
-                     nstart=to_vertex_dictionary(nstart), weight=weight,
-                     dangling=to_vertex_dictionary(dangling))
-
-    return [mgp.Record(node=k, rank=v) for k, v in pg.items()]

query_modules/wcc.py

@@ -1,6 +1,15 @@
+import sys
 import mgp
-
-import networkx as nx
+try:
+    import networkx as nx
+except ImportError as import_error:
+    sys.stderr.write(
+        '\n'
+        'NOTE: Please install networkx to be able to use wcc module.\n'
+        'Using Python:\n'
+        + sys.version +
+        '\n')
+    raise import_error
 
 
 @mgp.read_proc
@@ -28,8 +37,10 @@ def get_components(vertices: mgp.List[mgp.Vertex],
     vertices labeled `Person` and edges between such vertices can be obtained
     using the following openCypher query:
 
-    MATCH (n:Person)-[e]->(m:Person) WITH collect(n) AS nodes, collect(e) AS edges
-    CALL wcc.get_components(nodes, edges) YIELD * RETURN n_components, components;
+    MATCH (n:Person)-[e]->(m:Person)
+    WITH collect(n) AS nodes, collect(e) AS edges
+    CALL wcc.get_components(nodes, edges) YIELD *
+    RETURN n_components, components;
     '''
     g = nx.DiGraph()
     g.add_nodes_from(vertices)

release/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Install the license file.
 if (MG_ENTERPRISE)
-  install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE_ENTERPRISE_TRIAL.md
+  install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE_ENTERPRISE.md
           DESTINATION share/doc/memgraph RENAME copyright)
 else()
   install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE_COMMUNITY.md
@@ -11,16 +11,6 @@ endif()
 install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/memgraph.service
         DESTINATION /lib/systemd/system)
 
-# Install examples
-set(examples ${CMAKE_CURRENT_SOURCE_DIR}/examples)
-install(
-  CODE
-  "execute_process(COMMAND ${examples}/build_examples
-                   ${CMAKE_BINARY_DIR}/memgraph
-                   ${CMAKE_BINARY_DIR}/tools/src/mg_client
-                   WORKING_DIRECTORY ${examples})")
-install(DIRECTORY ${examples}/build/ DESTINATION share/memgraph/examples)
-
 # ---- Setup CPack --------
 
 # General setup
@@ -41,6 +31,7 @@ if (MG_ENTERPRISE)
   set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/enterprise/conffiles;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/enterprise/copyright;"
+      "${CMAKE_CURRENT_SOURCE_DIR}/debian/preinst;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/prerm;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/postrm;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/postinst;")
@@ -48,6 +39,7 @@ else()
   set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/community/conffiles;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/community/copyright;"
+      "${CMAKE_CURRENT_SOURCE_DIR}/debian/preinst;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/prerm;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/postrm;"
       "${CMAKE_CURRENT_SOURCE_DIR}/debian/postinst;")

release/LICENSE_COMMUNITY.md

@@ -1,33 +1,40 @@
-# User License Agreement
+# Memgraph Community User License Agreement
 
-1. Description
+This License Agreement governs your use of the Memgraph Community Release (the
+"Software") and documentation ("Documentation").
 
-THIS LICENSE AGREEMENT GOVERNS LICENSEE’S USE OF THE MEMGRAPH COMMUNITY
-RELEASE AND DOCUMENTATION.
+BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU ("LICENSEE") AGREE TO THESE
+TERMS.
 
-2. License Grant
+1. License Grant
 
 The Software and Documentation are provided to Licensee at no charge and are
 licensed, not sold to Licensee. No ownership of any part of the Software and
 Documentation is hereby transferred to Licensee. Subject to (i) the terms and
-conditions of this License Agreement, (ii) any additional license restrictions
-and parameters contained on Licensor’s quotation, website, or order form
-(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
+conditions of this License Agreement, and (ii) any additional license
+restrictions and parameters contained on Licensor’s quotation, website, or
+order form, Licensor hereby grants Licensee a personal, non-assignable,
 non-transferable and non-exclusive license to install, access and use the
 Software (in object code form only) and Documentation for Licensee’s internal
-business purposes only. All rights relating to the Software and Documentation
-that are not expressly licensed in this License Agreement, whether now existing
-or which may hereafter come into existence are reserved for Licensor. Licensee
-shall not remove, obscure, or alter any proprietary rights notices (including
-without limitation copyright and trademark notices), which may be affixed to or
-contained within the Software or Documentation.
+business purposes (including for use in a production environment) only. All
+rights relating to the Software and Documentation that are not expressly
+licensed in this License Agreement, whether now existing or which may hereafter
+come into existence are reserved for Licensor. Licensee shall not remove,
+obscure, or alter any proprietary rights notices (including without limitation
+copyright and trademark notices), which may be affixed to or contained within
+the Software or Documentation.
 
-3. Restrictions
+Licensor may terminate this License Agreement with immediate effect upon
+written notice to the Licensee. Upon termination Licensee shall delete all
+electronic copies of all or any part of the Software and/or the Documentation
+resident in its systems or elsewhere.
+
+2. Restrictions
 
 Licensee will not, directly or indirectly, (a) copy the Software or
 Documentation in any manner or for any purpose; (b) install, access or use any
 component of the Software or Documentation for any purpose not expressly
-granted in Section 2 above; (c) resell, distribute, publicly display or
+granted in Section 1 above; (c) resell, distribute, publicly display or
 publicly perform the Software or Documentation or any component thereof, by
 transfer, lease, loan or any other means, or make it available for use by
 others in any time-sharing, service bureau or similar arrangement; (d)
@@ -37,25 +44,55 @@ algorithms or techniques incorporated in the Software; (e) export the Software
 or Documentation in violation of any applicable laws or regulations; (f)
 modify, translate, adapt, or create derivative works from the Software or
 Documentation; (g) circumvent, disable or otherwise interfere with
-security-related features of the Software or Documentation; (h)
-reverse-engineer, disassemble, attempt to derive the source code; (i) use the
+security-related features of the Software or Documentation; (h) use the
 Software or Documentation for any illegal purpose, in any manner that is
 inconsistent with the terms of this License Agreement, or to engage in illegal
-activity; (j) remove or alter any trademark, logo, copyright or other
+activity; (i) remove or alter any trademark, logo, copyright or other
 proprietary notices, legends, symbols or labels on, or embedded in, the
-Software or Documentation; or (k) provide access to the Software or
+Software or Documentation; or (j) provide access to the Software or
 Documentation to third parties.
 
-4. Warranty Disclaimer
+3. Warranty Disclaimer
 
-THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED “AS IS” FOR
-DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
-PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
-IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
-THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
-NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
-INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
-WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
+THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" AND LICENSOR MAKES NO
+WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
+INFRINGEMENT OF THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER
+PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
+FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
+OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
 
-BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.
+4. Limitation of Liability
+
+Licensor shall not in any circumstances be liable, whether in tort (including
+for negligence or breach of statutory duty howsoever arising), contract,
+misrepresentation (whether innocent or negligent) or otherwise for: loss of
+profits, loss of business, depletion of goodwill or similar losses, loss of
+anticipated savings, loss of goods, loss or corruption of data or computer
+downtime, or any special, indirect, consequential or pure economic loss, costs,
+damages, charges or expenses.
+
+Licensor's total aggregate liability in contract, tort (including without
+limitation negligence or breach of statutory duty howsoever arising),
+misrepresentation (whether innocent or negligent), restitution or otherwise,
+arising in connection with the performance or contemplated performance of this
+License Agreement shall in all circumstances be limited to GBP10.00 (ten pounds
+sterling).
+
+Nothing in this License Agreement shall limit Licensor’s liability in the case
+of death or personal injury caused by negligence, fraud, or fraudulent
+misrepresentation, or where it otherwise cannot be limited by law.
+
+5. Technical Data
+
+Licensor may collect and use technical information (such as usage patterns)
+gathered when the Licensee downloads and uses the Software.  This is generally
+statistical data which does not identify an identified or identifiable
+individual.  It may also include Licensee’s IP address which is personal data
+and is processed in accordance with our Privacy Policy. We only use this
+technical information to improve our products.
+
+6. Law and Jurisdiction
+
+This License Agreement is governed by the laws of England and is subject to the
+non-exclusive jurisdiction of the courts of England.

release/LICENSE_ENTERPRISE.md

@@ -0,0 +1,671 @@
+# Memgraph Enterprise - Software Subscription Agreement
+
+Memgraph Limited is registered in England under registration 10195084 and has
+its registered office at Suite 4, Ironstone House, Ironstone Way, Brixworth,
+Northampton, NN6 9UD ("Memgraph").
+
+Memgraph agrees to license and/or grant you (the "Customer") access to the
+Software, as defined below and provide support and services to you only if you
+accept and agree to be bound by the terms and conditions in this Software
+Subscription Agreement (the "Agreement").  By signing an Order Document (as
+defined below), which is subject to and part of this Agreement, installing and
+using the Software or by downloading a trial version of the Software, you agree
+to be bound by the terms of this Agreement.
+
+Memgraph Enterprise Trial Users: If you receive free of charge trial access to
+the Software, you are deemed a "Customer" for purposes of this Agreement except
+that you are subject to the additional restrictions and limitations set forth
+in Section 3.2 below as to such Software.
+
+## 1. DEFINITIONS.
+
+1.1. "Applicable Laws" means (i) all applicable laws, statutes and regulations,
+and (ii) regulatory policies, guidelines and industry codes (in each case
+having the force of law), which apply to the provisions of the Software and
+Services and this Agreement.
+
+1.2. "Confidential Information" means all information disclosed
+by a party ("Disclosing Party") to the other party ("Receiving Party"), whether
+orally or in writing, that is designated as confidential or that reasonably
+should be understood to be confidential given the nature of the information and
+the circumstances of disclosure. Customer's Confidential Information includes
+Customer Data; Memgraph Confidential Information includes the Software and
+Services; and Confidential Information of each party includes the terms and
+conditions of this Agreement and all Orders (including pricing), as well as
+business and marketing plans, technology and technical information, product
+plans and designs, and business processes disclosed by such party. However,
+Confidential Information does not include any information that (i) is or
+becomes generally known to the public without breach of any obligation owed to
+the Disclosing Party, (ii) was known to the Receiving Party prior to its
+disclosure by the Disclosing Party without breach of any obligation owed to the
+Disclosing Party, (iii) is received from a third party without breach of any
+obligation owed to the Disclosing Party, or (iv) was independently developed by
+the Receiving Party.
+
+1.3. "Customer Data" means business information or other data loaded by or for
+Customer and/or processed by the Software.
+
+1.4. "Data Protection Legislation" means all applicable data protection and
+privacy legislation in force from time to time in the UK including the General
+Data Protection Regulation ((EU) 2016/679); the Data Protection Act 2018; the
+Privacy and Electronic Communications Directive 2002/58/EC (as updated by
+Directive 2009/136/EC) and the Privacy and Electronic Communications
+Regulations 2003 (SI 2003/2426) as amended.
+
+1.5. "Order Document" or "Order" means an order on the Memgraph form of
+provisioning document that is submitted by or on behalf of Customer, and
+executed by or on behalf of the parties referencing this Agreement and that
+specifies the Software and/or Services ordered by Customer, as well as the
+specific terms and conditions, for that particular transaction.
+
+1.6. "Services" means those services, including Support Services, which may be
+provided to Customer by Memgraph pursuant to the terms of this Agreement and
+are expressly limited to those services directly related to Customer's use of
+the Software, and expressly exclude any other services.
+
+1.7. "Software" means the Memgraph proprietary downloadable graph database
+enterprise software ("Enterprise Software") and the associated technical
+documentation located at https://docs.memgraph.com/ ("Documentation"), as well
+as software updates, upgrades, bug fixes, or modified versions thereof that
+Memgraph licenses or provides to Customer directly or indirectly throughout the
+Subscription Term.
+
+1.8. "Subscription Term" means the fixed term, of not less than one (1) year,
+designated in an Order Document beginning on the Effective Date and ending at
+the end of the period stated therein. If no expiration date is specified in an
+Order Document, the Subscription Term shall be a one (1) year period ("Minimum
+Subscription Term"). A "Subscription" is the binding, non-cancellable contract
+for the use of the Software for the Subscription Term as set forth in an Order
+Document.
+
+1.9. "Support" means the support and maintenance services, including any
+updates, upgrades, patches, enhancements and bug fixes for the Software that
+may be provided to Customer by Memgraph pursuant to the terms of this
+Agreement.
+
+1.10. "Users" means employees and Contractors of Customer that Customer has
+permitted or authorized to access and use of the Software on Customer's behalf
+pursuant to the terms of this Agreement.
+
+## 2. ORDERS, DELIVERY; SUPPORT.
+
+2.1. Delivery. Memgraph will promptly deliver the Software after it accepts an
+Order and Customer issues a purchase order referencing such Order. Unless
+otherwise stated in an Order, Customer is solely responsible for installing
+Software on Customer's own computer equipment. In some instances, Customer's
+purchasing relationship exists solely between Customer and an authorised
+reseller of Memgraph's Software and Services (a "Reseller"), in which case
+Sections 5.1-5.3 (Fees and Payment) will be inapplicable to such Order(s), and
+the Reseller shall be responsible for submitting Orders and the appropriate
+payment method therewith to Memgraph. An Order is not binding until Memgraph
+accepts and countersigns the Order.
+
+2.2 Support. Memgraph will use commercially reasonable efforts to provide
+Support to Customer in accordance with Memgraph's then-current terms and
+conditions set forth at
+https://download.memgraph.com/legal/memgraph-support-terms-and-conditions.pdf
+at the support tier stated in the applicable Order. The Support terms and
+conditions are subject to change at Memgraph's discretion; however, Memgraph
+will not materially reduce the level of Support during a Subscription Term for
+which Fees have been paid.
+
+## 3. LICENCE GRANTS; RESTRICTIONS AND PROPRIETARY RIGHTS.
+
+Customer's licence and access rights and benefits and Memgraph's obligations to
+Customer will vary depending on the product and the type of licence Memgraph is
+granting. If you purchased a licence to Memgraph Software, your licence will be
+subject to certain use and/or capacity restrictions, as identified on the
+applicable Order Document.
+
+3.1. Enterprise Software Licence. In consideration of the Fees paid hereunder
+and subject to the terms of this Agreement and the applicable Order, Memgraph
+grants Customer a world-wide, non-exclusive, non-transferable,
+non-sublicenseable, and limited licence during the applicable Subscription
+Term, to install, and use the Enterprise Software up to the maximum capacity
+("Licensed Capacity"), and subject to the usage rules, specified in the
+applicable Order Document, and to use Documentation solely for Customer's
+internal business purposes in connection with operation of the Enterprise
+Software.
+
+3.2. Enterprise Trial Licence. If the Customer downloads, accesses, or uses the
+Software under a trial licence ("Trial Licence"), then Customer may use one (1)
+copy of the Software in accordance with the terms and conditions of this
+Agreement for a thirty (30) day period, or such longer trial period represented
+by the applicable licence key issued by or expressly authorised by Memgraph
+(the "Trial Period"). Trial Licences are permitted solely for Customer's
+evaluation use to determine whether to purchase a Subscription to the Software.
+Customer may not use a Trial Licence for any other purpose. At the end of the
+Trial Period, the Trial Licence will expire (and this Agreement will terminate
+as to such Licence and continue to apply to any subsequent Subscription or use
+of the Software; if Customer decides not to obtain a Subscription upon
+expiration of the Trial Period, it will promptly cease using and will delete
+the Software from its computer systems. Memgraph has the right to terminate a
+Trial Licence at any time for any reason.
+
+3.3. NO OBLIGATIONS. NOTWITHSTANDING ANYTHING TO THE CONTRARY IN THIS AGREEMENT
+OR IN ANY ORDER DOCUMENT, MEMGRAPH WILL HAVE NO WARRANTY, INDEMNITY, SUPPORT,
+OR SERVICE LEVEL, OBLIGATIONS WITH RESPECT TO ANY ENTERPRISE TRIAL, OR OTHER
+NO-CHARGE SOFTWARE (INCLUDING TOOLS AND UTILITIES) LICENCES.
+
+3.4. General Restrictions. Customer acknowledges that the Software, and its
+structure, organization, and source code, constitute Memgraph's and its
+suppliers' valuable trade secrets, and that usage of the Software is subject to
+the following restrictions:
+
+3.4.1. Restrictions. Customer agrees not to, and not to authorize any third
+party to: (i) allow access or use of the Software by anyone other than its
+Users; (ii) distribute, embed, sell, rent, transfer, lease, lend, sublicense,
+loan, assign, pledge, grant a security interest in, or otherwise make the
+Software accessible or available to any third party; except to the limited
+extent expressly provided in Section 3.4.2, use the Software in any
+service-bureau, timesharing, outsourcing or similar arrangement; or (iii)
+disassemble, decompile, reverse engineer or otherwise attempt to derive the
+structure, sequence or organization of source code or APIs; (iv) remove or
+alter product identification, copyright, trademark or other proprietary
+markings contained in or on the Software; (v) modify, adapt, recast, transform
+or otherwise prepare a derivative work of the Software or portion thereof; (vi)
+conduct any competitive analysis, publish or share with any third party any
+results of any technical evaluation or tests performed on the Software, or
+disclose Software features, errors or bugs to a third party without Memgraph's
+prior written consent; or (vii) engage in any act designed to circumvent any
+restriction set forth in this Agreement, in the Software, or in an Order,
+including but not limited to restrictions related to Licensed Capacity.
+
+3.4.2. Internal Use Licences; Users. The Software is licensed for Customer's
+internal business use and not for distribution or use by third parties. For
+clarity, however, Customer may make available to third parties any
+Customer-hosted services or other Customer applications or services that make
+use of or incorporate the Software, provided and solely to the extent that (i)
+Customer's application or hosted service adds primary and significant
+functionality to the Software, (ii) Customer does not embed or otherwise
+distribute the Software to third parties, (iii) Customer does not provide third
+parties direct access to operate or control the Software itself; and (iv)
+Customer at all times remains in compliance with the terms of the applicable
+licence grants under this Agreement. Subject to the terms and conditions of
+this Agreement, in addition to Customer's employees, Customer may permit its
+independent contractors and consultants ("Contractors") to serve as Users.
+Customer will remain responsible for compliance by each of its Users (including
+but not limited to any Contractor Users) with all of the terms and conditions
+of this Agreement, and any use of the Software by any Contractors must be for
+the sole benefit of Customer.
+
+3.5. Ownership; Reservation of Rights. This is an agreement for use of Memgraph
+Software and not an agreement for sale. Customer acknowledges that it is
+obtaining only a limited right to use the Software on a licensed basis, and
+that irrespective of any use of the words "purchase", "sale" or like terms
+hereunder no ownership rights are being conveyed to Customer. Customer agrees
+that Memgraph or its suppliers retain all right, title and interest (including
+all patent, copyright, trade secret and other intellectual property rights) in
+and to the Memgraph Software. Nothing in this Section shall be deemed as
+granting Memgraph ownership of Customer Data or in any way impacting Customer's
+ownership of Customer Data.
+
+3.6. Third Party Code. The Software may contain or be provided with components
+which are licensed from third parties, including components subject to the
+terms and conditions of "open source" software licences ("Open Source
+Software"). Open Source Software may be identified in the Software,
+Documentation, or in a list of the Open Source Software provided to you upon
+your written request. To the extent required by the licence that accompanies
+the Open Source Software, the terms of such licence will apply in lieu of the
+terms of this Agreement with respect to such Open Source Software, including,
+without limitation, any provisions governing access to source code,
+modification, or reverse engineering.
+
+3.7. IP Ownership. The Parties agree that neither party shall gain, by virtue
+of this Agreement, any rights of ownership or any other interest, right or
+title of copyrights, patents, trade secrets, trademarks, or any other
+intellectual property rights owned by the other Party. Any and all new works
+developed in the course of performing obligations pursuant to this Agreement
+and all new inventions, innovations or ideas developed by a Party in the course
+of performance of its activities under this Agreement, will belong to that
+Party who develops the same. Notwithstanding anything to the contrary in this
+Section, the Parties understand and agree that any and all proprietary
+materials developed by a Party prior to this Agreement and any modifications,
+enhancements, improvements or inventions made to such proprietary materials
+shall be owned by that Party, regardless of which Party prepared or developed
+such modifications, enhancements, improvements or inventions.
+
+## 4. CUSTOMER DATA; OBLIGATIONS OF CUSTOMER AND MEMGRAPH.
+
+4.1. Customer shall retain all of its rights, title, and interest in and to its
+intellectual property rights in Customer Data. Customer grants to Memgraph a
+non-exclusive, worldwide, limited-term licence solely to host, copy, transmit
+and display Customer Data as reasonably necessary for Memgraph to support
+Customer's use of the Software, to ensure the security of and to administrate
+the Software, and to deliver Services in accordance with this Agreement or as
+otherwise outlined in https://memgraph.com/legal/privacy-policy/.
+
+4.2. Protection of Customer Data. Memgraph will maintain appropriate
+administrative, physical, and technical safeguards, consistent with generally
+prevailing industry standards, for protection of the security, confidentiality,
+and integrity of Customer Data, as described in the Documentation. Those
+safeguards will include, but will not be limited to, measures for preventing
+access, use, modification, or disclosure of Customer Data by Memgraph
+personnel, except as permitted by this Agreement.
+
+4.3. Personal data. Both parties will comply with all applicable requirements
+of the Data Protection Legislation. This section 4 is in addition to, and does
+not relieve, remove or replace, a party's obligations under the Data Protection
+Legislation. Notwithstanding the foregoing, the parties acknowledge that, in
+the ordinary course of providing the Services, Memgraph shall not process
+personal data (as defined in the Data Protection Legislation) on behalf of the
+Customer. In the event that the Customer requires Memgraph to process personal
+data on its behalf, it shall notify Memgraph and the parties shall execute such
+terms as necessary under applicable law.
+
+## 5. FEES AND PAYMENT.
+
+5.1. Fees. Customer will pay Memgraph the fees for the Licences and Services as
+set forth in the applicable Order ("Fees"). Customer acknowledges and agrees
+that if Customer's use of the Software exceeds the Licensed Capacity set forth
+on the applicable Orders or otherwise requires the payment of additional fees
+(per the terms of this Agreement), Customer shall be invoiced for such usage
+and Customer agrees to pay the additional fees in accordance with this Section.
+Notwithstanding the terms of Section 5.4 below (Reconciliation), Customer
+acknowledges and agrees that it is obligated to ensure that its Software usage
+does not exceed the Licensed Capacity and to promptly notify Memgraph of any
+such excess usage no more than thirty (30) days from the last day of the
+calendar month during which such excess usage occurred.
+
+5.2. Payment Terms. Except as otherwise specifically set forth on an Order
+Document, all fees are due and payable within thirty days after the date of
+invoice. Renewal Fees for any renewal Subscription Term (if purchased by
+Customer) will be due and payable within thirty (30) days of expiration of the
+then-current term. If Fees are not paid when due, or in the event of other
+breach of this Agreement, Customer shall discontinue use of the Software and
+Memgraph may suspend its performance, including its delivery of technical
+support of the Software or other Services without further notice and without
+penalty. All Orders (including multi-year Subscriptions with annual payment
+schedules) are non-cancellable and all amounts paid are non-refundable, unless
+otherwise expressly set forth herein. Any invoiced amount not received by the
+due date will accrue late interest at the rate of 1.5% of the outstanding
+balance per month, or the maximum rate permitted by applicable law, whichever
+is lower.
+
+5.3. Taxes. Fees are exclusive of taxes. Customer will pay any sales, use,
+value added, duties, fees and other governmental assessments or charges arising
+out of this Agreement and the transactions contemplated herein. Customer will
+make all payments free and clear of, and without reduction for, any withholding
+taxes.
+
+5.4. Reconciliation. At Memgraph's request from time to time, not exceeding
+once per quarter, Customer will provide Memgraph with a report detailing its
+use of the Software, including its non-production and/or production use and
+using the self-monitoring capabilities of the Software or other means, and
+Memgraph may inspect Customer's records related to such report not more
+frequently than annually to ensure payment of Fees. Any on-site review will be
+conducted during regular business hours at Customer's offices. The parties will
+use reasonable good faith efforts to promptly resolve any discrepancies between
+licensed usage and actual usage.
+
+## 6. APPLICABLE LAWS.
+
+6.1. Each Party shall perform this Agreement in accordance with all Applicable
+Laws. Without prejudice to the foregoing, each Party shall:
+
+6.1.1. comply with all Applicable Laws relating to anti-bribery,
+anti-corruption, anti-slavery and human trafficking, including the Bribery Act
+2010 and the Modern Slavery Act 2015 (the "Relevant Requirements");
+
+6.1.2. have and maintain in place throughout the Term its own policies and
+procedures, including adequate procedures under the Bribery Act 2010, to ensure
+compliance with the Relevant Requirements, and will enforce them where
+appropriate;
+
+6.1.3. (if not prohibited by law or regulation from doing so) promptly report
+to the other Party any request or demand for any undue financial or other
+advantage of any kind received by the reporting Party in connection with the
+performance of this Agreement; and
+
+6.1.4. (if not prohibited by law or regulation from doing so) notify the other
+Party (and email shall be sufficient for this purpose) as soon as it becomes
+aware of any actual or suspected slavery or human trafficking in a supply chain
+which has a connection with this Agreement.
+
+## 7. REPRESENTATIONS AND WARRANTIES.
+
+7.1. Mutual Representations and Warranties. Each Party represents and warrants
+to the other that: (i) it is a corporation lawfully incorporated and validly
+existing pursuant to the laws of its place of incorporation; (ii) it has all
+requisite power and authority, corporate or otherwise, to execute, deliver and
+perform its obligations under this Agreement; and (iii) this Agreement
+constitutes its legal, valid and binding obligations and may be enforced
+against it.
+
+7.2. Limited Memgraph Warranty; Remedies. Memgraph warrants that the Software,
+when used as permitted hereunder, will operate in all material respects as
+described in the applicable Documentation, and that Services will be provided
+in a professional manner consistent with industry standards.
+
+7.3. Limitations; Remedy. Memgraph does not warrant that the Software or the
+Services will be error-free, uninterrupted or meet Customer's specific
+requirements or that performance of the Services will be uninterrupted.
+Memgraph will have no warranty obligation under Section 7.2 for Customer's
+misuse or failure to use the Software in accordance with its Documentation or
+this Agreement. Customer's sole and exclusive remedy, and Memgraph's sole and
+exclusive obligation, for breach of warranty will be (i) during the thirty (30)
+day period following initial Delivery of the Software under an Order,
+Memgraph's correction of the program errors that cause the breach of warranty,
+or if Memgraph cannot substantially correct such breach in a commercially
+reasonable manner, a refund of the fees paid for the nonconforming Software
+(ii) during the remainder of the relevant Subscription Term, Memgraph's
+delivery of Support with respect to any such program errors. In the event of a
+refund remedy, Customer's licences and right to use the Software or receive
+Services will end. In the event of any noticed breach of warranty with respect
+to Services, Memgraph's sole and exclusive obligation shall be the
+re-performance of the deficient Services.
+
+7.4. Disclaimer. THIS SECTION 7 IS A LIMITED WARRANTY AND, EXCEPT EXPRESSLY AS
+SET FORTH IN SECTION 7.2, THE SOFTWARE, INCLUDING WITHOUT LIMITATION THE
+THIRD-PARTY CODE, AND ALL SERVICES ARE PROVIDED "AS IS". MEMGRAPH MAKES NO
+OTHER WARRANTIES OR REPRESENTATIONS, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE,
+AND DISCLAIMS ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE, TITLE, AND NON-INFRINGEMENT.
+
+## 8. INDEMNIFICATION.
+
+8.1. By Memgraph. Memgraph will defend against any action against Customer
+brought by a third party to the extent the action is based on a claim that the
+Software infringes a third party's patent, copyright or trademark (a "Claim")
+and indemnify Customer from the damages, liabilities, costs and expenses
+(including reasonable attorneys' fees) awarded against Customer or agreed in
+settlement by Customer resulting from such Claim. If your use of the Software
+is (or in Memgraph's opinion likely to be) enjoined, then Memgraph may, at its
+own expense and at its option: (i) substitute substantially similar
+functionality for the Software which renders it non-infringing; (ii) procure
+for Customer the right to continue to use the Software; or if (i) and (ii) are
+not commercially reasonable, terminate this Agreement and refund Customer any
+prepaid, unused (pro-rated) Fees for the duration of the then-current
+Subscription Term. The foregoing obligations of Memgraph will not apply: (i) if
+the Software is modified by any party other than Memgraph, but solely to the
+extent the alleged infringement is caused by such modification; (ii) if the
+Software is used in combination with other products or processes not provided
+or authorized by Memgraph, but solely to the extent the alleged infringement is
+caused by such combination; (iii) use of any version or release of Software
+other than the most current version or release made available to Customer by
+Memgraph, if its use would have avoided the infringement; (iv) any unauthorized
+use of the Software. THIS SECTION 8.1 SETS FORTH MEMGRAPH'S SOLE LIABILITY AND
+CUSTOMER'S SOLE AND EXCLUSIVE REMEDY WITH RESPECT TO ANY CLAIM OF INTELLECTUAL
+PROPERTY INFRINGEMENT.
+
+8.2. By Customer. Customer will indemnify and hold Memgraph and its suppliers
+harmless against any claims, liabilities, costs, and expenses (including
+reasonable attorneys' fees) that Memgraph or its suppliers may incur as a
+result of a third-party claim arising from or related to Customer Data, or
+misuse or unauthorised use of the Software by Customer or any User.
+
+8.3. Conditions. All defence and indemnity obligations under Sections 8.1 and
+8.2 are conditioned on the indemnitee (i) giving the indemnitor written notice
+of the relevant claim within thirty (30) days after the indemnitee receives
+notice of the Claim (or sooner if required by applicable law); (ii) reasonably
+cooperating with the indemnitor, at the indemnitor's expense, in the defence of
+the claim; and (iii) giving the indemnitor sole control of the defence and any
+settlement negotiations. The indemnitee may participate in the defence at its
+expense.
+
+## 9. LIMITATION OF LIABILITY.
+
+9.1. TO THE EXTENT PERMITTED BY LAW, NEITHER MEMGRAPH NOR CUSTOMER SHALL BE
+LIABLE TO THE OTHER OR ANY THIRD PARTY FOR LOST PROFITS (WHETHER DIRECT OR
+INDIRECT) OR LOSS OF USE OR DATA, SUBSTITUTE GOODS OR SERVICES, OR FOR
+INCIDENTAL, CONSEQUENTIAL, PUNITIVE, SPECIAL OR EXEMPLARY DAMAGES (INCLUDING
+DAMAGE TO BUSINESS, REPUTATION OR GOODWILL), OR INDIRECT DAMAGES OF ANY TYPE
+HOWEVER CAUSED, WHETHER BY BREACH OF WARRANTY, BREACH OF CONTRACT, IN TORT
+(INCLUDING NEGLIGENCE) OR ANY OTHER LEGAL OR EQUITABLE CAUSE OF ACTION EVEN IF
+SUCH PARTY HAS BEEN ADVISED OF SUCH DAMAGES IN ADVANCE OR IF SUCH DAMAGES WERE
+FORESEEABLE.
+
+9.2. LIMITATIONS ON DIRECT DAMAGES. EXCEPT FOR ANY EXCLUDED CLAIMS AND ANY
+DAMAGES THAT CANNOT BE LIMITED UNDER APPLICABLE LAW, IN NO EVENT WILL
+MEMGRAPH'S TOTAL AGGREGATE LIABILITY ARISING FROM OR RELATED TO THIS AGREEMENT,
+EXCEED AN AMOUNT EQUAL TO THE TOTAL AMOUNT OF FEES PAID OR PAYABLE BY CUSTOMER
+TO MEMGRAPH UNDER THIS AGREEMENT DURING THE TWELVE MONTHS PRECEDING THE CLAIM.
+THE FOREGOING LIMITATIONS SHALL NOT APPLY TO: (i) PAYMENTS TO A THIRD PARTY
+ARISING FROM A PARTY'S OBLIGATIONS UNDER SECTION 8 (INDEMNIFICATION); (ii)
+BREACH BY A PARTY OF SECTION 10 (CONFIDENTIAL INFORMATION), AND (iii)
+INFRINGEMENT BY A PARTY OF THE OTHER PARTY'S INTELLECTUAL PROPERTY RIGHTS
+(COLLECTIVELY EXCLUSIONS (i)-(iii) ARE REFERRED TO AS THE "EXCLUDED CLAIMS").
+WITH RESPECT TO ANY EXCLUDED CLAIMS, MEMGRAPH'S TOTAL AGGREGATE LIABILITY SHALL
+IN NO EVENT EXCEED £1,000,000.
+
+9.3. Nothing in this Agreement excludes the liability of Memgraph for death or
+personal injury caused by the Supplier's negligence; or for fraud or fraudulent
+misrepresentation.
+
+## 10. CONFIDENTIALITY.
+
+10.1. The Receiving Party will use the same degree of care that it uses to
+protect the confidentiality of its own confidential information of like kind
+(but not less than reasonable care) to (i) not use any confidential information
+of the Disclosing Party for any purpose outside the scope of this agreement and
+(ii) except as otherwise authorized by the Disclosing Party in writing, limit
+access to confidential information of the Disclosing Party to those of its and
+its affiliates' employees and contractors who need that access for purposes
+consistent with this agreement and who have signed confidentiality agreements
+with the receiving party containing protections not materially less protective
+of the confidential information than those herein. Neither party will disclose
+the terms of this agreement or any Orders to any third-party other than its
+affiliates, legal counsel, and accountants without the other party's prior
+written consent, provided that a party that makes any such disclosure to its
+affiliate, legal counsel or accountants will remain responsible for such
+affiliate's, legal counsel's, or accountant's compliance with this
+"confidentiality" section.
+
+10.2. Compelled Disclosure. The Receiving Party may disclose confidential
+information of the Disclosing Party to the extent compelled by law to do so,
+provided the Receiving Party gives the Disclosing Party prior notice of the
+compelled disclosure (to the extent legally permitted) and reasonable
+assistance, at the Disclosing Party's cost, if the Disclosing Party wishes to
+contest the disclosure. If the receiving party is compelled by law to disclose
+the Disclosing Party's confidential information as part of a civil proceeding
+to which the Disclosing Party is a party, and the Disclosing Party is not
+contesting the disclosure, the Disclosing Party will reimburse the Receiving
+Party for its reasonable cost of complying and providing secure access to that
+confidential information.
+
+## 11. TERMINATION.
+
+11.1. Term. The term ("Term") of this Agreement will commence on the Effective
+Date and continue until all Subscriptions, licence terms and Orders expire,
+unless earlier terminated in accordance with this Section 11.
+
+11.2. Termination for Cause. In the event of a material breach of this
+Agreement (excluding any breaches for which an exclusive remedy is expressly
+provided), the non-breaching party may terminate this Agreement if such breach
+is not cured within thirty (30) days after written notice thereof (except that
+for a breach of Section 3.4 ("General Restrictions"), there will be no cure
+period). For clarity, material breach of this Agreement includes, but is not
+limited to, failure to timely pay amounts due hereunder, exceeding the scope of
+any Licence granted hereunder (including the Licensed Capacity), violating the
+Licence restrictions, breach of Section 6.1 and failing to protect the other
+party's Confidential Information.
+
+11.3. Without affecting any other right or remedy available to it, either party
+may terminate this Agreement with immediate effect by giving written notice to
+the other party if the other party:
+
+11.3.1. suspends, or threatens to suspend, payment of its debts or is unable to
+pay its debts as they fall due or admits inability to pay its debts or is
+deemed unable to pay its debts within the meaning of section 123 of the
+Insolvency Act 1986, as if the words "it is proved to the satisfaction of the
+court" did not appear in sections 123(1)(e) or 123(2) of the Insolvency Act
+1986; or
+
+11.3.2. the other party commences negotiations with all or any class of its
+creditors with a view to rescheduling any of its debts, or makes a proposal for
+or enters into any compromise or arrangement with its creditors other than for
+the sole purpose of a scheme for a solvent amalgamation of that other party
+with one or more other companies or the solvent reconstruction of that other
+party; or
+
+11.3.3. a petition is filed, a notice is given, a resolution is passed, or an
+order is made, for or in connection with the winding up of that other party
+other than for the sole purpose of a scheme for a solvent amalgamation of that
+other party with one or more other companies or the solvent reconstruction of
+that other party; or
+
+11.3.4. an application is made to court, or an order is made, for the
+appointment of an administrator, or if a notice of intention to appoint an
+administrator is given or if an administrator is appointed, over the other
+party; or
+
+11.3.5. the holder of a qualifying floating charge over the assets of that
+other party has become entitled to appoint or has appointed an administrative
+receiver; or
+
+11.3.6. a person becomes entitled to appoint a receiver over the assets of the
+other party or a receiver is appointed over the assets of the other party; or
+
+11.3.7. a creditor or encumbrancer of the other party attaches or takes
+possession of, or a distress, execution, sequestration or other such process is
+levied or enforced on or sued against, the whole or any part of the other
+party's assets and such attachment or process is not discharged within 30 days;
+or
+
+11.3.8. any event occurs, or proceeding is taken, with respect to the other
+party in any jurisdiction to which it is subject that has an effect equivalent
+or similar to any of the events mentioned in clause 11.3.1 to clause 11.3.7
+(inclusive); or
+
+11.3.9. the other party suspends or ceases, or threatens to suspend or cease,
+carrying on all or a substantial part of its business.
+
+11.4. Effect of Termination. Upon the termination of this Agreement: (i) all
+licences will terminate; (ii) Customer will immediately discontinue all use of
+the affected Software, erase all full and partial copies of the Software and
+(subject to this Section) all other tangible embodiments of Memgraph
+Confidential Information in Customer's possession or control, and promptly
+certify the same to Memgraph; (iii) Memgraph may immediately cease providing
+the Services; (iv) (subject to this Section), Memgraph will return or delete
+all tangible embodiments of Customer Confidential Information in Memgraph's
+possession or control; and (v) Sections 1 ("Definitions"), 3.4 ("General
+Restrictions"), 3.5 ("Ownership; Reservation of Rights"), 5 ("Fees and
+Payment"), 7.3 ("Limitations"), 7.4 ("Disclaimer"), 8 ("Indemnification"), 9
+("Limitation of Liability"), 10 ("Confidentiality"), 11.4 ("Effect of
+Termination"), and 12 ("Miscellaneous") will survive. If a party's file
+retention policies or a valid legal order provides for backup or archival
+copies of files to be retained, such party will notify the other party of such
+policy or order, protect the other party's Confidential Information as required
+hereunder, and permanently erase, delete, or destroy such Confidential
+Information as soon as permissible under such policy or order.
+
+## 12. MISCELLANEOUS.
+
+12.1. Assignment. This Agreement will bind and inure to the benefit of each
+party's permitted successors and assigns. Memgraph may assign this Agreement to
+any affiliate or in connection with a merger, reorganization, acquisition, or
+other transfer of all or substantially all of Memgraph's assets or voting
+securities. Customer may not assign or transfer this Agreement, in whole or in
+part, without Memgraph's written consent except that Customer may assign its
+rights and obligations under this Agreement, in whole but not in part, without
+Memgraph's written consent in connection with any merger, consolidation, sale
+of all or substantially all of Customer's assets or voting stock, or any other
+similar transaction provided that: (i) the assignee is not a direct competitor
+of Memgraph; (ii) Customer provides prompt written notice of such assignment to
+Memgraph; (iii) the assignee is capable of fully performing Customer's
+obligations under this Agreement; and (iv) the assignee agrees to be bound by
+the terms and conditions of this Agreement. Any attempt to transfer or assign
+this Agreement without such written consent will be null and void.
+
+12.2. Force Majeure. Memgraph shall have no liability to the Customer under
+this Agreement if it is prevented from or delayed in performing its obligations
+under this Agreement, or from carrying on its business, by acts, events,
+omissions or accidents beyond its reasonable control, including strikes,
+lock-outs or other industrial disputes (whether involving the workforce of
+Memgraph or any other party), failure of a utility service or transport or
+telecommunications network, act of God, war, pandemic, riot, civil commotion,
+malicious damage, compliance with any law or governmental order, rule,
+regulation or direction, accident, breakdown of plant or machinery, fire,
+flood, storm or default of suppliers or subcontractors, provided that the
+Customer is notified of such an event and its expected duration.
+
+12.3. Governing Law. This Agreement and any dispute or claim arising out of or
+in connection with it or its subject matter or formation (including
+non-contractual disputes or claims) shall be governed by and construed in
+accordance with the law of England and Wales.
+
+12.4. Jurisdiction. Each party irrevocably agrees that the courts of England
+and Wales shall have exclusive jurisdiction to settle any dispute or claim
+arising out of or in connection with this Agreement or its subject matter or
+formation (including non-contractual disputes or claims).
+
+12.5. Severability; Waiver; Construction. If a court of competent jurisdiction
+adjudges any provision of this Agreement to be invalid or unenforceable, the
+remaining provisions of this Agreement, if capable of substantial performance,
+will continue in full force and effect without being impaired or invalidated in
+any way. The parties agree to replace any invalid provision with a valid
+provision that most closely approximates the intent and economic effect of the
+invalid provision. All waivers must be in writing. A party's consent to, or
+waiver of, enforcement of this Agreement on one occasion will not be deemed a
+waiver of any other provision or such provision on any other occasion. In this
+Agreement, the word "including" means "including but not limited to." No
+presumption will operate in favour of or against any party as a result of its
+role in drafting this Agreement.
+
+12.6. Subcontractors. Memgraph may use the services of subcontractors in
+connection with its performance of this Agreement, provided that Memgraph
+remains solely responsible for (i) compliance of any such subcontractor with
+the terms of this Agreement and (ii) the overall performance of Memgraph as
+required under this Agreement.
+
+12.7. Use of Aggregate Data. Customer agrees that Memgraph may collect, use and
+disclose quantitative data and metadata derived from the use of the Software
+(i) for its own internal, statistical analysis, (ii) to develop and improve the
+Software and (iii) to create and distribute reports and other materials
+regarding use of the Software. For clarity, any such data collected, used, and
+disclosed will be in anonymized aggregate form only and shall not identify
+Customer or its Users, or disclose any Customer Data.
+
+12.8. Independent Contractors. The parties are independent contractors. No
+agency, partnership, franchise, joint venture, or employment relationship is
+intended or created by this Agreement. Neither party has the power or authority
+to create or assume any obligation, or make any representations or warranties,
+on behalf of the other party.
+
+12.9. Publicity. Memgraph may, in conformity with Customer's trademark usage
+guidelines, use Customer's name and logo in Memgraph's sales and marketing
+materials, including in business presentations, Customer lists, and on
+websites. Neither party will issue a press release regarding this Agreement
+without the other party's prior written consent. Neither party will disclose
+the terms of this Agreement to any third party, except as required by law.
+
+12.10. Notice. Any notice, consent, or waiver hereunder must be in writing,
+addressed to the attention of "Legal Department" at the address set forth
+above, and delivered by personal delivery, reputable rapid courier, or
+certified/registered mail, return receipt requested, and will be deemed given
+upon personal delivery, one (1) day after deposit with an overnight domestic
+courier, two (2) days after deposit with an international courier, or five (5)
+days after deposit in the certified or registered mail. A party may specify a
+new address by providing notice to the other party in accordance with this
+Section.
+
+12.11. Compliance with Law. Each party will comply with all applicable laws,
+regulations, and orders of any governmental authority of competent jurisdiction
+in its performance under this Agreement, including but not limited to those
+applicable to data collection and the privacy and security of personal
+information, including trans-border data transfers and data breach notification
+requirements as required of each party by law.
+
+12.12. Supremacy; Modification. This Agreement will prevail over any written
+instrument submitted by Customer; the terms of any purchase order,
+acknowledgement, or similar document submitted by Customer to Memgraph will
+have no effect. If the express terms of an Order Document conflict with this
+Agreement, the terms on the Order Document will prevail, but only with respect
+to that Order Document. This Agreement cannot be varied or supplemented by
+course of dealing or by usage of trade. All modifications or amendments to this
+Agreement must be in writing and signed by both parties, except that subsequent
+renewals and purchases of additional Licensed Capacity can be procured by
+payment against an issued invoice as set forth in Section 5 ("Fees and
+payment") above.
+
+12.13. No Third Party Beneficiaries. This Agreement is not intended and shall
+not be construed to give any third party any interest or rights with respect to
+or in connection with any agreement or provision herein, except as expressly
+provided for in this Agreement.
+
+12.14. Entire Agreement. This Agreement in its original English text, sets
+forth the complete, exclusive, and final agreement of the parties concerning
+the subject matter hereof, supersedes, replaces, and merges all prior and
+contemporaneous agreements, communications, and understandings, both written
+and oral, between them concerning the subject matter hereof. This Agreement may
+be executed in counterparts.

release/LICENSE_ENTERPRISE_TRIAL.md

@@ -1,68 +0,0 @@
-# User License Agreement
-
-1. Description
-
-THIS LICENSE AGREEMENT GOVERNS LICENSEE’S USE OF THE MEMGRAPH ENTERPRISE
-RELEASE AND DOCUMENTATION UNDER A TRIAL LICENSE.
-
-2. Trial License Grant
-
-The Software and Documentation are provided to Licensee at no charge and are
-licensed, not sold to Licensee. No ownership of any part of the Software and
-Documentation is hereby transferred to Licensee. Subject to (i) the terms and
-conditions of this License Agreement, (ii) any additional license restrictions
-and parameters contained on Licensor’s quotation, website, or order form
-(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
-non-transferable and non-exclusive license to install, access and use the
-Software (in object code form only) and Documentation for Licensee’s internal
-business purposes only. All rights relating to the Software and Documentation
-that are not expressly licensed in this License Agreement, whether now existing
-or which may hereafter come into existence are reserved for Licensor. Licensee
-shall not remove, obscure, or alter any proprietary rights notices (including
-without limitation copyright and trademark notices), which may be affixed to or
-contained within the Software or Documentation.
-
-The Memgraph Enterprise Release Trial License is granted without charge for the
-trial or evaluation period specified when the Memgraph Enterprise Release was
-delivered, or if no term was specified, for thirty (30) calendar days. You may
-not use the Memgraph Enterprise Release under the Trial License more than once
-in any twelve (12) month period. Memgraph may revoke the Trial License at any
-time and for any reason.
-
-3. Restrictions
-
-Licensee will not, directly or indirectly, (a) copy the Software or
-Documentation in any manner or for any purpose; (b) install, access or use any
-component of the Software or Documentation for any purpose not expressly
-granted in Section 2 above; (c) resell, distribute, publicly display or
-publicly perform the Software or Documentation or any component thereof, by
-transfer, lease, loan or any other means, or make it available for use by
-others in any time-sharing, service bureau or similar arrangement; (d)
-disassemble, decrypt, extract, reverse engineer or reverse compile the
-Software, or otherwise attempt to discover the source code, confidential
-algorithms or techniques incorporated in the Software; (e) export the Software
-or Documentation in violation of any applicable laws or regulations; (f)
-modify, translate, adapt, or create derivative works from the Software or
-Documentation; (g) circumvent, disable or otherwise interfere with
-security-related features of the Software or Documentation; (h)
-reverse-engineer, disassemble, attempt to derive the source code; (i) use the
-Software or Documentation for any illegal purpose, in any manner that is
-inconsistent with the terms of this License Agreement, or to engage in illegal
-activity; (j) remove or alter any trademark, logo, copyright or other
-proprietary notices, legends, symbols or labels on, or embedded in, the
-Software or Documentation; or (k) provide access to the Software or
-Documentation to third parties.
-
-4. Warranty Disclaimer
-
-THE MEMGRAPH ENTERPRISE RELEASE AND DOCUMENTATION UNDER THE TRIAL LICENSE ARE
-PROVIDED “AS IS” FOR DEVELOPMENT, TESTING, AND EVALUATION PURPOSES ONLY. IT IS
-NOT LICENSED FOR PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL
-WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
-WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
-INFRINGEMENT OF THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER
-PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
-FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
-OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
-
-BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.

release/README.md

@@ -1,43 +1,4 @@
-# Memgraph Release Packaging
+# Memgraph Release
 
-Various tools and packaging configuration files should be put under this
-directory. Common files for all packages can be kept in the root of this
-directory, for example `memgraph.service`. If the common stuff should be
-grouped, it should be in a subdirectory. `examples` directory is one such
-case. Packaging specific stuff must have its own directory.
-
-Currently we support distributing Memgraph binary through the following
-packages.
-
-  * Debian package
-  * RPM package
-  * Docker image
-  * ArchLinux package
-
-## Release process
-
-While releasing an official version of Memgraph, there are two possible
-scenarios:
-  * First release in new major.minor series
-  * Patch release in existing major.minor series
-
-To release a new major.minor release of Memgraph you should execute the
-following steps:
-  1. Land all diffs that must be in the new release
-  2. Document all changes in `CHANGELOG.md` and land them
-  3. From the `master` branch, create a branch named `release/X.Y` and push it
-     to `origin`
-  4. Create the release packages triggering a `mg-master-release-branch-test`
-     using branch `release/X.Y` on Apollo
-  5. Enjoy
-
-To release a new patch release in an existing major.minor series you should
-execute the following steps:
-  1. Checkout to the `release/X.Y` branch
-  2. Cherry-pick all landed commits that should be included in the patch version
-  3. Document all changes in `CHANGELOG.md` and commit them
-  4. Edit the root `CMakeLists.txt` and set `MEMGRAPH_OVERRIDE_VERSION` to
-     `X.Y.patch` and commit the change
-  5. Create the release packages triggering a `mg-master-release-branch-test`
-     using branch `release/X.Y` on Apollo
-  6. Enjoy
+Please continue in
+[Notion](https://www.notion.so/memgraph/Release-621733518c82456ebc119d2a025b5dd3).

release/arch-pkg/PKGBUILD.proto

@@ -16,7 +16,7 @@ optdepends=()
 provides=()
 conflicts=()
 replaces=()
-backup=("etc/memgraph/memgraph.conf" "etc/logrotate.d/memgraph" "etc/logrotate.d/memgraph_audit")
+backup=("etc/memgraph/memgraph.conf" "etc/logrotate.d/memgraph")
 options=()
 install=memgraph.install
 changelog=

release/arch-pkg/memgraph.install

@@ -8,13 +8,9 @@ post_install() {
   chmod 750 /var/lib/memgraph || exit 1
   chown memgraph:adm /var/log/memgraph || exit 1
   chmod 750 /var/log/memgraph || exit 1
-  # Make examples directory immutable (optional)
-  chattr +i -R /usr/share/memgraph/examples || true
   echo "Enable and start 'memgraph.service' to use Memgraph" || exit 1
 }
 
 pre_remove() {
-  # Remove optional immutability from examples directory to allow removal
-  chattr -i -R /usr/share/memgraph/examples || true
   systemctl disable memgraph.service
 }

release/debian/enterprise/conffiles

@@ -1,4 +1,3 @@
 /etc/memgraph/memgraph.conf
 /etc/memgraph/auth_module/ldap.example.yaml
 /etc/logrotate.d/memgraph
-/etc/logrotate.d/memgraph_audit

release/debian/enterprise/copyright

@@ -1 +1 @@
-../../LICENSE_ENTERPRISE_TRIAL.md
+../../LICENSE_ENTERPRISE.md

release/debian/postinst

@@ -28,22 +28,6 @@ case "$1" in
         chown memgraph:adm /var/log/memgraph || exit 1
         chmod 750 /var/log/memgraph || exit 1
 
-        # Create telemetry directory in examples
-        for i in /usr/share/memgraph/examples/*; do
-            # The telemetry directory may already exist from some prior installation
-            if [ ! -d $i/telemetry ]; then
-                mkdir $i/telemetry || exit 1
-            fi
-        done
-
-        # Change ownership of all examples
-        chown -R memgraph:memgraph /usr/share/memgraph/examples || exit 1
-
-        # Make snapshots directory immutable (optional)
-        for i in /usr/share/memgraph/examples/*; do
-            chattr +i -R $i/snapshots || true
-        done
-
         # Generate SSL certificates
         if [ ! -d /etc/memgraph/ssl ]; then
             mkdir /etc/memgraph/ssl || exit 1
@@ -86,4 +70,9 @@ if [ -d /run/systemd/system ]; then
     deb-systemd-invoke start memgraph.service >/dev/null || true
 fi
 
+# Take a look at the preinst script for the detailed explanation.
+if dpkg-maintscript-helper supports rm_conffile 2>/dev/null; then
+    dpkg-maintscript-helper rm_conffile /etc/logrotate.d/memgraph_audit 1.1.999 -- "$@"
+fi
+
 exit 0

release/debian/postrm

@@ -30,12 +30,6 @@ case "$1" in
         if [ -d /etc/memgraph/ssl ]; then
             rm -rf /etc/memgraph/ssl
         fi
-        # Remove stored telemetry data
-        for i in /usr/share/memgraph/examples/*; do
-            if [ -d $i/telemetry ]; then
-                rm -rf $i/telemetry
-            fi
-        done
         # Don't remove the 'memgraph' user, since we cannot be sure whether it
         # existed before.
     ;;
@@ -71,4 +65,9 @@ if [ "$1" = "purge" ]; then
     fi
 fi
 
+# Take a look at the preinst script for the detailed explanation.
+if dpkg-maintscript-helper supports rm_conffile 2>/dev/null; then
+    dpkg-maintscript-helper rm_conffile /etc/logrotate.d/memgraph_audit 1.1.999 -- "$@"
+fi
+
 exit 0

release/debian/preinst

@@ -0,0 +1,19 @@
+#!/bin/sh
+# preinst script for memgraph
+#
+# see: dh_installdeb(1)
+
+set -e
+
+# Manage (remove) /etc/logrotate.d/memgraph_audit file because the whole
+# logrotate config is moved to /etc/logrotate.d/memgraph since v1.2.0.
+# Note: Only used to manage Memgraph Enterprise config but packaged into the
+# Memgraph Community as well.
+if dpkg-maintscript-helper supports rm_conffile 2>/dev/null; then
+    # 1.1.999 is chosen because it's high enough version number. It's highly
+    # unlikely (impossible) that the patch number in v1.1 reaches 999
+    # (it's 0 on 2020-10-17).
+    # Tested with: `dpkg --compare-versions -- "1.2" le-nl "1.1.999"`
+    # (used inside dpkg-maintscript-helper script).
+    dpkg-maintscript-helper rm_conffile /etc/logrotate.d/memgraph_audit 1.1.999 -- "$@"
+fi

release/debian/prerm

@@ -18,8 +18,6 @@ set -e
 
 case "$1" in
     remove|upgrade|deconfigure|failed-upgrade)
-        # Remove optional immutability from examples directory to allow removal
-        chattr -i -R /usr/share/memgraph/examples || true
         if [ -d /run/systemd/system ]; then
             deb-systemd-invoke stop memgraph.service >/dev/null
         fi