Disable diff build

Co-authored-by: antonio2368 <antonio2368@users.noreply.github.com>

.clang-tidy

@@ -54,7 +54,7 @@ Checks: '*,
          -readability-magic-numbers,
          -readability-named-parameter'
 WarningsAsErrors: ''
-HeaderFilterRegex: ''
+HeaderFilterRegex: 'src/.*'
 AnalyzeTemporaryDtors: false
 FormatStyle:     none
 CheckOptions:

.github/workflows/diff.yaml

@@ -1,16 +1,11 @@
 name: Diff
 
 on:
-  push:
-    paths-ignore:
-      - 'docs/**'
-      - '**/*.md'
-      - '.clang-*'
 
 jobs:
   community_build:
     name: "Community build"
-    runs-on: [self-hosted, General, Linux, X64, Debian10]
+    runs-on: [self-hosted, Linux, X64, Diff]
     env:
       THREADS: 24
 
@@ -54,8 +49,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create community DEB package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create community DEB package.
           mkdir output && cd output
           cpack -G DEB --config ../CPackConfig.cmake
 
@@ -65,9 +65,9 @@ jobs:
           name: "Community DEB package"
           path: build/output/memgraph*.deb
 
-  coverage_build:
-    name: "Coverage build"
-    runs-on: [self-hosted, General, Linux, X64, Debian10]
+  code_analysis:
+    name: "Code analysis"
+    runs-on: [self-hosted, Linux, X64, Diff]
     env:
       THREADS: 24
 
@@ -79,7 +79,7 @@ jobs:
           # branches and tags. (default: 1)
           fetch-depth: 0
 
-      - name: Build coverage binaries
+      - name: Build combined ASAN, UBSAN and coverage binaries
         run: |
           # Activate toolchain.
           source /opt/toolchain-v2/activate
@@ -87,9 +87,8 @@ jobs:
           # Initialize dependencies.
           ./init
 
-          # Build coverage binaries.
           cd build
-          cmake -DTEST_COVERAGE=ON ..
+          cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
           make -j$THREADS memgraph__unit
 
       - name: Run unit tests
@@ -97,9 +96,9 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Run unit tests.
+          # Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
           cd build
-          ctest -R memgraph__unit --output-on-failure -j$THREADS
+          LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
 
       - name: Compute code coverage
         run: |
@@ -120,9 +119,19 @@ jobs:
           name: "Code coverage"
           path: tools/github/generated/code_coverage.tar.gz
 
+      - name: Run clang-tidy
+        run: |
+          source /opt/toolchain-v2/activate
+
+          # Restrict clang-tidy results only to the modified parts
+          git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
+
+          # Fail if any warning is reported
+          ! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
+
   debug_build:
     name: "Debug build"
-    runs-on: [self-hosted, General, Linux, X64, Debian10]
+    runs-on: [self-hosted, Linux, X64, Diff]
     env:
       THREADS: 24
 
@@ -196,7 +205,7 @@ jobs:
 
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, General, Linux, X64, Debian10]
+    runs-on: [self-hosted, Linux, X64, Diff]
     env:
       THREADS: 24
 
@@ -208,21 +217,6 @@ jobs:
           # branches and tags. (default: 1)
           fetch-depth: 0
 
-      - name: Set up parent
-        run: |
-          # Remove parent folder (if it exists).
-          cd ..
-          if [ -d parent ]; then
-            rm -rf parent
-          fi
-
-          # Copy untouched repository to parent folder.
-          cp -r memgraph parent
-
-          # Checkout previous commit
-          cd parent
-          git checkout HEAD~1
-
       - name: Build release binaries
         run: |
           # Activate toolchain.
@@ -236,47 +230,6 @@ jobs:
           cmake -DCMAKE_BUILD_TYPE=release ..
           make -j$THREADS
 
-      - name: Build parent binaries
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v2/activate
-
-          # Initialize dependencies.
-          cd ../parent
-          ./init
-
-          # Build parent binaries.
-          cd build
-          cmake -DCMAKE_BUILD_TYPE=release ..
-          make -j$THREADS memgraph memgraph__macro_benchmark
-
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Run parent macro benchmark tests
-        run: |
-          cd ../parent/tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Compute macro benchmark summary
-        run: |
-          ./tools/github/macro_benchmark_summary \
-            --current tests/macro_benchmark/.harness_summary \
-            --previous ../parent/tests/macro_benchmark/.harness_summary \
-            --output macro_benchmark_summary.txt
-
-      - name: Save macro benchmark summary
-        uses: actions/upload-artifact@v2
-        with:
-          name: "Macro benchmark summary"
-          path: macro_benchmark_summary.txt
-
       - name: Run GQL Behave tests
         run: |
           cd tests/gql_behave
@@ -328,8 +281,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create enterprise DEB package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create enterprise DEB package.
           mkdir output && cd output
           cpack -G DEB --config ../CPackConfig.cmake
 
@@ -339,9 +297,18 @@ jobs:
           name: "Enterprise DEB package"
           path: build/output/memgraph*.deb
 
+      - name: Save test data
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: "Test data"
+          path: |
+            # multiple paths could be defined
+            build/logs
+
   release_jepsen_test:
     name: "Release Jepsen Test"
-    runs-on: [self-hosted, Linux, X64, Debian10, HP-DL360G6-v2-3]
+    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
     #continue-on-error: true
     env:
       THREADS: 24
@@ -378,3 +345,64 @@ jobs:
         with:
           name: "Jepsen Report"
           path: tests/jepsen/Jepsen.tar.gz
+
+  release_benchmarks:
+    name: "Release benchmarks"
+    runs-on: [self-hosted, Linux, X64, Diff, Gen7]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binarie.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Run macro benchmarks
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Upload macro benchmark results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "macro_benchmark" \
+                    --benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}"
+
+      - name: Run mgbench
+        run: |
+          cd tests/mgbench
+          ./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
+
+      - name: Upload mgbench results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "mgbench" \
+                    --benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}"

.github/workflows/full_clang_tidy.yaml

@@ -0,0 +1,44 @@
+name: Run clang-tidy on the full codebase
+
+on:
+  workflow_dispatch:
+
+jobs:
+  clang_tidy_check:
+    name: "Clang-tidy check"
+    runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
+    env:
+      THREADS: 24
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v2
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v2/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run clang-tidy
+        run: |
+          source /opt/toolchain-v2/activate
+
+          # The results are also written to standard output in order to retain them in the logs
+          ./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
+            tee ./build/full_clang_tidy_output.txt
+
+      - name: Summarize clang-tidy results
+        run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

.github/workflows/package_all.yaml

@@ -0,0 +1,248 @@
+name: Package All
+
+# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
+
+on: workflow_dispatch
+
+jobs:
+  centos-7_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community centos-7
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-7_community
+          path: build/output/centos-7/memgraph*.rpm
+
+  centos-8_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community centos-8
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-8_community
+          path: build/output/centos-8/memgraph*.rpm
+
+  debian-9_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community debian-9
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-9_community
+          path: build/output/debian-9/memgraph*.deb
+
+  debian-10_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community debian-10
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-10_community
+          path: build/output/debian-10/memgraph*.deb
+
+  docker_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          cd release/package
+          ./run.sh package community debian-10 --for-docker
+          ./run.sh docker
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: docker_community
+          path: build/output/docker/memgraph*.tar.gz
+
+  ubuntu-1804_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community ubuntu-18.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-1804_community
+          path: build/output/ubuntu-18.04/memgraph*.deb
+
+  ubuntu-2004_community:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package community ubuntu-20.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-2004_community
+          path: build/output/ubuntu-20.04/memgraph*.deb
+
+  centos-7_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise centos-7
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-7_enterprise
+          path: build/output/centos-7/memgraph*.rpm
+
+  centos-8_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise centos-8
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos-8_enterprise
+          path: build/output/centos-8/memgraph*.rpm
+
+  debian-9_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise debian-9
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-9_enterprise
+          path: build/output/debian-9/memgraph*.deb
+
+  debian-10_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise debian-10
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: debian-10_enterprise
+          path: build/output/debian-10/memgraph*.deb
+
+  docker_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          cd release/package
+          ./run.sh package enterprise debian-10 --for-docker
+          ./run.sh docker
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: docker_enterprise
+          path: build/output/docker/memgraph*.tar.gz
+
+  ubuntu-1804_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise ubuntu-18.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-1804_enterprise
+          path: build/output/ubuntu-18.04/memgraph*.deb
+
+  ubuntu-2004_enterprise:
+    runs-on: [self-hosted, DockerMgBuild]
+    timeout-minutes: 60
+    steps:
+      - name: "Set up repository"
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # Required because of release/get_version.py
+      - name: "Build package"
+        run: |
+          ./release/package/run.sh package enterprise ubuntu-20.04
+      - name: "Upload package"
+        uses: actions/upload-artifact@v2
+        with:
+          name: ubuntu-2004_enterprise
+          path: build/output/ubuntu-20.04/memgraph*.deb

.github/workflows/release_centos8.yaml

@@ -39,8 +39,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create community RPM package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create community RPM package.
           mkdir output && cd output
           cpack -G RPM --config ../CPackConfig.cmake
           rpmlint memgraph*.rpm
@@ -232,8 +237,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create enterprise RPM package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create enterprise RPM package.
           mkdir output && cd output
           cpack -G RPM --config ../CPackConfig.cmake
           rpmlint memgraph*.rpm

.github/workflows/release_debian10.yaml

@@ -1,4 +1,4 @@
-name: Release Debian10
+name: Release Debian 10
 
 on:
   workflow_dispatch:
@@ -39,8 +39,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create community DEB package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create community DEB package.
           mkdir output && cd output
           cpack -G DEB --config ../CPackConfig.cmake
 
@@ -231,8 +236,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create enterprise DEB package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create enterprise DEB package.
           mkdir output && cd output
           cpack -G DEB --config ../CPackConfig.cmake
 

.github/workflows/release_ubuntu2004.yaml

@@ -1,4 +1,4 @@
-name: Release Ubuntu20.04
+name: Release Ubuntu 20.04
 
 on:
   workflow_dispatch:
@@ -39,8 +39,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create community DEB package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create community DEB package.
           mkdir output && cd output
           cpack -G DEB --config ../CPackConfig.cmake
 
@@ -231,8 +236,13 @@ jobs:
           # Activate toolchain.
           source /opt/toolchain-v2/activate
 
-          # Create enterprise DEB package.
           cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create enterprise DEB package.
           mkdir output && cd output
           cpack -G DEB --config ../CPackConfig.cmake
 

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## Future
 
+### Major Feature and Improvements
+
+* Replaced mg_client with mgconsole
+
+### Bug Fixes
+
+* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
+  For example, parsing of `mgp.List[mgp.Map]` works now.
+* Fixed memory tracking issues. Some of the allocation and deallocation weren't
+  tracked during the query execution.
+* Fixed reading CSV files that are using CRLF as the newline symbol.
+* Fixed permission issues for `LOAD CSV`, `FREE MEMORY`, `LOCK DATA DIRECTORY`,
+  and replication queries.
+
+## v1.4.0
+
 ### Breaking Changes
 
 * Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.
@@ -10,7 +26,7 @@
 ### Major Feature and Improvements
 
 * Added replication to community version.
-* Add support for multiple query modules directories at the same time.
+* Added support for multiple query modules directories at the same time.
   You can now define multiple, comma-separated paths to directories from
   which the modules will be loaded using the `--query-modules-directory` flag.
 * Added support for programatically reading in data from CSV files through the
@@ -22,12 +38,15 @@
 * Added the memory limit and amount of currently allocated bytes in the result of `SHOW STORAGE INFO` query.
 * Added `QUERY MEMORY LIMIT num (KB|MB)` to Cypher queries which allows you to limit memory allocation for
   the entire query. It can be added only at the end of the entire Cypher query.
+* Added logs for the different parts of the recovery process. `INFO`, `DEBUG` and `TRACE` level all contain
+  additional information that is printed out while the recovery is in progress.
 
 ### Bug Fixes
 
 * Fixed garbage collector by correctly marking the oldest current timestamp
   after the database was recovered using the durability files.
 * Fixed reloading of the modules with changed result names.
+* Fixed profile query to show the correct name of the ScanAll operator variant.
 
 ## v1.3.0
 

CMakeLists.txt

@@ -312,8 +312,9 @@ if (UBSAN)
   # runtime library and c++ standard libraries are present.
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
-  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
-  # Make sure llvm-symbolizer binary is in path
+  # Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
+  # Make sure llvm-symbolizer binary is in path.
+  # To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
 endif()
 
 set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
@@ -335,3 +336,7 @@ endif()
 if(QUERY_MODULES)
   add_subdirectory(query_modules)
 endif()
+
+install(FILES ${CMAKE_BINARY_DIR}/bin/mgconsole
+  PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+  TYPE BIN)

environment/os/centos-7.sh

@@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
     libffi-devel libxml2-devel perl-Digest-MD5 # llvm
     libedit-devel pcre-devel automake bison # swig
 )
+
 TOOLCHAIN_RUN_DEPS=(
     make # generic build tools
     tar gzip bzip2 xz # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
     readline # for cmake and llvm
     libffi libxml2 # for llvm
 )
+
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkgconfig # build system
@@ -48,9 +50,11 @@ MEMGRAPH_BUILD_DEPS=(
     which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
     autoconf # for jemalloc code generation
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     local missing=""
     for pkg in $1; do
@@ -75,16 +79,13 @@ check() {
         exit 1
     fi
 }
+
 install() {
     cd "$DIR"
     if [ "$EUID" -ne 0 ]; then
         echo "Please run as root."
         exit 1
     fi
-    if [ "$SUDO_USER" == "" ]; then
-        echo "Please run as sudo."
-        exit 1
-    fi
     # If GitHub Actions runner is installed, append LANG to the environment.
     # Python related tests doesn't work the LANG export.
     if [ -d "/home/gh/actions-runner" ]; then
@@ -118,11 +119,16 @@ install() {
             continue
         fi
         if [ "$pkg" == PyYAML ]; then
-            sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
             continue
         fi
         yum install -y "$pkg"
     done
 }
+
 deps=$2"[*]"
 "$1" "${!deps}"

environment/os/centos-8.sh

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
     libffi-devel libxml2-devel # for llvm
     libedit-devel pcre-devel automake bison # for swig
 )
+
 TOOLCHAIN_RUN_DEPS=(
     make # generic build tools
     tar gzip bzip2 xz # used for archive unpacking
@@ -25,6 +26,7 @@ TOOLCHAIN_RUN_DEPS=(
     readline # for cmake and llvm
     libffi libxml2 # for llvm
 )
+
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkgconf-pkg-config # build system
@@ -47,9 +49,11 @@ MEMGRAPH_BUILD_DEPS=(
     sbcl # for custom Lisp C++ preprocessing
     autoconf # for jemalloc code generation
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     local missing=""
     for pkg in $1; do
@@ -68,16 +72,13 @@ check() {
         exit 1
     fi
 }
+
 install() {
     cd "$DIR"
     if [ "$EUID" -ne 0 ]; then
         echo "Please run as root."
         exit 1
     fi
-    if [ "$SUDO_USER" == "" ]; then
-        echo "Please run as sudo."
-        exit 1
-    fi
     # If GitHub Actions runner is installed, append LANG to the environment.
     # Python related tests doesn't work the LANG export.
     if [ -d "/home/gh/actions-runner" ]; then
@@ -86,6 +87,7 @@ install() {
         echo "NOTE: export LANG=en_US.utf8"
     fi
     dnf install -y epel-release
+    dnf install -y 'dnf-command(config-manager)'
     dnf config-manager --set-enabled powertools # Required to install texinfo.
     dnf update -y
     dnf install -y wget git python36 python3-pip
@@ -135,11 +137,16 @@ install() {
             continue
         fi
         if [ "$pkg" == PyYAML ]; then
-            sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
             continue
         fi
         dnf install -y "$pkg"
     done
 }
+
 deps=$2"[*]"
 "$1" "${!deps}"

environment/os/debian-10.sh

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
     libffi-dev libxml2-dev # for llvm
     libedit-dev libpcre3-dev automake bison # for swig
 )
+
 TOOLCHAIN_RUN_DEPS=(
     make # generic build tools
     tar gzip bzip2 xz-utils # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
     libreadline7 # for cmake and llvm
     libffi6 libxml2 # for llvm
 )
+
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkg-config # build system
@@ -45,12 +47,15 @@ MEMGRAPH_BUILD_DEPS=(
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     check_all_dpkg "$1"
 }
+
 install() {
     cat >/etc/apt/sources.list <<EOF
 deb http://deb.debian.org/debian/ buster main non-free contrib
@@ -83,5 +88,6 @@ EOF
         apt install -y "$pkg"
     done
 }
+
 deps=$2"[*]"
 "$1" "${!deps}"

environment/os/debian-9.sh

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
     libffi-dev libxml2-dev # for llvm
     libedit-dev libpcre3-dev automake bison # for swig
 )
+
 TOOLCHAIN_RUN_DEPS=(
     make # generic build tools
     tar gzip bzip2 xz-utils # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
     libreadline7 # for cmake and llvm
     libffi6 libxml2 # for llvm
 )
+
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkg-config # build system
@@ -43,14 +45,18 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
     autoconf # for jemalloc code generation
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     check_all_dpkg "$1"
 }
+
 install() {
     install_all_apt "$1"
 }
+
 deps=$2"[*]"
 "$1" "${!deps}"

environment/os/template.sh

@@ -8,23 +8,29 @@ source "$DIR/../util.sh"
 TOOLCHAIN_BUILD_DEPS=(
     pkg
 )
+
 TOOLCHAIN_RUN_DEPS=(
     pkg
 )
+
 MEMGRAPH_BUILD_DEPS=(
     pkg
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     echo "TODO: Implement ${FUNCNAME[0]}."
     exit 1
 }
+
 install() {
     echo "TODO: Implement ${FUNCNAME[0]}."
     exit 1
 }
+
 # http://ahmed.amayem.com/bash-indirect-expansion-exploration
 deps=$2"[*]"
 "$1" "${!deps}"

environment/os/ubuntu-18.04.sh

@@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
     libffi-dev libxml2-dev # llvm
     libedit-dev libpcre3-dev automake bison # swig
 )
+
 TOOLCHAIN_RUN_DEPS=(
     make # generic build tools
     tar gzip bzip2 xz-utils # used for archive unpacking
@@ -27,6 +28,7 @@ TOOLCHAIN_RUN_DEPS=(
     libreadline7 # for cmake and llvm
     libffi6 libxml2 # for llvm
 )
+
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkg-config # build system
@@ -44,14 +46,18 @@ MEMGRAPH_BUILD_DEPS=(
     mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
     autoconf # for jemalloc code generation
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     check_all_dpkg "$1"
 }
+
 install() {
     apt install -y $1
 }
+
 deps=$2"[*]"
 "$1" "${!deps}"

environment/os/ubuntu-20.04.sh

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
     libffi-dev libxml2-dev # for llvm
     libedit-dev libpcre3-dev automake bison # for swig
 )
+
 TOOLCHAIN_RUN_DEPS=(
     make # generic build tools
     tar gzip bzip2 xz-utils # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
     libreadline8 # for cmake and llvm
     libffi7 libxml2 # for llvm
 )
+
 MEMGRAPH_BUILD_DEPS=(
     git # source code control
     make pkg-config # build system
@@ -45,12 +47,15 @@ MEMGRAPH_BUILD_DEPS=(
     dotnet-sdk-3.1 golang nodejs npm
     autoconf # for jemalloc code generation
 )
+
 list() {
     echo "$1"
 }
+
 check() {
     check_all_dpkg "$1"
 }
+
 install() {
     cd "$DIR"
     apt update
@@ -75,5 +80,6 @@ install() {
         apt install -y "$pkg"
     done
 }
+
 deps=$2"[*]"
 "$1" "${!deps}"

environment/toolchain/v3.sh

@@ -0,0 +1,546 @@
+#!/bin/bash -e
+
+# helpers
+pushd () { command pushd "$@" > /dev/null; }
+popd () { command popd "$@" > /dev/null; }
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+CPUS=$( grep -c processor < /proc/cpuinfo )
+cd "$DIR"
+
+source "$DIR/../util.sh"
+DISTRO="$(operating_system)"
+
+# toolchain version
+TOOLCHAIN_VERSION=3
+
+# package versions used
+GCC_VERSION=10.2.0
+BINUTILS_VERSION=2.35.1
+case "$DISTRO" in
+    centos-7) # because GDB >= 9 does NOT compile with readline6.
+        GDB_VERSION=8.3
+    ;;
+    *)
+        GDB_VERSION=10.1
+    ;;
+esac
+CMAKE_VERSION=3.18.4
+CPPCHECK_VERSION=2.2
+LLVM_VERSION=11.0.0
+SWIG_VERSION=4.0.2 # used only for LLVM compilation
+
+# Check for the dependencies.
+echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
+$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
+echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
+$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
+
+# check installation directory
+NAME=toolchain-v$TOOLCHAIN_VERSION
+PREFIX=/opt/$NAME
+mkdir -p $PREFIX >/dev/null 2>/dev/null || true
+if [ ! -d $PREFIX ] || [ ! -w $PREFIX ]; then
+    echo "Please make sure that the directory '$PREFIX' exists and is writable by the current user!"
+    echo
+    echo "If unsure, execute these commands as root:"
+    echo "    mkdir $PREFIX && chown $USER:$USER $PREFIX"
+    echo
+    echo "Press <return> when you have created the directory and granted permissions."
+    # wait for the directory to be created
+    while true; do
+        read
+        if [ ! -d $PREFIX ] || [ ! -w $PREFIX ]; then
+            echo
+            echo "You can't continue before you have created the directory and granted permissions!"
+            echo
+            echo "Press <return> when you have created the directory and granted permissions."
+        else
+            break
+        fi
+    done
+fi
+
+# create archives directory
+mkdir -p archives
+
+# download all archives
+pushd archives
+if [ ! -f gcc-$GCC_VERSION.tar.gz ]; then
+    wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz
+fi
+if [ ! -f binutils-$BINUTILS_VERSION.tar.gz ]; then
+    wget https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.gz
+fi
+if [ ! -f gdb-$GDB_VERSION.tar.gz ]; then
+    wget https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.gz
+fi
+if [ ! -f cmake-$CMAKE_VERSION.tar.gz ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz
+fi
+if [ ! -f swig-$SWIG_VERSION.tar.gz ]; then
+    wget https://github.com/swig/swig/archive/rel-$SWIG_VERSION.tar.gz -O swig-$SWIG_VERSION.tar.gz
+fi
+if [ ! -f cppcheck-$CPPCHECK_VERSION.tar.gz ]; then
+    wget https://github.com/danmar/cppcheck/archive/$CPPCHECK_VERSION.tar.gz -O cppcheck-$CPPCHECK_VERSION.tar.gz
+fi
+if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
+fi
+if [ ! -f pahole-gdb-master.zip ]; then
+    wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
+fi
+
+# verify all archives
+# NOTE: Verification can fail if the archive is signed by another developer. I
+# haven't added commands to download all developer GnuPG keys because the
+# download is very slow. If the verification fails for you, figure out who has
+# signed the archive and download their public key instead.
+GPG="gpg --homedir .gnupg"
+KEYSERVER="hkp://keyserver.ubuntu.com"
+mkdir -p .gnupg
+chmod 700 .gnupg
+# verify gcc
+if [ ! -f gcc-$GCC_VERSION.tar.gz.sig ]; then
+    wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz.sig
+fi
+# list of valid gcc gnupg keys: https://gcc.gnu.org/mirrors.html
+$GPG --keyserver $KEYSERVER --recv-keys 0x3AB00996FC26A641
+$GPG --verify gcc-$GCC_VERSION.tar.gz.sig gcc-$GCC_VERSION.tar.gz
+# verify binutils
+if [ ! -f binutils-$BINUTILS_VERSION.tar.gz.sig ]; then
+    wget https://ftp.gnu.org/gnu/binutils/binutils-$BINUTILS_VERSION.tar.gz.sig
+fi
+$GPG --keyserver $KEYSERVER --recv-keys 0xDD9E3C4F
+$GPG --verify binutils-$BINUTILS_VERSION.tar.gz.sig binutils-$BINUTILS_VERSION.tar.gz
+# verify gdb
+if [ ! -f gdb-$GDB_VERSION.tar.gz.sig ]; then
+    wget https://ftp.gnu.org/gnu/gdb/gdb-$GDB_VERSION.tar.gz.sig
+fi
+$GPG --keyserver $KEYSERVER --recv-keys 0xFF325CF3
+$GPG --verify gdb-$GDB_VERSION.tar.gz.sig gdb-$GDB_VERSION.tar.gz
+# verify cmake
+if [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt ] || [ ! -f cmake-$CMAKE_VERSION-SHA-256.txt.asc ]; then
+    wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt
+    wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-SHA-256.txt.asc
+    # Because CentOS 7 doesn't have the `--ignore-missing` flag for `sha256sum`
+    # we filter out the missing files from the sums here manually.
+    cat cmake-$CMAKE_VERSION-SHA-256.txt | grep "cmake-$CMAKE_VERSION.tar.gz" > cmake-$CMAKE_VERSION-SHA-256-filtered.txt
+fi
+$GPG --keyserver $KEYSERVER --recv-keys 0xC6C265324BBEBDC350B513D02D2CEF1034921684
+sha256sum -c cmake-$CMAKE_VERSION-SHA-256-filtered.txt
+$GPG --verify cmake-$CMAKE_VERSION-SHA-256.txt.asc cmake-$CMAKE_VERSION-SHA-256.txt
+# verify llvm, cfe, lld, clang-tools-extra
+if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/lld-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
+    wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
+fi
+# list of valid llvm gnupg keys: https://releases.llvm.org/download.html
+$GPG --keyserver $KEYSERVER --recv-keys 0x345AD05D
+$GPG --verify llvm-$LLVM_VERSION.src.tar.xz.sig llvm-$LLVM_VERSION.src.tar.xz
+$GPG --verify clang-$LLVM_VERSION.src.tar.xz.sig clang-$LLVM_VERSION.src.tar.xz
+$GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
+$GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
+$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
+popd
+
+# create build directory
+mkdir -p build
+pushd build
+
+# compile gcc
+if [ ! -f $PREFIX/bin/gcc ]; then
+    if [ -d gcc-$GCC_VERSION ]; then
+        rm -rf gcc-$GCC_VERSION
+    fi
+    tar -xvf ../archives/gcc-$GCC_VERSION.tar.gz
+    pushd gcc-$GCC_VERSION
+    ./contrib/download_prerequisites
+    mkdir build && pushd build
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=gcc-8&arch=amd64&ver=8.3.0-6&stamp=1554588545
+    ../configure -v \
+        --build=x86_64-linux-gnu \
+        --host=x86_64-linux-gnu \
+        --target=x86_64-linux-gnu \
+        --prefix=$PREFIX \
+        --disable-multilib \
+        --with-system-zlib \
+        --enable-checking=release \
+        --enable-languages=c,c++,fortran \
+        --enable-gold=yes \
+        --enable-ld=yes \
+        --enable-lto \
+        --enable-bootstrap \
+        --disable-vtable-verify \
+        --disable-werror \
+        --without-included-gettext \
+        --enable-threads=posix \
+        --enable-nls \
+        --enable-clocale=gnu \
+        --enable-libstdcxx-debug \
+        --enable-libstdcxx-time=yes \
+        --enable-gnu-unique-object \
+        --enable-libmpx \
+        --enable-plugin \
+        --enable-default-pie \
+        --with-target-system-zlib \
+        --with-tune=generic \
+        --without-cuda-driver
+        #--program-suffix=$( printf "$GCC_VERSION" | cut -d '.' -f 1,2 ) \
+    make -j$CPUS
+    # make -k check # run test suite
+    make install
+    popd && popd
+fi
+
+# activate toolchain
+export PATH=$PREFIX/bin:$PATH
+export LD_LIBRARY_PATH=$PREFIX/lib64
+
+# compile binutils
+if [ ! -f $PREFIX/bin/ld.gold ]; then
+    if [ -d binutils-$BINUTILS_VERSION ]; then
+        rm -rf binutils-$BINUTILS_VERSION
+    fi
+    tar -xvf ../archives/binutils-$BINUTILS_VERSION.tar.gz
+    pushd binutils-$BINUTILS_VERSION
+    mkdir build && pushd build
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=binutils&arch=amd64&ver=2.32-7&stamp=1553247092
+    env \
+        CC=gcc \
+        CXX=g++ \
+        CFLAGS="-g -O2" \
+        CXXFLAGS="-g -O2" \
+        LDFLAGS="" \
+        ../configure \
+            --build=x86_64-linux-gnu \
+            --host=x86_64-linux-gnu \
+            --prefix=$PREFIX \
+            --enable-ld=default \
+            --enable-gold \
+            --enable-lto \
+            --enable-plugins \
+            --enable-shared \
+            --enable-threads \
+            --with-system-zlib \
+            --enable-deterministic-archives \
+            --disable-compressed-debug-sections \
+            --enable-new-dtags \
+            --disable-werror
+    make -j$CPUS
+    # make -k check # run test suite
+    make install
+    popd && popd
+fi
+
+# compile gdb
+if [ ! -f $PREFIX/bin/gdb ]; then
+    if [ -d gdb-$GDB_VERSION ]; then
+        rm -rf gdb-$GDB_VERSION
+    fi
+    tar -xvf ../archives/gdb-$GDB_VERSION.tar.gz
+    pushd gdb-$GDB_VERSION
+    mkdir build && pushd build
+    # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
+    env \
+        CC=gcc \
+        CXX=g++ \
+        CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+        CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+        CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
+        LDFLAGS="-Wl,-z,relro" \
+        PYTHON="" \
+        ../configure \
+            --build=x86_64-linux-gnu \
+            --host=x86_64-linux-gnu \
+            --prefix=$PREFIX \
+            --disable-maintainer-mode \
+            --disable-dependency-tracking \
+            --disable-silent-rules \
+            --disable-gdbtk \
+            --disable-shared \
+            --without-guile \
+            --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
+            --with-system-readline \
+            --with-expat \
+            --with-system-zlib \
+            --with-lzma \
+            --with-babeltrace \
+            --with-intel-pt \
+            --enable-tui \
+            --with-python=python3
+    make -j$CPUS
+    make install
+    popd && popd
+fi
+
+# install pahole
+if [ ! -d $PREFIX/share/pahole-gdb ]; then
+    unzip ../archives/pahole-gdb-master.zip
+    mv pahole-gdb-master $PREFIX/share/pahole-gdb
+fi
+
+# setup system gdbinit
+if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
+    mkdir -p $PREFIX/etc/gdb
+    cat >$PREFIX/etc/gdb/gdbinit <<EOF
+# improve formatting
+set print pretty on
+set print object on
+set print static-members on
+set print vtbl on
+set print demangle on
+set demangle-style gnu-v3
+set print sevenbit-strings off
+
+# load libstdc++ pretty printers
+add-auto-load-scripts-directory $PREFIX/lib64
+add-auto-load-safe-path $PREFIX
+
+# load pahole
+python
+sys.path.insert(0, "$PREFIX/share/pahole-gdb")
+import offsets
+import pahole
+end
+EOF
+fi
+
+# compile cmake
+if [ ! -f $PREFIX/bin/cmake ]; then
+    if [ -d cmake-$CMAKE_VERSION ]; then
+        rm -rf cmake-$CMAKE_VERSION
+    fi
+    tar -xvf ../archives/cmake-$CMAKE_VERSION.tar.gz
+    pushd cmake-$CMAKE_VERSION
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=cmake&arch=amd64&ver=3.13.4-1&stamp=1549799837
+    echo 'set(CMAKE_SKIP_RPATH ON CACHE BOOL "Skip rpath" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_USE_RELATIVE_PATHS ON CACHE BOOL "Use relative paths" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_C_FLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" CACHE STRING "C flags" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_CXX_FLAGS "-g -O2 -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2" CACHE STRING "C++ flags" FORCE)' >> build-flags.cmake
+    echo 'set(CMAKE_SKIP_BOOTSTRAP_TEST ON CACHE BOOL "Skip BootstrapTest" FORCE)' >> build-flags.cmake
+    echo 'set(BUILD_CursesDialog ON CACHE BOOL "Build curses GUI" FORCE)' >> build-flags.cmake
+    mkdir build && pushd build
+    ../bootstrap \
+        --prefix=$PREFIX \
+        --init=../build-flags.cmake \
+        --parallel=$CPUS \
+        --system-curl
+    make -j$CPUS
+    # make test # run test suite
+    make install
+    popd && popd
+fi
+
+# compile cppcheck
+if [ ! -f $PREFIX/bin/cppcheck ]; then
+    if [ -d cppcheck-$CPPCHECK_VERSION ]; then
+        rm -rf cppcheck-$CPPCHECK_VERSION
+    fi
+    tar -xvf ../archives/cppcheck-$CPPCHECK_VERSION.tar.gz
+    pushd cppcheck-$CPPCHECK_VERSION
+    env \
+        CC=gcc \
+        CXX=g++ \
+        PREFIX=$PREFIX \
+        FILESDIR=$PREFIX/share/cppcheck \
+        CFGDIR=$PREFIX/share/cppcheck/cfg \
+            make -j$CPUS
+    env \
+        CC=gcc \
+        CXX=g++ \
+        PREFIX=$PREFIX \
+        FILESDIR=$PREFIX/share/cppcheck \
+        CFGDIR=$PREFIX/share/cppcheck/cfg \
+            make install
+    popd
+fi
+
+# compile swig
+if [ ! -d swig-$SWIG_VERSION/install ]; then
+    if [ -d swig-$SWIG_VERSION ]; then
+        rm -rf swig-$SWIG_VERSION
+    fi
+    tar -xvf ../archives/swig-$SWIG_VERSION.tar.gz
+    mv swig-rel-$SWIG_VERSION swig-$SWIG_VERSION
+    pushd swig-$SWIG_VERSION
+    ./autogen.sh
+    mkdir build && pushd build
+    ../configure --prefix=$DIR/build/swig-$SWIG_VERSION/install
+    make -j$CPUS
+    make install
+    popd && popd
+fi
+
+# compile llvm
+if [ ! -f $PREFIX/bin/clang ]; then
+    if [ -d llvm-$LLVM_VERSION ]; then
+        rm -rf llvm-$LLVM_VERSION
+    fi
+    tar -xvf ../archives/llvm-$LLVM_VERSION.src.tar.xz
+    mv llvm-$LLVM_VERSION.src llvm-$LLVM_VERSION
+    tar -xvf ../archives/clang-$LLVM_VERSION.src.tar.xz
+    mv clang-$LLVM_VERSION.src llvm-$LLVM_VERSION/tools/clang
+    tar -xvf ../archives/lld-$LLVM_VERSION.src.tar.xz
+    mv lld-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/lld
+    tar -xvf ../archives/clang-tools-extra-$LLVM_VERSION.src.tar.xz
+    mv clang-tools-extra-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/tools/clang/tools/extra
+    tar -xvf ../archives/compiler-rt-$LLVM_VERSION.src.tar.xz
+    mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
+    pushd llvm-$LLVM_VERSION
+    mkdir build && pushd build
+    # activate swig
+    export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
+    # influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
+    cmake .. \
+        -DCMAKE_C_COMPILER=$PREFIX/bin/gcc \
+        -DCMAKE_CXX_COMPILER=$PREFIX/bin/g++ \
+        -DCMAKE_CXX_LINK_FLAGS="-L$PREFIX/lib64 -Wl,-rpath,$PREFIX/lib64" \
+        -DCMAKE_INSTALL_PREFIX=$PREFIX \
+        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+        -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O2 -DNDEBUG" \
+        -DCMAKE_CXX_FLAGS=' -fuse-ld=gold -fPIC -Wno-unused-command-line-argument -Wno-unknown-warning-option' \
+        -DCMAKE_C_FLAGS=' -fuse-ld=gold -fPIC -Wno-unused-command-line-argument -Wno-unknown-warning-option' \
+        -DLLVM_LINK_LLVM_DYLIB=ON \
+        -DLLVM_INSTALL_UTILS=ON \
+        -DLLVM_VERSION_SUFFIX= \
+        -DLLVM_BUILD_LLVM_DYLIB=ON \
+        -DLLVM_ENABLE_RTTI=ON \
+        -DLLVM_ENABLE_FFI=ON \
+        -DLLVM_BINUTILS_INCDIR=$PREFIX/include/ \
+        -DLLVM_USE_PERF=yes
+    make -j$CPUS
+    make -j$CPUS check-clang # run clang test suite
+    make -j$CPUS check-lld # run lld test suite
+    make install
+    popd && popd
+fi
+
+# create README
+if [ ! -f $PREFIX/README.md ]; then
+    cat >$PREFIX/README.md <<EOF
+# Memgraph Toolchain v$TOOLCHAIN_VERSION
+
+## Included tools
+
+ - GCC $GCC_VERSION
+ - Binutils $BINUTILS_VERSION
+ - GDB $GDB_VERSION
+ - CMake $CMAKE_VERSION
+ - Cppcheck $CPPCHECK_VERSION
+ - LLVM (Clang, LLD, compiler-rt, Clang tools extra) $LLVM_VERSION
+
+## Required libraries
+
+In order to be able to run all of these tools you should install the following
+packages:
+
+\`\`\`
+$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
+\`\`\`
+
+## Usage
+
+In order to use the toolchain you just have to source the activation script:
+
+\`\`\`
+source $PREFIX/activate
+\`\`\`
+EOF
+fi
+
+# create activation script
+if [ ! -f $PREFIX/activate ]; then
+    cat >$PREFIX/activate <<EOF
+# This file must be used with "source $PREFIX/activate" *from bash*
+# You can't run it directly!
+
+env_error="You already have an active virtual environment!"
+
+# zsh does not recognize the option -t of the command type
+# therefore we use the alternative whence -w
+if [[ "\$ZSH_NAME" == "zsh" ]]; then
+    # check for active virtual environments
+    if [ "\$( whence -w deactivate )" != "deactivate: none" ]; then
+        echo \$env_error
+        return 0;
+    fi
+# any other shell
+else
+    # check for active virtual environments
+    if [ "\$( type -t deactivate )" != "" ]; then
+        echo \$env_error
+        return 0
+    fi
+fi
+
+# check that we aren't root
+if [[ "\$USER" == "root" ]]; then
+    echo "You shouldn't use the toolchain as root!"
+    return 0
+fi
+
+# save original environment
+export ORIG_PATH=\$PATH
+export ORIG_PS1=\$PS1
+export ORIG_LD_LIBRARY_PATH=\$LD_LIBRARY_PATH
+
+# activate new environment
+export PATH=$PREFIX/bin:\$PATH
+export PS1="($NAME) \$PS1"
+export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64
+
+# disable root
+function su () {
+    echo "You don't want to use root functions while using the toolchain!"
+    return 1
+}
+function sudo () {
+    echo "You don't want to use root functions while using the toolchain!"
+    return 1
+}
+
+# create deactivation function
+function deactivate() {
+    export PATH=\$ORIG_PATH
+    export PS1=\$ORIG_PS1
+    export LD_LIBRARY_PATH=\$ORIG_LD_LIBRARY_PATH
+    unset ORIG_PATH ORIG_PS1 ORIG_LD_LIBRARY_PATH
+    unset -f su sudo deactivate
+}
+EOF
+fi
+
+# create toolchain archive
+if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
+    tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO.tar.gz -C /opt $NAME
+fi
+
+# output final instructions
+echo -e "\n\n"
+echo "All tools have been built. They are installed in '$PREFIX'."
+echo "In order to distribute the tools to someone else, an archive with the toolchain was created in the 'build' directory."
+echo "If you want to install the packed tools you should execute the following command:"
+echo
+echo "    tar -xvzf build/$NAME-binaries.tar.gz -C /opt"
+echo
+echo "Because the tools were built on this machine, you should probably change the permissions of the installation directory using:"
+echo
+echo "    OPTIONAL: chown -R root:root $PREFIX"
+echo
+echo "In order to use all of the newly compiled tools you should use the prepared activation script:"
+echo
+echo "    source $PREFIX/activate"
+echo
+echo "Or, for more advanced uses, you can add the following lines to your script:"
+echo
+echo "    export PATH=$PREFIX/bin:\$PATH"
+echo "    export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64"
+echo
+echo "Enjoy!"

include/mgp.py

@@ -683,7 +683,15 @@ def _typing_to_cypher_type(type_):
                         return _mgp.type_nullable(simple_type)
                     return _mgp.type_nullable(parse_typing(type_arg_as_str))
             elif type_as_str.startswith('typing.List'):
-                type_arg_as_str, = parse_type_args(type_as_str)
+                type_arg_as_str = parse_type_args(type_as_str)
+
+                if len(type_arg_as_str) > 1:
+                    # Nested object could be a type consisting of a list of types (e.g. mgp.Map)
+                    # so we need to join the parts.
+                    type_arg_as_str = ', '.join(type_arg_as_str)
+                else:
+                    type_arg_as_str = type_arg_as_str[0]
+
                 simple_type = get_simple_type(type_arg_as_str)
                 if simple_type is not None:
                     return _mgp.type_list(simple_type)

libs/CMakeLists.txt

@@ -209,6 +209,14 @@ import_external_library(mgclient STATIC
 find_package(OpenSSL REQUIRED)
 target_link_libraries(mgclient INTERFACE ${OPENSSL_LIBRARIES})
 
+add_external_project(mgconsole
+  SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/mgconsole
+  CMAKE_ARGS
+    -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}
+  BUILD_COMMAND $(MAKE) mgconsole)
+
+add_custom_target(mgconsole DEPENDS mgconsole-proj)
+
 # Setup spdlog
 import_external_library(spdlog STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/spdlog/${CMAKE_INSTALL_LIBDIR}/libspdlog.a

libs/setup.sh

@@ -2,8 +2,9 @@
 
 # Download external dependencies.
 
+local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
 working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd ${working_dir}
+cd "${working_dir}"
 
 # Clones a git repository and optionally cherry picks additional commits. The
 # function will try to preserve any local changes in the repo.
@@ -15,7 +16,11 @@ clone () {
   shift 3
   # Clone if there's no repo.
   if [[ ! -d "$dir_name" ]]; then
-    git clone "$git_repo" "$dir_name"
+    echo "Cloning from $git_repo"
+    # If the clone fails, it doesn't make sense to continue with the function
+    # execution but the whole script should continue executing because we might
+    # clone the same repo from a different source.
+    git clone "$git_repo" "$dir_name" || return 1
   fi
   pushd "$dir_name"
   # Just fetch new commits from remote repository. Don't merge/pull them in, so
@@ -29,12 +34,17 @@ clone () {
   # Stash regardless of local_changes, so that a user gets a message on stdout.
   git stash
   # Checkout the primary commit (there's no need to pull/merge).
-  git checkout $checkout_id
+  # The checkout fail should exit this script immediately because the target
+  # commit is not there and that will most likely create build-time errors.
+  git checkout "$checkout_id" || exit 1
   # Apply any optional cherry pick fixes.
   while [[ $# -ne 0 ]]; do
     local cherry_pick_id=$1
     shift
-    git cherry-pick -n $cherry_pick_id
+    # The cherry-pick fail should exit this script immediately because the
+    # target commit is not there and that will most likely create build-time
+    # errors.
+    git cherry-pick -n "$cherry_pick_id" || exit 1
   done
   # Reapply any local changes.
   if [[ $local_changes == true ]]; then
@@ -43,12 +53,97 @@ clone () {
   popd
 }
 
+file_get_try_double () {
+    primary_url="$1"
+    secondary_url="$2"
+    echo "Download primary from $primary_url secondary from $secondary_url"
+    if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
+    if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
+    filename="$(basename "$secondary_url")"
+    wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
+    echo ""
+}
+
+repo_clone_try_double () {
+    primary_url="$1"
+    secondary_url="$2"
+    folder_name="$3"
+    ref="$4"
+    echo "Cloning primary from $primary_url secondary from $secondary_url"
+    if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
+    if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
+    if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
+    if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
+    clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
+    echo ""
+}
+
+# List all dependencies.
+
+# The reason for introducing primary and secondary urls are:
+#   * HTTPS is hard to cache
+#   * Remote development workflow is more flexible if people don't have to connect to VPN
+#   * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
+#   * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
+
+# The goal of having primary urls is to have links to the "local" cache of
+# dependencies where these dependencies could be downloaded as fast as
+# possible. The actual cache server could be on your local machine, on a
+# dedicated machine inside the build cluster or on the actual build machine.
+# Download from primary_urls might fail because the cache is not installed.
+declare -A primary_urls=(
+  ["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
+  ["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
+  ["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
+  ["fmt"]="http://$local_cache_host/git/fmt.git"
+  ["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
+  ["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
+  ["gtest"]="http://$local_cache_host/git/googletest.git"
+  ["gflags"]="http://$local_cache_host/git/gflags.git"
+  ["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
+  ["bzip2"]="http://$local_cache_host/git/bzip2.git"
+  ["zlib"]="http://$local_cache_host/git/zlib.git"
+  ["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
+  ["mgclient"]="http://$local_cache_host/git/mgclient.git"
+  ["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
+  ["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
+  ["spdlog"]="http://$local_cache_host/git/spdlog"
+  ["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
+  ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
+  ["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
+)
+
+# The goal of secondary urls is to have links to the "source of truth" of
+# dependencies, e.g., Github or S3. Download from secondary urls, if happens
+# at all, should never fail. In other words, if it fails, the whole build
+# should fail.
+declare -A secondary_urls=(
+  ["antlr4-code"]="https://github.com/antlr/antlr4.git"
+  ["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
+  ["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
+  ["fmt"]="https://github.com/fmtlib/fmt.git"
+  ["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
+  ["gbenchmark"]="https://github.com/google/benchmark.git"
+  ["gtest"]="https://github.com/google/googletest.git"
+  ["gflags"]="https://github.com/memgraph/gflags.git"
+  ["libbcrypt"]="https://github.com/rg3/libbcrypt"
+  ["bzip2"]="https://github.com/VFR-maniac/bzip2"
+  ["zlib"]="https://github.com/madler/zlib.git"
+  ["rocksdb"]="https://github.com/facebook/rocksdb.git"
+  ["mgclient"]="https://github.com/memgraph/mgclient.git"
+  ["pymgclient"]="https://github.com/memgraph/pymgclient.git"
+  ["mgconsole"]="http://github.com/memgraph/mgconsole.git"
+  ["spdlog"]="https://github.com/gabime/spdlog"
+  ["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
+  ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
+  ["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
+)
+
 # antlr
-antlr_generator_filename="antlr-4.6-complete.jar"
-# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
-wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
+file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
+
 antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
-clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
+repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
 # fix missing include
 sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
 # remove shared library from install dependencies
@@ -56,74 +151,77 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
 
 # cppitertools v2.0 2019-12-23
 cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
-clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
+repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
 
 # fmt
-fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25) 
-clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
+fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
+repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
 
 # rapidcheck
 rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
-clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
+repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
 
 # google benchmark
 benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
-clone https://github.com/google/benchmark.git benchmark $benchmark_tag
+repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
 
 # google test
 googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
-clone https://github.com/google/googletest.git googletest $googletest_tag
+repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
 
 # google flags
 gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
-clone https://github.com/memgraph/gflags.git gflags $gflags_tag
+repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
 
 # libbcrypt
 libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
-clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
+repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
 
 # neo4j
-wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
-tar -xzf neo4j.tar.gz
-rm -rf neo4j
+file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
+tar -xzf neo4j-community-3.2.3-unix.tar.gz
 mv neo4j-community-3.2.3 neo4j
-rm neo4j.tar.gz
+rm neo4j-community-3.2.3-unix.tar.gz
 
 # nlohmann json
 # We wget header instead of cloning repo since repo is huge (lots of test data).
 # We use head on Sep 1, 2017 instead of last release since it was long time ago.
 mkdir -p json
 cd json
-wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
+file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
 cd ..
 
 bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580"  # v1.0.6 (May 26, 2011)
-clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
+repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
 
 zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
-clone https://github.com/madler/zlib.git zlib $zlib_tag
+repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
 # remove shared library from install dependencies
 sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
 
 rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
-clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
+repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
 # remove shared library from install dependencies
 sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
 
 # mgclient
 mgclient_tag="v1.2.0" # (2021-01-14)
-clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
+repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
 sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
 
 # pymgclient
 pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
-clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
+repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
+
+# mgconsole
+mgconsole_tag="5377df371375a73f5d7eb1d3a839dfcfd356438d" # (2021-05-25)
+repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag"
 
 spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
-clone https://github.com/gabime/spdlog spdlog $spdlog_tag
+repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
 
 jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
-clone https://github.com/jemalloc/jemalloc.git jemalloc $jemalloc_tag
+repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
 pushd jemalloc
 # ThreadPool select job randomly, and there can be some threads that had been
 # performed some memory heavy task before and will be inactive for some time,
@@ -138,5 +236,5 @@ pushd jemalloc
 # avoid spurious latencies and additional work associated with
 # MADV_DONTNEED. See
 # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
-./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000"
+./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
 popd

release/LICENSE_COMMUNITY.md

@@ -1,33 +1,40 @@
-# User License Agreement
+# Memgraph Community User License Agreement
 
-1. Description
+This License Agreement governs your use of the Memgraph Community Release (the
+"Software") and documentation ("Documentation").
 
-THIS LICENSE AGREEMENT GOVERNS LICENSEE’S USE OF THE MEMGRAPH COMMUNITY
-RELEASE AND DOCUMENTATION.
+BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU ("LICENSEE") AGREE TO THESE
+TERMS.
 
-2. License Grant
+1. License Grant
 
 The Software and Documentation are provided to Licensee at no charge and are
 licensed, not sold to Licensee. No ownership of any part of the Software and
 Documentation is hereby transferred to Licensee. Subject to (i) the terms and
-conditions of this License Agreement, (ii) any additional license restrictions
-and parameters contained on Licensor’s quotation, website, or order form
-(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
+conditions of this License Agreement, and (ii) any additional license
+restrictions and parameters contained on Licensor’s quotation, website, or
+order form, Licensor hereby grants Licensee a personal, non-assignable,
 non-transferable and non-exclusive license to install, access and use the
 Software (in object code form only) and Documentation for Licensee’s internal
-business purposes only. All rights relating to the Software and Documentation
-that are not expressly licensed in this License Agreement, whether now existing
-or which may hereafter come into existence are reserved for Licensor. Licensee
-shall not remove, obscure, or alter any proprietary rights notices (including
-without limitation copyright and trademark notices), which may be affixed to or
-contained within the Software or Documentation.
+business purposes (including for use in a production environment) only. All
+rights relating to the Software and Documentation that are not expressly
+licensed in this License Agreement, whether now existing or which may hereafter
+come into existence are reserved for Licensor. Licensee shall not remove,
+obscure, or alter any proprietary rights notices (including without limitation
+copyright and trademark notices), which may be affixed to or contained within
+the Software or Documentation.
 
-3. Restrictions
+Licensor may terminate this License Agreement with immediate effect upon
+written notice to the Licensee. Upon termination Licensee shall delete all
+electronic copies of all or any part of the Software and/or the Documentation
+resident in its systems or elsewhere.
+
+2. Restrictions
 
 Licensee will not, directly or indirectly, (a) copy the Software or
 Documentation in any manner or for any purpose; (b) install, access or use any
 component of the Software or Documentation for any purpose not expressly
-granted in Section 2 above; (c) resell, distribute, publicly display or
+granted in Section 1 above; (c) resell, distribute, publicly display or
 publicly perform the Software or Documentation or any component thereof, by
 transfer, lease, loan or any other means, or make it available for use by
 others in any time-sharing, service bureau or similar arrangement; (d)
@@ -37,25 +44,55 @@ algorithms or techniques incorporated in the Software; (e) export the Software
 or Documentation in violation of any applicable laws or regulations; (f)
 modify, translate, adapt, or create derivative works from the Software or
 Documentation; (g) circumvent, disable or otherwise interfere with
-security-related features of the Software or Documentation; (h)
-reverse-engineer, disassemble, attempt to derive the source code; (i) use the
+security-related features of the Software or Documentation; (h) use the
 Software or Documentation for any illegal purpose, in any manner that is
 inconsistent with the terms of this License Agreement, or to engage in illegal
-activity; (j) remove or alter any trademark, logo, copyright or other
+activity; (i) remove or alter any trademark, logo, copyright or other
 proprietary notices, legends, symbols or labels on, or embedded in, the
-Software or Documentation; or (k) provide access to the Software or
+Software or Documentation; or (j) provide access to the Software or
 Documentation to third parties.
 
-4. Warranty Disclaimer
+3. Warranty Disclaimer
 
-THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED “AS IS” FOR
-DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
-PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
-IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
-THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
-NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
-INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
-WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
+THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" AND LICENSOR MAKES NO
+WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
+INFRINGEMENT OF THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER
+PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
+FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
+OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
 
-BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.
+4. Limitation of Liability
+
+Licensor shall not in any circumstances be liable, whether in tort (including
+for negligence or breach of statutory duty howsoever arising), contract,
+misrepresentation (whether innocent or negligent) or otherwise for: loss of
+profits, loss of business, depletion of goodwill or similar losses, loss of
+anticipated savings, loss of goods, loss or corruption of data or computer
+downtime, or any special, indirect, consequential or pure economic loss, costs,
+damages, charges or expenses.
+
+Licensor's total aggregate liability in contract, tort (including without
+limitation negligence or breach of statutory duty howsoever arising),
+misrepresentation (whether innocent or negligent), restitution or otherwise,
+arising in connection with the performance or contemplated performance of this
+License Agreement shall in all circumstances be limited to GBP10.00 (ten pounds
+sterling).
+
+Nothing in this License Agreement shall limit Licensor’s liability in the case
+of death or personal injury caused by negligence, fraud, or fraudulent
+misrepresentation, or where it otherwise cannot be limited by law.
+
+5. Technical Data
+
+Licensor may collect and use technical information (such as usage patterns)
+gathered when the Licensee downloads and uses the Software.  This is generally
+statistical data which does not identify an identified or identifiable
+individual.  It may also include Licensee’s IP address which is personal data
+and is processed in accordance with our Privacy Policy. We only use this
+technical information to improve our products.
+
+6. Law and Jurisdiction
+
+This License Agreement is governed by the laws of England and is subject to the
+non-exclusive jurisdiction of the courts of England.

release/arch-pkg/package_arch

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 function print_help () {
-    echo "Usage: $0 MEMGPRAH_PACKAGE.tar.gz"
+    echo "Usage: $0 MEMGRAPH_PACKAGE.tar.gz"
     echo "Optional arguments:"
     echo -e "  -h|--help           Print help."
 }

release/docker/memgraph_community.dockerfile

@@ -1,4 +1,5 @@
 FROM debian:buster
+# NOTE: If you change the base distro update release/package as well.
 
 ARG deb_release
 

release/docker/memgraph_enterprise.dockerfile

@@ -1,4 +1,5 @@
 FROM debian:buster
+# NOTE: If you change the base distro update release/package as well.
 
 ARG deb_release
 

release/get_version.py

@@ -192,7 +192,19 @@ if args.version:
 try:
     current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
     if current_branch != "master":
-        get_output("git", "fetch", "origin", "master:master")
+        branches = get_output("git", "branch")
+        if "master" in branches:
+            # If master is present locally, the fetch is allowed to fail
+            # because this script will still be able to compare against the
+            # master branch.
+            try:
+                get_output("git", "fetch", "origin", "master:master")
+            except Exception:
+                pass
+        else:
+            # If master is not present locally, the fetch command has to
+            # succeed because something else will fail otherwise.
+            get_output("git", "fetch", "origin", "master:master")
 except Exception:
     print("Fatal error while ensuring local master branch.")
     sys.exit(1)

release/package/centos-7/Dockerfile

@@ -0,0 +1,12 @@
+FROM centos:7
+
+RUN yum -y update \
+    && yum install -y wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-7.tar.gz \
+         -O toolchain-v2-binaries-centos-7.tar.gz \
+    && tar xzvf toolchain-v2-binaries-centos-7.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

release/package/centos-8/Dockerfile

@@ -0,0 +1,12 @@
+FROM centos:8
+
+RUN dnf -y update \
+    && dnf install -y wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-8.tar.gz \
+         -O toolchain-v2-binaries-centos-8.tar.gz \
+    && tar xzvf toolchain-v2-binaries-centos-8.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

release/package/debian-10/Dockerfile

@@ -0,0 +1,15 @@
+FROM debian:10
+
+# Stops tzdata interactive configuration.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && apt install -y \
+    ca-certificates wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-10.tar.gz \
+         -O toolchain-v2-binaries-debian-10.tar.gz \
+    && tar xzvf toolchain-v2-binaries-debian-10.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

release/package/debian-9/Dockerfile

@@ -0,0 +1,15 @@
+FROM debian:9
+
+# Stops tzdata interactive configuration.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && apt install -y \
+    ca-certificates wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-9.tar.gz \
+         -O toolchain-v2-binaries-debian-9.tar.gz \
+    && tar xzvf toolchain-v2-binaries-debian-9.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

release/package/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3"
+services:
+  mgbuild_centos-7:
+    build:
+      context: centos-7
+    container_name: "mgbuild_centos-7"
+  mgbuild_centos-8:
+    build:
+      context: centos-8
+    container_name: "mgbuild_centos-8"
+  mgbuild_debian-9:
+    build:
+      context: debian-9
+    container_name: "mgbuild_debian-9"
+  mgbuild_debian-10:
+    build:
+      context: debian-10
+    container_name: "mgbuild_debian-10"
+  mgbuild_ubuntu-18.04:
+    build:
+      context: ubuntu-18.04
+    container_name: "mgbuild_ubuntu-18.04"
+  mgbuild_ubuntu-20.04:
+    build:
+      context: ubuntu-20.04
+    container_name: "mgbuild_ubuntu-20.04"

release/package/run.sh

@@ -0,0 +1,153 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SUPPORTED_OFFERING=(community enterprise)
+SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04)
+PROJECT_ROOT="$SCRIPT_DIR/../.."
+ACTIVATE_TOOLCHAIN="source /opt/toolchain-v2/activate"
+HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
+
+print_help () {
+    echo "$0 init|package {offering} {os} [--for-docker]|docker|test"
+    echo ""
+    echo "    offerings: ${SUPPORTED_OFFERING[*]}"
+    echo "    OSs: ${SUPPORTED_OS[*]}"
+    exit 1
+}
+
+make_package () {
+    offering="$1"
+    offering_flag=" -DMG_ENTERPRISE=OFF "
+    if [[ "$offering" == "enterprise" ]]; then
+        offering_flag=" -DMG_ENTERPRISE=ON "
+    fi
+    if [[ "$offering" == "community" ]]; then
+        offering_flag=" -DMG_ENTERPRISE=OFF "
+    fi
+    os="$2"
+    package_command=""
+    if [[ "$os" =~ ^"centos".* ]]; then
+        package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
+    fi
+    if [[ "$os" =~ ^"debian".* ]]; then
+        package_command=" cpack -G DEB --config ../CPackConfig.cmake "
+    fi
+    if [[ "$os" =~ ^"ubuntu".* ]]; then
+        package_command=" cpack -G DEB --config ../CPackConfig.cmake "
+    fi
+    docker_flag=" -DBUILD_FOR_DOCKER=OFF "
+    if [[ "$#" -gt 2 ]]; then
+        if [[ "$3" == "--for-docker" ]]; then
+            docker_flag=" -DBUILD_FOR_DOCKER=ON "
+        fi
+    fi
+    build_container="mgbuild_$os"
+    echo "Building Memgraph $offering for $os on $build_container..."
+
+    echo "Copying project files..."
+    # If master is not the current branch, fetch it, because the get_version
+    # script depends on it. If we are on master, the fetch command is going to
+    # fail so that's why there is the explicit check.
+    # Required here because Docker build container can't access remote.
+    cd "$PROJECT_ROOT"
+    if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
+        git fetch origin master:master
+    fi
+    docker exec "$build_container" mkdir -p /memgraph
+    docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
+
+    container_build_dir="/memgraph/build"
+    container_output_dir="$container_build_dir/output"
+
+    # TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
+    # image build phase, but that is not easy at this point because the
+    # environment/os/{os}.sh does not come within the toolchain package. When
+    # migrating to the next version of toolchain do that, and remove the
+    # TOOLCHAIN_RUN_DEPS installation from here.
+    echo "Installing dependencies..."
+    docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
+    docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
+
+    echo "Building targeted package..."
+    docker exec "$build_container" bash -c "cd /memgraph && ./init"
+    docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
+    docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $offering_flag $docker_flag .."
+    # ' is used instead of " because we need to run make within the allowed
+    # container resources.
+    # shellcheck disable=SC2016
+    docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc)'
+    docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc) -B mgconsole'
+    docker exec "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
+
+    echo "Copying targeted package to host..."
+    last_package_name=$(docker exec "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
+    # The operating system folder is introduced because multiple different
+    # packages could be preserved during the same build "session".
+    mkdir -p "$HOST_OUTPUT_DIR/$os"
+    package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
+    docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
+    echo "Package saved to $package_host_destination."
+}
+
+case "$1" in
+    init)
+        cd "$SCRIPT_DIR"
+        docker-compose build
+        docker-compose up -d
+    ;;
+
+    docker)
+        # NOTE: Docker is build on top of Debian 10 package.
+        based_on_os="debian-10"
+        # shellcheck disable=SC2012
+        last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
+        docker_build_folder="$PROJECT_ROOT/release/docker"
+        cd "$docker_build_folder"
+        ./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
+        # shellcheck disable=SC2012
+        docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
+        docker_host_folder="$HOST_OUTPUT_DIR/docker"
+        docker_host_image_path="$docker_host_folder/$docker_image_name"
+        mkdir -p "$docker_host_folder"
+        cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
+        echo "Docker images saved to $docker_host_image_path."
+    ;;
+
+    package)
+        shift 1
+        if [[ "$#" -lt 2 ]]; then
+            print_help
+        fi
+        offering="$1"
+        shift 1
+        is_offering_ok=false
+        for supported_offering in "${SUPPORTED_OFFERING[@]}"; do
+            if [[ "$supported_offering" == "${offering}" ]]; then
+                is_offering_ok=true
+            fi
+        done
+        os="$1"
+        shift 1
+        is_os_ok=false
+        for supported_os in "${SUPPORTED_OS[@]}"; do
+            if [[ "$supported_os" == "${os}" ]]; then
+                is_os_ok=true
+            fi
+        done
+        if [[ "$is_offering_ok" == true ]] && [[ "$is_os_ok" == true ]]; then
+            make_package "$offering" "$os" "$@"
+        else
+            print_help
+        fi
+    ;;
+
+    test)
+        echo "TODO(gitbuda): Test all packages on mgtest containers."
+    ;;
+
+    *)
+        print_help
+    ;;
+esac

release/package/ubuntu-18.04/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:18.04
+
+# Stops tzdata interactive configuration.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && apt install -y \
+    ca-certificates wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-18.04.tar.gz \
+         -O toolchain-v2-binaries-ubuntu-18.04.tar.gz \
+    && tar xzvf toolchain-v2-binaries-ubuntu-18.04.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

release/package/ubuntu-20.04/Dockerfile

@@ -0,0 +1,15 @@
+FROM ubuntu:20.04
+
+# Stops tzdata interactive configuration.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update && apt install -y \
+    ca-certificates wget git
+# Do NOT be smart here and clean the cache because the container is used in the
+# stateful context.
+
+RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-20.04.tar.gz \
+         -O toolchain-v2-binaries-ubuntu-20.04.tar.gz \
+    && tar xzvf toolchain-v2-binaries-ubuntu-20.04.tar.gz -C /opt
+
+ENTRYPOINT ["sleep", "infinity"]

src/auth/reference_modules/example.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 import json
 import io
 

src/auth/reference_modules/ldap.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/python3
 import json
 import io
 import ssl

src/query/frontend/ast/ast.lcp

@@ -2232,6 +2232,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
     AuthQuery::Privilege::AUTH,
     AuthQuery::Privilege::CONSTRAINT, AuthQuery::Privilege::DUMP,
     AuthQuery::Privilege::REPLICATION,
+    AuthQuery::Privilege::READ_FILE,
     AuthQuery::Privilege::LOCK_PATH,
     AuthQuery::Privilege::FREE_MEMORY};
 cpp<#

src/query/frontend/ast/ast_visitor.hpp

@@ -109,6 +109,6 @@ class ExpressionVisitor
 template <class TResult>
 class QueryVisitor
     : public ::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery, InfoQuery,
-                              ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery, LoadCsv, FreeMemoryQuery> {};
+                              ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery, FreeMemoryQuery> {};
 
 }  // namespace query

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -767,6 +767,10 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
   if (ctx->AUTH()) return AuthQuery::Privilege::AUTH;
   if (ctx->CONSTRAINT()) return AuthQuery::Privilege::CONSTRAINT;
   if (ctx->DUMP()) return AuthQuery::Privilege::DUMP;
+  if (ctx->REPLICATION()) return AuthQuery::Privilege::REPLICATION;
+  if (ctx->LOCK_PATH()) return AuthQuery::Privilege::LOCK_PATH;
+  if (ctx->READ_FILE()) return AuthQuery::Privilege::READ_FILE;
+  if (ctx->FREE_MEMORY()) return AuthQuery::Privilege::FREE_MEMORY;
   LOG_FATAL("Should not get here - unknown privilege!");
 }
 

src/query/frontend/opencypher/grammar/MemgraphCypher.g4

@@ -117,7 +117,7 @@ delimiter : literal ;
 
 quote : literal ;
 
-rowVar : variable ; 
+rowVar : variable ;
 
 userOrRoleName : symbolicName ;
 
@@ -146,8 +146,22 @@ denyPrivilege : DENY ( ALL PRIVILEGES | privileges=privilegeList ) TO userOrRole
 
 revokePrivilege : REVOKE ( ALL PRIVILEGES | privileges=privilegeList ) FROM userOrRole=userOrRoleName ;
 
-privilege : CREATE | DELETE | MATCH | MERGE | SET
-          | REMOVE | INDEX | STATS | AUTH | CONSTRAINT | DUMP ;
+privilege : CREATE
+          | DELETE
+          | MATCH
+          | MERGE
+          | SET
+          | REMOVE
+          | INDEX
+          | STATS
+          | AUTH
+          | CONSTRAINT
+          | DUMP
+          | REPLICATION
+          | LOCK_PATH
+          | READ_FILE
+          | FREE_MEMORY
+          ;
 
 privilegeList : privilege ( ',' privilege )* ;
 

src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4

@@ -10,6 +10,8 @@ lexer grammar MemgraphCypherLexer ;
 
 import CypherLexer ;
 
+UNDERSCORE : '_' ;
+
 ALTER          : A L T E R ;
 ASYNC          : A S Y N C ;
 AUTH           : A U T H ;
@@ -25,6 +27,7 @@ DROP           : D R O P ;
 DUMP           : D U M P ;
 FOR            : F O R ;
 FREE           : F R E E ;
+FREE_MEMORY     : F R E E UNDERSCORE M E M O R Y ;
 FROM           : F R O M ;
 GRANT          : G R A N T ;
 GRANTS         : G R A N T S ;
@@ -33,12 +36,14 @@ IDENTIFIED     : I D E N T I F I E D ;
 IGNORE         : I G N O R E ;
 LOAD           : L O A D ;
 LOCK           : L O C K ;
+LOCK_PATH      : L O C K UNDERSCORE P A T H ;
 MAIN           : M A I N ;
 MODE           : M O D E ;
 NO             : N O ;
 PASSWORD       : P A S S W O R D ;
 PORT           : P O R T ;
 PRIVILEGES     : P R I V I L E G E S ;
+READ_FILE      : R E A D UNDERSCORE F I L E ;
 REGISTER       : R E G I S T E R ;
 REPLICA        : R E P L I C A ;
 REPLICAS       : R E P L I C A S ;

src/query/frontend/semantic/required_privileges.cpp

@@ -51,74 +51,58 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
 
   void Visit(LockPathQuery &lock_path_query) override { AddPrivilege(AuthQuery::Privilege::LOCK_PATH); }
 
-  void Visit(LoadCsv &load_csv) override { AddPrivilege(AuthQuery::Privilege::READ_FILE); }
-
   void Visit(FreeMemoryQuery &free_memory_query) override { AddPrivilege(AuthQuery::Privilege::FREE_MEMORY); }
 
-  void Visit(ReplicationQuery &replication_query) override {
-    switch (replication_query.action_) {
-      case ReplicationQuery::Action::SET_REPLICATION_ROLE:
-        AddPrivilege(AuthQuery::Privilege::REPLICATION);
-        break;
-      case ReplicationQuery::Action::SHOW_REPLICATION_ROLE:
-        AddPrivilege(AuthQuery::Privilege::REPLICATION);
-        break;
-      case ReplicationQuery::Action::REGISTER_REPLICA:
-        AddPrivilege(AuthQuery::Privilege::REPLICATION);
-        break;
-      case ReplicationQuery::Action::DROP_REPLICA:
-        AddPrivilege(AuthQuery::Privilege::REPLICATION);
-        break;
-      case ReplicationQuery::Action::SHOW_REPLICAS:
-        AddPrivilege(AuthQuery::Privilege::REPLICATION);
-        break;
-    }
-  }
+  void Visit(ReplicationQuery &replication_query) override { AddPrivilege(AuthQuery::Privilege::REPLICATION); }
 
-  bool PreVisit(Create &) override {
+  bool PreVisit(Create & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::CREATE);
     return false;
   }
-  bool PreVisit(CallProcedure &) override {
+  bool PreVisit(CallProcedure & /*unused*/) override {
     // TODO: Corresponding privilege
     return false;
   }
-  bool PreVisit(Delete &) override {
+  bool PreVisit(Delete & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::DELETE);
     return false;
   }
-  bool PreVisit(Match &) override {
+  bool PreVisit(Match & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::MATCH);
     return false;
   }
-  bool PreVisit(Merge &) override {
+  bool PreVisit(Merge & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::MERGE);
     return false;
   }
-  bool PreVisit(SetProperty &) override {
+  bool PreVisit(SetProperty & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::SET);
     return false;
   }
-  bool PreVisit(SetProperties &) override {
+  bool PreVisit(SetProperties & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::SET);
     return false;
   }
-  bool PreVisit(SetLabels &) override {
+  bool PreVisit(SetLabels & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::SET);
     return false;
   }
-  bool PreVisit(RemoveProperty &) override {
+  bool PreVisit(RemoveProperty & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::REMOVE);
     return false;
   }
-  bool PreVisit(RemoveLabels &) override {
+  bool PreVisit(RemoveLabels & /*unused*/) override {
     AddPrivilege(AuthQuery::Privilege::REMOVE);
     return false;
   }
+  bool PreVisit(LoadCsv & /*unused*/) override {
+    AddPrivilege(AuthQuery::Privilege::READ_FILE);
+    return false;
+  }
 
-  bool Visit(Identifier &) override { return true; }
-  bool Visit(PrimitiveLiteral &) override { return true; }
-  bool Visit(ParameterLookup &) override { return true; }
+  bool Visit(Identifier & /*unused*/) override { return true; }
+  bool Visit(PrimitiveLiteral & /*unused*/) override { return true; }
+  bool Visit(ParameterLookup & /*unused*/) override { return true; }
 
  private:
   void AddPrivilege(AuthQuery::Privilege privilege) {

src/query/frontend/stripped_lexer_constants.hpp

@@ -79,15 +79,15 @@ class Trie {
 const int kBitsetSize = 65536;
 
 const trie::Trie kKeywords = {
-    "union",  "all",        "optional",  "match",   "unwind",     "as",       "merge",    "on",    "create",
-    "set",    "detach",     "delete",    "remove",  "with",       "distinct", "return",   "order", "by",
-    "skip",   "limit",      "ascending", "asc",     "descending", "desc",     "where",    "or",    "xor",
-    "and",    "not",        "in",        "starts",  "ends",       "contains", "is",       "null",  "case",
-    "when",   "then",       "else",      "end",     "count",      "filter",   "extract",  "any",   "none",
-    "single", "true",       "false",     "reduce",  "coalesce",   "user",     "password", "alter", "drop",
-    "show",   "stats",      "unique",    "explain", "profile",    "storage",  "index",    "info",  "exists",
-    "assert", "constraint", "node",      "key",     "dump",       "database", "call",     "yield", "memory",
-    "mb",     "kb",         "unlimited", "free",    "procedure",  "query"};
+    "union",  "all",        "optional",  "match",   "unwind",     "as",       "merge",       "on",        "create",
+    "set",    "detach",     "delete",    "remove",  "with",       "distinct", "return",      "order",     "by",
+    "skip",   "limit",      "ascending", "asc",     "descending", "desc",     "where",       "or",        "xor",
+    "and",    "not",        "in",        "starts",  "ends",       "contains", "is",          "null",      "case",
+    "when",   "then",       "else",      "end",     "count",      "filter",   "extract",     "any",       "none",
+    "single", "true",       "false",     "reduce",  "coalesce",   "user",     "password",    "alter",     "drop",
+    "show",   "stats",      "unique",    "explain", "profile",    "storage",  "index",       "info",      "exists",
+    "assert", "constraint", "node",      "key",     "dump",       "database", "call",        "yield",     "memory",
+    "mb",     "kb",         "unlimited", "free",    "procedure",  "query",    "free_memory", "read_file", "lock_path"};
 
 // Unicode codepoints that are allowed at the start of the unescaped name.
 const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

src/query/interpreter.cpp

@@ -27,6 +27,7 @@
 #include "utils/logging.hpp"
 #include "utils/memory.hpp"
 #include "utils/memory_tracker.hpp"
+#include "utils/readable_size.hpp"
 #include "utils/string.hpp"
 #include "utils/tsc.hpp"
 
@@ -603,8 +604,8 @@ struct PullPlanVector {
 
 struct PullPlan {
   explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
-                    DbAccessor *dba, InterpreterContext *interpreter_context,
-                    utils::MonotonicBufferResource *execution_memory);
+                    DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
+                    std::optional<size_t> memory_limit = {});
   std::optional<ExecutionContext> Pull(AnyStream *stream, std::optional<int> n,
                                        const std::vector<Symbol> &output_symbols,
                                        std::map<std::string, TypedValue> *summary);
@@ -614,6 +615,7 @@ struct PullPlan {
   plan::UniqueCursorPtr cursor_ = nullptr;
   Frame frame_;
   ExecutionContext ctx_;
+  std::optional<size_t> memory_limit_;
 
   // As it's possible to query execution using multiple pulls
   // we need the keep track of the total execution time across
@@ -630,11 +632,12 @@ struct PullPlan {
 };
 
 PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
-                   DbAccessor *dba, InterpreterContext *interpreter_context,
-                   utils::MonotonicBufferResource *execution_memory)
+                   DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
+                   const std::optional<size_t> memory_limit)
     : plan_(plan),
       cursor_(plan->plan().MakeCursor(execution_memory)),
-      frame_(plan->symbol_table().max_position(), execution_memory) {
+      frame_(plan->symbol_table().max_position(), execution_memory),
+      memory_limit_(memory_limit) {
   ctx_.db_accessor = dba;
   ctx_.symbol_table = plan->symbol_table();
   ctx_.evaluation_context.timestamp =
@@ -657,21 +660,25 @@ std::optional<ExecutionContext> PullPlan::Pull(AnyStream *stream, std::optional<
   // single `Pull`.
   constexpr size_t stack_size = 256 * 1024;
   char stack_data[stack_size];
+  utils::ResourceWithOutOfMemoryException resource_with_exception;
+  utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
+  // We can throw on every query because a simple queries for deleting will use only
+  // the stack allocated buffer.
+  // Also, we want to throw only when the query engine requests more memory and not the storage
+  // so we add the exception to the allocator.
+  // TODO (mferencevic): Tune the parameters accordingly.
+  utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
+  std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
+
+  if (memory_limit_) {
+    maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
+    ctx_.evaluation_context.memory = &*maybe_limited_resource;
+  } else {
+    ctx_.evaluation_context.memory = &pool_memory;
+  }
 
   // Returns true if a result was pulled.
-  const auto pull_result = [&]() -> bool {
-    // We can throw on every query because a simple queries for deleting will use only
-    // the stack allocated buffer.
-    // Also, we want to throw only when the query engine requests more memory and not the storage
-    // so we add the exception to the allocator.
-    utils::ResourceWithOutOfMemoryException resource_with_exception;
-    utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
-    // TODO (mferencevic): Tune the parameters accordingly.
-    utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
-    ctx_.evaluation_context.memory = &pool_memory;
-
-    return cursor_->Pull(frame_, ctx_);
-  };
+  const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); };
 
   const auto stream_values = [&]() {
     // TODO: The streamed values should also probably use the above memory.
@@ -828,10 +835,24 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
 
 PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
                                  InterpreterContext *interpreter_context, DbAccessor *dba,
-                                 utils::MonotonicBufferResource *execution_memory) {
-  auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage),
-                                utils::Downcast<CypherQuery>(parsed_query.query), parsed_query.parameters,
-                                &interpreter_context->plan_cache, dba, parsed_query.is_cacheable);
+                                 utils::MemoryResource *execution_memory) {
+  auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
+
+  Frame frame(0);
+  SymbolTable symbol_table;
+  EvaluationContext evaluation_context;
+  evaluation_context.timestamp =
+      std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
+          .count();
+  evaluation_context.parameters = parsed_query.parameters;
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
+  const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
+  if (memory_limit) {
+    spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit));
+  }
+
+  auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
+                                parsed_query.parameters, &interpreter_context->plan_cache, dba);
 
   summary->insert_or_assign("cost_estimate", plan->cost());
   auto rw_type_checker = plan::ReadWriteTypeChecker();
@@ -850,8 +871,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
         utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
   }
 
-  auto pull_plan =
-      std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory);
+  auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
+                                              execution_memory, memory_limit);
   return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
                        [pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
                            AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
@@ -865,7 +886,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
 
 PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
                                   InterpreterContext *interpreter_context, DbAccessor *dba,
-                                  utils::MonotonicBufferResource *execution_memory) {
+                                  utils::MemoryResource *execution_memory) {
   const std::string kExplainQueryStart = "explain ";
   MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
             "Expected stripped query to start with '{}'", kExplainQueryStart);
@@ -911,7 +932,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
 
 PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                   std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
-                                  DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
+                                  DbAccessor *dba, utils::MemoryResource *execution_memory) {
   const std::string kProfileQueryStart = "profile ";
 
   MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -949,6 +970,15 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
 
   auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
   MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
+  Frame frame(0);
+  SymbolTable symbol_table;
+  EvaluationContext evaluation_context;
+  evaluation_context.timestamp =
+      std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
+          .count();
+  evaluation_context.parameters = parsed_inner_query.parameters;
+  ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
+  const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
 
   auto cypher_query_plan = CypherQueryToPlan(
       parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
@@ -960,14 +990,14 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
       {"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
       std::move(parsed_query.required_privileges),
       [plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), summary, dba,
-       interpreter_context, execution_memory,
+       interpreter_context, execution_memory, memory_limit,
        // We want to execute the query we are profiling lazily, so we delay
        // the construction of the corresponding context.
        ctx = std::optional<ExecutionContext>{}, pull_plan = std::shared_ptr<PullPlanVector>(nullptr)](
           AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
         // No output symbols are given so that nothing is streamed.
         if (!ctx) {
-          ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory)
+          ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, memory_limit)
                     .Pull(stream, {}, {}, summary);
           pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(ctx->stats, ctx->profile_execution_time));
         }
@@ -985,7 +1015,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
 }
 
 PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
-                               utils::MonotonicBufferResource *execution_memory) {
+                               utils::MemoryResource *execution_memory) {
   return PreparedQuery{{"QUERY"},
                        std::move(parsed_query.required_privileges),
                        [pull_plan = std::make_shared<PullPlanDump>(dba)](
@@ -1000,7 +1030,7 @@ PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, T
 
 PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                 std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
-                                utils::MonotonicBufferResource *execution_memory) {
+                                utils::MemoryResource *execution_memory) {
   if (in_explicit_transaction) {
     throw IndexInMulticommandTxException();
   }
@@ -1069,7 +1099,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
 
 PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
-                               DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
+                               DbAccessor *dba, utils::MemoryResource *execution_memory) {
   if (in_explicit_transaction) {
     throw UserModificationInMulticommandTxException();
   }
@@ -1122,6 +1152,8 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_ex
                          return std::nullopt;
                        },
                        RWType::NONE};
+  // False positive report for the std::make_shared above
+  // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
 }
 
 PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
@@ -1180,7 +1212,7 @@ PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_exp
 
 PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
-                               storage::Storage *db, utils::MonotonicBufferResource *execution_memory) {
+                               storage::Storage *db, utils::MemoryResource *execution_memory) {
   if (in_explicit_transaction) {
     throw InfoInMulticommandTxException();
   }
@@ -1268,8 +1300,7 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
 
 PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
                                      std::map<std::string, TypedValue> *summary,
-                                     InterpreterContext *interpreter_context,
-                                     utils::MonotonicBufferResource *execution_memory) {
+                                     InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory) {
   if (in_explicit_transaction) {
     throw ConstraintInMulticommandTxException();
   }
@@ -1434,10 +1465,12 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
       in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
 
   // Handle transaction control queries.
-  auto query_upper = utils::Trim(utils::ToUpperCase(query_string));
 
-  if (query_upper == "BEGIN" || query_upper == "COMMIT" || query_upper == "ROLLBACK") {
-    query_execution->prepared_query.emplace(PrepareTransactionQuery(query_upper));
+  const auto upper_case_query = utils::ToUpperCase(query_string);
+  const auto trimmed_query = utils::Trim(upper_case_query);
+
+  if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") {
+    query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
     return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
   }
 

src/query/interpreter.hpp

@@ -317,7 +317,9 @@ class Interpreter final {
  private:
   struct QueryExecution {
     std::optional<PreparedQuery> prepared_query;
-    utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
+    utils::MonotonicBufferResource execution_monotonic_memory{kExecutionMemoryBlockSize};
+    utils::ResourceWithOutOfMemoryException execution_memory{&execution_monotonic_memory};
+
     std::map<std::string, TypedValue> summary;
 
     explicit QueryExecution() = default;
@@ -331,7 +333,7 @@ class Interpreter final {
       // destroy the prepared query which is using that instance
       // of execution memory.
       prepared_query.reset();
-      execution_memory.Release();
+      execution_monotonic_memory.Release();
     }
   };
 

src/query/plan/operator.cpp

@@ -324,11 +324,15 @@ VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, Exec
 template <class TVerticesFun>
 class ScanAllCursor : public Cursor {
  public:
-  explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices)
-      : output_symbol_(output_symbol), input_cursor_(std::move(input_cursor)), get_vertices_(std::move(get_vertices)) {}
+  explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices,
+                         const char *op_name)
+      : output_symbol_(output_symbol),
+        input_cursor_(std::move(input_cursor)),
+        get_vertices_(std::move(get_vertices)),
+        op_name_(op_name) {}
 
   bool Pull(Frame &frame, ExecutionContext &context) override {
-    SCOPED_PROFILE_OP("ScanAll");
+    SCOPED_PROFILE_OP(op_name_);
 
     if (MustAbort(context)) throw HintedAbortError();
 
@@ -364,6 +368,7 @@ class ScanAllCursor : public Cursor {
   TVerticesFun get_vertices_;
   std::optional<typename std::result_of<TVerticesFun(Frame &, ExecutionContext &)>::type::value_type> vertices_;
   std::optional<decltype(vertices_.value().begin())> vertices_it_;
+  const char *op_name_;
 };
 
 ScanAll::ScanAll(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, storage::View view)
@@ -379,7 +384,7 @@ UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const {
     return std::make_optional(db->Vertices(view_));
   };
   return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
-                                                                std::move(vertices));
+                                                                std::move(vertices), "ScanAll");
 }
 
 std::vector<Symbol> ScanAll::ModifiedSymbols(const SymbolTable &table) const {
@@ -402,7 +407,7 @@ UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const {
     return std::make_optional(db->Vertices(view_, label_));
   };
   return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
-                                                                std::move(vertices));
+                                                                std::move(vertices), "ScanAllByLabel");
 }
 
 // TODO(buda): Implement ScanAllByLabelProperty operator to iterate over
@@ -466,7 +471,7 @@ UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *m
     return std::make_optional(db->Vertices(view_, label_, property_, maybe_lower, maybe_upper));
   };
   return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
-                                                                std::move(vertices));
+                                                                std::move(vertices), "ScanAllByLabelPropertyRange");
 }
 
 ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr<LogicalOperator> &input,
@@ -498,7 +503,7 @@ UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *m
     return std::make_optional(db->Vertices(view_, label_, property_, storage::PropertyValue(value)));
   };
   return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
-                                                                std::move(vertices));
+                                                                std::move(vertices), "ScanAllByLabelPropertyValue");
 }
 
 ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol,
@@ -516,7 +521,7 @@ UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) c
     return std::make_optional(db->Vertices(view_, label_, property_));
   };
   return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
-                                                                std::move(vertices));
+                                                                std::move(vertices), "ScanAllByLabelProperty");
 }
 
 ScanAllById::ScanAllById(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, Expression *expression,
@@ -542,7 +547,7 @@ UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const {
     return std::vector<VertexAccessor>{*maybe_vertex};
   };
   return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
-                                                                std::move(vertices));
+                                                                std::move(vertices), "ScanAllById");
 }
 
 namespace {

src/query/plan/preprocess.cpp

@@ -522,7 +522,8 @@ std::vector<SingleQueryPart> CollectSingleQueryParts(SymbolTable &symbol_table,
         query_part->merge_matching.emplace_back(Matching{});
         AddMatching({merge->pattern_}, nullptr, symbol_table, storage, query_part->merge_matching.back());
       } else if (utils::IsSubtype(*clause, With::kType) || utils::IsSubtype(*clause, query::Unwind::kType) ||
-                 utils::IsSubtype(*clause, query::CallProcedure::kType)) {
+                 utils::IsSubtype(*clause, query::CallProcedure::kType) ||
+                 utils::IsSubtype(*clause, query::LoadCsv::kType)) {
         // This query part is done, continue with a new one.
         query_parts.emplace_back(SingleQueryPart{});
         query_part = &query_parts.back();

src/storage/v2/durability/durability.cpp

@@ -102,30 +102,47 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
 // recovery process.
 void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
                                   Constraints *constraints, utils::SkipList<Vertex> *vertices) {
+  spdlog::info("Recreating indices from metadata.");
   // Recover label indices.
+  spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
   for (const auto &item : indices_constraints.indices.label) {
     if (!indices->label_index.CreateIndex(item, vertices->access()))
       throw RecoveryFailure("The label index must be created here!");
+    spdlog::info("A label index is recreated from metadata.");
   }
+  spdlog::info("Label indices are recreated.");
 
   // Recover label+property indices.
+  spdlog::info("Recreating {} label+property indices from metadata.",
+               indices_constraints.indices.label_property.size());
   for (const auto &item : indices_constraints.indices.label_property) {
     if (!indices->label_property_index.CreateIndex(item.first, item.second, vertices->access()))
       throw RecoveryFailure("The label+property index must be created here!");
+    spdlog::info("A label+property index is recreated from metadata.");
   }
+  spdlog::info("Label+property indices are recreated.");
+  spdlog::info("Indices are recreated.");
 
+  spdlog::info("Recreating constraints from metadata.");
   // Recover existence constraints.
+  spdlog::info("Recreating {} existence constraints from metadata.", indices_constraints.constraints.existence.size());
   for (const auto &item : indices_constraints.constraints.existence) {
     auto ret = CreateExistenceConstraint(constraints, item.first, item.second, vertices->access());
     if (ret.HasError() || !ret.GetValue()) throw RecoveryFailure("The existence constraint must be created here!");
+    spdlog::info("A existence constraint is recreated from metadata.");
   }
+  spdlog::info("Existence constraints are recreated from metadata.");
 
   // Recover unique constraints.
+  spdlog::info("Recreating {} unique constraints from metadata.", indices_constraints.constraints.unique.size());
   for (const auto &item : indices_constraints.constraints.unique) {
     auto ret = constraints->unique_constraints.CreateConstraint(item.first, item.second, vertices->access());
     if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
       throw RecoveryFailure("The unique constraint must be created here!");
+    spdlog::info("A unique constraint is recreated from metadata.");
   }
+  spdlog::info("Unique constraints are recreated from metadata.");
+  spdlog::info("Constraints are recreated from metadata.");
 }
 
 std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_directory,
@@ -137,7 +154,12 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
                                         Indices *indices, Constraints *constraints, Config::Items items,
                                         uint64_t *wal_seq_num) {
   utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
-  if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) return std::nullopt;
+  spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
+               wal_directory);
+  if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) {
+    spdlog::warn("Snapshot or WAL directory don't exist, there is nothing to recover.");
+    return std::nullopt;
+  }
 
   auto snapshot_files = GetSnapshotFiles(snapshot_directory);
 
@@ -145,6 +167,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
   RecoveredIndicesAndConstraints indices_constraints;
   std::optional<uint64_t> snapshot_timestamp;
   if (!snapshot_files.empty()) {
+    spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory);
     // Order the files by name
     std::sort(snapshot_files.begin(), snapshot_files.end());
 
@@ -157,13 +180,13 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
         spdlog::warn("The snapshot file {} isn't related to the latest snapshot file!", path);
         continue;
       }
-      spdlog::info("Starting snapshot recovery from {}", path);
+      spdlog::info("Starting snapshot recovery from {}.", path);
       try {
         recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history, name_id_mapper, edge_count, items);
         spdlog::info("Snapshot recovery successful!");
         break;
       } catch (const RecoveryFailure &e) {
-        spdlog::warn("Couldn't recover snapshot from {} because of: {}", path, e.what());
+        spdlog::warn("Couldn't recover snapshot from {} because of: {}.", path, e.what());
         continue;
       }
     }
@@ -181,6 +204,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
       return recovered_snapshot->recovery_info;
     }
   } else {
+    spdlog::info("No snapshot file was found, collecting information from WAL directory {}.", wal_directory);
     std::error_code error_code;
     if (!utils::DirExists(wal_directory)) return std::nullopt;
     // We use this smaller struct that contains only a subset of information
@@ -206,7 +230,10 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
       }
     }
     MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message());
-    if (wal_files.empty()) return std::nullopt;
+    if (wal_files.empty()) {
+      spdlog::warn("No snapshot or WAL file found!");
+      return std::nullopt;
+    }
     std::sort(wal_files.begin(), wal_files.end());
     // UUID used for durability is the UUID of the last WAL file.
     // Same for the epoch id.
@@ -215,7 +242,10 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
   }
 
   auto maybe_wal_files = GetWalFiles(wal_directory, *uuid);
-  if (!maybe_wal_files) return std::nullopt;
+  if (!maybe_wal_files) {
+    spdlog::warn("Couldn't get WAL file info from the WAL directory!");
+    return std::nullopt;
+  }
 
   // Array of all discovered WAL files, ordered by sequence number.
   auto &wal_files = *maybe_wal_files;
@@ -232,6 +262,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
             "files that match the last WAL file!");
 
   if (!wal_files.empty()) {
+    spdlog::info("Checking WAL files.");
     {
       const auto &first_wal = wal_files[0];
       if (first_wal.seq_num != 0) {
@@ -255,6 +286,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
     }
     std::optional<uint64_t> previous_seq_num;
     auto last_loaded_timestamp = snapshot_timestamp;
+    spdlog::info("Trying to load WAL files.");
     for (auto &wal_file : wal_files) {
       if (previous_seq_num && (wal_file.seq_num - *previous_seq_num) > 1) {
         LOG_FATAL("You are missing a WAL file with the sequence number {}!", *previous_seq_num + 1);
@@ -290,6 +322,8 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
     // The sequence number needs to be recovered even though `LoadWal` didn't
     // load any deltas from that file.
     *wal_seq_num = *previous_seq_num + 1;
+
+    spdlog::info("All necessary WAL files are loaded successfully.");
   }
 
   RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices);

src/storage/v2/durability/snapshot.cpp

@@ -168,14 +168,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
   });
 
   // Read snapshot info.
-  auto info = ReadSnapshotInfo(path);
-
+  const auto info = ReadSnapshotInfo(path);
+  spdlog::info("Recovering {} vertices and {} edges.", info.vertices_count, info.edges_count);
   // Check for edges.
   bool snapshot_has_edges = info.offset_edges != 0;
 
   // Recover mapper.
   std::unordered_map<uint64_t, uint64_t> snapshot_id_map;
   {
+    spdlog::info("Recovering mapper metadata.");
     if (!snapshot.SetPosition(info.offset_mapper)) throw RecoveryFailure("Couldn't read data from snapshot!");
 
     auto marker = snapshot.ReadMarker();
@@ -191,6 +192,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
       if (!name) throw RecoveryFailure("Invalid snapshot data!");
       auto my_id = name_id_mapper->NameToId(*name);
       snapshot_id_map.emplace(*id, my_id);
+      SPDLOG_TRACE("Mapping \"{}\"from snapshot id {} to actual id {}.", *name, *id, my_id);
     }
   }
   auto get_label_from_id = [&snapshot_id_map](uint64_t snapshot_id) {
@@ -217,10 +219,11 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
     auto edge_acc = edges->access();
     uint64_t last_edge_gid = 0;
     if (snapshot_has_edges) {
+      spdlog::info("Recovering {} edges.", info.edges_count);
       if (!snapshot.SetPosition(info.offset_edges)) throw RecoveryFailure("Couldn't read data from snapshot!");
       for (uint64_t i = 0; i < info.edges_count; ++i) {
         {
-          auto marker = snapshot.ReadMarker();
+          const auto marker = snapshot.ReadMarker();
           if (!marker || *marker != Marker::SECTION_EDGE) throw RecoveryFailure("Invalid snapshot data!");
         }
 
@@ -230,6 +233,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
           if (!gid) throw RecoveryFailure("Invalid snapshot data!");
           if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
           last_edge_gid = *gid;
+          spdlog::debug("Recovering edge {} with properties.", *gid);
           auto [it, inserted] = edge_acc.insert(Edge{Gid::FromUint(*gid), nullptr});
           if (!inserted) throw RecoveryFailure("The edge must be inserted here!");
 
@@ -243,6 +247,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
               if (!key) throw RecoveryFailure("Invalid snapshot data!");
               auto value = snapshot.ReadPropertyValue();
               if (!value) throw RecoveryFailure("Invalid snapshot data!");
+              SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for edge {}.",
+                           name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
               props.SetProperty(get_property_from_id(*key), *value);
             }
           }
@@ -253,6 +259,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
           if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
           last_edge_gid = *gid;
 
+          spdlog::debug("Ensuring edge {} doesn't have any properties.", *gid);
           // Read properties.
           {
             auto props_size = snapshot.ReadUint();
@@ -264,12 +271,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
           }
         }
       }
+      spdlog::info("Edges are recovered.");
     }
 
     // Recover vertices (labels and properties).
     if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
     auto vertex_acc = vertices->access();
     uint64_t last_vertex_gid = 0;
+    spdlog::info("Recovering {} vertices.", info.vertices_count);
     for (uint64_t i = 0; i < info.vertices_count; ++i) {
       {
         auto marker = snapshot.ReadMarker();
@@ -283,10 +292,12 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         throw RecoveryFailure("Invalid snapshot data!");
       }
       last_vertex_gid = *gid;
+      spdlog::debug("Recovering vertex {}.", *gid);
       auto [it, inserted] = vertex_acc.insert(Vertex{Gid::FromUint(*gid), nullptr});
       if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
 
       // Recover labels.
+      spdlog::trace("Recovering labels for vertex {}.", *gid);
       {
         auto labels_size = snapshot.ReadUint();
         if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
@@ -295,11 +306,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         for (uint64_t j = 0; j < *labels_size; ++j) {
           auto label = snapshot.ReadUint();
           if (!label) throw RecoveryFailure("Invalid snapshot data!");
+          SPDLOG_TRACE("Recovered label \"{}\" for vertex {}.", name_id_mapper->IdToName(snapshot_id_map.at(*label)),
+                       *gid);
           labels.emplace_back(get_label_from_id(*label));
         }
       }
 
       // Recover properties.
+      spdlog::trace("Recovering properties for vertex {}.", *gid);
       {
         auto props_size = snapshot.ReadUint();
         if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
@@ -309,6 +323,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
           if (!key) throw RecoveryFailure("Invalid snapshot data!");
           auto value = snapshot.ReadPropertyValue();
           if (!value) throw RecoveryFailure("Invalid snapshot data!");
+          SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for vertex {}.",
+                       name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
           props.SetProperty(get_property_from_id(*key), *value);
         }
       }
@@ -339,8 +355,10 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
       }
     }
+    spdlog::info("Vertices are recovered.");
 
     // Recover vertices (in/out edges).
+    spdlog::info("Recovering connectivity.");
     if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
     for (auto &vertex : vertex_acc) {
       {
@@ -348,6 +366,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
       }
 
+      spdlog::trace("Recovering connectivity for vertex {}.", vertex.gid.AsUint());
       // Check vertex.
       auto gid = snapshot.ReadUint();
       if (!gid) throw RecoveryFailure("Invalid snapshot data!");
@@ -377,6 +396,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
 
       // Recover in edges.
       {
+        spdlog::trace("Recovering inbound edges for vertex {}.", vertex.gid.AsUint());
         auto in_size = snapshot.ReadUint();
         if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
         vertex.in_edges.reserve(*in_size);
@@ -404,12 +424,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
               edge_ref = EdgeRef(&*edge);
             }
           }
+          SPDLOG_TRACE("Recovered inbound edge {} with label \"{}\" from vertex {}.", *edge_gid,
+                       name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), from_vertex->gid.AsUint());
           vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &*from_vertex, edge_ref);
         }
       }
 
       // Recover out edges.
       {
+        spdlog::trace("Recovering outbound edges for vertex {}.", vertex.gid.AsUint());
         auto out_size = snapshot.ReadUint();
         if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
         vertex.out_edges.reserve(*out_size);
@@ -437,6 +460,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
               edge_ref = EdgeRef(&*edge);
             }
           }
+          SPDLOG_TRACE("Recovered outbound edge {} with label \"{}\" to vertex {}.", *edge_gid,
+                       name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), to_vertex->gid.AsUint());
           vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &*to_vertex, edge_ref);
         }
         // Increment edge count. We only increment the count here because the
@@ -444,6 +469,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         edge_count->fetch_add(*out_size, std::memory_order_acq_rel);
       }
     }
+    spdlog::info("Connectivity is recovered.");
 
     // Set initial values for edge/vertex ID generators.
     ret.next_edge_id = last_edge_gid + 1;
@@ -452,6 +478,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
 
   // Recover indices.
   {
+    spdlog::info("Recovering metadata of indices.");
     if (!snapshot.SetPosition(info.offset_indices)) throw RecoveryFailure("Couldn't read data from snapshot!");
 
     auto marker = snapshot.ReadMarker();
@@ -461,18 +488,22 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
     {
       auto size = snapshot.ReadUint();
       if (!size) throw RecoveryFailure("Invalid snapshot data!");
+      spdlog::info("Recovering metadata of {} label indices.", *size);
       for (uint64_t i = 0; i < *size; ++i) {
         auto label = snapshot.ReadUint();
         if (!label) throw RecoveryFailure("Invalid snapshot data!");
         AddRecoveredIndexConstraint(&indices_constraints.indices.label, get_label_from_id(*label),
                                     "The label index already exists!");
+        SPDLOG_TRACE("Recovered metadata of label index for :{}", name_id_mapper->IdToName(snapshot_id_map.at(*label)));
       }
+      spdlog::info("Metadata of label indices are recovered.");
     }
 
     // Recover label+property indices.
     {
       auto size = snapshot.ReadUint();
       if (!size) throw RecoveryFailure("Invalid snapshot data!");
+      spdlog::info("Recovering metadata of {} label+property indices.", *size);
       for (uint64_t i = 0; i < *size; ++i) {
         auto label = snapshot.ReadUint();
         if (!label) throw RecoveryFailure("Invalid snapshot data!");
@@ -481,12 +512,18 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         AddRecoveredIndexConstraint(&indices_constraints.indices.label_property,
                                     {get_label_from_id(*label), get_property_from_id(*property)},
                                     "The label+property index already exists!");
+        SPDLOG_TRACE("Recovered metadata of label+property index for :{}({})",
+                     name_id_mapper->IdToName(snapshot_id_map.at(*label)),
+                     name_id_mapper->IdToName(snapshot_id_map.at(*property)));
       }
+      spdlog::info("Metadata of label+property indices are recovered.");
     }
+    spdlog::info("Metadata of indices are recovered.");
   }
 
   // Recover constraints.
   {
+    spdlog::info("Recovering metadata of constraints.");
     if (!snapshot.SetPosition(info.offset_constraints)) throw RecoveryFailure("Couldn't read data from snapshot!");
 
     auto marker = snapshot.ReadMarker();
@@ -496,6 +533,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
     {
       auto size = snapshot.ReadUint();
       if (!size) throw RecoveryFailure("Invalid snapshot data!");
+      spdlog::info("Recovering metadata of {} existence constraints.", *size);
       for (uint64_t i = 0; i < *size; ++i) {
         auto label = snapshot.ReadUint();
         if (!label) throw RecoveryFailure("Invalid snapshot data!");
@@ -504,7 +542,11 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         AddRecoveredIndexConstraint(&indices_constraints.constraints.existence,
                                     {get_label_from_id(*label), get_property_from_id(*property)},
                                     "The existence constraint already exists!");
+        SPDLOG_TRACE("Recovered metadata of existence constraint for :{}({})",
+                     name_id_mapper->IdToName(snapshot_id_map.at(*label)),
+                     name_id_mapper->IdToName(snapshot_id_map.at(*property)));
       }
+      spdlog::info("Metadata of existence constraints are recovered.");
     }
 
     // Recover unique constraints.
@@ -513,6 +555,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
     if (*version >= kUniqueConstraintVersion) {
       auto size = snapshot.ReadUint();
       if (!size) throw RecoveryFailure("Invalid snapshot data!");
+      spdlog::info("Recovering metadata of {} unique constraints.", *size);
       for (uint64_t i = 0; i < *size; ++i) {
         auto label = snapshot.ReadUint();
         if (!label) throw RecoveryFailure("Invalid snapshot data!");
@@ -526,10 +569,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
         }
         AddRecoveredIndexConstraint(&indices_constraints.constraints.unique, {get_label_from_id(*label), properties},
                                     "The unique constraint already exists!");
+        SPDLOG_TRACE("Recovered metadata of unique constraints for :{}",
+                     name_id_mapper->IdToName(snapshot_id_map.at(*label)));
       }
+      spdlog::info("Metadata of unique constraints are recovered.");
     }
+    spdlog::info("Metadata of constraints are recovered.");
   }
 
+  spdlog::info("Recovering metadata.");
   // Recover epoch history
   {
     if (!snapshot.SetPosition(info.offset_epoch_history)) throw RecoveryFailure("Couldn't read data from snapshot!");
@@ -555,6 +603,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
     }
   }
 
+  spdlog::info("Metadata recovered.");
   // Recover timestamp.
   ret.next_timestamp = info.start_timestamp + 1;
 

src/storage/v2/durability/wal.cpp

@@ -610,6 +610,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
                      const std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
                      utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
                      Config::Items items) {
+  spdlog::info("Trying to load WAL file {}.", path);
   RecoveryInfo ret;
 
   Decoder wal;
@@ -622,13 +623,17 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
   ret.last_commit_timestamp = info.to_timestamp;
 
   // Check timestamp.
-  if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) return ret;
+  if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) {
+    spdlog::info("Skip loading WAL file because it is too old.");
+    return ret;
+  }
 
   // Recover deltas.
   wal.SetPosition(info.offset_deltas);
   uint64_t deltas_applied = 0;
   auto edge_acc = edges->access();
   auto vertex_acc = vertices->access();
+  spdlog::info("WAL file contains {} deltas.", info.num_deltas);
   for (uint64_t i = 0; i < info.num_deltas; ++i) {
     // Read WAL delta header to find out the delta timestamp.
     auto timestamp = ReadWalDeltaHeader(&wal);
@@ -839,7 +844,8 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
     }
   }
 
-  spdlog::info("Applied {} deltas from WAL", deltas_applied, path);
+  spdlog::info("Applied {} deltas from WAL. Skipped {} deltas, because they were too old.", deltas_applied,
+               info.num_deltas - deltas_applied);
 
   return ret;
 }

src/utils/csv_parsing.cpp

@@ -61,12 +61,7 @@ void Reader::TryInitializeHeader() {
 const Reader::Header &Reader::GetHeader() const { return header_; }
 
 namespace {
-enum class CsvParserState : uint8_t {
-  INITIAL_FIELD,
-  NEXT_FIELD,
-  QUOTING,
-  EXPECT_DELIMITER,
-};
+enum class CsvParserState : uint8_t { INITIAL_FIELD, NEXT_FIELD, QUOTING, EXPECT_DELIMITER, DONE };
 
 }  // namespace
 
@@ -89,7 +84,12 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
 
     std::string_view line_string_view = *maybe_line;
 
-    while (!line_string_view.empty()) {
+    // remove '\r' from the end in case we have dos file format
+    if (line_string_view.back() == '\r') {
+      line_string_view.remove_suffix(1);
+    }
+
+    while (state != CsvParserState::DONE && !line_string_view.empty()) {
       const auto c = line_string_view[0];
 
       // Line feeds and carriage returns are ignored in CSVs.
@@ -120,11 +120,11 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
             const auto delimiter_idx = line_string_view.find(*read_config_.delimiter);
             row.emplace_back(line_string_view.substr(0, delimiter_idx));
             if (delimiter_idx == std::string_view::npos) {
-              line_string_view.remove_prefix(line_string_view.size());
+              state = CsvParserState::DONE;
             } else {
               line_string_view.remove_prefix(delimiter_idx + read_config_.delimiter->size());
+              state = CsvParserState::NEXT_FIELD;
             }
-            state = CsvParserState::NEXT_FIELD;
           }
           break;
         }
@@ -159,15 +159,21 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
           }
           break;
         }
+        case CsvParserState::DONE: {
+          LOG_FATAL("Invalid state of the CSV parser!");
+        }
       }
     }
   } while (state == CsvParserState::QUOTING);
 
   switch (state) {
     case CsvParserState::INITIAL_FIELD:
-    case CsvParserState::NEXT_FIELD:
+    case CsvParserState::DONE:
     case CsvParserState::EXPECT_DELIMITER:
       break;
+    case CsvParserState::NEXT_FIELD:
+      row.emplace_back("");
+      break;
     case CsvParserState::QUOTING: {
       return ParseError(ParseError::ErrorCode::NO_CLOSING_QUOTE,
                         "There is no more data left to load while inside a quoted string. "

src/utils/logging.hpp

@@ -65,7 +65,7 @@ void Fatal(const char *msg, const Args &...msg_args) {
   do {                             \
     spdlog::critical(__VA_ARGS__); \
     std::terminate();              \
-  } while (0);
+  } while (0)
 
 #ifndef NDEBUG
 #define DLOG_FATAL(...) LOG_FATAL(__VA_ARGS__)

src/utils/memory.cpp

@@ -23,6 +23,9 @@ size_t GrowMonotonicBuffer(size_t current_size, size_t max_size) {
   return std::ceil(next_size);
 }
 
+__attribute__((no_sanitize("pointer-overflow"))) void CheckAllocationSizeOverflow(void *aligned_ptr, size_t bytes) {
+  if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
+}
 }  // namespace
 
 MonotonicBufferResource::MonotonicBufferResource(size_t initial_size) : initial_size_(initial_size) {}
@@ -121,7 +124,7 @@ void *MonotonicBufferResource::DoAllocate(size_t bytes, size_t alignment) {
     next_buffer_size_ = GrowMonotonicBuffer(next_buffer_size_, std::numeric_limits<size_t>::max() - sizeof(Buffer));
   }
   if (reinterpret_cast<char *>(aligned_ptr) < buffer_head) throw BadAlloc("Allocation alignment overflow");
-  if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
+  CheckAllocationSizeOverflow(aligned_ptr, bytes);
   allocated_ = reinterpret_cast<char *>(aligned_ptr) - data + bytes;
   return aligned_ptr;
 }

src/utils/new_delete.cpp

@@ -1,3 +1,4 @@
+#include <cstddef>
 #include <new>
 
 #if USE_JEMALLOC
@@ -10,7 +11,7 @@
 #include "utils/memory_tracker.hpp"
 
 namespace {
-void *newImpl(std::size_t size) {
+void *newImpl(const std::size_t size) {
   auto *ptr = malloc(size);
   if (LIKELY(ptr != nullptr)) {
     return ptr;
@@ -19,11 +20,26 @@ void *newImpl(std::size_t size) {
   throw std::bad_alloc{};
 }
 
-void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
+void *newImpl(const std::size_t size, const std::align_val_t align) {
+  auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
+  if (LIKELY(ptr != nullptr)) {
+    return ptr;
+  }
 
-void deleteImpl(void *ptr) noexcept { free(ptr); }
+  throw std::bad_alloc{};
+}
+
+void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
+void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept {
+  return aligned_alloc(size, static_cast<std::size_t>(align));
+}
 
 #if USE_JEMALLOC
+void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
+
+void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
+  dallocx(ptr, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
+}
 
 void deleteSized(void *ptr, const std::size_t size) noexcept {
   if (UNLIKELY(ptr == nullptr)) {
@@ -33,24 +49,43 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
   sdallocx(ptr, size, 0);
 }
 
+void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
+  if (UNLIKELY(ptr == nullptr)) {
+    return;
+  }
+
+  sdallocx(ptr, size, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
+}
+
 #else
+void deleteImpl(void *ptr) noexcept { free(ptr); }
+
+void deleteImpl(void *ptr, const std::align_val_t /*unused*/) noexcept { free(ptr); }
 
 void deleteSized(void *ptr, const std::size_t /*unused*/) noexcept { free(ptr); }
 
+void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t /*unused*/) noexcept { free(ptr); }
 #endif
 
-void TrackMemory(const size_t size) {
-  size_t actual_size = size;
-
+void TrackMemory(std::size_t size) {
 #if USE_JEMALLOC
   if (LIKELY(size != 0)) {
-    actual_size = nallocx(size, 0);
+    size = nallocx(size, 0);
   }
 #endif
-  utils::total_memory_tracker.Alloc(actual_size);
+  utils::total_memory_tracker.Alloc(size);
 }
 
-bool TrackMemoryNoExcept(const size_t size) {
+void TrackMemory(std::size_t size, const std::align_val_t align) {
+#if USE_JEMALLOC
+  if (LIKELY(size != 0)) {
+    size = nallocx(size, MALLOCX_ALIGN(align));  // NOLINT(hicpp-signed-bitwise)
+  }
+#endif
+  utils::total_memory_tracker.Alloc(size);
+}
+
+bool TrackMemoryNoExcept(const std::size_t size) {
   try {
     TrackMemory(size);
   } catch (...) {
@@ -60,7 +95,17 @@ bool TrackMemoryNoExcept(const size_t size) {
   return true;
 }
 
-void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0) noexcept {
+bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
+  try {
+    TrackMemory(size, align);
+  } catch (...) {
+    return false;
+  }
+
+  return true;
+}
+
+void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
   try {
 #if USE_JEMALLOC
     if (LIKELY(ptr != nullptr)) {
@@ -78,32 +123,74 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0)
   }
 }
 
+void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
+  try {
+#if USE_JEMALLOC
+    if (LIKELY(ptr != nullptr)) {
+      utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align)));  // NOLINT(hicpp-signed-bitwise)
+    }
+#else
+    if (size) {
+      utils::total_memory_tracker.Free(size);
+    } else {
+      // Innaccurate because malloc_usable_size() result is greater or equal to allocated size.
+      utils::total_memory_tracker.Free(malloc_usable_size(ptr));
+    }
+#endif
+  } catch (...) {
+  }
+}
+
 }  // namespace
 
-void *operator new(std::size_t size) {
+void *operator new(const std::size_t size) {
   TrackMemory(size);
   return newImpl(size);
 }
 
-void *operator new[](std::size_t size) {
+void *operator new[](const std::size_t size) {
   TrackMemory(size);
   return newImpl(size);
 }
 
-void *operator new(std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
+void *operator new(const std::size_t size, const std::align_val_t align) {
+  TrackMemory(size, align);
+  return newImpl(size, align);
+}
+
+void *operator new[](const std::size_t size, const std::align_val_t align) {
+  TrackMemory(size, align);
+  return newImpl(size, align);
+}
+
+void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
   if (LIKELY(TrackMemoryNoExcept(size))) {
     return newNoExcept(size);
   }
   return nullptr;
 }
 
-void *operator new[](std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
+void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
   if (LIKELY(TrackMemoryNoExcept(size))) {
     return newNoExcept(size);
   }
   return nullptr;
 }
 
+void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
+  if (LIKELY(TrackMemoryNoExcept(size, align))) {
+    return newNoExcept(size, align);
+  }
+  return nullptr;
+}
+
+void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
+  if (LIKELY(TrackMemoryNoExcept(size, align))) {
+    return newNoExcept(size, align);
+  }
+  return nullptr;
+}
+
 void operator delete(void *ptr) noexcept {
   UntrackMemory(ptr);
   deleteImpl(ptr);
@@ -114,12 +201,52 @@ void operator delete[](void *ptr) noexcept {
   deleteImpl(ptr);
 }
 
-void operator delete(void *ptr, std::size_t size) noexcept {
+void operator delete(void *ptr, const std::align_val_t align) noexcept {
+  UntrackMemory(ptr, align);
+  deleteImpl(ptr, align);
+}
+
+void operator delete[](void *ptr, const std::align_val_t align) noexcept {
+  UntrackMemory(ptr, align);
+  deleteImpl(ptr, align);
+}
+
+void operator delete(void *ptr, const std::size_t size) noexcept {
   UntrackMemory(ptr, size);
   deleteSized(ptr, size);
 }
 
-void operator delete[](void *ptr, std::size_t size) noexcept {
+void operator delete[](void *ptr, const std::size_t size) noexcept {
   UntrackMemory(ptr, size);
   deleteSized(ptr, size);
 }
+
+void operator delete(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
+  UntrackMemory(ptr, align, size);
+  deleteSized(ptr, size, align);
+}
+
+void operator delete[](void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
+  UntrackMemory(ptr, align, size);
+  deleteSized(ptr, size, align);
+}
+
+void operator delete(void *ptr, const std::nothrow_t & /*unused*/) noexcept {
+  UntrackMemory(ptr);
+  deleteImpl(ptr);
+}
+
+void operator delete[](void *ptr, const std::nothrow_t & /*unused*/) noexcept {
+  UntrackMemory(ptr);
+  deleteImpl(ptr);
+}
+
+void operator delete(void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
+  UntrackMemory(ptr, align);
+  deleteImpl(ptr, align);
+}
+
+void operator delete[](void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
+  UntrackMemory(ptr, align);
+  deleteImpl(ptr, align);
+}

tests/e2e/memory/memory_control.cpp

@@ -38,19 +38,5 @@ int main(int argc, char **argv) {
   }
 
   spdlog::info("Memgraph is out of memory");
-
-  spdlog::info("Cleaning up unused memory");
-  client->Execute("MATCH (n) DETACH DELETE n;");
-  client->DiscardAll();
-  client->Execute("FREE MEMORY;");
-  client->DiscardAll();
-
-  // now it should succeed
-  spdlog::info("Retrying the query with the memory cleaned up");
-  client->Execute(create_query);
-  if (!client->FetchOne()) {
-    LOG_FATAL("Memgraph is still out of memory");
-  }
-
   return 0;
 }

tests/e2e/memory/workloads.yaml

@@ -2,7 +2,8 @@ bolt_port: &bolt_port "7687"
 template_cluster: &template_cluster
   cluster:
     main:
-      args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"]
+      args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"]
+      log_file: "memory-e2e.log"
       setup_queries: []
       validation_queries: []
 

tests/e2e/replication/constraints.cpp

@@ -39,7 +39,7 @@ int main(int argc, char **argv) {
         const auto label_name = (*data)[0][1].ValueString();
         const auto property_name = (*data)[0][2].ValueList()[0].ValueString();
         if (label_name != "Node" || property_name != "id") {
-          LOG_FATAL("{} does NOT hava valid constraint created.", database_endpoint)
+          LOG_FATAL("{} does NOT hava valid constraint created.", database_endpoint);
         }
       } else {
         LOG_FATAL("Unable to get CONSTRAINT INFO from {}", database_endpoint);

tests/e2e/replication/workloads.yaml

@@ -11,19 +11,23 @@ template_validation_queries: &template_validation_queries
 template_cluster: &template_cluster
   cluster:
     replica_1:
-      args: ["--bolt-port", "7688"]
+      args: ["--bolt-port", "7688", "--log-level=TRACE"]
+      log_file: "replication-e2e-replica1.log"
       setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
       <<: *template_validation_queries
     replica_2:
-      args: ["--bolt-port", "7689"]
+      args: ["--bolt-port", "7689", "--log-level=TRACE"]
+      log_file: "replication-e2e-replica2.log"
       setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
       <<: *template_validation_queries
     replica_3:
-      args: ["--bolt-port", "7690"]
+      args: ["--bolt-port", "7690", "--log-level=TRACE"]
+      log_file: "replication-e2e-replica3.log"
       setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10003;"]
       <<: *template_validation_queries
     main:
-      args: ["--bolt-port", "7687"]
+      args: ["--bolt-port", "7687", "--log-level=TRACE"]
+      log_file: "replication-e2e-main.log"
       setup_queries: [
         "REGISTER REPLICA replica_1 SYNC WITH TIMEOUT 0 TO '127.0.0.1:10001'",
         "REGISTER REPLICA replica_2 SYNC WITH TIMEOUT 1 TO '127.0.0.1:10002'",

tests/e2e/runner.py

@@ -44,7 +44,9 @@ def run(args):
         for name, config in workload['cluster'].items():
             mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY)
             mg_instances[name] = mg_instance
-            mg_instance.start(args=config['args'])
+            log_file_path = os.path.join(BUILD_DIR, 'logs', config['log_file'])
+            binary_args = config['args'] + ["--log-file", log_file_path]
+            mg_instance.start(args=binary_args)
             for query in config['setup_queries']:
                 mg_instance.query(query)
         # Test.

tests/integration/ldap/runner.py

@@ -73,7 +73,7 @@ class Memgraph:
         virtualenv_bin = os.path.join(SCRIPT_DIR, "ve3", "bin", "python3")
         with open(script_file) as fin:
             data = fin.read()
-            data = data.replace("/usr/bin/env python3", virtualenv_bin)
+            data = data.replace("/usr/bin/python3", virtualenv_bin)
             data = data.replace("/etc/memgraph/auth/ldap.yaml",
                                 self._auth_config)
             with open(self._auth_module, "w") as fout:

tests/integration/telemetry/runner.py

@@ -45,7 +45,7 @@ def execute_test(**kwargs):
     server = None
     if start_server:
         server = subprocess.Popen(list(map(str, server_args)))
-        time.sleep(0.1)
+        time.sleep(0.4)
         assert server.poll() is None, "Server process died prematurely!"
 
     try:

tests/jepsen/src/jepsen/memgraph/bank.clj

@@ -164,4 +164,4 @@
                 {:bank     (bank-checker)
                  :timeline (timeline/html)})
    :generator (c/replication-gen (gen/mix [read-balances valid-transfer]))
-   :final-generator (gen/once read-balances)})
+   :final-generator {:gen (gen/once read-balances) :recovery-time 20}})

tests/jepsen/src/jepsen/memgraph/core.clj

@@ -22,7 +22,7 @@
   "A map of workload names to functions that can take opts and construct
    workloads."
    {:bank       bank/workload
-    :sequential sequential/workload
+    ;; :sequential sequential/workload (T0532-MG)
     :large      large/workload})
 
 (def nemesis-configuration
@@ -45,8 +45,8 @@
                                (gen/log "Healing cluster.")
                                (gen/nemesis (:final-generator nemesis))
                                (gen/log "Waiting for recovery")
-                               (gen/sleep 20)
-                               (gen/clients final-generator))
+                               (gen/sleep (:recovery-time final-generator))
+                               (gen/clients (:gen final-generator)))
                    gen)]
     (merge tests/noop-test
            opts

tests/jepsen/src/jepsen/memgraph/large.clj

@@ -103,4 +103,4 @@
                :timeline (timeline/html)})
    :generator (c/replication-gen
                 (gen/mix [read-nodes add-nodes]))
-   :final-generator (gen/once read-nodes)})
+   :final-generator {:gen (gen/once read-nodes) :recovery-time 40}})

tests/unit/bolt_session.cpp

@@ -383,15 +383,16 @@ TEST(BoltSession, ExecuteRunWrongMarker) {
 }
 
 TEST(BoltSession, ExecuteRunMissingData) {
+  std::array<uint8_t, 6> run_req_without_parameters{
+      run_req_header[0], run_req_header[1], run_req_header[2], 0x00, 0x00, 0x00};
   // test lengths, they test the following situations:
   // missing header data, missing query data, missing parameters
-  int len[] = {1, 2, 37};
-
+  int len[] = {1, 2, run_req_without_parameters.size()};
   for (int i = 0; i < 3; ++i) {
     INIT_VARS;
     ExecuteHandshake(input_stream, session, output);
     ExecuteInit(input_stream, session, output);
-    ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_header, len[i]), SessionException);
+    ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_without_parameters.data(), len[i]), SessionException);
 
     ASSERT_EQ(session.state_, State::Close);
     CheckFailureMessage(output);
@@ -871,7 +872,7 @@ TEST(BoltSession, Noop) {
     CheckFailureMessage(output);
 
     session.state_ = State::Result;
-    ExecuteCommand(input_stream, session, pullall_req, sizeof(v4::pullall_req));
+    ExecuteCommand(input_stream, session, pullall_req, sizeof(pullall_req));
     CheckSuccessMessage(output);
 
     ASSERT_THROW(ExecuteCommand(input_stream, session, v4_1::noop, sizeof(v4_1::noop)), SessionException);

tests/unit/cypher_main_visitor.cpp

@@ -2053,6 +2053,14 @@ TEST_P(CypherMainVisitorTest, GrantPrivilege) {
                    {AuthQuery::Privilege::CONSTRAINT});
   check_auth_query(&ast_generator, "GRANT DUMP TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
                    {AuthQuery::Privilege::DUMP});
+  check_auth_query(&ast_generator, "GRANT REPLICATION TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
+                   {AuthQuery::Privilege::REPLICATION});
+  check_auth_query(&ast_generator, "GRANT LOCK_PATH TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
+                   {AuthQuery::Privilege::LOCK_PATH});
+  check_auth_query(&ast_generator, "GRANT READ_FILE TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
+                   {AuthQuery::Privilege::READ_FILE});
+  check_auth_query(&ast_generator, "GRANT FREE_MEMORY TO user", AuthQuery::Action::GRANT_PRIVILEGE, "", "", "user", {},
+                   {AuthQuery::Privilege::FREE_MEMORY});
 }
 
 TEST_P(CypherMainVisitorTest, DenyPrivilege) {

tests/unit/query_plan.cpp

@@ -73,6 +73,12 @@ class TestPlanner : public ::testing::Test {};
 
 using PlannerTypes = ::testing::Types<Planner>;
 
+void DeleteListContent(std::list<BaseOpChecker *> *list) {
+  for (BaseOpChecker *ptr : *list) {
+    delete ptr;
+  }
+}
+
 TYPED_TEST_CASE(TestPlanner, PlannerTypes);
 
 TYPED_TEST(TestPlanner, MatchNodeReturn) {
@@ -223,6 +229,7 @@ TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) {
   auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
   std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()};
   CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce());
+  DeleteListContent(&optional);
 }
 
 TYPED_TEST(TestPlanner, MatchWhereReturn) {
@@ -549,10 +556,8 @@ TYPED_TEST(TestPlanner, MatchMerge) {
   auto acc = ExpectAccumulate({symbol_table.at(*ident_n)});
   auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
   CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce());
-  for (auto &op : on_match) delete op;
-  on_match.clear();
-  for (auto &op : on_create) delete op;
-  on_create.clear();
+  DeleteListContent(&on_match);
+  DeleteListContent(&on_create);
 }
 
 TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
@@ -564,6 +569,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
                                    WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r")));
   std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()};
   CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce());
+  DeleteListContent(&optional);
 }
 
 TYPED_TEST(TestPlanner, MatchUnwindReturn) {
@@ -705,6 +711,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) {
   // optional ScanAll.
   std::list<BaseOpChecker *> optional{new ExpectFilter(), new ExpectScanAll()};
   CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce());
+  DeleteListContent(&optional);
 }
 
 TYPED_TEST(TestPlanner, MatchReturnAsterisk) {
@@ -763,8 +770,8 @@ TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) {
   std::list<BaseOpChecker *> on_match{new ExpectScanAll(), new ExpectFilter()};
   std::list<BaseOpChecker *> on_create{new ExpectCreateNode()};
   CheckPlan<TypeParam>(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create));
-  for (auto &op : on_match) delete op;
-  for (auto &op : on_create) delete op;
+  DeleteListContent(&on_match);
+  DeleteListContent(&on_create);
 }
 
 TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
@@ -774,6 +781,7 @@ TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
       QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n")));
   std::list<BaseOpChecker *> optional{new ExpectScanAll()};
   CheckPlan<TypeParam>(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce());
+  DeleteListContent(&optional);
 }
 
 TYPED_TEST(TestPlanner, FunctionAggregationReturn) {

tests/unit/query_procedure_mgp_module.cpp

@@ -5,6 +5,8 @@
 
 #include "query/procedure/mg_procedure_impl.hpp"
 
+#include "test_utils.hpp"
+
 static void DummyCallback(const mgp_list *, const mgp_graph *, mgp_result *, mgp_memory *) {}
 
 TEST(Module, InvalidProcedureRegistration) {
@@ -53,7 +55,8 @@ TEST(Module, ProcedureSignature) {
   CheckSignature(proc, "proc() :: ()");
   mgp_proc_add_arg(proc, "arg1", mgp_type_number());
   CheckSignature(proc, "proc(arg1 :: NUMBER) :: ()");
-  mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
+  mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
+                       test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
   CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: ()");
   mgp_proc_add_result(proc, "res1", mgp_type_list(mgp_type_int()));
   CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: (res1 :: LIST OF INTEGER)");
@@ -69,7 +72,8 @@ TEST(Module, ProcedureSignature) {
                  "(res1 :: LIST OF INTEGER, DEPRECATED res2 :: STRING)");
   EXPECT_FALSE(mgp_proc_add_result(proc, "res2", mgp_type_any()));
   EXPECT_FALSE(mgp_proc_add_deprecated_result(proc, "res1", mgp_type_any()));
-  mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(), mgp_value_make_string("string=\"value\"", &memory));
+  mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(),
+                       test_utils::CreateValueOwningPtr(mgp_value_make_string("string=\"value\"", &memory)).get());
   CheckSignature(proc,
                  "proc(arg1 :: NUMBER, opt1 = Null :: ANY?, "
                  "opt2 = \"string=\\\"value\\\"\" :: STRING) :: "
@@ -80,6 +84,7 @@ TEST(Module, ProcedureSignatureOnlyOptArg) {
   mgp_memory memory{utils::NewDeleteResource()};
   mgp_module module(utils::NewDeleteResource());
   auto *proc = mgp_module_add_read_procedure(&module, "proc", DummyCallback);
-  mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
+  mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
+                       test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
   CheckSignature(proc, "proc(opt1 = Null :: ANY?) :: ()");
 }

tests/unit/query_procedure_mgp_type.cpp

@@ -1,7 +1,13 @@
+#include <functional>
+#include <memory>
+#include <utility>
+
 #include <gtest/gtest.h>
 
 #include "query/procedure/mg_procedure_impl.hpp"
 
+#include "test_utils.hpp"
+
 TEST(CypherType, PresentableNameSimpleTypes) {
   EXPECT_EQ(mgp_type_any()->impl->GetPresentableName(), "ANY");
   EXPECT_EQ(mgp_type_bool()->impl->GetPresentableName(), "BOOLEAN");
@@ -66,6 +72,7 @@ TEST(CypherType, NullSatisfiesType) {
         EXPECT_TRUE(null_type->impl->SatisfiesType(tv_null));
       }
     }
+    mgp_value_destroy(mgp_null);
   }
 }
 
@@ -101,6 +108,7 @@ TEST(CypherType, BoolSatisfiesType) {
   CheckNotSatisfiesTypesAndListAndNullable(mgp_bool, tv_bool,
                                            {mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
                                             mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_bool);
 }
 
 TEST(CypherType, IntSatisfiesType) {
@@ -111,6 +119,7 @@ TEST(CypherType, IntSatisfiesType) {
   CheckNotSatisfiesTypesAndListAndNullable(mgp_int, tv_int,
                                            {mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
                                             mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_int);
 }
 
 TEST(CypherType, DoubleSatisfiesType) {
@@ -121,6 +130,7 @@ TEST(CypherType, DoubleSatisfiesType) {
   CheckNotSatisfiesTypesAndListAndNullable(mgp_double, tv_double,
                                            {mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_map(),
                                             mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_double);
 }
 
 TEST(CypherType, StringSatisfiesType) {
@@ -131,12 +141,13 @@ TEST(CypherType, StringSatisfiesType) {
   CheckNotSatisfiesTypesAndListAndNullable(mgp_string, tv_string,
                                            {mgp_type_bool(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
                                             mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_string);
 }
 
 TEST(CypherType, MapSatisfiesType) {
   mgp_memory memory{utils::NewDeleteResource()};
   auto *map = mgp_map_make_empty(&memory);
-  mgp_map_insert(map, "key", mgp_value_make_int(42, &memory));
+  mgp_map_insert(map, "key", test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get());
   auto *mgp_map_v = mgp_value_make_map(map);
   const query::TypedValue tv_map(std::map<std::string, query::TypedValue>{{"key", query::TypedValue(42)}});
   CheckSatisfiesTypesAndNullable(mgp_map_v, tv_map, {mgp_type_any(), mgp_type_map()});
@@ -144,6 +155,7 @@ TEST(CypherType, MapSatisfiesType) {
       mgp_map_v, tv_map,
       {mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_node(),
        mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_map_v);
 }
 
 TEST(CypherType, VertexSatisfiesType) {
@@ -160,6 +172,7 @@ TEST(CypherType, VertexSatisfiesType) {
   CheckNotSatisfiesTypesAndListAndNullable(mgp_vertex_v, tv_vertex,
                                            {mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
                                             mgp_type_number(), mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_vertex_v);
 }
 
 TEST(CypherType, EdgeSatisfiesType) {
@@ -178,6 +191,7 @@ TEST(CypherType, EdgeSatisfiesType) {
   CheckNotSatisfiesTypesAndListAndNullable(mgp_edge_v, tv_edge,
                                            {mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
                                             mgp_type_number(), mgp_type_node(), mgp_type_path()});
+  mgp_value_destroy(mgp_edge_v);
 }
 
 TEST(CypherType, PathSatisfiesType) {
@@ -190,9 +204,13 @@ TEST(CypherType, PathSatisfiesType) {
   mgp_memory memory{utils::NewDeleteResource()};
   utils::Allocator<mgp_path> alloc(memory.impl);
   mgp_graph graph{&dba, storage::View::NEW};
-  auto *path = mgp_path_make_with_start(alloc.new_object<mgp_vertex>(v1, &graph), &memory);
+  auto *mgp_vertex_v = alloc.new_object<mgp_vertex>(v1, &graph);
+  auto path = mgp_path_make_with_start(mgp_vertex_v, &memory);
   ASSERT_TRUE(path);
-  ASSERT_TRUE(mgp_path_expand(path, alloc.new_object<mgp_edge>(edge, &graph)));
+  alloc.delete_object(mgp_vertex_v);
+  auto mgp_edge_v = alloc.new_object<mgp_edge>(edge, &graph);
+  ASSERT_TRUE(mgp_path_expand(path, mgp_edge_v));
+  alloc.delete_object(mgp_edge_v);
   auto *mgp_path_v = mgp_value_make_path(path);
   const query::TypedValue tv_path(query::Path(v1, edge, v2));
   CheckSatisfiesTypesAndNullable(mgp_path_v, tv_path, {mgp_type_any(), mgp_type_path()});
@@ -200,6 +218,7 @@ TEST(CypherType, PathSatisfiesType) {
       mgp_path_v, tv_path,
       {mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
        mgp_type_node(), mgp_type_relationship()});
+  mgp_value_destroy(mgp_path_v);
 }
 
 static std::vector<const mgp_type *> MakeListTypes(const std::vector<const mgp_type *> &element_types) {
@@ -224,6 +243,7 @@ TEST(CypherType, EmptyListSatisfiesType) {
   auto all_types = MakeListTypes(primitive_types);
   all_types.push_back(mgp_type_any());
   CheckSatisfiesTypesAndNullable(mgp_list_v, tv_list, all_types);
+  mgp_value_destroy(mgp_list_v);
 }
 
 TEST(CypherType, ListOfIntSatisfiesType) {
@@ -233,7 +253,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
   auto *mgp_list_v = mgp_value_make_list(list);
   query::TypedValue tv_list(std::vector<query::TypedValue>{});
   for (int64_t i = 0; i < elem_count; ++i) {
-    ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(i, &memory)));
+    ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(i, &memory)).get()));
     tv_list.ValueList().emplace_back(i);
     auto valid_types = MakeListTypes({mgp_type_any(), mgp_type_int(), mgp_type_number()});
     valid_types.push_back(mgp_type_any());
@@ -242,6 +262,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
                                              {mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
                                               mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
   }
+  mgp_value_destroy(mgp_list_v);
 }
 
 TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
@@ -251,10 +272,10 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
   auto *mgp_list_v = mgp_value_make_list(list);
   query::TypedValue tv_list(std::vector<query::TypedValue>{});
   // Add an int
-  ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(42, &memory)));
+  ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get()));
   tv_list.ValueList().emplace_back(42);
   // Add a boolean
-  ASSERT_TRUE(mgp_list_append(list, mgp_value_make_bool(1, &memory)));
+  ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_bool(1, &memory)).get()));
   tv_list.ValueList().emplace_back(true);
   auto valid_types = MakeListTypes({mgp_type_any()});
   valid_types.push_back(mgp_type_any());
@@ -264,6 +285,7 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
       mgp_list_v, tv_list,
       {mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
        mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
+  mgp_value_destroy(mgp_list_v);
 }
 
 TEST(CypherType, ListOfNullSatisfiesType) {
@@ -271,7 +293,7 @@ TEST(CypherType, ListOfNullSatisfiesType) {
   auto *list = mgp_list_make_empty(1, &memory);
   auto *mgp_list_v = mgp_value_make_list(list);
   query::TypedValue tv_list(std::vector<query::TypedValue>{});
-  ASSERT_TRUE(mgp_list_append(list, mgp_value_make_null(&memory)));
+  ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get()));
   tv_list.ValueList().emplace_back();
   // List with Null satisfies all nullable list element types
   std::vector<const mgp_type *> primitive_types{
@@ -295,4 +317,5 @@ TEST(CypherType, ListOfNullSatisfiesType) {
     EXPECT_FALSE(null_type->impl->SatisfiesType(*mgp_list_v)) << null_type->impl->GetPresentableName();
     EXPECT_FALSE(null_type->impl->SatisfiesType(tv_list));
   }
+  mgp_value_destroy(mgp_list_v);
 }

tests/unit/query_procedure_py_module.cpp

@@ -254,6 +254,7 @@ TEST(PyModule, PyObjectToMgpValue) {
   const mgp_value *v2 = mgp_map_at(map, "four");
   ASSERT_TRUE(mgp_value_is_double(v2));
   EXPECT_EQ(mgp_value_get_double(v2), 4.0);
+  mgp_value_destroy(value);
 }
 
 int main(int argc, char **argv) {

tests/unit/query_required_privileges.cpp

@@ -1,6 +1,7 @@
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
+#include "query/frontend/ast/ast_visitor.hpp"
 #include "query/frontend/semantic/required_privileges.hpp"
 #include "storage/v2/id_types.hpp"
 
@@ -131,3 +132,20 @@ TEST_F(TestPrivilegeExtractor, DumpDatabase) {
   auto *query = storage.Create<DumpQuery>();
   EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::DUMP));
 }
+
+TEST_F(TestPrivilegeExtractor, ReadFile) {
+  auto load_csv = storage.Create<LoadCsv>();
+  load_csv->row_var_ = IDENT("row");
+  auto *query = QUERY(SINGLE_QUERY(load_csv));
+  EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::READ_FILE));
+}
+
+TEST_F(TestPrivilegeExtractor, LockPathQuery) {
+  auto *query = storage.Create<LockPathQuery>();
+  EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::LOCK_PATH));
+}
+
+TEST_F(TestPrivilegeExtractor, FreeMemoryQuery) {
+  auto *query = storage.Create<FreeMemoryQuery>();
+  EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::FREE_MEMORY));
+}

tests/unit/storage_v2_durability.cpp

@@ -709,14 +709,15 @@ TEST_P(DurabilityTest, SnapshotFallback) {
         {.items = {.properties_on_edges = GetParam()},
          .durability = {.storage_directory = storage_directory,
                         .snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT,
-                        .snapshot_interval = std::chrono::milliseconds(2000)}});
+                        .snapshot_interval = std::chrono::milliseconds(3000)}});
     CreateBaseDataset(&store, GetParam());
-    std::this_thread::sleep_for(std::chrono::milliseconds(2500));
+    std::this_thread::sleep_for(std::chrono::milliseconds(3500));
+    ASSERT_EQ(GetSnapshotsList().size(), 1);
     CreateExtendedDataset(&store);
-    std::this_thread::sleep_for(std::chrono::milliseconds(2500));
+    std::this_thread::sleep_for(std::chrono::milliseconds(3000));
   }
 
-  ASSERT_GE(GetSnapshotsList().size(), 2);
+  ASSERT_EQ(GetSnapshotsList().size(), 2);
   ASSERT_EQ(GetBackupSnapshotsList().size(), 0);
   ASSERT_EQ(GetWalsList().size(), 0);
   ASSERT_EQ(GetBackupWalsList().size(), 0);
@@ -724,7 +725,7 @@ TEST_P(DurabilityTest, SnapshotFallback) {
   // Destroy last snapshot.
   {
     auto snapshots = GetSnapshotsList();
-    ASSERT_GE(snapshots.size(), 2);
+    ASSERT_EQ(snapshots.size(), 2);
     DestroySnapshot(*snapshots.begin());
   }
 

tests/unit/test_utils.hpp

@@ -0,0 +1,9 @@
+#include <memory>
+
+#include "query/procedure/mg_procedure_impl.hpp"
+
+namespace test_utils {
+using MgpValueOwningPtr = std::unique_ptr<mgp_value, void (*)(mgp_value *)>;
+
+MgpValueOwningPtr CreateValueOwningPtr(mgp_value *value) { return MgpValueOwningPtr(value, &mgp_value_destroy); }
+}  // namespace test_utils

tests/unit/typed_value.cpp

@@ -397,8 +397,8 @@ TEST_F(TypedValueLogicTest, LogicalXor) {
 
 // NOLINTNEXTLINE(hicpp-special-member-functions)
 TEST_F(AllTypesFixture, ConstructionWithMemoryResource) {
-  std::vector<TypedValue> values_with_custom_memory;
   utils::MonotonicBufferResource monotonic_memory(1024);
+  std::vector<TypedValue> values_with_custom_memory;
   for (const auto &value : values_) {
     EXPECT_EQ(value.GetMemoryResource(), utils::NewDeleteResource());
     TypedValue copy_constructed_value(value, &monotonic_memory);

tests/unit/utils_csv_parsing.cpp

@@ -4,7 +4,7 @@
 
 #include "utils/string.hpp"
 
-class CsvReaderTest : public ::testing::Test {
+class CsvReaderTest : public ::testing::TestWithParam<const char *> {
  protected:
   const std::filesystem::path csv_directory{std::filesystem::temp_directory_path() / "csv_testing"};
 
@@ -30,7 +30,9 @@ class CsvReaderTest : public ::testing::Test {
 namespace {
 class FileWriter {
  public:
-  explicit FileWriter(const std::filesystem::path path) { stream_.open(path); }
+  explicit FileWriter(const std::filesystem::path path, std::string newline = "\n") : newline_{std::move(newline)} {
+    stream_.open(path);
+  }
 
   FileWriter(const FileWriter &) = delete;
   FileWriter &operator=(const FileWriter &) = delete;
@@ -45,7 +47,7 @@ class FileWriter {
       return 0;
     }
 
-    stream_ << line << std::endl;
+    stream_ << line << newline_;
 
     // including the newline character
     return line.size() + 1;
@@ -53,6 +55,7 @@ class FileWriter {
 
  private:
   std::ofstream stream_;
+  std::string newline_;
 };
 
 std::string CreateRow(const std::vector<std::string> &columns, const std::string_view delim) {
@@ -69,10 +72,10 @@ auto ToPmrColumns(const std::vector<std::string> &columns) {
 
 }  // namespace
 
-TEST_F(CsvReaderTest, CommaDelimiter) {
+TEST_P(CsvReaderTest, CommaDelimiter) {
   // create a file with a single valid row;
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   const std::vector<std::string> columns{"A", "B", "C"};
   writer.WriteLine(CreateRow(columns, ","));
@@ -93,9 +96,9 @@ TEST_F(CsvReaderTest, CommaDelimiter) {
   ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
 }
 
-TEST_F(CsvReaderTest, SemicolonDelimiter) {
+TEST_P(CsvReaderTest, SemicolonDelimiter) {
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   utils::MemoryResource *mem(utils::NewDeleteResource());
 
@@ -116,12 +119,12 @@ TEST_F(CsvReaderTest, SemicolonDelimiter) {
   ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
 }
 
-TEST_F(CsvReaderTest, SkipBad) {
+TEST_P(CsvReaderTest, SkipBad) {
   // create a file with invalid first two rows (containing a string with a
   // missing closing quote);
   // the last row is valid;
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   utils::MemoryResource *mem(utils::NewDeleteResource());
 
@@ -161,11 +164,11 @@ TEST_F(CsvReaderTest, SkipBad) {
   }
 }
 
-TEST_F(CsvReaderTest, AllRowsValid) {
+TEST_P(CsvReaderTest, AllRowsValid) {
   // create a file with all rows valid;
   // parser should return 'std::nullopt'
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   utils::MemoryResource *mem(utils::NewDeleteResource());
 
@@ -190,11 +193,11 @@ TEST_F(CsvReaderTest, AllRowsValid) {
   }
 }
 
-TEST_F(CsvReaderTest, SkipAllRows) {
+TEST_P(CsvReaderTest, SkipAllRows) {
   // create a file with all rows invalid (containing a string with a missing closing quote);
   // parser should return 'std::nullopt'
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   utils::MemoryResource *mem(utils::NewDeleteResource());
 
@@ -217,9 +220,9 @@ TEST_F(CsvReaderTest, SkipAllRows) {
   ASSERT_EQ(parsed_row, std::nullopt);
 }
 
-TEST_F(CsvReaderTest, WithHeader) {
+TEST_P(CsvReaderTest, WithHeader) {
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   utils::MemoryResource *mem(utils::NewDeleteResource());
 
@@ -249,12 +252,12 @@ TEST_F(CsvReaderTest, WithHeader) {
   }
 }
 
-TEST_F(CsvReaderTest, MultilineQuotedString) {
+TEST_P(CsvReaderTest, MultilineQuotedString) {
   // create a file with first row valid and the second row containing a quoted
   // string spanning two lines;
   // parser should return two valid rows
   const auto filepath = csv_directory / "bla.csv";
-  auto writer = FileWriter(filepath);
+  auto writer = FileWriter(filepath, GetParam());
 
   utils::MemoryResource *mem(utils::NewDeleteResource());
 
@@ -283,3 +286,37 @@ TEST_F(CsvReaderTest, MultilineQuotedString) {
   parsed_row = reader.GetNextRow(mem);
   ASSERT_EQ(*parsed_row, ToPmrColumns(expected_multiline));
 }
+
+TEST_P(CsvReaderTest, EmptyColumns) {
+  // create a file with all rows valid;
+  // parser should return 'std::nullopt'
+  const auto filepath = csv_directory / "bla.csv";
+  auto writer = FileWriter(filepath, GetParam());
+
+  utils::MemoryResource *mem(utils::NewDeleteResource());
+
+  const utils::pmr::string delimiter{",", mem};
+  const utils::pmr::string quote{"\"", mem};
+
+  std::vector<std::vector<std::string>> expected_rows{{"", "B", "C"}, {"A", "", "C"}, {"A", "B", ""}};
+
+  for (const auto &row : expected_rows) {
+    writer.WriteLine(CreateRow(row, delimiter));
+  }
+
+  writer.Close();
+
+  const bool with_header = false;
+  const bool ignore_bad = false;
+  const csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
+  auto reader = csv::Reader(filepath, cfg);
+
+  for (const auto &expected_row : expected_rows) {
+    const auto pmr_expected_row = ToPmrColumns(expected_row);
+    const auto parsed_row = reader.GetNextRow(mem);
+    ASSERT_TRUE(parsed_row.has_value());
+    ASSERT_EQ(*parsed_row, pmr_expected_row);
+  }
+}
+
+INSTANTIATE_TEST_CASE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n"));

tests/unit/utils_memory.cpp

@@ -12,6 +12,7 @@ class TestMemory final : public utils::MemoryResource {
   size_t delete_count_{0};
 
  private:
+  static constexpr size_t kPadSize = 32;
   void *DoAllocate(size_t bytes, size_t alignment) override {
     new_count_++;
     EXPECT_TRUE(alignment != 0U && (alignment & (alignment - 1U)) == 0U) << "Alignment must be power of 2";
@@ -20,11 +21,11 @@ class TestMemory final : public utils::MemoryResource {
     EXPECT_TRUE(bytes + pad_size > bytes) << "TestMemory size overflow";
     EXPECT_TRUE(bytes + pad_size + alignment > bytes + alignment) << "TestMemory size overflow";
     EXPECT_TRUE(2U * alignment > alignment) << "TestMemory alignment overflow";
-    // Allocate a block containing extra alignment and pad_size bytes, but
+    // Allocate a block containing extra alignment and kPadSize bytes, but
     // aligned to 2 * alignment. Then we can offset the ptr so that it's never
     // aligned to 2 * alignment. This ought to make allocator alignment issues
     // more obvious.
-    void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + pad_size, 2U * alignment);
+    void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + kPadSize, 2U * alignment);
     // Clear allocated memory to 0xFF, marking the invalid region.
     memset(ptr, 0xFF, alignment + bytes + pad_size);
     // Offset the ptr so it's not aligned to 2 * alignment, but still aligned to
@@ -39,7 +40,8 @@ class TestMemory final : public utils::MemoryResource {
   void DoDeallocate(void *ptr, size_t bytes, size_t alignment) override {
     delete_count_++;
     // Deallocate the original ptr, before alignment adjustment.
-    return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, bytes, alignment);
+    return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, alignment + bytes + kPadSize,
+                                                  2U * alignment);
   }
 
   bool DoIsEqual(const utils::MemoryResource &other) const noexcept override { return this == &other; }

tools/bench-graph-client/main.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+
+"""
+Bench Graph client responsible for sending benchmarking data in JSON format to
+the Bench Graph server.
+"""
+
+import json
+import logging
+import os
+import requests
+import subprocess
+from datetime import datetime
+from argparse import ArgumentParser
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "")
+GITHUB_SHA = os.getenv("GITHUB_SHA", "")
+GITHUB_REF = os.getenv("GITHUB_REF", "")
+
+BENCH_GRAPH_SERVER_ENDPOINT = os.getenv(
+    "BENCH_GRAPH_SERVER_ENDPOINT",
+    "http://bench-graph-api:9001")
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    argp = ArgumentParser(description=__doc__)
+    argp.add_argument("--benchmark-name", type=str, required=True)
+    argp.add_argument("--benchmark-results-path", type=str, required=True)
+    argp.add_argument("--github-run-id", type=int, required=True)
+    argp.add_argument("--github-run-number", type=int, required=True)
+    return argp.parse_args()
+
+
+def post_measurement(args):
+    with open(args.benchmark_results_path, "r") as f:
+        data = json.load(f)
+        timestamp = datetime.now().timestamp()
+        branch = subprocess.run(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+            stdout=subprocess.PIPE,
+            check=True).stdout.decode("utf-8").strip()
+        req = requests.post(
+            f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements",
+            json={
+                "name": args.benchmark_name,
+                "timestamp": timestamp,
+                "git_repo": GITHUB_REPOSITORY,
+                "git_ref": GITHUB_REF,
+                "git_sha": GITHUB_SHA,
+                "github_run_id": args.github_run_id,
+                "github_run_number": args.github_run_number,
+                "results": data,
+                "git_branch": branch},
+            timeout=1)
+        assert req.status_code == 200, \
+            f"Uploading {args.benchmark_name} data failed."
+        log.info(f"{args.benchmark_name} data sent to "
+                 f"{BENCH_GRAPH_SERVER_ENDPOINT}")
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    logging.basicConfig(level=logging.INFO)
+    post_measurement(args)

tools/bench-graph-client/requirements.txt

@@ -0,0 +1 @@
+requests==2.25.1

tools/github/clang-tidy/clang-tidy-diff.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+#
+#===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===-----------------------------------------------------------------------===#
+
+r"""
+ClangTidy Diff Checker
+======================
+
+This script reads input from a unified diff, runs clang-tidy on all changed
+files and outputs clang-tidy warnings in changed lines only. This is useful to
+detect clang-tidy regressions in the lines touched by a specific patch.
+Example usage for git/svn users:
+
+  git diff -U0 HEAD^ | clang-tidy-diff.py -p1
+  svn diff --diff-cmd=diff -x-U0 | \
+      clang-tidy-diff.py -fix -checks=-*,modernize-use-override
+
+"""
+
+import argparse
+import glob
+import json
+import multiprocessing
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import traceback
+
+try:
+  import yaml
+except ImportError:
+  yaml = None
+
+is_py2 = sys.version[0] == '2'
+
+if is_py2:
+    import Queue as queue
+else:
+    import queue as queue
+
+
+def run_tidy(task_queue, lock, timeout):
+  watchdog = None
+  while True:
+    command = task_queue.get()
+    try:
+      proc = subprocess.Popen(command,
+                              stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+
+      if timeout is not None:
+        watchdog = threading.Timer(timeout, proc.kill)
+        watchdog.start()
+
+      stdout, stderr = proc.communicate()
+
+      with lock:
+        sys.stdout.write(stdout.decode('utf-8') + '\n')
+        sys.stdout.flush()
+        if stderr:
+          sys.stderr.write(stderr.decode('utf-8') + '\n')
+          sys.stderr.flush()
+    except Exception as e:
+      with lock:
+        sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
+    finally:
+      with lock:
+        if not (timeout is None or watchdog is None):
+          if not watchdog.is_alive():
+              sys.stderr.write('Terminated by timeout: ' +
+                               ' '.join(command) + '\n')
+          watchdog.cancel()
+      task_queue.task_done()
+
+
+def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
+  for _ in range(max_tasks):
+    t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
+    t.daemon = True
+    t.start()
+
+
+def merge_replacement_files(tmpdir, mergefile):
+  """Merge all replacement files in a directory into a single file"""
+  # The fixes suggested by clang-tidy >= 4.0.0 are given under
+  # the top level key 'Diagnostics' in the output yaml files
+  mergekey = "Diagnostics"
+  merged = []
+  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
+    content = yaml.safe_load(open(replacefile, 'r'))
+    if not content:
+      continue # Skip empty files.
+    merged.extend(content.get(mergekey, []))
+
+  if merged:
+    # MainSourceFile: The key is required by the definition inside
+    # include/clang/Tooling/ReplacementsYaml.h, but the value
+    # is actually never used inside clang-apply-replacements,
+    # so we set it to '' here.
+    output = {'MainSourceFile': '', mergekey: merged}
+    with open(mergefile, 'w') as out:
+      yaml.safe_dump(output, out)
+  else:
+    # Empty the file:
+    open(mergefile, 'w').close()
+
+
+def main():
+  parser = argparse.ArgumentParser(description=
+                                   'Run clang-tidy against changed files, and '
+                                   'output diagnostics only for modified '
+                                   'lines.')
+  parser.add_argument('-clang-tidy-binary', metavar='PATH',
+                      default='clang-tidy',
+                      help='path to clang-tidy binary')
+  parser.add_argument('-p', metavar='NUM', default=0,
+                      help='strip the smallest prefix containing P slashes')
+  parser.add_argument('-regex', metavar='PATTERN', default=None,
+                      help='custom pattern selecting file paths to check '
+                      '(case sensitive, overrides -iregex)')
+  parser.add_argument('-iregex', metavar='PATTERN', default=
+                      r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
+                      help='custom pattern selecting file paths to check '
+                      '(case insensitive, overridden by -regex)')
+  parser.add_argument('-j', type=int, default=1,
+                      help='number of tidy instances to be run in parallel.')
+  parser.add_argument('-timeout', type=int, default=None,
+                      help='timeout per each file in seconds.')
+  parser.add_argument('-fix', action='store_true', default=False,
+                      help='apply suggested fixes')
+  parser.add_argument('-checks',
+                      help='checks filter, when not specified, use clang-tidy '
+                      'default',
+                      default='')
+  parser.add_argument('-path', dest='build_path',
+                      help='Path used to read a compile command database.')
+  if yaml:
+    parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
+                        help='Create a yaml file to store suggested fixes in, '
+                        'which can be applied with clang-apply-replacements.')
+  parser.add_argument('-extra-arg', dest='extra_arg',
+                      action='append', default=[],
+                      help='Additional argument to append to the compiler '
+                      'command line.')
+  parser.add_argument('-extra-arg-before', dest='extra_arg_before',
+                      action='append', default=[],
+                      help='Additional argument to prepend to the compiler '
+                      'command line.')
+  parser.add_argument('-quiet', action='store_true', default=False,
+                      help='Run clang-tidy in quiet mode')
+  clang_tidy_args = []
+  argv = sys.argv[1:]
+  if '--' in argv:
+    clang_tidy_args.extend(argv[argv.index('--'):])
+    argv = argv[:argv.index('--')]
+
+  args = parser.parse_args(argv)
+
+  # Extract changed lines for each file.
+  filename = None
+  lines_by_file = {}
+  for line in sys.stdin:
+    match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
+    if match:
+      filename = match.group(2)
+    if filename is None:
+      continue
+
+    if args.regex is not None:
+      if not re.match('^%s$' % args.regex, filename):
+        continue
+    else:
+      if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
+        continue
+
+    match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
+    if match:
+      start_line = int(match.group(1))
+      line_count = 1
+      if match.group(3):
+        line_count = int(match.group(3))
+      if line_count == 0:
+        continue
+      end_line = start_line + line_count - 1
+      lines_by_file.setdefault(filename, []).append([start_line, end_line])
+
+  if not any(lines_by_file):
+    print("No relevant changes found.")
+    sys.exit(0)
+
+  max_task_count = args.j
+  if max_task_count == 0:
+      max_task_count = multiprocessing.cpu_count()
+  max_task_count = min(len(lines_by_file), max_task_count)
+
+  tmpdir = None
+  if yaml and args.export_fixes:
+    tmpdir = tempfile.mkdtemp()
+
+  # Tasks for clang-tidy.
+  task_queue = queue.Queue(max_task_count)
+  # A lock for console output.
+  lock = threading.Lock()
+
+  # Run a pool of clang-tidy workers.
+  start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
+
+  # Form the common args list.
+  common_clang_tidy_args = []
+  if args.fix:
+    common_clang_tidy_args.append('-fix')
+  if args.checks != '':
+    common_clang_tidy_args.append('-checks=' + args.checks)
+  if args.quiet:
+    common_clang_tidy_args.append('-quiet')
+  if args.build_path is not None:
+    common_clang_tidy_args.append('-p=%s' % args.build_path)
+  for arg in args.extra_arg:
+    common_clang_tidy_args.append('-extra-arg=%s' % arg)
+  for arg in args.extra_arg_before:
+    common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
+
+  for name in lines_by_file:
+    line_filter_json = json.dumps(
+      [{"name": name, "lines": lines_by_file[name]}],
+      separators=(',', ':'))
+
+    # Run clang-tidy on files containing changes.
+    command = [args.clang_tidy_binary]
+    command.append('-line-filter=' + line_filter_json)
+    if yaml and args.export_fixes:
+      # Get a temporary file. We immediately close the handle so clang-tidy can
+      # overwrite it.
+      (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
+      os.close(handle)
+      command.append('-export-fixes=' + tmp_name)
+    command.extend(common_clang_tidy_args)
+    command.append(name)
+    command.extend(clang_tidy_args)
+
+    task_queue.put(command)
+
+  # Wait for all threads to be done.
+  task_queue.join()
+
+  if yaml and args.export_fixes:
+    print('Writing fixes to ' + args.export_fixes + ' ...')
+    try:
+      merge_replacement_files(tmpdir, args.export_fixes)
+    except:
+      sys.stderr.write('Error exporting fixes.\n')
+      traceback.print_exc()
+
+  if tmpdir:
+    shutil.rmtree(tmpdir)
+
+
+if __name__ == '__main__':
+  main()

tools/github/clang-tidy/count_errors.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# the first sort | uniq is necessary, because the same occurrence of the same error
+# can be reported from headers when they are included in multiple source files
+`dirname ${BASH_SOURCE[0]}`/grep_error_lines.sh |
+    sort | uniq |
+    sed -E 's/.*\[(.*)\]\r?$/\1/g' | # extract the check name from [check-name]
+    sort | uniq -c |                 # count each type of check
+    sort -nr                         # sort them into descending order

tools/github/clang-tidy/grep_error_lines.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Matches timestamp like "2021-03-25T17:06:42.2621697Z"
+TIMESTAMP_PATTERN="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z"
+
+# Matches absolute file pathes with line and column identifier like
+# "/opt/actions-runner/_work/memgraph/memgraph/src/utils/exceptions.hpp:71:11:"
+FILE_ABSOLUTE_PATH_PATTERN="/[^:]+:\d+:\d+:"
+
+ERROR_OR_WARNING_PATTERN="(error|warning):"
+
+grep -P "^($TIMESTAMP_PATTERN )?$FILE_ABSOLUTE_PATH_PATTERN $ERROR_OR_WARNING_PATTERN.*$"

tools/github/clang-tidy/run-clang-tidy.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+#
+#===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+#===-----------------------------------------------------------------------===#
+# FIXME: Integrate with clang-tidy-diff.py
+
+
+"""
+Parallel clang-tidy runner
+==========================
+
+Runs clang-tidy over all files in a compilation database. Requires clang-tidy
+and clang-apply-replacements in $PATH.
+
+Example invocations.
+- Run clang-tidy on all files in the current working directory with a default
+  set of checks and show warnings in the cpp files and all project headers.
+    run-clang-tidy.py $PWD
+
+- Fix all header guards.
+    run-clang-tidy.py -fix -checks=-*,llvm-header-guard
+
+- Fix all header guards included from clang-tidy and header guards
+  for clang-tidy headers.
+    run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
+                      -header-filter=extra/clang-tidy
+
+Compilation database setup:
+http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
+"""
+
+from __future__ import print_function
+
+import argparse
+import glob
+import json
+import multiprocessing
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import traceback
+
+try:
+  import yaml
+except ImportError:
+  yaml = None
+
+is_py2 = sys.version[0] == '2'
+
+if is_py2:
+    import Queue as queue
+else:
+    import queue as queue
+
+
+def find_compilation_database(path):
+  """Adjusts the directory until a compilation database is found."""
+  result = './'
+  while not os.path.isfile(os.path.join(result, path)):
+    if os.path.realpath(result) == '/':
+      print('Error: could not find compilation database.')
+      sys.exit(1)
+    result += '../'
+  return os.path.realpath(result)
+
+
+def make_absolute(f, directory):
+  if os.path.isabs(f):
+    return f
+  return os.path.normpath(os.path.join(directory, f))
+
+
+def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
+                        header_filter, allow_enabling_alpha_checkers,
+                        extra_arg, extra_arg_before, quiet, config):
+  """Gets a command line for clang-tidy."""
+  start = [clang_tidy_binary]
+  if allow_enabling_alpha_checkers:
+    start.append('-allow-enabling-analyzer-alpha-checkers')
+  if header_filter is not None:
+    start.append('-header-filter=' + header_filter)
+  if checks:
+    start.append('-checks=' + checks)
+  if tmpdir is not None:
+    start.append('-export-fixes')
+    # Get a temporary file. We immediately close the handle so clang-tidy can
+    # overwrite it.
+    (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
+    os.close(handle)
+    start.append(name)
+  for arg in extra_arg:
+      start.append('-extra-arg=%s' % arg)
+  for arg in extra_arg_before:
+      start.append('-extra-arg-before=%s' % arg)
+  start.append('-p=' + build_path)
+  if quiet:
+      start.append('-quiet')
+  if config:
+      start.append('-config=' + config)
+  start.append(f)
+  return start
+
+
+def merge_replacement_files(tmpdir, mergefile):
+  """Merge all replacement files in a directory into a single file"""
+  # The fixes suggested by clang-tidy >= 4.0.0 are given under
+  # the top level key 'Diagnostics' in the output yaml files
+  mergekey = "Diagnostics"
+  merged=[]
+  for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
+    content = yaml.safe_load(open(replacefile, 'r'))
+    if not content:
+      continue # Skip empty files.
+    merged.extend(content.get(mergekey, []))
+
+  if merged:
+    # MainSourceFile: The key is required by the definition inside
+    # include/clang/Tooling/ReplacementsYaml.h, but the value
+    # is actually never used inside clang-apply-replacements,
+    # so we set it to '' here.
+    output = {'MainSourceFile': '', mergekey: merged}
+    with open(mergefile, 'w') as out:
+      yaml.safe_dump(output, out)
+  else:
+    # Empty the file:
+    open(mergefile, 'w').close()
+
+
+def check_clang_apply_replacements_binary(args):
+  """Checks if invoking supplied clang-apply-replacements binary works."""
+  try:
+    subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
+  except:
+    print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
+          'binary correctly specified?', file=sys.stderr)
+    traceback.print_exc()
+    sys.exit(1)
+
+
+def apply_fixes(args, tmpdir):
+  """Calls clang-apply-fixes on a given directory."""
+  invocation = [args.clang_apply_replacements_binary]
+  if args.format:
+    invocation.append('-format')
+  if args.style:
+    invocation.append('-style=' + args.style)
+  invocation.append(tmpdir)
+  subprocess.call(invocation)
+
+
+def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
+  """Takes filenames out of queue and runs clang-tidy on them."""
+  while True:
+    name = queue.get()
+    invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
+                                     tmpdir, build_path, args.header_filter,
+                                     args.allow_enabling_alpha_checkers,
+                                     args.extra_arg, args.extra_arg_before,
+                                     args.quiet, args.config)
+
+    proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output, err = proc.communicate()
+    if proc.returncode != 0:
+      failed_files.append(name)
+    with lock:
+      sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
+      if len(err) > 0:
+        sys.stdout.flush()
+        sys.stderr.write(err.decode('utf-8'))
+    queue.task_done()
+
+
+def main():
+  parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
+                                   'in a compilation database. Requires '
+                                   'clang-tidy and clang-apply-replacements in '
+                                   '$PATH.')
+  parser.add_argument('-allow-enabling-alpha-checkers',
+                      action='store_true', help='allow alpha checkers from '
+                                                'clang-analyzer.')
+  parser.add_argument('-clang-tidy-binary', metavar='PATH',
+                      default='clang-tidy-11',
+                      help='path to clang-tidy binary')
+  parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
+                      default='clang-apply-replacements-11',
+                      help='path to clang-apply-replacements binary')
+  parser.add_argument('-checks', default=None,
+                      help='checks filter, when not specified, use clang-tidy '
+                      'default')
+  parser.add_argument('-config', default=None,
+                      help='Specifies a configuration in YAML/JSON format: '
+                      '  -config="{Checks: \'*\', '
+                      '                       CheckOptions: [{key: x, '
+                      '                                       value: y}]}" '
+                      'When the value is empty, clang-tidy will '
+                      'attempt to find a file named .clang-tidy for '
+                      'each source file in its parent directories.')
+  parser.add_argument('-header-filter', default=None,
+                      help='regular expression matching the names of the '
+                      'headers to output diagnostics from. Diagnostics from '
+                      'the main file of each translation unit are always '
+                      'displayed.')
+  if yaml:
+    parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
+                        help='Create a yaml file to store suggested fixes in, '
+                        'which can be applied with clang-apply-replacements.')
+  parser.add_argument('-j', type=int, default=0,
+                      help='number of tidy instances to be run in parallel.')
+  parser.add_argument('files', nargs='*', default=['.*'],
+                      help='files to be processed (regex on path)')
+  parser.add_argument('-fix', action='store_true', help='apply fix-its')
+  parser.add_argument('-format', action='store_true', help='Reformat code '
+                      'after applying fixes')
+  parser.add_argument('-style', default='file', help='The style of reformat '
+                      'code after applying fixes')
+  parser.add_argument('-p', dest='build_path',
+                      help='Path used to read a compile command database.')
+  parser.add_argument('-extra-arg', dest='extra_arg',
+                      action='append', default=[],
+                      help='Additional argument to append to the compiler '
+                      'command line.')
+  parser.add_argument('-extra-arg-before', dest='extra_arg_before',
+                      action='append', default=[],
+                      help='Additional argument to prepend to the compiler '
+                      'command line.')
+  parser.add_argument('-quiet', action='store_true',
+                      help='Run clang-tidy in quiet mode')
+  args = parser.parse_args()
+
+  db_path = 'compile_commands.json'
+
+  if args.build_path is not None:
+    build_path = args.build_path
+  else:
+    # Find our database
+    build_path = find_compilation_database(db_path)
+
+  try:
+    invocation = [args.clang_tidy_binary, '-list-checks']
+    if args.allow_enabling_alpha_checkers:
+      invocation.append('-allow-enabling-analyzer-alpha-checkers')
+    invocation.append('-p=' + build_path)
+    if args.checks:
+      invocation.append('-checks=' + args.checks)
+    invocation.append('-')
+    if args.quiet:
+      # Even with -quiet we still want to check if we can call clang-tidy.
+      with open(os.devnull, 'w') as dev_null:
+        subprocess.check_call(invocation, stdout=dev_null)
+    else:
+      subprocess.check_call(invocation)
+  except:
+    print("Unable to run clang-tidy.", file=sys.stderr)
+    sys.exit(1)
+
+  # Load the database and extract all files.
+  database = json.load(open(os.path.join(build_path, db_path)))
+  files = [make_absolute(entry['file'], entry['directory'])
+           for entry in database]
+
+  max_task = args.j
+  if max_task == 0:
+    max_task = multiprocessing.cpu_count()
+
+  tmpdir = None
+  if args.fix or (yaml and args.export_fixes):
+    check_clang_apply_replacements_binary(args)
+    tmpdir = tempfile.mkdtemp()
+
+  # Build up a big regexy filter from all command line arguments.
+  file_name_re = re.compile('|'.join(args.files))
+
+  return_code = 0
+  try:
+    # Spin up a bunch of tidy-launching threads.
+    task_queue = queue.Queue(max_task)
+    # List of files with a non-zero return code.
+    failed_files = []
+    lock = threading.Lock()
+    for _ in range(max_task):
+      t = threading.Thread(target=run_tidy,
+                           args=(args, tmpdir, build_path, task_queue, lock, failed_files))
+      t.daemon = True
+      t.start()
+
+    # Fill the queue with files.
+    for name in files:
+      if file_name_re.search(name):
+        task_queue.put(name)
+
+    # Wait for all threads to be done.
+    task_queue.join()
+    if len(failed_files):
+      return_code = 1
+
+  except KeyboardInterrupt:
+    # This is a sad hack. Unfortunately subprocess goes
+    # bonkers with ctrl-c and we start forking merrily.
+    print('\nCtrl-C detected, goodbye.')
+    if tmpdir:
+      shutil.rmtree(tmpdir)
+    os.kill(0, 9)
+
+  if yaml and args.export_fixes:
+    print('Writing fixes to ' + args.export_fixes + ' ...')
+    try:
+      merge_replacement_files(tmpdir, args.export_fixes)
+    except:
+      print('Error exporting fixes.\n', file=sys.stderr)
+      traceback.print_exc()
+      return_code=1
+
+  if args.fix:
+    print('Applying fixes ...')
+    try:
+      apply_fixes(args, tmpdir)
+    except:
+      print('Error applying fixes.\n', file=sys.stderr)
+      traceback.print_exc()
+      return_code = 1
+
+  if tmpdir:
+    shutil.rmtree(tmpdir)
+  sys.exit(return_code)
+
+
+if __name__ == '__main__':
+  main()

tools/lsan.supp

@@ -0,0 +1,12 @@
+leak:antlr4::atn::ArrayPredictionContext::ArrayPredictionContext
+leak:std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<antlr4::atn::SingletonPredictionContext, std::allocator<antlr4::atn::SingletonPredictionContext>, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&>(antlr4::atn::SingletonPredictionContext*&, std::_Sp_alloc_shared_tag<std::allocator<antlr4::atn::SingletonPredictionContext> >, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&)
+leak:antlr4::atn::PredictionContext::mergeSingletons(std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
+leak:void std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::PredictionContext> >(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::PredictionContext>*, std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > > >, std::shared_ptr<antlr4::atn::PredictionContext>&&)
+leak:antlr4::atn::ParserATNSimulator::closureCheckingStopState(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::ATNConfigSet*, std::unordered_set<std::shared_ptr<antlr4::atn::ATNConfig>, antlr4::atn::ATNConfig::Hasher, antlr4::atn::ATNConfig::Comparer, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >&, bool, bool, int, bool)
+leak:antlr4::atn::ParserATNSimulator::computeReachSet(antlr4::atn::ATNConfigSet*, unsigned long, bool)
+leak:std::_Hashtable<unsigned long, std::pair<unsigned long const, antlr4::atn::ATNConfig*>, std::allocator<std::pair<unsigned long const, antlr4::atn::ATNConfig*> >, std::__detail::_Select1st, std::equal_to<unsigned long>, std::hash<unsigned long>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node(unsigned long const&, unsigned long, unsigned long, std::__detail::_Hash_node<std::pair<unsigned long const, antlr4::atn::ATNConfig*>, false>*, unsigned long)
+leak:void std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::ATNConfig> const&>(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::ATNConfig>*, std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > > >, std::shared_ptr<antlr4::atn::ATNConfig> const&)
+leak:antlr4::atn::ATNConfigSet::add(std::shared_ptr<antlr4::atn::ATNConfig> const&, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
+leak:antlr4::atn::ParserATNSimulator::getEpsilonTarget(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::Transition*, bool, bool, bool, bool)
+leak:antlr4::atn::PredictionContext::mergeArrays(std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
+leak:/lib/x86_64-linux-gnu/libpython3.

tools/src/CMakeLists.txt

@@ -3,28 +3,11 @@ set(VERSION_STRING ${MEMGRAPH_VERSION})
 configure_file(../../src/version.hpp.in version.hpp @ONLY)
 include_directories(${CMAKE_CURRENT_BINARY_DIR})
 
-# Memgraph Client Target
-add_executable(mg_client mg_client/main.cpp)
-set(CLIENT_LIBS mg-communication mg-io mg-utils)
-if (READLINE_FOUND)
-  list(APPEND CLIENT_LIBS readline)
-endif()
-target_link_libraries(mg_client ${CLIENT_LIBS})
-
 # Memgraph Dump Target
 add_executable(mg_dump mg_dump/main.cpp)
 target_include_directories(mg_dump PRIVATE ${MGCLIENT_INCLUDE_DIR})
 target_link_libraries(mg_dump gflags spdlog fmt mgclient pthread)
-
-# Strip the executable in release build.
-string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
-if (lower_build_type STREQUAL "release")
-  add_custom_command(TARGET mg_client POST_BUILD
-                     COMMAND strip -s mg_client
-                     COMMENT "Stripping symbols and sections from mg_client")
-endif()
-
-install(TARGETS mg_client RUNTIME DESTINATION bin)
+install(TARGETS mg_dump RUNTIME DESTINATION bin)
 
 # Target for building all the tool executables.
-add_custom_target(tools DEPENDS mg_client mg_dump)
+add_custom_target(tools DEPENDS mg_dump)

tools/src/mg_client/main.cpp

@@ -1,709 +0,0 @@
-#include <gflags/gflags.h>
-
-#include <pwd.h>
-#include <signal.h>
-#include <sys/ioctl.h>
-#include <termios.h>
-#include <unistd.h>
-#include <algorithm>
-#include <filesystem>
-#include <thread>
-#include <unordered_set>
-
-#include "communication/bolt/client.hpp"
-#include "communication/init.hpp"
-#include "io/network/endpoint.hpp"
-#include "io/network/utils.hpp"
-#include "utils/algorithm.hpp"
-#include "utils/file.hpp"
-#include "utils/flag_validation.hpp"
-#include "utils/logging.hpp"
-#include "utils/signals.hpp"
-#include "utils/string.hpp"
-#include "utils/terminate_handler.hpp"
-#include "utils/timer.hpp"
-#include "version.hpp"
-
-namespace fs = std::filesystem;
-
-volatile sig_atomic_t is_shutting_down = 0;
-
-// Usage strings.
-static const std::string kUsage =
-    "Memgraph bolt client.\n"
-    "The client can be run in interactive or non-interactive mode.\n";
-static const std::string kInteractiveUsage =
-    "In interactive mode, user can enter cypher queries and supported "
-    "commands.\n\n"
-    "Cypher queries can span through multiple lines and conclude with a\n"
-    "semi-colon (;). Each query is executed in the database and the results\n"
-    "are printed out.\n\n"
-    "The following interactive commands are supported:\n\n"
-    "\t:help\t Print out usage for interactive mode\n"
-    "\t:quit\t Exit the shell\n";
-
-// Supported commands.
-// Maybe also add reconnect?
-static const std::string kCommandQuit = ":quit";
-static const std::string kCommandHelp = ":help";
-
-// Supported formats.
-static const std::string kCsvFormat = "csv";
-static const std::string kTabularFormat = "tabular";
-
-DEFINE_string(host, "127.0.0.1",
-              "Server address. It can be a DNS resolvable hostname.");
-DEFINE_int32(port, 7687, "Server port");
-DEFINE_string(username, "", "Username for the database");
-DEFINE_string(password, "", "Password for the database");
-DEFINE_bool(use_ssl, true, "Use SSL when connecting to the server.");
-DEFINE_bool(fit_to_screen, false, "Fit output width to screen width.");
-DEFINE_VALIDATED_string(
-    output_format, "tabular",
-    "Query output format. Can be csv/tabular. If output format is "
-    "other than tabular `fit-to-screen` flag is ignored.",
-    {
-      if (value == kCsvFormat || value == kTabularFormat) {
-        return true;
-      }
-      return false;
-    });
-DEFINE_VALIDATED_string(csv_delimiter, ",",
-                        "Character used to separate fields.", {
-                          if (value.size() != 1) {
-                            return false;
-                          }
-                          return true;
-                        });
-DEFINE_string(
-    csv_escapechar, "",
-    "Character used to escape the quotechar(\") if csv-doublequote is false.");
-DEFINE_bool(
-    csv_doublequote, true,
-    "Controls how instances of quotechar(\") appearing inside a field should "
-    "themselves be quoted. When true, the character is doubled. When false, "
-    "the escapechar is used as a prefix to the quotechar. "
-    "If csv-doublequote is false 'csv-escapechar' must be set.");
-
-static bool ValidateCsvDoubleQuote() {
-  if (!FLAGS_csv_doublequote && FLAGS_csv_escapechar.size() != 1) {
-    return false;
-  }
-  return true;
-}
-
-#ifdef HAS_READLINE
-DEFINE_string(history, "~/.memgraph",
-              "Use the specified directory for saving history.");
-DEFINE_bool(no_history, false, "Do not save history.");
-
-// History default directory.
-static const std::string kDefaultHistoryBaseDir = "~";
-static const std::string kDefaultHistoryMemgraphDir = ".memgraph";
-// History filename.
-static const std::string kHistoryFilename = "client_history";
-#endif
-
-DECLARE_int32(min_log_level);
-
-// Unfinished query text from previous input.
-// e.g. Previous input was MATCH(n) RETURN n; MATCH
-// then default_text would be set to MATCH for next query.
-static std::string default_text;
-
-static const std::string kPrompt = "memgraph> ";
-static const std::string kMultilinePrompt = "       -> ";
-
-static void PrintHelp() { std::cout << kInteractiveUsage << std::endl; }
-
-static void PrintValue(std::ostream &os, const std::string &value) {
-  os << value;
-}
-
-static void PrintValue(std::ostream &os,
-                       const communication::bolt::Value &value) {
-  switch (value.type()) {
-    case communication::bolt::Value::Type::String:
-      os << value.ValueString();
-      return;
-    default:
-      os << value;
-      return;
-  }
-}
-
-static void EchoFailure(const std::string &failure_msg,
-                        const std::string &explanation) {
-  if (isatty(STDIN_FILENO)) {
-    std::cout << "\033[1;31m" << failure_msg << ": \033[0m";
-    std::cout << explanation << std::endl;
-  } else {
-    std::cerr << failure_msg << ": ";
-    std::cerr << explanation << std::endl;
-  }
-}
-
-static void EchoInfo(const std::string &message) {
-  if (isatty(STDIN_FILENO)) {
-    std::cout << message << std::endl;
-  }
-}
-
-static void SetStdinEcho(bool enable = true) {
-  struct termios tty;
-  tcgetattr(STDIN_FILENO, &tty);
-  if (!enable) {
-    tty.c_lflag &= ~ECHO;
-  } else {
-    tty.c_lflag |= ECHO;
-  }
-  tcsetattr(STDIN_FILENO, TCSANOW, &tty);
-}
-
-#ifdef HAS_READLINE
-
-#include "readline/history.h"
-#include "readline/readline.h"
-
-/// Helper function that sets default input for 'readline'
-static int SetDefaultText() {
-  rl_insert_text(default_text.c_str());
-  default_text = "";
-  rl_startup_hook = (rl_hook_func_t *)NULL;
-  return 0;
-}
-
-/// Memgraph and OpenCypher keywords.
-/// Copied from src/query/frontend/opencypher/grammar/Cypher.g4
-/// and src/query/frontend/grammar/MemgraphCypher.g4
-static const std::vector<std::string> kMemgraphKeywords{
-    "ALTER",    "AUTH",    "BATCH", "BATCHES", "CLEAR",     "DATA",
-    "DENY",     "DROP",    "FOR",   "FROM",    "GRANT",     "IDENTIFIED",
-    "INTERVAL", "K_TEST",  "KAFKA", "LOAD",    "PASSWORD",  "PRIVILEGES",
-    "REVOKE",   "ROLE",    "ROLES", "SIZE",    "START",     "STOP",
-    "STREAM",   "STREAMS", "TO",    "TOPIC",   "TRANSFORM", "USER",
-    "USERS"};
-static const std::vector<std::string> kCypherKeywords{
-    "ALL",        "AND",    "ANY",    "AS",         "ASC",      "ASCENDING",
-    "BFS",        "BY",     "CASE",   "CONTAINS",   "COUNT",    "CREATE",
-    "CYPHERNULL", "DELETE", "DESC",   "DESCENDING", "DETACH",   "DISTINCT",
-    "ELSE",       "END",    "ENDS",   "EXTRACT",    "FALSE",    "FILTER",
-    "IN",         "INDEX",  "IS",     "LIMIT",      "L_SKIP",   "MATCH",
-    "MERGE",      "NONE",   "NOT",    "ON",         "OPTIONAL", "OR",
-    "ORDER",      "REDUCE", "REMOVE", "RETURN",     "SET",      "SHOW",
-    "SINGLE",     "STARTS", "THEN",   "TRUE",       "UNION",    "UNWIND",
-    "WHEN",       "WHERE",  "WITH",   "WSHORTEST",  "XOR"};
-
-static char *CompletionGenerator(const char *text, int state) {
-  // This function is called with state=0 the first time; subsequent calls
-  // are with a nonzero state. state=0 can be used to perform one-time
-  // initialization for this completion session.
-  static std::vector<std::string> matches;
-  static size_t match_index = 0;
-
-  if (state == 0) {
-    // During initialization, compute the actual matches for 'text' and
-    // keep them in a static vector.
-    matches.clear();
-    match_index = 0;
-
-    // Collect a vector of matches: vocabulary words that begin with text.
-    std::string text_str = utils::ToUpperCase(std::string(text));
-    for (auto word : kCypherKeywords) {
-      if (word.size() >= text_str.size() &&
-          word.compare(0, text_str.size(), text_str) == 0) {
-        matches.push_back(word);
-      }
-    }
-    for (auto word : kMemgraphKeywords) {
-      if (word.size() >= text_str.size() &&
-          word.compare(0, text_str.size(), text_str) == 0) {
-        matches.push_back(word);
-      }
-    }
-  }
-
-  if (match_index >= matches.size()) {
-    // We return nullptr to notify the caller no more matches are available.
-    return nullptr;
-  } else {
-    // Return a malloc'd char* for the match. The caller frees it.
-    return strdup(matches[match_index++].c_str());
-  }
-}
-
-static char **Completer(const char *text, int start, int end) {
-  // Don't do filename completion even if our generator finds no matches.
-  rl_attempted_completion_over = 1;
-  // Note: returning nullptr here will make readline use the default filename
-  // completer. This note is copied from examples - I think because
-  // rl_attempted_completion_over is set to 1, filename completer won't be used.
-  return rl_completion_matches(text, CompletionGenerator);
-}
-
-/// Helper function that reads a line from the
-/// standard input using the 'readline' lib.
-/// Adds support for history and reverse-search.
-///
-/// @param prompt The prompt to display.
-/// @return  User input line, or nullopt on EOF.
-static std::optional<std::string> ReadLine(const std::string &prompt) {
-  if (default_text.size() > 0) {
-    // Initialize text with remainder of previous query.
-    rl_startup_hook = SetDefaultText;
-  }
-  char *line = readline(prompt.c_str());
-  if (!line) return std::nullopt;
-
-  std::string r_val(line);
-  if (!utils::Trim(r_val).empty()) add_history(line);
-  free(line);
-  return r_val;
-}
-
-#else
-
-/// Helper function that reads a line from the standard input
-/// using getline.
-/// @param prompt The prompt to display.
-/// @return User input line, or nullopt on EOF.
-static std::optional<std::string> ReadLine(const std::string &prompt) {
-  std::cout << prompt << default_text;
-  std::string line;
-  std::getline(std::cin, line);
-  if (std::cin.eof()) return std::nullopt;
-  line = default_text + line;
-  default_text = "";
-  return line;
-}
-
-#endif  // HAS_READLINE
-
-static std::optional<std::string> GetLine() {
-  std::string line;
-  std::getline(std::cin, line);
-  if (std::cin.eof()) return std::nullopt;
-  line = default_text + line;
-  default_text = "";
-  return line;
-}
-
-/// Helper function that parses user line input.
-/// @param line user input line.
-/// @param quote quote character or '\0'; if set line is inside quotation.
-/// @param escaped if set, next character should be escaped.
-/// @return pair of string and bool. string is parsed line and bool marks
-/// if query finished(Query finishes with ';') with this line.
-static std::pair<std::string, bool> ParseLine(const std::string &line,
-                                              char *quote, bool *escaped) {
-  // Parse line.
-  bool is_done = false;
-  std::stringstream parsed_line;
-  for (auto c : line) {
-    if (*quote && c == '\\') {
-      // Escaping is only used inside quotation to not end the quote
-      // when quotation char is escaped.
-      *escaped = !*escaped;
-      parsed_line << c;
-      continue;
-    } else if ((!*quote && (c == '\"' || c == '\'')) ||
-               (!*escaped && c == *quote)) {
-      *quote = *quote ? '\0' : c;
-    } else if (!*quote && c == ';') {
-      is_done = true;
-      break;
-    }
-    parsed_line << c;
-    *escaped = false;
-  }
-  return std::make_pair(parsed_line.str(), is_done);
-}
-
-static std::optional<std::string> GetQuery() {
-  char quote = '\0';
-  bool escaped = false;
-  auto ret = ParseLine(default_text, &quote, &escaped);
-  if (ret.second) {
-    auto idx = ret.first.size() + 1;
-    default_text = utils::Trim(default_text.substr(idx));
-    return ret.first;
-  }
-  std::stringstream query;
-  std::optional<std::string> line;
-  int line_cnt = 0;
-  auto is_done = false;
-  while (!is_done) {
-    if (!isatty(STDIN_FILENO)) {
-      line = GetLine();
-    } else {
-      line = ReadLine(line_cnt == 0 ? kPrompt : kMultilinePrompt);
-      if (line_cnt == 0 && line && line->size() > 0 && (*line)[0] == ':') {
-        auto trimmed_line = utils::Trim(*line);
-        if (trimmed_line == kCommandQuit) {
-          return std::nullopt;
-        } else if (trimmed_line == kCommandHelp) {
-          PrintHelp();
-          return "";
-        } else {
-          EchoFailure("Unsupported command", std::string(trimmed_line));
-          PrintHelp();
-          return "";
-        }
-      }
-    }
-    if (!line) return std::nullopt;
-    if (line->empty()) continue;
-    auto ret = ParseLine(*line, &quote, &escaped);
-    query << ret.first;
-    auto char_count = ret.first.size();
-    if (ret.second) {
-      is_done = true;
-      char_count += 1;  // ';' sign
-    } else {
-      // Query is multiline so append newline.
-      query << "\n";
-    }
-    if (char_count < line->size()) {
-      default_text = utils::Trim(line->substr(char_count));
-    }
-    ++line_cnt;
-  }
-  return query.str();
-}
-
-template <typename T>
-static void PrintRowTabular(const std::vector<T> &data, int total_width,
-                            int column_width, int num_columns,
-                            bool all_columns_fit, int margin = 1) {
-  if (!all_columns_fit) num_columns -= 1;
-  std::string data_output = std::string(total_width, ' ');
-  for (auto i = 0; i < total_width; i += column_width) {
-    data_output[i] = '|';
-    int idx = i / column_width;
-    if (idx < num_columns) {
-      std::stringstream field;
-      PrintValue(field, data[idx]);  // convert Value to string
-      std::string field_str(field.str());
-      if (field_str.size() > column_width - 2 * margin - 1) {
-        field_str.erase(column_width - 2 * margin - 1, std::string::npos);
-        field_str.replace(field_str.size() - 3, 3, "...");
-      }
-      data_output.replace(i + 1 + margin, field_str.size(), field_str);
-    }
-  }
-  if (!all_columns_fit) {
-    data_output.replace(total_width - column_width, 3, "...");
-  }
-  data_output[total_width - 1] = '|';
-  std::cout << data_output << std::endl;
-}
-
-/// Helper function for determining maximum length of data.
-/// @param data Vector of string representable elements. Elements should have
-/// operator '<<' implemented.
-/// @param margin Column margin width.
-/// @return length needed for representing max size element in vector. Plus
-/// one is added because of column start character '|'.
-template <typename T>
-static uint64_t GetMaxColumnWidth(const std::vector<T> &data, int margin = 1) {
-  uint64_t column_width = 0;
-  for (auto &elem : data) {
-    std::stringstream field;
-    PrintValue(field, elem);
-    column_width = std::max(column_width, field.str().size() + 2 * margin);
-  }
-  return column_width + 1;
-}
-
-static void PrintTabular(
-    const std::vector<std::string> &header,
-    const std::vector<std::vector<communication::bolt::Value>> &records) {
-  struct winsize w;
-  ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
-  bool all_columns_fit = true;
-
-  auto num_columns = header.size();
-  auto column_width = GetMaxColumnWidth(header);
-  for (size_t i = 0; i < records.size(); ++i) {
-    column_width = std::max(column_width, GetMaxColumnWidth(records[i]));
-  }
-  column_width = std::max(static_cast<uint64_t>(5),
-                          column_width);  // set column width to min 5
-  auto total_width = column_width * num_columns + 1;
-
-  // Fit to screen width.
-  if (FLAGS_fit_to_screen && total_width > w.ws_col) {
-    uint64_t lo = 5;
-    uint64_t hi = column_width;
-    uint64_t last = 5;
-    while (lo < hi) {
-      uint64_t mid = lo + (hi - lo) / 2;
-      uint64_t width = mid * num_columns + 1;
-      if (width <= w.ws_col) {
-        last = mid;
-        lo = mid + 1;
-      } else {
-        hi = mid - 1;
-      }
-    }
-    column_width = last;
-    total_width = column_width * num_columns + 1;
-    // All columns do not fit on screen.
-    while (total_width > w.ws_col && num_columns > 1) {
-      num_columns -= 1;
-      total_width = column_width * num_columns + 1;
-      all_columns_fit = false;
-    }
-  }
-
-  auto line_fill = std::string(total_width, '-');
-  for (auto i = 0; i < total_width; i += column_width) {
-    line_fill[i] = '+';
-  }
-  line_fill[total_width - 1] = '+';
-  std::cout << line_fill << std::endl;
-  // Print Header.
-  PrintRowTabular(header, total_width, column_width, num_columns,
-                  all_columns_fit);
-  std::cout << line_fill << std::endl;
-  // Print Records.
-  for (size_t i = 0; i < records.size(); ++i) {
-    PrintRowTabular(records[i], total_width, column_width, num_columns,
-                    all_columns_fit);
-  }
-  std::cout << line_fill << std::endl;
-}
-
-template <typename T>
-static std::vector<std::string> FormatCsvFields(const std::vector<T> &fields) {
-  std::vector<std::string> formatted;
-  formatted.reserve(fields.size());
-  for (auto &field : fields) {
-    std::stringstream field_stream;
-    field_stream << field;
-    std::string formatted_field(field_stream.str());
-    if (FLAGS_csv_doublequote) {
-      formatted_field = utils::Replace(formatted_field, "\"", "\"\"");
-    } else {
-      formatted_field =
-          utils::Replace(formatted_field, "\"", FLAGS_csv_escapechar + "\"");
-    }
-    formatted_field.insert(0, 1, '"');
-    formatted_field.append(1, '"');
-    formatted.push_back(formatted_field);
-  }
-  return formatted;
-}
-
-static void PrintCsv(
-    const std::vector<std::string> &header,
-    const std::vector<std::vector<communication::bolt::Value>> &records) {
-  // Print Header.
-  auto formatted_header = FormatCsvFields(header);
-  utils::PrintIterable(std::cout, formatted_header, FLAGS_csv_delimiter);
-  std::cout << std::endl;
-  // Print Records.
-  for (size_t i = 0; i < records.size(); ++i) {
-    auto formatted_row = FormatCsvFields(records[i]);
-    utils::PrintIterable(std::cout, formatted_row, FLAGS_csv_delimiter);
-    std::cout << std::endl;
-  }
-}
-
-static void Output(
-    const std::vector<std::string> &header,
-    const std::vector<std::vector<communication::bolt::Value>> &records) {
-  if (FLAGS_output_format == kTabularFormat) {
-    PrintTabular(header, records);
-  } else if (FLAGS_output_format == kCsvFormat) {
-    PrintCsv(header, records);
-  }
-}
-
-int main(int argc, char **argv) {
-  gflags::SetVersionString(version_string);
-  gflags::SetUsageMessage(kUsage);
-
-  gflags::ParseCommandLineFlags(&argc, &argv, true);
-  if (FLAGS_output_format == kCsvFormat && !ValidateCsvDoubleQuote()) {
-    EchoFailure(
-        "Unsupported combination of 'csv-doublequote' and 'csv-escapechar'\n"
-        "flags",
-        "Run './mg_client --help' for usage.");
-    return 1;
-  }
-  auto password = FLAGS_password;
-  if (isatty(STDIN_FILENO) && FLAGS_username.size() > 0 &&
-      password.size() == 0) {
-    SetStdinEcho(false);
-    auto password_optional = ReadLine("Password: ");
-    std::cout << std::endl;
-    if (password_optional) {
-      password = *password_optional;
-    } else {
-      EchoFailure(
-          "Password not submitted",
-          fmt::format("Requested password for username {}", FLAGS_username));
-      return 1;
-    }
-    SetStdinEcho(true);
-  }
-
-  spdlog::set_level(spdlog::level::err);
-
-  communication::SSLInit sslInit;
-
-#ifdef HAS_READLINE
-  using_history();
-  int history_len = 0;
-  rl_attempted_completion_function = Completer;
-  fs::path history_dir = FLAGS_history;
-  if (FLAGS_history ==
-      (kDefaultHistoryBaseDir + "/" + kDefaultHistoryMemgraphDir)) {
-    // Fetch home dir for user.
-    struct passwd *pw = getpwuid(getuid());
-    history_dir = fs::path(pw->pw_dir) / kDefaultHistoryMemgraphDir;
-  }
-  if (!utils::EnsureDir(history_dir)) {
-    EchoFailure("History directory doesn't exist", history_dir);
-    // Should program exit here or just continue with warning message?
-    return 1;
-  }
-  fs::path history_file = history_dir / kHistoryFilename;
-  // Read history file.
-  if (fs::exists(history_file)) {
-    auto ret = read_history(history_file.string().c_str());
-    if (ret != 0) {
-      EchoFailure("Unable to read history file", history_file);
-      // Should program exit here or just continue with warning message?
-      return 1;
-    }
-    history_len = history_length;
-  }
-
-  // Save history function. Used to save readline history after each query.
-  auto save_history = [&history_len, history_file] {
-    if (!FLAGS_no_history) {
-      int ret = 0;
-      // If there was no history, create history file.
-      // Otherwise, append to existing history.
-      if (history_len == 0) {
-        ret = write_history(history_file.string().c_str());
-      } else {
-        ret = append_history(1, history_file.string().c_str());
-      }
-      if (ret != 0) {
-        EchoFailure("Unable to save history to file", history_file);
-        return 1;
-      }
-      ++history_len;
-    }
-    return 0;
-  };
-#endif
-
-  // Prevent handling shutdown inside a shutdown. For example, SIGINT handler
-  // being interrupted by SIGTERM before is_shutting_down is set, thus causing
-  // double shutdown.
-  sigset_t block_shutdown_signals;
-  sigemptyset(&block_shutdown_signals);
-  sigaddset(&block_shutdown_signals, SIGTERM);
-  sigaddset(&block_shutdown_signals, SIGINT);
-
-  auto shutdown = [](int exit_code = 0) {
-    if (is_shutting_down) return;
-    is_shutting_down = 1;
-    std::quick_exit(exit_code);
-  };
-
-  utils::SignalHandler::RegisterHandler(utils::Signal::Terminate, shutdown,
-                                        block_shutdown_signals);
-  utils::SignalHandler::RegisterHandler(utils::Signal::Interupt, shutdown,
-                                        block_shutdown_signals);
-
-  // TODO handle endpoint exception.
-  // It has CHECK in constructor if address is not valid.
-  io::network::Endpoint endpoint(io::network::ResolveHostname(FLAGS_host),
-                                 FLAGS_port);
-  communication::ClientContext context(FLAGS_use_ssl);
-  communication::bolt::Client client(&context);
-
-  std::string bolt_client_version =
-      fmt::format("mg_client/{}", gflags::VersionString());
-  try {
-    client.Connect(endpoint, FLAGS_username, password, bolt_client_version);
-  } catch (const communication::bolt::ClientFatalException &e) {
-    EchoFailure("Connection failure", e.what());
-    return 1;
-  }
-
-  EchoInfo(fmt::format("mg_client {}", gflags::VersionString()));
-  EchoInfo("Type :help for shell usage");
-  EchoInfo("Quit the shell by typing Ctrl-D(eof) or :quit");
-  EchoInfo(fmt::format("Connected to 'memgraph://{}'", endpoint));
-  int num_retries = 3;
-  while (true) {
-    auto query = GetQuery();
-    if (!query) {
-      EchoInfo("Bye");
-      break;
-    }
-    if (query->empty()) continue;
-    try {
-      utils::Timer t;
-      auto ret = client.Execute(*query, {});
-      auto elapsed = t.Elapsed().count();
-      if (ret.records.size() > 0) Output(ret.fields, ret.records);
-      if (isatty(STDIN_FILENO)) {
-        std::string summary;
-        if (ret.records.size() == 0) {
-          summary = "Empty set";
-        } else if (ret.records.size() == 1) {
-          summary = std::to_string(ret.records.size()) + " row in set";
-        } else {
-          summary = std::to_string(ret.records.size()) + " rows in set";
-        }
-        std::cout << summary << " (" << fmt::format("{:.3f}", elapsed)
-                  << " sec)" << std::endl;
-#ifdef HAS_READLINE
-        auto history_ret = save_history();
-        if (history_ret != 0) return history_ret;
-#endif
-      }
-    } catch (const communication::bolt::ClientQueryException &e) {
-      if (!isatty(STDIN_FILENO)) {
-        EchoFailure("Failed query", *query);
-      }
-      EchoFailure("Client received exception", e.what());
-      if (!isatty(STDIN_FILENO)) {
-        return 1;
-      }
-    } catch (const communication::bolt::ClientFatalException &e) {
-      EchoFailure("Client received exception", e.what());
-      EchoInfo("Trying to reconnect");
-      bool is_connected = false;
-      client.Close();
-      while (num_retries > 0) {
-        --num_retries;
-        try {
-          client.Connect(endpoint, FLAGS_username, FLAGS_password,
-                         bolt_client_version);
-          is_connected = true;
-          break;
-        } catch (const communication::bolt::ClientFatalException &e) {
-          EchoFailure("Connection failure", e.what());
-        }
-        std::this_thread::sleep_for(std::chrono::seconds(1));
-      }
-      if (is_connected) {
-        num_retries = 3;
-        EchoInfo(fmt::format("Connected to 'memgraph://{}'", endpoint));
-      } else {
-        EchoFailure("Couldn't connect to",
-                    fmt::format("'memgraph://{}'", endpoint));
-        return 1;
-      }
-    }
-  }
-  return 0;
-}

tools/tests/test_mg_client

@@ -1,131 +0,0 @@
-#!/bin/bash
-
-## Helper functions
-
-function wait_for_server {
-    port=$1
-    while ! nc -z -w 1 127.0.0.1 $port; do
-        sleep 0.1
-    done
-    sleep 1
-}
-
-function echo_info { printf "\033[1;36m~~ $1 ~~\033[0m\n"; }
-function echo_success { printf "\033[1;32m~~ $1 ~~\033[0m\n\n"; }
-function echo_failure { printf "\033[1;31m~~ $1 ~~\033[0m\n\n"; }
-
-
-## Environment setup
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-cd "$DIR"
-
-# Create a temporary directory for output files
-tmpdir=/tmp/mg_client/output
-if [ -d $tmpdir ]; then
-    rm -rf $tmpdir
-fi
-mkdir -p $tmpdir
-cd $tmpdir
-
-# Find memgraph binaries.
-memgraph_dir="$DIR/../../build"
-
-# Find mg_client binaries.
-client_dir="$memgraph_dir/tools/src"
-if [ ! -d $client_dir ]; then
-    echo_failure "mg-client directory not found"
-    exit 1
-fi
-
-# Find tests dir.
-tests_dir="$DIR/client"
-if [ ! -d $tests_dir ]; then
-    echo_failure "Directory with tests not found"
-    exit 1
-fi
-
-# Find tests input files.
-input_dir="$tests_dir/input"
-if [ ! -d $input_dir ]; then
-    echo_failure "Directory with tests input files not found"
-    exit 1
-fi
-
-# Check and generate SSL certificates
-use_ssl=false
-key_file=""
-cert_file=""
-if [ "$1" == "--use-ssl" ]; then
-    use_ssl=true
-    key_file=".key.pem"
-    cert_file=".cert.pem"
-    openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
-        -subj "/C=HR/ST=Zagreb/L=Zagreb/O=Memgraph/CN=db.memgraph.com" \
-        -keyout $key_file -out $cert_file || exit 1
-fi
-
-
-## Startup
-
-# Start the memgraph process and wait for it to start.
-echo_info "Starting memgraph"
-$memgraph_dir/memgraph --bolt-cert-file=$cert_file --bolt-key-file=$key_file &
-pid=$!
-wait_for_server 7687
-echo_success "Started memgraph"
-
-
-## Tests
-
-echo_info "Running tests"
-echo  # Blank line
-
-client_flags="--use-ssl=$use_ssl"
-test_code=0
-for output_dir in $tests_dir/output_*; do
-    for filename in $input_dir/*; do
-        test_name=$(basename $filename)
-        test_name=${test_name%.*}
-        output_name="$test_name.txt"
-
-        output_format=$(basename $output_dir)
-        output_format=${output_format#*_}
-        run_flags="$client_flags --output-format=$output_format"
-
-        echo_info "Running test '$test_name' with $output_format output"
-        $client_dir/mg_client $run_flags < $filename > $tmpdir/$test_name
-        diff -b $tmpdir/$test_name $output_dir/$output_name
-        test_code=$?
-        if [ $test_code -ne 0 ]; then
-            echo_failure "Test '$test_name' with $output_format output failed"
-            break
-        else
-            echo_success "Test '$test_name' with $output_format output passed"
-        fi
-
-        # Clear database for each test.
-        $client_dir/mg_client $client_flags <<< "MATCH (n) DETACH DELETE n;" \
-                                &> /dev/null || exit 1
-    done
-    if [ $test_code -ne 0 ]; then
-        break
-    fi
-done
-
-
-## Cleanup
-echo_info "Starting test cleanup"
-
-# Shutdown the memgraph process.
-kill $pid
-wait -n
-code_mg=$?
-
-# Check memgraph exit code.
-if [ $code_mg -ne 0 ]; then
-    echo_failure "The memgraph process didn't terminate properly!"
-    exit $code_mg
-fi
-echo_success "Test cleanup done"
-
-exit $test_code