reduce number of steps

.github/workflows/diff.yaml

@@ -17,9 +17,205 @@ on:
       - "licenses/*"
 
 jobs:
+  community_build:
+    name: "Community build"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build community binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build community binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMG_ENTERPRISE=OFF ..
+          make -j$THREADS
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure -j$THREADS
+
+  code_analysis:
+    name: "Code analysis"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+        # This is also needed if we want do to comparison against other branches
+        # See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
+      - name: Fetch all history for all tags and branches
+        run: git fetch
+
+      - name: Initialize deps
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Initialize dependencies.
+          ./init
+
+      - name: Set base branch
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
+
+      - name: Set base branch # if we manually dispatch or push to master
+        if: ${{ github.event_name != 'pull_request' }}
+        run: |
+          echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
+
+      - name: Python code analysis
+        run: |
+          CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
+          for file in ${CHANGED_FILES}; do
+            echo ${file}
+            if [[ ${file} == *.py ]]; then
+              python3 -m black --check --diff ${file}
+              python3 -m isort --profile black --check-only --diff ${file}
+            fi
+          done
+
+      - name: Build combined ASAN, UBSAN and coverage binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          cd build
+          cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
+          make -j$THREADS memgraph__unit
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
+          cd build
+          LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
+
+      - name: Compute code coverage
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Compute code coverage.
+          cd tools/github
+          ./coverage_convert
+
+          # Package code coverage.
+          cd generated
+          tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
+
+      - name: Save code coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: "Code coverage"
+          path: tools/github/generated/code_coverage.tar.gz
+
+      - name: Run clang-tidy
+        run: |
+          source /opt/toolchain-v4/activate
+
+          # Restrict clang-tidy results only to the modified parts
+          git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build  -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
+
+          # Fail if any warning is reported
+          ! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
+
+  debug_build:
+    name: "Debug build"
+    runs-on: [self-hosted, Linux, X64, Diff]
+    env:
+      THREADS: 24
+      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build debug binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build debug binaries.
+          cd build
+          cmake ..
+          make -j$THREADS
+
+      - name: Run leftover CTest tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run leftover CTest tests (all except unit and benchmark tests).
+          cd build
+          ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
+
+      - name: Run drivers tests
+        run: |
+          ./tests/drivers/run.sh
+
+      - name: Run integration tests
+        run: |
+          tests/integration/run.sh
+
+      - name: Run cppcheck and clang-format
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run cppcheck and clang-format.
+          cd tools/github
+          ./cppcheck_and_clang_format diff
+
+      - name: Save cppcheck and clang-format errors
+        uses: actions/upload-artifact@v3
+        with:
+          name: "Code coverage"
+          path: tools/github/cppcheck_and_clang_format.txt
+
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, Linux, X64, gamora]
+    runs-on: [self-hosted, Linux, X64, Diff]
     env:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
@@ -129,3 +325,119 @@ jobs:
           path: |
             # multiple paths could be defined
             build/logs
+
+  release_jepsen_test:
+    name: "Release Jepsen Test"
+    runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
+    #continue-on-error: true
+    env:
+      THREADS: 24
+      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+          # Initialize dependencies.
+          ./init
+          # Build only memgraph release binarie.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+          make -j$THREADS memgraph
+
+      - name: Run Jepsen tests
+        run: |
+          cd tests/jepsen
+          ./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
+
+      - name: Save Jepsen report
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: "Jepsen Report"
+          path: tests/jepsen/Jepsen.tar.gz
+
+  release_benchmarks:
+    name: "Release benchmarks"
+    runs-on: [self-hosted, Linux, X64, Diff, Gen7]
+    env:
+      THREADS: 24
+      MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
+      MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
+
+    steps:
+      - name: Set up repository
+        uses: actions/checkout@v3
+        with:
+          # Number of commits to fetch. `0` indicates all history for all
+          # branches and tags. (default: 1)
+          fetch-depth: 0
+
+      - name: Build release binaries
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Initialize dependencies.
+          ./init
+
+          # Build only memgraph release binaries.
+          cd build
+          cmake -DCMAKE_BUILD_TYPE=release ..
+          make -j$THREADS
+
+      - name: Run macro benchmarks
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Get branch name (merge)
+        if: github.event_name != 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
+
+      - name: Get branch name (pull request)
+        if: github.event_name == 'pull_request'
+        shell: bash
+        run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
+
+      - name: Upload macro benchmark results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "macro_benchmark" \
+                    --benchmark-results "../../tests/macro_benchmark/.harness_summary" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"
+
+      # TODO (andi) No need for path flags and for --disk-storage and --in-memory-analytical
+      - name: Run mgbench
+        run: |
+          cd tests/mgbench
+          ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
+
+      - name: Upload mgbench results
+        run: |
+          cd tools/bench-graph-client
+          virtualenv -p python3 ve3
+          source ve3/bin/activate
+          pip install -r requirements.txt
+          ./main.py --benchmark-name "mgbench" \
+                    --benchmark-results "../../tests/mgbench/benchmark_result.json" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"

.github/workflows/release_debian10.yaml

@@ -178,7 +178,7 @@ jobs:
 
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Debian10, BigMemory]
+    runs-on: [self-hosted, Linux, X64, Debian10]
     env:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
@@ -231,73 +231,73 @@ jobs:
           name: "Enterprise DEB package"
           path: build/output/memgraph*.deb
 
-      - name: Run micro benchmark tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
+      # - name: Run micro benchmark tests
+      #   run: |
+      #     # Activate toolchain.
+      #     source /opt/toolchain-v4/activate
 
-          # Run micro benchmark tests.
-          cd build
-          # The `eval` benchmark needs a large stack limit.
-          ulimit -s 262144
-          ctest -R memgraph__benchmark -V
+      #     # Run micro benchmark tests.
+      #     cd build
+      #     # The `eval` benchmark needs a large stack limit.
+      #     ulimit -s 262144
+      #     ctest -R memgraph__benchmark -V
 
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
+      # - name: Run macro benchmark tests
+      #   run: |
+      #     cd tests/macro_benchmark
+      #     ./harness QuerySuite MemgraphRunner \
+      #       --groups aggregation 1000_create unwind_create dense_expand match \
+      #       --no-strict
 
-      - name: Run parallel macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QueryParallelSuite MemgraphRunner \
-            --groups aggregation_parallel create_parallel bfs_parallel \
-            --num-database-workers 9 --num-clients-workers 30 \
-            --no-strict
+      # - name: Run parallel macro benchmark tests
+      #   run: |
+      #     cd tests/macro_benchmark
+      #     ./harness QueryParallelSuite MemgraphRunner \
+      #       --groups aggregation_parallel create_parallel bfs_parallel \
+      #       --num-database-workers 9 --num-clients-workers 30 \
+      #       --no-strict
 
-      - name: Run GQL Behave tests
-        run: |
-          cd tests
-          ./setup.sh /opt/toolchain-v4/activate
-          cd gql_behave
-          ./continuous_integration
+      # - name: Run GQL Behave tests
+      #   run: |
+      #     cd tests
+      #     ./setup.sh /opt/toolchain-v4/activate
+      #     cd gql_behave
+      #     ./continuous_integration
 
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
+      # - name: Save quality assurance status
+      #   uses: actions/upload-artifact@v3
+      #   with:
+      #     name: "GQL Behave Status"
+      #     path: |
+      #       tests/gql_behave/gql_behave_status.csv
+      #       tests/gql_behave/gql_behave_status.html
 
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
+      # - name: Run unit tests
+      #   run: |
+      #     # Activate toolchain.
+      #     source /opt/toolchain-v4/activate
 
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
+      #     # Run unit tests.
+      #     cd build
+      #     ctest -R memgraph__unit --output-on-failure
 
-      - name: Run e2e tests
-        run: |
-          cd tests
-          ./setup.sh /opt/toolchain-v4/activate
-          source ve3/bin/activate_e2e
-          cd e2e
-          ./run.sh
+      # - name: Run e2e tests
+      #   run: |
+      #     cd tests
+      #     ./setup.sh /opt/toolchain-v4/activate
+      #     source ve3/bin/activate_e2e
+      #     cd e2e
+      #     ./run.sh
 
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
+      # - name: Run stress test (plain)
+      #   run: |
+      #     cd tests/stress
+      #     ./continuous_integration
 
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
+      # - name: Run stress test (SSL)
+      #   run: |
+      #     cd tests/stress
+      #     ./continuous_integration --use-ssl
 
       - name: Run stress test (large)
         run: |

licenses/BSL.txt

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT:   You may use the Licensed Work in accordance with the
                         3. using the Licensed Work to create a work or solution
                         which competes (or might reasonably be expected to
                         compete) with the Licensed Work.
-CHANGE DATE:            2027-08-12
+CHANGE DATE:            2027-30-10
 CHANGE LICENSE:         Apache License, Version 2.0
 
 For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -1276,59 +1276,28 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
       call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(result_alias));
     }
   } else {
-    call_proc->is_write_ = maybe_found->second->info.is_write;
-
-    auto *yield_ctx = ctx->yieldProcedureResults();
-    if (!yield_ctx) {
-      if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) {
-        throw SemanticException(
-            "CALL without YIELD may only be used on procedures which do not "
-            "return any result fields.");
-      }
-      // When we return, we will release the lock on modules. This means that
-      // someone may reload the procedure and change the result signature. But to
-      // keep the implementation simple, we ignore the case as the rest of the
-      // code doesn't really care whether we yield or not, so it should not break.
-      return call_proc;
+    const auto &maybe_found =
+        procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource());
+    if (!maybe_found) {
+      throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
     }
-    if (yield_ctx->getTokens(MemgraphCypher::ASTERISK).empty()) {
-      call_proc->result_fields_.reserve(yield_ctx->procedureResult().size());
-      call_proc->result_identifiers_.reserve(yield_ctx->procedureResult().size());
-      for (auto *result : yield_ctx->procedureResult()) {
-        MG_ASSERT(result->variable().size() == 1 || result->variable().size() == 2);
-        call_proc->result_fields_.push_back(std::any_cast<std::string>(result->variable()[0]->accept(this)));
-        std::string result_alias;
-        if (result->variable().size() == 2) {
-          result_alias = std::any_cast<std::string>(result->variable()[1]->accept(this));
-        } else {
-          result_alias = std::any_cast<std::string>(result->variable()[0]->accept(this));
-        }
-        call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(result_alias));
-      }
-    } else {
-      const auto &maybe_found =
-          procedure::FindProcedure(procedure::gModuleRegistry, call_proc->procedure_name_, utils::NewDeleteResource());
-      if (!maybe_found) {
-        throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
-      }
-      const auto &[module, proc] = *maybe_found;
-      call_proc->result_fields_.reserve(proc->results.size());
-      call_proc->result_identifiers_.reserve(proc->results.size());
-      for (const auto &[result_name, desc] : proc->results) {
-        bool is_deprecated = desc.second;
-        if (is_deprecated) continue;
-        call_proc->result_fields_.emplace_back(result_name);
-        call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(std::string(result_name)));
-      }
-      // When we leave the scope, we will release the lock on modules. This means
-      // that someone may reload the procedure and change its result signature. We
-      // are fine with this, because if new result fields were added then we yield
-      // the subset of those and that will appear to a user as if they used the
-      // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the
-      // new fields as well. In case the result signature has had some result
-      // fields removed, then the query execution will report an error that we are
-      // yielding missing fields. The user can then just retry the query.
+    const auto &[module, proc] = *maybe_found;
+    call_proc->result_fields_.reserve(proc->results.size());
+    call_proc->result_identifiers_.reserve(proc->results.size());
+    for (const auto &[result_name, desc] : proc->results) {
+      bool is_deprecated = desc.second;
+      if (is_deprecated) continue;
+      call_proc->result_fields_.emplace_back(result_name);
+      call_proc->result_identifiers_.push_back(storage_->Create<Identifier>(std::string(result_name)));
     }
+    // When we leave the scope, we will release the lock on modules. This means
+    // that someone may reload the procedure and change its result signature. We
+    // are fine with this, because if new result fields were added then we yield
+    // the subset of those and that will appear to a user as if they used the
+    // procedure before reload. Any subsequent `CALL ... YIELD *` will fetch the
+    // new fields as well. In case the result signature has had some result
+    // fields removed, then the query execution will report an error that we are
+    // yielding missing fields. The user can then just retry the query.
   }
 
   return call_proc;

tests/stress/continuous_integration

@@ -212,7 +212,6 @@ def run_stress_test_suite(args: Args) -> Optional[Dict[str, float]]:
                 print(f"Failed to execute {test_run} with following exception:", ex)
                 err = True
             finally:
-                stop_memgraph(memgraph_proc)
                 cleanup(memgraph_proc)
             if err:
                 return None

tests/stress/memory_tracker.py

@@ -260,11 +260,10 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None:
             # Problem with test using detach delete and memory tracker
             # is that memory tracker gets updated immediately
             # whereas RES takes some time
-            # Tries 10 times or fails
-            cnt_again = 10
+            cnt_again = 3
             skip_failure = False
-            # 10% is maximum diff for this test to pass
-            multiplier = 1.10
+            # 10% is maximum increment, afterwards is fail
+            multiplier = 1
             while cnt_again:
                 new_memory_tracker, new_res_data = get_storage_data(session)
 
@@ -278,6 +277,7 @@ def run_monitor_cleanup(repetition_count: int, sleep_sec: float) -> None:
                         f"RES data: {new_res_data}, multiplier: {multiplier}"
                     )
                     break
+                multiplier += 0.05
                 cnt_again -= 1
             if not skip_failure:
                 log.info(memory_tracker, initial_diff, res_data)

tests/stress/test_config.py

@@ -135,6 +135,25 @@ SMALL_DATASET = [
 # long_running runs for 8h
 LARGE_DATASET = (
     [
+        {
+            DatasetConstants.TEST: "long_running.cpp",
+            DatasetConstants.OPTIONS: [
+                "--vertex-count",
+                "200000",
+                "--edge-count",
+                "1000000",
+                "--max-time",
+                "480",
+                "--verify",
+                "300",
+                "--stats-file",
+                STATS_FILE,
+            ],
+            DatasetConstants.TIMEOUT: 500,
+            DatasetConstants.MODE: [get_default_database_mode()],
+        },
+    ]
+    + [
         {
             DatasetConstants.TEST: "bipartite.py",
             DatasetConstants.OPTIONS: ["--u-count", "300", "--v-count", "300"],
@@ -172,23 +191,4 @@ LARGE_DATASET = (
         },
     ]
     * 6
-    + [
-        {
-            DatasetConstants.TEST: "long_running.cpp",
-            DatasetConstants.OPTIONS: [
-                "--vertex-count",
-                "200000",
-                "--edge-count",
-                "1000000",
-                "--max-time",
-                "480",
-                "--verify",
-                "300",
-                "--stats-file",
-                STATS_FILE,
-            ],
-            DatasetConstants.TIMEOUT: 500,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-    ]
 )