Merge branch 'master' into orderby-workload

.github/workflows/daily_benchmark.yaml

@@ -75,6 +75,8 @@ jobs:
 
           ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results cartesian.json cartesian
 
+          ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results order_by.json order_by
+
       - name: Upload mgbench results
         run: |
           cd tools/bench-graph-client
@@ -104,3 +106,9 @@ jobs:
                     --github-run-id "${{ github.run_id }}" \
                     --github-run-number "${{ github.run_number }}" \
                     --head-branch-name "${{ env.BRANCH_NAME }}"
+
+          ./main.py --benchmark-name "order_by" \
+                    --benchmark-results "../../tests/mgbench/order_by.json" \
+                    --github-run-id "${{ github.run_id }}" \
+                    --github-run-number "${{ github.run_number }}" \
+                    --head-branch-name "${{ env.BRANCH_NAME }}"

.github/workflows/release_debian10.yaml

@@ -178,7 +178,7 @@ jobs:
 
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Debian10, nightcrawler]
+    runs-on: [self-hosted, Linux, X64, Debian10, BigMemory]
     env:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
@@ -201,16 +201,121 @@ jobs:
           # Initialize dependencies.
           ./init
 
+          # Set default build_type to Release
+          INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
+          BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
+
           # Build release binaries.
           cd build
-          cmake -DCMAKE_BUILD_TYPE=Debug ..
+          cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
           make -j$THREADS
 
+      - name: Create enterprise DEB package
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          cd build
+
+          # create mgconsole
+          # we use the -B to force the build
+          make -j$THREADS -B mgconsole
+
+          # Create enterprise DEB package.
+          mkdir output && cd output
+          cpack -G DEB --config ../CPackConfig.cmake
+
+      - name: Save enterprise DEB package
+        uses: actions/upload-artifact@v3
+        with:
+          name: "Enterprise DEB package"
+          path: build/output/memgraph*.deb
+
+      - name: Run micro benchmark tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run micro benchmark tests.
+          cd build
+          # The `eval` benchmark needs a large stack limit.
+          ulimit -s 262144
+          ctest -R memgraph__benchmark -V
+
+      - name: Run macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QuerySuite MemgraphRunner \
+            --groups aggregation 1000_create unwind_create dense_expand match \
+            --no-strict
+
+      - name: Run parallel macro benchmark tests
+        run: |
+          cd tests/macro_benchmark
+          ./harness QueryParallelSuite MemgraphRunner \
+            --groups aggregation_parallel create_parallel bfs_parallel \
+            --num-database-workers 9 --num-clients-workers 30 \
+            --no-strict
+
+      - name: Run GQL Behave tests
+        run: |
+          cd tests
+          ./setup.sh /opt/toolchain-v4/activate
+          cd gql_behave
+          ./continuous_integration
+
+      - name: Save quality assurance status
+        uses: actions/upload-artifact@v3
+        with:
+          name: "GQL Behave Status"
+          path: |
+            tests/gql_behave/gql_behave_status.csv
+            tests/gql_behave/gql_behave_status.html
+
+      - name: Run unit tests
+        run: |
+          # Activate toolchain.
+          source /opt/toolchain-v4/activate
+
+          # Run unit tests.
+          cd build
+          ctest -R memgraph__unit --output-on-failure
+
+      - name: Run e2e tests
+        run: |
+          cd tests
+          ./setup.sh /opt/toolchain-v4/activate
+          source ve3/bin/activate_e2e
+          cd e2e
+          ./run.sh
+
+      - name: Run stress test (plain)
+        run: |
+          cd tests/stress
+          ./continuous_integration
+
+      - name: Run stress test (SSL)
+        run: |
+          cd tests/stress
+          ./continuous_integration --use-ssl
+
       - name: Run stress test (large)
         run: |
           cd tests/stress
           ./continuous_integration --large-dataset
 
+      - name: Run durability test (plain)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 5
+
+      - name: Run durability test (large)
+        run: |
+          cd tests/stress
+          source ve3/bin/activate
+          python3 durability --num-steps 20
+
   release_jepsen_test:
     name: "Release Jepsen Test"
     runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]

.github/workflows/release_ubuntu2004.yaml

@@ -281,13 +281,6 @@ jobs:
           cd build
           ctest -R memgraph__unit --output-on-failure
 
-      - name: Ensure Kafka and Pulsar are up
-        run: |
-          cd tests/e2e/streams/kafka
-          docker-compose up -d
-          cd ../pulsar
-          docker-compose up -d
-
       - name: Run e2e tests
         run: |
           cd tests
@@ -296,14 +289,6 @@ jobs:
           cd e2e
           ./run.sh
 
-      - name: Ensure Kafka and Pulsar are down
-        if: always()
-        run: |
-          cd tests/e2e/streams/kafka
-          docker-compose down
-          cd ../pulsar
-          docker-compose down
-
       - name: Run stress test (plain)
         run: |
           cd tests/stress

ADRs/001_tantivy.md

@@ -1,32 +0,0 @@
-# Tantivy ADR
-
-**Author**
-Marko Budiselic (github.com/gitbuda)
-
-**Status**
-APPROVED
-
-**Date**
-January 5, 2024
-
-**Problem**
-
-For some of Memgraph workloads, text search is a required feature. We don't
-want to build a new text search engine because that's not Memgraph's core
-value.
-
-**Criteria**
-
-- easy integration with our C++ codebase
-- ability to operate in-memory and on-disk
-- sufficient features (regex, full-text search, fuzzy search, aggregations over
-  text data)
-- production-ready
-
-**Decision**
-
-All known C++ libraries are not production-ready. Recent Rust libraries, in
-particular [Tantivy](https://github.com/quickwit-oss/tantivy), seem to provide
-much more features, it is production ready. The way how we'll integrate Tantivy
-into the current Memgraph codebase is via
-[cxx](https://github.com/dtolnay/cxx). **We select Tantivy.**

src/memory/query_memory_control.cpp

@@ -1,4 +1,4 @@
-// Copyright 2024 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -36,22 +36,14 @@ namespace memgraph::memory {
 
 void QueriesMemoryControl::UpdateThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
   auto accessor = thread_id_to_transaction_id.access();
-  auto elem = accessor.find(thread_id);
-  if (elem == accessor.end()) {
-    accessor.insert({thread_id, {transaction_id, 1}});
-  } else {
-    elem->transaction_id.cnt++;
-  }
+  accessor.insert({thread_id, transaction_id});
 }
 
 void QueriesMemoryControl::EraseThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
   auto accessor = thread_id_to_transaction_id.access();
   auto elem = accessor.find(thread_id);
   MG_ASSERT(elem != accessor.end() && elem->transaction_id == transaction_id);
-  elem->transaction_id.cnt--;
-  if (elem->transaction_id.cnt == 0) {
-    accessor.remove(thread_id);
-  }
+  accessor.remove(thread_id);
 }
 
 void QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {

src/memory/query_memory_control.hpp

@@ -1,4 +1,4 @@
-// Copyright 2024 Memgraph Ltd.
+// Copyright 2023 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -78,20 +78,9 @@ class QueriesMemoryControl {
   bool IsThreadTracked();
 
  private:
-  struct TransactionId {
-    uint64_t id;
-    uint64_t cnt;
-
-    bool operator<(const TransactionId &other) const { return id < other.id; }
-    bool operator==(const TransactionId &other) const { return id == other.id; }
-
-    bool operator<(uint64_t other) const { return id < other; }
-    bool operator==(uint64_t other) const { return id == other; }
-  };
-
   struct ThreadIdToTransactionId {
     std::thread::id thread_id;
-    TransactionId transaction_id;
+    uint64_t transaction_id;
 
     bool operator<(const ThreadIdToTransactionId &other) const { return thread_id < other.thread_id; }
     bool operator==(const ThreadIdToTransactionId &other) const { return thread_id == other.thread_id; }
@@ -109,9 +98,6 @@ class QueriesMemoryControl {
 
     bool operator<(uint64_t other) const { return transaction_id < other; }
     bool operator==(uint64_t other) const { return transaction_id == other; }
-
-    bool operator<(TransactionId other) const { return transaction_id < other.id; }
-    bool operator==(TransactionId other) const { return transaction_id == other.id; }
   };
 
   utils::SkipList<ThreadIdToTransactionId> thread_id_to_transaction_id;

tests/drivers/run.sh

@@ -37,7 +37,7 @@ $binary_dir/memgraph \
     --bolt-cert-file="" \
     --log-file=$tmpdir/logs/memgarph.log \
     --also-log-to-stderr \
-    --log-level TRACE &
+    --log-level ERROR &
 pid=$!
 wait_for_server 7687
 

tests/drivers/run_v1.sh

@@ -33,7 +33,7 @@ $binary_dir/memgraph \
     --bolt-cert-file="" \
     --log-file=$tmpdir/logs/memgarph.log \
     --also-log-to-stderr \
-    --log-level TRACE &
+    --log-level ERROR &
 pid=$!
 wait_for_server 7687
 

tests/mgbench/workloads/order_by.py

@@ -0,0 +1,46 @@
+# Copyright 2023 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+from workloads.base import Workload
+
+
+class OrderBy(Workload):
+    NAME = "order_by"
+    CARDINALITY = 10000
+
+    def indexes_generator(self):
+        return [
+            ("CREATE INDEX ON :Node;", {}),
+            ("CREATE INDEX ON :Node(prop1);", {}),
+            ("CREATE INDEX ON :Node(prop2);", {}),
+            ("CREATE INDEX ON :Node(prop3);", {}),
+        ]
+
+    def dataset_generator(self):
+        queries = []
+        for i in range(0, OrderBy.CARDINALITY):
+            queries.append(
+                (
+                    """CREATE
+(:Node {prop1: $id, prop2: $id, prop3: $id
+});
+""",
+                    {"id": i},
+                )
+            )
+
+        return queries
+
+    def benchmark__test__order_by(self):
+        return (
+            "MATCH (n:Node) RETURN n ORDER BY n.prop1",
+            {},
+        )

tests/stress/continuous_integration

@@ -93,11 +93,9 @@ def start_memgraph(args: Args, memgraph_options: List[str]) -> Popen:
         "--storage-recover-on-startup=false",
         "--query-execution-timeout-sec=1200",
         "--bolt-server-name-for-init=Neo4j/",
-        "--log-level=TRACE",
-        "--also-log-to-stderr=true",
     ] + memgraph_options
-    # if not args.verbose:
-    #     cmd += ["--log-level", "WARNING"]
+    if not args.verbose:
+        cmd += ["--log-level", "WARNING"]
     if args.log_file:
         cmd += ["--log-file", args.log_file]
     if args.data_directory:
@@ -147,15 +145,10 @@ def run_test(args: Args, test: str, options: List[str], timeout: int, mode: Data
         + options
     )
     start = time.time()
-    ret_test = subprocess.run(cmd, cwd=SCRIPT_DIR, timeout=timeout * 60, capture_output=True)
+    ret_test = subprocess.run(cmd, cwd=SCRIPT_DIR, timeout=timeout * 60)
 
     if ret_test.returncode != 0:
-        # raise Exception("Test '{}' binary returned non-zero ({})!".format(test, ret_test.returncode))
-        raise Exception(
-            "Test '{}' binary returned non-zero ({}). Stdout: {}. Stderr: {}!".format(
-                test, ret_test.returncode, ret_test.stdout, ret_test.stderr
-            )
-        )
+        raise Exception("Test '{}' binary returned non-zero ({})!".format(test, ret_test.returncode))
 
     runtime = time.time() - start
     print("    Done after {:.3f} seconds".format(runtime))