typo

.github/workflows/release_debian10.yaml

@@ -178,7 +178,7 @@ jobs:
 
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Debian10, BigMemory]
+    runs-on: [self-hosted, Linux, X64, Debian10, nightcrawler]
     env:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
@@ -201,121 +201,16 @@ jobs:
           # Initialize dependencies.
           ./init
 
-          # Set default build_type to Release
-          INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
-          BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
-
           # Build release binaries.
           cd build
-          cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
+          cmake -DCMAKE_BUILD_TYPE=Debug ..
           make -j$THREADS
 
-      - name: Create enterprise DEB package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise DEB package.
-          mkdir output && cd output
-          cpack -G DEB --config ../CPackConfig.cmake
-
-      - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Enterprise DEB package"
-          path: build/output/memgraph*.deb
-
-      - name: Run micro benchmark tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run micro benchmark tests.
-          cd build
-          # The `eval` benchmark needs a large stack limit.
-          ulimit -s 262144
-          ctest -R memgraph__benchmark -V
-
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Run parallel macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QueryParallelSuite MemgraphRunner \
-            --groups aggregation_parallel create_parallel bfs_parallel \
-            --num-database-workers 9 --num-clients-workers 30 \
-            --no-strict
-
-      - name: Run GQL Behave tests
-        run: |
-          cd tests
-          ./setup.sh /opt/toolchain-v4/activate
-          cd gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Run e2e tests
-        run: |
-          cd tests
-          ./setup.sh /opt/toolchain-v4/activate
-          source ve3/bin/activate_e2e
-          cd e2e
-          ./run.sh
-
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
       - name: Run stress test (large)
         run: |
           cd tests/stress
           ./continuous_integration --large-dataset
 
-      - name: Run durability test (plain)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Run durability test (large)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 20
-
   release_jepsen_test:
     name: "Release Jepsen Test"
     runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]

.github/workflows/release_ubuntu2004.yaml

@@ -281,6 +281,13 @@ jobs:
           cd build
           ctest -R memgraph__unit --output-on-failure
 
+      - name: Ensure Kafka and Pulsar are up
+        run: |
+          cd tests/e2e/streams/kafka
+          docker-compose up -d
+          cd ../pulsar
+          docker-compose up -d
+
       - name: Run e2e tests
         run: |
           cd tests
@@ -289,6 +296,14 @@ jobs:
           cd e2e
           ./run.sh
 
+      - name: Ensure Kafka and Pulsar are down
+        if: always()
+        run: |
+          cd tests/e2e/streams/kafka
+          docker-compose down
+          cd ../pulsar
+          docker-compose down
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress

ADRs/001_tantivy.md

@@ -0,0 +1,32 @@
+# Tantivy ADR
+
+**Author**
+Marko Budiselic (github.com/gitbuda)
+
+**Status**
+APPROVED
+
+**Date**
+January 5, 2024
+
+**Problem**
+
+For some of Memgraph workloads, text search is a required feature. We don't
+want to build a new text search engine because that's not Memgraph's core
+value.
+
+**Criteria**
+
+- easy integration with our C++ codebase
+- ability to operate in-memory and on-disk
+- sufficient features (regex, full-text search, fuzzy search, aggregations over
+  text data)
+- production-ready
+
+**Decision**
+
+All known C++ libraries are not production-ready. Recent Rust libraries, in
+particular [Tantivy](https://github.com/quickwit-oss/tantivy), seem to provide
+much more features, it is production ready. The way how we'll integrate Tantivy
+into the current Memgraph codebase is via
+[cxx](https://github.com/dtolnay/cxx). **We select Tantivy.**

src/memory/query_memory_control.cpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -36,14 +36,22 @@ namespace memgraph::memory {
 
 void QueriesMemoryControl::UpdateThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
   auto accessor = thread_id_to_transaction_id.access();
-  accessor.insert({thread_id, transaction_id});
+  auto elem = accessor.find(thread_id);
+  if (elem == accessor.end()) {
+    accessor.insert({thread_id, {transaction_id, 1}});
+  } else {
+    elem->transaction_id.cnt++;
+  }
 }
 
 void QueriesMemoryControl::EraseThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
   auto accessor = thread_id_to_transaction_id.access();
   auto elem = accessor.find(thread_id);
   MG_ASSERT(elem != accessor.end() && elem->transaction_id == transaction_id);
-  accessor.remove(thread_id);
+  elem->transaction_id.cnt--;
+  if (elem->transaction_id.cnt == 0) {
+    accessor.remove(thread_id);
+  }
 }
 
 void QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {

src/memory/query_memory_control.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2024 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -78,9 +78,20 @@ class QueriesMemoryControl {
   bool IsThreadTracked();
 
  private:
+  struct TransactionId {
+    uint64_t id;
+    uint64_t cnt;
+
+    bool operator<(const TransactionId &other) const { return id < other.id; }
+    bool operator==(const TransactionId &other) const { return id == other.id; }
+
+    bool operator<(uint64_t other) const { return id < other; }
+    bool operator==(uint64_t other) const { return id == other; }
+  };
+
   struct ThreadIdToTransactionId {
     std::thread::id thread_id;
-    uint64_t transaction_id;
+    TransactionId transaction_id;
 
     bool operator<(const ThreadIdToTransactionId &other) const { return thread_id < other.thread_id; }
     bool operator==(const ThreadIdToTransactionId &other) const { return thread_id == other.thread_id; }
@@ -98,6 +109,9 @@ class QueriesMemoryControl {
 
     bool operator<(uint64_t other) const { return transaction_id < other; }
     bool operator==(uint64_t other) const { return transaction_id == other; }
+
+    bool operator<(TransactionId other) const { return transaction_id < other.id; }
+    bool operator==(TransactionId other) const { return transaction_id == other.id; }
   };
 
   utils::SkipList<ThreadIdToTransactionId> thread_id_to_transaction_id;

tests/drivers/run.sh

@@ -37,7 +37,7 @@ $binary_dir/memgraph \
     --bolt-cert-file="" \
     --log-file=$tmpdir/logs/memgarph.log \
     --also-log-to-stderr \
-    --log-level ERROR &
+    --log-level TRACE &
 pid=$!
 wait_for_server 7687
 

tests/drivers/run_v1.sh

@@ -33,7 +33,7 @@ $binary_dir/memgraph \
     --bolt-cert-file="" \
     --log-file=$tmpdir/logs/memgarph.log \
     --also-log-to-stderr \
-    --log-level ERROR &
+    --log-level TRACE &
 pid=$!
 wait_for_server 7687
 

tests/stress/continuous_integration

@@ -93,9 +93,11 @@ def start_memgraph(args: Args, memgraph_options: List[str]) -> Popen:
         "--storage-recover-on-startup=false",
         "--query-execution-timeout-sec=1200",
         "--bolt-server-name-for-init=Neo4j/",
+        "--log-level=TRACE",
+        "--also-log-to-stderr=true",
     ] + memgraph_options
-    if not args.verbose:
-        cmd += ["--log-level", "WARNING"]
+    # if not args.verbose:
+    #     cmd += ["--log-level", "WARNING"]
     if args.log_file:
         cmd += ["--log-file", args.log_file]
     if args.data_directory:
@@ -145,10 +147,15 @@ def run_test(args: Args, test: str, options: List[str], timeout: int, mode: Data
         + options
     )
     start = time.time()
-    ret_test = subprocess.run(cmd, cwd=SCRIPT_DIR, timeout=timeout * 60)
+    ret_test = subprocess.run(cmd, cwd=SCRIPT_DIR, timeout=timeout * 60, capture_output=True)
 
     if ret_test.returncode != 0:
-        raise Exception("Test '{}' binary returned non-zero ({})!".format(test, ret_test.returncode))
+        # raise Exception("Test '{}' binary returned non-zero ({})!".format(test, ret_test.returncode))
+        raise Exception(
+            "Test '{}' binary returned non-zero ({}). Stdout: {}. Stderr: {}!".format(
+                test, ret_test.returncode, ret_test.stdout, ret_test.stderr
+            )
+        )
 
     runtime = time.time() - start
     print("    Done after {:.3f} seconds".format(runtime))