Target nightcrawler for rel build

.github/workflows/release_debian10.yaml

@@ -178,7 +178,7 @@ jobs:
 
   release_build:
     name: "Release build"
-    runs-on: [self-hosted, Linux, X64, Debian10]
+    runs-on: [self-hosted, Linux, X64, Debian10, nightcrawler]
     env:
       THREADS: 24
       MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
@@ -210,112 +210,11 @@ jobs:
           cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
           make -j$THREADS
 
-      - name: Create enterprise DEB package
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          cd build
-
-          # create mgconsole
-          # we use the -B to force the build
-          make -j$THREADS -B mgconsole
-
-          # Create enterprise DEB package.
-          mkdir output && cd output
-          cpack -G DEB --config ../CPackConfig.cmake
-
-      - name: Save enterprise DEB package
-        uses: actions/upload-artifact@v3
-        with:
-          name: "Enterprise DEB package"
-          path: build/output/memgraph*.deb
-
-      - name: Run micro benchmark tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run micro benchmark tests.
-          cd build
-          # The `eval` benchmark needs a large stack limit.
-          ulimit -s 262144
-          ctest -R memgraph__benchmark -V
-
-      - name: Run macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QuerySuite MemgraphRunner \
-            --groups aggregation 1000_create unwind_create dense_expand match \
-            --no-strict
-
-      - name: Run parallel macro benchmark tests
-        run: |
-          cd tests/macro_benchmark
-          ./harness QueryParallelSuite MemgraphRunner \
-            --groups aggregation_parallel create_parallel bfs_parallel \
-            --num-database-workers 9 --num-clients-workers 30 \
-            --no-strict
-
-      - name: Run GQL Behave tests
-        run: |
-          cd tests
-          ./setup.sh /opt/toolchain-v4/activate
-          cd gql_behave
-          ./continuous_integration
-
-      - name: Save quality assurance status
-        uses: actions/upload-artifact@v3
-        with:
-          name: "GQL Behave Status"
-          path: |
-            tests/gql_behave/gql_behave_status.csv
-            tests/gql_behave/gql_behave_status.html
-
-      - name: Run unit tests
-        run: |
-          # Activate toolchain.
-          source /opt/toolchain-v4/activate
-
-          # Run unit tests.
-          cd build
-          ctest -R memgraph__unit --output-on-failure
-
-      - name: Run e2e tests
-        run: |
-          cd tests
-          ./setup.sh /opt/toolchain-v4/activate
-          source ve3/bin/activate_e2e
-          cd e2e
-          ./run.sh
-
-      - name: Run stress test (plain)
-        run: |
-          cd tests/stress
-          ./continuous_integration
-
-      - name: Run stress test (SSL)
-        run: |
-          cd tests/stress
-          ./continuous_integration --use-ssl
-
       - name: Run stress test (large)
         run: |
           cd tests/stress
           ./continuous_integration --large-dataset
 
-      - name: Run durability test (plain)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 5
-
-      - name: Run durability test (large)
-        run: |
-          cd tests/stress
-          source ve3/bin/activate
-          python3 durability --num-steps 20
-
   release_jepsen_test:
     name: "Release Jepsen Test"
     runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]

tests/stress/continuous_integration

@@ -93,17 +93,21 @@ def start_memgraph(args: Args, memgraph_options: List[str]) -> Popen:
         "--storage-recover-on-startup=false",
         "--query-execution-timeout-sec=1200",
         "--bolt-server-name-for-init=Neo4j/",
+        "--log-level=TRACE",
+        "--also-log-to-stderr",
     ] + memgraph_options
-    if not args.verbose:
+    # if not args.verbose:
-        cmd += ["--log-level", "WARNING"]
+    #     cmd += ["--log-level", "WARNING"]
     if args.log_file:
         cmd += ["--log-file", args.log_file]
     if args.data_directory:
         cmd += ["--data-directory", args.data_directory]
     if args.use_ssl:
         cmd += ["--bolt-cert-file", CERT_FILE, "--bolt-key-file", KEY_FILE]
+    print(f"CMD: {cmd}")
     memgraph_proc = subprocess.Popen(cmd, cwd=cwd)
     wait_for_server(7687)
+    print(f"Memgraph started with pid {memgraph_proc.pid}")
 
     assert memgraph_proc.poll() is None, "The database binary died prematurely!"
 
@@ -145,10 +149,14 @@ def run_test(args: Args, test: str, options: List[str], timeout: int, mode: Data
         + options
     )
     start = time.time()
-    ret_test = subprocess.run(cmd, cwd=SCRIPT_DIR, timeout=timeout * 60)
+    ret_test = subprocess.run(cmd, cwd=SCRIPT_DIR, timeout=timeout * 60, capture_output=True)
 
     if ret_test.returncode != 0:
-        raise Exception("Test '{}' binary returned non-zero ({})!".format(test, ret_test.returncode))
+        raise Exception(
+            "Test '{}' binary returned non-zero ({}). Stdout: {}. Stderr: {}!".format(
+                test, ret_test.returncode, ret_test.stdout, ret_test.stderr
+            )
+        )
 
     runtime = time.time() - start
     print("    Done after {:.3f} seconds".format(runtime))

tests/stress/test_config.py

@@ -133,62 +133,22 @@ SMALL_DATASET = [
 # bipartite.py and create_match.py run for approx. 15min
 # long_running runs for 5min x 6 times = 30min
 # long_running runs for 8h
-LARGE_DATASET = (
+LARGE_DATASET = [
-    [
+    {
-        {
+        DatasetConstants.TEST: "long_running.cpp",
-            DatasetConstants.TEST: "bipartite.py",
+        DatasetConstants.OPTIONS: [
-            DatasetConstants.OPTIONS: ["--u-count", "300", "--v-count", "300"],
+            "--vertex-count",
-            DatasetConstants.TIMEOUT: 30,
+            "200000",
-            DatasetConstants.MODE: [get_default_database_mode()],
+            "--edge-count",
-        },
+            "1000000",
-        {
+            "--max-time",
-            DatasetConstants.TEST: "detach_delete.py",
+            "480",
-            DatasetConstants.OPTIONS: ["--worker-count", "4", "--repetition-count", "300"],
+            "--verify",
-            DatasetConstants.TIMEOUT: 5,
+            "300",
-            DatasetConstants.MODE: [get_default_database_mode()],
+            "--stats-file",
-        },
+            STATS_FILE,
-        {
+        ],
-            DatasetConstants.TEST: "create_match.py",
+        DatasetConstants.TIMEOUT: 500,
-            DatasetConstants.OPTIONS: ["--vertex-count", "500000", "--create-pack-size", "500"],
+        DatasetConstants.MODE: [get_default_database_mode()],
-            DatasetConstants.TIMEOUT: 30,
+    },
-            DatasetConstants.MODE: [get_default_database_mode()],
+]
-        },
-    ]
-    + [
-        {
-            DatasetConstants.TEST: "long_running.cpp",
-            DatasetConstants.OPTIONS: [
-                "--vertex-count",
-                "10000",
-                "--edge-count",
-                "40000",
-                "--max-time",
-                "5",
-                "--verify",
-                "60",
-            ],
-            DatasetConstants.TIMEOUT: 16,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-    ]
-    * 6
-    + [
-        {
-            DatasetConstants.TEST: "long_running.cpp",
-            DatasetConstants.OPTIONS: [
-                "--vertex-count",
-                "200000",
-                "--edge-count",
-                "1000000",
-                "--max-time",
-                "480",
-                "--verify",
-                "300",
-                "--stats-file",
-                STATS_FILE,
-            ],
-            DatasetConstants.TIMEOUT: 500,
-            DatasetConstants.MODE: [get_default_database_mode()],
-        },
-    ]
-)