Compare commits
4 Commits
analyze-fl
...
detach-del
| Author | SHA1 | Date | |
|---|---|---|---|
|
4dc6400681 | ||
|
|
675c166254 | ||
|
|
70dabbd926 | ||
|
|
05c9c28dca |
649
.github/workflows/analyze_diff.yaml
vendored
649
.github/workflows/analyze_diff.yaml
vendored
@@ -1,649 +0,0 @@
|
||||
name: Analyze diff flakiness
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
# pull_request:
|
||||
# paths-ignore:
|
||||
# - "docs/**"
|
||||
# - "**/*.md"
|
||||
# - ".clang-format"
|
||||
# - "CODEOWNERS"
|
||||
# - "licenses/*"
|
||||
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build community binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build community binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMG_ENTERPRISE=OFF ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
code_analysis:
|
||||
name: "Code analysis"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
# This is also needed if we want do to comparison against other branches
|
||||
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
|
||||
- name: Fetch all history for all tags and branches
|
||||
run: git fetch
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Set base branch
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set base branch # if we manually dispatch or push to master
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
|
||||
|
||||
- name: Python code analysis
|
||||
run: |
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only --diff-filter=d)
|
||||
for file in ${CHANGED_FILES}; do
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
python3 -m black --check --diff ${file}
|
||||
python3 -m isort --profile black --check-only --diff ${file}
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
|
||||
cd build
|
||||
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
|
||||
|
||||
- name: Compute code coverage
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Compute code coverage.
|
||||
cd tools/github
|
||||
./coverage_convert
|
||||
|
||||
# Package code coverage.
|
||||
cd generated
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build debug binaries.
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run leftover CTest tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run leftover CTest tests (all except unit and benchmark tests).
|
||||
cd build
|
||||
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
|
||||
|
||||
- name: Run drivers tests
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run cppcheck and clang-format.
|
||||
cd tools/github
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: always()
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
./continuous_integration
|
||||
|
||||
- name: Run stress test (SSL)
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run durability test
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 5
|
||||
|
||||
- name: Create enterprise DEB package
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
cd build
|
||||
|
||||
# create mgconsole
|
||||
# we use the -B to force the build
|
||||
make -j$THREADS -B mgconsole
|
||||
|
||||
# Create enterprise DEB package.
|
||||
mkdir output && cd output
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
#
|
||||
# experimental_build_ha:
|
||||
# name: "High availability build"
|
||||
# runs-on: [self-hosted, Linux, X64, Diff]
|
||||
# strategy:
|
||||
# max-parallel: 1
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
# env:
|
||||
# THREADS: 24
|
||||
# MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
# MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
#
|
||||
# steps:
|
||||
# - name: Set up repository
|
||||
# uses: actions/checkout@v3
|
||||
# with:
|
||||
# # Number of commits to fetch. `0` indicates all history for all
|
||||
# # branches and tags. (default: 1)
|
||||
# fetch-depth: 0
|
||||
#
|
||||
# - name: Initialize deps
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Initialize dependencies.
|
||||
# ./init
|
||||
#
|
||||
# - name: Build release binaries
|
||||
# run: |
|
||||
# source /opt/toolchain-v4/activate
|
||||
# cd build
|
||||
# cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
|
||||
# make -j$THREADS
|
||||
# - name: Run unit tests
|
||||
# run: |
|
||||
# source /opt/toolchain-v4/activate
|
||||
# cd build
|
||||
# ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
# - name: Run e2e tests
|
||||
# run: |
|
||||
# cd tests
|
||||
# ./setup.sh /opt/toolchain-v4/activate
|
||||
# source ve3/bin/activate_e2e
|
||||
# cd e2e
|
||||
# ./run.sh "Coordinator"
|
||||
# ./run.sh "Client initiated failover"
|
||||
# ./run.sh "Uninitialized cluster"
|
||||
# - name: Save test data
|
||||
# uses: actions/upload-artifact@v3
|
||||
# if: always()
|
||||
# with:
|
||||
# name: "Test data"
|
||||
# path: |
|
||||
# # multiple paths could be defined
|
||||
# build/logs
|
||||
#
|
||||
# experimental_build_mt:
|
||||
# name: "MultiTenancy replication build"
|
||||
# runs-on: [self-hosted, Linux, X64, Diff]
|
||||
# strategy:
|
||||
# max-parallel: 1
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
# env:
|
||||
# THREADS: 24
|
||||
# MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
# MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
#
|
||||
# steps:
|
||||
# - name: Set up repository
|
||||
# uses: actions/checkout@v3
|
||||
# with:
|
||||
# # Number of commits to fetch. `0` indicates all history for all
|
||||
# # branches and tags. (default: 1)
|
||||
# fetch-depth: 0
|
||||
#
|
||||
#
|
||||
# - name: Initialize deps
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Initialize dependencies.
|
||||
# ./init
|
||||
#
|
||||
# - name: Build release binaries
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Build MT replication experimental binaries.
|
||||
# cd build
|
||||
# cmake -DCMAKE_BUILD_TYPE=Release -D MG_EXPERIMENTAL_REPLICATION_MULTITENANCY=ON ..
|
||||
# make -j$THREADS
|
||||
#
|
||||
# - name: Run unit tests
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Run unit tests.
|
||||
# cd build
|
||||
# ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
#
|
||||
# - name: Run e2e tests
|
||||
# run: |
|
||||
# cd tests
|
||||
# ./setup.sh /opt/toolchain-v4/activate
|
||||
# source ve3/bin/activate_e2e
|
||||
# cd e2e
|
||||
#
|
||||
# # Just the replication based e2e tests
|
||||
# ./run.sh "Replicate multitenancy"
|
||||
# ./run.sh "Show"
|
||||
# ./run.sh "Show while creating invalid state"
|
||||
# ./run.sh "Delete edge replication"
|
||||
# ./run.sh "Read-write benchmark"
|
||||
# ./run.sh "Index replication"
|
||||
# ./run.sh "Constraints"
|
||||
#
|
||||
# - name: Save test data
|
||||
# uses: actions/upload-artifact@v3
|
||||
# if: always()
|
||||
# with:
|
||||
# name: "Test data"
|
||||
# path: |
|
||||
# # multiple paths could be defined
|
||||
# build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
#continue-on-error: true
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Refresh Jepsen Cluster
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh cluster-refresh
|
||||
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v3
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build only memgraph release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Get branch name (merge)
|
||||
if: github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
|
||||
|
||||
- name: Get branch name (pull request)
|
||||
if: github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
|
||||
|
||||
- name: Upload macro benchmark results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
# TODO (andi) No need for path flags and for --disk-storage and --in-memory-analytical
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
72
.github/workflows/analyze_e2e.yaml
vendored
72
.github/workflows/analyze_e2e.yaml
vendored
@@ -1,72 +0,0 @@
|
||||
name: Analyze e2e flakiness
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
- ".clang-format"
|
||||
- "CODEOWNERS"
|
||||
- "licenses/*"
|
||||
|
||||
jobs:
|
||||
release_e2e_test:
|
||||
name: "Release End-to-end Test"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 5
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: always()
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
22
.github/workflows/diff.yaml
vendored
22
.github/workflows/diff.yaml
vendored
@@ -4,18 +4,18 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
# push:
|
||||
# branches:
|
||||
# - master
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
# pull_request:
|
||||
# paths-ignore:
|
||||
# - "docs/**"
|
||||
# - "**/*.md"
|
||||
# - ".clang-format"
|
||||
# - "CODEOWNERS"
|
||||
# - "licenses/*"
|
||||
#
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
- ".clang-format"
|
||||
- "CODEOWNERS"
|
||||
- "licenses/*"
|
||||
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -54,6 +54,13 @@ DEFINE_double(query_execution_timeout_sec, 600,
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(cartesian_product_enabled, true, "Enable cartesian product expansion.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_int64(maximum_deltas_per_transaction, -1, "Limit of deltas per transaction, default -1 (no limit)");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_int64(maximum_delete_deltas_per_transaction, -1,
|
||||
"Limit of delete deltas per transaction, default -1 (no limit)");
|
||||
|
||||
namespace {
|
||||
// Bolt server name
|
||||
constexpr auto kServerNameSettingKey = "server.name";
|
||||
@@ -73,12 +80,24 @@ constexpr auto kLogToStderrGFlagsKey = "also_log_to_stderr";
|
||||
constexpr auto kCartesianProductEnabledSettingKey = "cartesian-product-enabled";
|
||||
constexpr auto kCartesianProductEnabledGFlagsKey = "cartesian-product-enabled";
|
||||
|
||||
constexpr auto kMaximumDeltasPerTransactionSettingKey = "maximum-deltas-per-transaction";
|
||||
constexpr auto kMaximumDeltasPerTransactionGFlagsKey = "maximum-deltas-per-transaction";
|
||||
|
||||
constexpr auto kMaximumDeleteDeltasPerTransactionSettingKey = "maximum-delete-deltas-per-transaction";
|
||||
constexpr auto kMaximumDeleteDeltasPerTransactionGFlagsKey = "maximum-delete-deltas-per-transaction";
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
std::atomic<double> execution_timeout_sec_; // Local cache-like thing
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
std::atomic<bool> cartesian_product_enabled_{true}; // Local cache-like thing
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
std::atomic<int64_t> maximum_deltas_per_transaction_; // Local cache-like thing
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
std::atomic<int64_t> maximum_delete_deltas_per_transaction_; // Local cache-like thing
|
||||
|
||||
auto ToLLEnum(std::string_view val) {
|
||||
const auto ll_enum = memgraph::flags::LogLevelToEnum(val);
|
||||
if (!ll_enum) {
|
||||
@@ -186,6 +205,22 @@ void Initialize() {
|
||||
register_flag(
|
||||
kCartesianProductEnabledGFlagsKey, kCartesianProductEnabledSettingKey, !kRestore,
|
||||
[](const std::string &val) { cartesian_product_enabled_ = val == "true"; }, ValidBoolStr);
|
||||
|
||||
/*
|
||||
* Register maximum deltas per transaction
|
||||
*/
|
||||
register_flag(kMaximumDeltasPerTransactionGFlagsKey, kMaximumDeltasPerTransactionSettingKey, !kRestore,
|
||||
[&](const std::string &val) {
|
||||
maximum_deltas_per_transaction_ = std::stoll(val); // Cache for faster reads
|
||||
});
|
||||
|
||||
/*
|
||||
* Register maximum delete deltas per transaction
|
||||
*/
|
||||
register_flag(kMaximumDeleteDeltasPerTransactionGFlagsKey, kMaximumDeleteDeltasPerTransactionSettingKey, !kRestore,
|
||||
[&](const std::string &val) {
|
||||
maximum_delete_deltas_per_transaction_ = std::stoll(val); // Cache for faster reads
|
||||
});
|
||||
}
|
||||
|
||||
std::string GetServerName() {
|
||||
@@ -199,4 +234,8 @@ double GetExecutionTimeout() { return execution_timeout_sec_; }
|
||||
|
||||
bool GetCartesianProductEnabled() { return cartesian_product_enabled_; }
|
||||
|
||||
int64_t GetMaximumDeltasPerTransaction() { return maximum_deltas_per_transaction_; }
|
||||
|
||||
int64_t GetMaximumDeleteDeltasPerTransaction() { return maximum_delete_deltas_per_transaction_; }
|
||||
|
||||
} // namespace memgraph::flags::run_time
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -42,4 +42,18 @@ double GetExecutionTimeout();
|
||||
*/
|
||||
bool GetCartesianProductEnabled();
|
||||
|
||||
/**
|
||||
* @brief Get the maximum amount of deltas per transaction
|
||||
*
|
||||
* @return int64_t
|
||||
*/
|
||||
int64_t GetMaximumDeltasPerTransaction();
|
||||
|
||||
/**
|
||||
* @brief Get the maximum amount of delete deltas per transaction
|
||||
*
|
||||
* @return int64_t
|
||||
*/
|
||||
int64_t GetMaximumDeleteDeltasPerTransaction();
|
||||
|
||||
} // namespace memgraph::flags::run_time
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#######################
|
||||
# ######################
|
||||
find_package(gflags REQUIRED)
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
@@ -42,4 +42,4 @@ add_library(mg-storage-v2 STATIC
|
||||
inmemory/replication/recovery.cpp
|
||||
)
|
||||
|
||||
target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory)
|
||||
target_link_libraries(mg-storage-v2 mg::replication Threads::Threads mg-utils gflags absl::flat_hash_map mg-rpc mg-slk mg-events mg-memory mg-flags)
|
||||
|
||||
@@ -21,6 +21,9 @@
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/metadata_delta.hpp"
|
||||
|
||||
/// FLAGS ///
|
||||
#include "flags/run_time_configurable.hpp"
|
||||
|
||||
/// REPLICATION ///
|
||||
#include "dbms/inmemory/replication_handlers.hpp"
|
||||
#include "storage/v2/inmemory/replication/recovery.hpp"
|
||||
@@ -176,9 +179,9 @@ InMemoryStorage::~InMemoryStorage() {
|
||||
committed_transactions_.WithLock([](auto &transactions) { transactions.clear(); });
|
||||
}
|
||||
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(auto tag, InMemoryStorage *storage, IsolationLevel isolation_level,
|
||||
StorageMode storage_mode,
|
||||
memgraph::replication_coordination_glue::ReplicationRole replication_role)
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(
|
||||
auto tag, InMemoryStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode,
|
||||
memgraph::replication_coordination_glue::ReplicationRole replication_role)
|
||||
: Accessor(tag, storage, isolation_level, storage_mode, replication_role),
|
||||
config_(storage->config_.salient.items) {}
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(InMemoryAccessor &&other) noexcept
|
||||
@@ -1303,7 +1306,17 @@ Transaction InMemoryStorage::CreateTransaction(
|
||||
start_timestamp = timestamp_;
|
||||
}
|
||||
}
|
||||
return {transaction_id, start_timestamp, isolation_level, storage_mode, false};
|
||||
|
||||
auto maximum_deltas_per_transaction = flags::run_time::GetMaximumDeltasPerTransaction();
|
||||
auto maximum_delete_deltas_per_transaction = flags::run_time::GetMaximumDeleteDeltasPerTransaction();
|
||||
|
||||
return {transaction_id,
|
||||
start_timestamp,
|
||||
isolation_level,
|
||||
storage_mode,
|
||||
false,
|
||||
maximum_deltas_per_transaction,
|
||||
maximum_delete_deltas_per_transaction};
|
||||
}
|
||||
|
||||
void InMemoryStorage::SetStorageMode(StorageMode new_storage_mode) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,15 +14,20 @@
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <unordered_set>
|
||||
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/rocksdb_serialization.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
const std::unordered_set<Delta::Action> delete_delta_actions{Delta::Action::ADD_IN_EDGE, Delta::Action::ADD_OUT_EDGE,
|
||||
Delta::Action::RECREATE_OBJECT};
|
||||
|
||||
/// This function iterates through the undo buffers from an object (starting
|
||||
/// from the supplied delta) and determines what deltas should be applied to get
|
||||
/// the currently visible version of the object. When the function finds a delta
|
||||
@@ -104,6 +109,21 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
|
||||
return false;
|
||||
}
|
||||
|
||||
inline void IncrementDeltasChanged(Transaction *transaction, Delta *delta) {
|
||||
if (transaction->max_deltas > -1) {
|
||||
transaction->deltas_changed++;
|
||||
if (transaction->deltas_changed > transaction->max_deltas) {
|
||||
throw utils::BasicException(
|
||||
"You have reached the maximum number of deltas for a transaction, transaction will be rollbacked!");
|
||||
}
|
||||
}
|
||||
|
||||
if (transaction->max_delete_deltas > -1 && delete_delta_actions.contains(delta->action)) {
|
||||
throw utils::BasicException(
|
||||
"You have reached the maximum number of delete deltas for a transaction, transaction will be rollbacked!");
|
||||
}
|
||||
}
|
||||
|
||||
/// This function creates a `DELETE_OBJECT` delta in the transaction and returns
|
||||
/// a pointer to the created delta. It doesn't perform any linking of the delta
|
||||
/// and is primarily used to create the first delta for an object (that must be
|
||||
@@ -114,16 +134,13 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
|
||||
return nullptr;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
return &transaction->deltas.use().emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
}
|
||||
|
||||
inline Delta *CreateDeleteObjectDelta(Transaction *transaction, std::list<Delta> *deltas) {
|
||||
if (transaction->storage_mode == StorageMode::IN_MEMORY_ANALYTICAL) {
|
||||
return nullptr;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
return &deltas->emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(), transaction->command_id);
|
||||
auto *delta = &transaction->deltas.use().emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
|
||||
IncrementDeltasChanged(transaction, delta);
|
||||
|
||||
return delta;
|
||||
}
|
||||
|
||||
/// TODO: what if in-memory analytical
|
||||
@@ -165,9 +182,12 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
|
||||
return;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
|
||||
auto delta = &transaction->deltas.use().emplace_back(std::forward<Args>(args)..., transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
|
||||
IncrementDeltasChanged(transaction, delta);
|
||||
|
||||
// The operations are written in such order so that both `next` and `prev`
|
||||
// chains are valid at all times. The chains must be valid at all times
|
||||
// because garbage collection (which traverses the chains) is done
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -43,12 +43,15 @@ using PmrListDelta = utils::pmr::list<Delta>;
|
||||
|
||||
struct Transaction {
|
||||
Transaction(uint64_t transaction_id, uint64_t start_timestamp, IsolationLevel isolation_level,
|
||||
StorageMode storage_mode, bool edge_import_mode_active)
|
||||
StorageMode storage_mode, bool edge_import_mode_active, int64_t max_deltas = -1,
|
||||
int64_t max_delete_deltas = -1)
|
||||
: transaction_id(transaction_id),
|
||||
start_timestamp(start_timestamp),
|
||||
command_id(0),
|
||||
deltas(0),
|
||||
md_deltas(utils::NewDeleteResource()),
|
||||
max_deltas(max_deltas),
|
||||
max_delete_deltas(max_delete_deltas),
|
||||
must_abort(false),
|
||||
isolation_level(isolation_level),
|
||||
storage_mode(storage_mode),
|
||||
@@ -93,6 +96,10 @@ struct Transaction {
|
||||
|
||||
Bond<PmrListDelta> deltas;
|
||||
utils::pmr::list<MetadataDelta> md_deltas;
|
||||
int64_t max_deltas{-1};
|
||||
int64_t max_delete_deltas{-1};
|
||||
uint64_t deltas_changed{0};
|
||||
uint64_t delete_deltas_changed{0};
|
||||
bool must_abort{};
|
||||
IsolationLevel isolation_level{};
|
||||
StorageMode storage_mode{};
|
||||
|
||||
@@ -39,8 +39,8 @@ endfunction()
|
||||
|
||||
add_subdirectory(fine_grained_access)
|
||||
add_subdirectory(server)
|
||||
#add_subdirectory(replication)
|
||||
#add_subdirectory(memory)
|
||||
add_subdirectory(replication)
|
||||
add_subdirectory(memory)
|
||||
add_subdirectory(triggers)
|
||||
add_subdirectory(isolation_levels)
|
||||
add_subdirectory(streams)
|
||||
@@ -56,7 +56,7 @@ add_subdirectory(python_query_modules_reloading)
|
||||
add_subdirectory(analyze_graph)
|
||||
add_subdirectory(transaction_queue)
|
||||
add_subdirectory(mock_api)
|
||||
#add_subdirectory(graphql)
|
||||
add_subdirectory(graphql)
|
||||
add_subdirectory(disk_storage)
|
||||
add_subdirectory(load_csv)
|
||||
add_subdirectory(init_file_flags)
|
||||
@@ -76,6 +76,7 @@ add_subdirectory(queries)
|
||||
add_subdirectory(query_modules_storage_modes)
|
||||
add_subdirectory(garbage_collection)
|
||||
add_subdirectory(query_planning)
|
||||
add_subdirectory(maximum_deltas_restriction)
|
||||
|
||||
if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
add_subdirectory(high_availability_experimental)
|
||||
|
||||
@@ -90,6 +90,8 @@ startup_config_dict = {
|
||||
"TRACE",
|
||||
"Minimum log level. Allowed values: TRACE, DEBUG, INFO, WARNING, ERROR, CRITICAL",
|
||||
),
|
||||
"maximum_deltas_per_transaction": ("-1", "-1", "Limit of deltas per transaction, default -1 (no limit)"),
|
||||
"maximum_delete_deltas_per_transaction": ("-1", "-1", "Limit of deltas per transaction, default -1 (no limit)"),
|
||||
"memory_limit": (
|
||||
"0",
|
||||
"0",
|
||||
|
||||
@@ -48,6 +48,16 @@ read_query_modules_in_memory_cluster: &read_query_modules_in_memory_cluster
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
read_query_modules_disk_cluster: &read_query_modules_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_query_modules_in_memory_cluster: &update_query_modules_in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
@@ -56,6 +66,16 @@ update_query_modules_in_memory_cluster: &update_query_modules_in_memory_cluster
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_query_modules_disk_cluster: &update_query_modules_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
show_privileges_in_memory_cluster: &show_privileges_in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
@@ -64,6 +84,16 @@ show_privileges_in_memory_cluster: &show_privileges_in_memory_cluster
|
||||
setup_queries: *show_privileges_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
show_privileges_disk_cluster: &show_privileges_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *show_privileges_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
read_permission_in_memory_queries: &read_permission_in_memory_queries
|
||||
cluster:
|
||||
main:
|
||||
@@ -72,6 +102,16 @@ read_permission_in_memory_queries: &read_permission_in_memory_queries
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
read_permission_disk_queries: &read_permission_disk_queries
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
|
||||
create_delete_query_modules_in_memory_cluster: &create_delete_query_modules_in_memory_cluster
|
||||
cluster:
|
||||
@@ -81,6 +121,16 @@ create_delete_query_modules_in_memory_cluster: &create_delete_query_modules_in_m
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
create_delete_query_modules_disk_cluster: &create_delete_query_modules_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_permission_queries_in_memory_cluster: &update_permission_queries_in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
@@ -89,6 +139,15 @@ update_permission_queries_in_memory_cluster: &update_permission_queries_in_memor
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_permission_queries_disk_cluster: &update_permission_queries_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
workloads:
|
||||
- name: "read-query-modules"
|
||||
@@ -97,32 +156,68 @@ workloads:
|
||||
args: ["lba_procedures/read_query_modules.py"]
|
||||
<<: *read_query_modules_in_memory_cluster
|
||||
|
||||
- name: "read-query-modules on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/read_query_modules.py"]
|
||||
<<: *read_query_modules_disk_cluster
|
||||
|
||||
- name: "update-query-modules"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_query_modules.py"]
|
||||
<<: *update_query_modules_in_memory_cluster
|
||||
|
||||
- name: "update-query-modules on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_query_modules.py"]
|
||||
<<: *update_query_modules_disk_cluster
|
||||
|
||||
- name: "create-delete-query-modules"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/create_delete_query_modules.py"]
|
||||
<<: *create_delete_query_modules_in_memory_cluster
|
||||
|
||||
- name: "create-delete-query-modules on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/create_delete_query_modules.py"]
|
||||
<<: *create_delete_query_modules_disk_cluster
|
||||
|
||||
- name: "show-privileges"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/show_privileges.py"]
|
||||
<<: *show_privileges_in_memory_cluster
|
||||
|
||||
- name: "show-privileges on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/show_privileges.py"]
|
||||
<<: *show_privileges_disk_cluster
|
||||
|
||||
- name: "read-permission-queries"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/read_permission_queries.py"]
|
||||
<<: *read_permission_in_memory_queries
|
||||
|
||||
- name: "read-permission-queries on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/read_permission_queries.py"]
|
||||
<<: *read_permission_disk_queries
|
||||
|
||||
- name: "update-permission-queries"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_permission_queries.py"]
|
||||
<<: *update_permission_queries_in_memory_cluster
|
||||
|
||||
- name: "update-permission-queries on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_permission_queries.py"]
|
||||
<<: *update_permission_queries_disk_cluster
|
||||
|
||||
8
tests/e2e/maximum_deltas_restriction/CMakeLists.txt
Normal file
8
tests/e2e/maximum_deltas_restriction/CMakeLists.txt
Normal file
@@ -0,0 +1,8 @@
|
||||
function(copy_maximum_deltas_restriction_e2e_python_files FILE_NAME)
|
||||
copy_e2e_python_files(maximum_deltas_restriction ${FILE_NAME})
|
||||
endfunction()
|
||||
|
||||
copy_maximum_deltas_restriction_e2e_python_files(common.py)
|
||||
copy_maximum_deltas_restriction_e2e_python_files(maximum_deltas_restriction.py)
|
||||
|
||||
copy_e2e_files(maximum_deltas_restriction workloads.yaml)
|
||||
27
tests/e2e/maximum_deltas_restriction/common.py
Normal file
27
tests/e2e/maximum_deltas_restriction/common.py
Normal file
@@ -0,0 +1,27 @@
|
||||
# Copyright 2023 Memgraph Ltd.
|
||||
#
|
||||
# Use of this software is governed by the Business Source License
|
||||
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
# License, and you may not use this file except in compliance with the Business Source License.
|
||||
#
|
||||
# As of the Change Date specified in that file, in accordance with
|
||||
# the Business Source License, use of this software will be governed
|
||||
# by the Apache License, Version 2.0, included in the file
|
||||
# licenses/APL.txt.
|
||||
|
||||
import pytest
|
||||
from gqlalchemy import Memgraph
|
||||
|
||||
MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER = "SET DATABASE SETTING 'maximum-deltas-per-transaction' TO '$0';"
|
||||
MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER = "SET DATABASE SETTING 'maximum-delete-deltas-per-transaction' TO '$0';"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def memgraph(**kwargs) -> Memgraph:
|
||||
memgraph = Memgraph()
|
||||
|
||||
yield memgraph
|
||||
|
||||
memgraph.execute(MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "-1"))
|
||||
memgraph.execute(MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "-1"))
|
||||
memgraph.drop_database()
|
||||
@@ -0,0 +1,74 @@
|
||||
# Copyright 2023 Memgraph Ltd.
|
||||
#
|
||||
# Use of this software is governed by the Business Source License
|
||||
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
# License, and you may not use this file except in compliance with the Business Source License.
|
||||
#
|
||||
# As of the Change Date specified in that file, in accordance with
|
||||
# the Business Source License, use of this software will be governed
|
||||
# by the Apache License, Version 2.0, included in the file
|
||||
# licenses/APL.txt.
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from common import (
|
||||
MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER,
|
||||
MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER,
|
||||
memgraph,
|
||||
)
|
||||
from gqlalchemy import GQLAlchemyError
|
||||
|
||||
CREATE_EMPTY_NODES_PLACEHOLDER = "FOREACH (i in range(1, $0) | CREATE ());"
|
||||
CREATE_FULL_NODES_PLACEHOLDER = "FOREACH (i in range(1, $0) | CREATE (:Node {id: i}));"
|
||||
DELETE_EVERYTHING_QUERY = "MATCH (n) DETACH DELETE n;"
|
||||
|
||||
|
||||
def test_given_no_restrictions_on_the_database_when_executing_commands_then_everything_should_pass(memgraph):
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "100"))
|
||||
memgraph.execute(CREATE_FULL_NODES_PLACEHOLDER.replace("$0", "100"))
|
||||
memgraph.execute(DELETE_EVERYTHING_QUERY)
|
||||
|
||||
|
||||
def test_given_maximum_restriction_ingestion_fails_when_inserting_more_nodes(memgraph):
|
||||
memgraph.execute(MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
|
||||
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "100"))
|
||||
memgraph.execute(DELETE_EVERYTHING_QUERY)
|
||||
|
||||
with pytest.raises(GQLAlchemyError):
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "101"))
|
||||
|
||||
|
||||
def test_given_maximum_delete_restriction_deletion_fails_when_deleting_more_nodes(memgraph):
|
||||
memgraph.execute(MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
|
||||
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "2000"))
|
||||
|
||||
with pytest.raises(GQLAlchemyError):
|
||||
memgraph.execute(DELETE_EVERYTHING_QUERY)
|
||||
|
||||
|
||||
def test_given_maximum_delete_restriction_deletion_fails_when_deleting_more_edges(memgraph):
|
||||
memgraph.execute(MAXIMUM_DELETE_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
|
||||
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "2000"))
|
||||
memgraph.execute("CREATE (s:Supernode)")
|
||||
memgraph.execute("MATCH (s:Supernode) MATCH (n) CREATE (s)-[:HAS]->(n)")
|
||||
|
||||
with pytest.raises(GQLAlchemyError):
|
||||
memgraph.execute(DELETE_EVERYTHING_QUERY)
|
||||
|
||||
|
||||
def test_given_maximum_delta_restriction_fails_when_deleting_everything_on_batch_ingested_nodes(memgraph):
|
||||
memgraph.execute(MAXIMUM_DELTAS_RESTRICTION_PLACEHOLDER.replace("$0", "100"))
|
||||
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "90"))
|
||||
memgraph.execute(CREATE_EMPTY_NODES_PLACEHOLDER.replace("$0", "90"))
|
||||
|
||||
with pytest.raises(GQLAlchemyError):
|
||||
memgraph.execute(DELETE_EVERYTHING_QUERY)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-rA"]))
|
||||
14
tests/e2e/maximum_deltas_restriction/workloads.yaml
Normal file
14
tests/e2e/maximum_deltas_restriction/workloads.yaml
Normal file
@@ -0,0 +1,14 @@
|
||||
maximum_deltas_restriction_cluster: &maximum_deltas_restriction_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: ["--bolt-port", "7687", "--log-level=TRACE"]
|
||||
log_file: "maximum_deltas_restriction.log"
|
||||
setup_queries: []
|
||||
validation_queries: []
|
||||
|
||||
|
||||
workloads:
|
||||
- name: "Maximum deltas restriction"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
args: ["maximum_deltas_restriction/maximum_deltas_restriction.py"]
|
||||
<<: *maximum_deltas_restriction_cluster
|
||||
@@ -135,6 +135,11 @@ workloads:
|
||||
proc: "tests/e2e/memory/procedures/"
|
||||
<<: *in_memory_query_limit_cluster
|
||||
|
||||
- name: "Memory control query limit create"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *in_memory_query_limit_cluster
|
||||
|
||||
- name: "Memory control query limit create multi thread"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create_multi_thread"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
|
||||
@@ -26,3 +26,10 @@ workloads:
|
||||
proc: "query_modules/"
|
||||
args: ["query_modules/mgps_test.py"]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
- name: "Schema test"
|
||||
pre_set_workload: "tests/e2e/x.sh"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "query_modules/"
|
||||
args: ["query_modules/schema_test.py"]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
@@ -42,3 +42,21 @@ workloads:
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/pulsar_streams_tests.py"]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
- name: "Kafka streams start, stop and show for on-disk storage"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/kafka_streams_tests.py"]
|
||||
<<: *disk_cluster
|
||||
|
||||
- name: "Streams with users for on-disk storage"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/streams_owner_tests.py"]
|
||||
<<: *disk_cluster
|
||||
|
||||
- name: "Pulsar streams start, stop and show for on-disk storage"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/pulsar_streams_tests.py"]
|
||||
<<: *disk_cluster
|
||||
|
||||
Reference in New Issue
Block a user