Compare commits
17 Commits
jemalloc-n
...
analyze-fl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f5e3c26284 | ||
|
|
e6c8e7cfdc | ||
|
|
955609d036 | ||
|
|
6c46046f3d | ||
|
|
47db66c744 | ||
|
|
dcdeb19479 | ||
|
|
ff67e263cd | ||
|
|
4de5392aba | ||
|
|
916af3006c | ||
|
|
a7400e3b1d | ||
|
|
0e7c7c61ea | ||
|
|
35157a649e | ||
|
|
4548baab0f | ||
|
|
52383694f1 | ||
|
|
088773c0c8 | ||
|
|
d6d36e4277 | ||
|
|
6b9389c545 |
649
.github/workflows/analyze_diff.yaml
vendored
Normal file
649
.github/workflows/analyze_diff.yaml
vendored
Normal file
@@ -0,0 +1,649 @@
|
||||
name: Analyze diff flakiness
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
# pull_request:
|
||||
# paths-ignore:
|
||||
# - "docs/**"
|
||||
# - "**/*.md"
|
||||
# - ".clang-format"
|
||||
# - "CODEOWNERS"
|
||||
# - "licenses/*"
|
||||
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build community binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build community binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMG_ENTERPRISE=OFF ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
code_analysis:
|
||||
name: "Code analysis"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
# This is also needed if we want do to comparison against other branches
|
||||
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
|
||||
- name: Fetch all history for all tags and branches
|
||||
run: git fetch
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Set base branch
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set base branch # if we manually dispatch or push to master
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
|
||||
|
||||
- name: Python code analysis
|
||||
run: |
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only --diff-filter=d)
|
||||
for file in ${CHANGED_FILES}; do
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
python3 -m black --check --diff ${file}
|
||||
python3 -m isort --profile black --check-only --diff ${file}
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
|
||||
cd build
|
||||
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
|
||||
|
||||
- name: Compute code coverage
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Compute code coverage.
|
||||
cd tools/github
|
||||
./coverage_convert
|
||||
|
||||
# Package code coverage.
|
||||
cd generated
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build debug binaries.
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run leftover CTest tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run leftover CTest tests (all except unit and benchmark tests).
|
||||
cd build
|
||||
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
|
||||
|
||||
- name: Run drivers tests
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run cppcheck and clang-format.
|
||||
cd tools/github
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: always()
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
./continuous_integration
|
||||
|
||||
- name: Run stress test (SSL)
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run durability test
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 5
|
||||
|
||||
- name: Create enterprise DEB package
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
cd build
|
||||
|
||||
# create mgconsole
|
||||
# we use the -B to force the build
|
||||
make -j$THREADS -B mgconsole
|
||||
|
||||
# Create enterprise DEB package.
|
||||
mkdir output && cd output
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
#
|
||||
# experimental_build_ha:
|
||||
# name: "High availability build"
|
||||
# runs-on: [self-hosted, Linux, X64, Diff]
|
||||
# strategy:
|
||||
# max-parallel: 1
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
# env:
|
||||
# THREADS: 24
|
||||
# MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
# MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
#
|
||||
# steps:
|
||||
# - name: Set up repository
|
||||
# uses: actions/checkout@v3
|
||||
# with:
|
||||
# # Number of commits to fetch. `0` indicates all history for all
|
||||
# # branches and tags. (default: 1)
|
||||
# fetch-depth: 0
|
||||
#
|
||||
# - name: Initialize deps
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Initialize dependencies.
|
||||
# ./init
|
||||
#
|
||||
# - name: Build release binaries
|
||||
# run: |
|
||||
# source /opt/toolchain-v4/activate
|
||||
# cd build
|
||||
# cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
|
||||
# make -j$THREADS
|
||||
# - name: Run unit tests
|
||||
# run: |
|
||||
# source /opt/toolchain-v4/activate
|
||||
# cd build
|
||||
# ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
# - name: Run e2e tests
|
||||
# run: |
|
||||
# cd tests
|
||||
# ./setup.sh /opt/toolchain-v4/activate
|
||||
# source ve3/bin/activate_e2e
|
||||
# cd e2e
|
||||
# ./run.sh "Coordinator"
|
||||
# ./run.sh "Client initiated failover"
|
||||
# ./run.sh "Uninitialized cluster"
|
||||
# - name: Save test data
|
||||
# uses: actions/upload-artifact@v3
|
||||
# if: always()
|
||||
# with:
|
||||
# name: "Test data"
|
||||
# path: |
|
||||
# # multiple paths could be defined
|
||||
# build/logs
|
||||
#
|
||||
# experimental_build_mt:
|
||||
# name: "MultiTenancy replication build"
|
||||
# runs-on: [self-hosted, Linux, X64, Diff]
|
||||
# strategy:
|
||||
# max-parallel: 1
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
# env:
|
||||
# THREADS: 24
|
||||
# MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
# MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
#
|
||||
# steps:
|
||||
# - name: Set up repository
|
||||
# uses: actions/checkout@v3
|
||||
# with:
|
||||
# # Number of commits to fetch. `0` indicates all history for all
|
||||
# # branches and tags. (default: 1)
|
||||
# fetch-depth: 0
|
||||
#
|
||||
#
|
||||
# - name: Initialize deps
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Initialize dependencies.
|
||||
# ./init
|
||||
#
|
||||
# - name: Build release binaries
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Build MT replication experimental binaries.
|
||||
# cd build
|
||||
# cmake -DCMAKE_BUILD_TYPE=Release -D MG_EXPERIMENTAL_REPLICATION_MULTITENANCY=ON ..
|
||||
# make -j$THREADS
|
||||
#
|
||||
# - name: Run unit tests
|
||||
# run: |
|
||||
# # Activate toolchain.
|
||||
# source /opt/toolchain-v4/activate
|
||||
#
|
||||
# # Run unit tests.
|
||||
# cd build
|
||||
# ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
#
|
||||
# - name: Run e2e tests
|
||||
# run: |
|
||||
# cd tests
|
||||
# ./setup.sh /opt/toolchain-v4/activate
|
||||
# source ve3/bin/activate_e2e
|
||||
# cd e2e
|
||||
#
|
||||
# # Just the replication based e2e tests
|
||||
# ./run.sh "Replicate multitenancy"
|
||||
# ./run.sh "Show"
|
||||
# ./run.sh "Show while creating invalid state"
|
||||
# ./run.sh "Delete edge replication"
|
||||
# ./run.sh "Read-write benchmark"
|
||||
# ./run.sh "Index replication"
|
||||
# ./run.sh "Constraints"
|
||||
#
|
||||
# - name: Save test data
|
||||
# uses: actions/upload-artifact@v3
|
||||
# if: always()
|
||||
# with:
|
||||
# name: "Test data"
|
||||
# path: |
|
||||
# # multiple paths could be defined
|
||||
# build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
#continue-on-error: true
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Refresh Jepsen Cluster
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh cluster-refresh
|
||||
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v3
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
strategy:
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 90
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Build only memgraph release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Get branch name (merge)
|
||||
if: github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
|
||||
|
||||
- name: Get branch name (pull request)
|
||||
if: github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
|
||||
|
||||
- name: Upload macro benchmark results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
# TODO (andi) No need for path flags and for --disk-storage and --in-memory-analytical
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
72
.github/workflows/analyze_e2e.yaml
vendored
Normal file
72
.github/workflows/analyze_e2e.yaml
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
name: Analyze e2e flakiness
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
- ".clang-format"
|
||||
- "CODEOWNERS"
|
||||
- "licenses/*"
|
||||
|
||||
jobs:
|
||||
release_e2e_test:
|
||||
name: "Release End-to-end Test"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
strategy:
|
||||
max-parallel: 5
|
||||
fail-fast: false
|
||||
matrix:
|
||||
try_id: [1,2,3,4,5,6,7,8,9,10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: always()
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
22
.github/workflows/diff.yaml
vendored
22
.github/workflows/diff.yaml
vendored
@@ -4,18 +4,18 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
# push:
|
||||
# branches:
|
||||
# - master
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- "**/*.md"
|
||||
- ".clang-format"
|
||||
- "CODEOWNERS"
|
||||
- "licenses/*"
|
||||
|
||||
# pull_request:
|
||||
# paths-ignore:
|
||||
# - "docs/**"
|
||||
# - "**/*.md"
|
||||
# - ".clang-format"
|
||||
# - "CODEOWNERS"
|
||||
# - "licenses/*"
|
||||
#
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
|
||||
1
libs/.gitignore
vendored
1
libs/.gitignore
vendored
@@ -7,4 +7,3 @@
|
||||
!pulsar.patch
|
||||
!antlr4.10.1.patch
|
||||
!rocksdb8.1.1.patch
|
||||
!jemalloc-oversize_threshold.patch
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
diff --git a/src/ctl.c b/src/ctl.c
|
||||
index 48afaa61..1e545710 100644
|
||||
--- a/src/ctl.c
|
||||
+++ b/src/ctl.c
|
||||
@@ -2376,11 +2376,11 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
|
||||
|
||||
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
|
||||
MIB_UNSIGNED(arena_ind, 1);
|
||||
- if (arena_ind < narenas_total_get()) {
|
||||
+ if (arena_ind <= narenas_total_get()) {
|
||||
extent_hooks_t *old_extent_hooks;
|
||||
arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
|
||||
if (arena == NULL) {
|
||||
- if (arena_ind >= narenas_auto) {
|
||||
+ if (arena_ind >= narenas_auto+1) {
|
||||
ret = EFAULT;
|
||||
goto label_return;
|
||||
}
|
||||
@@ -2388,6 +2388,20 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
|
||||
(extent_hooks_t *)&extent_hooks_default;
|
||||
READ(old_extent_hooks, extent_hooks_t *);
|
||||
if (newp != NULL) {
|
||||
+ if (arena_ind == narenas_auto) {
|
||||
+ // init some number for huge arena
|
||||
+ arena_init_huge();
|
||||
+ //init arena really
|
||||
+ arena = arena_choose_huge(tsd);
|
||||
+ extent_hooks_t *new_extent_hooks
|
||||
+ JEMALLOC_CC_SILENCE_INIT(NULL);
|
||||
+ WRITE(new_extent_hooks, extent_hooks_t *);
|
||||
+ old_extent_hooks = extent_hooks_set(tsd, arena,
|
||||
+ new_extent_hooks);
|
||||
+ READ(old_extent_hooks, extent_hooks_t *);
|
||||
+ ret = 0;
|
||||
+ goto label_return;
|
||||
+ }
|
||||
/* Initialize a new arena as a side effect. */
|
||||
extent_hooks_t *new_extent_hooks
|
||||
JEMALLOC_CC_SILENCE_INIT(NULL);
|
||||
@@ -265,13 +265,11 @@ repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}"
|
||||
|
||||
# this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR
|
||||
pushd jemalloc
|
||||
patch -p1 < ../jemalloc-oversize_threshold.patch
|
||||
|
||||
./autogen.sh
|
||||
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
|
||||
./configure \
|
||||
--disable-cxx \
|
||||
--with-lg-page=12 \
|
||||
--with-lg-hugepage=21 \
|
||||
--enable-shared=no --prefix=$working_dir \
|
||||
--with-malloc-conf="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
|
||||
|
||||
@@ -57,19 +57,16 @@ struct UpdateAuthData : memgraph::system::ISystemAction {
|
||||
void DoDurability() override { /* Done during Auth execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
bool DoReplication(replication::ReplicationClient &client, replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const replication::UpdateAuthDataRes &response) { return response.success; };
|
||||
if (user_) {
|
||||
return client.SteamAndFinalizeDelta<replication::UpdateAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
*user_);
|
||||
check_response, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(), *user_);
|
||||
}
|
||||
if (role_) {
|
||||
return client.SteamAndFinalizeDelta<replication::UpdateAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
*role_);
|
||||
check_response, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(), *role_);
|
||||
}
|
||||
// Should never get here
|
||||
MG_ASSERT(false, "Trying to update auth data that is not a user nor a role");
|
||||
@@ -91,8 +88,7 @@ struct DropAuthData : memgraph::system::ISystemAction {
|
||||
void DoDurability() override { /* Done during Auth execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
bool DoReplication(replication::ReplicationClient &client, replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const replication::DropAuthDataRes &response) { return response.success; };
|
||||
|
||||
@@ -106,8 +102,7 @@ struct DropAuthData : memgraph::system::ISystemAction {
|
||||
break;
|
||||
}
|
||||
return client.SteamAndFinalizeDelta<replication::DropAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
type, name_);
|
||||
check_response, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(), type, name_);
|
||||
}
|
||||
void PostReplication(replication::RoleMainData &mainData) const override {}
|
||||
|
||||
|
||||
@@ -17,15 +17,8 @@
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req) {
|
||||
spdlog::error(fmt::format("Received {} with main_id: {} != current_main_uuid: {}", rpc_req, std::string(main_req_id),
|
||||
current_main_uuid.has_value() ? std::string(current_main_uuid.value()) : ""));
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::UpdateAuthDataReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
@@ -33,12 +26,6 @@ void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
using memgraph::replication::UpdateAuthDataRes;
|
||||
UpdateAuthDataRes res(false);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::UpdateAuthDataReq::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
@@ -66,8 +53,7 @@ void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::DropAuthDataReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
@@ -75,12 +61,6 @@ void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_s
|
||||
using memgraph::replication::DropAuthDataRes;
|
||||
DropAuthDataRes res(false);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::DropAuthDataRes::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
@@ -175,14 +155,14 @@ void Register(replication::RoleReplicaData const &data, system::ReplicaHandlerAc
|
||||
auth::SynchedAuth &auth) {
|
||||
// NOTE: Register even without license as the user could add a license at run-time
|
||||
data.server->rpc_server_.Register<replication::UpdateAuthDataRpc>(
|
||||
[&data, system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received UpdateAuthDataRpc");
|
||||
UpdateAuthDataHandler(system_state_access, data.uuid_, auth, req_reader, res_builder);
|
||||
UpdateAuthDataHandler(system_state_access, auth, req_reader, res_builder);
|
||||
});
|
||||
data.server->rpc_server_.Register<replication::DropAuthDataRpc>(
|
||||
[&data, system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received DropAuthDataRpc");
|
||||
DropAuthDataHandler(system_state_access, data.uuid_, auth, req_reader, res_builder);
|
||||
DropAuthDataHandler(system_state_access, auth, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -17,16 +17,10 @@
|
||||
#include "system/state.hpp"
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req);
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void UpdateAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void UpdateAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
void DropAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void DropAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
bool SystemRecoveryHandler(auth::SynchedAuth &auth, auth::Auth::Config auth_config,
|
||||
|
||||
@@ -89,7 +89,6 @@ void Load(auth::Auth::Config *self, memgraph::slk::Reader *reader) {
|
||||
|
||||
// Serialize code for UpdateAuthDataReq
|
||||
void Save(const memgraph::replication::UpdateAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
memgraph::slk::Save(self.expected_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.new_group_timestamp, builder);
|
||||
@@ -97,7 +96,6 @@ void Save(const memgraph::replication::UpdateAuthDataReq &self, memgraph::slk::B
|
||||
memgraph::slk::Save(self.role, builder);
|
||||
}
|
||||
void Load(memgraph::replication::UpdateAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
memgraph::slk::Load(&self->expected_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->new_group_timestamp, reader);
|
||||
@@ -115,7 +113,6 @@ void Load(memgraph::replication::UpdateAuthDataRes *self, memgraph::slk::Reader
|
||||
|
||||
// Serialize code for DropAuthDataReq
|
||||
void Save(const memgraph::replication::DropAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
memgraph::slk::Save(self.expected_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.new_group_timestamp, builder);
|
||||
@@ -123,7 +120,6 @@ void Save(const memgraph::replication::DropAuthDataReq &self, memgraph::slk::Bui
|
||||
memgraph::slk::Save(self.name, builder);
|
||||
}
|
||||
void Load(memgraph::replication::DropAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
memgraph::slk::Load(&self->expected_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->new_group_timestamp, reader);
|
||||
|
||||
@@ -27,22 +27,17 @@ struct UpdateAuthDataReq {
|
||||
static void Load(UpdateAuthDataReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const UpdateAuthDataReq &self, memgraph::slk::Builder *builder);
|
||||
UpdateAuthDataReq() = default;
|
||||
UpdateAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
auth::User user)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
UpdateAuthDataReq(std::string epoch_id, uint64_t expected_ts, uint64_t new_ts, auth::User user)
|
||||
: epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
user{std::move(user)} {}
|
||||
UpdateAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
auth::Role role)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
UpdateAuthDataReq(std::string epoch_id, uint64_t expected_ts, uint64_t new_ts, auth::Role role)
|
||||
: epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
role{std::move(role)} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
@@ -74,16 +69,13 @@ struct DropAuthDataReq {
|
||||
|
||||
enum class DataType { USER, ROLE };
|
||||
|
||||
DropAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
DataType type, std::string_view name)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
DropAuthDataReq(std::string epoch_id, uint64_t expected_ts, uint64_t new_ts, DataType type, std::string_view name)
|
||||
: epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
type{type},
|
||||
name{name} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
@@ -72,17 +71,16 @@ auto CoordinatorClient::SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCal
|
||||
|
||||
auto CoordinatorClient::ReplicationClientInfo() const -> ReplClientInfo { return config_.replication_client_info; }
|
||||
|
||||
auto CoordinatorClient::SendPromoteReplicaToMainRpc(const utils::UUID &uuid,
|
||||
ReplicationClientsInfo replication_clients_info) const -> bool {
|
||||
auto CoordinatorClient::SendPromoteReplicaToMainRpc(ReplicationClientsInfo replication_clients_info) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(uuid, std::move(replication_clients_info))};
|
||||
auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(std::move(replication_clients_info))};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful PromoteReplicaToMainRpc response!");
|
||||
spdlog::error("Failed to receive successful RPC failover response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("RPC error occurred while sending PromoteReplicaToMainRpc!");
|
||||
spdlog::error("RPC error occurred while sending failover RPC!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -103,19 +101,5 @@ auto CoordinatorClient::DemoteToReplica() const -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendSwapMainUUIDRpc(const utils::UUID &uuid) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<replication_coordination_glue::SwapMainUUIDRpc>(uuid)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC swapping of uuid response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occurred while sending swapping uuid RPC!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_data.hpp"
|
||||
@@ -33,94 +32,60 @@ CoordinatorData::CoordinatorData() {
|
||||
return *instance;
|
||||
};
|
||||
|
||||
replica_succ_cb_ = [this, find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
replica_succ_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing replica successful callback", instance_name);
|
||||
auto &instance = find_instance(coord_data, instance_name);
|
||||
|
||||
if (!instance.GetMainUUID().has_value() || main_uuid_ != instance.GetMainUUID().value()) {
|
||||
if (!instance.SendSwapAndUpdateUUID(main_uuid_)) {
|
||||
spdlog::error(
|
||||
fmt::format("Failed to swap uuid for replica instance {} which is alive", instance.InstanceName()));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
instance.OnSuccessPing();
|
||||
find_instance(coord_data, instance_name).OnSuccessPing();
|
||||
};
|
||||
|
||||
replica_fail_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing replica failure callback", instance_name);
|
||||
auto &instance = find_instance(coord_data, instance_name);
|
||||
instance.OnFailPing();
|
||||
// We need to restart main uuid from instance since it was "down" at least a second
|
||||
// There is slight delay, if we choose to use isAlive, instance can be down and back up in less than
|
||||
// our isAlive time difference, which would lead to instance setting UUID to nullopt and stopping accepting any
|
||||
// incoming RPCs from valid main
|
||||
// TODO(antoniofilipovic) this needs here more complex logic
|
||||
// We need to get id of main replica is listening to on successful ping
|
||||
// and swap it to correct uuid if it failed
|
||||
instance.SetNewMainUUID();
|
||||
find_instance(coord_data, instance_name).OnFailPing();
|
||||
};
|
||||
|
||||
main_succ_cb_ = [this, find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
main_succ_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing main successful callback", instance_name);
|
||||
|
||||
auto &instance = find_instance(coord_data, instance_name);
|
||||
|
||||
const auto &instance_uuid = instance.GetMainUUID();
|
||||
MG_ASSERT(instance_uuid.has_value(), "Instance must have uuid set");
|
||||
if (main_uuid_ == instance_uuid.value()) {
|
||||
if (instance.IsAlive() || !coord_data->ClusterHasAliveMain_()) {
|
||||
instance.OnSuccessPing();
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO(antoniof) make demoteToReplica idempotent since main can be demoted to replica but
|
||||
// swapUUID can fail
|
||||
bool const demoted = instance.DemoteToReplica(coord_data->replica_succ_cb_, coord_data->replica_fail_cb_);
|
||||
if (demoted) {
|
||||
instance.OnSuccessPing();
|
||||
spdlog::info("Instance {} demoted to replica", instance_name);
|
||||
} else {
|
||||
spdlog::error("Instance {} failed to become replica", instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!instance.SendSwapAndUpdateUUID(main_uuid_)) {
|
||||
spdlog::error(fmt::format("Failed to swap uuid for demoted main instance {}", instance.InstanceName()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
main_fail_cb_ = [this, find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
main_fail_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing main failure callback", instance_name);
|
||||
auto &instance = find_instance(coord_data, instance_name);
|
||||
instance.OnFailPing();
|
||||
const auto &instance_uuid = instance.GetMainUUID();
|
||||
MG_ASSERT(instance_uuid.has_value(), "Instance must have uuid set");
|
||||
find_instance(coord_data, instance_name).OnFailPing();
|
||||
|
||||
if (!instance.IsAlive() && main_uuid_ == instance_uuid.value()) {
|
||||
if (!coord_data->ClusterHasAliveMain_()) {
|
||||
spdlog::info("Cluster without main instance, trying automatic failover");
|
||||
coord_data->TryFailover();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
auto CoordinatorData::ClusterHasAliveMain_() const -> bool {
|
||||
auto const alive_main = [](CoordinatorInstance const &instance) { return instance.IsMain() && instance.IsAlive(); };
|
||||
return std::ranges::any_of(registered_instances_, alive_main);
|
||||
}
|
||||
|
||||
auto CoordinatorData::TryFailover() -> void {
|
||||
std::vector<CoordinatorInstance *> alive_registered_replica_instances{};
|
||||
std::ranges::transform(registered_instances_ | ranges::views::filter(&CoordinatorInstance::IsReplica) |
|
||||
ranges::views::filter(&CoordinatorInstance::IsAlive),
|
||||
std::back_inserter(alive_registered_replica_instances),
|
||||
[](CoordinatorInstance &instance) { return &instance; });
|
||||
auto replica_instances = registered_instances_ | ranges::views::filter(&CoordinatorInstance::IsReplica);
|
||||
|
||||
// TODO(antoniof) more complex logic of choosing replica instance
|
||||
CoordinatorInstance *chosen_replica_instance =
|
||||
!alive_registered_replica_instances.empty() ? alive_registered_replica_instances[0] : nullptr;
|
||||
|
||||
if (nullptr == chosen_replica_instance) {
|
||||
auto chosen_replica_instance = std::ranges::find_if(replica_instances, &CoordinatorInstance::IsAlive);
|
||||
if (chosen_replica_instance == replica_instances.end()) {
|
||||
spdlog::warn("Failover failed since all replicas are down!");
|
||||
return;
|
||||
}
|
||||
@@ -128,39 +93,21 @@ auto CoordinatorData::TryFailover() -> void {
|
||||
chosen_replica_instance->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&chosen_replica_instance] { chosen_replica_instance->ResumeFrequentCheck(); }};
|
||||
|
||||
utils::UUID potential_new_main_uuid = utils::UUID{};
|
||||
spdlog::trace("Generated potential new main uuid");
|
||||
|
||||
auto not_chosen_instance = [chosen_replica_instance](auto *instance) {
|
||||
return *instance != *chosen_replica_instance;
|
||||
};
|
||||
// If for some replicas swap fails, for others on successful ping we will revert back on next change
|
||||
// or we will do failover first again and then it will be consistent again
|
||||
for (auto *other_replica_instance : alive_registered_replica_instances | ranges::views::filter(not_chosen_instance)) {
|
||||
if (!other_replica_instance->SendSwapAndUpdateUUID(potential_new_main_uuid)) {
|
||||
spdlog::error(fmt::format("Failed to swap uuid for instance {} which is alive, aborting failover",
|
||||
other_replica_instance->InstanceName()));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<ReplClientInfo> repl_clients_info;
|
||||
repl_clients_info.reserve(registered_instances_.size() - 1);
|
||||
repl_clients_info.reserve(std::ranges::distance(replica_instances));
|
||||
|
||||
std::ranges::transform(registered_instances_ | ranges::views::filter([chosen_replica_instance](const auto &instance) {
|
||||
return *chosen_replica_instance != instance;
|
||||
}),
|
||||
auto const not_chosen_replica_instance = [&chosen_replica_instance](CoordinatorInstance const &instance) {
|
||||
return instance != *chosen_replica_instance;
|
||||
};
|
||||
|
||||
std::ranges::transform(registered_instances_ | ranges::views::filter(not_chosen_replica_instance),
|
||||
std::back_inserter(repl_clients_info),
|
||||
[](const CoordinatorInstance &instance) { return instance.ReplicationClientInfo(); });
|
||||
|
||||
if (!chosen_replica_instance->PromoteToMain(potential_new_main_uuid, std::move(repl_clients_info), main_succ_cb_,
|
||||
main_fail_cb_)) {
|
||||
if (!chosen_replica_instance->PromoteToMain(std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
spdlog::warn("Failover failed since promoting replica to main failed!");
|
||||
return;
|
||||
}
|
||||
chosen_replica_instance->SetNewMainUUID(potential_new_main_uuid);
|
||||
main_uuid_ = potential_new_main_uuid;
|
||||
|
||||
spdlog::info("Failover successful! Instance {} promoted to main.", chosen_replica_instance->InstanceName());
|
||||
}
|
||||
|
||||
@@ -213,28 +160,14 @@ auto CoordinatorData::SetInstanceToMain(std::string instance_name) -> SetInstanc
|
||||
auto const is_not_new_main = [&instance_name](CoordinatorInstance const &instance) {
|
||||
return instance.InstanceName() != instance_name;
|
||||
};
|
||||
|
||||
auto potential_new_main_uuid = utils::UUID{};
|
||||
spdlog::trace("Generated potential new main uuid");
|
||||
|
||||
for (auto &other_instance : registered_instances_ | ranges::views::filter(is_not_new_main)) {
|
||||
if (!other_instance.SendSwapAndUpdateUUID(potential_new_main_uuid)) {
|
||||
spdlog::error(
|
||||
fmt::format("Failed to swap uuid for instance {}, aborting failover", other_instance.InstanceName()));
|
||||
return SetInstanceToMainCoordinatorStatus::SWAP_UUID_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
std::ranges::transform(registered_instances_ | ranges::views::filter(is_not_new_main),
|
||||
std::back_inserter(repl_clients_info),
|
||||
[](const CoordinatorInstance &instance) { return instance.ReplicationClientInfo(); });
|
||||
|
||||
if (!new_main->PromoteToMain(potential_new_main_uuid, std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
if (!new_main->PromoteToMain(std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
|
||||
}
|
||||
|
||||
new_main->SetNewMainUUID(potential_new_main_uuid);
|
||||
main_uuid_ = potential_new_main_uuid;
|
||||
spdlog::info("Instance {} promoted to main", instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
#include "coordination/include/coordination/coordinator_server.hpp"
|
||||
#include "replication/state.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
@@ -33,29 +32,6 @@ void CoordinatorHandlers::Register(memgraph::coordination::CoordinatorServer &se
|
||||
spdlog::info("Received DemoteMainToReplicaRpc from coordinator server");
|
||||
CoordinatorHandlers::DemoteMainToReplicaHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<replication_coordination_glue::SwapMainUUIDRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received SwapMainUUIDRPC on coordinator server");
|
||||
CoordinatorHandlers::SwapMainUUIDHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsReplica()) {
|
||||
spdlog::error("Setting main uuid must be performed on replica.");
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
replication_coordination_glue::SwapMainUUIDReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
spdlog::info(fmt::format("Set replica data UUID to main uuid {}", std::string(req.uuid)));
|
||||
std::get<memgraph::replication::RoleReplicaData>(replication_handler.GetReplState().ReplicationData()).uuid_ =
|
||||
req.uuid;
|
||||
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHandler &replication_handler,
|
||||
@@ -75,7 +51,7 @@ void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHan
|
||||
.ip_address = req.replication_client_info.replication_ip_address,
|
||||
.port = req.replication_client_info.replication_port};
|
||||
|
||||
if (!replication_handler.SetReplicationRoleReplica(clients_config, std::nullopt)) {
|
||||
if (!replication_handler.SetReplicationRoleReplica(clients_config)) {
|
||||
spdlog::error("Demoting main to replica failed!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
@@ -91,17 +67,18 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
coordination::PromoteReplicaToMainReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
// This can fail because of disk. If it does, the cluster state could get inconsistent.
|
||||
// We don't handle disk issues.
|
||||
if (const bool success = replication_handler.DoReplicaToMainPromotion(req.main_uuid_); !success) {
|
||||
if (!replication_handler.DoReplicaToMainPromotion()) {
|
||||
spdlog::error("Promoting replica to main failed!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
coordination::PromoteReplicaToMainReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
auto const converter = [](const auto &repl_info_config) {
|
||||
return replication::ReplicationClientConfig{
|
||||
.name = repl_info_config.instance_name,
|
||||
@@ -113,7 +90,7 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
|
||||
|
||||
// registering replicas
|
||||
for (auto const &config : req.replication_clients_info | ranges::views::transform(converter)) {
|
||||
auto instance_client = replication_handler.RegisterReplica(config, false);
|
||||
auto instance_client = replication_handler.RegisterReplica(config);
|
||||
if (instance_client.HasError()) {
|
||||
using enum memgraph::replication::RegisterReplicaError;
|
||||
switch (instance_client.GetError()) {
|
||||
@@ -132,17 +109,13 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
|
||||
spdlog::error("Registered replica could not be persisted!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
case memgraph::query::RegisterReplicaError::ERROR_ACCEPTING_MAIN:
|
||||
spdlog::error("Replica didn't accept change of main!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
case memgraph::query::RegisterReplicaError::CONNECTION_FAILED:
|
||||
// Connection failure is not a fatal error
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
spdlog::error(fmt::format("FICO : Promote replica to main was success {}", std::string(req.main_uuid_)));
|
||||
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
|
||||
}
|
||||
|
||||
|
||||
@@ -49,9 +49,9 @@ auto CoordinatorInstance::IsMain() const -> bool {
|
||||
return replication_role_ == replication_coordination_glue::ReplicationRole::MAIN;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::PromoteToMain(utils::UUID uuid, ReplicationClientsInfo repl_clients_info,
|
||||
HealthCheckCallback main_succ_cb, HealthCheckCallback main_fail_cb) -> bool {
|
||||
if (!client_.SendPromoteReplicaToMainRpc(uuid, std::move(repl_clients_info))) {
|
||||
auto CoordinatorInstance::PromoteToMain(ReplicationClientsInfo repl_clients_info, HealthCheckCallback main_succ_cb,
|
||||
HealthCheckCallback main_fail_cb) -> bool {
|
||||
if (!client_.SendPromoteReplicaToMainRpc(std::move(repl_clients_info))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -80,17 +80,5 @@ auto CoordinatorInstance::ReplicationClientInfo() const -> CoordinatorClientConf
|
||||
return client_.ReplicationClientInfo();
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::GetClient() -> CoordinatorClient & { return client_; }
|
||||
void CoordinatorInstance::SetNewMainUUID(const std::optional<utils::UUID> &main_uuid) { main_uuid_ = main_uuid; }
|
||||
auto CoordinatorInstance::GetMainUUID() -> const std::optional<utils::UUID> & { return main_uuid_; }
|
||||
|
||||
auto CoordinatorInstance::SendSwapAndUpdateUUID(const utils::UUID &main_uuid) -> bool {
|
||||
if (!replication_coordination_glue::SendSwapMainUUIDRpc(client_.RpcClient(), main_uuid)) {
|
||||
return false;
|
||||
}
|
||||
SetNewMainUUID(main_uuid_);
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -77,12 +77,10 @@ void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::
|
||||
}
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid_, builder);
|
||||
memgraph::slk::Save(self.replication_clients_info, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid_, reader);
|
||||
memgraph::slk::Load(&self->replication_clients_info, reader);
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "replication_coordination_glue/messages.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
@@ -45,20 +44,13 @@ class CoordinatorClient {
|
||||
auto InstanceName() const -> std::string;
|
||||
auto SocketAddress() const -> std::string;
|
||||
|
||||
[[nodiscard]] auto SendPromoteReplicaToMainRpc(ReplicationClientsInfo replication_clients_info) const -> bool;
|
||||
[[nodiscard]] auto DemoteToReplica() const -> bool;
|
||||
auto SendPromoteReplicaToMainRpc(const utils::UUID &uuid, ReplicationClientsInfo replication_clients_info) const
|
||||
-> bool;
|
||||
|
||||
|
||||
auto SendSwapMainUUIDRpc(const utils::UUID &uuid) const -> bool;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
|
||||
auto SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCallback fail_cb) -> void;
|
||||
|
||||
auto RpcClient() -> rpc::Client & { return rpc_client_; }
|
||||
|
||||
friend bool operator==(CoordinatorClient const &first, CoordinatorClient const &second) {
|
||||
return first.config_ == second.config_;
|
||||
}
|
||||
|
||||
@@ -11,18 +11,17 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <list>
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
#include "coordination/coordinator_instance_status.hpp"
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "utils/rw_lock.hpp"
|
||||
#include "utils/thread_pool.hpp"
|
||||
|
||||
#include <list>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
class CoordinatorData {
|
||||
public:
|
||||
@@ -37,11 +36,12 @@ class CoordinatorData {
|
||||
auto ShowInstances() const -> std::vector<CoordinatorInstanceStatus>;
|
||||
|
||||
private:
|
||||
auto ClusterHasAliveMain_() const -> bool;
|
||||
|
||||
mutable utils::RWLock coord_data_lock_{utils::RWLock::Priority::READ};
|
||||
HealthCheckCallback main_succ_cb_, main_fail_cb_, replica_succ_cb_, replica_fail_cb_;
|
||||
// NOTE: Must be std::list because we rely on pointer stability
|
||||
std::list<CoordinatorInstance> registered_instances_;
|
||||
utils::UUID main_uuid_;
|
||||
};
|
||||
|
||||
struct CoordinatorMainReplicaData {
|
||||
|
||||
@@ -31,8 +31,6 @@ class CoordinatorHandlers {
|
||||
slk::Builder *res_builder);
|
||||
static void DemoteMainToReplicaHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
static void SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
#include "coordination/coordinator_cluster_config.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
@@ -45,7 +44,7 @@ class CoordinatorInstance {
|
||||
auto IsReplica() const -> bool;
|
||||
auto IsMain() const -> bool;
|
||||
|
||||
auto PromoteToMain(utils::UUID main_uuid, ReplicationClientsInfo repl_clients_info, HealthCheckCallback main_succ_cb,
|
||||
auto PromoteToMain(ReplicationClientsInfo repl_clients_info, HealthCheckCallback main_succ_cb,
|
||||
HealthCheckCallback main_fail_cb) -> bool;
|
||||
auto DemoteToReplica(HealthCheckCallback replica_succ_cb, HealthCheckCallback replica_fail_cb) -> bool;
|
||||
|
||||
@@ -54,25 +53,11 @@ class CoordinatorInstance {
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto GetClient() -> CoordinatorClient &;
|
||||
|
||||
void SetNewMainUUID(const std::optional<utils::UUID> &main_uuid = std::nullopt);
|
||||
auto GetMainUUID() -> const std::optional<utils::UUID> &;
|
||||
|
||||
auto SendSwapAndUpdateUUID(const utils::UUID &main_uuid) -> bool;
|
||||
|
||||
private:
|
||||
CoordinatorClient client_;
|
||||
replication_coordination_glue::ReplicationRole replication_role_;
|
||||
std::chrono::system_clock::time_point last_response_time_{};
|
||||
// TODO this needs to be atomic? What if instance is alive and then we read it and it has changed
|
||||
bool is_alive_{false};
|
||||
// for replica this is main uuid of current main
|
||||
// for "main" main this same as in CoordinatorData
|
||||
// it is set to nullopt when replica is down
|
||||
// TLDR; when replica is down and comes back up we reset uuid of main replica is listening to
|
||||
// so we need to send swap uuid again
|
||||
std::optional<utils::UUID> main_uuid_;
|
||||
|
||||
friend bool operator==(CoordinatorInstance const &first, CoordinatorInstance const &second) {
|
||||
return first.client_ == second.client_ && first.replication_role_ == second.replication_role_;
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
@@ -27,13 +26,10 @@ struct PromoteReplicaToMainReq {
|
||||
static void Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit PromoteReplicaToMainReq(const utils::UUID &uuid,
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
|
||||
: main_uuid_(uuid), replication_clients_info(std::move(replication_clients_info)) {}
|
||||
explicit PromoteReplicaToMainReq(std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
|
||||
: replication_clients_info(std::move(replication_clients_info)) {}
|
||||
PromoteReplicaToMainReq() = default;
|
||||
|
||||
// get uuid here
|
||||
utils::UUID main_uuid_;
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info;
|
||||
};
|
||||
|
||||
@@ -87,18 +83,21 @@ using DemoteMainToReplicaRpc = rpc::RequestResponse<DemoteMainToReplicaReq, Demo
|
||||
// SLK serialization declarations
|
||||
namespace memgraph::slk {
|
||||
|
||||
// PromoteReplicaToMainRpc
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// DemoteMainToReplicaRpc
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
} // namespace memgraph::slk
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ enum class SetInstanceToMainCoordinatorStatus : uint8_t {
|
||||
NOT_COORDINATOR,
|
||||
SUCCESS,
|
||||
COULD_NOT_PROMOTE_TO_MAIN,
|
||||
SWAP_UUID_FAILED
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -38,8 +38,6 @@ std::string RegisterReplicaErrorToString(query::RegisterReplicaError error) {
|
||||
return "CONNECTION_FAILED";
|
||||
case COULD_NOT_BE_PERSISTED:
|
||||
return "COULD_NOT_BE_PERSISTED";
|
||||
case ERROR_ACCEPTING_MAIN:
|
||||
return "ERROR_ACCEPTING_MAIN";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +52,7 @@ void RestoreReplication(replication::RoleMainData &mainData, DatabaseAccess db_a
|
||||
spdlog::info("Replica {} restoration started for {}.", instance_client.name_, db_acc->name());
|
||||
const auto &ret = db_acc->storage()->repl_storage_state_.replication_clients_.WithLock(
|
||||
[&, db_acc](auto &storage_clients) mutable -> utils::BasicResult<query::RegisterReplicaError> {
|
||||
auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client, mainData.uuid_);
|
||||
auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client);
|
||||
auto *storage = db_acc->storage();
|
||||
client->Start(storage, std::move(db_acc));
|
||||
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
|
||||
@@ -241,16 +239,14 @@ struct DropDatabase : memgraph::system::ISystemAction {
|
||||
void DoDurability() override { /* Done during DBMS execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
bool DoReplication(replication::ReplicationClient &client, replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const storage::replication::DropDatabaseRes &response) {
|
||||
return response.result != storage::replication::DropDatabaseRes::Result::FAILURE;
|
||||
};
|
||||
|
||||
return client.SteamAndFinalizeDelta<storage::replication::DropDatabaseRpc>(
|
||||
check_response, main_uuid, std::string(epoch.id()), txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
uuid_);
|
||||
check_response, epoch.id(), txn.last_committed_system_timestamp(), txn.timestamp(), uuid_);
|
||||
}
|
||||
void PostReplication(replication::RoleMainData &mainData) const override {}
|
||||
|
||||
@@ -327,16 +323,14 @@ struct CreateDatabase : memgraph::system::ISystemAction {
|
||||
// Done during dbms execution
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
bool DoReplication(replication::ReplicationClient &client, replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const storage::replication::CreateDatabaseRes &response) {
|
||||
return response.result != storage::replication::CreateDatabaseRes::Result::FAILURE;
|
||||
};
|
||||
|
||||
return client.SteamAndFinalizeDelta<storage::replication::CreateDatabaseRpc>(
|
||||
check_response, main_uuid, std::string(epoch.id()), txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
config_);
|
||||
check_response, epoch.id(), txn.last_committed_system_timestamp(), txn.timestamp(), config_);
|
||||
}
|
||||
|
||||
void PostReplication(replication::RoleMainData &mainData) const override {
|
||||
|
||||
@@ -29,7 +29,6 @@
|
||||
#include "kvstore/kvstore.hpp"
|
||||
#include "license/license.hpp"
|
||||
#include "replication/replication_client.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "system/system.hpp"
|
||||
@@ -262,16 +261,6 @@ class DbmsHandler {
|
||||
#endif
|
||||
}
|
||||
|
||||
replication::ReplicationState &ReplicationState() { return repl_state_; }
|
||||
replication::ReplicationState const &ReplicationState() const { return repl_state_; }
|
||||
|
||||
bool IsMain() const { return repl_state_.IsMain(); }
|
||||
bool IsReplica() const { return repl_state_.IsReplica(); }
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
// coordination::CoordinatorState &CoordinatorState() { return coordinator_state_; }
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief Return the statistics all databases.
|
||||
*
|
||||
|
||||
@@ -76,84 +76,47 @@ std::optional<DatabaseAccess> GetDatabaseAccessor(dbms::DbmsHandler *dbms_handle
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req) {
|
||||
spdlog::error("Received {} with main_id: {} != current_main_uuid: {}", rpc_req, std::string(main_req_id),
|
||||
current_main_uuid.has_value() ? std::string(current_main_uuid.value()) : "");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void InMemoryReplicationHandlers::Register(dbms::DbmsHandler *dbms_handler, replication::RoleReplicaData &data) {
|
||||
auto &server = *data.server;
|
||||
server.rpc_server_.Register<storage::replication::HeartbeatRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received HeartbeatRpc");
|
||||
InMemoryReplicationHandlers::HeartbeatHandler(dbms_handler, data.uuid_, req_reader, res_builder);
|
||||
});
|
||||
void InMemoryReplicationHandlers::Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server) {
|
||||
server.rpc_server_.Register<storage::replication::HeartbeatRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received HeartbeatRpc");
|
||||
InMemoryReplicationHandlers::HeartbeatHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::AppendDeltasRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
[dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received AppendDeltasRpc");
|
||||
InMemoryReplicationHandlers::AppendDeltasHandler(dbms_handler, data.uuid_, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::SnapshotRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received SnapshotRpc");
|
||||
InMemoryReplicationHandlers::SnapshotHandler(dbms_handler, data.uuid_, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::WalFilesRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received WalFilesRpc");
|
||||
InMemoryReplicationHandlers::WalFilesHandler(dbms_handler, data.uuid_, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::CurrentWalRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received CurrentWalRpc");
|
||||
InMemoryReplicationHandlers::CurrentWalHandler(dbms_handler, data.uuid_, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::TimestampRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received TimestampRpc");
|
||||
InMemoryReplicationHandlers::TimestampHandler(dbms_handler, data.uuid_, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<replication_coordination_glue::SwapMainUUIDRpc>(
|
||||
[&data, dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received SwapMainUUIDHandler");
|
||||
InMemoryReplicationHandlers::SwapMainUUIDHandler(dbms_handler, data, req_reader, res_builder);
|
||||
InMemoryReplicationHandlers::AppendDeltasHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::SnapshotRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received SnapshotRpc");
|
||||
InMemoryReplicationHandlers::SnapshotHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::WalFilesRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received WalFilesRpc");
|
||||
InMemoryReplicationHandlers::WalFilesHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::CurrentWalRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received CurrentWalRpc");
|
||||
InMemoryReplicationHandlers::CurrentWalHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::TimestampRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received TimestampRpc");
|
||||
InMemoryReplicationHandlers::TimestampHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::SwapMainUUIDHandler(dbms::DbmsHandler *dbms_handler,
|
||||
replication::RoleReplicaData &role_replica_data,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!dbms_handler->IsReplica()) {
|
||||
spdlog::error("Setting main uuid must be performed on replica.");
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
replication_coordination_glue::SwapMainUUIDReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
spdlog::info(fmt::format("Set replica data UUID to main uuid {}", std::string(req.uuid)));
|
||||
dbms_handler->ReplicationState().TryPersistRoleReplica(role_replica_data.config, req.uuid);
|
||||
role_replica_data.uuid_ = req.uuid;
|
||||
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handler,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::HeartbeatReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != *current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, storage::replication::HeartbeatReq::kType.name);
|
||||
if (!db_acc) {
|
||||
storage::replication::HeartbeatRes res{false, 0, ""};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
|
||||
auto const *storage = db_acc->get()->storage();
|
||||
storage::replication::HeartbeatRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load(),
|
||||
@@ -161,19 +124,10 @@ void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handl
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_handler,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::AppendDeltasReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, storage::replication::AppendDeltasReq::kType.name);
|
||||
storage::replication::AppendDeltasRes res{false, 0};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
|
||||
if (!db_acc) {
|
||||
storage::replication::AppendDeltasRes res{false, 0};
|
||||
@@ -233,9 +187,8 @@ void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_ha
|
||||
spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handler,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::SnapshotReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
|
||||
@@ -244,12 +197,6 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, storage::replication::SnapshotReq::kType.name);
|
||||
storage::replication::SnapshotRes res{false, 0};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
storage::replication::Decoder decoder(req_reader);
|
||||
|
||||
@@ -323,9 +270,8 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle
|
||||
spdlog::debug("Replication recovery from snapshot finished!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handler,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::WalFilesReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
|
||||
@@ -334,12 +280,6 @@ void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handle
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, storage::replication::WalFilesReq::kType.name);
|
||||
storage::replication::WalFilesRes res{false, 0};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
const auto wal_file_number = req.file_number;
|
||||
spdlog::debug("Received WAL files: {}", wal_file_number);
|
||||
@@ -358,9 +298,8 @@ void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handle
|
||||
spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_handler,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::CurrentWalReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
|
||||
@@ -370,13 +309,6 @@ void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_hand
|
||||
return;
|
||||
}
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, storage::replication::CurrentWalReq::kType.name);
|
||||
storage::replication::CurrentWalRes res{false, 0};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
storage::replication::Decoder decoder(req_reader);
|
||||
|
||||
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
|
||||
@@ -438,9 +370,8 @@ void InMemoryReplicationHandlers::LoadWal(storage::InMemoryStorage *storage, sto
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handler,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::TimestampReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
|
||||
@@ -450,20 +381,12 @@ void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handl
|
||||
return;
|
||||
}
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, storage::replication::TimestampReq::kType.name);
|
||||
storage::replication::CurrentWalRes res{false, 0};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
|
||||
auto const *storage = db_acc->get()->storage();
|
||||
storage::replication::TimestampRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
/////// AF how does this work, does it get all deltas at once or what?
|
||||
uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage *storage,
|
||||
storage::durability::BaseDecoder *decoder,
|
||||
const uint64_t version) {
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "replication/replication_server.hpp"
|
||||
#include "replication/state.hpp"
|
||||
#include "storage/v2/replication/serialization.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
@@ -24,30 +23,21 @@ class DbmsHandler;
|
||||
|
||||
class InMemoryReplicationHandlers {
|
||||
public:
|
||||
static void Register(dbms::DbmsHandler *dbms_handler, replication::RoleReplicaData &data);
|
||||
static void Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server);
|
||||
|
||||
private:
|
||||
// RPC handlers
|
||||
static void HeartbeatHandler(dbms::DbmsHandler *dbms_handler, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
static void HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
static void AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void SnapshotHandler(dbms::DbmsHandler *dbms_handler, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
static void SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void WalFilesHandler(dbms::DbmsHandler *dbms_handler, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
static void WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void CurrentWalHandler(dbms::DbmsHandler *dbms_handler, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
static void CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void TimestampHandler(dbms::DbmsHandler *dbms_handler, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void SwapMainUUIDHandler(dbms::DbmsHandler *dbms_handler, replication::RoleReplicaData &role_replica_data,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
static void TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder);
|
||||
|
||||
|
||||
@@ -21,8 +21,7 @@ namespace memgraph::dbms {
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
void CreateDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, DbmsHandler &dbms_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
using memgraph::storage::replication::CreateDatabaseRes;
|
||||
CreateDatabaseRes res(CreateDatabaseRes::Result::FAILURE);
|
||||
|
||||
@@ -36,12 +35,6 @@ void CreateDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
memgraph::storage::replication::CreateDatabaseReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, memgraph::storage::replication::CreateDatabaseReq::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
@@ -70,8 +63,7 @@ void CreateDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void DropDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, DbmsHandler &dbms_handler,
|
||||
void DropDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access, DbmsHandler &dbms_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
using memgraph::storage::replication::DropDatabaseRes;
|
||||
DropDatabaseRes res(DropDatabaseRes::Result::FAILURE);
|
||||
@@ -86,12 +78,6 @@ void DropDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_s
|
||||
memgraph::storage::replication::DropDatabaseReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, memgraph::storage::replication::DropDatabaseReq::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
@@ -191,14 +177,14 @@ void Register(replication::RoleReplicaData const &data, system::ReplicaHandlerAc
|
||||
dbms::DbmsHandler &dbms_handler) {
|
||||
// NOTE: Register even without license as the user could add a license at run-time
|
||||
data.server->rpc_server_.Register<storage::replication::CreateDatabaseRpc>(
|
||||
[&data, system_state_access, &dbms_handler](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &dbms_handler](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received CreateDatabaseRpc");
|
||||
CreateDatabaseHandler(system_state_access, data.uuid_, dbms_handler, req_reader, res_builder);
|
||||
CreateDatabaseHandler(system_state_access, dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
data.server->rpc_server_.Register<storage::replication::DropDatabaseRpc>(
|
||||
[&data, system_state_access, &dbms_handler](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &dbms_handler](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received DropDatabaseRpc");
|
||||
DropDatabaseHandler(system_state_access, data.uuid_, dbms_handler, req_reader, res_builder);
|
||||
DropDatabaseHandler(system_state_access, dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -17,21 +17,11 @@
|
||||
#include "system/state.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
inline void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req) {
|
||||
spdlog::error("Received {} with main_id: {} != current_main_uuid: {}", rpc_req, std::string(main_req_id),
|
||||
current_main_uuid.has_value() ? std::string(current_main_uuid.value()) : "");
|
||||
}
|
||||
|
||||
// RPC handlers
|
||||
void CreateDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, DbmsHandler &dbms_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
void DropDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, DbmsHandler &dbms_handler,
|
||||
DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
void DropDatabaseHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access, DbmsHandler &dbms_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
bool SystemRecoveryHandler(DbmsHandler &dbms_handler, const std::vector<storage::SalientConfig> &database_configs);
|
||||
|
||||
|
||||
@@ -29,15 +29,13 @@ struct CreateDatabaseReq {
|
||||
static void Load(CreateDatabaseReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const CreateDatabaseReq &self, memgraph::slk::Builder *builder);
|
||||
CreateDatabaseReq() = default;
|
||||
CreateDatabaseReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_group_timestamp,
|
||||
uint64_t new_group_timestamp, storage::SalientConfig config)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id(std::move(epoch_id)),
|
||||
CreateDatabaseReq(std::string_view epoch_id, uint64_t expected_group_timestamp, uint64_t new_group_timestamp,
|
||||
storage::SalientConfig config)
|
||||
: epoch_id(std::string(epoch_id)),
|
||||
expected_group_timestamp{expected_group_timestamp},
|
||||
new_group_timestamp(new_group_timestamp),
|
||||
config(std::move(config)) {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
@@ -67,15 +65,13 @@ struct DropDatabaseReq {
|
||||
static void Load(DropDatabaseReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const DropDatabaseReq &self, memgraph::slk::Builder *builder);
|
||||
DropDatabaseReq() = default;
|
||||
DropDatabaseReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_group_timestamp,
|
||||
uint64_t new_group_timestamp, const utils::UUID &uuid)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id(std::move(epoch_id)),
|
||||
DropDatabaseReq(std::string_view epoch_id, uint64_t expected_group_timestamp, uint64_t new_group_timestamp,
|
||||
const utils::UUID &uuid)
|
||||
: epoch_id(std::string(epoch_id)),
|
||||
expected_group_timestamp{expected_group_timestamp},
|
||||
new_group_timestamp(new_group_timestamp),
|
||||
uuid(uuid) {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -224,20 +224,6 @@ void SetHooks() {
|
||||
LOG_FATAL("Error setting custom hooks for jemalloc arena {}", i);
|
||||
}
|
||||
}
|
||||
auto last_arena = n_arenas;
|
||||
// For oversize arena, no need to read hooks, it will be intialized
|
||||
// with arena, just set our custom hooks
|
||||
std::string func_name = "arena." + std::to_string(last_arena) + ".extent_hooks";
|
||||
|
||||
if (err) {
|
||||
LOG_FATAL("Error setting jemalloc hooks for jemalloc arena {}", last_arena);
|
||||
}
|
||||
|
||||
err = mallctl(func_name.c_str(), nullptr, nullptr, &new_hooks, sizeof(new_hooks));
|
||||
|
||||
if (err) {
|
||||
LOG_FATAL("Error setting custom hooks for jemalloc oversize threshold arena {}", last_arena);
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -327,7 +327,7 @@ class ReplQueryHandler {
|
||||
.port = static_cast<uint16_t>(*port),
|
||||
};
|
||||
|
||||
if (!handler_->SetReplicationRoleReplica(config, std::nullopt)) {
|
||||
if (!handler_->SetReplicationRoleReplica(config)) {
|
||||
throw QueryRuntimeException("Couldn't set role to replica!");
|
||||
}
|
||||
}
|
||||
@@ -368,7 +368,7 @@ class ReplQueryHandler {
|
||||
.replica_check_frequency = replica_check_frequency,
|
||||
.ssl = std::nullopt};
|
||||
|
||||
const auto error = handler_->TryRegisterReplica(replication_config, true).HasError();
|
||||
const auto error = handler_->TryRegisterReplica(replication_config).HasError();
|
||||
|
||||
if (error) {
|
||||
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
|
||||
@@ -518,9 +518,7 @@ class CoordQueryHandler final : public query::CoordinatorQueryHandler {
|
||||
throw QueryRuntimeException("SET INSTANCE TO MAIN query can only be run on a coordinator!");
|
||||
case COULD_NOT_PROMOTE_TO_MAIN:
|
||||
throw QueryRuntimeException(
|
||||
"Couldn't set replica instance to main! Check coordinator and replica for more logs");
|
||||
case SWAP_UUID_FAILED:
|
||||
throw QueryRuntimeException("Couldn't set replica instance to main. Replicas didn't swap uuid of new main.");
|
||||
"Couldn't set replica instance to main!. Check coordinator and replica for more logs");
|
||||
case SUCCESS:
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
// BEGIN fwd declares
|
||||
namespace memgraph::replication {
|
||||
@@ -24,13 +23,7 @@ struct ReplicationClientConfig;
|
||||
|
||||
namespace memgraph::query {
|
||||
|
||||
enum class RegisterReplicaError : uint8_t {
|
||||
NAME_EXISTS,
|
||||
ENDPOINT_EXISTS,
|
||||
CONNECTION_FAILED,
|
||||
COULD_NOT_BE_PERSISTED,
|
||||
ERROR_ACCEPTING_MAIN
|
||||
};
|
||||
enum class RegisterReplicaError : uint8_t { NAME_EXISTS, ENDPOINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
|
||||
enum class UnregisterReplicaResult : uint8_t {
|
||||
NOT_MAIN,
|
||||
COULD_NOT_BE_PERSISTED,
|
||||
@@ -46,14 +39,13 @@ struct ReplicationQueryHandler {
|
||||
virtual bool SetReplicationRoleMain() = 0;
|
||||
|
||||
// as MAIN, become REPLICA
|
||||
virtual bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config,
|
||||
const std::optional<utils::UUID> &main_uuid) = 0;
|
||||
virtual bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) = 0;
|
||||
|
||||
// as MAIN, define and connect to REPLICAs
|
||||
virtual auto TryRegisterReplica(const memgraph::replication::ReplicationClientConfig &config, bool send_swap_uuid)
|
||||
virtual auto TryRegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> utils::BasicResult<RegisterReplicaError> = 0;
|
||||
|
||||
virtual auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config, bool send_swap_uuid)
|
||||
virtual auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> utils::BasicResult<RegisterReplicaError> = 0;
|
||||
|
||||
// as MAIN, remove a REPLICA connection
|
||||
|
||||
@@ -21,6 +21,6 @@ target_include_directories(mg-replication PUBLIC include)
|
||||
|
||||
find_package(fmt REQUIRED)
|
||||
target_link_libraries(mg-replication
|
||||
PUBLIC mg::utils mg::kvstore lib::json mg::rpc mg::slk mg::io mg::repl_coord_glue mg-flags
|
||||
PUBLIC mg::utils mg::kvstore lib::json mg::rpc mg::slk mg::io mg::repl_coord_glue
|
||||
PRIVATE fmt::fmt
|
||||
)
|
||||
|
||||
@@ -54,7 +54,7 @@ struct ReplicationClient {
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
// Nothing to do...wait for a reconnect
|
||||
// NOTE: Here we are communicating with the instance connection.
|
||||
// We don't have access to the underlying client; so the only thing we can do it
|
||||
// We don't have access to the undelying client; so the only thing we can do it
|
||||
// tell the callback that this is a reconnection and to check the state
|
||||
reconnect = true;
|
||||
}
|
||||
@@ -106,9 +106,6 @@ struct ReplicationClient {
|
||||
communication::ClientContext rpc_context_;
|
||||
rpc::Client rpc_client_;
|
||||
std::chrono::seconds replica_check_frequency_;
|
||||
// True only when we are migrating from V1 or V2 to V3 in replication durability
|
||||
// and we want to set replica to listen to main
|
||||
bool try_set_uuid{false};
|
||||
|
||||
// TODO: Better, this was the easiest place to put this
|
||||
enum class State {
|
||||
|
||||
@@ -21,12 +21,10 @@
|
||||
#include "status.hpp"
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <list>
|
||||
#include <optional>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
@@ -39,11 +37,7 @@ enum class RegisterReplicaError : uint8_t { NAME_EXISTS, ENDPOINT_EXISTS, COULD_
|
||||
|
||||
struct RoleMainData {
|
||||
RoleMainData() = default;
|
||||
explicit RoleMainData(ReplicationEpoch e, std::optional<utils::UUID> uuid = std::nullopt) : epoch_(std::move(e)) {
|
||||
if (uuid) {
|
||||
uuid_ = *uuid;
|
||||
}
|
||||
}
|
||||
explicit RoleMainData(ReplicationEpoch e) : epoch_(std::move(e)) {}
|
||||
~RoleMainData() = default;
|
||||
|
||||
RoleMainData(RoleMainData const &) = delete;
|
||||
@@ -53,14 +47,11 @@ struct RoleMainData {
|
||||
|
||||
ReplicationEpoch epoch_;
|
||||
std::list<ReplicationClient> registered_replicas_{}; // TODO: data race issues
|
||||
utils::UUID uuid_;
|
||||
};
|
||||
|
||||
struct RoleReplicaData {
|
||||
ReplicationServerConfig config;
|
||||
std::unique_ptr<ReplicationServer> server;
|
||||
// uuid of main replica is listening to
|
||||
std::optional<utils::UUID> uuid_;
|
||||
};
|
||||
|
||||
// Global (instance) level object
|
||||
@@ -92,19 +83,18 @@ struct ReplicationState {
|
||||
|
||||
bool HasDurability() const { return nullptr != durability_; }
|
||||
|
||||
bool TryPersistRoleMain(std::string new_epoch, utils::UUID main_uuid);
|
||||
bool TryPersistRoleReplica(const ReplicationServerConfig &config, const std::optional<utils::UUID> &main_uuid);
|
||||
bool TryPersistRoleMain(std::string new_epoch);
|
||||
bool TryPersistRoleReplica(const ReplicationServerConfig &config);
|
||||
bool TryPersistUnregisterReplica(std::string_view name);
|
||||
bool TryPersistRegisteredReplica(const ReplicationClientConfig &config, utils::UUID main_uuid);
|
||||
bool TryPersistRegisteredReplica(const ReplicationClientConfig &config);
|
||||
|
||||
// TODO: locked access
|
||||
auto ReplicationData() -> ReplicationData_t & { return replication_data_; }
|
||||
auto ReplicationData() const -> ReplicationData_t const & { return replication_data_; }
|
||||
utils::BasicResult<RegisterReplicaError, ReplicationClient *> RegisterReplica(const ReplicationClientConfig &config);
|
||||
|
||||
bool SetReplicationRoleMain(const utils::UUID &main_uuid);
|
||||
bool SetReplicationRoleReplica(const ReplicationServerConfig &config,
|
||||
const std::optional<utils::UUID> &main_uuid = std::nullopt);
|
||||
bool SetReplicationRoleMain();
|
||||
bool SetReplicationRoleReplica(const ReplicationServerConfig &config);
|
||||
|
||||
private:
|
||||
bool HandleVersionMigration(durability::ReplicationRoleEntry &data) const;
|
||||
|
||||
@@ -31,28 +31,25 @@ constexpr auto *kReplicationReplicaPrefix{"__replication_replica:"}; // introdu
|
||||
|
||||
enum class DurabilityVersion : uint8_t {
|
||||
V1, // no distinct key for replicas
|
||||
V2, // epoch, replica prefix introduced
|
||||
V3, // this version, main uuid introduced
|
||||
V2, // this version, epoch, replica prefix introduced
|
||||
};
|
||||
|
||||
// fragment of key: "__replication_role"
|
||||
struct MainRole {
|
||||
ReplicationEpoch epoch{};
|
||||
std::optional<utils::UUID> main_uuid{};
|
||||
friend bool operator==(MainRole const &, MainRole const &) = default;
|
||||
};
|
||||
|
||||
// fragment of key: "__replication_role"
|
||||
struct ReplicaRole {
|
||||
ReplicationServerConfig config{};
|
||||
std::optional<utils::UUID> main_uuid{};
|
||||
friend bool operator==(ReplicaRole const &, ReplicaRole const &) = default;
|
||||
};
|
||||
|
||||
// from key: "__replication_role"
|
||||
struct ReplicationRoleEntry {
|
||||
DurabilityVersion version =
|
||||
DurabilityVersion::V3; // if not latest then migration required for kReplicationReplicaPrefix
|
||||
DurabilityVersion::V2; // if not latest then migration required for kReplicationReplicaPrefix
|
||||
std::variant<MainRole, ReplicaRole> role;
|
||||
|
||||
friend bool operator==(ReplicationRoleEntry const &, ReplicationRoleEntry const &) = default;
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "replication/replication_server.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "replication_coordination_glue/messages.hpp"
|
||||
|
||||
namespace memgraph::replication {
|
||||
namespace {
|
||||
|
||||
@@ -10,15 +10,12 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "replication/state.hpp"
|
||||
#include <optional>
|
||||
|
||||
#include "flags/replication.hpp"
|
||||
#include "replication/replication_client.hpp"
|
||||
#include "replication/replication_server.hpp"
|
||||
#include "replication/status.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
constexpr auto kReplicationDirectory = std::string_view{"replication"};
|
||||
@@ -39,9 +36,9 @@ ReplicationState::ReplicationState(std::optional<std::filesystem::path> durabili
|
||||
durability_ = std::make_unique<kvstore::KVStore>(std::move(repl_dir));
|
||||
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
|
||||
|
||||
auto fetched_replication_data = FetchReplicationData();
|
||||
if (fetched_replication_data.HasError()) {
|
||||
switch (fetched_replication_data.GetError()) {
|
||||
auto replicationData = FetchReplicationData();
|
||||
if (replicationData.HasError()) {
|
||||
switch (replicationData.GetError()) {
|
||||
using enum ReplicationState::FetchReplicationError;
|
||||
case NOTHING_FETCHED: {
|
||||
spdlog::debug("Cannot find data needed for restore replication role in persisted metadata.");
|
||||
@@ -54,21 +51,15 @@ ReplicationState::ReplicationState(std::optional<std::filesystem::path> durabili
|
||||
}
|
||||
}
|
||||
}
|
||||
auto replication_data = std::move(fetched_replication_data).GetValue();
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (FLAGS_coordinator_server_port && std::holds_alternative<RoleReplicaData>(replication_data)) {
|
||||
std::get<RoleReplicaData>(replication_data).uuid_.reset();
|
||||
}
|
||||
#endif
|
||||
replication_data_ = std::move(replication_data);
|
||||
replication_data_ = std::move(replicationData).GetValue();
|
||||
}
|
||||
|
||||
bool ReplicationState::TryPersistRoleReplica(const ReplicationServerConfig &config,
|
||||
const std::optional<utils::UUID> &main_uuid) {
|
||||
bool ReplicationState::TryPersistRoleReplica(const ReplicationServerConfig &config) {
|
||||
if (!HasDurability()) return true;
|
||||
|
||||
auto data =
|
||||
durability::ReplicationRoleEntry{.role = durability::ReplicaRole{.config = config, .main_uuid = main_uuid}};
|
||||
auto data = durability::ReplicationRoleEntry{.role = durability::ReplicaRole{
|
||||
.config = config,
|
||||
}};
|
||||
|
||||
if (!durability_->Put(durability::kReplicationRoleName, nlohmann::json(data).dump())) {
|
||||
spdlog::error("Error when saving REPLICA replication role in settings.");
|
||||
@@ -87,11 +78,11 @@ bool ReplicationState::TryPersistRoleReplica(const ReplicationServerConfig &conf
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ReplicationState::TryPersistRoleMain(std::string new_epoch, utils::UUID main_uuid) {
|
||||
bool ReplicationState::TryPersistRoleMain(std::string new_epoch) {
|
||||
if (!HasDurability()) return true;
|
||||
|
||||
auto data = durability::ReplicationRoleEntry{
|
||||
.role = durability::MainRole{.epoch = ReplicationEpoch{std::move(new_epoch)}, .main_uuid = main_uuid}};
|
||||
auto data =
|
||||
durability::ReplicationRoleEntry{.role = durability::MainRole{.epoch = ReplicationEpoch{std::move(new_epoch)}}};
|
||||
|
||||
if (durability_->Put(durability::kReplicationRoleName, nlohmann::json(data).dump())) {
|
||||
role_persisted = RolePersisted::YES;
|
||||
@@ -137,8 +128,7 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t {
|
||||
return std::visit(
|
||||
utils::Overloaded{
|
||||
[&](durability::MainRole &&r) -> FetchReplicationResult_t {
|
||||
auto res =
|
||||
RoleMainData{std::move(r.epoch), r.main_uuid.has_value() ? r.main_uuid.value() : utils::UUID{}};
|
||||
auto res = RoleMainData{std::move(r.epoch)};
|
||||
auto b = durability_->begin(durability::kReplicationReplicaPrefix);
|
||||
auto e = durability_->end(durability::kReplicationReplicaPrefix);
|
||||
for (; b != e; ++b) {
|
||||
@@ -153,8 +143,6 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t {
|
||||
}
|
||||
// Instance clients
|
||||
res.registered_replicas_.emplace_back(data.config);
|
||||
// Bump for each replica uuid
|
||||
res.registered_replicas_.back().try_set_uuid = !r.main_uuid.has_value();
|
||||
} catch (...) {
|
||||
return FetchReplicationError::PARSE_ERROR;
|
||||
}
|
||||
@@ -162,9 +150,7 @@ auto ReplicationState::FetchReplicationData() -> FetchReplicationResult_t {
|
||||
return {std::move(res)};
|
||||
},
|
||||
[&](durability::ReplicaRole &&r) -> FetchReplicationResult_t {
|
||||
// False positive report for the std::make_unique
|
||||
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
|
||||
return {RoleReplicaData{r.config, std::make_unique<ReplicationServer>(r.config), r.main_uuid}};
|
||||
return {RoleReplicaData{r.config, std::make_unique<ReplicationServer>(r.config)}};
|
||||
},
|
||||
},
|
||||
std::move(data.role));
|
||||
@@ -206,29 +192,21 @@ bool ReplicationState::HandleVersionMigration(durability::ReplicationRoleEntry &
|
||||
[[fallthrough]];
|
||||
}
|
||||
case durability::DurabilityVersion::V2: {
|
||||
if (std::holds_alternative<durability::MainRole>(data.role)) {
|
||||
auto &main = std::get<durability::MainRole>(data.role);
|
||||
main.main_uuid = utils::UUID{};
|
||||
}
|
||||
data.version = durability::DurabilityVersion::V3;
|
||||
break;
|
||||
}
|
||||
case durability::DurabilityVersion::V3: {
|
||||
// do nothing - add code if V4 ever happens
|
||||
// do nothing - add code if V3 ever happens
|
||||
break;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ReplicationState::TryPersistRegisteredReplica(const ReplicationClientConfig &config, utils::UUID main_uuid) {
|
||||
bool ReplicationState::TryPersistRegisteredReplica(const ReplicationClientConfig &config) {
|
||||
if (!HasDurability()) return true;
|
||||
|
||||
// If any replicas are persisted then Role must be persisted
|
||||
if (role_persisted != RolePersisted::YES) {
|
||||
DMG_ASSERT(IsMain(), "MAIN is expected");
|
||||
auto epoch_str = std::string(std::get<RoleMainData>(replication_data_).epoch_.id());
|
||||
if (!TryPersistRoleMain(std::move(epoch_str), main_uuid)) return false;
|
||||
if (!TryPersistRoleMain(std::move(epoch_str))) return false;
|
||||
}
|
||||
|
||||
auto data = durability::ReplicationReplicaEntry{.config = config};
|
||||
@@ -239,28 +217,22 @@ bool ReplicationState::TryPersistRegisteredReplica(const ReplicationClientConfig
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ReplicationState::SetReplicationRoleMain(const utils::UUID &main_uuid) {
|
||||
bool ReplicationState::SetReplicationRoleMain() {
|
||||
auto new_epoch = utils::GenerateUUID();
|
||||
|
||||
if (!TryPersistRoleMain(new_epoch, main_uuid)) {
|
||||
if (!TryPersistRoleMain(new_epoch)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
replication_data_ = RoleMainData{ReplicationEpoch{new_epoch}, main_uuid};
|
||||
replication_data_ = RoleMainData{ReplicationEpoch{new_epoch}};
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ReplicationState::SetReplicationRoleReplica(const ReplicationServerConfig &config,
|
||||
const std::optional<utils::UUID> &main_uuid) {
|
||||
// False positive report for the std::make_unique
|
||||
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
|
||||
if (!TryPersistRoleReplica(config, main_uuid)) {
|
||||
bool ReplicationState::SetReplicationRoleReplica(const ReplicationServerConfig &config) {
|
||||
if (!TryPersistRoleReplica(config)) {
|
||||
return false;
|
||||
}
|
||||
// False positive report for the std::make_unique
|
||||
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
|
||||
replication_data_ = RoleReplicaData{config, std::make_unique<ReplicationServer>(config), std::nullopt};
|
||||
replication_data_ = RoleReplicaData{config, std::make_unique<ReplicationServer>(config)};
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -292,7 +264,7 @@ utils::BasicResult<RegisterReplicaError, ReplicationClient *> ReplicationState::
|
||||
}
|
||||
|
||||
// Durability
|
||||
if (!TryPersistRegisteredReplica(config, mainData.uuid_)) {
|
||||
if (!TryPersistRegisteredReplica(config)) {
|
||||
return RegisterReplicaError::COULD_NOT_BE_PERSISTED;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,28 +26,21 @@ constexpr auto *kSSLCertFile = "replica_ssl_cert_file";
|
||||
constexpr auto *kReplicationRole = "replication_role";
|
||||
constexpr auto *kEpoch = "epoch";
|
||||
constexpr auto *kVersion = "durability_version";
|
||||
constexpr auto *kMainUUID = "main_uuid";
|
||||
|
||||
void to_json(nlohmann::json &j, const ReplicationRoleEntry &p) {
|
||||
auto processMAIN = [&](MainRole const &main) {
|
||||
auto common = nlohmann::json{{kVersion, p.version},
|
||||
{kReplicationRole, replication_coordination_glue::ReplicationRole::MAIN},
|
||||
{kEpoch, main.epoch.id()}};
|
||||
if (p.version != DurabilityVersion::V1 && p.version != DurabilityVersion::V2) {
|
||||
MG_ASSERT(main.main_uuid.has_value(), "Main should have id ready on version >= V3");
|
||||
common[kMainUUID] = main.main_uuid.value();
|
||||
}
|
||||
j = std::move(common);
|
||||
j = nlohmann::json{{kVersion, p.version},
|
||||
{kReplicationRole, replication_coordination_glue::ReplicationRole::MAIN},
|
||||
{kEpoch, main.epoch.id()}};
|
||||
};
|
||||
auto processREPLICA = [&](ReplicaRole const &replica) {
|
||||
auto common = nlohmann::json{{kVersion, p.version},
|
||||
{kReplicationRole, replication_coordination_glue::ReplicationRole::REPLICA},
|
||||
{kIpAddress, replica.config.ip_address},
|
||||
{kPort, replica.config.port}};
|
||||
if (replica.main_uuid.has_value()) {
|
||||
common[kMainUUID] = replica.main_uuid.value();
|
||||
}
|
||||
j = std::move(common);
|
||||
j = nlohmann::json{
|
||||
{kVersion, p.version},
|
||||
{kReplicationRole, replication_coordination_glue::ReplicationRole::REPLICA},
|
||||
{kIpAddress, replica.config.ip_address},
|
||||
{kPort, replica.config.port}
|
||||
// TODO: SSL
|
||||
};
|
||||
};
|
||||
std::visit(utils::Overloaded{processMAIN, processREPLICA}, p.role);
|
||||
}
|
||||
@@ -63,12 +56,7 @@ void from_json(const nlohmann::json &j, ReplicationRoleEntry &p) {
|
||||
auto json_epoch = j.value(kEpoch, std::string{});
|
||||
auto epoch = ReplicationEpoch{};
|
||||
if (!json_epoch.empty()) epoch.SetEpoch(json_epoch);
|
||||
auto main_role = MainRole{.epoch = std::move(epoch)};
|
||||
|
||||
if (j.contains(kMainUUID)) {
|
||||
main_role.main_uuid = j.at(kMainUUID);
|
||||
}
|
||||
p = ReplicationRoleEntry{.version = version, .role = std::move(main_role)};
|
||||
p = ReplicationRoleEntry{.version = version, .role = MainRole{.epoch = std::move(epoch)}};
|
||||
break;
|
||||
}
|
||||
case memgraph::replication_coordination_glue::ReplicationRole::REPLICA: {
|
||||
@@ -78,13 +66,7 @@ void from_json(const nlohmann::json &j, ReplicationRoleEntry &p) {
|
||||
j.at(kIpAddress).get_to(ip_address);
|
||||
j.at(kPort).get_to(port);
|
||||
auto config = ReplicationServerConfig{.ip_address = std::move(ip_address), .port = port};
|
||||
auto replica_role = ReplicaRole{.config = std::move(config)};
|
||||
if (j.contains(kMainUUID)) {
|
||||
replica_role.main_uuid = j.at(kMainUUID);
|
||||
}
|
||||
|
||||
p = ReplicationRoleEntry{.version = version, .role = std::move(replica_role)};
|
||||
|
||||
p = ReplicationRoleEntry{.version = version, .role = ReplicaRole{.config = std::move(config)}};
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ target_sources(mg-repl_coord_glue
|
||||
messages.hpp
|
||||
mode.hpp
|
||||
role.hpp
|
||||
handler.hpp
|
||||
|
||||
PRIVATE
|
||||
messages.cpp
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
#pragma once
|
||||
|
||||
#include "rpc/client.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
#include "messages.hpp"
|
||||
#include "rpc/messages.hpp"
|
||||
|
||||
namespace memgraph::replication_coordination_glue {
|
||||
inline bool SendSwapMainUUIDRpc(memgraph::rpc::Client &rpc_client_, const memgraph::utils::UUID &uuid) {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<SwapMainUUIDRpc>(uuid)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC swapping of uuid response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (const memgraph::rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occurred while sending swapping uuid RPC!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
inline void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
FrequentHeartbeatReq req;
|
||||
FrequentHeartbeatReq::Load(&req, req_reader);
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
FrequentHeartbeatRes res{};
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
} // namespace memgraph::replication_coordination_glue
|
||||
@@ -29,25 +29,6 @@ void Load(memgraph::replication_coordination_glue::FrequentHeartbeatReq * /*self
|
||||
/* Nothing to serialize */
|
||||
}
|
||||
|
||||
// Serialize code for SwapMainUUIDRes
|
||||
|
||||
void Save(const memgraph::replication_coordination_glue::SwapMainUUIDRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::replication_coordination_glue::SwapMainUUIDRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
// Serialize code for SwapMainUUIDReq
|
||||
void Save(const memgraph::replication_coordination_glue::SwapMainUUIDReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::replication_coordination_glue::SwapMainUUIDReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
}
|
||||
|
||||
} // namespace memgraph::slk
|
||||
|
||||
namespace memgraph::replication_coordination_glue {
|
||||
@@ -58,10 +39,6 @@ constexpr utils::TypeInfo FrequentHeartbeatReq::kType{utils::TypeId::REP_FREQUEN
|
||||
constexpr utils::TypeInfo FrequentHeartbeatRes::kType{utils::TypeId::REP_FREQUENT_HEARTBEAT_RES, "FrequentHeartbeatRes",
|
||||
nullptr};
|
||||
|
||||
constexpr utils::TypeInfo SwapMainUUIDReq::kType{utils::TypeId::COORD_SWAP_UUID_REQ, "SwapUUIDReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo SwapMainUUIDRes::kType{utils::TypeId::COORD_SWAP_UUID_RES, "SwapUUIDRes", nullptr};
|
||||
|
||||
void FrequentHeartbeatReq::Save(const FrequentHeartbeatReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
@@ -75,16 +52,12 @@ void FrequentHeartbeatRes::Load(FrequentHeartbeatRes *self, memgraph::slk::Reade
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void SwapMainUUIDReq::Save(const SwapMainUUIDReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
FrequentHeartbeatReq req;
|
||||
FrequentHeartbeatReq::Load(&req, req_reader);
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
FrequentHeartbeatRes res{};
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void SwapMainUUIDReq::Load(SwapMainUUIDReq *self, memgraph::slk::Reader *reader) { memgraph::slk::Load(self, reader); }
|
||||
|
||||
void SwapMainUUIDRes::Save(const SwapMainUUIDRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void SwapMainUUIDRes::Load(SwapMainUUIDRes *self, memgraph::slk::Reader *reader) { memgraph::slk::Load(self, reader); }
|
||||
|
||||
} // namespace memgraph::replication_coordination_glue
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
|
||||
#include "rpc/messages.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
namespace memgraph::replication_coordination_glue {
|
||||
|
||||
@@ -37,34 +36,7 @@ struct FrequentHeartbeatRes {
|
||||
|
||||
using FrequentHeartbeatRpc = rpc::RequestResponse<FrequentHeartbeatReq, FrequentHeartbeatRes>;
|
||||
|
||||
struct SwapMainUUIDReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(SwapMainUUIDReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const SwapMainUUIDReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit SwapMainUUIDReq(const utils::UUID &uuid) : uuid(uuid) {}
|
||||
|
||||
SwapMainUUIDReq() = default;
|
||||
|
||||
utils::UUID uuid;
|
||||
};
|
||||
|
||||
struct SwapMainUUIDRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(SwapMainUUIDRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const SwapMainUUIDRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit SwapMainUUIDRes(bool success) : success(success) {}
|
||||
SwapMainUUIDRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using SwapMainUUIDRpc = rpc::RequestResponse<SwapMainUUIDReq, SwapMainUUIDRes>;
|
||||
void FrequentHeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
} // namespace memgraph::replication_coordination_glue
|
||||
|
||||
@@ -74,10 +46,4 @@ void Load(memgraph::replication_coordination_glue::FrequentHeartbeatRes *self, m
|
||||
void Save(const memgraph::replication_coordination_glue::FrequentHeartbeatReq & /*self*/,
|
||||
memgraph::slk::Builder * /*builder*/);
|
||||
void Load(memgraph::replication_coordination_glue::FrequentHeartbeatReq * /*self*/, memgraph::slk::Reader * /*reader*/);
|
||||
|
||||
// SwapMainUUIDRpc
|
||||
void Save(const memgraph::replication_coordination_glue::SwapMainUUIDReq &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::replication_coordination_glue::SwapMainUUIDReq *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::replication_coordination_glue::SwapMainUUIDRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::replication_coordination_glue::SwapMainUUIDRes *self, memgraph::slk::Reader *reader);
|
||||
} // namespace memgraph::slk
|
||||
|
||||
@@ -7,8 +7,8 @@ target_sources(mg-replication_handler
|
||||
include/replication_handler/system_rpc.hpp
|
||||
|
||||
PRIVATE
|
||||
system_replication.cpp
|
||||
replication_handler.cpp
|
||||
system_replication.cpp
|
||||
system_rpc.cpp
|
||||
)
|
||||
target_include_directories(mg-replication_handler PUBLIC include)
|
||||
|
||||
@@ -22,10 +22,10 @@ inline std::optional<query::RegisterReplicaError> HandleRegisterReplicaStatus(
|
||||
utils::BasicResult<replication::RegisterReplicaError, replication::ReplicationClient *> &instance_client);
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler, utils::UUID main_uuid,
|
||||
system::System *system, auth::SynchedAuth &auth);
|
||||
void StartReplicaClient(replication::ReplicationClient &client, system::System *system, dbms::DbmsHandler &dbms_handler,
|
||||
auth::SynchedAuth &auth);
|
||||
#else
|
||||
void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler, utils::UUID main_uuid);
|
||||
void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler);
|
||||
#endif
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
@@ -33,8 +33,8 @@ void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandle
|
||||
// When being called by interpreter no need to gain lock, it should already be under a system transaction
|
||||
// But concurrently the FrequentCheck is running and will need to lock before reading last_committed_system_timestamp_
|
||||
template <bool REQUIRE_LOCK = false>
|
||||
void SystemRestore(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler,
|
||||
const utils::UUID &main_uuid, system::System *system, auth::SynchedAuth &auth) {
|
||||
void SystemRestore(replication::ReplicationClient &client, system::System *system, dbms::DbmsHandler &dbms_handler,
|
||||
auth::SynchedAuth &auth) {
|
||||
// Check if system is up to date
|
||||
if (client.state_.WithLock(
|
||||
[](auto &state) { return state == memgraph::replication::ReplicationClient::State::READY; }))
|
||||
@@ -69,12 +69,12 @@ void SystemRestore(replication::ReplicationClient &client, dbms::DbmsHandler &db
|
||||
// Handle only default database is no license
|
||||
if (!license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
return client.rpc_client_.Stream<replication::SystemRecoveryRpc>(
|
||||
main_uuid, db_info.last_committed_timestamp, std::move(db_info.configs), auth::Auth::Config{},
|
||||
db_info.last_committed_timestamp, std::move(db_info.configs), auth::Auth::Config{},
|
||||
std::vector<auth::User>{}, std::vector<auth::Role>{});
|
||||
}
|
||||
return auth.WithLock([&](auto &locked_auth) {
|
||||
return client.rpc_client_.Stream<replication::SystemRecoveryRpc>(
|
||||
main_uuid, db_info.last_committed_timestamp, std::move(db_info.configs), locked_auth.GetConfig(),
|
||||
db_info.last_committed_timestamp, std::move(db_info.configs), locked_auth.GetConfig(),
|
||||
locked_auth.AllUsers(), locked_auth.AllRoles());
|
||||
});
|
||||
});
|
||||
@@ -109,32 +109,28 @@ struct ReplicationHandler : public memgraph::query::ReplicationQueryHandler {
|
||||
bool SetReplicationRoleMain() override;
|
||||
|
||||
// as MAIN, become REPLICA
|
||||
bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config,
|
||||
const std::optional<utils::UUID> &main_uuid) override;
|
||||
bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) override;
|
||||
|
||||
// as MAIN, define and connect to REPLICAs
|
||||
auto TryRegisterReplica(const memgraph::replication::ReplicationClientConfig &config, bool send_swap_uuid)
|
||||
auto TryRegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> memgraph::utils::BasicResult<memgraph::query::RegisterReplicaError> override;
|
||||
|
||||
auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config, bool send_swap_uuid)
|
||||
auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> memgraph::utils::BasicResult<memgraph::query::RegisterReplicaError> override;
|
||||
|
||||
// as MAIN, remove a REPLICA connection
|
||||
auto UnregisterReplica(std::string_view name) -> memgraph::query::UnregisterReplicaResult override;
|
||||
|
||||
bool DoReplicaToMainPromotion(const utils::UUID &main_uuid);
|
||||
bool DoReplicaToMainPromotion();
|
||||
|
||||
// Helper pass-through (TODO: remove)
|
||||
auto GetRole() const -> memgraph::replication_coordination_glue::ReplicationRole override;
|
||||
bool IsMain() const override;
|
||||
bool IsReplica() const override;
|
||||
|
||||
auto GetReplState() const -> const memgraph::replication::ReplicationState &;
|
||||
auto GetReplState() -> memgraph::replication::ReplicationState &;
|
||||
|
||||
private:
|
||||
template <bool HandleFailure>
|
||||
auto RegisterReplica_(const memgraph::replication::ReplicationClientConfig &config, bool send_swap_uuid)
|
||||
auto RegisterReplica_(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> memgraph::utils::BasicResult<memgraph::query::RegisterReplicaError> {
|
||||
MG_ASSERT(repl_state_.IsMain(), "Only main instance can register a replica!");
|
||||
|
||||
@@ -158,19 +154,10 @@ struct ReplicationHandler : public memgraph::query::ReplicationQueryHandler {
|
||||
if (!memgraph::dbms::allow_mt_repl && dbms_handler_.All().size() > 1) {
|
||||
spdlog::warn("Multi-tenant replication is currently not supported!");
|
||||
}
|
||||
const auto main_uuid =
|
||||
std::get<memgraph::replication::RoleMainData>(dbms_handler_.ReplicationState().ReplicationData()).uuid_;
|
||||
|
||||
if (send_swap_uuid) {
|
||||
if (!memgraph::replication_coordination_glue::SendSwapMainUUIDRpc(maybe_client.GetValue()->rpc_client_,
|
||||
main_uuid)) {
|
||||
return memgraph::query::RegisterReplicaError::ERROR_ACCEPTING_MAIN;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
// Update system before enabling individual storage <-> replica clients
|
||||
SystemRestore(*maybe_client.GetValue(), dbms_handler_, main_uuid, system_, auth_);
|
||||
SystemRestore(*maybe_client.GetValue(), system_, dbms_handler_, auth_);
|
||||
#endif
|
||||
|
||||
const auto dbms_error = HandleRegisterReplicaStatus(maybe_client);
|
||||
@@ -190,9 +177,8 @@ struct ReplicationHandler : public memgraph::query::ReplicationQueryHandler {
|
||||
if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
|
||||
|
||||
all_clients_good &= storage->repl_storage_state_.replication_clients_.WithLock(
|
||||
[storage, &instance_client_ptr, db_acc = std::move(db_acc),
|
||||
main_uuid](auto &storage_clients) mutable { // NOLINT
|
||||
auto client = std::make_unique<storage::ReplicationStorageClient>(*instance_client_ptr, main_uuid);
|
||||
[storage, &instance_client_ptr, db_acc = std::move(db_acc)](auto &storage_clients) mutable { // NOLINT
|
||||
auto client = std::make_unique<storage::ReplicationStorageClient>(*instance_client_ptr);
|
||||
// All good, start replica client
|
||||
client->Start(storage, std::move(db_acc));
|
||||
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
|
||||
@@ -215,9 +201,9 @@ struct ReplicationHandler : public memgraph::query::ReplicationQueryHandler {
|
||||
|
||||
// No client error, start instance level client
|
||||
#ifdef MG_ENTERPRISE
|
||||
StartReplicaClient(*instance_client_ptr, dbms_handler_, main_uuid, system_, auth_);
|
||||
StartReplicaClient(*instance_client_ptr, system_, dbms_handler_, auth_);
|
||||
#else
|
||||
StartReplicaClient(*instance_client_ptr, dbms_handler_, main_uuid);
|
||||
StartReplicaClient(*instance_client_ptr, dbms_handler_);
|
||||
#endif
|
||||
return {};
|
||||
}
|
||||
|
||||
@@ -17,23 +17,15 @@
|
||||
#include "system/state.hpp"
|
||||
|
||||
namespace memgraph::replication {
|
||||
|
||||
inline void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req) {
|
||||
spdlog::error("Received {} with main_id: {} != current_main_uuid: {}", rpc_req, std::string(main_req_id),
|
||||
current_main_uuid.has_value() ? std::string(current_main_uuid.value()) : "");
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void SystemHeartbeatHandler(uint64_t ts, const std::optional<utils::UUID> ¤t_main_uuid, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
void SystemHeartbeatHandler(uint64_t ts, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
void SystemRecoveryHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
std::optional<utils::UUID> ¤t_main_uuid, dbms::DbmsHandler &dbms_handler,
|
||||
auth::SynchedAuth &auth, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
dbms::DbmsHandler &dbms_handler, auth::SynchedAuth &auth, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
void Register(replication::RoleReplicaData const &data, dbms::DbmsHandler &dbms_handler, auth::SynchedAuth &auth);
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, replication::RoleReplicaData &data, auth::SynchedAuth &auth);
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, const replication::RoleReplicaData &data, auth::SynchedAuth &auth);
|
||||
#else
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, replication::RoleReplicaData &data);
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, const replication::RoleReplicaData &data);
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::replication
|
||||
|
||||
@@ -27,8 +27,6 @@ struct SystemHeartbeatReq {
|
||||
static void Load(SystemHeartbeatReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const SystemHeartbeatReq &self, memgraph::slk::Builder *builder);
|
||||
SystemHeartbeatReq() = default;
|
||||
explicit SystemHeartbeatReq(const utils::UUID &main_uuid) : main_uuid(main_uuid) {}
|
||||
utils::UUID main_uuid;
|
||||
};
|
||||
|
||||
struct SystemHeartbeatRes {
|
||||
@@ -52,17 +50,14 @@ struct SystemRecoveryReq {
|
||||
static void Load(SystemRecoveryReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const SystemRecoveryReq &self, memgraph::slk::Builder *builder);
|
||||
SystemRecoveryReq() = default;
|
||||
SystemRecoveryReq(const utils::UUID &main_uuid, uint64_t forced_group_timestamp,
|
||||
std::vector<storage::SalientConfig> database_configs, auth::Auth::Config auth_config,
|
||||
std::vector<auth::User> users, std::vector<auth::Role> roles)
|
||||
: main_uuid(main_uuid),
|
||||
forced_group_timestamp{forced_group_timestamp},
|
||||
SystemRecoveryReq(uint64_t forced_group_timestamp, std::vector<storage::SalientConfig> database_configs,
|
||||
auth::Auth::Config auth_config, std::vector<auth::User> users, std::vector<auth::Role> roles)
|
||||
: forced_group_timestamp{forced_group_timestamp},
|
||||
database_configs(std::move(database_configs)),
|
||||
auth_config(std::move(auth_config)),
|
||||
users{std::move(users)},
|
||||
roles{std::move(roles)} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
uint64_t forced_group_timestamp;
|
||||
std::vector<storage::SalientConfig> database_configs;
|
||||
auth::Auth::Config auth_config;
|
||||
|
||||
@@ -24,18 +24,14 @@ void RecoverReplication(memgraph::replication::ReplicationState &repl_state, mem
|
||||
*/
|
||||
|
||||
// Startup replication state (if recovered at startup)
|
||||
auto replica = [&dbms_handler, &auth](memgraph::replication::RoleReplicaData &data) {
|
||||
return StartRpcServer(dbms_handler, data, auth);
|
||||
auto replica = [&dbms_handler, &auth](memgraph::replication::RoleReplicaData const &data) {
|
||||
return memgraph::replication::StartRpcServer(dbms_handler, data, auth);
|
||||
};
|
||||
|
||||
// Replication recovery and frequent check start
|
||||
auto main = [system, &dbms_handler, &auth](memgraph::replication::RoleMainData &mainData) {
|
||||
for (auto &client : mainData.registered_replicas_) {
|
||||
if (client.try_set_uuid &&
|
||||
replication_coordination_glue::SendSwapMainUUIDRpc(client.rpc_client_, mainData.uuid_)) {
|
||||
client.try_set_uuid = false;
|
||||
}
|
||||
SystemRestore(client, dbms_handler, mainData.uuid_, system, auth);
|
||||
memgraph::replication::SystemRestore(client, system, dbms_handler, auth);
|
||||
}
|
||||
// DBMS here
|
||||
dbms_handler.ForEach([&mainData](memgraph::dbms::DatabaseAccess db_acc) {
|
||||
@@ -43,7 +39,7 @@ void RecoverReplication(memgraph::replication::ReplicationState &repl_state, mem
|
||||
});
|
||||
|
||||
for (auto &client : mainData.registered_replicas_) {
|
||||
StartReplicaClient(client, dbms_handler, mainData.uuid_, system, auth);
|
||||
memgraph::replication::StartReplicaClient(client, system, dbms_handler, auth);
|
||||
}
|
||||
|
||||
// Warning
|
||||
@@ -66,7 +62,7 @@ void RecoverReplication(memgraph::replication::ReplicationState &repl_state, mem
|
||||
void RecoverReplication(memgraph::replication::ReplicationState &repl_state,
|
||||
memgraph::dbms::DbmsHandler &dbms_handler) {
|
||||
// Startup replication state (if recovered at startup)
|
||||
auto replica = [&dbms_handler](memgraph::replication::RoleReplicaData &data) {
|
||||
auto replica = [&dbms_handler](memgraph::replication::RoleReplicaData const &data) {
|
||||
return memgraph::replication::StartRpcServer(dbms_handler, data);
|
||||
};
|
||||
|
||||
@@ -75,11 +71,7 @@ void RecoverReplication(memgraph::replication::ReplicationState &repl_state,
|
||||
dbms::DbmsHandler::RecoverStorageReplication(dbms_handler.Get(), mainData);
|
||||
|
||||
for (auto &client : mainData.registered_replicas_) {
|
||||
if (client.try_set_uuid &&
|
||||
replication_coordination_glue::SendSwapMainUUIDRpc(client.rpc_client_, mainData.uuid_)) {
|
||||
client.try_set_uuid = false;
|
||||
}
|
||||
memgraph::replication::StartReplicaClient(client, dbms_handler, mainData.uuid_);
|
||||
memgraph::replication::StartReplicaClient(client, dbms_handler);
|
||||
}
|
||||
|
||||
// Warning
|
||||
@@ -120,11 +112,10 @@ inline std::optional<query::RegisterReplicaError> HandleRegisterReplicaStatus(
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler, utils::UUID main_uuid,
|
||||
system::System *system, auth::SynchedAuth &auth) {
|
||||
void StartReplicaClient(replication::ReplicationClient &client, system::System *system, dbms::DbmsHandler &dbms_handler,
|
||||
auth::SynchedAuth &auth) {
|
||||
#else
|
||||
void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler,
|
||||
utils::UUID main_uuid) {
|
||||
void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandler &dbms_handler) {
|
||||
#endif
|
||||
// No client error, start instance level client
|
||||
auto const &endpoint = client.rpc_client_.Endpoint();
|
||||
@@ -133,12 +124,8 @@ void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandle
|
||||
#ifdef MG_ENTERPRISE
|
||||
system = system,
|
||||
#endif
|
||||
license = license::global_license_checker.IsEnterpriseValidFast(),
|
||||
main_uuid](bool reconnect, replication::ReplicationClient &client) mutable {
|
||||
if (client.try_set_uuid &&
|
||||
memgraph::replication_coordination_glue::SendSwapMainUUIDRpc(client.rpc_client_, main_uuid)) {
|
||||
client.try_set_uuid = false;
|
||||
}
|
||||
license = license::global_license_checker.IsEnterpriseValidFast()](
|
||||
bool reconnect, replication::ReplicationClient &client) mutable {
|
||||
// Working connection
|
||||
// Check if system needs restoration
|
||||
if (reconnect) {
|
||||
@@ -151,7 +138,7 @@ void StartReplicaClient(replication::ReplicationClient &client, dbms::DbmsHandle
|
||||
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
|
||||
}
|
||||
#ifdef MG_ENTERPRISE
|
||||
SystemRestore<true>(client, dbms_handler, main_uuid, system, auth);
|
||||
SystemRestore<true>(client, system, dbms_handler, auth);
|
||||
#endif
|
||||
// Check if any database has been left behind
|
||||
dbms_handler.ForEach([&name = client.name_, reconnect](dbms::DatabaseAccess db_acc) {
|
||||
@@ -187,15 +174,14 @@ bool ReplicationHandler::SetReplicationRoleMain() {
|
||||
};
|
||||
|
||||
auto const replica_handler = [this](memgraph::replication::RoleReplicaData const &) {
|
||||
return DoReplicaToMainPromotion(utils::UUID{});
|
||||
return DoReplicaToMainPromotion();
|
||||
};
|
||||
|
||||
// TODO: under lock
|
||||
return std::visit(memgraph::utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData());
|
||||
}
|
||||
|
||||
bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config,
|
||||
const std::optional<utils::UUID> &main_uuid) {
|
||||
bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) {
|
||||
// We don't want to restart the server if we're already a REPLICA
|
||||
if (repl_state_.IsReplica()) {
|
||||
return false;
|
||||
@@ -212,26 +198,27 @@ bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::
|
||||
std::get<memgraph::replication::RoleMainData>(repl_state_.ReplicationData()).registered_replicas_.clear();
|
||||
|
||||
// Creates the server
|
||||
repl_state_.SetReplicationRoleReplica(config, main_uuid);
|
||||
repl_state_.SetReplicationRoleReplica(config);
|
||||
|
||||
// Start
|
||||
const auto success = std::visit(memgraph::utils::Overloaded{[](memgraph::replication::RoleMainData &) {
|
||||
// ASSERT
|
||||
return false;
|
||||
},
|
||||
[this](memgraph::replication::RoleReplicaData &data) {
|
||||
const auto success =
|
||||
std::visit(memgraph::utils::Overloaded{[](memgraph::replication::RoleMainData const &) {
|
||||
// ASSERT
|
||||
return false;
|
||||
},
|
||||
[this](memgraph::replication::RoleReplicaData const &data) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
return StartRpcServer(dbms_handler_, data, auth_);
|
||||
return StartRpcServer(dbms_handler_, data, auth_);
|
||||
#else
|
||||
return StartRpcServer(dbms_handler_, data);
|
||||
return StartRpcServer(dbms_handler_, data);
|
||||
#endif
|
||||
}},
|
||||
repl_state_.ReplicationData());
|
||||
}},
|
||||
repl_state_.ReplicationData());
|
||||
// TODO Handle error (restore to main?)
|
||||
return success;
|
||||
}
|
||||
|
||||
bool ReplicationHandler::DoReplicaToMainPromotion(const utils::UUID &main_uuid) {
|
||||
bool ReplicationHandler::DoReplicaToMainPromotion() {
|
||||
// STEP 1) bring down all REPLICA servers
|
||||
dbms_handler_.ForEach([](dbms::DatabaseAccess db_acc) {
|
||||
auto *storage = db_acc->storage();
|
||||
@@ -241,7 +228,7 @@ bool ReplicationHandler::DoReplicaToMainPromotion(const utils::UUID &main_uuid)
|
||||
|
||||
// STEP 2) Change to MAIN
|
||||
// TODO: restore replication servers if false?
|
||||
if (!repl_state_.SetReplicationRoleMain(main_uuid)) {
|
||||
if (!repl_state_.SetReplicationRoleMain()) {
|
||||
// TODO: Handle recovery on failure???
|
||||
return false;
|
||||
}
|
||||
@@ -257,16 +244,14 @@ bool ReplicationHandler::DoReplicaToMainPromotion(const utils::UUID &main_uuid)
|
||||
};
|
||||
|
||||
// as MAIN, define and connect to REPLICAs
|
||||
auto ReplicationHandler::TryRegisterReplica(const memgraph::replication::ReplicationClientConfig &config,
|
||||
bool send_swap_uuid)
|
||||
auto ReplicationHandler::TryRegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> memgraph::utils::BasicResult<memgraph::query::RegisterReplicaError> {
|
||||
return RegisterReplica_<false>(config, send_swap_uuid);
|
||||
return RegisterReplica_<false>(config);
|
||||
}
|
||||
|
||||
auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config,
|
||||
bool send_swap_uuid)
|
||||
auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> memgraph::utils::BasicResult<memgraph::query::RegisterReplicaError> {
|
||||
return RegisterReplica_<true>(config, send_swap_uuid);
|
||||
return RegisterReplica_<true>(config);
|
||||
}
|
||||
|
||||
auto ReplicationHandler::UnregisterReplica(std::string_view name) -> memgraph::query::UnregisterReplicaResult {
|
||||
@@ -299,10 +284,6 @@ auto ReplicationHandler::GetRole() const -> memgraph::replication_coordination_g
|
||||
return repl_state_.GetRole();
|
||||
}
|
||||
|
||||
auto ReplicationHandler::GetReplState() const -> const memgraph::replication::ReplicationState & { return repl_state_; }
|
||||
|
||||
auto ReplicationHandler::GetReplState() -> memgraph::replication::ReplicationState & { return repl_state_; }
|
||||
|
||||
bool ReplicationHandler::IsMain() const { return repl_state_.IsMain(); }
|
||||
|
||||
bool ReplicationHandler::IsReplica() const { return repl_state_.IsReplica(); }
|
||||
|
||||
@@ -21,8 +21,7 @@
|
||||
namespace memgraph::replication {
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void SystemHeartbeatHandler(const uint64_t ts, const std::optional<utils::UUID> ¤t_main_uuid,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
void SystemHeartbeatHandler(const uint64_t ts, slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::SystemHeartbeatRes res{0};
|
||||
|
||||
// Ignore if no license
|
||||
@@ -31,23 +30,17 @@ void SystemHeartbeatHandler(const uint64_t ts, const std::optional<utils::UUID>
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
replication::SystemHeartbeatReq req;
|
||||
replication::SystemHeartbeatReq::Load(&req, req_reader);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::SystemHeartbeatRes::kType.name);
|
||||
replication::SystemHeartbeatRes res(-1);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
res = replication::SystemHeartbeatRes{ts};
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void SystemRecoveryHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, dbms::DbmsHandler &dbms_handler,
|
||||
auth::SynchedAuth &auth, slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
dbms::DbmsHandler &dbms_handler, auth::SynchedAuth &auth, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
using memgraph::replication::SystemRecoveryRes;
|
||||
SystemRecoveryRes res(SystemRecoveryRes::Result::FAILURE);
|
||||
|
||||
@@ -56,11 +49,6 @@ void SystemRecoveryHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
memgraph::replication::SystemRecoveryReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, SystemRecoveryReq::kType.name);
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* DBMS
|
||||
*/
|
||||
@@ -86,16 +74,15 @@ void Register(replication::RoleReplicaData const &data, dbms::DbmsHandler &dbms_
|
||||
auto system_state_access = dbms_handler.system_->CreateSystemStateAccess();
|
||||
|
||||
// System
|
||||
// TODO: remove, as this is not used
|
||||
data.server->rpc_server_.Register<replication::SystemHeartbeatRpc>(
|
||||
[&data, system_state_access](auto *req_reader, auto *res_builder) {
|
||||
[system_state_access](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received SystemHeartbeatRpc");
|
||||
SystemHeartbeatHandler(system_state_access.LastCommitedTS(), data.uuid_, req_reader, res_builder);
|
||||
SystemHeartbeatHandler(system_state_access.LastCommitedTS(), req_reader, res_builder);
|
||||
});
|
||||
data.server->rpc_server_.Register<replication::SystemRecoveryRpc>(
|
||||
[&data, system_state_access, &dbms_handler, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &dbms_handler, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received SystemRecoveryRpc");
|
||||
SystemRecoveryHandler(system_state_access, data.uuid_, dbms_handler, auth, req_reader, res_builder);
|
||||
SystemRecoveryHandler(system_state_access, dbms_handler, auth, req_reader, res_builder);
|
||||
});
|
||||
|
||||
// DBMS
|
||||
@@ -107,12 +94,13 @@ void Register(replication::RoleReplicaData const &data, dbms::DbmsHandler &dbms_
|
||||
#endif
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, replication::RoleReplicaData &data, auth::SynchedAuth &auth) {
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, const replication::RoleReplicaData &data,
|
||||
auth::SynchedAuth &auth) {
|
||||
#else
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, replication::RoleReplicaData &data) {
|
||||
bool StartRpcServer(dbms::DbmsHandler &dbms_handler, const replication::RoleReplicaData &data) {
|
||||
#endif
|
||||
// Register storage handlers
|
||||
dbms::InMemoryReplicationHandlers::Register(&dbms_handler, data);
|
||||
dbms::InMemoryReplicationHandlers::Register(&dbms_handler, *data.server);
|
||||
#ifdef MG_ENTERPRISE
|
||||
// Register system handlers
|
||||
Register(data, dbms_handler, auth);
|
||||
@@ -124,5 +112,4 @@ bool StartRpcServer(dbms::DbmsHandler &dbms_handler, replication::RoleReplicaDat
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace memgraph::replication
|
||||
|
||||
@@ -29,16 +29,15 @@ void Load(memgraph::replication::SystemHeartbeatRes *self, memgraph::slk::Reader
|
||||
}
|
||||
|
||||
// Serialize code for SystemHeartbeatReq
|
||||
void Save(const memgraph::replication::SystemHeartbeatReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
void Save(const memgraph::replication::SystemHeartbeatReq & /*self*/, memgraph::slk::Builder * /*builder*/) {
|
||||
/* Nothing to serialize */
|
||||
}
|
||||
void Load(memgraph::replication::SystemHeartbeatReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
void Load(memgraph::replication::SystemHeartbeatReq * /*self*/, memgraph::slk::Reader * /*reader*/) {
|
||||
/* Nothing to serialize */
|
||||
}
|
||||
|
||||
// Serialize code for SystemRecoveryReq
|
||||
void Save(const memgraph::replication::SystemRecoveryReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.forced_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.database_configs, builder);
|
||||
memgraph::slk::Save(self.auth_config, builder);
|
||||
@@ -47,7 +46,6 @@ void Save(const memgraph::replication::SystemRecoveryReq &self, memgraph::slk::B
|
||||
}
|
||||
|
||||
void Load(memgraph::replication::SystemRecoveryReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->forced_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->database_configs, reader);
|
||||
memgraph::slk::Load(&self->auth_config, reader);
|
||||
|
||||
@@ -214,6 +214,7 @@ class Client {
|
||||
// Build and send the request.
|
||||
slk::Save(req_type.id, handler.GetBuilder());
|
||||
slk::Save(rpc::current_version, handler.GetBuilder());
|
||||
|
||||
TRequestResponse::Request::Save(request, handler.GetBuilder());
|
||||
|
||||
// Return the handler to the user.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -28,9 +28,6 @@ constexpr auto v1 = Version{2023'10'30'0'2'13};
|
||||
// for any TypeIds that get added.
|
||||
constexpr auto v2 = Version{2023'12'07'0'2'14};
|
||||
|
||||
// To each RPC main uuid was added
|
||||
constexpr auto v3 = Version{2024'02'02'0'2'14};
|
||||
|
||||
constexpr auto current_version = v3;
|
||||
constexpr auto current_version = v2;
|
||||
|
||||
} // namespace memgraph::rpc
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -57,11 +57,9 @@ class PreviousPtr {
|
||||
explicit Pointer(Edge *edge) : type(Type::EDGE), edge(edge) {}
|
||||
|
||||
Type type{Type::NULLPTR};
|
||||
union {
|
||||
Delta *delta = nullptr;
|
||||
Vertex *vertex;
|
||||
Edge *edge;
|
||||
};
|
||||
Delta *delta{nullptr};
|
||||
Vertex *vertex{nullptr};
|
||||
Edge *edge{nullptr};
|
||||
};
|
||||
|
||||
PreviousPtr() : storage_(0) {}
|
||||
@@ -159,51 +157,59 @@ struct Delta {
|
||||
// DELETE_DESERIALIZED_OBJECT is used to load data from disk committed by past txs.
|
||||
// Because of this object was created in past txs, we create timestamp by ourselves inside instead of having it from
|
||||
// current tx. This timestamp we got from RocksDB timestamp stored in key.
|
||||
Delta(DeleteDeserializedObjectTag /*tag*/, uint64_t ts, std::optional<std::string> old_disk_key)
|
||||
: timestamp(new std::atomic<uint64_t>(ts)), command_id(0), old_disk_key{.value = std::move(old_disk_key)} {}
|
||||
Delta(DeleteDeserializedObjectTag /*tag*/, uint64_t ts, const std::optional<std::string> &old_disk_key)
|
||||
: action(Action::DELETE_DESERIALIZED_OBJECT),
|
||||
timestamp(new std::atomic<uint64_t>(ts)),
|
||||
command_id(0),
|
||||
old_disk_key(old_disk_key) {}
|
||||
|
||||
Delta(DeleteObjectTag /*tag*/, std::atomic<uint64_t> *timestamp, uint64_t command_id)
|
||||
: timestamp(timestamp), command_id(command_id), action(Action::DELETE_OBJECT) {}
|
||||
: action(Action::DELETE_OBJECT), timestamp(timestamp), command_id(command_id) {}
|
||||
|
||||
Delta(RecreateObjectTag /*tag*/, std::atomic<uint64_t> *timestamp, uint64_t command_id)
|
||||
: timestamp(timestamp), command_id(command_id), action(Action::RECREATE_OBJECT) {}
|
||||
: action(Action::RECREATE_OBJECT), timestamp(timestamp), command_id(command_id) {}
|
||||
|
||||
Delta(AddLabelTag /*tag*/, LabelId label, std::atomic<uint64_t> *timestamp, uint64_t command_id)
|
||||
: timestamp(timestamp), command_id(command_id), label{.action = Action::ADD_LABEL, .value = label} {}
|
||||
: action(Action::ADD_LABEL), timestamp(timestamp), command_id(command_id), label(label) {}
|
||||
|
||||
Delta(RemoveLabelTag /*tag*/, LabelId label, std::atomic<uint64_t> *timestamp, uint64_t command_id)
|
||||
: timestamp(timestamp), command_id(command_id), label{.action = Action::REMOVE_LABEL, .value = label} {}
|
||||
: action(Action::REMOVE_LABEL), timestamp(timestamp), command_id(command_id), label(label) {}
|
||||
|
||||
Delta(SetPropertyTag /*tag*/, PropertyId key, PropertyValue value, std::atomic<uint64_t> *timestamp,
|
||||
Delta(SetPropertyTag /*tag*/, PropertyId key, const PropertyValue &value, std::atomic<uint64_t> *timestamp,
|
||||
uint64_t command_id)
|
||||
: timestamp(timestamp),
|
||||
command_id(command_id),
|
||||
property{
|
||||
.action = Action::SET_PROPERTY, .key = key, .value = std::make_unique<PropertyValue>(std::move(value))} {}
|
||||
: action(Action::SET_PROPERTY), timestamp(timestamp), command_id(command_id), property({key, value}) {}
|
||||
|
||||
Delta(SetPropertyTag /*tag*/, PropertyId key, PropertyValue &&value, std::atomic<uint64_t> *timestamp,
|
||||
uint64_t command_id)
|
||||
: action(Action::SET_PROPERTY), timestamp(timestamp), command_id(command_id), property({key, std::move(value)}) {}
|
||||
|
||||
Delta(AddInEdgeTag /*tag*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge, std::atomic<uint64_t> *timestamp,
|
||||
uint64_t command_id)
|
||||
: timestamp(timestamp),
|
||||
: action(Action::ADD_IN_EDGE),
|
||||
timestamp(timestamp),
|
||||
command_id(command_id),
|
||||
vertex_edge{.action = Action::ADD_IN_EDGE, .edge_type = edge_type, vertex, edge} {}
|
||||
vertex_edge({edge_type, vertex, edge}) {}
|
||||
|
||||
Delta(AddOutEdgeTag /*tag*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge, std::atomic<uint64_t> *timestamp,
|
||||
uint64_t command_id)
|
||||
: timestamp(timestamp),
|
||||
: action(Action::ADD_OUT_EDGE),
|
||||
timestamp(timestamp),
|
||||
command_id(command_id),
|
||||
vertex_edge{.action = Action::ADD_OUT_EDGE, .edge_type = edge_type, vertex, edge} {}
|
||||
vertex_edge({edge_type, vertex, edge}) {}
|
||||
|
||||
Delta(RemoveInEdgeTag /*tag*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge, std::atomic<uint64_t> *timestamp,
|
||||
uint64_t command_id)
|
||||
: timestamp(timestamp),
|
||||
: action(Action::REMOVE_IN_EDGE),
|
||||
timestamp(timestamp),
|
||||
command_id(command_id),
|
||||
vertex_edge{.action = Action::REMOVE_IN_EDGE, .edge_type = edge_type, vertex, edge} {}
|
||||
vertex_edge({edge_type, vertex, edge}) {}
|
||||
|
||||
Delta(RemoveOutEdgeTag /*tag*/, EdgeTypeId edge_type, Vertex *vertex, EdgeRef edge, std::atomic<uint64_t> *timestamp,
|
||||
uint64_t command_id)
|
||||
: timestamp(timestamp),
|
||||
: action(Action::REMOVE_OUT_EDGE),
|
||||
timestamp(timestamp),
|
||||
command_id(command_id),
|
||||
vertex_edge{.action = Action::REMOVE_OUT_EDGE, .edge_type = edge_type, vertex, edge} {}
|
||||
vertex_edge({edge_type, vertex, edge}) {}
|
||||
|
||||
Delta(const Delta &) = delete;
|
||||
Delta(Delta &&) = delete;
|
||||
@@ -222,16 +228,18 @@ struct Delta {
|
||||
case Action::REMOVE_OUT_EDGE:
|
||||
break;
|
||||
case Action::DELETE_DESERIALIZED_OBJECT:
|
||||
old_disk_key.value.reset();
|
||||
old_disk_key.reset();
|
||||
delete timestamp;
|
||||
timestamp = nullptr;
|
||||
break;
|
||||
case Action::SET_PROPERTY:
|
||||
property.value.reset();
|
||||
property.value.~PropertyValue();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Action action;
|
||||
|
||||
// TODO: optimize with in-place copy
|
||||
std::atomic<uint64_t> *timestamp;
|
||||
uint64_t command_id;
|
||||
@@ -239,22 +247,13 @@ struct Delta {
|
||||
std::atomic<Delta *> next{nullptr};
|
||||
|
||||
union {
|
||||
Action action;
|
||||
std::optional<std::string> old_disk_key;
|
||||
LabelId label;
|
||||
struct {
|
||||
Action action = Action::DELETE_DESERIALIZED_OBJECT;
|
||||
std::optional<std::string> value;
|
||||
} old_disk_key;
|
||||
struct {
|
||||
Action action;
|
||||
LabelId value;
|
||||
} label;
|
||||
struct {
|
||||
Action action;
|
||||
PropertyId key;
|
||||
std::unique_ptr<storage::PropertyValue> value;
|
||||
storage::PropertyValue value;
|
||||
} property;
|
||||
struct {
|
||||
Action action;
|
||||
EdgeTypeId edge_type;
|
||||
Vertex *vertex;
|
||||
EdgeRef edge;
|
||||
|
||||
@@ -137,14 +137,14 @@ bool VertexHasLabel(const Vertex &vertex, LabelId label, Transaction *transactio
|
||||
ApplyDeltasForRead(transaction, delta, view, [&deleted, &has_label, label](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL: {
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
}
|
||||
@@ -177,7 +177,7 @@ PropertyValue GetVertexProperty(const Vertex &vertex, PropertyId property, Trans
|
||||
switch (delta.action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
if (delta.property.key == property) {
|
||||
value = *delta.property.value;
|
||||
value = delta.property.value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1682,9 +1682,9 @@ utils::BasicResult<StorageManipulationError, void> DiskStorage::DiskAccessor::Co
|
||||
} break;
|
||||
}
|
||||
}
|
||||
} else if (transaction_.deltas.empty() ||
|
||||
} else if (transaction_.deltas.use().empty() ||
|
||||
(!edge_import_mode_active &&
|
||||
std::all_of(transaction_.deltas.begin(), transaction_.deltas.end(), [](const Delta &delta) {
|
||||
std::all_of(transaction_.deltas.use().begin(), transaction_.deltas.use().end(), [](const Delta &delta) {
|
||||
return delta.action == Delta::Action::DELETE_DESERIALIZED_OBJECT;
|
||||
}))) {
|
||||
} else {
|
||||
@@ -1812,7 +1812,7 @@ void DiskStorage::DiskAccessor::UpdateObjectsCountOnAbort() {
|
||||
auto *disk_storage = static_cast<DiskStorage *>(storage_);
|
||||
uint64_t transaction_id = transaction_.transaction_id;
|
||||
|
||||
for (const auto &delta : transaction_.deltas) {
|
||||
for (const auto &delta : transaction_.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -580,7 +580,7 @@ void EncodeDelta(BaseEncoder *encoder, NameIdMapper *name_id_mapper, SalientConf
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
encoder->WriteMarker(VertexActionToMarker(delta.action));
|
||||
encoder->WriteUint(vertex.gid.AsUint());
|
||||
encoder->WriteString(name_id_mapper->IdToName(delta.label.value.AsUint()));
|
||||
encoder->WriteString(name_id_mapper->IdToName(delta.label.AsUint()));
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
|
||||
@@ -237,7 +237,7 @@ Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view)
|
||||
switch (delta.action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
if (delta.property.key == property) {
|
||||
*value = *delta.property.value;
|
||||
*value = delta.property.value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -281,15 +281,15 @@ Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view)
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
auto it = properties.find(delta.property.key);
|
||||
if (it != properties.end()) {
|
||||
if (delta.property.value->IsNull()) {
|
||||
if (delta.property.value.IsNull()) {
|
||||
// remove the property
|
||||
properties.erase(it);
|
||||
} else {
|
||||
// set the value
|
||||
it->second = *delta.property.value;
|
||||
it->second = delta.property.value;
|
||||
}
|
||||
} else if (!delta.property.value->IsNull()) {
|
||||
properties.emplace(delta.property.key, *delta.property.value);
|
||||
} else if (!delta.property.value.IsNull()) {
|
||||
properties.emplace(delta.property.key, delta.property.value);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -23,24 +23,24 @@
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
#define STORAGE_DEFINE_ID_TYPE(name, type_store, type_conv, parse) \
|
||||
#define STORAGE_DEFINE_ID_TYPE(name) \
|
||||
class name final { \
|
||||
private: \
|
||||
explicit name(type_store id) : id_(id) {} \
|
||||
explicit name(uint64_t id) : id_(id) {} \
|
||||
\
|
||||
public: \
|
||||
/* Default constructor to allow serialization or preallocation. */ \
|
||||
name() = default; \
|
||||
\
|
||||
static name FromUint(type_store id) { return name{id}; } \
|
||||
static name FromInt(type_conv id) { return name{utils::MemcpyCast<type_store>(id)}; } \
|
||||
type_store AsUint() const { return id_; } \
|
||||
type_conv AsInt() const { return utils::MemcpyCast<type_conv>(id_); } \
|
||||
static name FromString(std::string_view id) { return name{parse(id)}; } \
|
||||
static name FromUint(uint64_t id) { return name{id}; } \
|
||||
static name FromInt(int64_t id) { return name{utils::MemcpyCast<uint64_t>(id)}; } \
|
||||
uint64_t AsUint() const { return id_; } \
|
||||
int64_t AsInt() const { return utils::MemcpyCast<int64_t>(id_); } \
|
||||
static name FromString(std::string_view id) { return name{utils::ParseStringToUint64(id)}; } \
|
||||
std::string ToString() const { return std::to_string(id_); } \
|
||||
\
|
||||
private: \
|
||||
type_store id_; \
|
||||
uint64_t id_; \
|
||||
}; \
|
||||
static_assert(std::is_trivially_copyable_v<name>, "storage::" #name " must be trivially copyable!"); \
|
||||
inline bool operator==(const name &first, const name &second) { return first.AsUint() == second.AsUint(); } \
|
||||
@@ -50,10 +50,10 @@ namespace memgraph::storage {
|
||||
inline bool operator<=(const name &first, const name &second) { return first.AsUint() <= second.AsUint(); } \
|
||||
inline bool operator>=(const name &first, const name &second) { return first.AsUint() >= second.AsUint(); }
|
||||
|
||||
STORAGE_DEFINE_ID_TYPE(Gid, uint64_t, int64_t, utils::ParseStringToUint64);
|
||||
STORAGE_DEFINE_ID_TYPE(LabelId, uint32_t, int32_t, utils::ParseStringToUint32);
|
||||
STORAGE_DEFINE_ID_TYPE(PropertyId, uint32_t, int32_t, utils::ParseStringToUint32);
|
||||
STORAGE_DEFINE_ID_TYPE(EdgeTypeId, uint32_t, int32_t, utils::ParseStringToUint32);
|
||||
STORAGE_DEFINE_ID_TYPE(Gid);
|
||||
STORAGE_DEFINE_ID_TYPE(LabelId);
|
||||
STORAGE_DEFINE_ID_TYPE(PropertyId);
|
||||
STORAGE_DEFINE_ID_TYPE(EdgeTypeId);
|
||||
|
||||
#undef STORAGE_DEFINE_ID_TYPE
|
||||
|
||||
|
||||
@@ -72,13 +72,13 @@ inline bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t tim
|
||||
return AnyVersionSatisfiesPredicate<interesting>(timestamp, delta, [&has_label, &deleted, label](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
}
|
||||
break;
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
}
|
||||
@@ -135,20 +135,20 @@ inline bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, Prop
|
||||
timestamp, delta, [&has_label, ¤t_value_equal_to_value, &deleted, label, key, &value](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
}
|
||||
break;
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
}
|
||||
break;
|
||||
case Delta::Action::SET_PROPERTY:
|
||||
if (delta.property.key == key) {
|
||||
current_value_equal_to_value = *delta.property.value == value;
|
||||
current_value_equal_to_value = delta.property.value == value;
|
||||
}
|
||||
break;
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/replication/recovery.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
@@ -27,8 +26,7 @@ namespace memgraph::storage {
|
||||
// contained in the internal buffer and the file.
|
||||
class InMemoryCurrentWalHandler {
|
||||
public:
|
||||
explicit InMemoryCurrentWalHandler(const utils::UUID &main_uuid, InMemoryStorage const *storage,
|
||||
rpc::Client &rpc_client);
|
||||
explicit InMemoryCurrentWalHandler(InMemoryStorage const *storage, rpc::Client &rpc_client);
|
||||
void AppendFilename(const std::string &filename);
|
||||
|
||||
void AppendSize(size_t size);
|
||||
@@ -45,9 +43,8 @@ class InMemoryCurrentWalHandler {
|
||||
};
|
||||
|
||||
////// CurrentWalHandler //////
|
||||
InMemoryCurrentWalHandler::InMemoryCurrentWalHandler(const utils::UUID &main_uuid, InMemoryStorage const *storage,
|
||||
rpc::Client &rpc_client)
|
||||
: stream_(rpc_client.Stream<replication::CurrentWalRpc>(main_uuid, storage->uuid())) {}
|
||||
InMemoryCurrentWalHandler::InMemoryCurrentWalHandler(InMemoryStorage const *storage, rpc::Client &rpc_client)
|
||||
: stream_(rpc_client.Stream<replication::CurrentWalRpc>(storage->uuid())) {}
|
||||
|
||||
void InMemoryCurrentWalHandler::AppendFilename(const std::string &filename) {
|
||||
replication::Encoder encoder(stream_.GetBuilder());
|
||||
@@ -72,10 +69,10 @@ void InMemoryCurrentWalHandler::AppendBufferData(const uint8_t *buffer, const si
|
||||
replication::CurrentWalRes InMemoryCurrentWalHandler::Finalize() { return stream_.AwaitResponse(); }
|
||||
|
||||
////// ReplicationClient Helpers //////
|
||||
replication::WalFilesRes TransferWalFiles(const utils::UUID &main_uuid, const utils::UUID &uuid, rpc::Client &client,
|
||||
replication::WalFilesRes TransferWalFiles(const utils::UUID &uuid, rpc::Client &client,
|
||||
const std::vector<std::filesystem::path> &wal_files) {
|
||||
MG_ASSERT(!wal_files.empty(), "Wal files list is empty!");
|
||||
auto stream = client.Stream<replication::WalFilesRpc>(main_uuid, uuid, wal_files.size());
|
||||
auto stream = client.Stream<replication::WalFilesRpc>(uuid, wal_files.size());
|
||||
replication::Encoder encoder(stream.GetBuilder());
|
||||
for (const auto &wal : wal_files) {
|
||||
spdlog::debug("Sending wal file: {}", wal);
|
||||
@@ -84,17 +81,16 @@ replication::WalFilesRes TransferWalFiles(const utils::UUID &main_uuid, const ut
|
||||
return stream.AwaitResponse();
|
||||
}
|
||||
|
||||
replication::SnapshotRes TransferSnapshot(const utils::UUID &main_uuid, const utils::UUID &uuid, rpc::Client &client,
|
||||
replication::SnapshotRes TransferSnapshot(const utils::UUID &uuid, rpc::Client &client,
|
||||
const std::filesystem::path &path) {
|
||||
auto stream = client.Stream<replication::SnapshotRpc>(main_uuid, uuid);
|
||||
auto stream = client.Stream<replication::SnapshotRpc>(uuid);
|
||||
replication::Encoder encoder(stream.GetBuilder());
|
||||
encoder.WriteFile(path);
|
||||
return stream.AwaitResponse();
|
||||
}
|
||||
|
||||
uint64_t ReplicateCurrentWal(const utils::UUID &main_uuid, const InMemoryStorage *storage, rpc::Client &client,
|
||||
durability::WalFile const &wal_file) {
|
||||
InMemoryCurrentWalHandler stream{main_uuid, storage, client};
|
||||
uint64_t ReplicateCurrentWal(const InMemoryStorage *storage, rpc::Client &client, durability::WalFile const &wal_file) {
|
||||
InMemoryCurrentWalHandler stream{storage, client};
|
||||
stream.AppendFilename(wal_file.Path().filename());
|
||||
utils::InputFile file;
|
||||
MG_ASSERT(file.Open(wal_file.Path()), "Failed to open current WAL file at {}!", wal_file.Path());
|
||||
|
||||
@@ -19,14 +19,13 @@ class InMemoryStorage;
|
||||
|
||||
////// ReplicationClient Helpers //////
|
||||
|
||||
replication::WalFilesRes TransferWalFiles(const utils::UUID &main_uuid, const utils::UUID &uuid, rpc::Client &client,
|
||||
replication::WalFilesRes TransferWalFiles(const utils::UUID &uuid, rpc::Client &client,
|
||||
const std::vector<std::filesystem::path> &wal_files);
|
||||
|
||||
replication::SnapshotRes TransferSnapshot(const utils::UUID &main_uuid, const utils::UUID &uuid, rpc::Client &client,
|
||||
replication::SnapshotRes TransferSnapshot(const utils::UUID &uuid, rpc::Client &client,
|
||||
const std::filesystem::path &path);
|
||||
|
||||
uint64_t ReplicateCurrentWal(const utils::UUID &main_uuid, const InMemoryStorage *storage, rpc::Client &client,
|
||||
durability::WalFile const &wal_file);
|
||||
uint64_t ReplicateCurrentWal(const InMemoryStorage *storage, rpc::Client &client, durability::WalFile const &wal_file);
|
||||
|
||||
auto GetRecoverySteps(uint64_t replica_commit, utils::FileRetainer::FileLocker *file_locker,
|
||||
const InMemoryStorage *storage) -> std::vector<RecoveryStep>;
|
||||
|
||||
@@ -176,9 +176,9 @@ InMemoryStorage::~InMemoryStorage() {
|
||||
committed_transactions_.WithLock([](auto &transactions) { transactions.clear(); });
|
||||
}
|
||||
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(
|
||||
auto tag, InMemoryStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode,
|
||||
memgraph::replication_coordination_glue::ReplicationRole replication_role)
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(auto tag, InMemoryStorage *storage, IsolationLevel isolation_level,
|
||||
StorageMode storage_mode,
|
||||
memgraph::replication_coordination_glue::ReplicationRole replication_role)
|
||||
: Accessor(tag, storage, isolation_level, storage_mode, replication_role),
|
||||
config_(storage->config_.salient.items) {}
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(InMemoryAccessor &&other) noexcept
|
||||
@@ -757,7 +757,7 @@ utils::BasicResult<StorageManipulationError, void> InMemoryStorage::InMemoryAcce
|
||||
auto *mem_storage = static_cast<InMemoryStorage *>(storage_);
|
||||
|
||||
// TODO: duplicated transaction finalisation in md_deltas and deltas processing cases
|
||||
if (transaction_.deltas.empty() && transaction_.md_deltas.empty()) {
|
||||
if (transaction_.deltas.use().empty() && transaction_.md_deltas.empty()) {
|
||||
// We don't have to update the commit timestamp here because no one reads
|
||||
// it.
|
||||
mem_storage->commit_log_->MarkFinished(transaction_.start_timestamp);
|
||||
@@ -836,37 +836,25 @@ utils::BasicResult<StorageManipulationError, void> InMemoryStorage::InMemoryAcce
|
||||
// Replica can log only the write transaction received from Main
|
||||
// so the Wal files are consistent
|
||||
if (is_main_or_replica_write) {
|
||||
could_replicate_all_sync_replicas =
|
||||
mem_storage->AppendToWal(transaction_, *commit_timestamp_, std::move(db_acc));
|
||||
could_replicate_all_sync_replicas = mem_storage->AppendToWal(transaction_, *commit_timestamp_,
|
||||
std::move(db_acc)); // protected by engine_guard
|
||||
|
||||
// TODO: release lock, and update all deltas to have a local copy of the commit timestamp
|
||||
MG_ASSERT(transaction_.commit_timestamp != nullptr, "Invalid database state!");
|
||||
transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release);
|
||||
transaction_.commit_timestamp->store(*commit_timestamp_,
|
||||
std::memory_order_release); // protected by engine_guard
|
||||
// Replica can only update the last commit timestamp with
|
||||
// the commits received from main.
|
||||
// Update the last commit timestamp
|
||||
mem_storage->repl_storage_state_.last_commit_timestamp_.store(*commit_timestamp_);
|
||||
mem_storage->repl_storage_state_.last_commit_timestamp_.store(
|
||||
*commit_timestamp_); // protected by engine_guard
|
||||
}
|
||||
// Release engine lock because we don't have to hold it anymore
|
||||
engine_guard.unlock();
|
||||
|
||||
// TODO: can and should this be moved earlier?
|
||||
mem_storage->commit_log_->MarkFinished(start_timestamp);
|
||||
|
||||
// while still holding engine lock
|
||||
// and after durability + replication
|
||||
// check if we can fast discard deltas (ie. do not hand over to GC)
|
||||
bool no_older_transactions = mem_storage->commit_log_->OldestActive() == *commit_timestamp_;
|
||||
bool no_newer_transactions = mem_storage->transaction_id_ == transaction_.transaction_id + 1;
|
||||
if (no_older_transactions && no_newer_transactions) [[unlikely]] {
|
||||
// STEP 0) Can only do fast discard if GC is not running
|
||||
// We can't unlink our transcations deltas until all of the older deltas in GC have been unlinked
|
||||
// must do a try here, to avoid deadlock between transactions `engine_lock_` and the GC `gc_lock_`
|
||||
auto gc_guard = std::unique_lock{mem_storage->gc_lock_, std::defer_lock};
|
||||
if (gc_guard.try_lock()) {
|
||||
FastDiscardOfDeltas(*commit_timestamp_, std::move(gc_guard));
|
||||
}
|
||||
}
|
||||
}
|
||||
} // Release engine lock because we don't have to hold it anymore
|
||||
}
|
||||
|
||||
if (unique_constraint_violation) {
|
||||
Abort();
|
||||
@@ -885,332 +873,241 @@ utils::BasicResult<StorageManipulationError, void> InMemoryStorage::InMemoryAcce
|
||||
return {};
|
||||
}
|
||||
|
||||
void InMemoryStorage::InMemoryAccessor::FastDiscardOfDeltas(uint64_t oldest_active_timestamp,
|
||||
std::unique_lock<std::mutex> /*gc_guard*/) {
|
||||
auto *mem_storage = static_cast<InMemoryStorage *>(storage_);
|
||||
std::list<Gid> current_deleted_edges;
|
||||
std::list<Gid> current_deleted_vertices;
|
||||
|
||||
auto const unlink_remove_clear = [&](std::deque<Delta> &deltas) {
|
||||
for (auto &delta : deltas) {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::NULLPTR:
|
||||
case PreviousPtr::Type::DELTA:
|
||||
break;
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
// safe because no other txn can be reading this while we have engine lock
|
||||
auto &vertex = *prev.vertex;
|
||||
vertex.delta = nullptr;
|
||||
if (vertex.deleted) {
|
||||
DMG_ASSERT(delta.action == Delta::Action::RECREATE_OBJECT);
|
||||
current_deleted_vertices.push_back(vertex.gid);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::EDGE: {
|
||||
// safe because no other txn can be reading this while we have engine lock
|
||||
auto &edge = *prev.edge;
|
||||
edge.delta = nullptr;
|
||||
if (edge.deleted) {
|
||||
DMG_ASSERT(delta.action == Delta::Action::RECREATE_OBJECT);
|
||||
current_deleted_edges.push_back(edge.gid);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// delete deltas
|
||||
deltas.clear();
|
||||
};
|
||||
|
||||
// STEP 1) ensure everything in GC is gone
|
||||
|
||||
// 1.a) old garbage_undo_buffers are safe to remove
|
||||
// we are the only transaction, no one is reading those unlinked deltas
|
||||
mem_storage->garbage_undo_buffers_.WithLock([&](auto &garbage_undo_buffers) { garbage_undo_buffers.clear(); });
|
||||
|
||||
// 1.b.0) old committed_transactions_ need mininal unlinking + remove + clear
|
||||
// must be done before this transactions delta unlinking
|
||||
auto linked_undo_buffers = std::list<GCDeltas>{};
|
||||
mem_storage->committed_transactions_.WithLock(
|
||||
[&](auto &committed_transactions) { committed_transactions.swap(linked_undo_buffers); });
|
||||
|
||||
// 1.b.1) unlink, gathering the removals
|
||||
for (auto &gc_deltas : linked_undo_buffers) {
|
||||
unlink_remove_clear(gc_deltas.deltas_);
|
||||
}
|
||||
// 1.b.2) clear the list of deltas deques
|
||||
linked_undo_buffers.clear();
|
||||
|
||||
// STEP 2) this transactions deltas also mininal unlinking + remove + clear
|
||||
unlink_remove_clear(transaction_.deltas);
|
||||
|
||||
// STEP 3) skip_list removals
|
||||
if (!current_deleted_vertices.empty()) {
|
||||
// 3.a) clear from indexes first
|
||||
std::stop_source dummy;
|
||||
mem_storage->indices_.RemoveObsoleteEntries(oldest_active_timestamp, dummy.get_token());
|
||||
auto *mem_unique_constraints =
|
||||
static_cast<InMemoryUniqueConstraints *>(mem_storage->constraints_.unique_constraints_.get());
|
||||
mem_unique_constraints->RemoveObsoleteEntries(oldest_active_timestamp, dummy.get_token());
|
||||
|
||||
// 3.b) remove from veretex skip_list
|
||||
auto vertex_acc = mem_storage->vertices_.access();
|
||||
for (auto gid : current_deleted_vertices) {
|
||||
vertex_acc.remove(gid);
|
||||
}
|
||||
}
|
||||
|
||||
if (!current_deleted_edges.empty()) {
|
||||
// 3.c) remove from edge skip_list
|
||||
auto edge_acc = mem_storage->edges_.access();
|
||||
for (auto gid : current_deleted_edges) {
|
||||
edge_acc.remove(gid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryStorage::InMemoryAccessor::Abort() {
|
||||
MG_ASSERT(is_transaction_active_, "The transaction is already terminated!");
|
||||
|
||||
auto *mem_storage = static_cast<InMemoryStorage *>(storage_);
|
||||
// We collect vertices and edges we've created here and then splice them into
|
||||
// `deleted_vertices_` and `deleted_edges_` lists, instead of adding them one
|
||||
// by one and acquiring lock every time.
|
||||
std::list<Gid> my_deleted_vertices;
|
||||
std::list<Gid> my_deleted_edges;
|
||||
|
||||
// if we have no deltas then no need to do any undo work during Abort
|
||||
// note: this check also saves on unnecessary contention on `engine_lock_`
|
||||
if (!transaction_.deltas.empty()) {
|
||||
// CONSTRAINTS
|
||||
if (transaction_.constraint_verification_info.NeedsUniqueConstraintVerification()) {
|
||||
// Need to remove elements from constraints before handling of the deltas, so the elements match the correct
|
||||
// values
|
||||
auto vertices_to_check = transaction_.constraint_verification_info.GetVerticesForUniqueConstraintChecking();
|
||||
auto vertices_to_check_v = std::vector<Vertex const *>{vertices_to_check.begin(), vertices_to_check.end()};
|
||||
storage_->constraints_.AbortEntries(vertices_to_check_v, transaction_.start_timestamp);
|
||||
}
|
||||
std::map<LabelId, std::vector<Vertex *>> label_cleanup;
|
||||
std::map<LabelId, std::vector<std::pair<PropertyValue, Vertex *>>> label_property_cleanup;
|
||||
std::map<PropertyId, std::vector<std::pair<PropertyValue, Vertex *>>> property_cleanup;
|
||||
|
||||
const auto index_stats = storage_->indices_.Analysis();
|
||||
// CONSTRAINTS
|
||||
if (transaction_.constraint_verification_info.NeedsUniqueConstraintVerification()) {
|
||||
// Need to remove elements from constraints before handling of the deltas, so the elements match the correct
|
||||
// values
|
||||
auto vertices_to_check = transaction_.constraint_verification_info.GetVerticesForUniqueConstraintChecking();
|
||||
auto vertices_to_check_v = std::vector<Vertex const *>{vertices_to_check.begin(), vertices_to_check.end()};
|
||||
storage_->constraints_.AbortEntries(vertices_to_check_v, transaction_.start_timestamp);
|
||||
}
|
||||
|
||||
// We collect vertices and edges we've created here and then splice them into
|
||||
// `deleted_vertices_` and `deleted_edges_` lists, instead of adding them one
|
||||
// by one and acquiring lock every time.
|
||||
std::list<Gid> my_deleted_vertices;
|
||||
std::list<Gid> my_deleted_edges;
|
||||
const auto index_stats = storage_->indices_.Analysis();
|
||||
|
||||
std::map<LabelId, std::vector<Vertex *>> label_cleanup;
|
||||
std::map<LabelId, std::vector<std::pair<PropertyValue, Vertex *>>> label_property_cleanup;
|
||||
std::map<PropertyId, std::vector<std::pair<PropertyValue, Vertex *>>> property_cleanup;
|
||||
for (const auto &delta : transaction_.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
auto *vertex = prev.vertex;
|
||||
auto guard = std::unique_lock{vertex->lock};
|
||||
Delta *current = vertex->delta;
|
||||
while (current != nullptr &&
|
||||
current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) {
|
||||
switch (current->action) {
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
|
||||
MG_ASSERT(it != vertex->labels.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->labels.rbegin());
|
||||
vertex->labels.pop_back();
|
||||
|
||||
for (const auto &delta : transaction_.deltas) {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
auto *vertex = prev.vertex;
|
||||
auto guard = std::unique_lock{vertex->lock};
|
||||
Delta *current = vertex->delta;
|
||||
while (current != nullptr &&
|
||||
current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) {
|
||||
switch (current->action) {
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label.value);
|
||||
MG_ASSERT(it != vertex->labels.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->labels.rbegin());
|
||||
vertex->labels.pop_back();
|
||||
|
||||
// For label index
|
||||
// check if there is a label index for the label and add entry if so
|
||||
// For property label index
|
||||
// check if we care about the label; this will return all the propertyIds we care about and then get
|
||||
// the current property value
|
||||
if (std::binary_search(index_stats.label.begin(), index_stats.label.end(), current->label.value)) {
|
||||
label_cleanup[current->label.value].emplace_back(vertex);
|
||||
}
|
||||
const auto &properties = index_stats.property_label.l2p.find(current->label.value);
|
||||
if (properties != index_stats.property_label.l2p.end()) {
|
||||
for (const auto &property : properties->second) {
|
||||
auto current_value = vertex->properties.GetProperty(property);
|
||||
if (!current_value.IsNull()) {
|
||||
label_property_cleanup[current->label.value].emplace_back(std::move(current_value), vertex);
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
// For label index
|
||||
// check if there is a label index for the label and add entry if so
|
||||
// For property label index
|
||||
// check if we care about the label; this will return all the propertyIds we care about and then get
|
||||
// the current property value
|
||||
if (std::binary_search(index_stats.label.begin(), index_stats.label.end(), current->label)) {
|
||||
label_cleanup[current->label].emplace_back(vertex);
|
||||
}
|
||||
case Delta::Action::ADD_LABEL: {
|
||||
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label.value);
|
||||
MG_ASSERT(it == vertex->labels.end(), "Invalid database state!");
|
||||
vertex->labels.push_back(current->label.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
// For label index nothing
|
||||
// For property label index
|
||||
// check if we care about the property, this will return all the labels and then get current property
|
||||
// value
|
||||
const auto &labels = index_stats.property_label.p2l.find(current->property.key);
|
||||
if (labels != index_stats.property_label.p2l.end()) {
|
||||
auto current_value = vertex->properties.GetProperty(current->property.key);
|
||||
const auto &properties = index_stats.property_label.l2p.find(current->label);
|
||||
if (properties != index_stats.property_label.l2p.end()) {
|
||||
for (const auto &property : properties->second) {
|
||||
auto current_value = vertex->properties.GetProperty(property);
|
||||
if (!current_value.IsNull()) {
|
||||
property_cleanup[current->property.key].emplace_back(std::move(current_value), vertex);
|
||||
label_property_cleanup[current->label].emplace_back(std::move(current_value), vertex);
|
||||
}
|
||||
}
|
||||
// Setting the correct value
|
||||
vertex->properties.SetProperty(current->property.key, *current->property.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_IN_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
|
||||
MG_ASSERT(it == vertex->in_edges.end(), "Invalid database state!");
|
||||
vertex->in_edges.push_back(link);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_OUT_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
|
||||
MG_ASSERT(it == vertex->out_edges.end(), "Invalid database state!");
|
||||
vertex->out_edges.push_back(link);
|
||||
// Increment edge count. We only increment the count here because
|
||||
// the information in `ADD_IN_EDGE` and `Edge/RECREATE_OBJECT` is
|
||||
// redundant. Also, `Edge/RECREATE_OBJECT` isn't available when
|
||||
// edge properties are disabled.
|
||||
storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_IN_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
|
||||
MG_ASSERT(it != vertex->in_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->in_edges.rbegin());
|
||||
vertex->in_edges.pop_back();
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
|
||||
MG_ASSERT(it != vertex->out_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->out_edges.rbegin());
|
||||
vertex->out_edges.pop_back();
|
||||
// Decrement edge count. We only decrement the count here because
|
||||
// the information in `REMOVE_IN_EDGE` and `Edge/DELETE_OBJECT` is
|
||||
// redundant. Also, `Edge/DELETE_OBJECT` isn't available when edge
|
||||
// properties are disabled.
|
||||
storage_->edge_count_.fetch_add(-1, std::memory_order_acq_rel);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_DESERIALIZED_OBJECT:
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
vertex->deleted = true;
|
||||
my_deleted_vertices.push_back(vertex->gid);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
vertex->deleted = false;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
current = current->next.load(std::memory_order_acquire);
|
||||
}
|
||||
vertex->delta = current;
|
||||
if (current != nullptr) {
|
||||
current->prev.Set(vertex);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::EDGE: {
|
||||
auto *edge = prev.edge;
|
||||
auto guard = std::lock_guard{edge->lock};
|
||||
Delta *current = edge->delta;
|
||||
while (current != nullptr &&
|
||||
current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) {
|
||||
switch (current->action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
edge->properties.SetProperty(current->property.key, *current->property.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_DESERIALIZED_OBJECT:
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
edge->deleted = true;
|
||||
my_deleted_edges.push_back(edge->gid);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
edge->deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
LOG_FATAL("Invalid database state!");
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL: {
|
||||
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
|
||||
MG_ASSERT(it == vertex->labels.end(), "Invalid database state!");
|
||||
vertex->labels.push_back(current->label);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
// For label index nothing
|
||||
// For property label index
|
||||
// check if we care about the property, this will return all the labels and then get current property
|
||||
// value
|
||||
const auto &labels = index_stats.property_label.p2l.find(current->property.key);
|
||||
if (labels != index_stats.property_label.p2l.end()) {
|
||||
auto current_value = vertex->properties.GetProperty(current->property.key);
|
||||
if (!current_value.IsNull()) {
|
||||
property_cleanup[current->property.key].emplace_back(std::move(current_value), vertex);
|
||||
}
|
||||
}
|
||||
// Setting the correct value
|
||||
vertex->properties.SetProperty(current->property.key, current->property.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_IN_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
|
||||
MG_ASSERT(it == vertex->in_edges.end(), "Invalid database state!");
|
||||
vertex->in_edges.push_back(link);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_OUT_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
|
||||
MG_ASSERT(it == vertex->out_edges.end(), "Invalid database state!");
|
||||
vertex->out_edges.push_back(link);
|
||||
// Increment edge count. We only increment the count here because
|
||||
// the information in `ADD_IN_EDGE` and `Edge/RECREATE_OBJECT` is
|
||||
// redundant. Also, `Edge/RECREATE_OBJECT` isn't available when
|
||||
// edge properties are disabled.
|
||||
storage_->edge_count_.fetch_add(1, std::memory_order_acq_rel);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_IN_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
|
||||
MG_ASSERT(it != vertex->in_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->in_edges.rbegin());
|
||||
vertex->in_edges.pop_back();
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
std::tuple<EdgeTypeId, Vertex *, EdgeRef> link{current->vertex_edge.edge_type,
|
||||
current->vertex_edge.vertex, current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
|
||||
MG_ASSERT(it != vertex->out_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->out_edges.rbegin());
|
||||
vertex->out_edges.pop_back();
|
||||
// Decrement edge count. We only decrement the count here because
|
||||
// the information in `REMOVE_IN_EDGE` and `Edge/DELETE_OBJECT` is
|
||||
// redundant. Also, `Edge/DELETE_OBJECT` isn't available when edge
|
||||
// properties are disabled.
|
||||
storage_->edge_count_.fetch_add(-1, std::memory_order_acq_rel);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_DESERIALIZED_OBJECT:
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
vertex->deleted = true;
|
||||
my_deleted_vertices.push_back(vertex->gid);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
vertex->deleted = false;
|
||||
break;
|
||||
}
|
||||
current = current->next.load(std::memory_order_acquire);
|
||||
}
|
||||
edge->delta = current;
|
||||
if (current != nullptr) {
|
||||
current->prev.Set(edge);
|
||||
}
|
||||
|
||||
break;
|
||||
current = current->next.load(std::memory_order_acquire);
|
||||
}
|
||||
case PreviousPtr::Type::DELTA:
|
||||
// pointer probably couldn't be set because allocation failed
|
||||
case PreviousPtr::Type::NULLPTR:
|
||||
break;
|
||||
vertex->delta = current;
|
||||
if (current != nullptr) {
|
||||
current->prev.Set(vertex);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::EDGE: {
|
||||
auto *edge = prev.edge;
|
||||
auto guard = std::lock_guard{edge->lock};
|
||||
Delta *current = edge->delta;
|
||||
while (current != nullptr &&
|
||||
current->timestamp->load(std::memory_order_acquire) == transaction_.transaction_id) {
|
||||
switch (current->action) {
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
edge->properties.SetProperty(current->property.key, current->property.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_DESERIALIZED_OBJECT:
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
edge->deleted = true;
|
||||
my_deleted_edges.push_back(edge->gid);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
edge->deleted = false;
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
case Delta::Action::ADD_LABEL:
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
case Delta::Action::REMOVE_IN_EDGE:
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
LOG_FATAL("Invalid database state!");
|
||||
break;
|
||||
}
|
||||
}
|
||||
current = current->next.load(std::memory_order_acquire);
|
||||
}
|
||||
edge->delta = current;
|
||||
if (current != nullptr) {
|
||||
current->prev.Set(edge);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case PreviousPtr::Type::DELTA:
|
||||
// pointer probably couldn't be set because allocation failed
|
||||
case PreviousPtr::Type::NULLPTR:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
auto *mem_storage = static_cast<InMemoryStorage *>(storage_);
|
||||
{
|
||||
auto engine_guard = std::unique_lock(storage_->engine_lock_);
|
||||
uint64_t mark_timestamp = storage_->timestamp_;
|
||||
// Take garbage_undo_buffers lock while holding the engine lock to make
|
||||
// sure that entries are sorted by mark timestamp in the list.
|
||||
mem_storage->garbage_undo_buffers_.WithLock([&](auto &garbage_undo_buffers) {
|
||||
// Release engine lock because we don't have to hold it anymore and
|
||||
// emplace back could take a long time.
|
||||
engine_guard.unlock();
|
||||
|
||||
garbage_undo_buffers.emplace_back(mark_timestamp, std::move(transaction_.deltas),
|
||||
std::move(transaction_.commit_timestamp));
|
||||
});
|
||||
|
||||
/// We MUST unlink (aka. remove) entries in indexes and constraints
|
||||
/// before we unlink (aka. remove) vertices from storage
|
||||
/// this is because they point into vertices skip_list
|
||||
|
||||
// INDICES
|
||||
for (auto const &[label, vertices] : label_cleanup) {
|
||||
storage_->indices_.AbortEntries(label, vertices, transaction_.start_timestamp);
|
||||
}
|
||||
for (auto const &[label, prop_vertices] : label_property_cleanup) {
|
||||
storage_->indices_.AbortEntries(label, prop_vertices, transaction_.start_timestamp);
|
||||
}
|
||||
for (auto const &[property, prop_vertices] : property_cleanup) {
|
||||
storage_->indices_.AbortEntries(property, prop_vertices, transaction_.start_timestamp);
|
||||
}
|
||||
|
||||
// VERTICES
|
||||
{
|
||||
auto vertices_acc = mem_storage->vertices_.access();
|
||||
for (auto gid : my_deleted_vertices) {
|
||||
vertices_acc.remove(gid);
|
||||
}
|
||||
}
|
||||
|
||||
// EDGES
|
||||
{
|
||||
auto engine_guard = std::unique_lock(storage_->engine_lock_);
|
||||
uint64_t mark_timestamp = storage_->timestamp_;
|
||||
// Take garbage_undo_buffers lock while holding the engine lock to make
|
||||
// sure that entries are sorted by mark timestamp in the list.
|
||||
mem_storage->garbage_undo_buffers_.WithLock([&](auto &garbage_undo_buffers) {
|
||||
// Release engine lock because we don't have to hold it anymore and
|
||||
// emplace back could take a long time.
|
||||
engine_guard.unlock();
|
||||
|
||||
garbage_undo_buffers.emplace_back(mark_timestamp, std::move(transaction_.deltas),
|
||||
std::move(transaction_.commit_timestamp));
|
||||
});
|
||||
|
||||
/// We MUST unlink (aka. remove) entries in indexes and constraints
|
||||
/// before we unlink (aka. remove) vertices from storage
|
||||
/// this is because they point into vertices skip_list
|
||||
|
||||
// INDICES
|
||||
for (auto const &[label, vertices] : label_cleanup) {
|
||||
storage_->indices_.AbortEntries(label, vertices, transaction_.start_timestamp);
|
||||
}
|
||||
for (auto const &[label, prop_vertices] : label_property_cleanup) {
|
||||
storage_->indices_.AbortEntries(label, prop_vertices, transaction_.start_timestamp);
|
||||
}
|
||||
for (auto const &[property, prop_vertices] : property_cleanup) {
|
||||
storage_->indices_.AbortEntries(property, prop_vertices, transaction_.start_timestamp);
|
||||
}
|
||||
|
||||
// VERTICES
|
||||
{
|
||||
auto vertices_acc = mem_storage->vertices_.access();
|
||||
for (auto gid : my_deleted_vertices) {
|
||||
vertices_acc.remove(gid);
|
||||
}
|
||||
}
|
||||
|
||||
// EDGES
|
||||
{
|
||||
auto edges_acc = mem_storage->edges_.access();
|
||||
for (auto gid : my_deleted_edges) {
|
||||
edges_acc.remove(gid);
|
||||
}
|
||||
auto edges_acc = mem_storage->edges_.access();
|
||||
for (auto gid : my_deleted_edges) {
|
||||
edges_acc.remove(gid);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1224,7 +1121,7 @@ void InMemoryStorage::InMemoryAccessor::FinalizeTransaction() {
|
||||
auto *mem_storage = static_cast<InMemoryStorage *>(storage_);
|
||||
mem_storage->commit_log_->MarkFinished(*commit_timestamp_);
|
||||
|
||||
if (!transaction_.deltas.empty()) {
|
||||
if (!transaction_.deltas.use().empty()) {
|
||||
// Only hand over delta to be GC'ed if there was any deltas
|
||||
mem_storage->committed_transactions_.WithLock([&](auto &committed_transactions) {
|
||||
// using mark of 0 as GC will assign a mark_timestamp after unlinking
|
||||
@@ -1565,7 +1462,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock<utils::ResourceLock> main_
|
||||
// chain in a broken state.
|
||||
// The chain can be only read without taking any locks.
|
||||
|
||||
for (Delta &delta : linked_entry->deltas_) {
|
||||
for (Delta &delta : linked_entry->deltas_.use()) {
|
||||
while (true) {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
@@ -1847,7 +1744,6 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
// A single transaction will always be contained in a single WAL file.
|
||||
auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire);
|
||||
|
||||
//////// AF only this calls initialize transaction
|
||||
repl_storage_state_.InitializeTransaction(wal_file_->SequenceNumber(), this, db_acc);
|
||||
|
||||
auto append_deltas = [&](auto callback) {
|
||||
@@ -1885,7 +1781,7 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
|
||||
// 1. Process all Vertex deltas and store all operations that create vertices
|
||||
// and modify vertex data.
|
||||
for (const auto &delta : transaction.deltas) {
|
||||
for (const auto &delta : transaction.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!");
|
||||
if (prev.type != PreviousPtr::Type::VERTEX) continue;
|
||||
@@ -1908,7 +1804,7 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
});
|
||||
}
|
||||
// 2. Process all Vertex deltas and store all operations that create edges.
|
||||
for (const auto &delta : transaction.deltas) {
|
||||
for (const auto &delta : transaction.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!");
|
||||
if (prev.type != PreviousPtr::Type::VERTEX) continue;
|
||||
@@ -1930,7 +1826,7 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
});
|
||||
}
|
||||
// 3. Process all Edge deltas and store all operations that modify edge data.
|
||||
for (const auto &delta : transaction.deltas) {
|
||||
for (const auto &delta : transaction.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!");
|
||||
if (prev.type != PreviousPtr::Type::EDGE) continue;
|
||||
@@ -1952,7 +1848,7 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
});
|
||||
}
|
||||
// 4. Process all Vertex deltas and store all operations that delete edges.
|
||||
for (const auto &delta : transaction.deltas) {
|
||||
for (const auto &delta : transaction.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!");
|
||||
if (prev.type != PreviousPtr::Type::VERTEX) continue;
|
||||
@@ -1974,7 +1870,7 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
});
|
||||
}
|
||||
// 5. Process all Vertex deltas and store all operations that delete vertices.
|
||||
for (const auto &delta : transaction.deltas) {
|
||||
for (const auto &delta : transaction.deltas.use()) {
|
||||
auto prev = delta.prev.Get();
|
||||
MG_ASSERT(prev.type != PreviousPtr::Type::NULLPTR, "Invalid pointer!");
|
||||
if (prev.type != PreviousPtr::Type::VERTEX) continue;
|
||||
@@ -1998,7 +1894,7 @@ bool InMemoryStorage::AppendToWal(const Transaction &transaction, uint64_t final
|
||||
};
|
||||
|
||||
// Handle MVCC deltas
|
||||
if (!transaction.deltas.empty()) {
|
||||
if (!transaction.deltas.use().empty()) {
|
||||
append_deltas([&](const Delta &delta, const auto &parent, uint64_t timestamp) {
|
||||
wal_file_->AppendDelta(delta, parent, timestamp);
|
||||
repl_storage_state_.AppendDelta(delta, parent, timestamp);
|
||||
|
||||
@@ -302,9 +302,6 @@ class InMemoryStorage final : public Storage {
|
||||
/// @throw std::bad_alloc
|
||||
Result<EdgeAccessor> CreateEdgeEx(VertexAccessor *from, VertexAccessor *to, EdgeTypeId edge_type, storage::Gid gid);
|
||||
|
||||
/// Duiring commit, in some cases you do not need to hand over deltas to GC
|
||||
/// in those cases this method is a light weight way to unlink and discard our deltas
|
||||
void FastDiscardOfDeltas(uint64_t oldest_active_timestamp, std::unique_lock<std::mutex> gc_guard);
|
||||
SalientConfig::Items config_;
|
||||
};
|
||||
|
||||
@@ -432,15 +429,16 @@ class InMemoryStorage final : public Storage {
|
||||
utils::Scheduler gc_runner_;
|
||||
std::mutex gc_lock_;
|
||||
|
||||
using BondPmrLd = Bond<utils::pmr::list<Delta>>;
|
||||
struct GCDeltas {
|
||||
GCDeltas(uint64_t mark_timestamp, std::deque<Delta> deltas, std::unique_ptr<std::atomic<uint64_t>> commit_timestamp)
|
||||
GCDeltas(uint64_t mark_timestamp, BondPmrLd deltas, std::unique_ptr<std::atomic<uint64_t>> commit_timestamp)
|
||||
: mark_timestamp_{mark_timestamp}, deltas_{std::move(deltas)}, commit_timestamp_{std::move(commit_timestamp)} {}
|
||||
|
||||
GCDeltas(GCDeltas &&) = default;
|
||||
GCDeltas &operator=(GCDeltas &&) = default;
|
||||
|
||||
uint64_t mark_timestamp_{}; //!< a timestamp no active transaction currently has
|
||||
std::deque<Delta> deltas_; //!< the deltas that need cleaning
|
||||
BondPmrLd deltas_; //!< the deltas that need cleaning
|
||||
std::unique_ptr<std::atomic<uint64_t>> commit_timestamp_{}; //!< the timestamp the deltas are pointing at
|
||||
};
|
||||
|
||||
|
||||
@@ -80,7 +80,7 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
auto pos = FindPropertyPosition(property_array, delta->property.key);
|
||||
if (pos) {
|
||||
current_value_equal_to_value[*pos] = *delta->property.value == value_array[*pos];
|
||||
current_value_equal_to_value[*pos] = delta->property.value == value_array[*pos];
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -96,14 +96,14 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL: {
|
||||
if (delta->label.value == label) {
|
||||
if (delta->label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
if (delta->label.value == label) {
|
||||
if (delta->label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
break;
|
||||
@@ -190,13 +190,13 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, const std::
|
||||
}
|
||||
switch (delta->action) {
|
||||
case Delta::Action::ADD_LABEL:
|
||||
if (delta->label.value == label) {
|
||||
if (delta->label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
}
|
||||
break;
|
||||
case Delta::Action::REMOVE_LABEL:
|
||||
if (delta->label.value == label) {
|
||||
if (delta->label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
}
|
||||
@@ -204,7 +204,7 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, const std::
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
auto pos = FindPropertyPosition(property_array, delta->property.key);
|
||||
if (pos) {
|
||||
current_value_equal_to_value[*pos] = *delta->property.value == values[*pos];
|
||||
current_value_equal_to_value[*pos] = delta->property.value == values[*pos];
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -114,8 +114,8 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
|
||||
return nullptr;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
return &transaction->deltas.use().emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
}
|
||||
|
||||
inline Delta *CreateDeleteObjectDelta(Transaction *transaction, std::list<Delta> *deltas) {
|
||||
@@ -133,19 +133,19 @@ inline Delta *CreateDeleteDeserializedObjectDelta(Transaction *transaction, std:
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
// Should use utils::DecodeFixed64(ts.c_str()) once we will move to RocksDB real timestamps
|
||||
uint64_t ts_id = utils::ParseStringToUint64(ts);
|
||||
return &transaction->deltas.emplace_back(Delta::DeleteDeserializedObjectTag(), ts_id, std::move(old_disk_key));
|
||||
return &transaction->deltas.use().emplace_back(Delta::DeleteDeserializedObjectTag(), ts_id, old_disk_key);
|
||||
}
|
||||
|
||||
inline Delta *CreateDeleteDeserializedObjectDelta(std::list<Delta> *deltas, std::optional<std::string> old_disk_key,
|
||||
std::string &&ts) {
|
||||
// Should use utils::DecodeFixed64(ts.c_str()) once we will move to RocksDB real timestamps
|
||||
uint64_t ts_id = utils::ParseStringToUint64(ts);
|
||||
return &deltas->emplace_back(Delta::DeleteDeserializedObjectTag(), ts_id, std::move(old_disk_key));
|
||||
return &deltas->emplace_back(Delta::DeleteDeserializedObjectTag(), ts_id, old_disk_key);
|
||||
}
|
||||
|
||||
inline Delta *CreateDeleteDeserializedIndexObjectDelta(std::list<Delta> &deltas,
|
||||
std::optional<std::string> old_disk_key, const uint64_t ts) {
|
||||
return &deltas.emplace_back(Delta::DeleteDeserializedObjectTag(), ts, std::move(old_disk_key));
|
||||
return &deltas.emplace_back(Delta::DeleteDeserializedObjectTag(), ts, old_disk_key);
|
||||
}
|
||||
|
||||
/// TODO: what if in-memory analytical
|
||||
@@ -165,8 +165,8 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
|
||||
return;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
auto delta = &transaction->deltas.use().emplace_back(std::forward<Args>(args)..., transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
|
||||
// The operations are written in such order so that both `next` and `prev`
|
||||
// chains are valid at all times. The chains must be valid at all times
|
||||
|
||||
@@ -57,24 +57,38 @@ class PropertyValue {
|
||||
PropertyValue() : type_(Type::Null) {}
|
||||
|
||||
// constructors for primitive types
|
||||
explicit PropertyValue(const bool value) : bool_v{.val_ = value} {}
|
||||
explicit PropertyValue(const int value) : int_v{.val_ = value} {}
|
||||
explicit PropertyValue(const int64_t value) : int_v{.val_ = value} {}
|
||||
explicit PropertyValue(const double value) : double_v{.val_ = value} {}
|
||||
explicit PropertyValue(const TemporalData value) : temporal_data_v{.val_ = value} {}
|
||||
explicit PropertyValue(const bool value) : type_(Type::Bool) { bool_v = value; }
|
||||
explicit PropertyValue(const int value) : type_(Type::Int) { int_v = value; }
|
||||
explicit PropertyValue(const int64_t value) : type_(Type::Int) { int_v = value; }
|
||||
explicit PropertyValue(const double value) : type_(Type::Double) { double_v = value; }
|
||||
explicit PropertyValue(const TemporalData value) : type_{Type::TemporalData} { temporal_data_v = value; }
|
||||
|
||||
// copy constructors for non-primitive types
|
||||
/// @throw std::bad_alloc
|
||||
explicit PropertyValue(std::string value) : string_v{.val_ = std::move(value)} {}
|
||||
explicit PropertyValue(const std::string &value) : type_(Type::String) { new (&string_v) std::string(value); }
|
||||
/// @throw std::bad_alloc
|
||||
/// @throw std::length_error if length of value exceeds
|
||||
/// std::string::max_length().
|
||||
explicit PropertyValue(std::string_view value) : string_v{.val_ = std::string(value)} {}
|
||||
explicit PropertyValue(char const *value) : string_v{.val_ = std::string(value)} {}
|
||||
explicit PropertyValue(const char *value) : type_(Type::String) { new (&string_v) std::string(value); }
|
||||
/// @throw std::bad_alloc
|
||||
explicit PropertyValue(std::vector<PropertyValue> value) : list_v{.val_ = std::move(value)} {}
|
||||
explicit PropertyValue(const std::vector<PropertyValue> &value) : type_(Type::List) {
|
||||
new (&list_v) std::vector<PropertyValue>(value);
|
||||
}
|
||||
/// @throw std::bad_alloc
|
||||
explicit PropertyValue(std::map<std::string, PropertyValue> value) : map_v{.val_ = std::move(value)} {}
|
||||
explicit PropertyValue(const std::map<std::string, PropertyValue> &value) : type_(Type::Map) {
|
||||
new (&map_v) std::map<std::string, PropertyValue>(value);
|
||||
}
|
||||
|
||||
// move constructors for non-primitive types
|
||||
explicit PropertyValue(std::string &&value) noexcept : type_(Type::String) {
|
||||
new (&string_v) std::string(std::move(value));
|
||||
}
|
||||
explicit PropertyValue(std::vector<PropertyValue> &&value) noexcept : type_(Type::List) {
|
||||
new (&list_v) std::vector<PropertyValue>(std::move(value));
|
||||
}
|
||||
explicit PropertyValue(std::map<std::string, PropertyValue> &&value) noexcept : type_(Type::Map) {
|
||||
new (&map_v) std::map<std::string, PropertyValue>(std::move(value));
|
||||
}
|
||||
|
||||
// copy constructor
|
||||
/// @throw std::bad_alloc
|
||||
@@ -112,21 +126,21 @@ class PropertyValue {
|
||||
if (type_ != Type::Bool) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a bool!");
|
||||
}
|
||||
return bool_v.val_;
|
||||
return bool_v;
|
||||
}
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
int64_t ValueInt() const {
|
||||
if (type_ != Type::Int) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't an int!");
|
||||
}
|
||||
return int_v.val_;
|
||||
return int_v;
|
||||
}
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
double ValueDouble() const {
|
||||
if (type_ != Type::Double) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a double!");
|
||||
}
|
||||
return double_v.val_;
|
||||
return double_v;
|
||||
}
|
||||
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
@@ -135,7 +149,7 @@ class PropertyValue {
|
||||
throw PropertyValueException("The value isn't a temporal data!");
|
||||
}
|
||||
|
||||
return temporal_data_v.val_;
|
||||
return temporal_data_v;
|
||||
}
|
||||
|
||||
// const value getters for non-primitive types
|
||||
@@ -144,7 +158,7 @@ class PropertyValue {
|
||||
if (type_ != Type::String) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a string!");
|
||||
}
|
||||
return string_v.val_;
|
||||
return string_v;
|
||||
}
|
||||
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
@@ -152,7 +166,7 @@ class PropertyValue {
|
||||
if (type_ != Type::List) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a list!");
|
||||
}
|
||||
return list_v.val_;
|
||||
return list_v;
|
||||
}
|
||||
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
@@ -160,7 +174,7 @@ class PropertyValue {
|
||||
if (type_ != Type::Map) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a map!");
|
||||
}
|
||||
return map_v.val_;
|
||||
return map_v;
|
||||
}
|
||||
|
||||
// reference value getters for non-primitive types
|
||||
@@ -169,7 +183,7 @@ class PropertyValue {
|
||||
if (type_ != Type::String) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a string!");
|
||||
}
|
||||
return string_v.val_;
|
||||
return string_v;
|
||||
}
|
||||
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
@@ -177,7 +191,7 @@ class PropertyValue {
|
||||
if (type_ != Type::List) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a list!");
|
||||
}
|
||||
return list_v.val_;
|
||||
return list_v;
|
||||
}
|
||||
|
||||
/// @throw PropertyValueException if value isn't of correct type.
|
||||
@@ -185,45 +199,23 @@ class PropertyValue {
|
||||
if (type_ != Type::Map) [[unlikely]] {
|
||||
throw PropertyValueException("The value isn't a map!");
|
||||
}
|
||||
return map_v.val_;
|
||||
return map_v;
|
||||
}
|
||||
|
||||
private:
|
||||
void DestroyValue() noexcept;
|
||||
|
||||
// NOTE: this may look strange but it is for better data layout
|
||||
// https://eel.is/c++draft/class.union#general-note-1
|
||||
union {
|
||||
Type type_;
|
||||
struct {
|
||||
Type type_ = Type::Bool;
|
||||
bool val_;
|
||||
} bool_v;
|
||||
struct {
|
||||
Type type_ = Type::Int;
|
||||
int64_t val_;
|
||||
} int_v;
|
||||
struct {
|
||||
Type type_ = Type::Double;
|
||||
double val_;
|
||||
} double_v;
|
||||
struct {
|
||||
Type type_ = Type::String;
|
||||
std::string val_;
|
||||
} string_v;
|
||||
struct {
|
||||
Type type_ = Type::List;
|
||||
std::vector<PropertyValue> val_;
|
||||
} list_v;
|
||||
struct {
|
||||
Type type_ = Type::Map;
|
||||
std::map<std::string, PropertyValue> val_;
|
||||
} map_v;
|
||||
struct {
|
||||
Type type_ = Type::TemporalData;
|
||||
TemporalData val_;
|
||||
} temporal_data_v;
|
||||
bool bool_v;
|
||||
int64_t int_v;
|
||||
double double_v;
|
||||
std::string string_v;
|
||||
std::vector<PropertyValue> list_v;
|
||||
std::map<std::string, PropertyValue> map_v;
|
||||
TemporalData temporal_data_v;
|
||||
};
|
||||
|
||||
Type type_;
|
||||
};
|
||||
|
||||
// stream output
|
||||
@@ -348,25 +340,25 @@ inline PropertyValue::PropertyValue(const PropertyValue &other) : type_(other.ty
|
||||
case Type::Null:
|
||||
return;
|
||||
case Type::Bool:
|
||||
this->bool_v.val_ = other.bool_v.val_;
|
||||
this->bool_v = other.bool_v;
|
||||
return;
|
||||
case Type::Int:
|
||||
this->int_v.val_ = other.int_v.val_;
|
||||
this->int_v = other.int_v;
|
||||
return;
|
||||
case Type::Double:
|
||||
this->double_v.val_ = other.double_v.val_;
|
||||
this->double_v = other.double_v;
|
||||
return;
|
||||
case Type::String:
|
||||
new (&string_v.val_) std::string(other.string_v.val_);
|
||||
new (&string_v) std::string(other.string_v);
|
||||
return;
|
||||
case Type::List:
|
||||
new (&list_v.val_) std::vector<PropertyValue>(other.list_v.val_);
|
||||
new (&list_v) std::vector<PropertyValue>(other.list_v);
|
||||
return;
|
||||
case Type::Map:
|
||||
new (&map_v.val_) std::map<std::string, PropertyValue>(other.map_v.val_);
|
||||
new (&map_v) std::map<std::string, PropertyValue>(other.map_v);
|
||||
return;
|
||||
case Type::TemporalData:
|
||||
this->temporal_data_v.val_ = other.temporal_data_v.val_;
|
||||
this->temporal_data_v = other.temporal_data_v;
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -376,28 +368,28 @@ inline PropertyValue::PropertyValue(PropertyValue &&other) noexcept : type_(std:
|
||||
case Type::Null:
|
||||
break;
|
||||
case Type::Bool:
|
||||
bool_v.val_ = other.bool_v.val_;
|
||||
bool_v = other.bool_v;
|
||||
break;
|
||||
case Type::Int:
|
||||
int_v.val_ = other.int_v.val_;
|
||||
int_v = other.int_v;
|
||||
break;
|
||||
case Type::Double:
|
||||
double_v.val_ = other.double_v.val_;
|
||||
double_v = other.double_v;
|
||||
break;
|
||||
case Type::String:
|
||||
std::construct_at(&string_v.val_, std::move(other.string_v.val_));
|
||||
std::destroy_at(&other.string_v.val_);
|
||||
std::construct_at(&string_v, std::move(other.string_v));
|
||||
std::destroy_at(&other.string_v);
|
||||
break;
|
||||
case Type::List:
|
||||
std::construct_at(&list_v.val_, std::move(other.list_v.val_));
|
||||
std::destroy_at(&other.list_v.val_);
|
||||
std::construct_at(&list_v, std::move(other.list_v));
|
||||
std::destroy_at(&other.list_v);
|
||||
break;
|
||||
case Type::Map:
|
||||
std::construct_at(&map_v.val_, std::move(other.map_v.val_));
|
||||
std::destroy_at(&other.map_v.val_);
|
||||
std::construct_at(&map_v, std::move(other.map_v));
|
||||
std::destroy_at(&other.map_v);
|
||||
break;
|
||||
case Type::TemporalData:
|
||||
temporal_data_v.val_ = other.temporal_data_v.val_;
|
||||
temporal_data_v = other.temporal_data_v;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -412,25 +404,25 @@ inline PropertyValue &PropertyValue::operator=(const PropertyValue &other) {
|
||||
case Type::Null:
|
||||
break;
|
||||
case Type::Bool:
|
||||
this->bool_v.val_ = other.bool_v.val_;
|
||||
this->bool_v = other.bool_v;
|
||||
break;
|
||||
case Type::Int:
|
||||
this->int_v.val_ = other.int_v.val_;
|
||||
this->int_v = other.int_v;
|
||||
break;
|
||||
case Type::Double:
|
||||
this->double_v.val_ = other.double_v.val_;
|
||||
this->double_v = other.double_v;
|
||||
break;
|
||||
case Type::String:
|
||||
new (&string_v.val_) std::string(other.string_v.val_);
|
||||
new (&string_v) std::string(other.string_v);
|
||||
break;
|
||||
case Type::List:
|
||||
new (&list_v.val_) std::vector<PropertyValue>(other.list_v.val_);
|
||||
new (&list_v) std::vector<PropertyValue>(other.list_v);
|
||||
break;
|
||||
case Type::Map:
|
||||
new (&map_v.val_) std::map<std::string, PropertyValue>(other.map_v.val_);
|
||||
new (&map_v) std::map<std::string, PropertyValue>(other.map_v);
|
||||
break;
|
||||
case Type::TemporalData:
|
||||
this->temporal_data_v.val_ = other.temporal_data_v.val_;
|
||||
this->temporal_data_v = other.temporal_data_v;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -446,28 +438,28 @@ inline PropertyValue &PropertyValue::operator=(PropertyValue &&other) noexcept {
|
||||
case Type::Null:
|
||||
break;
|
||||
case Type::Bool:
|
||||
bool_v.val_ = other.bool_v.val_;
|
||||
bool_v = other.bool_v;
|
||||
break;
|
||||
case Type::Int:
|
||||
int_v.val_ = other.int_v.val_;
|
||||
int_v = other.int_v;
|
||||
break;
|
||||
case Type::Double:
|
||||
double_v.val_ = other.double_v.val_;
|
||||
double_v = other.double_v;
|
||||
break;
|
||||
case Type::String:
|
||||
string_v.val_ = std::move(other.string_v.val_);
|
||||
std::destroy_at(&other.string_v.val_);
|
||||
string_v = std::move(other.string_v);
|
||||
std::destroy_at(&other.string_v);
|
||||
break;
|
||||
case Type::List:
|
||||
list_v.val_ = std::move(other.list_v.val_);
|
||||
std::destroy_at(&other.list_v.val_);
|
||||
list_v = std::move(other.list_v);
|
||||
std::destroy_at(&other.list_v);
|
||||
break;
|
||||
case Type::Map:
|
||||
map_v.val_ = std::move(other.map_v.val_);
|
||||
std::destroy_at(&other.map_v.val_);
|
||||
map_v = std::move(other.map_v);
|
||||
std::destroy_at(&other.map_v);
|
||||
break;
|
||||
case Type::TemporalData:
|
||||
temporal_data_v.val_ = other.temporal_data_v.val_;
|
||||
temporal_data_v = other.temporal_data_v;
|
||||
break;
|
||||
}
|
||||
other.type_ = Type::Null;
|
||||
@@ -490,13 +482,13 @@ inline void PropertyValue::DestroyValue() noexcept {
|
||||
|
||||
// destructor for non primitive types since we used placement new
|
||||
case Type::String:
|
||||
std::destroy_at(&string_v.val_);
|
||||
std::destroy_at(&string_v);
|
||||
return;
|
||||
case Type::List:
|
||||
std::destroy_at(&list_v.val_);
|
||||
std::destroy_at(&list_v);
|
||||
return;
|
||||
case Type::Map:
|
||||
std::destroy_at(&map_v.val_);
|
||||
std::destroy_at(&map_v);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
@@ -26,9 +25,8 @@ template <typename>
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
ReplicationStorageClient::ReplicationStorageClient(::memgraph::replication::ReplicationClient &client,
|
||||
utils::UUID main_uuid)
|
||||
: client_{client}, main_uuid_(main_uuid) {}
|
||||
ReplicationStorageClient::ReplicationStorageClient(::memgraph::replication::ReplicationClient &client)
|
||||
: client_{client} {}
|
||||
|
||||
void ReplicationStorageClient::UpdateReplicaState(Storage *storage, DatabaseAccessProtector db_acc) {
|
||||
uint64_t current_commit_timestamp{kTimestampInitialId};
|
||||
@@ -36,13 +34,14 @@ void ReplicationStorageClient::UpdateReplicaState(Storage *storage, DatabaseAcce
|
||||
auto &replStorageState = storage->repl_storage_state_;
|
||||
|
||||
auto hb_stream{client_.rpc_client_.Stream<replication::HeartbeatRpc>(
|
||||
main_uuid_, storage->uuid(), replStorageState.last_commit_timestamp_, std::string{replStorageState.epoch_.id()})};
|
||||
storage->uuid(), replStorageState.last_commit_timestamp_, std::string{replStorageState.epoch_.id()})};
|
||||
|
||||
const auto replica = hb_stream.AwaitResponse();
|
||||
|
||||
#ifdef MG_ENTERPRISE // Multi-tenancy is only supported in enterprise
|
||||
if (!replica.success) { // Replica is missing the current database
|
||||
client_.state_.WithLock([&](auto &state) {
|
||||
spdlog::debug("Replica '{}' can't respond or missing database '{}' - '{}'", client_.name_, storage->name(),
|
||||
spdlog::debug("Replica '{}' missing database '{}' - '{}'", client_.name_, storage->name(),
|
||||
std::string{storage->uuid()});
|
||||
state = memgraph::replication::ReplicationClient::State::BEHIND;
|
||||
});
|
||||
@@ -96,7 +95,7 @@ TimestampInfo ReplicationStorageClient::GetTimestampInfo(Storage const *storage)
|
||||
info.current_number_of_timestamp_behind_master = 0;
|
||||
|
||||
try {
|
||||
auto stream{client_.rpc_client_.Stream<replication::TimestampRpc>(main_uuid_, storage->uuid())};
|
||||
auto stream{client_.rpc_client_.Stream<replication::TimestampRpc>(storage->uuid())};
|
||||
const auto response = stream.AwaitResponse();
|
||||
const auto is_success = response.success;
|
||||
|
||||
@@ -174,7 +173,7 @@ void ReplicationStorageClient::StartTransactionReplication(const uint64_t curren
|
||||
case READY:
|
||||
MG_ASSERT(!replica_stream_);
|
||||
try {
|
||||
replica_stream_.emplace(storage, client_.rpc_client_, current_wal_seq_num, main_uuid_);
|
||||
replica_stream_.emplace(storage, client_.rpc_client_, current_wal_seq_num);
|
||||
*locked_state = REPLICATING;
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
*locked_state = MAYBE_BEHIND;
|
||||
@@ -184,9 +183,6 @@ void ReplicationStorageClient::StartTransactionReplication(const uint64_t curren
|
||||
}
|
||||
}
|
||||
|
||||
//////// AF: you can't finialize transaction replication if you are not replicating
|
||||
/////// AF: if there is no stream or it is Defunct than we need to set replica in MAYBE_BEHIND -> is that even used
|
||||
/////// AF:
|
||||
bool ReplicationStorageClient::FinalizeTransactionReplication(Storage *storage, DatabaseAccessProtector db_acc) {
|
||||
// We can only check the state because it guarantees to be only
|
||||
// valid during a single transaction replication (if the assumption
|
||||
@@ -260,38 +256,36 @@ void ReplicationStorageClient::RecoverReplica(uint64_t replica_commit, memgraph:
|
||||
spdlog::trace("Recovering in step: {}", i++);
|
||||
try {
|
||||
rpc::Client &rpcClient = client_.rpc_client_;
|
||||
std::visit(
|
||||
utils::Overloaded{
|
||||
[&replica_commit, mem_storage, &rpcClient, main_uuid = main_uuid_](RecoverySnapshot const &snapshot) {
|
||||
spdlog::debug("Sending the latest snapshot file: {}", snapshot);
|
||||
auto response = TransferSnapshot(main_uuid, mem_storage->uuid(), rpcClient, snapshot);
|
||||
replica_commit = response.current_commit_timestamp;
|
||||
},
|
||||
[&replica_commit, mem_storage, &rpcClient, main_uuid = main_uuid_](RecoveryWals const &wals) {
|
||||
spdlog::debug("Sending the latest wal files");
|
||||
auto response = TransferWalFiles(main_uuid, mem_storage->uuid(), rpcClient, wals);
|
||||
replica_commit = response.current_commit_timestamp;
|
||||
spdlog::debug("Wal files successfully transferred.");
|
||||
},
|
||||
[&replica_commit, mem_storage, &rpcClient,
|
||||
main_uuid = main_uuid_](RecoveryCurrentWal const ¤t_wal) {
|
||||
std::unique_lock transaction_guard(mem_storage->engine_lock_);
|
||||
if (mem_storage->wal_file_ &&
|
||||
mem_storage->wal_file_->SequenceNumber() == current_wal.current_wal_seq_num) {
|
||||
utils::OnScopeExit on_exit([mem_storage]() { mem_storage->wal_file_->EnableFlushing(); });
|
||||
mem_storage->wal_file_->DisableFlushing();
|
||||
transaction_guard.unlock();
|
||||
spdlog::debug("Sending current wal file");
|
||||
replica_commit = ReplicateCurrentWal(main_uuid, mem_storage, rpcClient, *mem_storage->wal_file_);
|
||||
} else {
|
||||
spdlog::debug("Cannot recover using current wal file");
|
||||
}
|
||||
},
|
||||
[](auto const &in) {
|
||||
static_assert(always_false_v<decltype(in)>, "Missing type from variant visitor");
|
||||
},
|
||||
},
|
||||
recovery_step);
|
||||
std::visit(utils::Overloaded{
|
||||
[&replica_commit, mem_storage, &rpcClient](RecoverySnapshot const &snapshot) {
|
||||
spdlog::debug("Sending the latest snapshot file: {}", snapshot);
|
||||
auto response = TransferSnapshot(mem_storage->uuid(), rpcClient, snapshot);
|
||||
replica_commit = response.current_commit_timestamp;
|
||||
},
|
||||
[&replica_commit, mem_storage, &rpcClient](RecoveryWals const &wals) {
|
||||
spdlog::debug("Sending the latest wal files");
|
||||
auto response = TransferWalFiles(mem_storage->uuid(), rpcClient, wals);
|
||||
replica_commit = response.current_commit_timestamp;
|
||||
spdlog::debug("Wal files successfully transferred.");
|
||||
},
|
||||
[&replica_commit, mem_storage, &rpcClient](RecoveryCurrentWal const ¤t_wal) {
|
||||
std::unique_lock transaction_guard(mem_storage->engine_lock_);
|
||||
if (mem_storage->wal_file_ &&
|
||||
mem_storage->wal_file_->SequenceNumber() == current_wal.current_wal_seq_num) {
|
||||
utils::OnScopeExit on_exit([mem_storage]() { mem_storage->wal_file_->EnableFlushing(); });
|
||||
mem_storage->wal_file_->DisableFlushing();
|
||||
transaction_guard.unlock();
|
||||
spdlog::debug("Sending current wal file");
|
||||
replica_commit = ReplicateCurrentWal(mem_storage, rpcClient, *mem_storage->wal_file_);
|
||||
} else {
|
||||
spdlog::debug("Cannot recover using current wal file");
|
||||
}
|
||||
},
|
||||
[](auto const &in) {
|
||||
static_assert(always_false_v<decltype(in)>, "Missing type from variant visitor");
|
||||
},
|
||||
},
|
||||
recovery_step);
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
replica_state_.WithLock([](auto &val) { val = replication::ReplicaState::MAYBE_BEHIND; });
|
||||
LogRpcFailure();
|
||||
@@ -320,12 +314,10 @@ void ReplicationStorageClient::RecoverReplica(uint64_t replica_commit, memgraph:
|
||||
}
|
||||
|
||||
////// ReplicaStream //////
|
||||
ReplicaStream::ReplicaStream(Storage *storage, rpc::Client &rpc_client, const uint64_t current_seq_num,
|
||||
utils::UUID main_uuid)
|
||||
ReplicaStream::ReplicaStream(Storage *storage, rpc::Client &rpc_client, const uint64_t current_seq_num)
|
||||
: storage_{storage},
|
||||
stream_(rpc_client.Stream<replication::AppendDeltasRpc>(
|
||||
main_uuid, storage->uuid(), storage->repl_storage_state_.last_commit_timestamp_.load(), current_seq_num)),
|
||||
main_uuid_(main_uuid) {
|
||||
storage->uuid(), storage->repl_storage_state_.last_commit_timestamp_.load(), current_seq_num)) {
|
||||
replication::Encoder encoder{stream_.GetBuilder()};
|
||||
encoder.WriteString(storage->repl_storage_state_.epoch_.id());
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@
|
||||
#include "utils/scheduler.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
#include "utils/thread_pool.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <concepts>
|
||||
@@ -49,7 +48,7 @@ class ReplicationStorageClient;
|
||||
// Handler used for transferring the current transaction.
|
||||
class ReplicaStream {
|
||||
public:
|
||||
explicit ReplicaStream(Storage *storage, rpc::Client &rpc_client, uint64_t current_seq_num, utils::UUID main_uuid);
|
||||
explicit ReplicaStream(Storage *storage, rpc::Client &rpc_client, uint64_t current_seq_num);
|
||||
|
||||
/// @throw rpc::RpcFailedException
|
||||
void AppendDelta(const Delta &delta, const Vertex &vertex, uint64_t final_commit_timestamp);
|
||||
@@ -73,7 +72,6 @@ class ReplicaStream {
|
||||
private:
|
||||
Storage *storage_;
|
||||
rpc::Client::StreamHandler<replication::AppendDeltasRpc> stream_;
|
||||
utils::UUID main_uuid_;
|
||||
};
|
||||
|
||||
template <typename F>
|
||||
@@ -86,7 +84,7 @@ class ReplicationStorageClient {
|
||||
friend struct ::memgraph::replication::ReplicationClient;
|
||||
|
||||
public:
|
||||
explicit ReplicationStorageClient(::memgraph::replication::ReplicationClient &client, utils::UUID main_uuid);
|
||||
explicit ReplicationStorageClient(::memgraph::replication::ReplicationClient &client);
|
||||
|
||||
ReplicationStorageClient(ReplicationStorageClient const &) = delete;
|
||||
ReplicationStorageClient &operator=(ReplicationStorageClient const &) = delete;
|
||||
@@ -204,8 +202,6 @@ class ReplicationStorageClient {
|
||||
replica_stream_; // Currently active stream (nullopt if not in use), note: a single stream per rpc client
|
||||
mutable utils::Synchronized<replication::ReplicaState, utils::SpinLock> replica_state_{
|
||||
replication::ReplicaState::MAYBE_BEHIND};
|
||||
|
||||
const utils::UUID main_uuid_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage
|
||||
|
||||
@@ -114,12 +114,10 @@ void Load(memgraph::storage::replication::TimestampRes *self, memgraph::slk::Rea
|
||||
// Serialize code for TimestampReq
|
||||
|
||||
void Save(const memgraph::storage::replication::TimestampReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::storage::replication::TimestampReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
}
|
||||
|
||||
@@ -138,12 +136,10 @@ void Load(memgraph::storage::replication::CurrentWalRes *self, memgraph::slk::Re
|
||||
// Serialize code for CurrentWalReq
|
||||
|
||||
void Save(const memgraph::storage::replication::CurrentWalReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::storage::replication::CurrentWalReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
}
|
||||
|
||||
@@ -162,13 +158,11 @@ void Load(memgraph::storage::replication::WalFilesRes *self, memgraph::slk::Read
|
||||
// Serialize code for WalFilesReq
|
||||
|
||||
void Save(const memgraph::storage::replication::WalFilesReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
memgraph::slk::Save(self.file_number, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::storage::replication::WalFilesReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
memgraph::slk::Load(&self->file_number, reader);
|
||||
}
|
||||
@@ -188,12 +182,10 @@ void Load(memgraph::storage::replication::SnapshotRes *self, memgraph::slk::Read
|
||||
// Serialize code for SnapshotReq
|
||||
|
||||
void Save(const memgraph::storage::replication::SnapshotReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::storage::replication::SnapshotReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
}
|
||||
|
||||
@@ -214,14 +206,12 @@ void Load(memgraph::storage::replication::HeartbeatRes *self, memgraph::slk::Rea
|
||||
// Serialize code for HeartbeatReq
|
||||
|
||||
void Save(const memgraph::storage::replication::HeartbeatReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
memgraph::slk::Save(self.main_commit_timestamp, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::storage::replication::HeartbeatReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
memgraph::slk::Load(&self->main_commit_timestamp, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
@@ -242,14 +232,12 @@ void Load(memgraph::storage::replication::AppendDeltasRes *self, memgraph::slk::
|
||||
// Serialize code for AppendDeltasReq
|
||||
|
||||
void Save(const memgraph::storage::replication::AppendDeltasReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
memgraph::slk::Save(self.previous_commit_timestamp, builder);
|
||||
memgraph::slk::Save(self.seq_num, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::storage::replication::AppendDeltasReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
memgraph::slk::Load(&self->previous_commit_timestamp, reader);
|
||||
memgraph::slk::Load(&self->seq_num, reader);
|
||||
|
||||
@@ -32,11 +32,9 @@ struct AppendDeltasReq {
|
||||
static void Load(AppendDeltasReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const AppendDeltasReq &self, memgraph::slk::Builder *builder);
|
||||
AppendDeltasReq() = default;
|
||||
AppendDeltasReq(const utils::UUID &main_uuid, const utils::UUID &uuid, uint64_t previous_commit_timestamp,
|
||||
uint64_t seq_num)
|
||||
: main_uuid{main_uuid}, uuid{uuid}, previous_commit_timestamp(previous_commit_timestamp), seq_num(seq_num) {}
|
||||
AppendDeltasReq(const utils::UUID &uuid, uint64_t previous_commit_timestamp, uint64_t seq_num)
|
||||
: uuid{uuid}, previous_commit_timestamp(previous_commit_timestamp), seq_num(seq_num) {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
utils::UUID uuid;
|
||||
uint64_t previous_commit_timestamp;
|
||||
uint64_t seq_num;
|
||||
@@ -65,11 +63,9 @@ struct HeartbeatReq {
|
||||
static void Load(HeartbeatReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const HeartbeatReq &self, memgraph::slk::Builder *builder);
|
||||
HeartbeatReq() = default;
|
||||
HeartbeatReq(const utils::UUID &main_uuid, const utils::UUID &uuid, uint64_t main_commit_timestamp,
|
||||
std::string epoch_id)
|
||||
: main_uuid(main_uuid), uuid{uuid}, main_commit_timestamp(main_commit_timestamp), epoch_id(std::move(epoch_id)) {}
|
||||
HeartbeatReq(const utils::UUID &uuid, uint64_t main_commit_timestamp, std::string epoch_id)
|
||||
: uuid{uuid}, main_commit_timestamp(main_commit_timestamp), epoch_id(std::move(epoch_id)) {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
utils::UUID uuid;
|
||||
uint64_t main_commit_timestamp;
|
||||
std::string epoch_id;
|
||||
@@ -99,9 +95,8 @@ struct SnapshotReq {
|
||||
static void Load(SnapshotReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const SnapshotReq &self, memgraph::slk::Builder *builder);
|
||||
SnapshotReq() = default;
|
||||
explicit SnapshotReq(const utils::UUID &main_uuid, const utils::UUID &uuid) : main_uuid{main_uuid}, uuid{uuid} {}
|
||||
explicit SnapshotReq(const utils::UUID &uuid) : uuid{uuid} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
utils::UUID uuid;
|
||||
};
|
||||
|
||||
@@ -128,10 +123,8 @@ struct WalFilesReq {
|
||||
static void Load(WalFilesReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const WalFilesReq &self, memgraph::slk::Builder *builder);
|
||||
WalFilesReq() = default;
|
||||
explicit WalFilesReq(const utils::UUID &main_uuid, const utils::UUID &uuid, uint64_t file_number)
|
||||
: main_uuid{main_uuid}, uuid{uuid}, file_number(file_number) {}
|
||||
explicit WalFilesReq(const utils::UUID &uuid, uint64_t file_number) : uuid{uuid}, file_number(file_number) {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
utils::UUID uuid;
|
||||
uint64_t file_number;
|
||||
};
|
||||
@@ -159,9 +152,8 @@ struct CurrentWalReq {
|
||||
static void Load(CurrentWalReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const CurrentWalReq &self, memgraph::slk::Builder *builder);
|
||||
CurrentWalReq() = default;
|
||||
explicit CurrentWalReq(const utils::UUID &main_uuid, const utils::UUID &uuid) : main_uuid(main_uuid), uuid{uuid} {}
|
||||
explicit CurrentWalReq(const utils::UUID &uuid) : uuid{uuid} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
utils::UUID uuid;
|
||||
};
|
||||
|
||||
@@ -188,9 +180,8 @@ struct TimestampReq {
|
||||
static void Load(TimestampReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const TimestampReq &self, memgraph::slk::Builder *builder);
|
||||
TimestampReq() = default;
|
||||
explicit TimestampReq(const utils::UUID &main_uuid, const utils::UUID &uuid) : main_uuid(main_uuid), uuid{uuid} {}
|
||||
explicit TimestampReq(const utils::UUID &uuid) : uuid{uuid} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
utils::UUID uuid;
|
||||
};
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
#include <atomic>
|
||||
#include <limits>
|
||||
#include <list>
|
||||
#include <memory>
|
||||
|
||||
#include "utils/memory.hpp"
|
||||
@@ -38,6 +39,7 @@ namespace memgraph::storage {
|
||||
|
||||
const uint64_t kTimestampInitialId = 0;
|
||||
const uint64_t kTransactionInitialId = 1ULL << 63U;
|
||||
using PmrListDelta = utils::pmr::list<Delta>;
|
||||
|
||||
struct Transaction {
|
||||
Transaction(uint64_t transaction_id, uint64_t start_timestamp, IsolationLevel isolation_level,
|
||||
@@ -45,6 +47,7 @@ struct Transaction {
|
||||
: transaction_id(transaction_id),
|
||||
start_timestamp(start_timestamp),
|
||||
command_id(0),
|
||||
deltas(0),
|
||||
md_deltas(utils::NewDeleteResource()),
|
||||
must_abort(false),
|
||||
isolation_level(isolation_level),
|
||||
@@ -88,7 +91,7 @@ struct Transaction {
|
||||
std::unique_ptr<std::atomic<uint64_t>> commit_timestamp{};
|
||||
uint64_t command_id{};
|
||||
|
||||
std::deque<Delta> deltas;
|
||||
Bond<PmrListDelta> deltas;
|
||||
utils::pmr::list<MetadataDelta> md_deltas;
|
||||
bool must_abort{};
|
||||
IsolationLevel isolation_level{};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -76,13 +76,13 @@ inline auto HasLabel_ActionMethod(bool &has_label, LabelId label) {
|
||||
// clang-format off
|
||||
return utils::Overloaded{
|
||||
ActionMethod<REMOVE_LABEL>([&, label](Delta const &delta) {
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
}
|
||||
}),
|
||||
ActionMethod<ADD_LABEL>([&, label](Delta const &delta) {
|
||||
if (delta.label.value == label) {
|
||||
if (delta.label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
}
|
||||
@@ -96,14 +96,14 @@ inline auto Labels_ActionMethod(std::vector<LabelId> &labels) {
|
||||
// clang-format off
|
||||
return utils::Overloaded{
|
||||
ActionMethod<REMOVE_LABEL>([&](Delta const &delta) {
|
||||
auto it = std::find(labels.begin(), labels.end(), delta.label.value);
|
||||
auto it = std::find(labels.begin(), labels.end(), delta.label);
|
||||
DMG_ASSERT(it != labels.end(), "Invalid database state!");
|
||||
*it = labels.back();
|
||||
labels.pop_back();
|
||||
}),
|
||||
ActionMethod<ADD_LABEL>([&](Delta const &delta) {
|
||||
DMG_ASSERT(std::find(labels.begin(), labels.end(), delta.label.value) == labels.end(), "Invalid database state!");
|
||||
labels.emplace_back(delta.label.value);
|
||||
DMG_ASSERT(std::find(labels.begin(), labels.end(), delta.label) == labels.end(), "Invalid database state!");
|
||||
labels.emplace_back(delta.label);
|
||||
})
|
||||
};
|
||||
// clang-format on
|
||||
@@ -113,7 +113,7 @@ inline auto PropertyValue_ActionMethod(PropertyValue &value, PropertyId property
|
||||
using enum Delta::Action;
|
||||
return ActionMethod<SET_PROPERTY>([&, property](Delta const &delta) {
|
||||
if (delta.property.key == property) {
|
||||
value = *delta.property.value;
|
||||
value = delta.property.value;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -121,7 +121,7 @@ inline auto PropertyValue_ActionMethod(PropertyValue &value, PropertyId property
|
||||
inline auto PropertyValueMatch_ActionMethod(bool &match, PropertyId property, PropertyValue const &value) {
|
||||
using enum Delta::Action;
|
||||
return ActionMethod<SET_PROPERTY>([&, property](Delta const &delta) {
|
||||
if (delta.property.key == property) match = (value == *delta.property.value);
|
||||
if (delta.property.key == property) match = (value == delta.property.value);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -130,15 +130,15 @@ inline auto Properties_ActionMethod(std::map<PropertyId, PropertyValue> &propert
|
||||
return ActionMethod<SET_PROPERTY>([&](Delta const &delta) {
|
||||
auto it = properties.find(delta.property.key);
|
||||
if (it != properties.end()) {
|
||||
if (delta.property.value->IsNull()) {
|
||||
if (delta.property.value.IsNull()) {
|
||||
// remove the property
|
||||
properties.erase(it);
|
||||
} else {
|
||||
// set the value
|
||||
it->second = *delta.property.value;
|
||||
it->second = delta.property.value;
|
||||
}
|
||||
} else if (!delta.property.value->IsNull()) {
|
||||
properties.emplace(delta.property.key, *delta.property.value);
|
||||
} else if (!delta.property.value.IsNull()) {
|
||||
properties.emplace(delta.property.key, delta.property.value);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -27,7 +27,7 @@ struct ISystemAction {
|
||||
virtual void DoDurability() = 0;
|
||||
|
||||
/// Prepare the RPC payload that will be sent to all replicas clients
|
||||
virtual bool DoReplication(memgraph::replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
virtual bool DoReplication(memgraph::replication::ReplicationClient &client,
|
||||
memgraph::replication::ReplicationEpoch const &epoch,
|
||||
Transaction const &system_tx) const = 0;
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ struct DoReplication {
|
||||
auto sync_status = AllSyncReplicaStatus::AllCommitsConfirmed;
|
||||
|
||||
for (auto &client : main_data_.registered_replicas_) {
|
||||
bool completed = action.DoReplication(client, main_data_.uuid_, main_data_.epoch_, system_tx);
|
||||
bool completed = action.DoReplication(client, main_data_.epoch_, system_tx);
|
||||
if (!completed && client.mode_ == replication_coordination_glue::ReplicationMode::SYNC) {
|
||||
sync_status = AllSyncReplicaStatus::SomeCommitsUnconfirmed;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -21,7 +21,7 @@ inline std::optional<std::string> GetOldDiskKeyOrNull(storage::Delta *head) {
|
||||
head = head->next;
|
||||
}
|
||||
if (head->action == storage::Delta::Action::DELETE_DESERIALIZED_OBJECT) {
|
||||
return head->old_disk_key.value;
|
||||
return head->old_disk_key;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -338,13 +338,6 @@ inline uint64_t ParseStringToUint64(const std::string_view s) {
|
||||
throw utils::ParseException(s);
|
||||
}
|
||||
|
||||
inline uint32_t ParseStringToUint32(const std::string_view s) {
|
||||
if (uint32_t value = 0; std::from_chars(s.data(), s.data() + s.size(), value).ec == std::errc{}) {
|
||||
return value;
|
||||
}
|
||||
throw utils::ParseException(s);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a double floating point value from a string using classic locale.
|
||||
* Note, the current implementation copies the given string which may perform a
|
||||
|
||||
@@ -97,16 +97,12 @@ enum class TypeId : uint64_t {
|
||||
REP_UPDATE_AUTH_DATA_RES,
|
||||
REP_DROP_AUTH_DATA_REQ,
|
||||
REP_DROP_AUTH_DATA_RES,
|
||||
REP_TRY_SET_MAIN_UUID_REQ,
|
||||
REP_TRY_SET_MAIN_UUID_RES,
|
||||
|
||||
// Coordinator
|
||||
COORD_FAILOVER_REQ,
|
||||
COORD_FAILOVER_RES,
|
||||
COORD_SET_REPL_MAIN_REQ,
|
||||
COORD_SET_REPL_MAIN_RES,
|
||||
COORD_SWAP_UUID_REQ,
|
||||
COORD_SWAP_UUID_RES,
|
||||
|
||||
// AST
|
||||
AST_LABELIX = 3000,
|
||||
|
||||
@@ -39,8 +39,8 @@ endfunction()
|
||||
|
||||
add_subdirectory(fine_grained_access)
|
||||
add_subdirectory(server)
|
||||
add_subdirectory(replication)
|
||||
add_subdirectory(memory)
|
||||
#add_subdirectory(replication)
|
||||
#add_subdirectory(memory)
|
||||
add_subdirectory(triggers)
|
||||
add_subdirectory(isolation_levels)
|
||||
add_subdirectory(streams)
|
||||
@@ -56,7 +56,7 @@ add_subdirectory(python_query_modules_reloading)
|
||||
add_subdirectory(analyze_graph)
|
||||
add_subdirectory(transaction_queue)
|
||||
add_subdirectory(mock_api)
|
||||
add_subdirectory(graphql)
|
||||
#add_subdirectory(graphql)
|
||||
add_subdirectory(disk_storage)
|
||||
add_subdirectory(load_csv)
|
||||
add_subdirectory(init_file_flags)
|
||||
|
||||
@@ -3,7 +3,6 @@ find_package(gflags REQUIRED)
|
||||
copy_e2e_python_files(ha_experimental coordinator.py)
|
||||
copy_e2e_python_files(ha_experimental automatic_failover.py)
|
||||
copy_e2e_python_files(ha_experimental manual_setting_replicas.py)
|
||||
copy_e2e_python_files(ha_experimental not_replicate_from_old_main.py)
|
||||
copy_e2e_python_files(ha_experimental common.py)
|
||||
copy_e2e_python_files(ha_experimental workloads.yaml)
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import interactive_mg_runner
|
||||
import pytest
|
||||
@@ -132,7 +131,6 @@ def test_replication_works_on_failover():
|
||||
mg_sleep_and_assert(expected_data_on_new_main, retrieve_data_show_replicas)
|
||||
|
||||
interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
|
||||
|
||||
expected_data_on_new_main = [
|
||||
("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"),
|
||||
("instance_3", "127.0.0.1:10003", "sync", 0, 0, "ready"),
|
||||
@@ -143,8 +141,8 @@ def test_replication_works_on_failover():
|
||||
execute_and_fetch_all(new_main_cursor, "CREATE ();")
|
||||
|
||||
# 6
|
||||
alive_replica_cursor = connect(host="localhost", port=7689).cursor()
|
||||
res = execute_and_fetch_all(alive_replica_cursor, "MATCH (n) RETURN count(n) as count;")[0][0]
|
||||
alive_replica_cursror = connect(host="localhost", port=7689).cursor()
|
||||
res = execute_and_fetch_all(alive_replica_cursror, "MATCH (n) RETURN count(n) as count;")[0][0]
|
||||
assert res == 1, "Vertex should be replicated"
|
||||
interactive_mg_runner.stop_all(MEMGRAPH_INSTANCES_DESCRIPTION)
|
||||
|
||||
@@ -346,60 +344,65 @@ def test_automatic_failover_main_back_as_replica():
|
||||
mg_sleep_and_assert([("replica",)], retrieve_data_show_repl_role_instance3)
|
||||
|
||||
|
||||
def test_replica_instance_restarts_replication_works():
|
||||
def test_automatic_failover_main_back_as_main():
|
||||
safe_execute(shutil.rmtree, TEMP_DIR)
|
||||
interactive_mg_runner.start_all(MEMGRAPH_INSTANCES_DESCRIPTION)
|
||||
|
||||
cursor = connect(host="localhost", port=7690).cursor()
|
||||
interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_1")
|
||||
interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_2")
|
||||
interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
|
||||
|
||||
def show_repl_cluster():
|
||||
return sorted(list(execute_and_fetch_all(cursor, "SHOW REPLICATION CLUSTER;")))
|
||||
coord_cursor = connect(host="localhost", port=7690).cursor()
|
||||
|
||||
expected_data_up = [
|
||||
def retrieve_data_show_repl_cluster():
|
||||
return sorted(list(execute_and_fetch_all(coord_cursor, "SHOW REPLICATION CLUSTER;")))
|
||||
|
||||
expected_data_all_down = [
|
||||
("instance_1", "127.0.0.1:10011", False, "unknown"),
|
||||
("instance_2", "127.0.0.1:10012", False, "unknown"),
|
||||
("instance_3", "127.0.0.1:10013", False, "unknown"),
|
||||
]
|
||||
|
||||
mg_sleep_and_assert(expected_data_all_down, retrieve_data_show_repl_cluster)
|
||||
|
||||
interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_3")
|
||||
expected_data_main_back = [
|
||||
("instance_1", "127.0.0.1:10011", False, "unknown"),
|
||||
("instance_2", "127.0.0.1:10012", False, "unknown"),
|
||||
("instance_3", "127.0.0.1:10013", True, "main"),
|
||||
]
|
||||
mg_sleep_and_assert(expected_data_main_back, retrieve_data_show_repl_cluster)
|
||||
|
||||
instance3_cursor = connect(host="localhost", port=7687).cursor()
|
||||
|
||||
def retrieve_data_show_repl_role_instance3():
|
||||
return sorted(list(execute_and_fetch_all(instance3_cursor, "SHOW REPLICATION ROLE;")))
|
||||
|
||||
mg_sleep_and_assert([("main",)], retrieve_data_show_repl_role_instance3)
|
||||
|
||||
interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_1")
|
||||
interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_2")
|
||||
|
||||
expected_data_replicas_back = [
|
||||
("instance_1", "127.0.0.1:10011", True, "replica"),
|
||||
("instance_2", "127.0.0.1:10012", True, "replica"),
|
||||
("instance_3", "127.0.0.1:10013", True, "main"),
|
||||
]
|
||||
mg_sleep_and_assert(expected_data_up, show_repl_cluster)
|
||||
|
||||
interactive_mg_runner.kill(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_1")
|
||||
mg_sleep_and_assert(expected_data_replicas_back, retrieve_data_show_repl_cluster)
|
||||
|
||||
expected_data_down = [
|
||||
("instance_1", "127.0.0.1:10011", False, "unknown"),
|
||||
("instance_2", "127.0.0.1:10012", True, "replica"),
|
||||
("instance_3", "127.0.0.1:10013", True, "main"),
|
||||
]
|
||||
mg_sleep_and_assert(expected_data_down, show_repl_cluster)
|
||||
|
||||
interactive_mg_runner.start(MEMGRAPH_INSTANCES_DESCRIPTION, "instance_1")
|
||||
|
||||
mg_sleep_and_assert(expected_data_up, show_repl_cluster)
|
||||
|
||||
expected_data_on_main_show_replicas = [
|
||||
("instance_1", "127.0.0.1:10001", "sync", 0, 0, "ready"),
|
||||
("instance_2", "127.0.0.1:10002", "sync", 0, 0, "ready"),
|
||||
]
|
||||
instance3_cursor = connect(host="localhost", port=7687).cursor()
|
||||
instance1_cursor = connect(host="localhost", port=7688).cursor()
|
||||
|
||||
def retrieve_data_show_repl_role_instance1():
|
||||
return sorted(list(execute_and_fetch_all(instance3_cursor, "SHOW REPLICAS;")))
|
||||
|
||||
mg_sleep_and_assert(expected_data_on_main_show_replicas, retrieve_data_show_repl_role_instance1)
|
||||
instance2_cursor = connect(host="localhost", port=7689).cursor()
|
||||
|
||||
def retrieve_data_show_repl_role_instance1():
|
||||
return sorted(list(execute_and_fetch_all(instance1_cursor, "SHOW REPLICATION ROLE;")))
|
||||
|
||||
expected_data_replica = [("replica",)]
|
||||
mg_sleep_and_assert(expected_data_replica, retrieve_data_show_repl_role_instance1)
|
||||
def retrieve_data_show_repl_role_instance2():
|
||||
return sorted(list(execute_and_fetch_all(instance2_cursor, "SHOW REPLICATION ROLE;")))
|
||||
|
||||
execute_and_fetch_all(instance3_cursor, "CREATE ();")
|
||||
|
||||
def retrieve_data_replica():
|
||||
return execute_and_fetch_all(instance1_cursor, "MATCH (n) RETURN count(n);")[0][0]
|
||||
|
||||
expected_data_replica = 1
|
||||
mg_sleep_and_assert(expected_data_replica, retrieve_data_replica)
|
||||
mg_sleep_and_assert([("replica",)], retrieve_data_show_repl_role_instance1)
|
||||
mg_sleep_and_assert([("replica",)], retrieve_data_show_repl_role_instance2)
|
||||
mg_sleep_and_assert([("main",)], retrieve_data_show_repl_role_instance3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
# Copyright 2024 Memgraph Ltd.
|
||||
#
|
||||
# Use of this software is governed by the Business Source License
|
||||
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
# License, and you may not use this file except in compliance with the Business Source License.
|
||||
#
|
||||
# As of the Change Date specified in that file, in accordance with
|
||||
# the Business Source License, use of this software will be governed
|
||||
# by the Apache License, Version 2.0, included in the file
|
||||
# licenses/APL.txt.
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import interactive_mg_runner
|
||||
import pytest
|
||||
from common import execute_and_fetch_all
|
||||
from mg_utils import mg_sleep_and_assert
|
||||
|
||||
interactive_mg_runner.SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
interactive_mg_runner.PROJECT_DIR = os.path.normpath(
|
||||
os.path.join(interactive_mg_runner.SCRIPT_DIR, "..", "..", "..", "..")
|
||||
)
|
||||
interactive_mg_runner.BUILD_DIR = os.path.normpath(os.path.join(interactive_mg_runner.PROJECT_DIR, "build"))
|
||||
interactive_mg_runner.MEMGRAPH_BINARY = os.path.normpath(os.path.join(interactive_mg_runner.BUILD_DIR, "memgraph"))
|
||||
|
||||
MEMGRAPH_FIRST_CLUSTER_DESCRIPTION = {
|
||||
"shared_replica": {
|
||||
"args": ["--bolt-port", "7688", "--log-level", "TRACE"],
|
||||
"log_file": "replica2.log",
|
||||
"setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"],
|
||||
},
|
||||
"main1": {
|
||||
"args": ["--bolt-port", "7687", "--log-level", "TRACE"],
|
||||
"log_file": "main.log",
|
||||
"setup_queries": ["REGISTER REPLICA shared_replica SYNC TO '127.0.0.1:10001' ;"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
MEMGRAPH_INSTANCES_DESCRIPTION = {
|
||||
"replica": {
|
||||
"args": ["--bolt-port", "7689", "--log-level", "TRACE"],
|
||||
"log_file": "replica.log",
|
||||
"setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"],
|
||||
},
|
||||
"main_2": {
|
||||
"args": ["--bolt-port", "7690", "--log-level", "TRACE"],
|
||||
"log_file": "main_2.log",
|
||||
"setup_queries": [
|
||||
"REGISTER REPLICA shared_replica SYNC TO '127.0.0.1:10001' ;",
|
||||
"REGISTER REPLICA replica SYNC TO '127.0.0.1:10002' ; ",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_replication_works_on_failover(connection):
|
||||
# Goal of this test is to check that after changing `shared_replica`
|
||||
# to be part of new cluster, `main` (old cluster) can't write any more to it
|
||||
|
||||
# 1
|
||||
interactive_mg_runner.start_all_keep_others(MEMGRAPH_FIRST_CLUSTER_DESCRIPTION)
|
||||
|
||||
# 2
|
||||
main_cursor = connection(7687, "main1").cursor()
|
||||
expected_data_on_main = [
|
||||
("shared_replica", "127.0.0.1:10001", "sync", 0, 0, "ready"),
|
||||
]
|
||||
actual_data_on_main = sorted(list(execute_and_fetch_all(main_cursor, "SHOW REPLICAS;")))
|
||||
assert actual_data_on_main == expected_data_on_main
|
||||
|
||||
# 3
|
||||
interactive_mg_runner.start_all_keep_others(MEMGRAPH_INSTANCES_DESCRIPTION)
|
||||
|
||||
# 4
|
||||
new_main_cursor = connection(7690, "main_2").cursor()
|
||||
|
||||
def retrieve_data_show_replicas():
|
||||
return sorted(list(execute_and_fetch_all(new_main_cursor, "SHOW REPLICAS;")))
|
||||
|
||||
expected_data_on_new_main = [
|
||||
("replica", "127.0.0.1:10002", "sync", 0, 0, "ready"),
|
||||
("shared_replica", "127.0.0.1:10001", "sync", 0, 0, "ready"),
|
||||
]
|
||||
mg_sleep_and_assert(expected_data_on_new_main, retrieve_data_show_replicas)
|
||||
|
||||
# 5
|
||||
shared_replica_cursor = connection(7688, "shared_replica").cursor()
|
||||
|
||||
with pytest.raises(Exception) as e:
|
||||
execute_and_fetch_all(main_cursor, "CREATE ();")
|
||||
assert (
|
||||
str(e.value)
|
||||
== "Replication Exception: At least one SYNC replica has not confirmed committing last transaction. Check the status of the replicas using 'SHOW REPLICAS' query."
|
||||
)
|
||||
|
||||
res = execute_and_fetch_all(main_cursor, "MATCH (n) RETURN count(n) as count;")[0][0]
|
||||
assert res == 1, "Vertex should be created"
|
||||
|
||||
res = execute_and_fetch_all(shared_replica_cursor, "MATCH (n) RETURN count(n) as count;")[0][0]
|
||||
assert res == 0, "Vertex shouldn't be replicated"
|
||||
|
||||
# 7
|
||||
execute_and_fetch_all(new_main_cursor, "CREATE ();")
|
||||
|
||||
res = execute_and_fetch_all(new_main_cursor, "MATCH (n) RETURN count(n) as count;")[0][0]
|
||||
assert res == 1, "Vertex should be created"
|
||||
|
||||
res = execute_and_fetch_all(shared_replica_cursor, "MATCH (n) RETURN count(n) as count;")[0][0]
|
||||
assert res == 1, "Vertex should be replicated"
|
||||
|
||||
interactive_mg_runner.stop_all()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-rA"]))
|
||||
@@ -35,7 +35,3 @@ workloads:
|
||||
- name: "Disabled manual setting of replication cluster"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
args: ["high_availability_experimental/manual_setting_replicas.py"]
|
||||
|
||||
- name: "Not replicate from old main"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
args: ["high_availability_experimental/not_replicate_from_old_main.py"]
|
||||
|
||||
@@ -208,11 +208,6 @@ def start_all(context, procdir="", keep_directories=True):
|
||||
start_instance(context, key, procdir)
|
||||
|
||||
|
||||
def start_all_keep_others(context, procdir="", keep_directories=True):
|
||||
for key, _ in context.items():
|
||||
start_instance(context, key, procdir)
|
||||
|
||||
|
||||
def start(context, name, procdir=""):
|
||||
if name != "all":
|
||||
start_instance(context, name, procdir)
|
||||
|
||||
@@ -48,16 +48,6 @@ read_query_modules_in_memory_cluster: &read_query_modules_in_memory_cluster
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
read_query_modules_disk_cluster: &read_query_modules_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_query_modules_in_memory_cluster: &update_query_modules_in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
@@ -66,16 +56,6 @@ update_query_modules_in_memory_cluster: &update_query_modules_in_memory_cluster
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_query_modules_disk_cluster: &update_query_modules_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
show_privileges_in_memory_cluster: &show_privileges_in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
@@ -84,16 +64,6 @@ show_privileges_in_memory_cluster: &show_privileges_in_memory_cluster
|
||||
setup_queries: *show_privileges_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
show_privileges_disk_cluster: &show_privileges_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *show_privileges_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
read_permission_in_memory_queries: &read_permission_in_memory_queries
|
||||
cluster:
|
||||
main:
|
||||
@@ -102,16 +72,6 @@ read_permission_in_memory_queries: &read_permission_in_memory_queries
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
read_permission_disk_queries: &read_permission_disk_queries
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
|
||||
create_delete_query_modules_in_memory_cluster: &create_delete_query_modules_in_memory_cluster
|
||||
cluster:
|
||||
@@ -121,16 +81,6 @@ create_delete_query_modules_in_memory_cluster: &create_delete_query_modules_in_m
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
create_delete_query_modules_disk_cluster: &create_delete_query_modules_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_permission_queries_in_memory_cluster: &update_permission_queries_in_memory_cluster
|
||||
cluster:
|
||||
main:
|
||||
@@ -139,15 +89,6 @@ update_permission_queries_in_memory_cluster: &update_permission_queries_in_memor
|
||||
setup_queries: *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
update_permission_queries_disk_cluster: &update_permission_queries_disk_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: *args
|
||||
log_file: "disk-lba-e2e.log"
|
||||
setup_queries:
|
||||
- "STORAGE MODE ON_DISK_TRANSACTIONAL"
|
||||
- *query_modules_setup_queries
|
||||
validation_queries: []
|
||||
|
||||
workloads:
|
||||
- name: "read-query-modules"
|
||||
@@ -156,68 +97,32 @@ workloads:
|
||||
args: ["lba_procedures/read_query_modules.py"]
|
||||
<<: *read_query_modules_in_memory_cluster
|
||||
|
||||
- name: "read-query-modules on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/read_query_modules.py"]
|
||||
<<: *read_query_modules_disk_cluster
|
||||
|
||||
- name: "update-query-modules"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_query_modules.py"]
|
||||
<<: *update_query_modules_in_memory_cluster
|
||||
|
||||
- name: "update-query-modules on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_query_modules.py"]
|
||||
<<: *update_query_modules_disk_cluster
|
||||
|
||||
- name: "create-delete-query-modules"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/create_delete_query_modules.py"]
|
||||
<<: *create_delete_query_modules_in_memory_cluster
|
||||
|
||||
- name: "create-delete-query-modules on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/create_delete_query_modules.py"]
|
||||
<<: *create_delete_query_modules_disk_cluster
|
||||
|
||||
- name: "show-privileges"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/show_privileges.py"]
|
||||
<<: *show_privileges_in_memory_cluster
|
||||
|
||||
- name: "show-privileges on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/show_privileges.py"]
|
||||
<<: *show_privileges_disk_cluster
|
||||
|
||||
- name: "read-permission-queries"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/read_permission_queries.py"]
|
||||
<<: *read_permission_in_memory_queries
|
||||
|
||||
- name: "read-permission-queries on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/read_permission_queries.py"]
|
||||
<<: *read_permission_disk_queries
|
||||
|
||||
- name: "update-permission-queries"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_permission_queries.py"]
|
||||
<<: *update_permission_queries_in_memory_cluster
|
||||
|
||||
- name: "update-permission-queries on disk"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/lba_procedures/procedures/"
|
||||
args: ["lba_procedures/update_permission_queries.py"]
|
||||
<<: *update_permission_queries_disk_cluster
|
||||
|
||||
@@ -135,11 +135,6 @@ workloads:
|
||||
proc: "tests/e2e/memory/procedures/"
|
||||
<<: *in_memory_query_limit_cluster
|
||||
|
||||
- name: "Memory control query limit create"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
<<: *in_memory_query_limit_cluster
|
||||
|
||||
- name: "Memory control query limit create multi thread"
|
||||
binary: "tests/e2e/memory/memgraph__e2e__memory__limit_query_alloc_create_multi_thread"
|
||||
args: ["--bolt-port", *bolt_port]
|
||||
|
||||
@@ -26,10 +26,3 @@ workloads:
|
||||
proc: "query_modules/"
|
||||
args: ["query_modules/mgps_test.py"]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
- name: "Schema test"
|
||||
pre_set_workload: "tests/e2e/x.sh"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "query_modules/"
|
||||
args: ["query_modules/schema_test.py"]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
@@ -42,21 +42,3 @@ workloads:
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/pulsar_streams_tests.py"]
|
||||
<<: *in_memory_cluster
|
||||
|
||||
- name: "Kafka streams start, stop and show for on-disk storage"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/kafka_streams_tests.py"]
|
||||
<<: *disk_cluster
|
||||
|
||||
- name: "Streams with users for on-disk storage"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/streams_owner_tests.py"]
|
||||
<<: *disk_cluster
|
||||
|
||||
- name: "Pulsar streams start, stop and show for on-disk storage"
|
||||
binary: "tests/e2e/pytest_runner.sh"
|
||||
proc: "tests/e2e/streams/transformations/"
|
||||
args: ["streams/pulsar_streams_tests.py"]
|
||||
<<: *disk_cluster
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
#include "replication/state.hpp"
|
||||
#include "replication/status.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <fstream>
|
||||
@@ -49,17 +48,6 @@ TEST(ReplicationDurability, V2Main) {
|
||||
ASSERT_EQ(role_entry, deser);
|
||||
}
|
||||
|
||||
TEST(ReplicationDurability, V3Main) {
|
||||
auto const role_entry = ReplicationRoleEntry{
|
||||
.version = DurabilityVersion::V3,
|
||||
.role = MainRole{.epoch = ReplicationEpoch{"TEST_STRING"}, .main_uuid = memgraph::utils::UUID{}}};
|
||||
nlohmann::json j;
|
||||
to_json(j, role_entry);
|
||||
ReplicationRoleEntry deser;
|
||||
from_json(j, deser);
|
||||
ASSERT_EQ(role_entry, deser);
|
||||
}
|
||||
|
||||
TEST(ReplicationDurability, V1Replica) {
|
||||
auto const role_entry =
|
||||
ReplicationRoleEntry{.version = DurabilityVersion::V1,
|
||||
@@ -86,33 +74,6 @@ TEST(ReplicationDurability, V2Replica) {
|
||||
ASSERT_EQ(role_entry, deser);
|
||||
}
|
||||
|
||||
TEST(ReplicationDurability, V3ReplicaNoMain) {
|
||||
auto const role_entry =
|
||||
ReplicationRoleEntry{.version = DurabilityVersion::V3,
|
||||
.role = ReplicaRole{
|
||||
.config = ReplicationServerConfig{.ip_address = "000.123.456.789", .port = 2023},
|
||||
}};
|
||||
nlohmann::json j;
|
||||
to_json(j, role_entry);
|
||||
ReplicationRoleEntry deser;
|
||||
from_json(j, deser);
|
||||
ASSERT_EQ(role_entry, deser);
|
||||
}
|
||||
|
||||
TEST(ReplicationDurability, V3ReplicaMain) {
|
||||
auto const role_entry =
|
||||
ReplicationRoleEntry{.version = DurabilityVersion::V2,
|
||||
.role = ReplicaRole{
|
||||
.config = ReplicationServerConfig{.ip_address = "000.123.456.789", .port = 2023},
|
||||
.main_uuid = memgraph::utils::UUID{},
|
||||
}};
|
||||
nlohmann::json j;
|
||||
to_json(j, role_entry);
|
||||
ReplicationRoleEntry deser;
|
||||
from_json(j, deser);
|
||||
ASSERT_EQ(role_entry, deser);
|
||||
}
|
||||
|
||||
TEST(ReplicationDurability, ReplicaEntrySync) {
|
||||
using namespace std::chrono_literals;
|
||||
using namespace std::string_literals;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -419,9 +419,9 @@ TEST(PropertyStore, IntEncoding) {
|
||||
{memgraph::storage::PropertyId::FromUint(1048576UL), memgraph::storage::PropertyValue(1048576L)},
|
||||
{memgraph::storage::PropertyId::FromUint(std::numeric_limits<uint32_t>::max()),
|
||||
memgraph::storage::PropertyValue(std::numeric_limits<int32_t>::max())},
|
||||
{memgraph::storage::PropertyId::FromUint(1048577UL), memgraph::storage::PropertyValue(4294967296L)},
|
||||
{memgraph::storage::PropertyId::FromUint(1048578UL), memgraph::storage::PropertyValue(137438953472L)},
|
||||
{memgraph::storage::PropertyId::FromUint(std::numeric_limits<uint32_t>::max()),
|
||||
{memgraph::storage::PropertyId::FromUint(4294967296UL), memgraph::storage::PropertyValue(4294967296L)},
|
||||
{memgraph::storage::PropertyId::FromUint(137438953472UL), memgraph::storage::PropertyValue(137438953472L)},
|
||||
{memgraph::storage::PropertyId::FromUint(std::numeric_limits<uint64_t>::max()),
|
||||
memgraph::storage::PropertyValue(std::numeric_limits<int64_t>::max())}};
|
||||
|
||||
memgraph::storage::PropertyStore props;
|
||||
|
||||
@@ -142,21 +142,17 @@ TEST_F(ReplicationTest, BasicSynchronousReplicationTest) {
|
||||
MinMemgraph replica(repl_conf);
|
||||
|
||||
auto replica_store_handler = replica.repl_handler;
|
||||
replica_store_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
|
||||
const auto ® = main.repl_handler.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = "REPLICA",
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true);
|
||||
const auto ® = main.repl_handler.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = "REPLICA",
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
ASSERT_FALSE(reg.HasError()) << (int)reg.GetError();
|
||||
|
||||
// vertex create
|
||||
@@ -439,38 +435,30 @@ TEST_F(ReplicationTest, MultipleSynchronousReplicationTest) {
|
||||
MinMemgraph replica1(repl_conf);
|
||||
MinMemgraph replica2(repl2_conf);
|
||||
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
});
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
})
|
||||
.HasError());
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
})
|
||||
.HasError());
|
||||
|
||||
const auto *vertex_label = "label";
|
||||
@@ -597,21 +585,17 @@ TEST_F(ReplicationTest, RecoveryProcess) {
|
||||
MinMemgraph replica(repl_conf);
|
||||
auto replica_store_handler = replica.repl_handler;
|
||||
|
||||
replica_store_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
})
|
||||
.HasError());
|
||||
|
||||
ASSERT_EQ(main.db.storage()->GetReplicaState(replicas[0]), ReplicaState::RECOVERY);
|
||||
@@ -676,22 +660,18 @@ TEST_F(ReplicationTest, BasicAsynchronousReplicationTest) {
|
||||
MinMemgraph replica_async(repl_conf);
|
||||
|
||||
auto replica_store_handler = replica_async.repl_handler;
|
||||
replica_store_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
std::nullopt);
|
||||
replica_store_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
});
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = "REPLICA_ASYNC",
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = "REPLICA_ASYNC",
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
})
|
||||
.HasError());
|
||||
|
||||
static constexpr size_t vertices_create_num = 10;
|
||||
@@ -726,41 +706,33 @@ TEST_F(ReplicationTest, EpochTest) {
|
||||
MinMemgraph main(main_conf);
|
||||
MinMemgraph replica1(repl_conf);
|
||||
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
|
||||
MinMemgraph replica2(repl2_conf);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = 10001,
|
||||
},
|
||||
std::nullopt);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = 10001,
|
||||
});
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
})
|
||||
.HasError());
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = 10001,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = 10001,
|
||||
})
|
||||
.HasError());
|
||||
|
||||
std::optional<Gid> vertex_gid;
|
||||
@@ -789,14 +761,12 @@ TEST_F(ReplicationTest, EpochTest) {
|
||||
ASSERT_TRUE(replica1.repl_handler.SetReplicationRoleMain());
|
||||
|
||||
ASSERT_FALSE(replica1.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = 10001,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = 10001,
|
||||
})
|
||||
|
||||
.HasError());
|
||||
|
||||
@@ -819,21 +789,17 @@ TEST_F(ReplicationTest, EpochTest) {
|
||||
ASSERT_FALSE(acc->Commit().HasError());
|
||||
}
|
||||
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
ASSERT_TRUE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
})
|
||||
|
||||
.HasError());
|
||||
|
||||
@@ -858,43 +824,35 @@ TEST_F(ReplicationTest, ReplicationInformation) {
|
||||
MinMemgraph replica1(repl_conf);
|
||||
|
||||
uint16_t replica1_port = 10001;
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
});
|
||||
|
||||
uint16_t replica2_port = 10002;
|
||||
MinMemgraph replica2(repl2_conf);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
},
|
||||
std::nullopt);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
});
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
})
|
||||
|
||||
.HasError());
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
})
|
||||
|
||||
.HasError());
|
||||
|
||||
@@ -923,41 +881,33 @@ TEST_F(ReplicationTest, ReplicationReplicaWithExistingName) {
|
||||
MinMemgraph replica1(repl_conf);
|
||||
|
||||
uint16_t replica1_port = 10001;
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
});
|
||||
|
||||
uint16_t replica2_port = 10002;
|
||||
MinMemgraph replica2(repl2_conf);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
},
|
||||
std::nullopt);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
});
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica1_port,
|
||||
})
|
||||
.HasError());
|
||||
|
||||
ASSERT_TRUE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = replica2_port,
|
||||
})
|
||||
.GetError() == RegisterReplicaError::NAME_EXISTS);
|
||||
}
|
||||
|
||||
@@ -966,41 +916,33 @@ TEST_F(ReplicationTest, ReplicationReplicaWithExistingEndPoint) {
|
||||
|
||||
MinMemgraph main(main_conf);
|
||||
MinMemgraph replica1(repl_conf);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
});
|
||||
|
||||
MinMemgraph replica2(repl2_conf);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
},
|
||||
std::nullopt);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
});
|
||||
|
||||
ASSERT_FALSE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
})
|
||||
.HasError());
|
||||
|
||||
ASSERT_TRUE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
},
|
||||
true)
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::ASYNC,
|
||||
.ip_address = local_host,
|
||||
.port = common_port,
|
||||
})
|
||||
.GetError() == RegisterReplicaError::ENDPOINT_EXISTS);
|
||||
}
|
||||
|
||||
@@ -1023,38 +965,30 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartupAfterDroppingReplica) {
|
||||
std::optional<MinMemgraph> main(main_config);
|
||||
MinMemgraph replica1(replica1_config);
|
||||
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
|
||||
MinMemgraph replica2(replica2_config);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
std::nullopt);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
});
|
||||
|
||||
auto res = main->repl_handler.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true);
|
||||
auto res = main->repl_handler.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
ASSERT_FALSE(res.HasError()) << (int)res.GetError();
|
||||
res = main->repl_handler.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
true);
|
||||
res = main->repl_handler.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
});
|
||||
ASSERT_FALSE(res.HasError()) << (int)res.GetError();
|
||||
|
||||
auto replica_infos = main->db.storage()->ReplicasInfo();
|
||||
@@ -1088,38 +1022,30 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartup) {
|
||||
std::optional<MinMemgraph> main(main_config);
|
||||
MinMemgraph replica1(repl_conf);
|
||||
|
||||
replica1.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
std::nullopt);
|
||||
replica1.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
|
||||
MinMemgraph replica2(repl2_conf);
|
||||
|
||||
replica2.repl_handler.SetReplicationRoleReplica(
|
||||
ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
std::nullopt);
|
||||
auto res = main->repl_handler.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true);
|
||||
replica2.repl_handler.SetReplicationRoleReplica(ReplicationServerConfig{
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
});
|
||||
auto res = main->repl_handler.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[0],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
});
|
||||
ASSERT_FALSE(res.HasError());
|
||||
res = main->repl_handler.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
},
|
||||
true);
|
||||
res = main->repl_handler.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = replicas[1],
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[1],
|
||||
});
|
||||
ASSERT_FALSE(res.HasError());
|
||||
|
||||
auto replica_infos = main->db.storage()->ReplicasInfo();
|
||||
@@ -1157,13 +1083,11 @@ TEST_F(ReplicationTest, AddingInvalidReplica) {
|
||||
MinMemgraph main(main_conf);
|
||||
|
||||
ASSERT_TRUE(main.repl_handler
|
||||
.TryRegisterReplica(
|
||||
ReplicationClientConfig{
|
||||
.name = "REPLICA",
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
},
|
||||
true)
|
||||
.GetError() == RegisterReplicaError::ERROR_ACCEPTING_MAIN);
|
||||
.TryRegisterReplica(ReplicationClientConfig{
|
||||
.name = "REPLICA",
|
||||
.mode = ReplicationMode::SYNC,
|
||||
.ip_address = local_host,
|
||||
.port = ports[0],
|
||||
})
|
||||
.GetError() == RegisterReplicaError::CONNECTION_FAILED);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -154,8 +154,8 @@ class DeltaGenerator final {
|
||||
|
||||
void Finalize(bool append_transaction_end = true) {
|
||||
auto commit_timestamp = gen_->timestamp_++;
|
||||
if (transaction_.deltas.empty()) return;
|
||||
for (const auto &delta : transaction_.deltas) {
|
||||
if (transaction_.deltas.use().empty()) return;
|
||||
for (const auto &delta : transaction_.deltas.use()) {
|
||||
auto owner = delta.prev.Get();
|
||||
while (owner.type == memgraph::storage::PreviousPtr::Type::DELTA) {
|
||||
owner = owner.delta->prev.Get();
|
||||
@@ -171,7 +171,7 @@ class DeltaGenerator final {
|
||||
if (append_transaction_end) {
|
||||
gen_->wal_file_.AppendTransactionEnd(commit_timestamp);
|
||||
if (gen_->valid_) {
|
||||
gen_->UpdateStats(commit_timestamp, transaction_.deltas.size() + 1);
|
||||
gen_->UpdateStats(commit_timestamp, transaction_.deltas.use().size() + 1);
|
||||
for (auto &data : data_) {
|
||||
if (data.type == memgraph::storage::durability::WalDeltaData::Type::VERTEX_SET_PROPERTY) {
|
||||
// We need to put the final property value into the SET_PROPERTY
|
||||
|
||||
Reference in New Issue
Block a user