Compare commits

..

4 Commits

Author SHA1 Message Date
Andi Skrgat
b812417d86 typo 2024-01-18 11:21:27 +01:00
Andreja Tonev
ed35069dd0 TRACE logs for driver tests 2024-01-18 11:19:42 +01:00
Andi Skrgat
7b1a7f8bf6 Fix build mode 2024-01-18 11:16:32 +01:00
Andi Skrgat
49d25761a3 Logs in debug mode 2024-01-18 11:10:13 +01:00
318 changed files with 2628 additions and 10207 deletions

View File

@@ -16,7 +16,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -27,7 +27,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -65,7 +65,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -96,7 +96,7 @@ jobs:
- name: Python code analysis
run: |
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only --diff-filter=d)
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
for file in ${CHANGED_FILES}; do
echo ${file}
if [[ ${file} == *.py ]]; then
@@ -137,9 +137,9 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Code coverage(Code analysis)"
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Run clang-tidy
@@ -162,7 +162,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -208,9 +208,9 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Code coverage(Debug build)"
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
release_build:
@@ -223,7 +223,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -251,7 +251,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "GQL Behave Status"
path: |
@@ -268,7 +268,6 @@ jobs:
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Ensure Kafka and Pulsar are up
if: false
run: |
cd tests/e2e/streams/kafka
docker-compose up -d
@@ -276,7 +275,6 @@ jobs:
docker-compose up -d
- name: Run e2e tests
if: false
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
@@ -285,7 +283,7 @@ jobs:
./run.sh
- name: Ensure Kafka and Pulsar are down
if: false
if: always()
run: |
cd tests/e2e/streams/kafka
docker-compose down
@@ -325,128 +323,16 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
if: always()
with:
name: "Test data(Release build)"
path: |
# multiple paths could be defined
build/logs
experimental_build_ha:
name: "High availability build"
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
source /opt/toolchain-v4/activate
./init
cd build
cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
make -j$THREADS
- name: Run unit tests
run: |
source /opt/toolchain-v4/activate
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run e2e tests
if: false
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate_e2e
cd e2e
./run.sh "Coordinator"
./run.sh "Client initiated failover"
./run.sh "Uninitialized cluster"
- name: Save test data
uses: actions/upload-artifact@v4
if: always()
with:
name: "Test data(High availability build)"
path: |
# multiple paths could be defined
build/logs
experimental_build_mt:
name: "MultiTenancy replication build"
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build MT replication experimental binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=Release -D MG_EXPERIMENTAL_REPLICATION_MULTITENANCY=ON ..
make -j$THREADS
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run e2e tests
if: false
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate_e2e
cd e2e
# Just the replication based e2e tests
./run.sh "Replicate multitenancy"
./run.sh "Show"
./run.sh "Show while creating invalid state"
./run.sh "Delete edge replication"
./run.sh "Read-write benchmark"
./run.sh "Index replication"
./run.sh "Constraints"
- name: Save test data
uses: actions/upload-artifact@v4
if: always()
with:
name: "Test data(MultiTenancy replication build)"
name: "Test data"
path: |
# multiple paths could be defined
build/logs
@@ -462,7 +348,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -479,18 +365,13 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
make -j$THREADS memgraph
- name: Refresh Jepsen Cluster
run: |
cd tests/jepsen
./run.sh cluster-refresh
- name: Run Jepsen tests
run: |
cd tests/jepsen
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: "Jepsen Report"
@@ -506,7 +387,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -42,14 +42,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: amzn-2
path: build/output/amzn-2/memgraph*.rpm
@@ -60,14 +60,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
@@ -78,14 +78,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
@@ -96,14 +96,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
@@ -114,14 +114,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
@@ -132,14 +132,14 @@ jobs:
timeout-minutes: 120
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: debian-11-aarch64
path: build/output/debian-11-arm/memgraph*.deb
@@ -150,14 +150,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
@@ -168,7 +168,7 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -177,7 +177,7 @@ jobs:
./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: docker
path: build/output/docker/memgraph*.tar.gz
@@ -188,14 +188,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: fedora-36
path: build/output/fedora-36/memgraph*.rpm
@@ -206,14 +206,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ubuntu-18.04
path: build/output/ubuntu-18.04/memgraph*.deb
@@ -224,14 +224,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ubuntu-20.04
path: build/output/ubuntu-20.04/memgraph*.deb
@@ -242,14 +242,14 @@ jobs:
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04
path: build/output/ubuntu-22.04/memgraph*.deb
@@ -260,14 +260,14 @@ jobs:
timeout-minutes: 120
steps:
- name: "Set up repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04-aarch64
path: build/output/ubuntu-22.04-arm/memgraph*.deb
@@ -279,7 +279,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v4
uses: actions/download-artifact@v3
with:
# name: # if name input parameter is not provided, all artifacts are downloaded
# and put in directories named after each one.

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -14,21 +14,19 @@ on:
schedule:
- cron: "0 22 * * *"
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
timeout-minutes: 960
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -42,6 +40,10 @@ jobs:
# Initialize dependencies.
./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
@@ -63,11 +65,10 @@ jobs:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -109,19 +110,22 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Code coverage(Coverage build)"
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
debug_build:
name: "Debug build"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -153,6 +157,10 @@ jobs:
run: |
./tests/drivers/run.sh
- name: Run integration tests
run: |
tests/integration/run.sh
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
@@ -163,49 +171,23 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Code coverage(Debug build)"
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
debug_integration_test:
name: "Debug integration tests"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run integration tests
run: |
tests/integration/run.sh
release_build:
name: "Release build"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
runs-on: [self-hosted, Linux, X64, Debian10, nightcrawler]
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
timeout-minutes: 960
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -221,210 +203,26 @@ jobs:
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
cmake -DCMAKE_BUILD_TYPE=Debug ..
make -j$THREADS
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create enterprise DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v4
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Run GQL Behave tests
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v4
with:
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure
release_benchmark_tests:
name: "Release Benchmark Tests"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run micro benchmark tests.
cd build
# The `eval` benchmark needs a large stack limit.
ulimit -s 262144
ctest -R memgraph__benchmark -V
- name: Run macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Run parallel macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QueryParallelSuite MemgraphRunner \
--groups aggregation_parallel create_parallel bfs_parallel \
--num-database-workers 9 --num-clients-workers 30 \
--no-strict
release_e2e_test:
if: false
name: "Release End-to-end Test"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
- name: Ensure Kafka and Pulsar are up
run: |
cd tests/e2e/streams/kafka
docker-compose up -d
cd ../pulsar
docker-compose up -d
- name: Run e2e tests
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate_e2e
cd e2e
./run.sh
- name: Ensure Kafka and Pulsar are down
if: always()
run: |
cd tests/e2e/streams/kafka
docker-compose down
cd ../pulsar
docker-compose down
release_durability_stress_tests:
name: "Release durability and stress tests"
runs-on: [self-hosted, Linux, X64, Debian10]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
- name: Run stress test (plain)
- name: Run stress test (large)
run: |
cd tests/stress
./continuous_integration
- name: Run stress test (SSL)
run: |
cd tests/stress
./continuous_integration --use-ssl
- name: Run durability test (plain)
run: |
cd tests/stress
source ve3/bin/activate
python3 durability --num-steps 5
- name: Run durability test (large)
run: |
cd tests/stress
source ve3/bin/activate
python3 durability --num-steps 20
./continuous_integration --large-dataset
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
timeout-minutes: 60
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -437,23 +235,22 @@ jobs:
# Initialize dependencies.
./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build only memgraph release binary.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS memgraph
- name: Refresh Jepsen Cluster
run: |
cd tests/jepsen
./run.sh cluster-refresh
- name: Run Jepsen tests
run: |
cd tests/jepsen
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
if: ${{ always() }}
with:
name: "Jepsen Report"

View File

@@ -19,7 +19,7 @@ jobs:
DOCKER_REPOSITORY_NAME: memgraph
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2

View File

@@ -20,7 +20,7 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2

View File

@@ -1,190 +0,0 @@
name: Publish memgraph-builder docker images
on:
workflow_dispatch:
inputs:
toolchain_version:
type: choice
description: "Toolchain version for the builder"
options:
- v4
- v5
target_os:
type: choice
description: "Target OS for the builder"
default: 'debian-11'
options:
- all
- all-amd64
- all-arm
- amzn-2
- centos-7
- centos-9
- debian-10
- debian-11
- debian-11-arm
- fedora-36
- ubuntu-18_04
- ubuntu-20_04
- ubuntu-22_04
- ubuntu-22_04-arm
force_release:
type: boolean
description: "Overwrite existing image on dockerhub"
required: false
default: false
jobs:
amd_build:
if: ${{ !startsWith(github.event.inputs.target_os, 'all') && !endsWith(github.event.inputs.target_os, '-arm') }}
name: "Build and publish specific amd64 image"
runs-on: [self-hosted, DockerMgBuild, X64]
env:
DOCKER_ORG: "memgraph"
DOCKER_REPO: "memgraph-builder"
DOCKER_IMAGE_TAG: "${{ github.event.inputs.toolchain_version }}_${{ github.event.inputs.target_os }}"
ARCHITECTURE: "amd64"
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check if specified image is already pushed
run: |
if [[ $FORCE_RELEASE = true ]]; then
EXISTS=1
else
EXISTS=$(docker manifest inspect $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG > /dev/null; echo $?)
fi
if [[ $EXISTS -eq 0 ]]; then
echo "Image $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG already exists, skipping next step ..."
fi
echo "EXISTS=$EXISTS" >> $GITHUB_ENV
- name: "Build and publish image"
if: env.EXISTS == 1
run: |
echo "Building and publishing $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG"
./release/package/run.sh build ${{ github.event.inputs.toolchain_version }} ${{ github.event.inputs.target_os }} $ARCHITECTURE --publish
arm_build:
if: ${{ !startsWith(github.event.inputs.target_os, 'all') && endsWith(github.event.inputs.target_os, '-arm') }}
name: "Build and publish specific arm image"
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
env:
DOCKER_ORG: "memgraph"
DOCKER_REPO: "memgraph-builder"
DOCKER_IMAGE_TAG: "${{ github.event.inputs.toolchain_version }}_${{ github.event.inputs.target_os }}"
ARCHITECTURE: "arm64"
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos
run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check if specified image is already pushed
run: |
if [[ $FORCE_RELEASE = true ]]; then
EXISTS=1
else
EXISTS=$(docker manifest inspect $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG > /dev/null; echo $?)
fi
if [[ $EXISTS -eq 0 ]]; then
echo "Image $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG already exists, skipping next step ..."
fi
echo "EXISTS=$EXISTS" >> $GITHUB_ENV
- name: "Build and publish image"
if: env.EXISTS == 1
run: |
echo "Building and publishing $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG"
./release/package/run.sh build ${{ github.event.inputs.toolchain_version }} ${{ github.event.inputs.target_os }} $ARCHITECTURE --publish
all_amd_build:
if: ${{ github.event.inputs.target_os == 'all-amd64' || github.event.inputs.target_os == 'all'}}
name: "Build and publish all amd64 images"
runs-on: [self-hosted, DockerMgBuild, X64]
strategy:
matrix:
target_os: [amzn-2, centos-7, centos-9, debian-10, debian-11, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04]
env:
DOCKER_ORG: "memgraph"
DOCKER_REPO: "memgraph-builder"
DOCKER_IMAGE_TAG: "${{ github.event.inputs.toolchain_version }}_${{ github.event.inputs.target_os }}"
ARCHITECTURE: "amd64"
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check if specified image is already pushed
run: |
if [[ $FORCE_RELEASE = true ]]; then
EXISTS=1
else
EXISTS=$(docker manifest inspect $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG > /dev/null; echo $?)
fi
if [[ $EXISTS -eq 0 ]]; then
echo "Image $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG already exists, skipping next step ..."
fi
echo "EXISTS=$EXISTS" >> $GITHUB_ENV
- name: "Build and publish image"
if: env.EXISTS == 1
run: |
echo "Building and publishing $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG"
./release/package/run.sh build ${{ github.event.inputs.toolchain_version }} ${{ github.event.inputs.target_os }} $ARCHITECTURE --publish
all_arm_build:
if: ${{ github.event.inputs.target_os == 'all-arm' || github.event.inputs.target_os == 'all'}}
name: "Build and publish all arm images"
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
strategy:
matrix:
target_os: [debian-11-arm, ubuntu-22_04-arm]
env:
DOCKER_ORG: "memgraph"
DOCKER_REPO: "memgraph-builder"
DOCKER_IMAGE_TAG: "${{ github.event.inputs.toolchain_version }}_${{ github.event.inputs.target_os }}"
ARCHITECTURE: "arm64"
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos
run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check if specified image is already pushed
run: |
if [[ $FORCE_RELEASE = true ]]; then
EXISTS=1
else
EXISTS=$(docker manifest inspect $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG > /dev/null; echo $?)
fi
if [[ $EXISTS -eq 0 ]]; then
echo "Image $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG already exists, skipping next step ..."
fi
echo "EXISTS=$EXISTS" >> $GITHUB_ENV
- name: "Build and publish image"
if: env.EXISTS == 1
run: |
echo "Building and publishing $DOCKER_ORG/$DOCKER_REPO:$DOCKER_IMAGE_TAG"
./release/package/run.sh build ${{ github.event.inputs.toolchain_version }} ${{ github.event.inputs.target_os }} $ARCHITECTURE --publish

View File

@@ -14,21 +14,19 @@ on:
schedule:
- cron: "0 22 * * *"
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
timeout-minutes: 960
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -42,6 +40,10 @@ jobs:
# Initialize dependencies.
./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
@@ -59,11 +61,14 @@ jobs:
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -105,19 +110,22 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Code coverage(Coverage build)"
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
debug_build:
name: "Debug build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -149,6 +157,10 @@ jobs:
run: |
./tests/drivers/run.sh
- name: Run integration tests
run: |
tests/integration/run.sh
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
@@ -159,49 +171,23 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Code coverage(Debug build)"
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
debug_integration_test:
name: "Debug integration tests"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run integration tests
run: |
tests/integration/run.sh
release_build:
name: "Release build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
timeout-minutes: 960
steps:
- name: Set up repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -215,6 +201,10 @@ jobs:
# Initialize dependencies.
./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
@@ -236,60 +226,11 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Run GQL Behave tests
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v4
with:
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure
release_benchmark_tests:
name: "Release Benchmark Tests"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
- name: Run micro benchmark tests
run: |
# Activate toolchain.
@@ -316,31 +257,29 @@ jobs:
--num-database-workers 9 --num-clients-workers 30 \
--no-strict
release_e2e_test:
if: false
name: "Release End-to-end Test"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
- name: Run GQL Behave tests
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration
steps:
- name: Set up repository
uses: actions/checkout@v4
- name: Save quality assurance status
uses: actions/upload-artifact@v3
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Build release binaries
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries
# Run unit tests.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
ctest -R memgraph__unit --output-on-failure
- name: Ensure Kafka and Pulsar are up
run: |
@@ -365,32 +304,6 @@ jobs:
cd ../pulsar
docker-compose down
release_durability_stress_tests:
name: "Release durability and stress tests"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
timeout-minutes: 90
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
- name: Run stress test (plain)
run: |
cd tests/stress
@@ -401,6 +314,11 @@ jobs:
cd tests/stress
./continuous_integration --use-ssl
- name: Run stress test (large)
run: |
cd tests/stress
./continuous_integration --large-dataset
- name: Run durability test (plain)
run: |
cd tests/stress

View File

@@ -1,62 +0,0 @@
name: Stress test large
on:
workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule:
- cron: "0 22 * * *"
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
jobs:
stress_test_large:
name: "Stress test large"
timeout-minutes: 600
strategy:
matrix:
os: [Debian10, Ubuntu20.04]
extra: [BigMemory, Gen8]
exclude:
- os: Debian10
extra: Gen8
- os: Ubuntu20.04
extra: BigMemory
runs-on: [self-hosted, Linux, X64, "${{ matrix.os }}", "${{ matrix.extra }}"]
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS
- name: Run stress test (large)
run: |
cd tests/stress
./continuous_integration --large-dataset

View File

@@ -15,7 +15,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: dawidd6/action-download-artifact@v4
uses: dawidd6/action-download-artifact@v2
with:
workflow: package_all.yaml
workflow_conclusion: success

View File

@@ -1,34 +0,0 @@
# NuRaft ADR
**Author**
Marko Budiselic (github.com/gitbuda)
**Status**
PROPOSED
**Date**
January 10, 2024
**Problem**
In order to enhance Memgraph to have High Availability features as requested by
customers, we want to have reliable coordinators backed by RAFT consensus algorithm. Implementing
RAFT to be correct and performant is a very challenging task. Skillful Memgraph
engineers already tried 3 times and failed to deliver in a reasonable timeframe
all three times (approximately 4 person-weeks of engineering work each time).
**Criteria**
- easy integration with our C++ codebase
- heavily tested in production environments
- implementation of performance optimizations on top of the canonical Raft
implementation
**Decision**
There are a few, robust C++ implementations of Raft but as a part of other
projects or bigger libraries. **We select
[NuRaft](https://github.com/eBay/NuRaft)** because it focuses on delivering
Raft without bloatware, and it's used by
[Clickhouse](https://github.com/ClickHouse/ClickHouse) (an comparable peer to
Memgraph, a very well-established product).

View File

@@ -271,17 +271,6 @@ endif()
set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
add_subdirectory(libs EXCLUDE_FROM_ALL)
option(MG_EXPERIMENTAL_HIGH_AVAILABILITY "Feature flag for experimental high availability" OFF)
if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_HIGH_AVAILABILITY)
set(MG_EXPERIMENTAL_HIGH_AVAILABILITY OFF)
message(FATAL_ERROR "MG_EXPERIMENTAL_HIGH_AVAILABILITY must be used with enterpise version of the code.")
endif ()
if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
add_compile_definitions(MG_EXPERIMENTAL_HIGH_AVAILABILITY)
endif ()
# Optional subproject configuration -------------------------------------------
option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
option(TOOLS "Build tools binaries" ON)
@@ -290,18 +279,6 @@ option(ASAN "Build with Address Sanitizer. To get a reasonable performance optio
option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
# Build feature flags
option(MG_EXPERIMENTAL_REPLICATION_MULTITENANCY "Feature flag for experimental replicaition of multitenacy" OFF)
if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
set(MG_EXPERIMENTAL_REPLICATION_MULTITENANCY OFF)
message(FATAL_ERROR "MG_EXPERIMENTAL_REPLICATION_MULTITENANCY with community edition build isn't possible")
endif ()
if (MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
add_compile_definitions(MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
endif ()
if (TEST_COVERAGE)
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (NOT lower_build_type STREQUAL "debug")

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2028-21-01
CHANGE DATE: 2027-08-12
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -10,21 +10,15 @@ SUPPORTED_OS=(
fedora-36
amzn-2
)
SUPPORTED_TOOLCAHINS=(
v4
v5
)
SUPPORTED_BUILD_TYPES=(
Debug
Release
RelWithDebInfo
)
SUPPORTED_ARCHITECTURES=(
amd64
arm64
)
PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION:-toolchain-v4}"
TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
@@ -36,14 +30,6 @@ print_help () {
echo " Build types: ${SUPPORTED_BUILD_TYPES[*]}"
exit 1
}
print_help_build () {
echo "$0 build {toolchain_version} {os} {architecture} [--publish]"
echo ""
echo " Toolchain versions: ${SUPPORTED_TOOLCAHINS[*]}"
echo " OSs: ${SUPPORTED_OS[*]}"
echo " Architectures: ${SUPPORTED_ARCHITECTURES[*]}"
exit 1
}
make_package () {
os="$1"
@@ -201,26 +187,15 @@ case "$1" in
build)
shift 1
if [[ "$#" -lt 3 || "$#" -gt 4 ]]; then
print_help_build
if [[ "$#" -ne 2 ]]; then
print_help
fi
# in the vX format, e.g. v5
toolchain_version="$1"
# a name of the os folder, e.g. ubuntu-22.04-arm
os="$2"
cd "$SCRIPT_DIR/$os"
# architecture for which the image will be built, e.g. linux/arm64
architecture="$3"
if [[ "$#" -eq 4 ]]; then
if [[ "$4" == "--publish" ]]; then
docker buildx build --file Dockerfile --build-arg TOOLCHAIN_VERSION="toolchain-$toolchain_version" --platform "linux/$architecture" --tag "memgraph/memgraph-builder:${toolchain_version}_$os" --push .
else
print_help_build
exit
fi
else
docker buildx build --file Dockerfile --build-arg TOOLCHAIN_VERSION="toolchain-$toolchain_version" --platform "linux/$architecture" --tag "memgraph/memgraph-builder:${toolchain_version}_$os" .
fi
docker build -f Dockerfile --build-arg TOOLCHAIN_VERSION="toolchain-$toolchain_version" -t "memgraph/memgraph-builder:${toolchain_version}_$os" .
;;
test)

View File

@@ -22,8 +22,6 @@ add_subdirectory(dbms)
add_subdirectory(flags)
add_subdirectory(distributed)
add_subdirectory(replication)
add_subdirectory(coordination)
add_subdirectory(replication_coordination_glue)
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -8,15 +8,17 @@
#include "auth/auth.hpp"
#include <cstring>
#include <iostream>
#include <limits>
#include <utility>
#include <fmt/format.h>
#include "auth/crypto.hpp"
#include "auth/exceptions.hpp"
#include "license/license.hpp"
#include "utils/flag_validation.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/settings.hpp"
#include "utils/string.hpp"
@@ -44,9 +46,6 @@ namespace memgraph::auth {
const std::string kUserPrefix = "user:";
const std::string kRolePrefix = "role:";
const std::string kLinkPrefix = "link:";
const std::string kVersion = "version";
static constexpr auto kVersionV1 = "V1";
/**
* All data stored in the `Auth` storage is stored in an underlying
@@ -65,59 +64,7 @@ static constexpr auto kVersionV1 = "V1";
* key="link:<username>", value="<rolename>"
*/
namespace {
void MigrateVersions(kvstore::KVStore &store) {
static constexpr auto kPasswordHashV0V1 = "password_hash";
auto version_str = store.Get(kVersion);
if (!version_str) {
using namespace std::string_literals;
// pre versioning, add version to the store
auto puts = std::map<std::string, std::string>{{kVersion, kVersionV1}};
// also add hash kind into durability
auto it = store.begin(kUserPrefix);
auto const e = store.end(kUserPrefix);
if (it != e) {
const auto hash_algo = CurrentHashAlgorithm();
spdlog::info("Updating auth durability, assuming previously stored as {}", AsString(hash_algo));
for (; it != e; ++it) {
auto const &[key, value] = *it;
try {
auto user_data = nlohmann::json::parse(value);
auto password_hash = user_data[kPasswordHashV0V1];
if (!password_hash.is_string()) {
throw AuthException("Couldn't load user data!");
}
// upgrade the password_hash to include the hash algortihm
if (password_hash.empty()) {
user_data[kPasswordHashV0V1] = nullptr;
} else {
user_data[kPasswordHashV0V1] = HashedPassword{hash_algo, password_hash};
}
puts.emplace(key, user_data.dump());
} catch (const nlohmann::json::parse_error &e) {
throw AuthException("Couldn't load user data!");
}
}
}
// Perform migration to V1
store.PutMultiple(puts);
version_str = kVersionV1;
}
}
}; // namespace
Auth::Auth(std::string storage_directory, Config config)
: storage_(std::move(storage_directory)), module_(FLAGS_auth_module_executable), config_{std::move(config)} {
MigrateVersions(storage_);
}
Auth::Auth(const std::string &storage_directory) : storage_(storage_directory), module_(FLAGS_auth_module_executable) {}
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
if (module_.IsUsed()) {
@@ -166,7 +113,7 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
return std::nullopt;
}
} else {
UpdatePassword(*user, password);
user->UpdatePassword(password);
}
if (FLAGS_auth_module_manage_roles) {
if (!rolename.empty()) {
@@ -208,10 +155,6 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
username, "https://memgr.ph/auth"));
return std::nullopt;
}
if (user->UpgradeHash(password)) {
SaveUser(*user);
}
return user;
}
}
@@ -254,46 +197,13 @@ void Auth::SaveUser(const User &user) {
}
}
void Auth::UpdatePassword(auth::User &user, const std::optional<std::string> &password) {
// Check if null
if (!password) {
if (!config_.password_permit_null) {
throw AuthException("Null passwords aren't permitted!");
}
} else {
// Check if compliant with our filter
if (config_.custom_password_regex) {
if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
license_check_result.HasError()) {
throw AuthException(
"Custom password regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
glue::kDefaultPasswordRegex,
license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
}
}
if (!std::regex_match(*password, config_.password_regex)) {
throw AuthException(
"The user password doesn't conform to the required strength! Regex: "
"\"{}\"",
config_.password_regex_str);
}
}
// All checks passed; update
user.UpdatePassword(password);
}
std::optional<User> Auth::AddUser(const std::string &username, const std::optional<std::string> &password) {
if (!NameRegexMatch(username)) {
throw AuthException("Invalid user name.");
}
auto existing_user = GetUser(username);
if (existing_user) return std::nullopt;
auto existing_role = GetRole(username);
if (existing_role) return std::nullopt;
auto new_user = User(username);
UpdatePassword(new_user, password);
new_user.UpdatePassword(password);
SaveUser(new_user);
return new_user;
}
@@ -345,11 +255,10 @@ void Auth::SaveRole(const Role &role) {
}
std::optional<Role> Auth::AddRole(const std::string &rolename) {
if (!NameRegexMatch(rolename)) {
throw AuthException("Invalid role name.");
}
if (auto existing_role = GetRole(rolename)) return std::nullopt;
if (auto existing_user = GetUser(rolename)) return std::nullopt;
auto existing_role = GetRole(rolename);
if (existing_role) return std::nullopt;
auto existing_user = GetUser(rolename);
if (existing_user) return std::nullopt;
auto new_role = Role(rolename);
SaveRole(new_role);
return new_role;
@@ -376,7 +285,8 @@ std::vector<auth::Role> Auth::AllRoles() const {
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
auto rolename = it->first.substr(kRolePrefix.size());
if (rolename != utils::ToLowerCase(rolename)) continue;
if (auto role = GetRole(rolename)) {
auto role = GetRole(rolename);
if (role) {
ret.push_back(*role);
} else {
throw AuthException("Couldn't load role '{}'!", rolename);
@@ -386,14 +296,15 @@ std::vector<auth::Role> Auth::AllRoles() const {
}
std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig) const {
const auto rolename = utils::ToLowerCase(rolename_orig);
auto rolename = utils::ToLowerCase(rolename_orig);
std::vector<auth::User> ret;
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix); ++it) {
auto username = it->first.substr(kLinkPrefix.size());
if (username != utils::ToLowerCase(username)) continue;
if (it->second != utils::ToLowerCase(it->second)) continue;
if (it->second == rolename) {
if (auto user = GetUser(username)) {
auto user = GetUser(username);
if (user) {
ret.push_back(std::move(*user));
} else {
throw AuthException("Couldn't load user '{}'!", username);
@@ -405,7 +316,8 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
#ifdef MG_ENTERPRISE
bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name) {
if (auto user = GetUser(name)) {
auto user = GetUser(name);
if (user) {
if (db == kAllDatabases) {
user->db_access().GrantAll();
} else {
@@ -418,7 +330,8 @@ bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name) {
}
bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name) {
if (auto user = GetUser(name)) {
auto user = GetUser(name);
if (user) {
if (db == kAllDatabases) {
user->db_access().DenyAll();
} else {
@@ -433,15 +346,17 @@ bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name
void Auth::DeleteDatabase(const std::string &db) {
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
auto username = it->first.substr(kUserPrefix.size());
if (auto user = GetUser(username)) {
auto user = GetUser(username);
if (user) {
user->db_access().Delete(db);
SaveUser(*user);
}
}
}
bool Auth::SetMainDatabase(std::string_view db, const std::string &name) {
if (auto user = GetUser(name)) {
bool Auth::SetMainDatabase(const std::string &db, const std::string &name) {
auto user = GetUser(name);
if (user) {
if (!user->db_access().SetDefault(db)) {
throw AuthException("Couldn't set default database '{}' for user '{}'!", db, name);
}
@@ -452,19 +367,4 @@ bool Auth::SetMainDatabase(std::string_view db, const std::string &name) {
}
#endif
bool Auth::NameRegexMatch(const std::string &user_or_role) const {
if (config_.custom_name_regex) {
if (const auto license_check_result =
memgraph::license::global_license_checker.IsEnterpriseValid(memgraph::utils::global_settings);
license_check_result.HasError()) {
throw memgraph::auth::AuthException(
"Custom user/role regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-user-or-role-name-regex\") to its default value (\"{}\") or remove the flag.\n{}",
glue::kDefaultUserRoleRegex,
memgraph::license::LicenseCheckErrorToString(license_check_result.GetError(), "user/role regex"));
}
}
return std::regex_match(user_or_role, config_.name_regex);
}
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -10,13 +10,11 @@
#include <mutex>
#include <optional>
#include <regex>
#include <vector>
#include "auth/exceptions.hpp"
#include "auth/models.hpp"
#include "auth/module.hpp"
#include "glue/auth_global.hpp"
#include "kvstore/kvstore.hpp"
#include "utils/settings.hpp"
@@ -33,40 +31,7 @@ static const constexpr char *const kAllDatabases = "*";
*/
class Auth final {
public:
struct Config {
Config() {}
Config(std::string name_regex, std::string password_regex, bool password_permit_null)
: name_regex_str{std::move(name_regex)},
password_regex_str{std::move(password_regex)},
password_permit_null{password_permit_null},
custom_name_regex{name_regex_str != glue::kDefaultUserRoleRegex},
name_regex{name_regex_str},
custom_password_regex{password_regex_str != glue::kDefaultPasswordRegex},
password_regex{password_regex_str} {}
std::string name_regex_str{glue::kDefaultUserRoleRegex};
std::string password_regex_str{glue::kDefaultPasswordRegex};
bool password_permit_null{true};
private:
friend class Auth;
bool custom_name_regex{false};
std::regex name_regex{name_regex_str};
bool custom_password_regex{false};
std::regex password_regex{password_regex_str};
};
explicit Auth(std::string storage_directory, Config config);
/**
* @brief Set the Config object
*
* @param config
*/
void SetConfig(Config config) {
// NOTE: The Auth class itself is not thread-safe, higher-level code needs to synchronize it when using it.
config_ = std::move(config);
}
explicit Auth(const std::string &storage_directory);
/**
* Authenticates a user using his username and password.
@@ -120,14 +85,6 @@ class Auth final {
*/
bool RemoveUser(const std::string &username);
/**
* @brief
*
* @param user
* @param password
*/
void UpdatePassword(auth::User &user, const std::optional<std::string> &password);
/**
* Gets all users from the storage.
*
@@ -238,24 +195,14 @@ class Auth final {
* @return true on success
* @throw AuthException if unable to find or update the user
*/
bool SetMainDatabase(std::string_view db, const std::string &name);
bool SetMainDatabase(const std::string &db, const std::string &name);
#endif
private:
/**
* @brief
*
* @param user_or_role
* @return true
* @return false
*/
bool NameRegexMatch(const std::string &user_or_role) const;
// Even though the `kvstore::KVStore` class is guaranteed to be thread-safe,
// Auth is not thread-safe because modifying users and roles might require
// more than one operation on the storage.
kvstore::KVStore storage_;
auth::Module module_;
Config config_;
};
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -22,14 +22,10 @@
namespace {
using namespace std::literals;
constexpr auto kHashAlgo = "hash_algo";
constexpr auto kPasswordHash = "password_hash";
inline constexpr std::array password_hash_mappings{
std::pair{"bcrypt"sv, memgraph::auth::PasswordHashAlgorithm::BCRYPT},
std::pair{"sha256"sv, memgraph::auth::PasswordHashAlgorithm::SHA256},
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordHashAlgorithm::SHA256_MULTIPLE}};
inline constexpr std::array password_encryption_mappings{
std::pair{"bcrypt"sv, memgraph::auth::PasswordEncryptionAlgorithm::BCRYPT},
std::pair{"sha256"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256},
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256_MULTIPLE}};
inline constexpr uint64_t ONE_SHA_ITERATION = 1;
inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
@@ -39,7 +35,7 @@ inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
"The password encryption algorithm used for authentication.", {
if (const auto result =
memgraph::utils::IsValidEnumValueString(value, password_hash_mappings);
memgraph::utils::IsValidEnumValueString(value, password_encryption_mappings);
result.HasError()) {
const auto error = result.GetError();
switch (error) {
@@ -49,7 +45,7 @@ DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
}
case memgraph::utils::ValidationError::InvalidValue: {
std::cout << "Invalid value for password encryption algorithm. Allowed values: "
<< memgraph::utils::GetAllowedEnumValuesString(password_hash_mappings)
<< memgraph::utils::GetAllowedEnumValuesString(password_encryption_mappings)
<< std::endl;
break;
}
@@ -62,7 +58,7 @@ DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
namespace memgraph::auth {
namespace BCrypt {
std::string HashPassword(const std::string &password) {
std::string EncryptPassword(const std::string &password) {
char salt[BCRYPT_HASHSIZE];
char hash[BCRYPT_HASHSIZE];
@@ -90,30 +86,16 @@ bool VerifyPassword(const std::string &password, const std::string &hash) {
} // namespace BCrypt
namespace SHA {
namespace {
constexpr auto SHA_LENGTH = 64U;
constexpr auto SALT_SIZE = 16U;
constexpr auto SALT_SIZE_DURABLE = SALT_SIZE * 2;
#if OPENSSL_VERSION_MAJOR >= 3
std::string HashPasswordOpenSSL3(std::string_view password, const uint64_t number_of_iterations,
std::string_view salt) {
std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t number_of_iterations) {
unsigned char hash[SHA256_DIGEST_LENGTH];
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
EVP_MD *md = EVP_MD_fetch(nullptr, "SHA2-256", nullptr);
EVP_DigestInit_ex(ctx, md, nullptr);
if (!salt.empty()) {
DMG_ASSERT(salt.size() == SALT_SIZE);
EVP_DigestUpdate(ctx, salt.data(), salt.size());
}
for (auto i = 0; i < number_of_iterations; i++) {
EVP_DigestUpdate(ctx, password.data(), password.size());
EVP_DigestUpdate(ctx, password.c_str(), password.size());
}
EVP_DigestFinal_ex(ctx, hash, nullptr);
@@ -121,11 +103,6 @@ std::string HashPasswordOpenSSL3(std::string_view password, const uint64_t numbe
EVP_MD_CTX_free(ctx);
std::stringstream result_stream;
for (unsigned char salt_char : salt) {
result_stream << std::hex << std::setw(2) << std::setfill('0') << (((unsigned int)salt_char) & 0xFFU);
}
for (auto hash_char : hash) {
result_stream << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
}
@@ -133,27 +110,17 @@ std::string HashPasswordOpenSSL3(std::string_view password, const uint64_t numbe
return result_stream.str();
}
#else
std::string HashPasswordOpenSSL1_1(std::string_view password, const uint64_t number_of_iterations,
std::string_view salt) {
std::string EncryptPasswordOpenSSL1_1(const std::string &password, const uint64_t number_of_iterations) {
unsigned char hash[SHA256_DIGEST_LENGTH];
SHA256_CTX sha256;
SHA256_Init(&sha256);
if (!salt.empty()) {
DMG_ASSERT(salt.size() == SALT_SIZE);
SHA256_Update(&sha256, salt.data(), salt.size());
}
for (auto i = 0; i < number_of_iterations; i++) {
SHA256_Update(&sha256, password.data(), password.size());
SHA256_Update(&sha256, password.c_str(), password.size());
}
SHA256_Final(hash, &sha256);
std::stringstream ss;
for (unsigned char salt_char : salt) {
ss << std::hex << std::setw(2) << std::setfill('0') << (((unsigned int)salt_char) & 0xFFU);
}
for (auto hash_char : hash) {
ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
}
@@ -162,144 +129,55 @@ std::string HashPasswordOpenSSL1_1(std::string_view password, const uint64_t num
}
#endif
std::string HashPassword(std::string_view password, const uint64_t number_of_iterations, std::string_view salt) {
std::string EncryptPassword(const std::string &password, const uint64_t number_of_iterations) {
#if OPENSSL_VERSION_MAJOR >= 3
return HashPasswordOpenSSL3(password, number_of_iterations, salt);
return EncryptPasswordOpenSSL3(password, number_of_iterations);
#else
return HashPasswordOpenSSL1_1(password, number_of_iterations, salt);
return EncryptPasswordOpenSSL1_1(password, number_of_iterations);
#endif
}
auto ExtractSalt(std::string_view salt_durable) -> std::array<char, SALT_SIZE> {
static_assert(SALT_SIZE_DURABLE % 2 == 0);
static_assert(SALT_SIZE_DURABLE / 2 == SALT_SIZE);
MG_ASSERT(salt_durable.size() == SALT_SIZE_DURABLE);
auto const *b = salt_durable.cbegin();
auto const *const e = salt_durable.cend();
auto salt = std::array<char, SALT_SIZE>{};
auto *inserter = salt.begin();
auto const toval = [](char a) -> uint8_t {
if ('0' <= a && a <= '9') {
return a - '0';
}
if ('a' <= a && a <= 'f') {
return 10 + (a - 'a');
}
MG_ASSERT(false, "Currupt hash, can't extract salt");
__builtin_unreachable();
};
for (; b != e; b += 2, ++inserter) {
*inserter = static_cast<char>(static_cast<uint8_t>(toval(b[0]) << 4U) | toval(b[1]));
}
return salt;
}
bool IsSalted(std::string_view hash) { return hash.size() == SHA_LENGTH + SALT_SIZE_DURABLE; }
bool VerifyPassword(std::string_view password, std::string_view hash, const uint64_t number_of_iterations) {
auto password_hash = std::invoke([&] {
if (hash.size() == SHA_LENGTH) [[unlikely]] {
// Just SHA256
return HashPassword(password, number_of_iterations, {});
} else {
// SHA256 + SALT
MG_ASSERT(IsSalted(hash));
auto const salt_durable = std::string_view{hash.data(), SALT_SIZE_DURABLE};
std::array<char, SALT_SIZE> salt = ExtractSalt(salt_durable);
return HashPassword(password, number_of_iterations, {salt.data(), salt.size()});
}
});
bool VerifyPassword(const std::string &password, const std::string &hash, const uint64_t number_of_iterations) {
auto password_hash = EncryptPassword(password, number_of_iterations);
return password_hash == hash;
}
} // namespace
} // namespace SHA
HashedPassword HashPassword(const std::string &password, std::optional<PasswordHashAlgorithm> override_algo) {
auto const hash_algo = override_algo.value_or(CurrentHashAlgorithm());
auto password_hash = std::invoke([&] {
switch (hash_algo) {
case PasswordHashAlgorithm::BCRYPT: {
return BCrypt::HashPassword(password);
}
case PasswordHashAlgorithm::SHA256:
case PasswordHashAlgorithm::SHA256_MULTIPLE: {
auto gen = std::mt19937(std::random_device{}());
auto salt = std::array<char, SHA::SALT_SIZE>{};
auto dis = std::uniform_int_distribution<unsigned char>(0, 255);
std::generate(salt.begin(), salt.end(), [&]() { return dis(gen); });
auto iterations = (hash_algo == PasswordHashAlgorithm::SHA256) ? ONE_SHA_ITERATION : MULTIPLE_SHA_ITERATIONS;
return SHA::HashPassword(password, iterations, {salt.data(), salt.size()});
}
}
});
return HashedPassword{hash_algo, std::move(password_hash)};
};
bool VerifyPassword(const std::string &password, const std::string &hash) {
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
FLAGS_password_encryption_algorithm, password_encryption_mappings);
namespace {
auto InternalParseHashAlgorithm(std::string_view algo) -> PasswordHashAlgorithm {
auto maybe_parsed = utils::StringToEnum<PasswordHashAlgorithm>(algo, password_hash_mappings);
if (!maybe_parsed) {
throw AuthException("Invalid password encryption '{}'!", algo);
if (!password_encryption_algorithm.has_value()) {
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
return *maybe_parsed;
}
PasswordHashAlgorithm &InternalCurrentHashAlgorithm() {
static auto current = PasswordHashAlgorithm::BCRYPT;
static std::once_flag flag;
std::call_once(flag, [] { current = InternalParseHashAlgorithm(FLAGS_password_encryption_algorithm); });
return current;
}
} // namespace
auto CurrentHashAlgorithm() -> PasswordHashAlgorithm { return InternalCurrentHashAlgorithm(); }
void SetHashAlgorithm(std::string_view algo) {
auto &current = InternalCurrentHashAlgorithm();
current = InternalParseHashAlgorithm(algo);
}
auto AsString(PasswordHashAlgorithm hash_algo) -> std::string_view {
return *utils::EnumToString<PasswordHashAlgorithm>(hash_algo, password_hash_mappings);
}
bool HashedPassword::VerifyPassword(const std::string &password) {
switch (hash_algo) {
case PasswordHashAlgorithm::BCRYPT:
return BCrypt::VerifyPassword(password, password_hash);
case PasswordHashAlgorithm::SHA256:
return SHA::VerifyPassword(password, password_hash, ONE_SHA_ITERATION);
case PasswordHashAlgorithm::SHA256_MULTIPLE:
return SHA::VerifyPassword(password, password_hash, MULTIPLE_SHA_ITERATIONS);
switch (password_encryption_algorithm.value()) {
case PasswordEncryptionAlgorithm::BCRYPT:
return BCrypt::VerifyPassword(password, hash);
case PasswordEncryptionAlgorithm::SHA256:
return SHA::VerifyPassword(password, hash, ONE_SHA_ITERATION);
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
return SHA::VerifyPassword(password, hash, MULTIPLE_SHA_ITERATIONS);
}
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
void to_json(nlohmann::json &j, const HashedPassword &p) {
j = nlohmann::json{{kHashAlgo, p.hash_algo}, {kPasswordHash, p.password_hash}};
}
std::string EncryptPassword(const std::string &password) {
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
FLAGS_password_encryption_algorithm, password_encryption_mappings);
void from_json(const nlohmann::json &j, HashedPassword &p) {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
PasswordHashAlgorithm hash_algo;
j.at(kHashAlgo).get_to(hash_algo);
auto password_hash = j.value(kPasswordHash, std::string());
p = HashedPassword{hash_algo, std::move(password_hash)};
}
if (!password_encryption_algorithm.has_value()) {
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
bool HashedPassword::IsSalted() const {
switch (hash_algo) {
case PasswordHashAlgorithm::BCRYPT:
return true;
case PasswordHashAlgorithm::SHA256:
case PasswordHashAlgorithm::SHA256_MULTIPLE:
return SHA::IsSalted(password_hash);
switch (password_encryption_algorithm.value()) {
case PasswordEncryptionAlgorithm::BCRYPT:
return BCrypt::EncryptPassword(password);
case PasswordEncryptionAlgorithm::SHA256:
return SHA::EncryptPassword(password, ONE_SHA_ITERATION);
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
return SHA::EncryptPassword(password, MULTIPLE_SHA_ITERATIONS);
}
}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -8,45 +8,14 @@
#pragma once
#include <json/json.hpp>
#include <optional>
#include <string>
namespace memgraph::auth {
/// Need to be stable, auth durability depends on this
enum class PasswordHashAlgorithm : uint8_t { BCRYPT = 0, SHA256 = 1, SHA256_MULTIPLE = 2 };
enum class PasswordEncryptionAlgorithm : uint8_t { BCRYPT, SHA256, SHA256_MULTIPLE };
void SetHashAlgorithm(std::string_view algo);
/// @throw AuthException if unable to encrypt the password.
std::string EncryptPassword(const std::string &password);
auto CurrentHashAlgorithm() -> PasswordHashAlgorithm;
auto AsString(PasswordHashAlgorithm hash_algo) -> std::string_view;
struct HashedPassword {
HashedPassword() = default;
HashedPassword(PasswordHashAlgorithm hash_algo, std::string password_hash)
: hash_algo{hash_algo}, password_hash{std::move(password_hash)} {}
HashedPassword(HashedPassword const &) = default;
HashedPassword(HashedPassword &&) = default;
HashedPassword &operator=(HashedPassword const &) = default;
HashedPassword &operator=(HashedPassword &&) = default;
friend bool operator==(HashedPassword const &, HashedPassword const &) = default;
bool VerifyPassword(const std::string &password);
bool IsSalted() const;
auto HashAlgo() const -> PasswordHashAlgorithm { return hash_algo; }
friend void to_json(nlohmann::json &j, const HashedPassword &p);
friend void from_json(const nlohmann::json &j, HashedPassword &p);
private:
PasswordHashAlgorithm hash_algo{PasswordHashAlgorithm::BCRYPT};
std::string password_hash{};
};
/// @throw AuthException if unable to hash the password.
HashedPassword HashPassword(const std::string &password, std::optional<PasswordHashAlgorithm> override_algo = {});
/// @throw AuthException if unable to verify the password.
bool VerifyPassword(const std::string &password, const std::string &hash);
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -9,6 +9,7 @@
#include "auth/models.hpp"
#include <cstdint>
#include <regex>
#include <utility>
#include <gflags/gflags.h>
@@ -20,26 +21,22 @@
#include "query/constants.hpp"
#include "spdlog/spdlog.h"
#include "utils/cast.hpp"
#include "utils/logging.hpp"
#include "utils/settings.hpp"
#include "utils/string.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(auth_password_permit_null, true, "Set to false to disable null passwords.");
inline constexpr std::string_view default_password_regex = ".+";
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
"The regular expression that should be used to match the entire "
"entered password to ensure its strength.");
namespace memgraph::auth {
namespace {
constexpr auto kRoleName = "rolename";
constexpr auto kPermissions = "permissions";
constexpr auto kGrants = "grants";
constexpr auto kDenies = "denies";
constexpr auto kUsername = "username";
constexpr auto kPasswordHash = "password_hash";
#ifdef MG_ENTERPRISE
constexpr auto kGlobalPermission = "global_permission";
constexpr auto kFineGrainedAccessHandler = "fine_grained_access_handler";
constexpr auto kAllowAll = "allow_all";
constexpr auto kDefault = "default";
constexpr auto kDatabases = "databases";
#endif
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
Permission::CREATE,
@@ -65,8 +62,7 @@ const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
Permission::TRANSACTION_MANAGEMENT,
Permission::STORAGE_MODE,
Permission::MULTI_DATABASE_EDIT,
Permission::MULTI_DATABASE_USE,
Permission::COORDINATOR};
Permission::MULTI_DATABASE_USE};
} // namespace
@@ -122,8 +118,6 @@ std::string PermissionToString(Permission permission) {
return "MULTI_DATABASE_EDIT";
case Permission::MULTI_DATABASE_USE:
return "MULTI_DATABASE_USE";
case Permission::COORDINATOR:
return "COORDINATOR";
}
}
@@ -248,9 +242,8 @@ std::vector<Permission> Permissions::GetDenies() const {
nlohmann::json Permissions::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data[kGrants] = grants_;
data[kDenies] = denies_;
data["grants"] = grants_;
data["denies"] = denies_;
return data;
}
@@ -258,10 +251,10 @@ Permissions Permissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!");
}
if (!data[kGrants].is_number_unsigned() || !data[kDenies].is_number_unsigned()) {
if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) {
throw AuthException("Couldn't load permissions data!");
}
return Permissions{data[kGrants], data[kDenies]};
return Permissions{data["grants"], data["denies"]};
}
uint64_t Permissions::grants() const { return grants_; }
@@ -323,8 +316,8 @@ nlohmann::json FineGrainedAccessPermissions::Serialize() const {
return {};
}
nlohmann::json data = nlohmann::json::object();
data[kPermissions] = permissions_;
data[kGlobalPermission] = global_permission_.has_value() ? global_permission_.value() : -1;
data["permissions"] = permissions_;
data["global_permission"] = global_permission_.has_value() ? global_permission_.value() : -1;
return data;
}
@@ -337,13 +330,13 @@ FineGrainedAccessPermissions FineGrainedAccessPermissions::Deserialize(const nlo
}
std::optional<uint64_t> global_permission;
if (data[kGlobalPermission].empty() || data[kGlobalPermission] == -1) {
if (data["global_permission"].empty() || data["global_permission"] == -1) {
global_permission = std::nullopt;
} else {
global_permission = data[kGlobalPermission];
global_permission = data["global_permission"];
}
return FineGrainedAccessPermissions(data[kPermissions], global_permission);
return FineGrainedAccessPermissions(data["permissions"], global_permission);
}
const std::unordered_map<std::string, uint64_t> &FineGrainedAccessPermissions::GetPermissions() const {
@@ -449,13 +442,13 @@ const FineGrainedAccessPermissions &Role::GetFineGrainedAccessEdgeTypePermission
nlohmann::json Role::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data[kRoleName] = rolename_;
data[kPermissions] = permissions_.Serialize();
data["rolename"] = rolename_;
data["permissions"] = permissions_.Serialize();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data[kFineGrainedAccessHandler] = fine_grained_access_handler_.Serialize();
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
} else {
data[kFineGrainedAccessHandler] = {};
data["fine_grained_access_handler"] = {};
}
#endif
return data;
@@ -465,21 +458,21 @@ Role Role::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load role data!");
}
if (!data[kRoleName].is_string() || !data[kPermissions].is_object()) {
if (!data["rolename"].is_string() || !data["permissions"].is_object()) {
throw AuthException("Couldn't load role data!");
}
auto permissions = Permissions::Deserialize(data[kPermissions]);
auto permissions = Permissions::Deserialize(data["permissions"]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
FineGrainedAccessHandler fine_grained_access_handler;
// We can have an empty fine_grained if the user was created without a valid license
if (data[kFineGrainedAccessHandler].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
if (data["fine_grained_access_handler"].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
}
return {data[kRoleName], permissions, std::move(fine_grained_access_handler)};
return {data["rolename"], permissions, std::move(fine_grained_access_handler)};
}
#endif
return {data[kRoleName], permissions};
return {data["rolename"], permissions};
}
bool operator==(const Role &first, const Role &second) {
@@ -493,13 +486,13 @@ bool operator==(const Role &first, const Role &second) {
}
#ifdef MG_ENTERPRISE
void Databases::Add(std::string_view db) {
void Databases::Add(const std::string &db) {
if (allow_all_) {
grants_dbs_.clear();
allow_all_ = false;
}
grants_dbs_.emplace(db);
denies_dbs_.erase(std::string{db}); // TODO: C++23 use transparent key compare
denies_dbs_.erase(db);
}
void Databases::Remove(const std::string &db) {
@@ -530,13 +523,13 @@ void Databases::DenyAll() {
denies_dbs_.clear();
}
bool Databases::SetDefault(std::string_view db) {
bool Databases::SetDefault(const std::string &db) {
if (!Contains(db)) return false;
default_db_ = db;
return true;
}
[[nodiscard]] bool Databases::Contains(std::string_view db) const {
[[nodiscard]] bool Databases::Contains(const std::string &db) const {
return !denies_dbs_.contains(db) && (allow_all_ || grants_dbs_.contains(db));
}
@@ -549,10 +542,10 @@ const std::string &Databases::GetDefault() const {
nlohmann::json Databases::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data[kGrants] = grants_dbs_;
data[kDenies] = denies_dbs_;
data[kAllowAll] = allow_all_;
data[kDefault] = default_db_;
data["grants"] = grants_dbs_;
data["denies"] = denies_dbs_;
data["allow_all"] = allow_all_;
data["default"] = default_db_;
return data;
}
@@ -560,22 +553,22 @@ Databases Databases::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load database data!");
}
if (!data[kGrants].is_structured() || !data[kDenies].is_structured() || !data[kAllowAll].is_boolean() ||
!data[kDefault].is_string()) {
if (!data["grants"].is_structured() || !data["denies"].is_structured() || !data["allow_all"].is_boolean() ||
!data["default"].is_string()) {
throw AuthException("Couldn't load database data!");
}
return {data[kAllowAll], data[kGrants], data[kDenies], data[kDefault]};
return {data["allow_all"], data["grants"], data["denies"], data["default"]};
}
#endif
User::User() = default;
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions)
User::User(const std::string &username, std::string password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(std::move(password_hash)), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
User::User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions,
User::User(const std::string &username, std::string password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
: username_(utils::ToLowerCase(username)),
password_hash_(std::move(password_hash)),
@@ -585,16 +578,38 @@ User::User(const std::string &username, std::optional<HashedPassword> password_h
#endif
bool User::CheckPassword(const std::string &password) {
return password_hash_ ? password_hash_->VerifyPassword(password) : true;
if (password_hash_.empty()) return true;
return VerifyPassword(password, password_hash_);
}
void User::UpdatePassword(const std::optional<std::string> &password,
std::optional<PasswordHashAlgorithm> algo_override) {
void User::UpdatePassword(const std::optional<std::string> &password) {
if (!password) {
password_hash_.reset();
if (!FLAGS_auth_password_permit_null) {
throw AuthException("Null passwords aren't permitted!");
}
password_hash_ = "";
return;
}
password_hash_ = HashPassword(*password, algo_override);
if (FLAGS_auth_password_strength_regex != default_password_regex) {
if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
license_check_result.HasError()) {
throw AuthException(
"Custom password regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
default_password_regex,
license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
}
}
std::regex re(FLAGS_auth_password_strength_regex);
if (!std::regex_match(*password, re)) {
throw AuthException(
"The user password doesn't conform to the required strength! Regex: "
"\"{}\"",
FLAGS_auth_password_strength_regex);
}
password_hash_ = EncryptPassword(*password);
}
void User::SetRole(const Role &role) { role_.emplace(role); }
@@ -653,20 +668,16 @@ const Role *User::role() const {
nlohmann::json User::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data[kUsername] = username_;
if (password_hash_.has_value()) {
data[kPasswordHash] = *password_hash_;
} else {
data[kPasswordHash] = nullptr;
}
data[kPermissions] = permissions_.Serialize();
data["username"] = username_;
data["password_hash"] = password_hash_;
data["permissions"] = permissions_.Serialize();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data[kFineGrainedAccessHandler] = fine_grained_access_handler_.Serialize();
data[kDatabases] = database_access_.Serialize();
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
data["databases"] = database_access_.Serialize();
} else {
data[kFineGrainedAccessHandler] = {};
data[kDatabases] = {};
data["fine_grained_access_handler"] = {};
data["databases"] = {};
}
#endif
// The role shouldn't be serialized here, it is stored as a foreign key.
@@ -677,23 +688,15 @@ User User::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load user data!");
}
auto password_hash_json = data[kPasswordHash];
if (!data[kUsername].is_string() || !(password_hash_json.is_object() || password_hash_json.is_null()) ||
!data[kPermissions].is_object()) {
if (!data["username"].is_string() || !data["password_hash"].is_string() || !data["permissions"].is_object()) {
throw AuthException("Couldn't load user data!");
}
std::optional<HashedPassword> password_hash{};
if (password_hash_json.is_object()) {
password_hash = password_hash_json.get<HashedPassword>();
}
auto permissions = Permissions::Deserialize(data[kPermissions]);
auto permissions = Permissions::Deserialize(data["permissions"]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
Databases db_access;
if (data[kDatabases].is_structured()) {
db_access = Databases::Deserialize(data[kDatabases]);
if (data["databases"].is_structured()) {
db_access = Databases::Deserialize(data["databases"]);
} else {
// Back-compatibility
spdlog::warn("User without specified database access. Given access to the default database.");
@@ -702,13 +705,13 @@ User User::Deserialize(const nlohmann::json &data) {
}
FineGrainedAccessHandler fine_grained_access_handler;
// We can have an empty fine_grained if the user was created without a valid license
if (data[kFineGrainedAccessHandler].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
if (data["fine_grained_access_handler"].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
}
return {data[kUsername], std::move(password_hash), permissions, std::move(fine_grained_access_handler), db_access};
return {data["username"], data["password_hash"], permissions, std::move(fine_grained_access_handler), db_access};
}
#endif
return {data[kUsername], std::move(password_hash), permissions};
return {data["username"], data["password_hash"], permissions};
}
bool operator==(const User &first, const User &second) {

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -15,7 +15,6 @@
#include <json/json.hpp>
#include <utility>
#include "crypto.hpp"
#include "dbms/constants.hpp"
#include "utils/logging.hpp"
@@ -49,7 +48,6 @@ enum class Permission : uint64_t {
STORAGE_MODE = 1U << 22U,
MULTI_DATABASE_EDIT = 1U << 23U,
MULTI_DATABASE_USE = 1U << 24U,
COORDINATOR = 1U << 25U,
};
// clang-format on
@@ -248,7 +246,7 @@ bool operator==(const Role &first, const Role &second);
#ifdef MG_ENTERPRISE
class Databases final {
public:
Databases() : grants_dbs_{std::string{dbms::kDefaultDB}}, allow_all_(false), default_db_(dbms::kDefaultDB) {}
Databases() : grants_dbs_({dbms::kDefaultDB}), allow_all_(false), default_db_(dbms::kDefaultDB) {}
Databases(const Databases &) = default;
Databases &operator=(const Databases &) = default;
@@ -261,7 +259,7 @@ class Databases final {
*
* @param db name of the database to grant access to
*/
void Add(std::string_view db);
void Add(const std::string &db);
/**
* @brief Remove database to the list of granted access.
@@ -293,7 +291,7 @@ class Databases final {
/**
* @brief Set the default database.
*/
bool SetDefault(std::string_view db);
bool SetDefault(const std::string &db);
/**
* @brief Checks if access is grated to the database.
@@ -301,7 +299,7 @@ class Databases final {
* @param db name of the database
* @return true if allow_all and not denied or granted
*/
bool Contains(std::string_view db) const;
bool Contains(const std::string &db) const;
bool GetAllowAll() const { return allow_all_; }
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
@@ -314,7 +312,7 @@ class Databases final {
private:
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
std::string default_db = std::string{dbms::kDefaultDB})
std::string default_db = dbms::kDefaultDB)
: grants_dbs_(std::move(grant)),
denies_dbs_(std::move(deny)),
allow_all_(allow_all),
@@ -333,9 +331,9 @@ class User final {
User();
explicit User(const std::string &username);
User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions);
User(const std::string &username, std::string password_hash, const Permissions &permissions);
#ifdef MG_ENTERPRISE
User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions,
User(const std::string &username, std::string password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
#endif
User(const User &) = default;
@@ -347,18 +345,8 @@ class User final {
/// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password);
bool UpgradeHash(const std::string password) {
if (!password_hash_) return false;
if (password_hash_->IsSalted()) return false;
auto const algo = password_hash_->HashAlgo();
UpdatePassword(password, algo);
return true;
}
/// @throw AuthException if unable to set the password.
void UpdatePassword(const std::optional<std::string> &password = {},
std::optional<PasswordHashAlgorithm> algo_override = std::nullopt);
void UpdatePassword(const std::optional<std::string> &password = std::nullopt);
void SetRole(const Role &role);
@@ -393,7 +381,7 @@ class User final {
private:
std::string username_;
std::optional<HashedPassword> password_hash_;
std::string password_hash_;
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -170,7 +170,6 @@ inline State HandleFailure(TSession &session, const std::exception &e) {
spdlog::trace("Error trace: {}", p->trace());
}
session.encoder_buffer_.Clear();
auto code_message = ExceptionToErrorMessage(e);
bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
if (!fail_sent) {

View File

@@ -44,7 +44,7 @@ class ResultStreamFaker {
std::vector<memgraph::communication::bolt::Value> bvalues;
bvalues.reserve(values.size());
for (const auto &value : values) {
auto maybe_value = memgraph::glue::ToBoltValue(value, store_, memgraph::storage::View::NEW);
auto maybe_value = memgraph::glue::ToBoltValue(value, *store_, memgraph::storage::View::NEW);
MG_ASSERT(maybe_value.HasValue());
bvalues.push_back(std::move(*maybe_value));
}
@@ -56,7 +56,7 @@ class ResultStreamFaker {
void Summary(const std::map<std::string, memgraph::query::TypedValue> &summary) {
std::map<std::string, memgraph::communication::bolt::Value> bsummary;
for (const auto &item : summary) {
auto maybe_value = memgraph::glue::ToBoltValue(item.second, store_, memgraph::storage::View::NEW);
auto maybe_value = memgraph::glue::ToBoltValue(item.second, *store_, memgraph::storage::View::NEW);
MG_ASSERT(maybe_value.HasValue());
bsummary.insert({item.first, std::move(*maybe_value)});
}

View File

@@ -1,29 +0,0 @@
add_library(mg-coordination STATIC)
add_library(mg::coordination ALIAS mg-coordination)
target_sources(mg-coordination
PUBLIC
include/coordination/coordinator_client.hpp
include/coordination/coordinator_state.hpp
include/coordination/coordinator_rpc.hpp
include/coordination/coordinator_server.hpp
include/coordination/coordinator_config.hpp
include/coordination/coordinator_exceptions.hpp
include/coordination/coordinator_instance.hpp
include/coordination/coordinator_slk.hpp
include/coordination/coordinator_data.hpp
include/coordination/constants.hpp
include/coordination/failover_status.hpp
include/coordination/coordinator_cluster_config.hpp
PRIVATE
coordinator_client.cpp
coordinator_state.cpp
coordinator_rpc.cpp
coordinator_server.cpp
coordinator_data.cpp
)
target_include_directories(mg-coordination PUBLIC include)
target_link_libraries(mg-coordination
PUBLIC mg::utils mg::rpc mg::slk mg::io mg::repl_coord_glue lib::rangev3
)

View File

@@ -1,109 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_client.hpp"
#include "coordination/coordinator_config.hpp"
#include "coordination/coordinator_rpc.hpp"
#include "replication_coordination_glue/messages.hpp"
namespace memgraph::coordination {
namespace {
auto CreateClientContext(const memgraph::coordination::CoordinatorClientConfig &config)
-> communication::ClientContext {
return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file}
: communication::ClientContext{};
}
} // namespace
CoordinatorClient::CoordinatorClient(CoordinatorData *coord_data, CoordinatorClientConfig config,
HealthCheckCallback succ_cb, HealthCheckCallback fail_cb)
: rpc_context_{CreateClientContext(config)},
rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port),
&rpc_context_},
config_{std::move(config)},
coord_data_{coord_data},
succ_cb_{std::move(succ_cb)},
fail_cb_{std::move(fail_cb)} {}
auto CoordinatorClient::InstanceName() const -> std::string { return config_.instance_name; }
auto CoordinatorClient::SocketAddress() const -> std::string { return rpc_client_.Endpoint().SocketAddress(); }
void CoordinatorClient::StartFrequentCheck() {
MG_ASSERT(config_.health_check_frequency_sec > std::chrono::seconds(0),
"Health check frequency must be greater than 0");
instance_checker_.Run(
"Coord checker", config_.health_check_frequency_sec, [this, instance_name = config_.instance_name] {
try {
spdlog::trace("Sending frequent heartbeat to machine {} on {}", instance_name,
rpc_client_.Endpoint().SocketAddress());
auto stream{rpc_client_.Stream<memgraph::replication_coordination_glue::FrequentHeartbeatRpc>()};
stream.AwaitResponse();
succ_cb_(coord_data_, instance_name);
} catch (const rpc::RpcFailedException &) {
fail_cb_(coord_data_, instance_name);
}
});
}
void CoordinatorClient::StopFrequentCheck() { instance_checker_.Stop(); }
void CoordinatorClient::PauseFrequentCheck() { instance_checker_.Pause(); }
void CoordinatorClient::ResumeFrequentCheck() { instance_checker_.Resume(); }
auto CoordinatorClient::SetSuccCallback(HealthCheckCallback succ_cb) -> void { succ_cb_ = std::move(succ_cb); }
auto CoordinatorClient::SetFailCallback(HealthCheckCallback fail_cb) -> void { fail_cb_ = std::move(fail_cb); }
auto CoordinatorClient::ReplicationClientInfo() const -> const CoordinatorClientConfig::ReplicationClientInfo & {
return config_.replication_client_info;
}
auto CoordinatorClient::ResetReplicationClientInfo() -> void {
// TODO (antoniofilipovic) Sync with Andi on this one
// config_.replication_client_info.reset();
}
auto CoordinatorClient::SendPromoteReplicaToMainRpc(
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info) const -> bool {
try {
auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(std::move(replication_clients_info))};
if (!stream.AwaitResponse().success) {
spdlog::error("Failed to receive successful RPC failover response!");
return false;
}
return true;
} catch (const rpc::RpcFailedException &) {
spdlog::error("RPC error occurred while sending failover RPC!");
}
return false;
}
auto CoordinatorClient::SendSetToReplicaRpc(CoordinatorClient::ReplClientInfo replication_client_info) const -> bool {
try {
auto stream{rpc_client_.Stream<SetMainToReplicaRpc>(std::move(replication_client_info))};
if (!stream.AwaitResponse().success) {
spdlog::error("Failed to set main to replica!");
return false;
}
spdlog::info("Sent request RPC from coordinator to instance to set it as replica!");
return true;
} catch (const rpc::RpcFailedException &) {
spdlog::error("Failed to send failover RPC from coordinator to new main!");
}
return false;
}
} // namespace memgraph::coordination
#endif

View File

@@ -1,220 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "coordination/coordinator_instance.hpp"
#include "coordination/register_main_replica_coordinator_status.hpp"
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_data.hpp"
#include <range/v3/view.hpp>
#include <shared_mutex>
namespace memgraph::coordination {
CoordinatorData::CoordinatorData() {
auto find_instance = [](CoordinatorData *coord_data, std::string_view instance_name) -> CoordinatorInstance & {
auto instance = std::ranges::find_if(
coord_data->registered_instances_,
[instance_name](const CoordinatorInstance &instance) { return instance.InstanceName() == instance_name; });
MG_ASSERT(instance != coord_data->registered_instances_.end(), "Instance {} not found during callback!",
instance_name);
return *instance;
};
replica_succ_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
auto lock = std::lock_guard{coord_data->coord_data_lock_};
spdlog::trace("Instance {} performing replica successful callback", instance_name);
auto &instance = find_instance(coord_data, instance_name);
MG_ASSERT(instance.IsReplica(), "Instance {} is not a replica!", instance_name);
instance.UpdateLastResponseTime();
};
replica_fail_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
auto lock = std::lock_guard{coord_data->coord_data_lock_};
spdlog::trace("Instance {} performing replica failure callback", instance_name);
auto &instance = find_instance(coord_data, instance_name);
MG_ASSERT(instance.IsReplica(), "Instance {} is not a replica!", instance_name);
instance.UpdateInstanceStatus();
};
main_succ_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
auto lock = std::lock_guard{coord_data->coord_data_lock_};
spdlog::trace("Instance {} performing main successful callback", instance_name);
auto &instance = find_instance(coord_data, instance_name);
MG_ASSERT(instance.IsMain(), "Instance {} is not a main!", instance_name);
instance.UpdateLastResponseTime();
};
main_fail_cb_ = [this, find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
auto lock = std::lock_guard{coord_data->coord_data_lock_};
spdlog::trace("Instance {} performing main failure callback", instance_name);
auto &instance = find_instance(coord_data, instance_name);
MG_ASSERT(instance.IsMain(), "Instance {} is not a main!", instance_name);
if (bool main_alive = instance.UpdateInstanceStatus(); !main_alive) {
spdlog::info("Main instance {} is not alive, starting automatic failover", instance_name);
switch (auto failover_status = DoFailover(); failover_status) {
using enum DoFailoverStatus;
case ALL_REPLICAS_DOWN:
spdlog::warn("Failover aborted since all replicas are down!");
break;
case MAIN_ALIVE:
spdlog::warn("Failover aborted since main is alive!");
break;
case RPC_FAILED:
spdlog::warn("Failover aborted since promoting replica to main failed!");
break;
case SUCCESS:
break;
}
}
};
}
auto CoordinatorData::DoFailover() -> DoFailoverStatus {
using ReplicationClientInfo = CoordinatorClientConfig::ReplicationClientInfo;
auto replica_instances = registered_instances_ | ranges::views::filter(&CoordinatorInstance::IsReplica);
auto chosen_replica_instance = std::ranges::find_if(replica_instances, &CoordinatorInstance::IsAlive);
if (chosen_replica_instance == replica_instances.end()) {
return DoFailoverStatus::ALL_REPLICAS_DOWN;
}
chosen_replica_instance->PrepareForFailover();
std::vector<ReplicationClientInfo> repl_clients_info;
repl_clients_info.reserve(std::ranges::distance(replica_instances));
auto const not_chosen_replica_instance = [&chosen_replica_instance](const CoordinatorInstance &instance) {
return instance != *chosen_replica_instance;
};
auto const not_main = [](const CoordinatorInstance &instance) { return !instance.IsMain(); };
// TODO (antoniofilipovic): Should we send also data on old MAIN???
// TODO: (andi) Don't send replicas which aren't alive
for (const auto &unchosen_replica_instance :
replica_instances | ranges::views::filter(not_chosen_replica_instance) | ranges::views::filter(not_main)) {
repl_clients_info.emplace_back(unchosen_replica_instance.client_.ReplicationClientInfo());
}
if (!chosen_replica_instance->client_.SendPromoteReplicaToMainRpc(std::move(repl_clients_info))) {
chosen_replica_instance->RestoreAfterFailedFailover();
return DoFailoverStatus::RPC_FAILED;
}
auto old_main = std::ranges::find_if(registered_instances_, &CoordinatorInstance::IsMain);
// TODO: (andi) For performing restoration we will have to improve this
old_main->client_.PauseFrequentCheck();
chosen_replica_instance->PostFailover(main_succ_cb_, main_fail_cb_);
return DoFailoverStatus::SUCCESS;
}
auto CoordinatorData::ShowInstances() const -> std::vector<CoordinatorInstanceStatus> {
std::vector<CoordinatorInstanceStatus> instances_status;
instances_status.reserve(registered_instances_.size());
auto const stringify_repl_role = [](const CoordinatorInstance &instance) -> std::string {
if (!instance.IsAlive()) return "";
if (instance.IsMain()) return "main";
return "replica";
};
auto const instance_to_status =
[&stringify_repl_role](const CoordinatorInstance &instance) -> CoordinatorInstanceStatus {
return {.instance_name = instance.InstanceName(),
.socket_address = instance.SocketAddress(),
.replication_role = stringify_repl_role(instance),
.is_alive = instance.IsAlive()};
};
{
auto lock = std::shared_lock{coord_data_lock_};
std::ranges::transform(registered_instances_, std::back_inserter(instances_status), instance_to_status);
}
return instances_status;
}
auto CoordinatorData::SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus {
auto lock = std::lock_guard{coord_data_lock_};
// Find replica we already registered
auto registered_replica = std::find_if(
registered_instances_.begin(), registered_instances_.end(),
[instance_name](const CoordinatorInstance &instance) { return instance.InstanceName() == instance_name; });
// if replica not found...
if (registered_replica == registered_instances_.end()) {
spdlog::error("You didn't register instance with given name {}", instance_name);
return SetInstanceToMainCoordinatorStatus::NO_INSTANCE_WITH_NAME;
}
registered_replica->client_.PauseFrequentCheck();
std::vector<CoordinatorClientConfig::ReplicationClientInfo> repl_clients_info;
repl_clients_info.reserve(registered_instances_.size() - 1);
std::ranges::for_each(registered_instances_,
[registered_replica, &repl_clients_info](const CoordinatorInstance &replica) {
if (replica != *registered_replica) {
repl_clients_info.emplace_back(replica.client_.ReplicationClientInfo());
}
});
// PROMOTE REPLICA TO MAIN
// THIS SHOULD FAIL HERE IF IT IS DOWN
if (auto result = registered_replica->client_.SendPromoteReplicaToMainRpc(std::move(repl_clients_info)); !result) {
registered_replica->client_.ResumeFrequentCheck();
return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
}
registered_replica->client_.SetSuccCallback(main_succ_cb_);
registered_replica->client_.SetFailCallback(main_fail_cb_);
registered_replica->replication_role_ = replication_coordination_glue::ReplicationRole::MAIN;
registered_replica->client_.ResumeFrequentCheck();
return SetInstanceToMainCoordinatorStatus::SUCCESS;
}
auto CoordinatorData::RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus {
auto lock = std::lock_guard{coord_data_lock_};
if (std::ranges::any_of(registered_instances_, [&config](const CoordinatorInstance &instance) {
return instance.InstanceName() == config.instance_name;
})) {
return RegisterInstanceCoordinatorStatus::NAME_EXISTS;
}
if (std::ranges::any_of(registered_instances_, [&config](const CoordinatorInstance &instance) {
spdlog::trace("Comparing {} with {}", instance.SocketAddress(), config.SocketAddress());
return instance.SocketAddress() == config.SocketAddress();
})) {
return RegisterInstanceCoordinatorStatus::END_POINT_EXISTS;
}
CoordinatorClientConfig::ReplicationClientInfo replication_client_info_copy = config.replication_client_info;
// TODO (antoniofilipovic) create and then push back
auto *instance = &registered_instances_.emplace_back(this, std::move(config), replica_succ_cb_, replica_fail_cb_,
replication_coordination_glue::ReplicationRole::REPLICA);
if (auto res = instance->client_.SendSetToReplicaRpc(replication_client_info_copy); !res) {
return RegisterInstanceCoordinatorStatus::RPC_FAILED;
}
instance->client_.StartFrequentCheck();
return RegisterInstanceCoordinatorStatus::SUCCESS;
}
} // namespace memgraph::coordination
#endif

View File

@@ -1,107 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_rpc.hpp"
#include "coordination/coordinator_slk.hpp"
#include "slk/serialization.hpp"
namespace memgraph {
namespace coordination {
void PromoteReplicaToMainReq::Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void PromoteReplicaToMainReq::Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
void PromoteReplicaToMainRes::Save(const PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void PromoteReplicaToMainRes::Load(PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
void SetMainToReplicaReq::Save(const SetMainToReplicaReq &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void SetMainToReplicaReq::Load(SetMainToReplicaReq *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
void SetMainToReplicaRes::Save(const SetMainToReplicaRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void SetMainToReplicaRes::Load(SetMainToReplicaRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
} // namespace coordination
constexpr utils::TypeInfo coordination::PromoteReplicaToMainReq::kType{utils::TypeId::COORD_FAILOVER_REQ,
"CoordPromoteReplicaToMainReq", nullptr};
constexpr utils::TypeInfo coordination::PromoteReplicaToMainRes::kType{utils::TypeId::COORD_FAILOVER_RES,
"CoordPromoteReplicaToMainRes", nullptr};
constexpr utils::TypeInfo coordination::SetMainToReplicaReq::kType{utils::TypeId::COORD_SET_REPL_MAIN_REQ,
"CoordSetReplMainReq", nullptr};
constexpr utils::TypeInfo coordination::SetMainToReplicaRes::kType{utils::TypeId::COORD_SET_REPL_MAIN_RES,
"CoordSetReplMainRes", nullptr};
namespace slk {
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.success, builder);
}
void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(&self->success, reader);
}
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.replication_clients_info, builder);
}
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(&self->replication_clients_info, reader);
}
void Save(const memgraph::coordination::SetMainToReplicaReq &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.replication_client_info, builder);
}
void Load(memgraph::coordination::SetMainToReplicaReq *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(&self->replication_client_info, reader);
}
void Save(const memgraph::coordination::SetMainToReplicaRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.success, builder);
}
void Load(memgraph::coordination::SetMainToReplicaRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(&self->success, reader);
}
} // namespace slk
} // namespace memgraph
#endif

View File

@@ -1,57 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_server.hpp"
#include "replication_coordination_glue/messages.hpp"
namespace memgraph::coordination {
namespace {
auto CreateServerContext(const memgraph::coordination::CoordinatorServerConfig &config)
-> communication::ServerContext {
return (config.ssl) ? communication::ServerContext{config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file,
config.ssl->verify_peer}
: communication::ServerContext{};
}
// NOTE: The coordinator server doesn't more than 1 processing thread - each replica can
// have only a single coordinator server. Also, the single-threaded guarantee
// simplifies the rest of the implementation.
constexpr auto kCoordinatorServerThreads = 1;
} // namespace
CoordinatorServer::CoordinatorServer(const CoordinatorServerConfig &config)
: rpc_server_context_{CreateServerContext(config)},
rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_,
kCoordinatorServerThreads} {
rpc_server_.Register<replication_coordination_glue::FrequentHeartbeatRpc>([](auto *req_reader, auto *res_builder) {
spdlog::debug("Received FrequentHeartbeatRpc on coordinator server");
replication_coordination_glue::FrequentHeartbeatHandler(req_reader, res_builder);
});
}
CoordinatorServer::~CoordinatorServer() {
if (rpc_server_.IsRunning()) {
auto const &endpoint = rpc_server_.endpoint();
spdlog::trace("Closing coordinator server on {}:{}", endpoint.address, endpoint.port);
rpc_server_.Shutdown();
}
rpc_server_.AwaitShutdown();
}
bool CoordinatorServer::Start() { return rpc_server_.Start(); }
} // namespace memgraph::coordination
#endif

View File

@@ -1,89 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_state.hpp"
#include "coordination/coordinator_config.hpp"
#include "coordination/register_main_replica_coordinator_status.hpp"
#include "flags/replication.hpp"
#include "spdlog/spdlog.h"
#include "utils/logging.hpp"
#include "utils/variant_helpers.hpp"
#include <algorithm>
namespace memgraph::coordination {
CoordinatorState::CoordinatorState() {
MG_ASSERT(!(FLAGS_coordinator && FLAGS_coordinator_server_port),
"Instance cannot be a coordinator and have registered coordinator server.");
spdlog::info("Executing coordinator constructor");
if (FLAGS_coordinator_server_port) {
spdlog::info("Coordinator server port set");
auto const config = CoordinatorServerConfig{
.ip_address = kDefaultReplicationServerIp,
.port = static_cast<uint16_t>(FLAGS_coordinator_server_port),
};
spdlog::info("Executing coordinator constructor main replica");
data_ = CoordinatorMainReplicaData{.coordinator_server_ = std::make_unique<CoordinatorServer>(config)};
}
}
auto CoordinatorState::RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus {
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
"Coordinator cannot register replica since variant holds wrong alternative");
return std::visit(
memgraph::utils::Overloaded{
[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
return RegisterInstanceCoordinatorStatus::NOT_COORDINATOR;
},
[config](CoordinatorData &coordinator_data) { return coordinator_data.RegisterInstance(config); }},
data_);
}
auto CoordinatorState::SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus {
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
"Coordinator cannot register replica since variant holds wrong alternative");
return std::visit(
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
return SetInstanceToMainCoordinatorStatus::NOT_COORDINATOR;
},
[&instance_name](CoordinatorData &coordinator_data) {
return coordinator_data.SetInstanceToMain(instance_name);
}},
data_);
}
auto CoordinatorState::ShowInstances() const -> std::vector<CoordinatorInstanceStatus> {
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
"Can't call show instances on data_, as variant holds wrong alternative");
return std::get<CoordinatorData>(data_).ShowInstances();
}
[[nodiscard]] auto CoordinatorState::DoFailover() -> DoFailoverStatus {
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_), "Cannot do failover since variant holds wrong alternative");
auto &coord_state = std::get<CoordinatorData>(data_);
return coord_state.DoFailover();
}
auto CoordinatorState::GetCoordinatorServer() const -> CoordinatorServer & {
MG_ASSERT(std::holds_alternative<CoordinatorMainReplicaData>(data_),
"Cannot get coordinator server since variant holds wrong alternative");
return *std::get<CoordinatorMainReplicaData>(data_).coordinator_server_;
}
} // namespace memgraph::coordination
#endif

View File

@@ -1,22 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::coordination {
#ifdef MG_EXPERIMENTAL_HIGH_AVAILABILITY
constexpr bool allow_ha = true;
#else
constexpr bool allow_ha = false;
#endif
} // namespace memgraph::coordination

View File

@@ -1,77 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_config.hpp"
#include "rpc/client.hpp"
#include "utils/scheduler.hpp"
namespace memgraph::coordination {
class CoordinatorData;
using HealthCheckCallback = std::function<void(CoordinatorData *, std::string_view)>;
class CoordinatorClient {
public:
using ReplClientInfo = CoordinatorClientConfig::ReplicationClientInfo;
using ReplicationClientsInfo = std::vector<ReplClientInfo>;
explicit CoordinatorClient(CoordinatorData *coord_data_, CoordinatorClientConfig config, HealthCheckCallback succ_cb,
HealthCheckCallback fail_cb);
~CoordinatorClient() = default;
CoordinatorClient(CoordinatorClient &) = delete;
CoordinatorClient &operator=(CoordinatorClient const &) = delete;
CoordinatorClient(CoordinatorClient &&) noexcept = delete;
CoordinatorClient &operator=(CoordinatorClient &&) noexcept = delete;
void StartFrequentCheck();
void StopFrequentCheck();
void PauseFrequentCheck();
void ResumeFrequentCheck();
auto InstanceName() const -> std::string;
auto SocketAddress() const -> std::string;
auto SendPromoteReplicaToMainRpc(ReplicationClientsInfo replication_clients_info) const -> bool;
auto ReplicationClientInfo() const -> const ReplClientInfo &;
auto ResetReplicationClientInfo() -> void;
auto SendSetToReplicaRpc(ReplClientInfo replication_client_info) const -> bool;
auto SetSuccCallback(HealthCheckCallback succ_cb) -> void;
auto SetFailCallback(HealthCheckCallback fail_cb) -> void;
friend bool operator==(CoordinatorClient const &first, CoordinatorClient const &second) {
return first.config_ == second.config_;
}
private:
utils::Scheduler instance_checker_;
// TODO: (andi) Pimpl?
communication::ClientContext rpc_context_;
mutable rpc::Client rpc_client_;
CoordinatorClientConfig config_;
CoordinatorData *coord_data_;
HealthCheckCallback succ_cb_;
HealthCheckCallback fail_cb_;
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,22 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
namespace memgraph::coordination {
struct CoordinatorClusterConfig {
static constexpr int alive_response_time_difference_sec_{5};
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,78 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "replication_coordination_glue/mode.hpp"
#include <chrono>
#include <cstdint>
#include <optional>
#include <string>
namespace memgraph::coordination {
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
struct CoordinatorClientConfig {
std::string instance_name;
std::string ip_address;
uint16_t port{};
std::chrono::seconds health_check_frequency_sec{1};
auto SocketAddress() const -> std::string { return ip_address + ":" + std::to_string(port); }
// Info which coordinator will send to new main when performing failover
struct ReplicationClientInfo {
// Must be the same as CoordinatorClientConfig's instance_name
std::string instance_name;
replication_coordination_glue::ReplicationMode replication_mode{};
std::string replication_ip_address;
uint16_t replication_port{};
friend bool operator==(ReplicationClientInfo const &, ReplicationClientInfo const &) = default;
};
// Each instance has replication config in case it fails
ReplicationClientInfo replication_client_info;
struct SSL {
std::string key_file;
std::string cert_file;
friend bool operator==(const SSL &, const SSL &) = default;
};
std::optional<SSL> ssl;
friend bool operator==(CoordinatorClientConfig const &, CoordinatorClientConfig const &) = default;
};
struct CoordinatorServerConfig {
std::string ip_address;
uint16_t port{};
struct SSL {
std::string key_file;
std::string cert_file;
std::string ca_file;
bool verify_peer{};
friend bool operator==(SSL const &, SSL const &) = default;
};
std::optional<SSL> ssl;
friend bool operator==(CoordinatorServerConfig const &, CoordinatorServerConfig const &) = default;
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,49 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_instance.hpp"
#include "coordination/coordinator_instance_status.hpp"
#include "coordination/coordinator_server.hpp"
#include "coordination/failover_status.hpp"
#include "coordination/register_main_replica_coordinator_status.hpp"
#include "utils/rw_lock.hpp"
#include <list>
namespace memgraph::coordination {
class CoordinatorData {
public:
CoordinatorData();
[[nodiscard]] auto DoFailover() -> DoFailoverStatus;
[[nodiscard]] auto RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
[[nodiscard]] auto SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
auto ShowInstances() const -> std::vector<CoordinatorInstanceStatus>;
private:
mutable utils::RWLock coord_data_lock_{utils::RWLock::Priority::READ};
HealthCheckCallback main_succ_cb_, main_fail_cb_, replica_succ_cb_, replica_fail_cb_;
// Must be std::list because we rely on pointer stability
std::list<CoordinatorInstance> registered_instances_;
};
struct CoordinatorMainReplicaData {
std::unique_ptr<CoordinatorServer> coordinator_server_;
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,32 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "utils/exceptions.hpp"
namespace memgraph::coordination {
class CoordinatorFailoverException final : public utils::BasicException {
public:
explicit CoordinatorFailoverException(const std::string_view what) noexcept
: BasicException("Failover didn't complete successfully: " + std::string(what)) {}
template <class... Args>
explicit CoordinatorFailoverException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
: CoordinatorFailoverException(fmt::format(fmt, std::forward<Args>(args)...)) {}
SPECIALIZE_GET_EXCEPTION_NAME(CoordinatorFailoverException)
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,77 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_client.hpp"
#include "coordination/coordinator_cluster_config.hpp"
#include "replication_coordination_glue/role.hpp"
namespace memgraph::coordination {
class CoordinatorData;
class CoordinatorInstance {
public:
CoordinatorInstance(CoordinatorData *data, CoordinatorClientConfig config, HealthCheckCallback succ_cb,
HealthCheckCallback fail_cb, replication_coordination_glue::ReplicationRole replication_role)
: client_(data, std::move(config), std::move(succ_cb), std::move(fail_cb)),
replication_role_(replication_role),
is_alive_(true) {}
CoordinatorInstance(CoordinatorInstance const &other) = delete;
CoordinatorInstance &operator=(CoordinatorInstance const &other) = delete;
CoordinatorInstance(CoordinatorInstance &&other) noexcept = delete;
CoordinatorInstance &operator=(CoordinatorInstance &&other) noexcept = delete;
~CoordinatorInstance() = default;
auto UpdateInstanceStatus() -> bool {
is_alive_ = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - last_response_time_)
.count() < CoordinatorClusterConfig::alive_response_time_difference_sec_;
return is_alive_;
}
auto UpdateLastResponseTime() -> void { last_response_time_ = std::chrono::system_clock::now(); }
auto InstanceName() const -> std::string { return client_.InstanceName(); }
auto SocketAddress() const -> std::string { return client_.SocketAddress(); }
auto IsAlive() const -> bool { return is_alive_; }
auto IsReplica() const -> bool {
return replication_role_ == replication_coordination_glue::ReplicationRole::REPLICA;
}
auto IsMain() const -> bool { return replication_role_ == replication_coordination_glue::ReplicationRole::MAIN; }
auto PrepareForFailover() -> void { client_.PauseFrequentCheck(); }
auto RestoreAfterFailedFailover() -> void { client_.ResumeFrequentCheck(); }
auto PostFailover(HealthCheckCallback main_succ_cb, HealthCheckCallback main_fail_cb) -> void {
replication_role_ = replication_coordination_glue::ReplicationRole::MAIN;
client_.SetSuccCallback(std::move(main_succ_cb));
client_.SetFailCallback(std::move(main_fail_cb));
// Comment with Andi but we shouldn't delete this, what if this MAIN FAILS AGAIN
// client_.ResetReplicationClientInfo();
client_.ResumeFrequentCheck();
}
CoordinatorClient client_;
replication_coordination_glue::ReplicationRole replication_role_;
std::chrono::system_clock::time_point last_response_time_{};
bool is_alive_{false};
friend bool operator==(CoordinatorInstance const &first, CoordinatorInstance const &second) {
return first.client_ == second.client_ && first.replication_role_ == second.replication_role_;
}
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,31 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "io/network/endpoint.hpp"
#include <string_view>
namespace memgraph::coordination {
struct CoordinatorInstanceStatus {
std::string instance_name;
std::string socket_address;
std::string replication_role;
bool is_alive;
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,104 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_config.hpp"
#include "rpc/messages.hpp"
#include "slk/serialization.hpp"
namespace memgraph::coordination {
struct PromoteReplicaToMainReq {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
static void Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
explicit PromoteReplicaToMainReq(std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
: replication_clients_info(std::move(replication_clients_info)) {}
PromoteReplicaToMainReq() = default;
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info;
};
struct PromoteReplicaToMainRes {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
static void Save(const PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
explicit PromoteReplicaToMainRes(bool success) : success(success) {}
PromoteReplicaToMainRes() = default;
bool success;
};
using PromoteReplicaToMainRpc = rpc::RequestResponse<PromoteReplicaToMainReq, PromoteReplicaToMainRes>;
struct SetMainToReplicaReq {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(SetMainToReplicaReq *self, memgraph::slk::Reader *reader);
static void Save(const SetMainToReplicaReq &self, memgraph::slk::Builder *builder);
explicit SetMainToReplicaReq(CoordinatorClientConfig::ReplicationClientInfo replication_client_info)
: replication_client_info(std::move(replication_client_info)) {}
SetMainToReplicaReq() = default;
CoordinatorClientConfig::ReplicationClientInfo replication_client_info;
};
struct SetMainToReplicaRes {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(SetMainToReplicaRes *self, memgraph::slk::Reader *reader);
static void Save(const SetMainToReplicaRes &self, memgraph::slk::Builder *builder);
explicit SetMainToReplicaRes(bool success) : success(success) {}
SetMainToReplicaRes() = default;
bool success;
};
using SetMainToReplicaRpc = rpc::RequestResponse<SetMainToReplicaReq, SetMainToReplicaRes>;
} // namespace memgraph::coordination
// SLK serialization declarations
namespace memgraph::slk {
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
void Save(const memgraph::coordination::SetMainToReplicaRes &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::SetMainToReplicaRes *self, memgraph::slk::Reader *reader);
void Save(const memgraph::coordination::SetMainToReplicaReq &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::SetMainToReplicaReq *self, memgraph::slk::Reader *reader);
} // namespace memgraph::slk
#endif

View File

@@ -1,44 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_config.hpp"
#include "rpc/server.hpp"
namespace memgraph::coordination {
class CoordinatorServer {
public:
explicit CoordinatorServer(const CoordinatorServerConfig &config);
CoordinatorServer(const CoordinatorServer &) = delete;
CoordinatorServer(CoordinatorServer &&) = delete;
CoordinatorServer &operator=(const CoordinatorServer &) = delete;
CoordinatorServer &operator=(CoordinatorServer &&) = delete;
virtual ~CoordinatorServer();
bool Start();
template <typename TRequestResponse, typename F>
void Register(F &&callback) {
rpc_server_.Register<TRequestResponse>(std::forward<F>(callback));
}
private:
communication::ServerContext rpc_server_context_;
rpc::Server rpc_server_;
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,38 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_config.hpp"
#include "slk/serialization.hpp"
#include "slk/streams.hpp"
namespace memgraph::slk {
using ReplicationClientInfo = coordination::CoordinatorClientConfig::ReplicationClientInfo;
inline void Save(const ReplicationClientInfo &obj, Builder *builder) {
Save(obj.instance_name, builder);
Save(obj.replication_mode, builder);
Save(obj.replication_ip_address, builder);
Save(obj.replication_port, builder);
}
inline void Load(ReplicationClientInfo *obj, Reader *reader) {
Load(&obj->instance_name, reader);
Load(&obj->replication_mode, reader);
Load(&obj->replication_ip_address, reader);
Load(&obj->replication_port, reader);
}
} // namespace memgraph::slk
#endif

View File

@@ -1,53 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_data.hpp"
#include "coordination/coordinator_instance_status.hpp"
#include "coordination/coordinator_server.hpp"
#include "coordination/failover_status.hpp"
#include "coordination/register_main_replica_coordinator_status.hpp"
#include <variant>
namespace memgraph::coordination {
class CoordinatorState {
public:
CoordinatorState();
~CoordinatorState() = default;
CoordinatorState(const CoordinatorState &) = delete;
CoordinatorState &operator=(const CoordinatorState &) = delete;
CoordinatorState(CoordinatorState &&) noexcept = delete;
CoordinatorState &operator=(CoordinatorState &&) noexcept = delete;
[[nodiscard]] auto RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
[[nodiscard]] auto SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
auto ShowInstances() const -> std::vector<CoordinatorInstanceStatus>;
// The client code must check that the server exists before calling this method.
auto GetCoordinatorServer() const -> CoordinatorServer &;
[[nodiscard]] auto DoFailover() -> DoFailoverStatus;
private:
std::variant<CoordinatorData, CoordinatorMainReplicaData> data_;
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,21 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include <cstdint>
namespace memgraph::coordination {
enum class DoFailoverStatus : uint8_t { SUCCESS, ALL_REPLICAS_DOWN, MAIN_ALIVE, RPC_FAILED };
} // namespace memgraph::coordination
#endif

View File

@@ -1,37 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include <cstdint>
namespace memgraph::coordination {
enum class RegisterInstanceCoordinatorStatus : uint8_t {
NAME_EXISTS,
END_POINT_EXISTS,
COULD_NOT_BE_PERSISTED,
NOT_COORDINATOR,
RPC_FAILED,
SUCCESS
};
enum class SetInstanceToMainCoordinatorStatus : uint8_t {
NO_INSTANCE_WITH_NAME,
NOT_COORDINATOR,
SUCCESS,
COULD_NOT_PROMOTE_TO_MAIN,
};
} // namespace memgraph::coordination
#endif

View File

@@ -1,2 +1,3 @@
add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp coordinator_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp coordinator_handlers.cpp)
target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query mg-replication mg-coordination)
add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp)
target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query)

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,8 +13,7 @@
namespace memgraph::dbms {
constexpr std::string_view kDefaultDB = "memgraph"; //!< Name of the default database
constexpr std::string_view kMultiTenantDir = "databases"; //!< Name of the multi-tenant directory
constexpr static const char *kDefaultDB = "memgraph"; //!< Name of the default database
#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY
constexpr bool allow_mt_repl = true;

View File

@@ -1,38 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "coordination/register_main_replica_coordinator_status.hpp"
#ifdef MG_ENTERPRISE
#include "dbms/coordinator_handler.hpp"
#include "dbms/dbms_handler.hpp"
namespace memgraph::dbms {
CoordinatorHandler::CoordinatorHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {}
auto CoordinatorHandler::RegisterInstance(memgraph::coordination::CoordinatorClientConfig config)
-> coordination::RegisterInstanceCoordinatorStatus {
return dbms_handler_.CoordinatorState().RegisterInstance(config);
}
auto CoordinatorHandler::SetInstanceToMain(std::string instance_name)
-> coordination::SetInstanceToMainCoordinatorStatus {
return dbms_handler_.CoordinatorState().SetInstanceToMain(std::move(instance_name));
}
auto CoordinatorHandler::ShowInstances() const -> std::vector<coordination::CoordinatorInstanceStatus> {
return dbms_handler_.CoordinatorState().ShowInstances();
}
} // namespace memgraph::dbms
#endif

View File

@@ -1,47 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "utils/result.hpp"
#include "coordination/coordinator_config.hpp"
#include "coordination/coordinator_instance_status.hpp"
#include "coordination/failover_status.hpp"
#include "coordination/register_main_replica_coordinator_status.hpp"
#include <cstdint>
#include <optional>
#include <vector>
namespace memgraph::dbms {
class DbmsHandler;
class CoordinatorHandler {
public:
explicit CoordinatorHandler(DbmsHandler &dbms_handler);
auto RegisterInstance(coordination::CoordinatorClientConfig config)
-> coordination::RegisterInstanceCoordinatorStatus;
auto SetInstanceToMain(std::string instance_name) -> coordination::SetInstanceToMainCoordinatorStatus;
auto ShowInstances() const -> std::vector<coordination::CoordinatorInstanceStatus>;
private:
DbmsHandler &dbms_handler_;
};
} // namespace memgraph::dbms
#endif

View File

@@ -1,153 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#ifdef MG_ENTERPRISE
#include "dbms/coordinator_handlers.hpp"
#include "dbms/utils.hpp"
#include "coordination/coordinator_exceptions.hpp"
#include "coordination/coordinator_rpc.hpp"
#include "dbms/dbms_handler.hpp"
#include "dbms/replication_client.hpp"
#include "range/v3/view.hpp"
namespace memgraph::dbms {
void CoordinatorHandlers::Register(DbmsHandler &dbms_handler) {
auto &server = dbms_handler.CoordinatorState().GetCoordinatorServer();
server.Register<coordination::PromoteReplicaToMainRpc>(
[&dbms_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
spdlog::info("Received PromoteReplicaToMainRpc");
CoordinatorHandlers::PromoteReplicaToMainHandler(dbms_handler, req_reader, res_builder);
});
server.Register<coordination::SetMainToReplicaRpc>(
[&dbms_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
spdlog::info("Received SetMainToReplicaRpc from coordinator server");
CoordinatorHandlers::SetMainToReplicaHandler(dbms_handler, req_reader, res_builder);
});
}
void CoordinatorHandlers::SetMainToReplicaHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
auto &repl_state = dbms_handler.ReplicationState();
if (!repl_state.IsMain()) {
spdlog::error("Setting to replica must be performed on main.");
slk::Save(coordination::SetMainToReplicaRes{false}, res_builder);
return;
}
coordination::SetMainToReplicaReq req;
slk::Load(&req, req_reader);
replication::ReplicationServerConfig clients_config{.ip_address = req.replication_client_info.replication_ip_address,
.port = req.replication_client_info.replication_port};
if (bool success = memgraph::dbms::SetReplicationRoleReplica(dbms_handler, clients_config); !success) {
spdlog::error("Setting main to replica failed!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
}
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
}
void CoordinatorHandlers::PromoteReplicaToMainHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
auto &repl_state = dbms_handler.ReplicationState();
if (!repl_state.IsReplica()) {
spdlog::error("Failover must be performed on replica!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
}
auto repl_server_config = std::get<replication::RoleReplicaData>(repl_state.ReplicationData()).config;
// This can fail because of disk. If it does, the cluster state could get inconsistent.
// We don't handle disk issues.
if (bool success = memgraph::dbms::DoReplicaToMainPromotion(dbms_handler); !success) {
spdlog::error("Promoting replica to main failed!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
}
coordination::PromoteReplicaToMainReq req;
slk::Load(&req, req_reader);
auto const converter = [](const auto &repl_info_config) {
return replication::ReplicationClientConfig{
.name = repl_info_config.instance_name,
.mode = repl_info_config.replication_mode,
.ip_address = repl_info_config.replication_ip_address,
.port = repl_info_config.replication_port,
};
};
MG_ASSERT(
std::get<replication::RoleMainData>(repl_state.ReplicationData()).registered_replicas_.empty(),
"No replicas should be registered after promoting replica to main and before registering replication clients!");
// registering replicas
for (auto const &config : req.replication_clients_info | ranges::views::transform(converter)) {
auto instance_client = repl_state.RegisterReplica(config);
if (instance_client.HasError()) {
switch (instance_client.GetError()) {
// Can't happen, we are already replica
case memgraph::replication::RegisterReplicaError::NOT_MAIN:
spdlog::error("Failover must be performed to main!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
// Can't happen, checked on the coordinator side
case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
spdlog::error("Replica with the same name already exists!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
// Can't happen, checked on the coordinator side
case memgraph::replication::RegisterReplicaError::ENDPOINT_EXISTS:
spdlog::error("Replica with the same endpoint already exists!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
// We don't handle disk issues
case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
spdlog::error("Registered replica could not be persisted!");
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
case memgraph::replication::RegisterReplicaError::SUCCESS:
break;
}
}
if (!allow_mt_repl && dbms_handler.All().size() > 1) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
auto &instance_client_ref = *instance_client.GetValue();
// Update system before enabling individual storage <-> replica clients
dbms_handler.SystemRestore(instance_client_ref);
// TODO: (andi) Policy for register all databases
// Will be resolved after deciding about choosing new replica
const bool all_clients_good = memgraph::dbms::RegisterAllDatabasesClients(dbms_handler, instance_client_ref);
MG_ASSERT(all_clients_good, "Failed to register one or more databases to the REPLICA \"{}\".", config.name);
StartReplicaClient(dbms_handler, instance_client_ref);
}
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
}
} // namespace memgraph::dbms
#endif

View File

@@ -1,34 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifdef MG_ENTERPRISE
#include "slk/serialization.hpp"
namespace memgraph::dbms {
class DbmsHandler;
class CoordinatorHandlers {
public:
static void Register(DbmsHandler &dbms_handler);
private:
static void PromoteReplicaToMainHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder);
static void SetMainToReplicaHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
};
} // namespace memgraph::dbms
#endif

View File

@@ -26,7 +26,7 @@ Database::Database(storage::Config config, replication::ReplicationState &repl_s
streams_{config.durability.storage_directory / "streams"},
plan_cache_{FLAGS_query_plan_cache_max_size},
repl_state_(&repl_state) {
if (config.salient.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk ||
if (config.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk ||
utils::DirExists(config.disk.main_storage_directory)) {
storage_ = std::make_unique<storage::DiskStorage>(std::move(config));
} else {

View File

@@ -81,14 +81,7 @@ class Database {
*
* @return const std::string&
*/
const std::string &name() const { return storage_->name(); }
/**
* @brief Unique storage identified (uuid)
*
* @return const utils::UUID&
*/
const utils::UUID &uuid() const { return storage_->uuid(); }
const std::string &id() const { return storage_->id(); }
/**
* @brief Returns the storage configuration
@@ -110,7 +103,7 @@ class Database {
* @param force_directory Use the configured directory, do not try to decipher the multi-db version
* @return DatabaseInfo
*/
DatabaseInfo GetInfo(bool force_directory, replication_coordination_glue::ReplicationRole replication_role) const {
DatabaseInfo GetInfo(bool force_directory, replication::ReplicationRole replication_role) const {
DatabaseInfo info;
info.storage_info = storage_->GetInfo(force_directory, replication_role);
info.triggers = trigger_store_.GetTriggerInfo().size();

View File

@@ -51,7 +51,7 @@ class DatabaseHandler : public Handler<Database> {
* @param config Storage configuration
* @return HandlerT::NewResult
*/
HandlerT::NewResult New(storage::Config config, replication::ReplicationState &repl_state) {
HandlerT::NewResult New(std::string_view name, storage::Config config, replication::ReplicationState &repl_state) {
// Control that no one is using the same data directory
if (std::any_of(begin(), end(), [&](auto &elem) {
auto db_acc = elem.second.access();
@@ -61,7 +61,8 @@ class DatabaseHandler : public Handler<Database> {
spdlog::info("Tried to generate new storage using a claimed directory.");
return NewError::EXISTS;
}
return HandlerT::New(std::piecewise_construct, config.salient.name, config, repl_state);
config.name = name; // Set storage id via config
return HandlerT::New(std::piecewise_construct, name, config, repl_state);
}
/**

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,205 +11,56 @@
#include "dbms/dbms_handler.hpp"
#include "dbms/coordinator_handlers.hpp"
#include "flags/replication.hpp"
#include <cstdint>
#include <filesystem>
#include "dbms/constants.hpp"
#include "dbms/global.hpp"
#include "dbms/replication_client.hpp"
#include "spdlog/spdlog.h"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/uuid.hpp"
namespace memgraph::dbms {
#ifdef MG_ENTERPRISE
namespace {
constexpr std::string_view kDBPrefix = "database:"; // Key prefix for database durability
constexpr std::string_view kLastCommitedSystemTsKey = "last_commited_system_ts"; // Key for timestamp durability
} // namespace
struct Durability {
enum class DurabilityVersion : uint8_t {
V0 = 0,
V1,
};
struct VersionException : public utils::BasicException {
VersionException() : utils::BasicException("Unsupported durability version!") {}
};
struct UnknownVersionException : public utils::BasicException {
UnknownVersionException() : utils::BasicException("Unable to parse the durability version!") {}
};
struct MigrationException : public utils::BasicException {
MigrationException() : utils::BasicException("Failed to migrate to the current durability version!") {}
};
static DurabilityVersion VersionCheck(std::optional<std::string_view> val) {
if (!val) {
return DurabilityVersion::V0;
}
if (val == "V1") {
return DurabilityVersion::V1;
}
throw UnknownVersionException();
};
static auto GenKey(std::string_view name) -> std::string { return fmt::format("{}{}", kDBPrefix, name); }
static auto GenVal(utils::UUID uuid, std::filesystem::path rel_dir) {
nlohmann::json json;
json["uuid"] = uuid;
json["rel_dir"] = rel_dir;
// TODO: Serialize the configuration
return json.dump();
}
static void Migrate(kvstore::KVStore *durability, const std::filesystem::path &root) {
const auto ver_val = durability->Get("version");
const auto ver = VersionCheck(ver_val);
std::map<std::string, std::string> to_put;
std::vector<std::string> to_delete;
// Update from V0 to V1
if (ver == DurabilityVersion::V0) {
for (const auto &[key, val] : *durability) {
if (key == "version") continue; // Reserved key
// Generate a UUID
auto const uuid = utils::UUID();
// New json values
auto new_key = GenKey(key);
auto path = root;
if (key != kDefaultDB) { // Special case for non-default DBs
// Move directory to new UUID dir
path = root / kMultiTenantDir / std::string{uuid};
std::filesystem::path old_dir(root / kMultiTenantDir / key);
std::error_code ec;
std::filesystem::rename(old_dir, path, ec);
MG_ASSERT(!ec, "Failed to upgrade durability: cannot move default directory.");
}
// Generate json and update value
auto new_data = GenVal(uuid, std::filesystem::relative(path, root));
to_put.emplace(std::move(new_key), std::move(new_data));
to_delete.emplace_back(key);
}
}
// Set version
durability->Put("version", "V1");
// Update to the new key-value pairs
if (!durability->PutAndDeleteMultiple(to_put, to_delete)) {
throw MigrationException();
}
}
};
DbmsHandler::DbmsHandler(
storage::Config config,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
bool recovery_on_startup)
: default_config_{std::move(config)}, repl_state_{ReplicationStateRootPath(default_config_)} {
bool recovery_on_startup, bool delete_on_drop)
: default_config_{std::move(config)},
delete_on_drop_(delete_on_drop),
repl_state_{ReplicationStateRootPath(default_config_)} {
// TODO: Decouple storage config from dbms config
// TODO: Save individual db configs inside the kvstore and restore from there
/*
* FILESYSTEM MANIPULATION
*/
const auto &root = default_config_.durability.storage_directory;
storage::UpdatePaths(default_config_, root);
const auto &db_dir = default_config_.durability.storage_directory / kMultiTenantDir;
// TODO: Unify durability and wal
storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases");
const auto &db_dir = default_config_.durability.storage_directory;
const auto durability_dir = db_dir / ".durability";
utils::EnsureDirOrDie(db_dir);
utils::EnsureDirOrDie(durability_dir);
durability_ = std::make_unique<kvstore::KVStore>(durability_dir);
/*
* DURABILITY
*/
// Migrate durability
Durability::Migrate(durability_.get(), root);
auto directories = std::set{std::string{kDefaultDB}};
// Generate the default database
MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB.");
// Recover previous databases
if (recovery_on_startup) {
auto it = durability_->begin(std::string(kDBPrefix));
auto end = durability_->end(std::string(kDBPrefix));
for (; it != end; ++it) {
const auto &[key, config_json] = *it;
const auto name = key.substr(kDBPrefix.size());
auto json = nlohmann::json::parse(config_json);
const auto uuid = json.at("uuid").get<utils::UUID>();
const auto rel_dir = json.at("rel_dir").get<std::filesystem::path>();
spdlog::info("Restoring database {} at {}.", name, rel_dir);
auto new_db = New_(name, uuid, rel_dir);
MG_ASSERT(!new_db.HasError(), "Failed while creating database {}.", name);
directories.emplace(rel_dir.filename());
for (const auto &[name, _] : *durability_) {
if (name == kDefaultDB) continue; // Already set
spdlog::info("Restoring database {}.", name);
MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name);
spdlog::info("Database {} restored.", name);
}
// Read the last timestamp
auto lcst = durability_->Get(kLastCommitedSystemTsKey);
if (lcst) {
last_commited_system_timestamp_ = std::stoul(*lcst);
system_timestamp_ = last_commited_system_timestamp_;
}
} else { // Clear databases from the durability list and auth
auto locked_auth = auth->Lock();
auto it = durability_->begin(std::string{kDBPrefix});
auto end = durability_->end(std::string{kDBPrefix});
for (; it != end; ++it) {
const auto &[key, _] = *it;
const auto name = key.substr(kDBPrefix.size());
for (const auto &[name, _] : *durability_) {
if (name == kDefaultDB) continue;
locked_auth->DeleteDatabase(name);
durability_->Delete(key);
}
// Delete the last timestamp
durability_->Delete(kLastCommitedSystemTsKey);
}
/*
* DATABASES CLEAN UP
*/
// Clean the unused directories
for (const auto &entry : std::filesystem::directory_iterator(db_dir)) {
const auto &name = entry.path().filename().string();
if (entry.is_directory() && !name.empty() && name.front() != '.') {
auto itr = directories.find(name);
if (itr == directories.end()) {
std::error_code dummy;
std::filesystem::remove_all(entry, dummy);
} else {
directories.erase(itr);
}
durability_->Delete(name);
}
}
/*
* DEFAULT DB SETUP
*/
// Setup the default DB
SetupDefault_();
/*
* REPLICATION RECOVERY AND STARTUP
*/
// Startup replication state (if recovered at startup)
auto replica = [this](replication::RoleReplicaData const &data) { return StartRpcServer(*this, data); };
// Replication recovery and frequent check start
auto main = [this](replication::RoleMainData &data) {
for (auto &client : data.registered_replicas_) {
SystemRestore(client);
auto replica = [this](replication::RoleReplicaData const &data) {
// Register handlers
InMemoryReplicationHandlers::Register(this, *data.server);
if (!data.server->Start()) {
spdlog::error("Unable to start the replication server.");
return false;
}
ForEach([this](DatabaseAccess db) { RecoverReplication(db); });
return true;
};
// Replication frequent check start
auto main = [this](replication::RoleMainData &data) {
for (auto &client : data.registered_replicas_) {
StartReplicaClient(*this, client);
}
@@ -218,232 +69,7 @@ DbmsHandler::DbmsHandler(
// Startup proccess for main/replica
MG_ASSERT(std::visit(memgraph::utils::Overloaded{replica, main}, repl_state_.ReplicationData()),
"Replica recovery failure!");
// Warning
if (default_config_.durability.snapshot_wal_mode == storage::Config::Durability::SnapshotWalMode::DISABLED &&
repl_state_.IsMain()) {
spdlog::warn(
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please "
"consider "
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
"without write-ahead logs this instance is not replicating any data.");
}
// MAIN or REPLICA instance
if (FLAGS_coordinator_server_port) {
CoordinatorHandlers::Register(*this);
MG_ASSERT(coordinator_state_.GetCoordinatorServer().Start(), "Failed to start coordinator server!");
}
}
DbmsHandler::DeleteResult DbmsHandler::TryDelete(std::string_view db_name) {
std::lock_guard<LockT> wr(lock_);
if (db_name == kDefaultDB) {
// MSG cannot delete the default db
return DeleteError::DEFAULT_DB;
}
// Get DB config for the UUID and disk clean up
const auto conf = db_handler_.GetConfig(db_name);
if (!conf) {
return DeleteError::NON_EXISTENT;
}
const auto &storage_path = conf->durability.storage_directory;
const auto &uuid = conf->salient.uuid;
// Check if db exists
try {
// Low level handlers
if (!db_handler_.TryDelete(db_name)) {
return DeleteError::USING;
}
} catch (utils::BasicException &) {
return DeleteError::NON_EXISTENT;
}
// Remove from durability list
if (durability_) durability_->Delete(Durability::GenKey(db_name));
// Delete disk storage
std::error_code ec;
(void)std::filesystem::remove_all(storage_path, ec);
if (ec) {
spdlog::error(R"(Failed to clean disk while deleting database "{}" stored in {})", db_name, storage_path);
}
// Success
// Save delta
if (system_transaction_) {
system_transaction_->delta.emplace(SystemTransaction::Delta::drop_database, uuid);
}
return {};
}
DbmsHandler::DeleteResult DbmsHandler::Delete(std::string_view db_name) {
auto wr = std::lock_guard(lock_);
return Delete_(db_name);
}
DbmsHandler::DeleteResult DbmsHandler::Delete(utils::UUID uuid) {
auto wr = std::lock_guard(lock_);
std::string db_name;
try {
const auto db = Get_(uuid);
db_name = db->name();
} catch (const UnknownDatabaseException &) {
return DeleteError::NON_EXISTENT;
}
return Delete_(db_name);
}
DbmsHandler::NewResultT DbmsHandler::New_(storage::Config storage_config) {
auto new_db = db_handler_.New(storage_config, repl_state_);
if (new_db.HasValue()) { // Success
// Save delta
if (system_transaction_) {
system_transaction_->delta.emplace(SystemTransaction::Delta::create_database, storage_config.salient);
}
UpdateDurability(storage_config);
return new_db.GetValue();
}
return new_db.GetError();
}
DbmsHandler::DeleteResult DbmsHandler::Delete_(std::string_view db_name) {
if (db_name == kDefaultDB) {
// MSG cannot delete the default db
return DeleteError::DEFAULT_DB;
}
const auto storage_path = StorageDir_(db_name);
if (!storage_path) return DeleteError::NON_EXISTENT;
{
auto db = db_handler_.Get(db_name);
if (!db) return DeleteError::NON_EXISTENT;
// TODO: ATM we assume REPLICA won't have streams,
// this is a best effort approach just in case they do
// there is still subtle data race we stream manipulation
// can occur while we are dropping the database
db->prepare_for_deletion();
auto &database = *db->get();
database.streams()->StopAll();
database.streams()->DropAll();
database.thread_pool()->Shutdown();
}
// Remove from durability list
if (durability_) durability_->Delete(Durability::GenKey(db_name));
// Check if db exists
// Low level handlers
db_handler_.DeferDelete(db_name, [storage_path = *storage_path, db_name = std::string{db_name}]() {
// Delete disk storage
std::error_code ec;
(void)std::filesystem::remove_all(storage_path, ec);
if (ec) {
spdlog::error(R"(Failed to clean disk while deleting database "{}" stored in {})", db_name, storage_path);
}
});
return {}; // Success
}
void DbmsHandler::UpdateDurability(const storage::Config &config, std::optional<std::filesystem::path> rel_dir) {
if (!durability_) return;
// Save database in a list of active databases
const auto &key = Durability::GenKey(config.salient.name);
if (rel_dir == std::nullopt)
rel_dir =
std::filesystem::relative(config.durability.storage_directory, default_config_.durability.storage_directory);
const auto &val = Durability::GenVal(config.salient.uuid, *rel_dir);
durability_->Put(key, val);
}
AllSyncReplicaStatus DbmsHandler::Commit() {
if (system_transaction_ == std::nullopt || system_transaction_->delta == std::nullopt)
return AllSyncReplicaStatus::AllCommitsConfirmed; // Nothing to commit
const auto &delta = *system_transaction_->delta;
auto sync_status = AllSyncReplicaStatus::AllCommitsConfirmed;
// TODO Create a system client that can handle all of this automatically
switch (delta.action) {
using enum SystemTransaction::Delta::Action;
case CREATE_DATABASE: {
// Replication
auto main_handler = [&](memgraph::replication::RoleMainData &main_data) {
// TODO: data race issue? registered_replicas_ access not protected
// This is sync in any case, as this is the startup
for (auto &client : main_data.registered_replicas_) {
bool completed = SteamAndFinalizeDelta<storage::replication::CreateDatabaseRpc>(
client,
[](const storage::replication::CreateDatabaseRes &response) {
return response.result != storage::replication::CreateDatabaseRes::Result::FAILURE;
},
std::string(main_data.epoch_.id()), last_commited_system_timestamp_,
system_transaction_->system_timestamp, delta.config);
// TODO: reduce duplicate code
if (!completed && client.mode_ == replication_coordination_glue::ReplicationMode::SYNC) {
sync_status = AllSyncReplicaStatus::SomeCommitsUnconfirmed;
}
}
// Sync database with REPLICAs
RecoverReplication(Get_(delta.config.name));
};
auto replica_handler = [](memgraph::replication::RoleReplicaData &) { /* Nothing to do */ };
std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData());
} break;
case DROP_DATABASE: {
// Replication
auto main_handler = [&](memgraph::replication::RoleMainData &main_data) {
// TODO: data race issue? registered_replicas_ access not protected
// This is sync in any case, as this is the startup
for (auto &client : main_data.registered_replicas_) {
bool completed = SteamAndFinalizeDelta<storage::replication::DropDatabaseRpc>(
client,
[](const storage::replication::DropDatabaseRes &response) {
return response.result != storage::replication::DropDatabaseRes::Result::FAILURE;
},
std::string(main_data.epoch_.id()), last_commited_system_timestamp_,
system_transaction_->system_timestamp, delta.uuid);
// TODO: reduce duplicate code
if (!completed && client.mode_ == replication_coordination_glue::ReplicationMode::SYNC) {
sync_status = AllSyncReplicaStatus::SomeCommitsUnconfirmed;
}
}
};
auto replica_handler = [](memgraph::replication::RoleReplicaData &) { /* Nothing to do */ };
std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData());
} break;
}
durability_->Put(kLastCommitedSystemTsKey, std::to_string(system_transaction_->system_timestamp));
last_commited_system_timestamp_ = system_transaction_->system_timestamp;
ResetSystemTransaction();
return sync_status;
}
#else // not MG_ENTERPRISE
AllSyncReplicaStatus DbmsHandler::Commit() {
if (system_transaction_ == std::nullopt || system_transaction_->delta == std::nullopt) {
return AllSyncReplicaStatus::AllCommitsConfirmed; // Nothing to commit
}
const auto &delta = *system_transaction_->delta;
switch (delta.action) {
using enum SystemTransaction::Delta::Action;
case CREATE_DATABASE:
case DROP_DATABASE:
/* Community edition doesn't support multi-tenant replication */
break;
}
last_commited_system_timestamp_ = system_transaction_->system_timestamp;
ResetSystemTransaction();
return AllSyncReplicaStatus::AllCommitsConfirmed;
}
#endif
} // namespace memgraph::dbms

View File

@@ -12,37 +12,34 @@
#pragma once
#include <algorithm>
#include <atomic>
#include <concepts>
#include <cstdint>
#include <filesystem>
#include <memory>
#include <mutex>
#include <optional>
#include <ostream>
#include <stdexcept>
#include <system_error>
#include <utility>
#include <unordered_map>
#include "auth/auth.hpp"
#include "constants.hpp"
#include "dbms/database.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "dbms/replication_handler.hpp"
#include "kvstore/kvstore.hpp"
#include "replication/replication_client.hpp"
#include "storage/v2/config.hpp"
#include "storage/v2/replication/enums.hpp"
#include "storage/v2/replication/rpc.hpp"
#include "storage/v2/transaction.hpp"
#include "utils/thread_pool.hpp"
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_state.hpp"
#include "dbms/database_handler.hpp"
#endif
#include "dbms/transaction.hpp"
#include "dbms/replication_client.hpp"
#include "global.hpp"
#include "query/config.hpp"
#include "query/interpreter_context.hpp"
#include "spdlog/spdlog.h"
#include "storage/v2/durability/durability.hpp"
#include "storage/v2/durability/paths.hpp"
#include "storage/v2/isolation_level.hpp"
#include "utils/exceptions.hpp"
#include "utils/file.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
#include "utils/rw_lock.hpp"
@@ -51,11 +48,6 @@
namespace memgraph::dbms {
enum class AllSyncReplicaStatus {
AllCommitsConfirmed,
SomeCommitsUnconfirmed,
};
struct Statistics {
uint64_t num_vertex; //!< Sum of vertexes in every database
uint64_t num_edges; //!< Sum of edges in every database
@@ -110,10 +102,11 @@ class DbmsHandler {
* @param configs storage configuration
* @param auth pointer to the global authenticator
* @param recovery_on_startup restore databases (and its content) and authentication data
* @param delete_on_drop when dropping delete any associated directories on disk
*/
DbmsHandler(storage::Config config,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
bool recovery_on_startup); // TODO If more arguments are added use a config struct
bool recovery_on_startup, bool delete_on_drop); // TODO If more arguments are added use a config strut
#else
/**
* @brief Initialize the handler. A single database is supported in community edition.
@@ -123,12 +116,10 @@ class DbmsHandler {
DbmsHandler(storage::Config config)
: repl_state_{ReplicationStateRootPath(config)},
db_gatekeeper_{[&] {
config.salient.name = kDefaultDB;
config.name = kDefaultDB;
return std::move(config);
}(),
repl_state_} {
RecoverReplication(Get());
}
repl_state_} {}
#endif
#ifdef MG_ENTERPRISE
@@ -140,56 +131,9 @@ class DbmsHandler {
*/
NewResultT New(const std::string &name) {
std::lock_guard<LockT> wr(lock_);
const auto uuid = utils::UUID{};
return New_(name, uuid);
return New_(name, name);
}
/**
* @brief Create new if name/uuid do not match any database. Drop and recreate if database already present.
* @note Default database is not dropped, only its UUID is updated and only if the database is clean.
*
* @param config desired salient config
* @return NewResultT context on success, error on failure
*/
NewResultT Update(const storage::SalientConfig &config) {
std::lock_guard<LockT> wr(lock_);
auto new_db = New_(config);
if (new_db.HasValue() || new_db.GetError() != NewError::EXISTS) {
// NOTE: If db already exists we retry below
return new_db;
}
spdlog::debug("Trying to create db '{}' on replica which already exists.", config.name);
auto db = Get_(config.name);
if (db->uuid() == config.uuid) { // Same db
return db;
}
spdlog::debug("Different UUIDs");
// TODO: Fix this hack
if (config.name == kDefaultDB) {
if (db->storage()->repl_storage_state_.last_commit_timestamp_ != storage::kTimestampInitialId) {
spdlog::debug("Default storage is not clean, cannot update UUID...");
return NewError::GENERIC; // Update error
}
spdlog::debug("Update default db's UUID");
// Default db cannot be deleted and remade, have to just update the UUID
db->storage()->config_.salient.uuid = config.uuid;
UpdateDurability(db->storage()->config_, ".");
return db;
}
spdlog::debug("Drop database and recreate with the correct UUID");
// Defer drop
(void)Delete_(db->name());
// Second attempt
return New_(config);
}
void UpdateDurability(const storage::Config &config, std::optional<std::filesystem::path> rel_dir = {});
/**
* @brief Get the context associated with the "name" database
*
@@ -201,19 +145,6 @@ class DbmsHandler {
std::shared_lock<LockT> rd(lock_);
return Get_(name);
}
/**
* @brief Get the context associated with the UUID database
*
* @param uuid
* @return DatabaseAccess
* @throw UnknownDatabaseException if database not found
*/
DatabaseAccess Get(const utils::UUID &uuid) {
std::shared_lock<LockT> rd(lock_);
return Get_(uuid);
}
#else
/**
* @brief Get the context associated with the default database
@@ -229,28 +160,50 @@ class DbmsHandler {
#ifdef MG_ENTERPRISE
/**
* @brief Attempt to delete database.
* @brief Delete database.
*
* @param db_name database name
* @return DeleteResult error on failure
*/
DeleteResult TryDelete(std::string_view db_name);
DeleteResult Delete(const std::string &db_name) {
std::lock_guard<LockT> wr(lock_);
if (db_name == kDefaultDB) {
// MSG cannot delete the default db
return DeleteError::DEFAULT_DB;
}
/**
* @brief Delete or defer deletion of database.
*
* @param db_name database name
* @return DeleteResult error on failure
*/
DeleteResult Delete(std::string_view db_name);
const auto storage_path = StorageDir_(db_name);
if (!storage_path) return DeleteError::NON_EXISTENT;
/**
* @brief Delete or defer deletion of database.
*
* @param uuid database UUID
* @return DeleteResult error on failure
*/
DeleteResult Delete(utils::UUID uuid);
// Check if db exists
try {
// Low level handlers
if (!db_handler_.Delete(db_name)) {
return DeleteError::USING;
}
} catch (utils::BasicException &) {
return DeleteError::NON_EXISTENT;
}
// Remove from durability list
if (durability_) durability_->Delete(db_name);
// Delete disk storage
if (delete_on_drop_) {
std::error_code ec;
(void)std::filesystem::remove_all(*storage_path, ec);
if (ec) {
spdlog::error("Failed to clean disk while deleting database \"{}\".", db_name);
defunct_dbs_.emplace(db_name);
return DeleteError::DISK_FAIL;
}
}
// Delete from defunct_dbs_ (in case a second delete call was successful)
defunct_dbs_.erase(db_name);
return {}; // Success
}
#endif
/**
@@ -263,7 +216,7 @@ class DbmsHandler {
std::shared_lock<LockT> rd(lock_);
return db_handler_.All();
#else
return {db_gatekeeper_.access()->get()->name()};
return {db_gatekeeper_.access()->get()->id()};
#endif
}
@@ -273,10 +226,6 @@ class DbmsHandler {
bool IsMain() const { return repl_state_.IsMain(); }
bool IsReplica() const { return repl_state_.IsReplica(); }
#ifdef MG_ENTERPRISE
coordination::CoordinatorState &CoordinatorState() { return coordinator_state_; }
#endif
/**
* @brief Return the statistics all databases.
*
@@ -356,7 +305,7 @@ class DbmsHandler {
auto db_acc_opt = db_gk.access();
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->name());
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id());
auto storage_accessor = db_acc->Access();
auto dba = memgraph::query::DbAccessor{storage_accessor.get()};
db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker);
@@ -381,7 +330,7 @@ class DbmsHandler {
auto db_acc = db_gk.access();
if (db_acc) {
auto *db = db_acc->get();
spdlog::debug("Restoring streams for database \"{}\"", db->name());
spdlog::debug("Restoring streams for database \"{}\"", db->id());
db->streams()->RestoreStreams(*db_acc, ic);
}
}
@@ -392,7 +341,7 @@ class DbmsHandler {
*
* @param f
*/
void ForEach(std::invocable<DatabaseAccess> auto f) {
void ForEach(auto f) {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
@@ -402,103 +351,33 @@ class DbmsHandler {
#endif
auto db_acc = db_gk.access();
if (db_acc) { // This isn't an error, just a defunct db
f(*db_acc);
f(db_acc->get());
}
}
}
void NewSystemTransaction() {
DMG_ASSERT(!system_transaction_, "Already running a system transaction");
system_transaction_.emplace(++system_timestamp_);
}
void ResetSystemTransaction() { system_transaction_.reset(); }
//! \tparam RPC An rpc::RequestResponse
//! \tparam Args the args type
//! \param client the client to use for rpc communication
//! \param check predicate to check response is ok
//! \param args arguments to forward to the rpc request
//! \return If replica stream is completed or enqueued
template <typename RPC, typename... Args>
bool SteamAndFinalizeDelta(auto &client, auto &&check, Args &&...args) {
try {
auto stream = client.rpc_client_.template Stream<RPC>(std::forward<Args>(args)...);
auto task = [&client, check = std::forward<decltype(check)>(check), stream = std::move(stream)]() mutable {
if (stream.IsDefunct()) {
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
return false;
}
try {
if (check(stream.AwaitResponse())) {
return true;
}
} catch (memgraph::rpc::GenericRpcFailedException const &e) {
// swallow error, fallthrough to error handling
}
// This replica needs SYSTEM recovery
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
return false;
};
if (client.mode_ == memgraph::replication_coordination_glue::ReplicationMode::ASYNC) {
client.thread_pool_.AddTask([task = utils::CopyMovableFunctionWrapper{std::move(task)}]() mutable { task(); });
return true;
}
return task();
} catch (memgraph::rpc::GenericRpcFailedException const &e) {
// This replica needs SYSTEM recovery
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
return false;
}
};
AllSyncReplicaStatus Commit();
auto LastCommitedTS() const -> uint64_t { return last_commited_system_timestamp_; }
void SetLastCommitedTS(uint64_t new_ts) { last_commited_system_timestamp_.store(new_ts); }
/**
* @brief todo
*
* @param f
*/
void ForOne(auto f) {
#ifdef MG_ENTERPRISE
// When being called by intepreter no need to gain lock, it should already be under a system transaction
// But concurrently the FrequentCheck is running and will need to lock before reading last_commited_system_timestamp_
template <bool REQUIRE_LOCK = false>
void SystemRestore(replication::ReplicationClient &client) {
// Check if system is up to date
if (client.state_.WithLock(
[](auto &state) { return state == memgraph::replication::ReplicationClient::State::READY; }))
return;
// Try to recover...
{
auto [database_configs, last_commited_system_timestamp] = std::invoke([&] {
auto sys_guard =
std::unique_lock{system_lock_, std::defer_lock}; // ensure no other system transaction in progress
if constexpr (REQUIRE_LOCK) {
sys_guard.lock();
}
auto configs = std::vector<storage::SalientConfig>{};
ForEach([&configs](DatabaseAccess acc) { configs.emplace_back(acc->config().salient); });
return std::pair{configs, last_commited_system_timestamp_.load()};
});
try {
auto stream = client.rpc_client_.Stream<storage::replication::SystemRecoveryRpc>(last_commited_system_timestamp,
std::move(database_configs));
const auto response = stream.AwaitResponse();
if (response.result == storage::replication::SystemRecoveryRes::Result::FAILURE) {
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
return;
}
} catch (memgraph::rpc::GenericRpcFailedException const &e) {
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
return;
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
auto db_acc = db_gk.access();
if (db_acc) { // This isn't an error, just a defunct db
if (f(db_acc->get())) break; // Run until the first successful one
}
}
// Successfully recovered
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::READY; });
}
#else
{
auto db_acc = db_gatekeeper_.access();
MG_ASSERT(db_acc, "Should always have the database");
f(db_acc->get());
}
#endif
}
private:
#ifdef MG_ENTERPRISE
@@ -508,7 +387,7 @@ class DbmsHandler {
* @param name Database name
* @return std::optional<std::filesystem::path>
*/
std::optional<std::filesystem::path> StorageDir_(std::string_view name) {
std::optional<std::filesystem::path> StorageDir_(const std::string &name) {
const auto conf = db_handler_.GetConfig(name);
if (conf) {
return conf->durability.storage_directory;
@@ -521,108 +400,105 @@ class DbmsHandler {
* @brief Create a new Database associated with the "name" database
*
* @param name name of the database
* @param uuid undelying RocksDB directory
* @return NewResultT context on success, error on failure
*/
NewResultT New_(std::string_view name, utils::UUID uuid, std::optional<std::filesystem::path> rel_dir = {}) {
auto config_copy = default_config_;
config_copy.salient.name = name;
config_copy.salient.uuid = uuid;
spdlog::debug("Creating database '{}' - '{}'", name, std::string{uuid});
if (rel_dir) {
storage::UpdatePaths(config_copy, default_config_.durability.storage_directory / *rel_dir);
} else {
storage::UpdatePaths(config_copy,
default_config_.durability.storage_directory / kMultiTenantDir / std::string{uuid});
}
return New_(std::move(config_copy));
}
NewResultT New_(const std::string &name) { return New_(name, name); }
/**
* @brief Create a new Database using the passed configuration
* @brief Create a new Database associated with the "name" database
*
* @param config configuration to be used
* @param name name of the database
* @param storage_subdir undelying RocksDB directory
* @return NewResultT context on success, error on failure
*/
NewResultT New_(const storage::SalientConfig &config) {
NewResultT New_(const std::string &name, std::filesystem::path storage_subdir) {
auto config_copy = default_config_;
config_copy.salient = config; // name, uuid, mode, etc
UpdatePaths(config_copy, config_copy.durability.storage_directory / kMultiTenantDir / std::string{config.uuid});
return New_(std::move(config_copy));
storage::UpdatePaths(config_copy, default_config_.durability.storage_directory / storage_subdir);
return New_(name, config_copy);
}
/**
* @brief Create a new Database associated with the "name" database
*
* @param name name of the database
* @param storage_config storage configuration
* @return NewResultT context on success, error on failure
*/
NewResultT New_(storage::Config storage_config);
NewResultT New_(const std::string &name, storage::Config &storage_config) {
if (defunct_dbs_.contains(name)) {
spdlog::warn("Failed to generate database due to the unknown state of the previously defunct database \"{}\".",
name);
return NewError::DEFUNCT;
}
// TODO: new overload of Delete_ with DatabaseAccess
DeleteResult Delete_(std::string_view db_name);
auto new_db = db_handler_.New(name, storage_config, repl_state_);
if (new_db.HasValue()) {
// Success
if (durability_) durability_->Put(name, "ok"); // TODO: Serialize the configuration?
return new_db.GetValue();
}
return new_db.GetError();
}
/**
* @brief Create a new Database associated with the default database
*
* @return NewResultT context on success, error on failure
*/
void SetupDefault_() {
try {
Get(kDefaultDB);
} catch (const UnknownDatabaseException &) {
// No default DB restored, create it
MG_ASSERT(New_(kDefaultDB, {/* random UUID */}, ".").HasValue(), "Failed while creating the default database");
}
NewResultT NewDefault_() {
// Create the default DB in the root (this is how it was done pre multi-tenancy)
auto res = New_(kDefaultDB, "..");
if (res.HasValue()) {
// For back-compatibility...
// Recreate the dbms layout for the default db and symlink to the root
const auto dir = StorageDir_(kDefaultDB);
MG_ASSERT(dir, "Failed to find storage path.");
const auto main_dir = *dir / "databases" / kDefaultDB;
// For back-compatibility...
// Recreate the dbms layout for the default db and symlink to the root
const auto dir = StorageDir_(kDefaultDB);
MG_ASSERT(dir, "Failed to find storage path.");
const auto main_dir = *dir / kMultiTenantDir / kDefaultDB;
if (!std::filesystem::exists(main_dir)) {
std::filesystem::create_directory(main_dir);
}
if (!std::filesystem::exists(main_dir)) {
std::filesystem::create_directory(main_dir);
}
// Force link on-disk directories
const auto conf = db_handler_.GetConfig(kDefaultDB);
MG_ASSERT(conf, "No configuration for the default database.");
const auto &tmp_conf = conf->disk;
std::vector<std::filesystem::path> to_link{
tmp_conf.main_storage_directory, tmp_conf.label_index_directory,
tmp_conf.label_property_index_directory, tmp_conf.unique_constraints_directory,
tmp_conf.name_id_mapper_directory, tmp_conf.id_name_mapper_directory,
tmp_conf.durability_directory, tmp_conf.wal_directory,
};
// Force link on-disk directories
const auto conf = db_handler_.GetConfig(kDefaultDB);
MG_ASSERT(conf, "No configuration for the default database.");
const auto &tmp_conf = conf->disk;
std::vector<std::filesystem::path> to_link{
tmp_conf.main_storage_directory, tmp_conf.label_index_directory,
tmp_conf.label_property_index_directory, tmp_conf.unique_constraints_directory,
tmp_conf.name_id_mapper_directory, tmp_conf.id_name_mapper_directory,
tmp_conf.durability_directory, tmp_conf.wal_directory,
};
// Add in-memory paths
// Some directories are redundant (skip those)
const std::vector<std::string> skip{".lock", "audit_log", "auth", "databases", "internal_modules", "settings"};
for (auto const &item : std::filesystem::directory_iterator{*dir}) {
const auto dir_name = std::filesystem::relative(item.path(), item.path().parent_path());
if (std::find(skip.begin(), skip.end(), dir_name) != skip.end()) continue;
to_link.push_back(item.path());
}
// Add in-memory paths
// Some directories are redundant (skip those)
const std::vector<std::string> skip{".lock", "audit_log", "auth", "databases", "internal_modules", "settings"};
for (auto const &item : std::filesystem::directory_iterator{*dir}) {
const auto dir_name = std::filesystem::relative(item.path(), item.path().parent_path());
if (std::find(skip.begin(), skip.end(), dir_name) != skip.end()) continue;
to_link.push_back(item.path());
}
// Symlink to root dir
for (auto const &item : to_link) {
const auto dir_name = std::filesystem::relative(item, item.parent_path());
const auto link = main_dir / dir_name;
const auto to = std::filesystem::relative(item, main_dir);
if (!std::filesystem::is_symlink(link) && !std::filesystem::exists(link)) {
std::filesystem::create_directory_symlink(to, link);
} else { // Check existing link
std::error_code ec;
const auto test_link = std::filesystem::read_symlink(link, ec);
if (ec || test_link != to) {
MG_ASSERT(false,
"Memgraph storage directory incompatible with new version.\n"
"Please use a clean directory or remove \"{}\" and try again.",
link.string());
// Symlink to root dir
for (auto const &item : to_link) {
const auto dir_name = std::filesystem::relative(item, item.parent_path());
const auto link = main_dir / dir_name;
const auto to = std::filesystem::relative(item, main_dir);
if (!std::filesystem::is_symlink(link) && !std::filesystem::exists(link)) {
std::filesystem::create_directory_symlink(to, link);
} else { // Check existing link
std::error_code ec;
const auto test_link = std::filesystem::read_symlink(link, ec);
if (ec || test_link != to) {
MG_ASSERT(false,
"Memgraph storage directory incompatible with new version.\n"
"Please use a clean directory or remove \"{}\" and try again.",
link.string());
}
}
}
}
return res;
}
/**
@@ -640,57 +516,17 @@ class DbmsHandler {
throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name);
}
/**
* @brief Get the context associated with the UUID database
*
* @param uuid
* @return DatabaseAccess
* @throw UnknownDatabaseException if database not found
*/
DatabaseAccess Get_(const utils::UUID &uuid) {
// TODO Speed up
for (auto &[_, db_gk] : db_handler_) {
auto acc = db_gk.access();
if (acc->get()->uuid() == uuid) {
return std::move(*acc);
}
}
throw UnknownDatabaseException("Tried to retrieve an unknown database with UUID \"{}\".", std::string{uuid});
}
#endif
void RecoverReplication(DatabaseAccess db_acc) {
if (allow_mt_repl || db_acc->name() == dbms::kDefaultDB) {
// Handle global replication state
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
// RECOVER REPLICA CONNECTIONS
memgraph::dbms::RestoreReplication(repl_state_, std::move(db_acc));
} else if (const ::memgraph::replication::RoleMainData *data =
std::get_if<::memgraph::replication::RoleMainData>(&repl_state_.ReplicationData());
data && !data->registered_replicas_.empty()) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
}
#ifdef MG_ENTERPRISE
mutable LockT lock_{utils::RWLock::Priority::READ}; //!< protective lock
storage::Config default_config_; //!< Storage configuration used when creating new databases
DatabaseHandler db_handler_; //!< multi-tenancy storage handler
std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation)
coordination::CoordinatorState coordinator_state_; //!< Replication coordinator
bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory
std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures
#endif
// TODO: Make an api
public:
utils::ResourceLock system_lock_{}; //!> Ensure exclusive access for system queries
private:
std::optional<SystemTransaction> system_transaction_; //!< Current system transaction (only one at a time)
uint64_t system_timestamp_{storage::kTimestampInitialId}; //!< System timestamp
std::atomic_uint64_t last_commited_system_timestamp_{
storage::kTimestampInitialId}; //!< Last commited system timestamp
replication::ReplicationState repl_state_; //!< Global replication state
#ifndef MG_ENTERPRISE
mutable utils::Gatekeeper<Database> db_gatekeeper_; //!< Single databases gatekeeper
#endif
}; // namespace memgraph::dbms
};
} // namespace memgraph::dbms

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,7 +21,6 @@
#include "utils/exceptions.hpp"
#include "utils/gatekeeper.hpp"
#include "utils/result.hpp"
#include "utils/thread_pool.hpp"
namespace memgraph::dbms {
@@ -83,7 +82,7 @@ class Handler {
* @return true on success
* @throw BasicException
*/
bool TryDelete(std::string_view name) {
bool Delete(const std::string &name) {
if (auto itr = items_.find(name); itr != items_.end()) {
auto db_acc = itr->second.access();
if (db_acc && db_acc->try_delete()) {
@@ -93,42 +92,9 @@ class Handler {
}
return false;
}
// TODO: Change to return enum
throw utils::BasicException("Unknown item \"{}\".", name);
}
/**
* @brief Delete or defunct the context associated with the name.
*
* @param name Name associated with the context to delete
* @param post_delete_func What to do after deletion has happened
*/
template <typename Func>
void DeferDelete(std::string_view name, Func &&post_delete_func) {
auto itr = items_.find(name);
if (itr == items_.end()) return;
auto db_acc = itr->second.access();
if (!db_acc) return;
if (db_acc->try_delete()) {
// Delete the database now
db_acc->reset();
post_delete_func();
} else {
// Defer deletion
db_acc->reset();
// TODO: Make sure this shuts down correctly
auto task = [gk = std::move(itr->second), post_delete_func = std::forward<Func>(post_delete_func)]() mutable {
gk.~Gatekeeper<T>();
post_delete_func();
};
defer_pool_.AddTask(utils::CopyMovableFunctionWrapper{std::move(task)});
}
// In any case remove from handled map
items_.erase(itr);
}
/**
* @brief Check if a name is already used.
*
@@ -154,7 +120,6 @@ class Handler {
private:
std::unordered_map<std::string, utils::Gatekeeper<T>, string_hash, std::equal_to<>>
items_; //!< map to all active items
utils::ThreadPool defer_pool_{1};
};
} // namespace memgraph::dbms

View File

@@ -10,7 +10,6 @@
// licenses/APL.txt.
#include "dbms/inmemory/replication_handlers.hpp"
#include <chrono>
#include <optional>
#include "dbms/constants.hpp"
#include "dbms/dbms_handler.hpp"
@@ -23,7 +22,7 @@
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/inmemory/unique_constraints.hpp"
using memgraph::replication_coordination_glue::ReplicationRole;
using memgraph::replication::ReplicationRole;
using memgraph::storage::Delta;
using memgraph::storage::EdgeAccessor;
using memgraph::storage::EdgeRef;
@@ -50,29 +49,29 @@ std::pair<uint64_t, WalDeltaData> ReadDelta(storage::durability::BaseDecoder *de
}
};
std::optional<DatabaseAccess> GetDatabaseAccessor(dbms::DbmsHandler *dbms_handler, const utils::UUID &uuid) {
std::optional<DatabaseAccess> GetDatabaseAccessor(dbms::DbmsHandler *dbms_handler, std::string_view db_name) {
try {
#ifdef MG_ENTERPRISE
auto acc = dbms_handler->Get(uuid);
if (!acc) {
spdlog::error("Failed to get access to UUID ", std::string{uuid});
return std::nullopt;
}
auto acc = dbms_handler->Get(db_name);
#else
auto acc = dbms_handler->Get();
if (!acc) {
spdlog::warn("Failed to get access to the default db.");
if (db_name != dbms::kDefaultDB) {
spdlog::warn("Trying to replicate a non-default database on a community replica.");
return std::nullopt;
}
auto acc = dbms_handler->Get();
#endif
if (!acc) {
spdlog::error("Failed to get access to ", db_name);
return std::nullopt;
}
auto *inmem_storage = dynamic_cast<storage::InMemoryStorage *>(acc.get()->storage());
if (!inmem_storage || inmem_storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
spdlog::error("Database is not IN_MEMORY_TRANSACTIONAL.");
spdlog::error("Database \"{}\" is not IN_MEMORY_TRANSACTIONAL.", db_name);
return std::nullopt;
}
return std::optional{std::move(acc)};
} catch (const dbms::UnknownDatabaseException &e) {
spdlog::warn("No database with UUID \"{}\" on replica!", std::string{uuid});
spdlog::warn("No database \"{}\" on replica!", db_name);
return std::nullopt;
}
}
@@ -110,16 +109,13 @@ void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handl
slk::Builder *res_builder) {
storage::replication::HeartbeatReq req;
slk::Load(&req, req_reader);
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
if (!db_acc) {
storage::replication::HeartbeatRes res{false, 0, ""};
slk::Save(res, res_builder);
return;
}
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
auto const *storage = db_acc->get()->storage();
storage::replication::HeartbeatRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load(),
storage::replication::HeartbeatRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load(),
std::string{storage->repl_storage_state_.epoch_.id()}};
slk::Save(res, res_builder);
}
@@ -128,12 +124,8 @@ void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_ha
slk::Builder *res_builder) {
storage::replication::AppendDeltasReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
if (!db_acc) {
storage::replication::AppendDeltasRes res{false, 0};
slk::Save(res, res_builder);
return;
}
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
storage::replication::Decoder decoder(req_reader);
@@ -173,7 +165,7 @@ void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_ha
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
}
storage::replication::AppendDeltasRes res{false, repl_storage_state.last_commit_timestamp_.load()};
storage::replication::AppendDeltasRes res{storage->id(), false, repl_storage_state.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
return;
}
@@ -182,7 +174,7 @@ void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_ha
storage, &decoder,
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
storage::replication::AppendDeltasRes res{true, repl_storage_state.last_commit_timestamp_.load()};
storage::replication::AppendDeltasRes res{storage->id(), true, repl_storage_state.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!");
}
@@ -191,12 +183,8 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle
slk::Builder *res_builder) {
storage::replication::SnapshotReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
if (!db_acc) {
storage::replication::SnapshotRes res{false, 0};
slk::Save(res, res_builder);
return;
}
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
storage::replication::Decoder decoder(req_reader);
@@ -244,7 +232,8 @@ void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handle
}
storage_guard.unlock();
storage::replication::SnapshotRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::SnapshotRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::trace("Deleting old snapshot files due to snapshot recovery.");
@@ -274,12 +263,8 @@ void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handle
slk::Builder *res_builder) {
storage::replication::WalFilesReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
if (!db_acc) {
storage::replication::WalFilesRes res{false, 0};
slk::Save(res, res_builder);
return;
}
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
const auto wal_file_number = req.file_number;
spdlog::debug("Received WAL files: {}", wal_file_number);
@@ -293,7 +278,8 @@ void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handle
LoadWal(storage, &decoder);
}
storage::replication::WalFilesRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::WalFilesRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!");
}
@@ -302,12 +288,8 @@ void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_hand
slk::Builder *res_builder) {
storage::replication::CurrentWalReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
if (!db_acc) {
storage::replication::CurrentWalRes res{false, 0};
slk::Save(res, res_builder);
return;
}
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
storage::replication::Decoder decoder(req_reader);
@@ -316,7 +298,8 @@ void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_hand
LoadWal(storage, &decoder);
storage::replication::CurrentWalRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::CurrentWalRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::debug("Replication recovery from current WAL ended successfully, replica is now up to date!");
}
@@ -335,8 +318,6 @@ void InMemoryReplicationHandlers::LoadWal(storage::InMemoryStorage *storage, sto
}
auto &replica_epoch = storage->repl_storage_state_.epoch_;
if (wal_info.epoch_id != replica_epoch.id()) {
// questionable behaviour, we trust that any change in epoch implies change in who is MAIN
// when we use high availability, this assumption need to be checked.
auto prev_epoch = replica_epoch.SetEpoch(wal_info.epoch_id);
storage->repl_storage_state_.AddEpochToHistoryForce(prev_epoch);
}
@@ -374,16 +355,13 @@ void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handl
slk::Builder *res_builder) {
storage::replication::TimestampReq req;
slk::Load(&req, req_reader);
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.uuid);
if (!db_acc) {
storage::replication::TimestampRes res{false, 0};
slk::Save(res, res_builder);
return;
}
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
auto const *storage = db_acc->get()->storage();
storage::replication::TimestampRes res{true, storage->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::TimestampRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
}
@@ -527,7 +505,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
case WalDeltaData::Type::EDGE_SET_PROPERTY: {
spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
if (!storage->config_.salient.items.properties_on_edges)
if (!storage->config_.items.properties_on_edges)
throw utils::BasicException(
"Can't set properties on edges because properties on edges "
"are disabled!");
@@ -594,8 +572,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
spdlog::trace(" Transaction end");
if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
throw utils::BasicException("Invalid commit data!");
auto ret = commit_timestamp_and_accessor->second.Commit(
{.desired_commit_timestamp = commit_timestamp_and_accessor->first, .is_main = false});
auto ret =
commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first, false /* not main */);
if (ret.HasError())
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
commit_timestamp_and_accessor = std::nullopt;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,6 +12,7 @@
#pragma once
#include "replication/replication_server.hpp"
#include "replication/state.hpp"
#include "storage/v2/replication/serialization.hpp"
namespace memgraph::storage {

View File

@@ -24,7 +24,8 @@ namespace memgraph::dbms {
inline std::unique_ptr<storage::Storage> CreateInMemoryStorage(storage::Config config,
::memgraph::replication::ReplicationState &repl_state) {
const auto name = config.salient.name;
const auto wal_mode = config.durability.snapshot_wal_mode;
const auto name = config.name;
auto storage = std::make_unique<storage::InMemoryStorage>(std::move(config));
// Connect replication state and storage
@@ -33,6 +34,24 @@ inline std::unique_ptr<storage::Storage> CreateInMemoryStorage(storage::Config c
return storage->CreateSnapshot(repl_state.GetRole());
});
if (allow_mt_repl || name == dbms::kDefaultDB) {
// Handle global replication state
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
// RECOVER REPLICA CONNECTIONS
memgraph::dbms::RestoreReplication(repl_state, *storage);
} else if (const ::memgraph::replication::RoleMainData *data =
std::get_if<::memgraph::replication::RoleMainData>(&repl_state.ReplicationData());
data && !data->registered_replicas_.empty()) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
if (wal_mode == storage::Config::Durability::SnapshotWalMode::DISABLED && repl_state.IsMain()) {
spdlog::warn(
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider "
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
"without write-ahead logs this instance is not replicating any data.");
}
return std::move(storage);
}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,7 +10,6 @@
// licenses/APL.txt.
#include "dbms/replication_client.hpp"
#include "replication/replication_client.hpp"
namespace memgraph::dbms {
@@ -18,26 +17,18 @@ void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClien
// No client error, start instance level client
auto const &endpoint = client.rpc_client_.Endpoint();
spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port);
client.StartFrequentCheck([&dbms_handler](bool reconnect, replication::ReplicationClient &client) {
// Working connection
// Check if system needs restoration
if (reconnect) {
client.state_.WithLock([](auto &state) { state = memgraph::replication::ReplicationClient::State::BEHIND; });
}
#ifdef MG_ENTERPRISE
dbms_handler.SystemRestore<true>(client);
#endif
// Check if any database has been left behind
dbms_handler.ForEach([&name = client.name_, reconnect](dbms::DatabaseAccess db_acc) {
client.StartFrequentCheck([&dbms_handler](std::string_view name) {
// Working connection, check if any database has been left behind
dbms_handler.ForEach([name](dbms::Database *db) {
// Specific database <-> replica client
db_acc->storage()->repl_storage_state_.WithClient(name, [&](storage::ReplicationStorageClient *client) {
if (reconnect || client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
db->storage()->repl_storage_state_.WithClient(name, [&](storage::ReplicationStorageClient *client) {
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
// Database <-> replica might be behind, check and recover
client->TryCheckReplicaStateAsync(db_acc->storage(), db_acc);
client->TryCheckReplicaStateAsync(db->storage());
}
});
});
});
} // namespace memgraph::dbms
}
} // namespace memgraph::dbms

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,21 +11,15 @@
#include "dbms/replication_handler.hpp"
#include <algorithm>
#include "dbms/constants.hpp"
#include "dbms/dbms_handler.hpp"
#include "dbms/global.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "dbms/inmemory/storage_helper.hpp"
#include "dbms/replication_client.hpp"
#include "dbms/utils.hpp"
#include "replication/messages.hpp"
#include "replication/state.hpp"
#include "spdlog/spdlog.h"
#include "storage/v2/config.hpp"
#include "storage/v2/replication/rpc.hpp"
#include "utils/on_scope_exit.hpp"
using memgraph::replication::ReplicationClientConfig;
using memgraph::replication::ReplicationState;
using memgraph::replication::RoleMainData;
using memgraph::replication::RoleReplicaData;
@@ -38,8 +32,8 @@ std::string RegisterReplicaErrorToString(RegisterReplicaError error) {
using enum RegisterReplicaError;
case NAME_EXISTS:
return "NAME_EXISTS";
case ENDPOINT_EXISTS:
return "ENDPOINT_EXISTS";
case END_POINT_EXISTS:
return "END_POINT_EXISTS";
case CONNECTION_FAILED:
return "CONNECTION_FAILED";
case COULD_NOT_BE_PERSISTED:
@@ -51,13 +45,34 @@ std::string RegisterReplicaErrorToString(RegisterReplicaError error) {
ReplicationHandler::ReplicationHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {}
bool ReplicationHandler::SetReplicationRoleMain() {
auto const main_handler = [](RoleMainData &) {
auto const main_handler = [](RoleMainData const &) {
// If we are already MAIN, we don't want to change anything
return false;
};
auto const replica_handler = [this](RoleReplicaData const &) {
return memgraph::dbms::DoReplicaToMainPromotion(dbms_handler_);
// STEP 1) bring down all REPLICA servers
dbms_handler_.ForEach([](Database *db) {
auto *storage = db->storage();
// Remember old epoch + storage timestamp association
storage->PrepareForNewEpoch();
});
// STEP 2) Change to MAIN
// TODO: restore replication servers if false?
if (!dbms_handler_.ReplicationState().SetReplicationRoleMain()) {
// TODO: Handle recovery on failure???
return false;
}
// STEP 3) We are now MAIN, update storage local epoch
const auto &epoch =
std::get<RoleMainData>(std::as_const(dbms_handler_.ReplicationState()).ReplicationData()).epoch_;
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
storage->repl_storage_state_.epoch_ = epoch;
});
return true;
};
// TODO: under lock
@@ -74,8 +89,8 @@ bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::
// TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are
// deleting the replica.
// Remove database specific clients
dbms_handler_.ForEach([&](DatabaseAccess db_acc) {
auto *storage = db_acc->storage();
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); });
});
// Remove instance level clients
@@ -90,7 +105,15 @@ bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::
// ASSERT
return false;
},
[this](RoleReplicaData const &data) { return StartRpcServer(dbms_handler_, data); }},
[this](RoleReplicaData const &data) {
// Register handlers
InMemoryReplicationHandlers::Register(&dbms_handler_, *data.server);
if (!data.server->Start()) {
spdlog::error("Unable to start the replication server.");
return false;
}
return true;
}},
dbms_handler_.ReplicationState().ReplicationData());
// TODO Handle error (restore to main?)
return success;
@@ -100,48 +123,60 @@ auto ReplicationHandler::RegisterReplica(const memgraph::replication::Replicatio
-> memgraph::utils::BasicResult<RegisterReplicaError> {
MG_ASSERT(dbms_handler_.ReplicationState().IsMain(), "Only main instance can register a replica!");
auto maybe_client = dbms_handler_.ReplicationState().RegisterReplica(config);
if (maybe_client.HasError()) {
switch (maybe_client.GetError()) {
auto instance_client = dbms_handler_.ReplicationState().RegisterReplica(config);
if (instance_client.HasError()) switch (instance_client.GetError()) {
case memgraph::replication::RegisterReplicaError::NOT_MAIN:
MG_ASSERT(false, "Only main instance can register a replica!");
return {};
case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
return memgraph::dbms::RegisterReplicaError::NAME_EXISTS;
case memgraph::replication::RegisterReplicaError::ENDPOINT_EXISTS:
return memgraph::dbms::RegisterReplicaError::ENDPOINT_EXISTS;
case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS:
return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS;
case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
case memgraph::replication::RegisterReplicaError::SUCCESS:
break;
}
}
if (!allow_mt_repl && dbms_handler_.All().size() > 1) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
#ifdef MG_ENTERPRISE
// Update system before enabling individual storage <-> replica clients
dbms_handler_.SystemRestore(*maybe_client.GetValue());
#endif
bool all_clients_good = true;
const auto dbms_error = memgraph::dbms::HandleRegisterReplicaStatus(maybe_client);
if (dbms_error.has_value()) {
return *dbms_error;
}
auto &instance_client_ptr = maybe_client.GetValue();
const bool all_clients_good = memgraph::dbms::RegisterAllDatabasesClients(dbms_handler_, *instance_client_ptr);
// Add database specific clients (NOTE Currently all databases are connected to each replica)
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
if (!allow_mt_repl && storage->id() != kDefaultDB) {
return;
}
// TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
all_clients_good &=
storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) {
auto client = std::make_unique<storage::ReplicationStorageClient>(*instance_client.GetValue());
client->Start(storage);
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
// Failed to start due to branching of MAIN and REPLICA
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
return false;
}
storage_clients.push_back(std::move(client));
return true;
});
});
// NOTE Currently if any databases fails, we revert back
if (!all_clients_good) {
spdlog::error("Failed to register all databases on the REPLICA \"{}\"", config.name);
spdlog::error("Failed to register all databases to the REPLICA \"{}\"", config.name);
UnregisterReplica(config.name);
return RegisterReplicaError::CONNECTION_FAILED;
}
// No client error, start instance level client
StartReplicaClient(dbms_handler_, *instance_client_ptr);
StartReplicaClient(dbms_handler_, *instance_client.GetValue());
return {};
}
@@ -154,8 +189,8 @@ auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterR
return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED;
}
// Remove database specific clients
dbms_handler_.ForEach([name](DatabaseAccess db_acc) {
db_acc->storage()->repl_storage_state_.replication_clients_.WithLock([&name](auto &clients) {
dbms_handler_.ForEach([name](Database *db) {
db->storage()->repl_storage_state_.replication_clients_.WithLock([&name](auto &clients) {
std::erase_if(clients, [name](const auto &client) { return client->Name() == name; });
});
});
@@ -169,7 +204,7 @@ auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterR
dbms_handler_.ReplicationState().ReplicationData());
}
auto ReplicationHandler::GetRole() const -> memgraph::replication_coordination_glue::ReplicationRole {
auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole {
return dbms_handler_.ReplicationState().GetRole();
}
@@ -179,20 +214,20 @@ bool ReplicationHandler::IsReplica() const { return dbms_handler_.ReplicationSta
// Per storage
// NOTE Storage will connect to all replicas. Future work might change this
void RestoreReplication(replication::ReplicationState &repl_state, DatabaseAccess db_acc) {
void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage) {
spdlog::info("Restoring replication role.");
/// MAIN
auto const recover_main = [db_acc = std::move(db_acc)](RoleMainData &mainData) mutable { // NOLINT
auto const recover_main = [&storage](RoleMainData &mainData) {
// Each individual client has already been restored and started. Here we just go through each database and start its
// client
for (auto &instance_client : mainData.registered_replicas_) {
spdlog::info("Replica {} restoration started for {}.", instance_client.name_, db_acc->name());
const auto &ret = db_acc->storage()->repl_storage_state_.replication_clients_.WithLock(
[&, db_acc](auto &storage_clients) mutable -> utils::BasicResult<RegisterReplicaError> {
spdlog::info("Replica {} restoration started for {}.", instance_client.name_, storage.id());
const auto &ret = storage.repl_storage_state_.replication_clients_.WithLock(
[&](auto &storage_clients) -> utils::BasicResult<RegisterReplicaError> {
auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client);
auto *storage = db_acc->storage();
client->Start(storage, std::move(db_acc));
client->Start(&storage);
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
// Failed to start due to branching of MAIN and REPLICA
@@ -209,7 +244,7 @@ void RestoreReplication(replication::ReplicationState &repl_state, DatabaseAcces
LOG_FATAL("Failure when restoring replica {}: {}.", instance_client.name_,
RegisterReplicaErrorToString(ret.GetError()));
}
spdlog::info("Replica {} restored for {}.", instance_client.name_, db_acc->name());
spdlog::info("Replica {} restored for {}.", instance_client.name_, storage.id());
}
spdlog::info("Replication role restored to MAIN.");
};
@@ -224,177 +259,4 @@ void RestoreReplication(replication::ReplicationState &repl_state, DatabaseAcces
},
repl_state.ReplicationData());
}
namespace system_replication {
#ifdef MG_ENTERPRISE
void SystemHeartbeatHandler(const uint64_t ts, slk::Reader *req_reader, slk::Builder *res_builder) {
replication::SystemHeartbeatReq req;
replication::SystemHeartbeatReq::Load(&req, req_reader);
replication::SystemHeartbeatRes res(ts);
memgraph::slk::Save(res, res_builder);
}
void CreateDatabaseHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder) {
memgraph::storage::replication::CreateDatabaseReq req;
memgraph::slk::Load(&req, req_reader);
using memgraph::storage::replication::CreateDatabaseRes;
CreateDatabaseRes res(CreateDatabaseRes::Result::FAILURE);
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
// of the set of databases. Hence no history exists to maintain regarding epoch change.
// If MAIN has changed we need to check this new group_timestamp is consistent with
// what we have so far.
if (req.expected_group_timestamp != dbms_handler.LastCommitedTS()) {
spdlog::debug("CreateDatabaseHandler: bad expected timestamp {},{}", req.expected_group_timestamp,
dbms_handler.LastCommitedTS());
memgraph::slk::Save(res, res_builder);
return;
}
try {
// Create new
auto new_db = dbms_handler.Update(req.config);
if (new_db.HasValue()) {
// Successfully create db
dbms_handler.SetLastCommitedTS(req.new_group_timestamp);
res = CreateDatabaseRes(CreateDatabaseRes::Result::SUCCESS);
spdlog::debug("CreateDatabaseHandler: SUCCESS updated LCTS to {}", req.new_group_timestamp);
}
} catch (...) {
// Failure
}
memgraph::slk::Save(res, res_builder);
}
void DropDatabaseHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder) {
memgraph::storage::replication::DropDatabaseReq req;
memgraph::slk::Load(&req, req_reader);
using memgraph::storage::replication::DropDatabaseRes;
DropDatabaseRes res(DropDatabaseRes::Result::FAILURE);
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
// of the set of databases. Hence no history exists to maintain regarding epoch change.
// If MAIN has changed we need to check this new group_timestamp is consistent with
// what we have so far.
if (req.expected_group_timestamp != dbms_handler.LastCommitedTS()) {
spdlog::debug("DropDatabaseHandler: bad expected timestamp {},{}", req.expected_group_timestamp,
dbms_handler.LastCommitedTS());
memgraph::slk::Save(res, res_builder);
return;
}
try {
// NOTE: Single communication channel can exist at a time, no other database can be deleted/created at the moment.
auto new_db = dbms_handler.Delete(req.uuid);
if (new_db.HasError()) {
if (new_db.GetError() == DeleteError::NON_EXISTENT) {
// Nothing to drop
dbms_handler.SetLastCommitedTS(req.new_group_timestamp);
res = DropDatabaseRes(DropDatabaseRes::Result::NO_NEED);
}
} else {
// Successfully drop db
dbms_handler.SetLastCommitedTS(req.new_group_timestamp);
res = DropDatabaseRes(DropDatabaseRes::Result::SUCCESS);
spdlog::debug("DropDatabaseHandler: SUCCESS updated LCTS to {}", req.new_group_timestamp);
}
} catch (...) {
// Failure
}
memgraph::slk::Save(res, res_builder);
}
void SystemRecoveryHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder) {
// TODO Speed up
memgraph::storage::replication::SystemRecoveryReq req;
memgraph::slk::Load(&req, req_reader);
using memgraph::storage::replication::SystemRecoveryRes;
SystemRecoveryRes res(SystemRecoveryRes::Result::FAILURE);
utils::OnScopeExit send_on_exit([&]() { memgraph::slk::Save(res, res_builder); });
// Get all current dbs
auto old = dbms_handler.All();
// Check/create the incoming dbs
for (const auto &config : req.database_configs) {
// Missing db
try {
if (dbms_handler.Update(config).HasError()) {
spdlog::debug("SystemRecoveryHandler: Failed to update database \"{}\".", config.name);
return; // Send failure on exit
}
} catch (const UnknownDatabaseException &) {
spdlog::debug("SystemRecoveryHandler: UnknownDatabaseException");
return; // Send failure on exit
}
const auto it = std::find(old.begin(), old.end(), config.name);
if (it != old.end()) old.erase(it);
}
// Delete all the leftover old dbs
for (const auto &remove_db : old) {
const auto del = dbms_handler.Delete(remove_db);
if (del.HasError()) {
// Some errors are not terminal
if (del.GetError() == DeleteError::DEFAULT_DB || del.GetError() == DeleteError::NON_EXISTENT) {
spdlog::debug("SystemRecoveryHandler: Dropped database \"{}\".", remove_db);
continue;
}
spdlog::debug("SystemRecoveryHandler: Failed to drop database \"{}\".", remove_db);
return; // Send failure on exit
}
}
// Successfully recovered
dbms_handler.SetLastCommitedTS(req.forced_group_timestamp);
spdlog::debug("SystemRecoveryHandler: SUCCESS updated LCTS to {}", req.forced_group_timestamp);
res = SystemRecoveryRes(SystemRecoveryRes::Result::SUCCESS);
}
#endif
void Register(replication::RoleReplicaData const &data, dbms::DbmsHandler &dbms_handler) {
#ifdef MG_ENTERPRISE
data.server->rpc_server_.Register<replication::SystemHeartbeatRpc>(
[&dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received SystemHeartbeatRpc");
SystemHeartbeatHandler(dbms_handler.LastCommitedTS(), req_reader, res_builder);
});
data.server->rpc_server_.Register<storage::replication::CreateDatabaseRpc>(
[&dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received CreateDatabaseRpc");
CreateDatabaseHandler(dbms_handler, req_reader, res_builder);
});
data.server->rpc_server_.Register<storage::replication::DropDatabaseRpc>(
[&dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received DropDatabaseRpc");
DropDatabaseHandler(dbms_handler, req_reader, res_builder);
});
data.server->rpc_server_.Register<storage::replication::SystemRecoveryRpc>(
[&dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received SystemRecoveryRpc");
SystemRecoveryHandler(dbms_handler, req_reader, res_builder);
});
#endif
}
} // namespace system_replication
bool StartRpcServer(DbmsHandler &dbms_handler, const replication::RoleReplicaData &data) {
// Register handlers
InMemoryReplicationHandlers::Register(&dbms_handler, *data.server);
system_replication::Register(data, dbms_handler);
// Start server
if (!data.server->Start()) {
spdlog::error("Unable to start the replication server.");
return false;
}
return true;
}
} // namespace memgraph::dbms

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,10 +11,11 @@
#pragma once
#include "replication_coordination_glue/role.hpp"
#include "dbms/database.hpp"
#include "replication/role.hpp"
#include "storage/v2/storage.hpp"
#include "utils/result.hpp"
// BEGIN fwd declares
namespace memgraph::replication {
struct ReplicationState;
struct ReplicationServerConfig;
@@ -22,11 +23,9 @@ struct ReplicationClientConfig;
} // namespace memgraph::replication
namespace memgraph::dbms {
class DbmsHandler;
enum class RegisterReplicaError : uint8_t { NAME_EXISTS, ENDPOINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
enum class UnregisterReplicaResult : uint8_t {
NOT_MAIN,
COULD_NOT_BE_PERSISTED,
@@ -53,7 +52,7 @@ struct ReplicationHandler {
auto UnregisterReplica(std::string_view name) -> UnregisterReplicaResult;
// Helper pass-through (TODO: remove)
auto GetRole() const -> memgraph::replication_coordination_glue::ReplicationRole;
auto GetRole() const -> memgraph::replication::ReplicationRole;
bool IsMain() const;
bool IsReplica() const;
@@ -63,20 +62,6 @@ struct ReplicationHandler {
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
/// TODO: extend to do multiple storages
void RestoreReplication(replication::ReplicationState &repl_state, DatabaseAccess db_acc);
namespace system_replication {
// System handlers
#ifdef MG_ENTERPRISE
void CreateDatabaseHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
void SystemHeartbeatHandler(uint64_t ts, slk::Reader *req_reader, slk::Builder *res_builder);
void SystemRecoveryHandler(DbmsHandler &dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
#endif
/// Register all DBMS level RPC handlers
void Register(replication::RoleReplicaData const &data, DbmsHandler &dbms_handler);
} // namespace system_replication
bool StartRpcServer(DbmsHandler &dbms_handler, const replication::RoleReplicaData &data);
void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage);
} // namespace memgraph::dbms

View File

@@ -1,64 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include "storage/v2/config.hpp"
namespace memgraph::dbms {
struct SystemTransaction {
struct Delta {
enum class Action {
CREATE_DATABASE,
DROP_DATABASE,
};
static constexpr struct CreateDatabase {
} create_database;
static constexpr struct DropDatabase {
} drop_database;
Delta(CreateDatabase /*tag*/, storage::SalientConfig config)
: action(Action::CREATE_DATABASE), config(std::move(config)) {}
Delta(DropDatabase /*tag*/, const utils::UUID &uuid) : action(Action::DROP_DATABASE), uuid(uuid) {}
Delta(const Delta &) = delete;
Delta(Delta &&) = delete;
Delta &operator=(const Delta &) = delete;
Delta &operator=(Delta &&) = delete;
~Delta() {
switch (action) {
case Action::CREATE_DATABASE:
std::destroy_at(&config);
break;
case Action::DROP_DATABASE:
break;
// Some deltas might have special destructor handling
}
}
Action action;
union {
storage::SalientConfig config;
utils::UUID uuid;
};
};
explicit SystemTransaction(uint64_t timestamp) : system_timestamp(timestamp) {}
// Currently system transitions support a single delta
std::optional<Delta> delta{};
uint64_t system_timestamp;
};
} // namespace memgraph::dbms

View File

@@ -1,133 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "dbms/dbms_handler.hpp"
#include "dbms/replication_handler.hpp"
#include "replication/include/replication/state.hpp"
#include "utils/result.hpp"
namespace memgraph::dbms {
inline bool DoReplicaToMainPromotion(dbms::DbmsHandler &dbms_handler) {
auto &repl_state = dbms_handler.ReplicationState();
// STEP 1) bring down all REPLICA servers
dbms_handler.ForEach([](DatabaseAccess db_acc) {
auto *storage = db_acc->storage();
// Remember old epoch + storage timestamp association
storage->PrepareForNewEpoch();
});
// STEP 2) Change to MAIN
// TODO: restore replication servers if false?
if (!repl_state.SetReplicationRoleMain()) {
// TODO: Handle recovery on failure???
return false;
}
// STEP 3) We are now MAIN, update storage local epoch
const auto &epoch =
std::get<replication::RoleMainData>(std::as_const(dbms_handler.ReplicationState()).ReplicationData()).epoch_;
dbms_handler.ForEach([&](DatabaseAccess db_acc) {
auto *storage = db_acc->storage();
storage->repl_storage_state_.epoch_ = epoch;
});
return true;
};
inline bool SetReplicationRoleReplica(dbms::DbmsHandler &dbms_handler,
const memgraph::replication::ReplicationServerConfig &config) {
if (dbms_handler.ReplicationState().IsReplica()) {
return false;
}
// TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are
// deleting the replica.
// Remove database specific clients
dbms_handler.ForEach([&](DatabaseAccess db_acc) {
auto *storage = db_acc->storage();
storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); });
});
// Remove instance level clients
std::get<replication::RoleMainData>(dbms_handler.ReplicationState().ReplicationData()).registered_replicas_.clear();
// Creates the server
dbms_handler.ReplicationState().SetReplicationRoleReplica(config);
// Start
const auto success = std::visit(utils::Overloaded{[](replication::RoleMainData const &) {
// ASSERT
return false;
},
[&dbms_handler](replication::RoleReplicaData const &data) {
return StartRpcServer(dbms_handler, data);
}},
dbms_handler.ReplicationState().ReplicationData());
// TODO Handle error (restore to main?)
return success;
}
inline bool RegisterAllDatabasesClients(dbms::DbmsHandler &dbms_handler,
replication::ReplicationClient &instance_client) {
if (!allow_mt_repl && dbms_handler.All().size() > 1) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
bool all_clients_good = true;
// Add database specific clients (NOTE Currently all databases are connected to each replica)
dbms_handler.ForEach([&](DatabaseAccess db_acc) {
auto *storage = db_acc->storage();
if (!allow_mt_repl && storage->name() != kDefaultDB) {
return;
}
// TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
all_clients_good &= storage->repl_storage_state_.replication_clients_.WithLock(
[storage, &instance_client, db_acc = std::move(db_acc)](auto &storage_clients) mutable { // NOLINT
auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client);
// All good, start replica client
client->Start(storage, std::move(db_acc));
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
// Failed to start due an error like branching of MAIN and REPLICA
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
return false; // TODO: sometimes we need to still add to storage_clients
}
storage_clients.push_back(std::move(client));
return true;
});
});
return all_clients_good;
}
inline std::optional<RegisterReplicaError> HandleRegisterReplicaStatus(
utils::BasicResult<replication::RegisterReplicaError, replication::ReplicationClient *> &instance_client) {
if (instance_client.HasError()) switch (instance_client.GetError()) {
case replication::RegisterReplicaError::NOT_MAIN:
MG_ASSERT(false, "Only main instance can register a replica!");
return {};
case replication::RegisterReplicaError::NAME_EXISTS:
return dbms::RegisterReplicaError::NAME_EXISTS;
case replication::RegisterReplicaError::ENDPOINT_EXISTS:
return dbms::RegisterReplicaError::ENDPOINT_EXISTS;
case replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
return dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
case replication::RegisterReplicaError::SUCCESS:
break;
}
return {};
}
} // namespace memgraph::dbms

View File

@@ -6,7 +6,6 @@ add_library(mg-flags STATIC audit.cpp
memory_limit.cpp
run_time_configurable.cpp
storage_mode.cpp
query.cpp
replication.cpp)
query.cpp)
target_include_directories(mg-flags PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-flags PUBLIC spdlog::spdlog mg-settings mg-utils)

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,6 +17,5 @@
#include "flags/log_level.hpp"
#include "flags/memory_limit.hpp"
#include "flags/query.hpp"
#include "flags/replication.hpp"
#include "flags/run_time_configurable.hpp"
#include "flags/storage_mode.hpp"

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -131,6 +131,12 @@ DEFINE_uint64(storage_recovery_thread_count,
DEFINE_bool(storage_enable_schema_metadata, false,
"Controls whether metadata should be collected about the resident labels and edge types.");
#ifdef MG_ENTERPRISE
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_delete_on_drop, true,
"If set to true the query 'DROP DATABASE x' will delete the underlying storage as well.");
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(telemetry_enabled, false,
"Set to true to enable telemetry. We collect information about the "
@@ -156,6 +162,13 @@ DEFINE_string(pulsar_service_url, "", "Default URL used while connecting to Puls
// Query flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(replication_replica_check_frequency_sec, 1,
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
"The MAIN instance allocates a new thread for each REPLICA.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(replication_restore_state_on_startup, false, "Restore replication state on startup, e.g. recover replica");
DEFINE_VALIDATED_string(query_modules_directory, "",
"Directory where modules with custom query procedures are stored. "
"NOTE: Multiple comma-separated directories can be defined.",
@@ -195,9 +208,3 @@ DEFINE_HIDDEN_string(organization_name, "", "Organization name.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(auth_user_or_role_name_regex, memgraph::glue::kDefaultUserRoleRegex.data(),
"Set to the regular expression that each user or role name must fulfill.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(auth_password_permit_null, true, "Set to false to disable null passwords.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(auth_password_strength_regex, memgraph::glue::kDefaultPasswordRegex.data(),
"The regular expression that should be used to match the entire "
"entered password to ensure its strength.");

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -84,6 +84,10 @@ DECLARE_bool(storage_parallel_schema_recovery);
DECLARE_uint64(storage_recovery_thread_count);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(storage_enable_schema_metadata);
#ifdef MG_ENTERPRISE
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(storage_delete_on_drop);
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(telemetry_enabled);
@@ -112,13 +116,14 @@ namespace memgraph::flags {
auto ParseQueryModulesDirectory() -> std::vector<std::filesystem::path>;
} // namespace memgraph::flags
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint64(replication_replica_check_frequency_sec);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(replication_restore_state_on_startup);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(license_key);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(organization_name);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(auth_user_or_role_name_regex);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(auth_password_permit_null);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(auth_password_strength_regex);

View File

@@ -1,26 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "replication.hpp"
#ifdef MG_ENTERPRISE
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(coordinator, false, "Controls whether the instance is a replication coordinator.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(coordinator_server_port, 0, "Port on which coordinator servers will be started.");
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(replication_replica_check_frequency_sec, 1,
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
"The MAIN instance allocates a new thread for each REPLICA.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(replication_restore_state_on_startup, false, "Restore replication state on startup, e.g. recover replica");

View File

@@ -1,26 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "gflags/gflags.h"
#ifdef MG_ENTERPRISE
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(coordinator);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint32(coordinator_server_port);
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint64(replication_replica_check_frequency_sec);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(replication_restore_state_on_startup);

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,14 +17,20 @@
#include "gflags/gflags.h"
#include <array>
inline constexpr std::array storage_mode_mappings{
std::pair{std::string_view{"IN_MEMORY_TRANSACTIONAL"}, memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL},
std::pair{std::string_view{"IN_MEMORY_ANALYTICAL"}, memgraph::storage::StorageMode::IN_MEMORY_ANALYTICAL},
std::pair{std::string_view{"ON_DISK_TRANSACTIONAL"}, memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL}};
const std::string storage_mode_help_string =
fmt::format("Default storage mode Memgraph uses. Allowed values: {}",
memgraph::utils::GetAllowedEnumValuesString(memgraph::storage::storage_mode_mappings));
memgraph::utils::GetAllowedEnumValuesString(storage_mode_mappings));
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_string(storage_mode, "IN_MEMORY_TRANSACTIONAL", storage_mode_help_string.c_str(), {
if (const auto result = memgraph::utils::IsValidEnumValueString(value, memgraph::storage::storage_mode_mappings);
result.HasError()) {
if (const auto result = memgraph::utils::IsValidEnumValueString(value, storage_mode_mappings); result.HasError()) {
switch (result.GetError()) {
case memgraph::utils::ValidationError::EmptyValue: {
std::cout << "Storage mode cannot be empty." << std::endl;
@@ -32,7 +38,7 @@ DEFINE_VALIDATED_string(storage_mode, "IN_MEMORY_TRANSACTIONAL", storage_mode_he
}
case memgraph::utils::ValidationError::InvalidValue: {
std::cout << "Invalid value for storage mode. Allowed values: "
<< memgraph::utils::GetAllowedEnumValuesString(memgraph::storage::storage_mode_mappings) << std::endl;
<< memgraph::utils::GetAllowedEnumValuesString(storage_mode_mappings) << std::endl;
break;
}
}
@@ -42,8 +48,8 @@ DEFINE_VALIDATED_string(storage_mode, "IN_MEMORY_TRANSACTIONAL", storage_mode_he
});
memgraph::storage::StorageMode memgraph::flags::ParseStorageMode() {
const auto storage_mode = memgraph::utils::StringToEnum<memgraph::storage::StorageMode>(
FLAGS_storage_mode, memgraph::storage::storage_mode_mappings);
const auto storage_mode =
memgraph::utils::StringToEnum<memgraph::storage::StorageMode>(FLAGS_storage_mode, storage_mode_mappings);
MG_ASSERT(storage_mode, "Invalid storage mode");
return *storage_mode;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -80,7 +80,7 @@ std::vector<memgraph::communication::bolt::Value> TypedValueResultStreamBase::De
std::vector<memgraph::communication::bolt::Value> decoded_values;
decoded_values.reserve(values.size());
for (const auto &v : values) {
auto maybe_value = memgraph::glue::ToBoltValue(v, storage_, memgraph::storage::View::NEW);
auto maybe_value = memgraph::glue::ToBoltValue(v, *storage_, memgraph::storage::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:
@@ -112,14 +112,14 @@ std::string SessionHL::GetDefaultDB() {
if (user_.has_value()) {
return user_->db_access().GetDefault();
}
return std::string{memgraph::dbms::kDefaultDB};
return memgraph::dbms::kDefaultDB;
}
#endif
std::string SessionHL::GetCurrentDB() const {
if (!interpreter_.current_db_.db_acc_) return "";
const auto *db = interpreter_.current_db_.db_acc_->get();
return db->name();
return db->id();
}
std::optional<std::string> SessionHL::GetServerNameForInit() {
@@ -167,10 +167,10 @@ std::map<std::string, memgraph::communication::bolt::Value> SessionHL::Discard(s
std::map<std::string, memgraph::communication::bolt::Value> SessionHL::Pull(SessionHL::TEncoder *encoder,
std::optional<int> n,
std::optional<int> qid) {
// TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt)
auto *db = interpreter_.current_db_.db_acc_->get();
try {
auto &db = interpreter_.current_db_.db_acc_;
auto *storage = db ? db->get()->storage() : nullptr;
TypedValueResultStream<TEncoder> stream(encoder, storage);
TypedValueResultStream<TEncoder> stream(encoder, db->storage());
return DecodeSummary(interpreter_.Pull(&stream, n, qid));
} catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
@@ -193,17 +193,17 @@ std::pair<std::vector<std::string>, std::optional<int>> SessionHL::Interpret(
for (const auto &[key, bolt_param] : params) {
params_pv.emplace(key, ToPropertyValue(bolt_param));
}
#ifdef MG_ENTERPRISE
const std::string *username{nullptr};
if (user_) {
username = &user_->username();
}
#ifdef MG_ENTERPRISE
// TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt)
auto *db = interpreter_.current_db_.db_acc_->get();
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
auto &db = interpreter_.current_db_.db_acc_;
audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query,
memgraph::storage::PropertyValue(params_pv), db ? db->get()->name() : "no known database");
memgraph::storage::PropertyValue(params_pv), db->id());
}
#endif
try {
@@ -351,11 +351,11 @@ SessionHL::~SessionHL() {
std::map<std::string, memgraph::communication::bolt::Value> SessionHL::DecodeSummary(
const std::map<std::string, memgraph::query::TypedValue> &summary) {
auto &db_acc = interpreter_.current_db_.db_acc_;
auto *storage = db_acc ? db_acc->get()->storage() : nullptr;
// TODO: Update once interpreter can handle non-database queries (db_acc will be nullopt)
auto *db = interpreter_.current_db_.db_acc_->get();
std::map<std::string, memgraph::communication::bolt::Value> decoded_summary;
for (const auto &kv : summary) {
auto maybe_value = ToBoltValue(kv.second, storage, memgraph::storage::View::NEW);
auto maybe_value = ToBoltValue(kv.second, *db->storage(), memgraph::storage::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -66,8 +66,6 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::MULTI_DATABASE_EDIT;
case query::AuthQuery::Privilege::MULTI_DATABASE_USE:
return auth::Permission::MULTI_DATABASE_USE;
case query::AuthQuery::Privilege::COORDINATOR:
return auth::Permission::COORDINATOR;
}
}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,5 +13,4 @@
namespace memgraph::glue {
inline constexpr std::string_view kDefaultUserRoleRegex = "[a-zA-Z0-9_.+-@]+";
static constexpr std::string_view kDefaultPasswordRegex = ".+";
} // namespace memgraph::glue

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -249,10 +249,25 @@ std::vector<std::vector<memgraph::query::TypedValue>> ShowFineGrainedRolePrivile
namespace memgraph::glue {
AuthQueryHandler::AuthQueryHandler(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth)
: auth_(auth) {}
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::string name_regex_string)
: auth_(auth), name_regex_string_(std::move(name_regex_string)), name_regex_(name_regex_string_) {}
bool AuthQueryHandler::CreateUser(const std::string &username, const std::optional<std::string> &password) {
if (name_regex_string_ != kDefaultUserRoleRegex) {
if (const auto license_check_result =
memgraph::license::global_license_checker.IsEnterpriseValid(memgraph::utils::global_settings);
license_check_result.HasError()) {
throw memgraph::auth::AuthException(
"Custom user/role regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-user-or-role-name-regex\") to its default value (\"{}\") or remove the flag.\n{}",
kDefaultUserRoleRegex,
memgraph::license::LicenseCheckErrorToString(license_check_result.GetError(), "user/role regex"));
}
}
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
try {
const auto [first_user, user_added] = std::invoke([&, this] {
auto locked_auth = auth_->Lock();
@@ -279,7 +294,7 @@ bool AuthQueryHandler::CreateUser(const std::string &username, const std::option
);
#ifdef MG_ENTERPRISE
GrantDatabaseToUser(auth::kAllDatabases, username);
SetMainDatabase(dbms::kDefaultDB, username);
SetMainDatabase(username, dbms::kDefaultDB);
#endif
}
@@ -290,6 +305,9 @@ bool AuthQueryHandler::CreateUser(const std::string &username, const std::option
}
bool AuthQueryHandler::DropUser(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
@@ -301,13 +319,16 @@ bool AuthQueryHandler::DropUser(const std::string &username) {
}
void AuthQueryHandler::SetPassword(const std::string &username, const std::optional<std::string> &password) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist.", username);
}
locked_auth->UpdatePassword(*user, password);
user->UpdatePassword(password);
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
@@ -315,6 +336,9 @@ void AuthQueryHandler::SetPassword(const std::string &username, const std::optio
}
bool AuthQueryHandler::CreateRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
return locked_auth->AddRole(rolename).has_value();
@@ -325,6 +349,9 @@ bool AuthQueryHandler::CreateRole(const std::string &rolename) {
#ifdef MG_ENTERPRISE
bool AuthQueryHandler::RevokeDatabaseFromUser(const std::string &db, const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
@@ -336,6 +363,9 @@ bool AuthQueryHandler::RevokeDatabaseFromUser(const std::string &db, const std::
}
bool AuthQueryHandler::GrantDatabaseToUser(const std::string &db, const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
@@ -348,6 +378,9 @@ bool AuthQueryHandler::GrantDatabaseToUser(const std::string &db, const std::str
std::vector<std::vector<memgraph::query::TypedValue>> AuthQueryHandler::GetDatabasePrivileges(
const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto user = locked_auth->GetUser(username);
@@ -360,7 +393,10 @@ std::vector<std::vector<memgraph::query::TypedValue>> AuthQueryHandler::GetDatab
}
}
bool AuthQueryHandler::SetMainDatabase(std::string_view db, const std::string &username) {
bool AuthQueryHandler::SetMainDatabase(const std::string &db, const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
@@ -381,6 +417,9 @@ void AuthQueryHandler::DeleteDatabase(std::string_view db) {
#endif
bool AuthQueryHandler::DropRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
auto role = locked_auth->GetRole(rolename);
@@ -426,6 +465,9 @@ std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetRolenames() {
}
std::optional<std::string> AuthQueryHandler::GetRolenameForUser(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto user = locked_auth->GetUser(username);
@@ -443,6 +485,9 @@ std::optional<std::string> AuthQueryHandler::GetRolenameForUser(const std::strin
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetUsernamesForRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto role = locked_auth->GetRole(rolename);
@@ -462,6 +507,12 @@ std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetUsernamesForRole(c
}
void AuthQueryHandler::SetRole(const std::string &username, const std::string &rolename) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
@@ -484,6 +535,9 @@ void AuthQueryHandler::SetRole(const std::string &username, const std::string &r
}
void AuthQueryHandler::ClearRole(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
@@ -498,6 +552,9 @@ void AuthQueryHandler::ClearRole(const std::string &username) {
}
std::vector<std::vector<memgraph::query::TypedValue>> AuthQueryHandler::GetPrivileges(const std::string &user_or_role) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
auto locked_auth = auth_->ReadLock();
std::vector<std::vector<memgraph::query::TypedValue>> grants;
@@ -647,6 +704,9 @@ void AuthQueryHandler::EditPermissions(
const TEditFineGrainedPermissionsFun &edit_fine_grained_permissions_fun
#endif
) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
std::vector<memgraph::auth::Permission> permissions;
permissions.reserve(privileges.size());

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,9 +24,12 @@ namespace memgraph::glue {
class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
std::string name_regex_string_;
std::regex name_regex_;
public:
AuthQueryHandler(memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth);
AuthQueryHandler(memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::string name_regex_string);
bool CreateUser(const std::string &username, const std::optional<std::string> &password) override;
@@ -41,7 +44,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
std::vector<std::vector<memgraph::query::TypedValue>> GetDatabasePrivileges(const std::string &username) override;
bool SetMainDatabase(std::string_view db, const std::string &username) override;
bool SetMainDatabase(const std::string &db, const std::string &username) override;
void DeleteDatabase(std::string_view db) override;
#endif

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -73,14 +73,8 @@ storage::Result<communication::bolt::Edge> ToBoltEdge(const query::EdgeAccessor
return ToBoltEdge(edge.impl_, db, view);
}
storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage::Storage *db, storage::View view) {
auto check_db = [db]() {
if (db == nullptr) [[unlikely]]
throw communication::bolt::ValueException("Database needed for TypeValue conversion.");
};
storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage::Storage &db, storage::View view) {
switch (value.type()) {
// No database needed
case query::TypedValue::Type::Null:
return Value();
case query::TypedValue::Type::Bool:
@@ -91,29 +85,7 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage
return Value(value.ValueDouble());
case query::TypedValue::Type::String:
return Value(std::string(value.ValueString()));
case query::TypedValue::Type::Date:
return Value(value.ValueDate());
case query::TypedValue::Type::LocalTime:
return Value(value.ValueLocalTime());
case query::TypedValue::Type::LocalDateTime:
return Value(value.ValueLocalDateTime());
case query::TypedValue::Type::Duration:
return Value(value.ValueDuration());
// Database potentially not required
case query::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return Value(std::move(map));
}
// Database is required
case query::TypedValue::Type::List: {
check_db();
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
@@ -123,35 +95,44 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage
}
return Value(std::move(values));
}
case query::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return Value(std::move(map));
}
case query::TypedValue::Type::Vertex: {
check_db();
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), *db, view);
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::TypedValue::Type::Edge: {
check_db();
auto maybe_edge = ToBoltEdge(value.ValueEdge(), *db, view);
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::TypedValue::Type::Path: {
check_db();
auto maybe_path = ToBoltPath(value.ValuePath(), *db, view);
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::TypedValue::Type::Graph: {
check_db();
auto maybe_graph = ToBoltGraph(value.ValueGraph(), *db, view);
case query::TypedValue::Type::Date:
return Value(value.ValueDate());
case query::TypedValue::Type::LocalTime:
return Value(value.ValueLocalTime());
case query::TypedValue::Type::LocalDateTime:
return Value(value.ValueLocalDateTime());
case query::TypedValue::Type::Duration:
return Value(value.ValueDuration());
case query::TypedValue::Type::Function:
throw communication::bolt::ValueException("Unsupported conversion from TypedValue::Function to Value");
case query::TypedValue::Type::Graph:
auto maybe_graph = ToBoltGraph(value.ValueGraph(), db, view);
if (maybe_graph.HasError()) return maybe_graph.GetError();
return Value(std::move(*maybe_graph));
}
// Unsupported conversions
case query::TypedValue::Type::Function: {
throw communication::bolt::ValueException("Unsupported conversion from TypedValue::Function to Value");
}
}
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -65,7 +65,7 @@ storage::Result<std::map<std::string, communication::bolt::Value>> ToBoltGraph(c
/// @param storage::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::Result<communication::bolt::Value> ToBoltValue(const query::TypedValue &value, const storage::Storage *db,
storage::Result<communication::bolt::Value> ToBoltValue(const query::TypedValue &value, const storage::Storage &db,
storage::View view);
query::TypedValue ToTypedValue(const communication::bolt::Value &value);

View File

@@ -8,5 +8,4 @@ find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-io STATIC ${io_src_files})
add_library(mg::io ALIAS mg-io)
target_link_libraries(mg-io stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@@ -166,7 +166,7 @@ bool Endpoint::IsResolvableAddress(const std::string &address, uint16_t port) {
}
std::optional<std::pair<std::string, uint16_t>> Endpoint::ParseSocketOrAddress(
const std::string &address, const std::optional<uint16_t> default_port) {
const std::string &address, const std::optional<uint16_t> default_port = {}) {
const std::string delimiter = ":";
std::vector<std::string> parts = utils::Split(address, delimiter);
if (parts.size() == 1) {

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,7 +51,7 @@ KVStore &KVStore::operator=(KVStore &&other) {
return *this;
}
bool KVStore::Put(std::string_view key, std::string_view value) {
bool KVStore::Put(const std::string &key, const std::string &value) {
auto s = pimpl_->db->Put(rocksdb::WriteOptions(), key, value);
return s.ok();
}
@@ -65,7 +65,7 @@ bool KVStore::PutMultiple(const std::map<std::string, std::string> &items) {
return s.ok();
}
std::optional<std::string> KVStore::Get(std::string_view key) const noexcept {
std::optional<std::string> KVStore::Get(const std::string &key) const noexcept {
std::string value;
auto s = pimpl_->db->Get(rocksdb::ReadOptions(), key, &value);
if (!s.ok()) return std::nullopt;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -61,7 +61,7 @@ class KVStore final {
* @return true if the value has been successfully stored.
* In case of any error false is going to be returned.
*/
bool Put(std::string_view key, std::string_view value);
bool Put(const std::string &key, const std::string &value);
/**
* Store values under the given keys.
@@ -81,7 +81,7 @@ class KVStore final {
* @return Value for the given key. std::nullopt in case of any error
* OR the value doesn't exist.
*/
std::optional<std::string> Get(std::string_view key) const noexcept;
std::optional<std::string> Get(const std::string &key) const noexcept;
/**
* Deletes the key and corresponding value from storage.

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -92,7 +92,7 @@ void RegisterLicenseSettings(LicenseChecker &license_checker, utils::Settings &s
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
LicenseChecker global_license_checker;
LicenseChecker::~LicenseChecker() { Finalize(); }
LicenseChecker::~LicenseChecker() { scheduler_.Stop(); }
std::pair<std::string, std::string> LicenseChecker::ExtractLicenseInfo(const utils::Settings &settings) const {
if (license_info_override_) {

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -89,8 +89,6 @@ struct LicenseChecker {
utils::Synchronized<std::optional<LicenseInfo>, utils::SpinLock> &GetLicenseInfo();
void Finalize() { scheduler_.Stop(); }
private:
std::pair<std::string, std::string> ExtractLicenseInfo(const utils::Settings &settings) const;
void RevalidateLicense(const utils::Settings &settings);

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,11 +11,13 @@
#include <cstdint>
#include "audit/log.hpp"
#include "communication/metrics.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "dbms/constants.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "flags/all.hpp"
#include "flags/run_time_configurable.hpp"
#include "glue/MonitoringServerT.hpp"
#include "glue/ServerT.hpp"
#include "glue/auth_checker.hpp"
@@ -31,9 +33,9 @@
#include "query/procedure/module.hpp"
#include "query/procedure/py_module.hpp"
#include "requests/requests.hpp"
#include "storage/v2/durability/durability.hpp"
#include "telemetry/telemetry.hpp"
#include "utils/signals.hpp"
#include "utils/skip_list.hpp"
#include "utils/sysinfo/memory.hpp"
#include "utils/system_info.hpp"
#include "utils/terminate_handler.hpp"
@@ -71,7 +73,7 @@ void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbm
spdlog::warn("{} The rest of the init-file will be run.", e.what());
}
if (audit_log) {
audit_log->Record("", "", line, {}, std::string{memgraph::dbms::kDefaultDB});
audit_log->Record("", "", line, {}, memgraph::dbms::kDefaultDB);
}
}
}
@@ -255,8 +257,6 @@ int main(int argc, char **argv) {
// register all runtime settings
memgraph::license::RegisterLicenseSettings(memgraph::license::global_license_checker,
memgraph::utils::global_settings);
memgraph::utils::OnScopeExit global_license_finalizer([] { memgraph::license::global_license_checker.Finalize(); });
memgraph::flags::run_time::Initialize();
memgraph::license::global_license_checker.CheckEnvLicense();
@@ -298,7 +298,8 @@ int main(int argc, char **argv) {
memgraph::storage::Config db_config{
.gc = {.type = memgraph::storage::Config::Gc::Type::PERIODIC,
.interval = std::chrono::seconds(FLAGS_storage_gc_cycle_sec)},
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges,
.enable_schema_metadata = FLAGS_storage_enable_schema_metadata},
.durability = {.storage_directory = FLAGS_data_directory,
.recover_on_startup = FLAGS_storage_recover_on_startup || FLAGS_data_recovery_on_startup,
.snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
@@ -320,9 +321,7 @@ int main(int argc, char **argv) {
.id_name_mapper_directory = FLAGS_data_directory + "/rocksdb_id_name_mapper",
.durability_directory = FLAGS_data_directory + "/rocksdb_durability",
.wal_directory = FLAGS_data_directory + "/rocksdb_wal"},
.salient.items = {.properties_on_edges = FLAGS_storage_properties_on_edges,
.enable_schema_metadata = FLAGS_storage_enable_schema_metadata},
.salient.storage_mode = memgraph::flags::ParseStorageMode()};
.storage_mode = memgraph::flags::ParseStorageMode()};
memgraph::utils::Scheduler jemalloc_purge_scheduler;
jemalloc_purge_scheduler.Run("Jemalloc purge", std::chrono::seconds(FLAGS_storage_gc_cycle_sec),
@@ -357,10 +356,11 @@ int main(int argc, char **argv) {
.stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)};
auto auth_glue =
[](memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::unique_ptr<memgraph::query::AuthQueryHandler> &ah, std::unique_ptr<memgraph::query::AuthChecker> &ac) {
[flag = FLAGS_auth_user_or_role_name_regex](
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::unique_ptr<memgraph::query::AuthQueryHandler> &ah, std::unique_ptr<memgraph::query::AuthChecker> &ac) {
// Glue high level auth implementations to the query side
ah = std::make_unique<memgraph::glue::AuthQueryHandler>(auth);
ah = std::make_unique<memgraph::glue::AuthQueryHandler>(auth, flag);
ac = std::make_unique<memgraph::glue::AuthChecker>(auth);
// Handle users passed via arguments
auto *maybe_username = std::getenv(kMgUser);
@@ -376,10 +376,9 @@ int main(int argc, char **argv) {
}
};
memgraph::auth::Auth::Config auth_config{FLAGS_auth_user_or_role_name_regex, FLAGS_auth_password_strength_regex,
FLAGS_auth_password_permit_null};
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> auth_{
data_directory / "auth", auth_config};
// WIP
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> auth_{data_directory /
"auth"};
std::unique_ptr<memgraph::query::AuthQueryHandler> auth_handler;
std::unique_ptr<memgraph::query::AuthChecker> auth_checker;
auth_glue(&auth_, auth_handler, auth_checker);
@@ -387,7 +386,7 @@ int main(int argc, char **argv) {
memgraph::dbms::DbmsHandler dbms_handler(db_config
#ifdef MG_ENTERPRISE
,
&auth_, FLAGS_data_recovery_on_startup
&auth_, FLAGS_data_recovery_on_startup, FLAGS_storage_delete_on_drop
#endif
);
auto db_acc = dbms_handler.Get();
@@ -547,7 +546,6 @@ int main(int argc, char **argv) {
memgraph::query::procedure::gModuleRegistry.UnloadAllModules();
python_gc_scheduler.Stop();
Py_END_ALLOW_THREADS;
// Shutdown Python
Py_Finalize();

View File

@@ -32,7 +32,7 @@
#include "utils/timer.hpp"
#include "version.hpp"
using memgraph::replication_coordination_glue::ReplicationRole;
using memgraph::replication::ReplicationRole;
bool ValidateControlCharacter(const char *flagname, const std::string &value) {
if (value.empty()) {
@@ -707,11 +707,12 @@ int main(int argc, char *argv[]) {
std::unordered_map<NodeId, memgraph::storage::Gid> node_id_map;
memgraph::storage::Config config{
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
.durability = {.storage_directory = FLAGS_data_directory,
.recover_on_startup = false,
.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::DISABLED,
.snapshot_on_exit = true},
.salient = {.items = {.properties_on_edges = FLAGS_storage_properties_on_edges}},
};
memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)};
auto store = memgraph::dbms::CreateInMemoryStorage(config, repl_state);

View File

@@ -57,7 +57,7 @@ class AuthQueryHandler {
/// Return true if main database set successfully
/// @throw QueryRuntimeException if an error ocurred.
virtual bool SetMainDatabase(std::string_view db, const std::string &username) = 0;
virtual bool SetMainDatabase(const std::string &db, const std::string &username) = 0;
/// Delete database from all users
/// @throw QueryRuntimeException if an error ocurred.

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -72,9 +72,8 @@ class TypedValueVectorCompare final {
/// Raise QueryRuntimeException if the value for symbol isn't of expected type.
inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue::Type expected) {
if (value.type() != expected) [[unlikely]] {
if (value.type() != expected)
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
}
inline void ProcessError(const storage::Error error) {

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -541,10 +541,7 @@ class DbAccessor final {
void AdvanceCommand() { accessor_->AdvanceCommand(); }
utils::BasicResult<storage::StorageManipulationError, void> Commit(storage::CommitReplArgs reparg = {},
storage::DatabaseAccessProtector db_acc = {}) {
return accessor_->Commit(std::move(reparg), std::move(db_acc));
}
utils::BasicResult<storage::StorageManipulationError, void> Commit() { return accessor_->Commit(); }
void Abort() { accessor_->Abort(); }

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -195,12 +195,6 @@ class DatabaseContextRequiredException : public QueryRuntimeException {
SPECIALIZE_GET_EXCEPTION_NAME(DatabaseContextRequiredException)
};
class ConcurrentSystemQueriesException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
SPECIALIZE_GET_EXCEPTION_NAME(ConcurrentSystemQueriesException)
};
class WriteVertexOperationInEdgeImportModeException : public QueryException {
public:
WriteVertexOperationInEdgeImportModeException()
@@ -259,13 +253,6 @@ class ReplicationModificationInMulticommandTxException : public QueryException {
SPECIALIZE_GET_EXCEPTION_NAME(ReplicationModificationInMulticommandTxException)
};
class CoordinatorModificationInMulticommandTxException : public QueryException {
public:
CoordinatorModificationInMulticommandTxException()
: QueryException("Coordinator clause not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(CoordinatorModificationInMulticommandTxException)
};
class ReplicationDisabledOnDiskStorage : public QueryException {
public:
ReplicationDisabledOnDiskStorage() : QueryException("Replication is not supported while in on-disk storage mode.") {}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -239,9 +239,6 @@ constexpr utils::TypeInfo query::DumpQuery::kType{utils::TypeId::AST_DUMP_QUERY,
constexpr utils::TypeInfo query::ReplicationQuery::kType{utils::TypeId::AST_REPLICATION_QUERY, "ReplicationQuery",
&query::Query::kType};
constexpr utils::TypeInfo query::CoordinatorQuery::kType{utils::TypeId::AST_COORDINATOR_QUERY, "CoordinatorQuery",
&query::Query::kType};
constexpr utils::TypeInfo query::LockPathQuery::kType{utils::TypeId::AST_LOCK_PATH_QUERY, "LockPathQuery",
&query::Query::kType};

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -1209,8 +1209,7 @@ class PropertyLookup : public memgraph::query::Expression {
}
protected:
PropertyLookup(Expression *expression, PropertyIx property)
: expression_(expression), property_(std::move(property)) {}
PropertyLookup(Expression *expression, PropertyIx property) : expression_(expression), property_(property) {}
private:
friend class AstStorage;
@@ -1806,9 +1805,9 @@ class EdgeAtom : public memgraph::query::PatternAtom {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class Type : uint8_t { SINGLE, DEPTH_FIRST, BREADTH_FIRST, WEIGHTED_SHORTEST_PATH, ALL_SHORTEST_PATHS };
enum class Type { SINGLE, DEPTH_FIRST, BREADTH_FIRST, WEIGHTED_SHORTEST_PATH, ALL_SHORTEST_PATHS };
enum class Direction : uint8_t { IN, OUT, BOTH };
enum class Direction { IN, OUT, BOTH };
/// Lambda for use in filtering or weight calculation during variable expand.
struct Lambda {
@@ -2217,7 +2216,7 @@ class IndexQuery : public memgraph::query::Query {
protected:
IndexQuery(Action action, LabelIx label, std::vector<PropertyIx> properties)
: action_(action), label_(std::move(label)), properties_(std::move(properties)) {}
: action_(action), label_(label), properties_(properties) {}
private:
friend class AstStorage;
@@ -2850,7 +2849,6 @@ class AuthQuery : public memgraph::query::Query {
TRANSACTION_MANAGEMENT,
MULTI_DATABASE_EDIT,
MULTI_DATABASE_USE,
COORDINATOR
};
enum class FineGrainedPrivilege { NOTHING, READ, UPDATE, CREATE_DELETE };
@@ -2929,8 +2927,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {AuthQuery::Privilege::
AuthQuery::Privilege::TRANSACTION_MANAGEMENT,
AuthQuery::Privilege::STORAGE_MODE,
AuthQuery::Privilege::MULTI_DATABASE_EDIT,
AuthQuery::Privilege::MULTI_DATABASE_USE,
AuthQuery::Privilege::COORDINATOR};
AuthQuery::Privilege::MULTI_DATABASE_USE};
class DatabaseInfoQuery : public memgraph::query::Query {
public:
@@ -3042,9 +3039,8 @@ class ReplicationQuery : public memgraph::query::Query {
memgraph::query::ReplicationQuery::Action action_;
memgraph::query::ReplicationQuery::ReplicationRole role_;
std::string instance_name_;
std::string replica_name_;
memgraph::query::Expression *socket_address_{nullptr};
memgraph::query::Expression *coordinator_socket_address_{nullptr};
memgraph::query::Expression *port_{nullptr};
memgraph::query::ReplicationQuery::SyncMode sync_mode_;
@@ -3052,53 +3048,10 @@ class ReplicationQuery : public memgraph::query::Query {
ReplicationQuery *object = storage->Create<ReplicationQuery>();
object->action_ = action_;
object->role_ = role_;
object->instance_name_ = instance_name_;
object->replica_name_ = replica_name_;
object->socket_address_ = socket_address_ ? socket_address_->Clone(storage) : nullptr;
object->port_ = port_ ? port_->Clone(storage) : nullptr;
object->sync_mode_ = sync_mode_;
object->coordinator_socket_address_ =
coordinator_socket_address_ ? coordinator_socket_address_->Clone(storage) : nullptr;
return object;
}
private:
friend class AstStorage;
};
class CoordinatorQuery : public memgraph::query::Query {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class Action {
REGISTER_INSTANCE,
SET_INSTANCE_TO_MAIN,
SHOW_REPLICATION_CLUSTER,
};
enum class SyncMode { SYNC, ASYNC };
CoordinatorQuery() = default;
DEFVISITABLE(QueryVisitor<void>);
memgraph::query::CoordinatorQuery::Action action_;
std::string instance_name_;
memgraph::query::Expression *replication_socket_address_{nullptr};
memgraph::query::Expression *coordinator_socket_address_{nullptr};
memgraph::query::CoordinatorQuery::SyncMode sync_mode_;
CoordinatorQuery *Clone(AstStorage *storage) const override {
auto *object = storage->Create<CoordinatorQuery>();
object->action_ = action_;
object->instance_name_ = instance_name_;
object->replication_socket_address_ =
replication_socket_address_ ? replication_socket_address_->Clone(storage) : nullptr;
object->sync_mode_ = sync_mode_;
object->coordinator_socket_address_ =
coordinator_socket_address_ ? coordinator_socket_address_->Clone(storage) : nullptr;
return object;
}
@@ -3659,7 +3612,7 @@ class MultiDatabaseQuery : public memgraph::query::Query {
DEFVISITABLE(QueryVisitor<void>);
enum class Action { CREATE, USE, DROP, SHOW };
enum class Action { CREATE, USE, DROP };
memgraph::query::MultiDatabaseQuery::Action action_;
std::string db_name_;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -108,7 +108,6 @@ class MultiDatabaseQuery;
class ShowDatabasesQuery;
class EdgeImportModeQuery;
class PatternComprehension;
class CoordinatorQuery;
using TreeCompositeVisitor = utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@@ -147,7 +146,6 @@ class QueryVisitor
SystemInfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery, StreamQuery,
SettingQuery, VersionQuery, ShowConfigQuery, TransactionQueueQuery, StorageModeQuery,
AnalyzeGraphQuery, MultiDatabaseQuery, ShowDatabasesQuery, EdgeImportModeQuery,
CoordinatorQuery> {};
AnalyzeGraphQuery, MultiDatabaseQuery, ShowDatabasesQuery, EdgeImportModeQuery> {};
} // namespace memgraph::query

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -302,13 +302,6 @@ antlrcpp::Any CypherMainVisitor::visitReplicationQuery(MemgraphCypher::Replicati
return replication_query;
}
antlrcpp::Any CypherMainVisitor::visitCoordinatorQuery(MemgraphCypher::CoordinatorQueryContext *ctx) {
MG_ASSERT(ctx->children.size() == 1, "CoordinatorQuery should have exactly one child!");
auto *coordinator_query = std::any_cast<CoordinatorQuery *>(ctx->children[0]->accept(this));
query_ = coordinator_query;
return coordinator_query;
}
antlrcpp::Any CypherMainVisitor::visitEdgeImportModeQuery(MemgraphCypher::EdgeImportModeQueryContext *ctx) {
auto *edge_import_mode_query = storage_->Create<EdgeImportModeQuery>();
if (ctx->ACTIVE()) {
@@ -323,34 +316,24 @@ antlrcpp::Any CypherMainVisitor::visitEdgeImportModeQuery(MemgraphCypher::EdgeIm
antlrcpp::Any CypherMainVisitor::visitSetReplicationRole(MemgraphCypher::SetReplicationRoleContext *ctx) {
auto *replication_query = storage_->Create<ReplicationQuery>();
replication_query->action_ = ReplicationQuery::Action::SET_REPLICATION_ROLE;
auto set_replication_port = [replication_query, ctx, this]() -> void {
if (ctx->port->numberLiteral() && ctx->port->numberLiteral()->integerLiteral()) {
replication_query->port_ = std::any_cast<Expression *>(ctx->port->accept(this));
} else {
throw SyntaxException("Port must be an integer literal!");
}
};
if (ctx->MAIN()) {
replication_query->role_ = ReplicationQuery::ReplicationRole::MAIN;
if (ctx->WITH() || ctx->PORT()) {
throw SemanticException("Main can't set a port!");
}
replication_query->role_ = ReplicationQuery::ReplicationRole::MAIN;
} else if (ctx->REPLICA()) {
replication_query->role_ = ReplicationQuery::ReplicationRole::REPLICA;
if (ctx->WITH() && ctx->PORT()) {
set_replication_port();
} else {
throw SemanticException("Replica must set a port!");
if (ctx->port->numberLiteral() && ctx->port->numberLiteral()->integerLiteral()) {
replication_query->port_ = std::any_cast<Expression *>(ctx->port->accept(this));
} else {
throw SyntaxException("Port must be an integer literal!");
}
}
}
return replication_query;
}
antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowReplicationRoleContext * /*ctx*/) {
antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowReplicationRoleContext *ctx) {
auto *replication_query = storage_->Create<ReplicationQuery>();
replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICATION_ROLE;
return replication_query;
@@ -359,7 +342,7 @@ antlrcpp::Any CypherMainVisitor::visitShowReplicationRole(MemgraphCypher::ShowRe
antlrcpp::Any CypherMainVisitor::visitRegisterReplica(MemgraphCypher::RegisterReplicaContext *ctx) {
auto *replication_query = storage_->Create<ReplicationQuery>();
replication_query->action_ = ReplicationQuery::Action::REGISTER_REPLICA;
replication_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
replication_query->replica_name_ = std::any_cast<std::string>(ctx->replicaName()->symbolicName()->accept(this));
if (ctx->SYNC()) {
replication_query->sync_mode_ = memgraph::query::ReplicationQuery::SyncMode::SYNC;
} else if (ctx->ASYNC()) {
@@ -368,67 +351,26 @@ antlrcpp::Any CypherMainVisitor::visitRegisterReplica(MemgraphCypher::RegisterRe
if (!ctx->socketAddress()->literal()->StringLiteral()) {
throw SemanticException("Socket address should be a string literal!");
} else {
replication_query->socket_address_ = std::any_cast<Expression *>(ctx->socketAddress()->accept(this));
}
replication_query->socket_address_ = std::any_cast<Expression *>(ctx->socketAddress()->accept(this));
return replication_query;
}
// License check is done in the interpreter.
antlrcpp::Any CypherMainVisitor::visitRegisterInstanceOnCoordinator(
MemgraphCypher::RegisterInstanceOnCoordinatorContext *ctx) {
auto *coordinator_query = storage_->Create<CoordinatorQuery>();
if (!ctx->replicationSocketAddress()->literal()->StringLiteral()) {
throw SemanticException("Replication socket address should be a string literal!");
}
if (!ctx->coordinatorSocketAddress()->literal()->StringLiteral()) {
throw SemanticException("Coordinator socket address should be a string literal!");
}
coordinator_query->action_ = CoordinatorQuery::Action::REGISTER_INSTANCE;
coordinator_query->replication_socket_address_ =
std::any_cast<Expression *>(ctx->replicationSocketAddress()->accept(this));
coordinator_query->coordinator_socket_address_ =
std::any_cast<Expression *>(ctx->coordinatorSocketAddress()->accept(this));
coordinator_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
if (ctx->ASYNC()) {
coordinator_query->sync_mode_ = memgraph::query::CoordinatorQuery::SyncMode::ASYNC;
} else {
coordinator_query->sync_mode_ = memgraph::query::CoordinatorQuery::SyncMode::SYNC;
}
return coordinator_query;
}
// License check is done in the interpreter
antlrcpp::Any CypherMainVisitor::visitShowReplicationCluster(MemgraphCypher::ShowReplicationClusterContext * /*ctx*/) {
auto *coordinator_query = storage_->Create<CoordinatorQuery>();
coordinator_query->action_ = CoordinatorQuery::Action::SHOW_REPLICATION_CLUSTER;
return coordinator_query;
}
antlrcpp::Any CypherMainVisitor::visitDropReplica(MemgraphCypher::DropReplicaContext *ctx) {
auto *replication_query = storage_->Create<ReplicationQuery>();
replication_query->action_ = ReplicationQuery::Action::DROP_REPLICA;
replication_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
replication_query->replica_name_ = std::any_cast<std::string>(ctx->replicaName()->symbolicName()->accept(this));
return replication_query;
}
antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasContext * /*ctx*/) {
antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasContext *ctx) {
auto *replication_query = storage_->Create<ReplicationQuery>();
replication_query->action_ = ReplicationQuery::Action::SHOW_REPLICAS;
return replication_query;
}
// License check is done in the interpreter
antlrcpp::Any CypherMainVisitor::visitSetInstanceToMain(MemgraphCypher::SetInstanceToMainContext *ctx) {
auto *coordinator_query = storage_->Create<CoordinatorQuery>();
coordinator_query->action_ = CoordinatorQuery::Action::SET_INSTANCE_TO_MAIN;
coordinator_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
query_ = coordinator_query;
return coordinator_query;
}
antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) {
auto *lock_query = storage_->Create<LockPathQuery>();
if (ctx->STATUS()) {
@@ -1696,7 +1638,6 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
if (ctx->STORAGE_MODE()) return AuthQuery::Privilege::STORAGE_MODE;
if (ctx->MULTI_DATABASE_EDIT()) return AuthQuery::Privilege::MULTI_DATABASE_EDIT;
if (ctx->MULTI_DATABASE_USE()) return AuthQuery::Privilege::MULTI_DATABASE_USE;
if (ctx->COORDINATOR()) return AuthQuery::Privilege::COORDINATOR;
LOG_FATAL("Should not get here - unknown privilege!");
}
@@ -1811,11 +1752,7 @@ antlrcpp::Any CypherMainVisitor::visitReturnBody(MemgraphCypher::ReturnBodyConte
body.skip = static_cast<Expression *>(std::any_cast<Expression *>(ctx->skip()->accept(this)));
}
if (ctx->limit()) {
if (ctx->limit()->expression()) {
body.limit = std::any_cast<Expression *>(ctx->limit()->accept(this));
} else {
body.limit = std::any_cast<ParameterLookup *>(ctx->limit()->accept(this));
}
body.limit = static_cast<Expression *>(std::any_cast<Expression *>(ctx->limit()->accept(this)));
}
std::tie(body.all_identifiers, body.named_expressions) =
std::any_cast<std::pair<bool, std::vector<NamedExpression *>>>(ctx->returnItems()->accept(this));
@@ -2951,14 +2888,6 @@ antlrcpp::Any CypherMainVisitor::visitDropDatabase(MemgraphCypher::DropDatabaseC
return mdb_query;
}
antlrcpp::Any CypherMainVisitor::visitShowDatabase(MemgraphCypher::ShowDatabaseContext * /*ctx*/) {
auto *mdb_query = storage_->Create<MultiDatabaseQuery>();
mdb_query->db_name_ = "";
mdb_query->action_ = MultiDatabaseQuery::Action::SHOW;
query_ = mdb_query;
return mdb_query;
}
antlrcpp::Any CypherMainVisitor::visitShowDatabases(MemgraphCypher::ShowDatabasesContext * /*ctx*/) {
query_ = storage_->Create<ShowDatabasesQuery>();
return query_;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -233,26 +233,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitShowReplicas(MemgraphCypher::ShowReplicasContext *ctx) override;
/**
* @return CoordinatorQuery*
*/
antlrcpp::Any visitCoordinatorQuery(MemgraphCypher::CoordinatorQueryContext *ctx) override;
/**
* @return CoordinatorQuery*
*/
antlrcpp::Any visitRegisterInstanceOnCoordinator(MemgraphCypher::RegisterInstanceOnCoordinatorContext *ctx) override;
/**
* @return CoordinatorQuery*
*/
antlrcpp::Any visitSetInstanceToMain(MemgraphCypher::SetInstanceToMainContext *ctx) override;
/**
* @return CoordinatorQuery*
*/
antlrcpp::Any visitShowReplicationCluster(MemgraphCypher::ShowReplicationClusterContext *ctx) override;
/**
* @return LockPathQuery*
*/
@@ -1017,11 +997,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitDropDatabase(MemgraphCypher::DropDatabaseContext *ctx) override;
/**
* @return MultiDatabaseQuery*
*/
antlrcpp::Any visitShowDatabase(MemgraphCypher::ShowDatabaseContext *ctx) override;
/**
* @return ShowDatabasesQuery*
*/

View File

@@ -143,7 +143,7 @@ order : ORDER BY sortItem ( ',' sortItem )* ;
skip : L_SKIP expression ;
limit : LIMIT ( expression | parameter ) ;
limit : LIMIT expression ;
sortItem : expression ( ASCENDING | ASC | DESCENDING | DESC )? ;

View File

@@ -102,7 +102,6 @@ FILTER : F I L T E R ;
IN : I N ;
INDEX : I N D E X ;
INFO : I N F O ;
INSTANCE : I N S T A N C E ;
IS : I S ;
KB : K B ;
KEY : K E Y ;
@@ -123,7 +122,6 @@ PROCEDURE : P R O C E D U R E ;
PROFILE : P R O F I L E ;
QUERY : Q U E R Y ;
REDUCE : R E D U C E ;
REGISTER : R E G I S T E R;
REMOVE : R E M O V E ;
RETURN : R E T U R N ;
SET : S E T ;

Some files were not shown because too many files have changed in this diff Show More