Compare commits
4 Commits
isolate-cu
...
detach-del
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4dc6400681 | ||
|
|
675c166254 | ||
|
|
70dabbd926 | ||
|
|
05c9c28dca |
@@ -6,7 +6,6 @@ Checks: '*,
|
||||
-altera-unroll-loops,
|
||||
-android-*,
|
||||
-cert-err58-cpp,
|
||||
-cppcoreguidelines-avoid-do-while,
|
||||
-cppcoreguidelines-avoid-c-arrays,
|
||||
-cppcoreguidelines-avoid-goto,
|
||||
-cppcoreguidelines-avoid-magic-numbers,
|
||||
@@ -61,7 +60,6 @@ Checks: '*,
|
||||
-readability-implicit-bool-conversion,
|
||||
-readability-magic-numbers,
|
||||
-readability-named-parameter,
|
||||
-readability-identifier-length,
|
||||
-misc-no-recursion,
|
||||
-concurrency-mt-unsafe,
|
||||
-bugprone-easily-swappable-parameters'
|
||||
|
||||
1
.github/ISSUE_TEMPLATE/bug_report.md
vendored
1
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -3,6 +3,7 @@ name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ""
|
||||
labels: bug
|
||||
assignees: gitbuda
|
||||
---
|
||||
|
||||
**Memgraph version**
|
||||
|
||||
115
.github/workflows/diff.yaml
vendored
115
.github/workflows/diff.yaml
vendored
@@ -268,6 +268,7 @@ jobs:
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
if: false
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
@@ -275,6 +276,7 @@ jobs:
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
if: false
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
@@ -283,6 +285,7 @@ jobs:
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: false
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
@@ -336,6 +339,118 @@ jobs:
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
experimental_build_ha:
|
||||
name: "High availability build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
./init
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
|
||||
make -j$THREADS
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
- name: Run e2e tests
|
||||
if: false
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh "Coordinator"
|
||||
./run.sh "Client initiated failover"
|
||||
./run.sh "Uninitialized cluster"
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data(High availability build)"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
experimental_build_mt:
|
||||
name: "MultiTenancy replication build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build MT replication experimental binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release -D MG_EXPERIMENTAL_REPLICATION_MULTITENANCY=ON ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
- name: Run e2e tests
|
||||
if: false
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
|
||||
# Just the replication based e2e tests
|
||||
./run.sh "Replicate multitenancy"
|
||||
./run.sh "Show"
|
||||
./run.sh "Show while creating invalid state"
|
||||
./run.sh "Delete edge replication"
|
||||
./run.sh "Read-write benchmark"
|
||||
./run.sh "Index replication"
|
||||
./run.sh "Constraints"
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data(MultiTenancy replication build)"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
|
||||
208
.github/workflows/release_build_test.yaml
vendored
208
.github/workflows/release_build_test.yaml
vendored
@@ -1,208 +0,0 @@
|
||||
name: Release build test
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
type: choice
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
|
||||
push:
|
||||
branches:
|
||||
- "release/**"
|
||||
tags:
|
||||
- "v*.*.*-rc*"
|
||||
- "v*.*-rc*"
|
||||
schedule:
|
||||
# UTC
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
|
||||
|
||||
jobs:
|
||||
Debian10:
|
||||
uses: ./.github/workflows/release_debian10.yaml
|
||||
with:
|
||||
build_type: ${{ github.event.inputs.build_type || 'Release' }}
|
||||
secrets: inherit
|
||||
|
||||
Ubuntu20_04:
|
||||
uses: ./.github/workflows/release_ubuntu2004.yaml
|
||||
with:
|
||||
build_type: ${{ github.event.inputs.build_type || 'Release' }}
|
||||
secrets: inherit
|
||||
|
||||
PackageDebian10:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [Debian10]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-10 $BUILD_TYPE
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "deps.memgraph.io"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output"
|
||||
DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/"
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-10
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
PackageUbuntu20_04:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [Ubuntu20_04]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04 $BUILD_TYPE
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "deps.memgraph.io"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output"
|
||||
DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/"
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
PackageUbuntu20_04_ARM:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [Ubuntu20_04]
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64]
|
||||
# M1 Mac mini is sometimes slower
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04-arm $BUILD_TYPE
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04-aarch64
|
||||
path: build/output/ubuntu-22.04-arm/memgraph*.deb
|
||||
|
||||
PushToS3Ubuntu20_04_ARM:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [PackageUbuntu20_04_ARM]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download package
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04-aarch64
|
||||
path: build/output/release
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "deps.memgraph.io"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output/release"
|
||||
DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/"
|
||||
|
||||
PackageDebian11:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [Debian10, Ubuntu20_04]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 $BUILD_TYPE
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "deps.memgraph.io"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output"
|
||||
DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/"
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
PackageDebian11_ARM:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [Debian10, Ubuntu20_04]
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64]
|
||||
# M1 Mac mini is sometimes slower
|
||||
timeout-minutes: 150
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm $BUILD_TYPE
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
PushToS3Debian11_ARM:
|
||||
if: github.ref_type == 'tag'
|
||||
needs: [PackageDebian11_ARM]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download package
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/release
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "deps.memgraph.io"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output/release"
|
||||
DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/"
|
||||
21
.github/workflows/release_debian10.yaml
vendored
21
.github/workflows/release_debian10.yaml
vendored
@@ -1,12 +1,6 @@
|
||||
name: Release Debian 10
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
build_type:
|
||||
type: string
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
@@ -17,8 +11,10 @@ on:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
env:
|
||||
OS: "Debian10"
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
@@ -115,7 +111,7 @@ jobs:
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage(Coverage build)-${{ env.OS }}"
|
||||
name: "Code coverage(Coverage build)"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
debug_build:
|
||||
@@ -169,7 +165,7 @@ jobs:
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage(Debug build)-${{ env.OS }}"
|
||||
name: "Code coverage(Debug build)"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
debug_integration_test:
|
||||
@@ -246,7 +242,7 @@ jobs:
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Enterprise DEB package-${{ env.OS}}"
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
@@ -259,7 +255,7 @@ jobs:
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "GQL Behave Status-${{ env.OS }}"
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
@@ -325,6 +321,7 @@ jobs:
|
||||
--no-strict
|
||||
|
||||
release_e2e_test:
|
||||
if: false
|
||||
name: "Release End-to-end Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
timeout-minutes: 60
|
||||
@@ -459,5 +456,5 @@ jobs:
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report-${{ env.OS }}"
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
19
.github/workflows/release_ubuntu2004.yaml
vendored
19
.github/workflows/release_ubuntu2004.yaml
vendored
@@ -1,12 +1,6 @@
|
||||
name: Release Ubuntu 20.04
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
build_type:
|
||||
type: string
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
@@ -17,8 +11,10 @@ on:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
env:
|
||||
OS: "Ubuntu 20.04"
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
@@ -111,7 +107,7 @@ jobs:
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage(Coverage build)-${{ env.OS }}"
|
||||
name: "Code coverage(Coverage build)"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
debug_build:
|
||||
@@ -165,7 +161,7 @@ jobs:
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage(Debug build)-${{ env.OS }}"
|
||||
name: "Code coverage(Debug build)"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
debug_integration_test:
|
||||
@@ -242,7 +238,7 @@ jobs:
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Enterprise DEB package-${{ env.OS }}"
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
@@ -255,7 +251,7 @@ jobs:
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "GQL Behave Status-${{ env.OS }}"
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
@@ -321,6 +317,7 @@ jobs:
|
||||
--no-strict
|
||||
|
||||
release_e2e_test:
|
||||
if: false
|
||||
name: "Release End-to-end Test"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
timeout-minutes: 60
|
||||
|
||||
8
.github/workflows/stress_test_large.yaml
vendored
8
.github/workflows/stress_test_large.yaml
vendored
@@ -1,7 +1,4 @@
|
||||
name: Stress test large
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -13,10 +10,7 @@ on:
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*-rc*"
|
||||
- "v*.*-rc*"
|
||||
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
|
||||
@@ -211,13 +211,8 @@ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
|
||||
# ** Static linking is allowed only for executables! **
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
|
||||
|
||||
# Use lld linker to speedup build and use less memory.
|
||||
add_link_options(-fuse-ld=lld)
|
||||
# NOTE: Moving to latest Clang (probably starting from 15), lld stopped to work
|
||||
# without explicit link_directories call.
|
||||
string(REPLACE ":" " " LD_LIBS $ENV{LD_LIBRARY_PATH})
|
||||
separate_arguments(LD_LIBS)
|
||||
link_directories(${LD_LIBS})
|
||||
# Use lld linker to speedup build
|
||||
add_link_options(-fuse-ld=lld) # TODO: use mold linker
|
||||
|
||||
# release flags
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
|
||||
@@ -276,6 +271,18 @@ endif()
|
||||
set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
|
||||
add_subdirectory(libs EXCLUDE_FROM_ALL)
|
||||
|
||||
option(MG_EXPERIMENTAL_HIGH_AVAILABILITY "Feature flag for experimental high availability" OFF)
|
||||
|
||||
if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
set(MG_EXPERIMENTAL_HIGH_AVAILABILITY OFF)
|
||||
message(FATAL_ERROR "MG_EXPERIMENTAL_HIGH_AVAILABILITY must be used with enterpise version of the code.")
|
||||
endif ()
|
||||
|
||||
if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
add_compile_definitions(MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
endif ()
|
||||
|
||||
# Optional subproject configuration -------------------------------------------
|
||||
option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
|
||||
option(TOOLS "Build tools binaries" ON)
|
||||
option(QUERY_MODULES "Build query modules containing custom procedures" ON)
|
||||
@@ -284,6 +291,16 @@ option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option
|
||||
option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
|
||||
|
||||
# Build feature flags
|
||||
option(MG_EXPERIMENTAL_REPLICATION_MULTITENANCY "Feature flag for experimental replicaition of multitenacy" OFF)
|
||||
|
||||
if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
|
||||
set(MG_EXPERIMENTAL_REPLICATION_MULTITENANCY OFF)
|
||||
message(FATAL_ERROR "MG_EXPERIMENTAL_REPLICATION_MULTITENANCY with community edition build isn't possible")
|
||||
endif ()
|
||||
|
||||
if (MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
|
||||
add_compile_definitions(MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
|
||||
endif ()
|
||||
|
||||
if (TEST_COVERAGE)
|
||||
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -7,7 +9,7 @@ check_operating_system "amzn-2"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
git gcc gcc-c++ make # generic build tools
|
||||
gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
@@ -45,7 +47,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel
|
||||
openssl
|
||||
libseccomp-devel
|
||||
python3 python3-pip nmap-ncat # for tests
|
||||
#
|
||||
@@ -62,8 +63,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
cyrus-sasl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -43,7 +45,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel
|
||||
openssl
|
||||
libseccomp-devel
|
||||
python3 python-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
#
|
||||
@@ -62,8 +63,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
cyrus-sasl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -7,10 +9,8 @@ check_operating_system "centos-9"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
wget # used for archive download
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
# NOTE: Pure libcurl conflicts with libcurl-minimal
|
||||
libcurl-devel # cmake build requires it
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
@@ -64,8 +64,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
cyrus-sasl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
@@ -125,9 +123,7 @@ install() {
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
# --nobest is used because of libipt because we install custom versions
|
||||
# because libipt-devel is not available on CentOS 9 Stream
|
||||
yum update -y --nobest
|
||||
yum update -y
|
||||
yum install -y wget git python3 python3-pip
|
||||
|
||||
for pkg in $1; do
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
# IMPORTANT: Deprecated since memgraph v2.12.0.
|
||||
|
||||
check_operating_system "debian-10"
|
||||
check_architecture "x86_64"
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
# IMPORTANT: Deprecated since memgraph v2.12.0.
|
||||
|
||||
check_operating_system "debian-11"
|
||||
check_architecture "arm64" "aarch64"
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -59,8 +61,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libsasl2-dev
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "debian-12"
|
||||
check_architecture "arm64" "aarch64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg # used for archive signature verification
|
||||
tar gzip bzip2 xz-utils unzip # used for archive unpacking
|
||||
zlib1g-dev # zlib library used for all builds
|
||||
libexpat1-dev liblzma-dev python3-dev texinfo # for gdb
|
||||
libcurl4-openssl-dev # for cmake
|
||||
libreadline-dev # for cmake and llvm
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
|
||||
curl # snappy
|
||||
file # for libunwind
|
||||
libssl-dev # for libevent
|
||||
libgmp-dev
|
||||
gperf # for proxygen
|
||||
git # for fbthrift
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
zlib1g # zlib library used for all builds
|
||||
libexpat1 liblzma5 python3 # for gdb
|
||||
libcurl4 # for cmake
|
||||
file # for CPack
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi8 libxml2 # for llvm
|
||||
libssl-dev # for libevent
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
uuid-dev default-jre-headless # required by antlr
|
||||
libreadline-dev # for memgraph console
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat # tests are using nc to wait for memgraph
|
||||
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless custom-maven3.9.3 # for driver tests
|
||||
dotnet-sdk-7.0 golang custom-golang1.18.9 nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
libsasl2-dev
|
||||
)
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
NEW_DEPS=(
|
||||
wget curl tar gzip
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == custom-maven3.9.3 ]; then
|
||||
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == custom-golang1.18.9 ]; then
|
||||
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
done
|
||||
if [ "$missing" != "" ]; then
|
||||
echo "MISSING PACKAGES: $missing"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
apt update
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
apt install -y wget
|
||||
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == custom-maven3.9.3 ]; then
|
||||
install_custom_maven "3.9.3"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == custom-golang1.18.9 ]; then
|
||||
install_custom_golang "1.18.9"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-7.0 ]; then
|
||||
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https dotnet-sdk-7.0
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -1,136 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "debian-12"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg # used for archive signature verification
|
||||
tar gzip bzip2 xz-utils unzip # used for archive unpacking
|
||||
zlib1g-dev # zlib library used for all builds
|
||||
libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
|
||||
libcurl4-openssl-dev # for cmake
|
||||
libreadline-dev # for cmake and llvm
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
|
||||
curl # snappy
|
||||
file # for libunwind
|
||||
libssl-dev # for libevent
|
||||
libgmp-dev
|
||||
gperf # for proxygen
|
||||
git # for fbthrift
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
zlib1g # zlib library used for all builds
|
||||
libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
|
||||
libcurl4 # for cmake
|
||||
file # for CPack
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi8 libxml2 # for llvm
|
||||
libssl-dev # for libevent
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make cmake pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
uuid-dev default-jre-headless # required by antlr
|
||||
libreadline-dev # for memgraph console
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat-traditional # tests are using nc to wait for memgraph
|
||||
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless custom-maven3.9.3 # for driver tests
|
||||
dotnet-sdk-7.0 golang custom-golang1.18.9 nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
libsasl2-dev
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
NEW_DEPS=(
|
||||
wget curl tar gzip
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == custom-maven3.9.3 ]; then
|
||||
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == custom-golang1.18.9 ]; then
|
||||
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
done
|
||||
if [ "$missing" != "" ]; then
|
||||
echo "MISSING PACKAGES: $missing"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
apt update
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
apt install -y wget
|
||||
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == custom-maven3.9.3 ]; then
|
||||
install_custom_maven "3.9.3"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == custom-golang1.18.9 ]; then
|
||||
install_custom_golang "1.18.9"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-7.0 ]; then
|
||||
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https dotnet-sdk-7.0
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
# IMPORTANT: Deprecated since memgraph v2.12.0.
|
||||
|
||||
check_operating_system "fedora-36"
|
||||
check_architecture "x86_64"
|
||||
|
||||
@@ -27,7 +27,6 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libipt libipt-devel # intel
|
||||
patch
|
||||
perl # for openssl
|
||||
git
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -25,7 +27,6 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libipt libipt-devel # intel
|
||||
patch
|
||||
perl # for openssl
|
||||
git
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
@@ -57,16 +58,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
NEW_DEPS=(
|
||||
wget curl tar gzip
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "fedora-39"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
|
||||
curl libcurl-devel # for cmake
|
||||
readline-devel # for cmake and llvm
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel pcre2-devel automake bison # for swig
|
||||
file
|
||||
openssl-devel
|
||||
gmp-devel
|
||||
gperf
|
||||
diffutils
|
||||
libipt libipt-devel # intel
|
||||
patch
|
||||
perl # for openssl
|
||||
git
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz # used for archive unpacking
|
||||
zlib # zlib library used for all builds
|
||||
expat xz-libs python3 # for gdb
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
openssl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconf-pkg-config # build system
|
||||
wget # for downloading libs
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
|
||||
libcurl-devel # mg-requests
|
||||
rpm-build rpmlint # for RPM package building
|
||||
doxygen graphviz # source documentation generators
|
||||
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
NEW_DEPS=(
|
||||
wget curl tar gzip
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
if [ -v LD_LIBRARY_PATH ]; then
|
||||
# On Fedora 38 yum/dnf and python11 use newer glibc which is not compatible
|
||||
# with ours, so we need to momentarely disable env
|
||||
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
|
||||
LD_LIBRARY_PATH=""
|
||||
fi
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
done
|
||||
if [ "$missing" != "" ]; then
|
||||
echo "MISSING PACKAGES: $missing"
|
||||
exit 1
|
||||
fi
|
||||
if [ -v OLD_LD_LIBRARY_PATH ]; then
|
||||
echo "Restoring LD_LIBRARY_PATH..."
|
||||
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests don't work without the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
dnf update -y
|
||||
for pkg in $1; do
|
||||
dnf install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -1,188 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
# TODO(gitbuda): Rocky gets automatically updates -> figure out how to handle it.
|
||||
check_operating_system "rocky-9.3"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
wget # used for archive download
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
# NOTE: Pure libcurl conflicts with libcurl-minimal
|
||||
libcurl-devel # cmake build requires it
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
expat-devel xz-devel python3-devel perl-Unicode-EastAsianWidth texinfo libbabeltrace-devel # for gdb
|
||||
readline-devel # for cmake and llvm
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel pcre2-devel automake bison # for swig
|
||||
file
|
||||
openssl-devel
|
||||
gmp-devel
|
||||
gperf
|
||||
diffutils
|
||||
libipt libipt-devel # intel
|
||||
patch
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz # used for archive unpacking
|
||||
zlib # zlib library used for all builds
|
||||
expat xz-libs python3 # for gdb
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
openssl-devel
|
||||
perl # for openssl
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make cmake pkgconf-pkg-config # build system
|
||||
wget # for downloading libs
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
#
|
||||
# IMPORTANT: python3-yaml does NOT exist on CentOS
|
||||
# Install it manually using `pip3 install PyYAML`
|
||||
#
|
||||
PyYAML # Package name here does not correspond to the yum package!
|
||||
libcurl-devel # mg-requests
|
||||
rpm-build rpmlint # for RPM package building
|
||||
doxygen graphviz # source documentation generators
|
||||
which nodejs golang custom-golang1.18.9 # for driver tests
|
||||
zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.3 # for driver tests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
cyrus-sasl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp
|
||||
)
|
||||
|
||||
NEW_DEPS=(
|
||||
wget curl tar gzip
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == custom-maven3.9.3 ]; then
|
||||
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == custom-golang1.18.9 ]; then
|
||||
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == "PyYAML" ]; then
|
||||
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == "python3-virtualenv" ]; then
|
||||
continue
|
||||
fi
|
||||
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
done
|
||||
if [ "$missing" != "" ]; then
|
||||
echo "MISSING PACKAGES: $missing"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
yum update -y
|
||||
yum install -y wget git python3 python3-pip
|
||||
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == custom-maven3.9.3 ]; then
|
||||
install_custom_maven "3.9.3"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == custom-golang1.18.9 ]; then
|
||||
install_custom_golang "1.18.9"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == perl-Unicode-EastAsianWidth ]; then
|
||||
if ! dnf list installed perl-Unicode-EastAsianWidth >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://dl.rockylinux.org/pub/rocky/9/CRB/x86_64/os/Packages/p/perl-Unicode-EastAsianWidth-12.0-7.el9.noarch.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == texinfo ]; then
|
||||
if ! dnf list installed texinfo >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://dl.rockylinux.org/pub/rocky/9/CRB/x86_64/os/Packages/t/texinfo-6.7-15.el9.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libbabeltrace-devel ]; then
|
||||
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://dl.rockylinux.org/pub/rocky/9/devel/x86_64/os/Packages/l/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libipt-devel ]; then
|
||||
if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://dl.rockylinux.org/pub/rocky/9/devel/x86_64/os/Packages/l/libipt-devel-2.0.4-5.el9.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == PyYAML ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user PyYAML
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == python3-virtualenv ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install virtualenv
|
||||
pip3 install virtualenvwrapper
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
yum install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -18,10 +20,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
# IMPORTANT: Deprecated since memgraph v2.12.0.
|
||||
|
||||
check_operating_system "ubuntu-18.04"
|
||||
check_architecture "x86_64"
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -58,8 +60,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libsasl2-dev
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp2
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -58,8 +60,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libsasl2-dev
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp2
|
||||
)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
@@ -58,8 +60,6 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libsasl2-dev
|
||||
)
|
||||
|
||||
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
|
||||
|
||||
MEMGRAPH_RUN_DEPS=(
|
||||
logrotate openssl python3 libseccomp2
|
||||
)
|
||||
|
||||
1
environment/toolchain/.gitignore
vendored
1
environment/toolchain/.gitignore
vendored
@@ -2,4 +2,3 @@ archives
|
||||
build
|
||||
output
|
||||
*.tar.gz
|
||||
tmp_build.sh
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
|
||||
# NOTE: Copy this under memgraph/environment/toolchain/vN/tmp_build.sh, edit and test.
|
||||
|
||||
pushd () { command pushd "$@" > /dev/null; }
|
||||
popd () { command popd "$@" > /dev/null; }
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
CPUS=$( grep -c processor < /proc/cpuinfo )
|
||||
cd "$DIR"
|
||||
source "$DIR/../../util.sh"
|
||||
DISTRO="$(operating_system)"
|
||||
TOOLCHAIN_VERSION=5
|
||||
NAME=toolchain-v$TOOLCHAIN_VERSION
|
||||
PREFIX=/opt/$NAME
|
||||
function log_tool_name () {
|
||||
echo ""
|
||||
echo ""
|
||||
echo "#### $1 ####"
|
||||
echo ""
|
||||
echo ""
|
||||
}
|
||||
|
||||
# HERE: Remove/clear dependencies from a given toolchain.
|
||||
|
||||
mkdir -p archives && pushd archives
|
||||
# HERE: Download dependencies here.
|
||||
popd
|
||||
|
||||
mkdir -p build
|
||||
pushd build
|
||||
source $PREFIX/activate
|
||||
export CC=$PREFIX/bin/clang
|
||||
export CXX=$PREFIX/bin/clang++
|
||||
export CFLAGS="$CFLAGS -fPIC"
|
||||
export PATH=$PREFIX/bin:$PATH
|
||||
export LD_LIBRARY_PATH=$PREFIX/lib64
|
||||
COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
|
||||
-DCMAKE_PREFIX_PATH=$PREFIX
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
-DCMAKE_C_COMPILER=$CC
|
||||
-DCMAKE_CXX_COMPILER=$CXX
|
||||
-DBUILD_SHARED_LIBS=OFF
|
||||
-DCMAKE_CXX_STANDARD=20
|
||||
-DBUILD_TESTING=OFF
|
||||
-DCMAKE_REQUIRED_INCLUDES=$PREFIX/include
|
||||
-DCMAKE_POSITION_INDEPENDENT_CODE=ON"
|
||||
|
||||
# HERE: Add dependencies to test below.
|
||||
@@ -307,7 +307,7 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
|
||||
fi
|
||||
|
||||
log_tool_name "GDB $GDB_VERSION"
|
||||
if [[ ! -f "$PREFIX/bin/gdb" && "$DISTRO" -ne "amzn-2" ]]; then
|
||||
if [ ! -f $PREFIX/bin/gdb ]; then
|
||||
if [ -d gdb-$GDB_VERSION ]; then
|
||||
rm -rf gdb-$GDB_VERSION
|
||||
fi
|
||||
@@ -671,6 +671,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
|
||||
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
|
||||
# WANGLE_SHA256=0e493c03572bb27fe9ca03a9da5023e52fde99c95abdcaa919bb6190e7e69532
|
||||
|
||||
FLEX_VERSION=2.6.4
|
||||
FMT_SHA256=78b8c0a72b1c35e4443a7e308df52498252d1cefc2b08c9a97bc9ee6cfe61f8b
|
||||
FMT_VERSION=10.1.1
|
||||
# NOTE: spdlog depends on exact fmt versions -> UPGRADE fmt and spdlog TOGETHER.
|
||||
@@ -689,8 +690,8 @@ LZ4_VERSION=1.9.4
|
||||
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
|
||||
SNAPPY_VERSION=1.1.9
|
||||
XZ_VERSION=5.2.5 # for LZMA
|
||||
ZLIB_VERSION=1.3.1
|
||||
ZSTD_VERSION=1.5.5
|
||||
ZLIB_VERSION=1.3
|
||||
ZSTD_VERSION=1.5.0
|
||||
|
||||
pushd archives
|
||||
if [ ! -f boost_$BOOST_VERSION_UNDERSCORES.tar.gz ]; then
|
||||
@@ -699,7 +700,7 @@ if [ ! -f boost_$BOOST_VERSION_UNDERSCORES.tar.gz ]; then
|
||||
wget https://boostorg.jfrog.io/artifactory/main/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORES.tar.gz -O boost_$BOOST_VERSION_UNDERSCORES.tar.gz
|
||||
fi
|
||||
if [ ! -f bzip2-$BZIP2_VERSION.tar.gz ]; then
|
||||
wget https://sourceware.org/pub/bzip2/bzip2-$BZIP2_VERSION.tar.gz -O bzip2-$BZIP2_VERSION.tar.gz
|
||||
wget https://sourceforge.net/projects/bzip2/files/bzip2-$BZIP2_VERSION.tar.gz -O bzip2-$BZIP2_VERSION.tar.gz
|
||||
fi
|
||||
if [ ! -f double-conversion-$DOUBLE_CONVERSION_VERSION.tar.gz ]; then
|
||||
wget https://github.com/google/double-conversion/archive/refs/tags/v$DOUBLE_CONVERSION_VERSION.tar.gz -O double-conversion-$DOUBLE_CONVERSION_VERSION.tar.gz
|
||||
@@ -707,7 +708,9 @@ fi
|
||||
if [ ! -f fizz-$FBLIBS_VERSION.tar.gz ]; then
|
||||
wget https://github.com/facebookincubator/fizz/releases/download/v$FBLIBS_VERSION/fizz-v$FBLIBS_VERSION.tar.gz -O fizz-$FBLIBS_VERSION.tar.gz
|
||||
fi
|
||||
|
||||
if [ ! -f flex-$FLEX_VERSION.tar.gz ]; then
|
||||
wget https://github.com/westes/flex/releases/download/v$FLEX_VERSION/flex-$FLEX_VERSION.tar.gz -O flex-$FLEX_VERSION.tar.gz
|
||||
fi
|
||||
if [ ! -f fmt-$FMT_VERSION.tar.gz ]; then
|
||||
wget https://github.com/fmtlib/fmt/archive/refs/tags/$FMT_VERSION.tar.gz -O fmt-$FMT_VERSION.tar.gz
|
||||
fi
|
||||
@@ -762,6 +765,14 @@ echo "$BZIP2_SHA256 bzip2-$BZIP2_VERSION.tar.gz" | sha256sum -c
|
||||
echo "$DOUBLE_CONVERSION_SHA256 double-conversion-$DOUBLE_CONVERSION_VERSION.tar.gz" | sha256sum -c
|
||||
# verify fizz
|
||||
echo "$FIZZ_SHA256 fizz-$FBLIBS_VERSION.tar.gz" | sha256sum -c
|
||||
# verify flex
|
||||
if [ ! -f flex-$FLEX_VERSION.tar.gz.sig ]; then
|
||||
wget https://github.com/westes/flex/releases/download/v$FLEX_VERSION/flex-$FLEX_VERSION.tar.gz.sig
|
||||
fi
|
||||
if false; then
|
||||
$GPG --keyserver $KEYSERVER --recv-keys 0xE4B29C8D64885307
|
||||
$GPG --verify flex-$FLEX_VERSION.tar.gz.sig flex-$FLEX_VERSION.tar.gz
|
||||
fi
|
||||
# verify fmt
|
||||
echo "$FMT_SHA256 fmt-$FMT_VERSION.tar.gz" | sha256sum -c
|
||||
# verify spdlog
|
||||
@@ -1014,6 +1025,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
|
||||
if [ -d gflags ]; then
|
||||
rm -rf gflags
|
||||
fi
|
||||
|
||||
git clone https://github.com/memgraph/gflags.git gflags
|
||||
pushd gflags
|
||||
git checkout $GFLAGS_COMMIT_HASH
|
||||
@@ -1022,7 +1034,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
|
||||
cmake .. $COMMON_CMAKE_FLAGS \
|
||||
-DREGISTER_INSTALL_PREFIX=OFF \
|
||||
-DBUILD_gflags_nothreads_LIB=OFF \
|
||||
-DGFLAGS_NO_FILENAMES=1
|
||||
-DGFLAGS_NO_FILENAMES=0
|
||||
make -j$CPUS install
|
||||
popd && popd
|
||||
fi
|
||||
@@ -1220,6 +1232,18 @@ if false; then
|
||||
fi
|
||||
fi
|
||||
|
||||
log_tool_name "flex $FLEX_VERSION"
|
||||
if [ ! -f $PREFIX/include/FlexLexer.h ]; then
|
||||
if [ -d flex-$FLEX_VERSION ]; then
|
||||
rm -rf flex-$FLEX_VERSION
|
||||
fi
|
||||
tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
|
||||
pushd flex-$FLEX_VERSION
|
||||
./configure $COMMON_CONFIGURE_FLAGS
|
||||
make -j$CPUS install
|
||||
popd
|
||||
fi
|
||||
|
||||
popd
|
||||
# NOTE: It's important/clean (e.g., easier upload to S3) to have a separated
|
||||
# folder to the output archive.
|
||||
|
||||
@@ -20,18 +20,14 @@ if [ ! -f "$INPUT" ]; then
|
||||
fi
|
||||
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create constraints manually if needed${COLOR_NULL}"
|
||||
|
||||
echo 'CREATE INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' > "$OUTPUT"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create consraints manually if needed${COLOR_NULL}"
|
||||
|
||||
sed -e 's/^:begin/BEGIN/g; s/^BEGIN$/BEGIN;/g;' \
|
||||
-e 's/^:commit/COMMIT/g; s/^COMMIT$/COMMIT;/g;' \
|
||||
-e '/^CALL/d; /^SCHEMA AWAIT/d;' \
|
||||
-e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
|
||||
-e 's/) ON (n./(/g;' \
|
||||
-e '/^CREATE CONSTRAINT/d; /^DROP CONSTRAINT/d;' "$INPUT" >> "$OUTPUT"
|
||||
|
||||
echo 'DROP INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' >> "$OUTPUT"
|
||||
-e '/^CREATE CONSTRAINT/d; /^DROP CONSTRAINT/d;' "$INPUT" > "$OUTPUT"
|
||||
|
||||
echo ""
|
||||
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher file under $OUTPUT"
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
COLOR_ORANGE="\e[38;5;208m"
|
||||
COLOR_GREEN="\e[38;5;35m"
|
||||
COLOR_RED="\e[0;31m"
|
||||
COLOR_NULL="\e[0m"
|
||||
|
||||
print_help() {
|
||||
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 input_file_schema_path input_file_nodes_path input_file_relationships_path input_file_cleanup_path output_file_path"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$#" -ne 5 ]; then
|
||||
print_help
|
||||
fi
|
||||
INPUT_SCHEMA="$1"
|
||||
INPUT_NODES="$2"
|
||||
INPUT_RELATIONSHIPS="$3"
|
||||
INPUT_CLEANUP="$4"
|
||||
OUTPUT="$5"
|
||||
|
||||
if [ ! -f "$INPUT_SCHEMA" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_NODES" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_RELATIONSHIPS" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_CLEANUP" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create constraints manually if needed${COLOR_NULL}"
|
||||
|
||||
|
||||
echo 'CREATE INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' > "$OUTPUT"
|
||||
|
||||
sed -e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
|
||||
-e 's/) ON (n./(/g;' \
|
||||
-e '/^CREATE CONSTRAINT/d' $INPUT_SCHEMA >> "$OUTPUT"
|
||||
|
||||
cat "$INPUT_NODES" >> "$OUTPUT"
|
||||
cat "$INPUT_RELATIONSHIPS" >> "$OUTPUT"
|
||||
|
||||
sed -e '/^DROP CONSTRAINT/d' "$INPUT_CLEANUP" >> "$OUTPUT"
|
||||
|
||||
echo 'DROP INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' >> "$OUTPUT"
|
||||
|
||||
echo ""
|
||||
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher file under $OUTPUT"
|
||||
echo ""
|
||||
echo "Please import data by executing => \`cat $OUTPUT | mgconsole\`"
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
COLOR_ORANGE="\e[38;5;208m"
|
||||
COLOR_GREEN="\e[38;5;35m"
|
||||
COLOR_RED="\e[0;31m"
|
||||
COLOR_NULL="\e[0m"
|
||||
|
||||
print_help() {
|
||||
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 input_file_schema_path input_file_nodes_path input_file_relationships_path input_file_cleanup_path output_file_schema_path output_file_nodes_path output_file_relationships_path output_file_cleanup_path"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$#" -ne 8 ]; then
|
||||
print_help
|
||||
fi
|
||||
INPUT_SCHEMA="$1"
|
||||
INPUT_NODES="$2"
|
||||
INPUT_RELATIONSHIPS="$3"
|
||||
INPUT_CLEANUP="$4"
|
||||
OUTPUT_SCHEMA="$5"
|
||||
OUTPUT_NODES="$6"
|
||||
OUTPUT_RELATIONSHIPS="$7"
|
||||
OUTPUT_CLEANUP="$8"
|
||||
|
||||
if [ ! -f "$INPUT_SCHEMA" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_NODES" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_RELATIONSHIPS" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_CLEANUP" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create constraints manually if needed${COLOR_NULL}"
|
||||
|
||||
|
||||
echo 'CREATE INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' > "$OUTPUT_SCHEMA"
|
||||
|
||||
sed -e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
|
||||
-e 's/) ON (n./(/g;' \
|
||||
-e '/^CREATE CONSTRAINT/d' $INPUT_SCHEMA >> "$OUTPUT_SCHEMA"
|
||||
|
||||
cat "$INPUT_NODES" > "$OUTPUT_NODES"
|
||||
cat "$INPUT_RELATIONSHIPS" > "$OUTPUT_RELATIONSHIPS"
|
||||
|
||||
sed -e '/^DROP CONSTRAINT/d' "$INPUT_CLEANUP" >> "$OUTPUT_CLEANUP"
|
||||
|
||||
echo 'DROP INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' >> "$OUTPUT_CLEANUP"
|
||||
|
||||
echo ""
|
||||
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher files under $OUTPUT_SCHEMA, $OUTPUT_NODES, $OUTPUT_RELATIONSHIPS and $OUTPUT_CLEANUP"
|
||||
echo ""
|
||||
echo "Please import data by executing => \`cat $OUTPUT_SCHEMA | mgconsole\`, \`cat $OUTPUT_NODES | mgconsole\`, \`cat $OUTPUT_RELATIONSHIPS | mgconsole\` and \`cat $OUTPUT_CLEANUP | mgconsole\`"
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -283,7 +283,7 @@ inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memor
|
||||
}
|
||||
|
||||
// mgp_graph
|
||||
|
||||
|
||||
inline bool graph_is_transactional(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_transactional, graph); }
|
||||
|
||||
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
|
||||
|
||||
1
libs/.gitignore
vendored
1
libs/.gitignore
vendored
@@ -7,4 +7,3 @@
|
||||
!pulsar.patch
|
||||
!antlr4.10.1.patch
|
||||
!rocksdb8.1.1.patch
|
||||
!nuraft2.1.0.patch
|
||||
|
||||
@@ -16,7 +16,7 @@ set(GFLAGS_NOTHREADS OFF)
|
||||
|
||||
# NOTE: config/generate.py depends on the gflags help XML format.
|
||||
find_package(gflags REQUIRED)
|
||||
find_package(fmt 8.0.1 REQUIRED)
|
||||
find_package(fmt 8.0.1)
|
||||
find_package(ZLIB 1.2.11 REQUIRED)
|
||||
|
||||
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
@@ -5,7 +5,7 @@ index ee9b58c..31359a9 100644
|
||||
@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU "Enables PMU usage on ARM platforms" OFF)
|
||||
# | Library Build and Install Properties |
|
||||
# +--------------------------------------------------------+
|
||||
|
||||
|
||||
-add_library(rdtsc SHARED
|
||||
+add_library(rdtsc
|
||||
src/cycles.c
|
||||
@@ -14,7 +14,7 @@ index ee9b58c..31359a9 100644
|
||||
@@ -72,15 +72,6 @@ target_include_directories(rdtsc
|
||||
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
|
||||
)
|
||||
|
||||
|
||||
-# Install directory changes depending on build mode
|
||||
-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug")
|
||||
- # During debug, the library will be installed into a local directory
|
||||
@@ -27,15 +27,3 @@ index ee9b58c..31359a9 100644
|
||||
# Specifying what to export when installing (GNUInstallDirs required)
|
||||
install(TARGETS rdtsc
|
||||
EXPORT librstsc-config
|
||||
diff --git a/include/librdtsc/common_timer.h b/include/librdtsc/common_timer.h
|
||||
index a6922d8..080dc77 100644
|
||||
--- a/include/librdtsc/common_timer.h
|
||||
+++ b/include/librdtsc/common_timer.h
|
||||
@@ -2,6 +2,7 @@
|
||||
#define LIBRDTSC_COMMON_TIMER_H
|
||||
|
||||
#include <librdtsc/common.h>
|
||||
+#include <librdtsc/cycles.h>
|
||||
|
||||
extern uint64_t rdtsc_get_tsc_freq_arch();
|
||||
extern uint64_t rdtsc_get_tsc_freq();
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
diff --git a/include/libnuraft/asio_service_options.hxx b/include/libnuraft/asio_service_options.hxx
|
||||
index 8fe1ec9..9497355 100644
|
||||
--- a/include/libnuraft/asio_service_options.hxx
|
||||
+++ b/include/libnuraft/asio_service_options.hxx
|
||||
@@ -17,6 +17,7 @@ limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
+#include <cstdint>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <system_error>
|
||||
diff --git a/include/libnuraft/callback.hxx b/include/libnuraft/callback.hxx
|
||||
index 7b71624..d48c1e2 100644
|
||||
--- a/include/libnuraft/callback.hxx
|
||||
+++ b/include/libnuraft/callback.hxx
|
||||
@@ -18,6 +18,7 @@ limitations under the License.
|
||||
#ifndef _CALLBACK_H_
|
||||
#define _CALLBACK_H_
|
||||
|
||||
+#include <cstdint>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
21
libs/rocksdb.patch
Normal file
21
libs/rocksdb.patch
Normal file
@@ -0,0 +1,21 @@
|
||||
diff --git a/CMakeLists.txt b/CMakeLists.txt
|
||||
index 6761929..6a369af 100644
|
||||
--- a/CMakeLists.txt
|
||||
+++ b/CMakeLists.txt
|
||||
@@ -220,6 +220,7 @@ else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
|
||||
endif()
|
||||
endif()
|
||||
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-copy -Wno-unused-but-set-variable")
|
||||
endif()
|
||||
|
||||
include(CheckCCompilerFlag)
|
||||
@@ -997,7 +998,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
|
||||
|
||||
if(ROCKSDB_BUILD_SHARED)
|
||||
install(
|
||||
- TARGETS ${ROCKSDB_SHARED_LIB}
|
||||
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
|
||||
EXPORT RocksDBTargets
|
||||
COMPONENT runtime
|
||||
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
@@ -123,10 +123,10 @@ declare -A primary_urls=(
|
||||
["pulsar"]="http://$local_cache_host/git/pulsar.git"
|
||||
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
|
||||
["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
|
||||
["absl"]="http://$local_cache_host/git/abseil-cpp.git"
|
||||
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
|
||||
["range-v3"]="http://$local_cache_host/git/range-v3.git"
|
||||
["nuraft"]="http://$local_cache_host/git/NuRaft.git"
|
||||
["absl"]="https://$local_cache_host/git/abseil-cpp.git"
|
||||
["jemalloc"]="https://$local_cache_host/git/jemalloc.git"
|
||||
["range-v3"]="https://$local_cache_host/git/ericniebler/range-v3.git"
|
||||
["nuraft"]="https://$local_cache_host/git/eBay/NuRaft.git"
|
||||
)
|
||||
|
||||
# The goal of secondary urls is to have links to the "source of truth" of
|
||||
@@ -168,11 +168,12 @@ pushd antlr4
|
||||
git apply ../antlr4.10.1.patch
|
||||
popd
|
||||
|
||||
cppitertools_ref="v2.1" # 2021-01-15
|
||||
# cppitertools v2.0 2019-12-23
|
||||
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
|
||||
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
|
||||
|
||||
# rapidcheck
|
||||
rapidcheck_tag="1c91f40e64d87869250cfb610376c629307bf77d" # (2023-08-15)
|
||||
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
|
||||
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
|
||||
|
||||
# google benchmark
|
||||
@@ -220,7 +221,7 @@ repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient
|
||||
mgconsole_tag="v1.4.0" # (2023-05-21)
|
||||
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
|
||||
|
||||
spdlog_tag="v1.12.0" # (2022-11-02)
|
||||
spdlog_tag="v1.9.2" # (2021-08-12)
|
||||
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag" true
|
||||
|
||||
# librdkafka
|
||||
@@ -269,8 +270,6 @@ pushd jemalloc
|
||||
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
|
||||
./configure \
|
||||
--disable-cxx \
|
||||
--with-lg-page=12 \
|
||||
--with-lg-hugepage=21 \
|
||||
--enable-shared=no --prefix=$working_dir \
|
||||
--with-malloc-conf="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
|
||||
@@ -285,6 +284,5 @@ repo_clone_try_double "${primary_urls[range-v3]}" "${secondary_urls[range-v3]}"
|
||||
nuraft_tag="v2.1.0"
|
||||
repo_clone_try_double "${primary_urls[nuraft]}" "${secondary_urls[nuraft]}" "nuraft" "$nuraft_tag" true
|
||||
pushd nuraft
|
||||
git apply ../nuraft2.1.0.patch
|
||||
./prepare.sh
|
||||
popd
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -9,11 +9,10 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <boost/functional/hash.hpp>
|
||||
#include <mgp.hpp>
|
||||
#include "utils/string.hpp"
|
||||
|
||||
#include <unordered_set>
|
||||
#include <optional>
|
||||
|
||||
namespace Schema {
|
||||
|
||||
@@ -38,7 +37,6 @@ constexpr std::string_view kParameterIndices = "indices";
|
||||
constexpr std::string_view kParameterUniqueConstraints = "unique_constraints";
|
||||
constexpr std::string_view kParameterExistenceConstraints = "existence_constraints";
|
||||
constexpr std::string_view kParameterDropExisting = "drop_existing";
|
||||
constexpr int kInitialNumberOfPropertyOccurances = 1;
|
||||
|
||||
std::string TypeOf(const mgp::Type &type);
|
||||
|
||||
@@ -110,79 +108,83 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t
|
||||
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
|
||||
}
|
||||
|
||||
struct PropertyInfo {
|
||||
std::unordered_set<std::string> property_types; // property types
|
||||
int64_t number_of_property_occurrences = 0;
|
||||
struct Property {
|
||||
std::string name;
|
||||
mgp::Value value;
|
||||
|
||||
PropertyInfo() = default;
|
||||
explicit PropertyInfo(std::string &&property_type)
|
||||
: property_types({std::move(property_type)}),
|
||||
number_of_property_occurrences(Schema::kInitialNumberOfPropertyOccurances) {}
|
||||
};
|
||||
|
||||
struct LabelsInfo {
|
||||
std::unordered_map<std::string, PropertyInfo> properties; // key is a property name
|
||||
int64_t number_of_label_occurrences = 0;
|
||||
Property(const std::string &name, mgp::Value &&value) : name(name), value(std::move(value)) {}
|
||||
};
|
||||
|
||||
struct LabelsHash {
|
||||
std::size_t operator()(const std::set<std::string> &s) const { return boost::hash_range(s.begin(), s.end()); }
|
||||
std::size_t operator()(const std::set<std::string> &set) const {
|
||||
std::size_t seed = set.size();
|
||||
for (const auto &i : set) {
|
||||
seed ^= std::hash<std::string>{}(i) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
|
||||
struct LabelsComparator {
|
||||
bool operator()(const std::set<std::string> &lhs, const std::set<std::string> &rhs) const { return lhs == rhs; }
|
||||
};
|
||||
|
||||
struct PropertyComparator {
|
||||
bool operator()(const Property &lhs, const Property &rhs) const { return lhs.name < rhs.name; }
|
||||
};
|
||||
|
||||
struct PropertyInfo {
|
||||
std::set<Property, PropertyComparator> properties;
|
||||
bool mandatory;
|
||||
};
|
||||
|
||||
void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result,
|
||||
mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
std::unordered_map<std::set<std::string>, LabelsInfo, LabelsHash, LabelsComparator> node_types_properties;
|
||||
std::unordered_map<std::set<std::string>, PropertyInfo, LabelsHash, LabelsComparator> node_types_properties;
|
||||
|
||||
for (const auto node : mgp::Graph(memgraph_graph).Nodes()) {
|
||||
for (auto node : mgp::Graph(memgraph_graph).Nodes()) {
|
||||
std::set<std::string> labels_set = {};
|
||||
for (const auto label : node.Labels()) {
|
||||
for (auto label : node.Labels()) {
|
||||
labels_set.emplace(label);
|
||||
}
|
||||
|
||||
node_types_properties[labels_set].number_of_label_occurrences++;
|
||||
if (node_types_properties.find(labels_set) == node_types_properties.end()) {
|
||||
node_types_properties[labels_set] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
|
||||
}
|
||||
|
||||
if (node.Properties().empty()) {
|
||||
node_types_properties[labels_set].mandatory = false; // if there is node with no property, it is not mandatory
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &labels_info = node_types_properties.at(labels_set);
|
||||
for (const auto &[key, prop] : node.Properties()) {
|
||||
auto prop_type = TypeOf(prop.Type());
|
||||
if (labels_info.properties.find(key) == labels_info.properties.end()) {
|
||||
labels_info.properties[key] = PropertyInfo{std::move(prop_type)};
|
||||
} else {
|
||||
labels_info.properties[key].property_types.emplace(prop_type);
|
||||
labels_info.properties[key].number_of_property_occurrences++;
|
||||
auto &property_info = node_types_properties.at(labels_set);
|
||||
for (auto &[key, prop] : node.Properties()) {
|
||||
property_info.properties.emplace(key, std::move(prop));
|
||||
if (property_info.mandatory) {
|
||||
property_info.mandatory =
|
||||
property_info.properties.size() == 1; // if there is only one property, it is mandatory
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &[node_type, labels_info] : node_types_properties) { // node type is a set of labels
|
||||
for (auto &[labels, property_info] : node_types_properties) {
|
||||
std::string label_type;
|
||||
auto labels_list = mgp::List();
|
||||
for (const auto &label : node_type) {
|
||||
mgp::List labels_list = mgp::List();
|
||||
for (auto const &label : labels) {
|
||||
label_type += ":`" + std::string(label) + "`";
|
||||
labels_list.AppendExtend(mgp::Value(label));
|
||||
}
|
||||
for (const auto &prop : labels_info.properties) {
|
||||
auto prop_types = mgp::List();
|
||||
for (const auto &prop_type : prop.second.property_types) {
|
||||
prop_types.AppendExtend(mgp::Value(prop_type));
|
||||
}
|
||||
bool mandatory = prop.second.number_of_property_occurrences == labels_info.number_of_label_occurrences;
|
||||
for (auto const &prop : property_info.properties) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesNode(record, label_type, labels_list, prop.first, prop_types, mandatory);
|
||||
ProcessPropertiesNode(record, label_type, labels_list, prop.name, TypeOf(prop.value.Type()),
|
||||
property_info.mandatory);
|
||||
}
|
||||
if (labels_info.properties.empty()) {
|
||||
if (property_info.properties.empty()) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesNode<mgp::List>(record, label_type, labels_list, "", mgp::List(), false);
|
||||
ProcessPropertiesNode<std::string>(record, label_type, labels_list, "", "", false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,45 +197,40 @@ void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph,
|
||||
void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
|
||||
std::unordered_map<std::string, LabelsInfo> rel_types_properties;
|
||||
std::unordered_map<std::string, PropertyInfo> rel_types_properties;
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
const auto graph = mgp::Graph(memgraph_graph);
|
||||
for (const auto rel : graph.Relationships()) {
|
||||
const mgp::Graph graph = mgp::Graph(memgraph_graph);
|
||||
for (auto rel : graph.Relationships()) {
|
||||
std::string rel_type = std::string(rel.Type());
|
||||
|
||||
rel_types_properties[rel_type].number_of_label_occurrences++;
|
||||
if (rel_types_properties.find(rel_type) == rel_types_properties.end()) {
|
||||
rel_types_properties[rel_type] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
|
||||
}
|
||||
|
||||
if (rel.Properties().empty()) {
|
||||
rel_types_properties[rel_type].mandatory = false; // if there is rel with no property, it is not mandatory
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &labels_info = rel_types_properties.at(rel_type);
|
||||
auto &property_info = rel_types_properties.at(rel_type);
|
||||
for (auto &[key, prop] : rel.Properties()) {
|
||||
auto prop_type = TypeOf(prop.Type());
|
||||
if (labels_info.properties.find(key) == labels_info.properties.end()) {
|
||||
labels_info.properties[key] = PropertyInfo{std::move(prop_type)};
|
||||
} else {
|
||||
labels_info.properties[key].property_types.emplace(prop_type);
|
||||
labels_info.properties[key].number_of_property_occurrences++;
|
||||
property_info.properties.emplace(key, std::move(prop));
|
||||
if (property_info.mandatory) {
|
||||
property_info.mandatory =
|
||||
property_info.properties.size() == 1; // if there is only one property, it is mandatory
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &[rel_type, labels_info] : rel_types_properties) {
|
||||
std::string type_str = ":`" + std::string(rel_type) + "`";
|
||||
for (const auto &prop : labels_info.properties) {
|
||||
auto prop_types = mgp::List();
|
||||
for (const auto &prop_type : prop.second.property_types) {
|
||||
prop_types.AppendExtend(mgp::Value(prop_type));
|
||||
}
|
||||
bool mandatory = prop.second.number_of_property_occurrences == labels_info.number_of_label_occurrences;
|
||||
for (auto &[type, property_info] : rel_types_properties) {
|
||||
std::string type_str = ":`" + std::string(type) + "`";
|
||||
for (auto const &prop : property_info.properties) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesRel(record, type_str, prop.first, prop_types, mandatory);
|
||||
ProcessPropertiesRel(record, type_str, prop.name, TypeOf(prop.value.Type()), property_info.mandatory);
|
||||
}
|
||||
if (labels_info.properties.empty()) {
|
||||
if (property_info.properties.empty()) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesRel<mgp::List>(record, type_str, "", mgp::List(), false);
|
||||
ProcessPropertiesRel<std::string>(record, type_str, "", "", false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,42 +35,16 @@ DEFINE_VALIDATED_string(auth_module_executable, "", "Absolute path to the auth m
|
||||
}
|
||||
return true;
|
||||
});
|
||||
DEFINE_bool(auth_module_create_missing_user, true, "Set to false to disable creation of missing users.");
|
||||
DEFINE_bool(auth_module_create_missing_role, true, "Set to false to disable creation of missing roles.");
|
||||
DEFINE_bool(auth_module_manage_roles, true, "Set to false to disable management of roles through the auth module.");
|
||||
DEFINE_VALIDATED_int32(auth_module_timeout_ms, 10000,
|
||||
"Timeout (in milliseconds) used when waiting for a "
|
||||
"response from the auth module.",
|
||||
FLAG_IN_RANGE(100, 1800000));
|
||||
|
||||
// DEPRECATED FLAGS
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
|
||||
DEFINE_VALIDATED_HIDDEN_bool(auth_module_create_missing_user, true,
|
||||
"Set to false to disable creation of missing users.", {
|
||||
spdlog::warn(
|
||||
"auth_module_create_missing_user flag is deprecated. It not possible to create "
|
||||
"users through the module anymore.");
|
||||
return true;
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
|
||||
DEFINE_VALIDATED_HIDDEN_bool(auth_module_create_missing_role, true,
|
||||
"Set to false to disable creation of missing roles.", {
|
||||
spdlog::warn(
|
||||
"auth_module_create_missing_role flag is deprecated. It not possible to create "
|
||||
"roles through the module anymore.");
|
||||
return true;
|
||||
});
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
|
||||
DEFINE_VALIDATED_HIDDEN_bool(
|
||||
auth_module_manage_roles, true, "Set to false to disable management of roles through the auth module.", {
|
||||
spdlog::warn(
|
||||
"auth_module_manage_roles flag is deprecated. It not possible to create roles through the module anymore.");
|
||||
return true;
|
||||
});
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
const Auth::Epoch Auth::kStartEpoch = 1;
|
||||
|
||||
namespace {
|
||||
#ifdef MG_ENTERPRISE
|
||||
/**
|
||||
@@ -83,19 +57,16 @@ struct UpdateAuthData : memgraph::system::ISystemAction {
|
||||
void DoDurability() override { /* Done during Auth execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
bool DoReplication(replication::ReplicationClient &client, replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const replication::UpdateAuthDataRes &response) { return response.success; };
|
||||
if (user_) {
|
||||
return client.SteamAndFinalizeDelta<replication::UpdateAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
*user_);
|
||||
check_response, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(), *user_);
|
||||
}
|
||||
if (role_) {
|
||||
return client.SteamAndFinalizeDelta<replication::UpdateAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
*role_);
|
||||
check_response, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(), *role_);
|
||||
}
|
||||
// Should never get here
|
||||
MG_ASSERT(false, "Trying to update auth data that is not a user nor a role");
|
||||
@@ -117,8 +88,7 @@ struct DropAuthData : memgraph::system::ISystemAction {
|
||||
void DoDurability() override { /* Done during Auth execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
bool DoReplication(replication::ReplicationClient &client, replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const replication::DropAuthDataRes &response) { return response.success; };
|
||||
|
||||
@@ -132,8 +102,7 @@ struct DropAuthData : memgraph::system::ISystemAction {
|
||||
break;
|
||||
}
|
||||
return client.SteamAndFinalizeDelta<replication::DropAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
type, name_);
|
||||
check_response, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(), type, name_);
|
||||
}
|
||||
void PostReplication(replication::RoleMainData &mainData) const override {}
|
||||
|
||||
@@ -218,17 +187,6 @@ void MigrateVersions(kvstore::KVStore &store) {
|
||||
version_str = kVersionV1;
|
||||
}
|
||||
}
|
||||
|
||||
auto ParseJson(std::string_view str) {
|
||||
nlohmann::json data;
|
||||
try {
|
||||
data = nlohmann::json::parse(str);
|
||||
} catch (const nlohmann::json::parse_error &e) {
|
||||
throw AuthException("Couldn't load auth data!");
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
}; // namespace
|
||||
|
||||
Auth::Auth(std::string storage_directory, Config config)
|
||||
@@ -236,11 +194,8 @@ Auth::Auth(std::string storage_directory, Config config)
|
||||
MigrateVersions(storage_);
|
||||
}
|
||||
|
||||
std::optional<UserOrRole> Auth::Authenticate(const std::string &username, const std::string &password) {
|
||||
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
|
||||
if (module_.IsUsed()) {
|
||||
/*
|
||||
* MODULE AUTH STORAGE
|
||||
*/
|
||||
const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
|
||||
if (license_check_result.HasError()) {
|
||||
spdlog::warn(license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
|
||||
@@ -265,64 +220,108 @@ std::optional<UserOrRole> Auth::Authenticate(const std::string &username, const
|
||||
auto is_authenticated = ret_authenticated.get<bool>();
|
||||
const auto &rolename = ret_role.get<std::string>();
|
||||
|
||||
// Check if role is present
|
||||
auto role = GetRole(rolename);
|
||||
if (!role) {
|
||||
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the role '{}' doesn't exist.",
|
||||
username, rolename, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Authenticate the user.
|
||||
if (!is_authenticated) return std::nullopt;
|
||||
|
||||
return RoleWUsername{username, std::move(*role)};
|
||||
}
|
||||
/**
|
||||
* TODO
|
||||
* The auth module should not update auth data.
|
||||
* There is now way to replicate it and we should not be storing sensitive data if we don't have to.
|
||||
*/
|
||||
|
||||
/*
|
||||
* LOCAL AUTH STORAGE
|
||||
*/
|
||||
auto user = GetUser(username);
|
||||
if (!user) {
|
||||
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the user doesn't exist.", username,
|
||||
"https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
if (!user->CheckPassword(password)) {
|
||||
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the password is not correct.",
|
||||
username, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
if (user->UpgradeHash(password)) {
|
||||
// Find or create the user and return it.
|
||||
auto user = GetUser(username);
|
||||
if (!user) {
|
||||
if (FLAGS_auth_module_create_missing_user) {
|
||||
user = AddUser(username, password);
|
||||
if (!user) {
|
||||
spdlog::warn(utils::MessageWithLink(
|
||||
"Couldn't create the missing user '{}' using the auth module because the user already exists as a role.",
|
||||
username, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
} else {
|
||||
spdlog::warn(utils::MessageWithLink(
|
||||
"Couldn't authenticate user '{}' using the auth module because the user doesn't exist.", username,
|
||||
"https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
} else {
|
||||
UpdatePassword(*user, password);
|
||||
}
|
||||
if (FLAGS_auth_module_manage_roles) {
|
||||
if (!rolename.empty()) {
|
||||
auto role = GetRole(rolename);
|
||||
if (!role) {
|
||||
if (FLAGS_auth_module_create_missing_role) {
|
||||
role = AddRole(rolename);
|
||||
if (!role) {
|
||||
spdlog::warn(
|
||||
utils::MessageWithLink("Couldn't authenticate user '{}' using the auth module because the user's "
|
||||
"role '{}' already exists as a user.",
|
||||
username, rolename, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
SaveRole(*role);
|
||||
} else {
|
||||
spdlog::warn(utils::MessageWithLink(
|
||||
"Couldn't authenticate user '{}' using the auth module because the user's role '{}' doesn't exist.",
|
||||
username, rolename, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
user->SetRole(*role);
|
||||
} else {
|
||||
user->ClearRole();
|
||||
}
|
||||
}
|
||||
SaveUser(*user);
|
||||
}
|
||||
return user;
|
||||
} else {
|
||||
auto user = GetUser(username);
|
||||
if (!user) {
|
||||
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the user doesn't exist.", username,
|
||||
"https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
if (!user->CheckPassword(password)) {
|
||||
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the password is not correct.",
|
||||
username, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
if (user->UpgradeHash(password)) {
|
||||
SaveUser(*user);
|
||||
}
|
||||
|
||||
return user;
|
||||
return user;
|
||||
}
|
||||
}
|
||||
|
||||
void Auth::LinkUser(User &user) const {
|
||||
auto link = storage_.Get(kLinkPrefix + user.username());
|
||||
std::optional<User> Auth::GetUser(const std::string &username_orig) const {
|
||||
auto username = utils::ToLowerCase(username_orig);
|
||||
auto existing_user = storage_.Get(kUserPrefix + username);
|
||||
if (!existing_user) return std::nullopt;
|
||||
|
||||
nlohmann::json data;
|
||||
try {
|
||||
data = nlohmann::json::parse(*existing_user);
|
||||
} catch (const nlohmann::json::parse_error &e) {
|
||||
throw AuthException("Couldn't load user data!");
|
||||
}
|
||||
|
||||
auto user = User::Deserialize(data);
|
||||
auto link = storage_.Get(kLinkPrefix + username);
|
||||
|
||||
if (link) {
|
||||
auto role = GetRole(*link);
|
||||
if (role) {
|
||||
user.SetRole(*role);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<User> Auth::GetUser(const std::string &username_orig) const {
|
||||
if (module_.IsUsed()) return std::nullopt; // User's are not supported when using module
|
||||
auto username = utils::ToLowerCase(username_orig);
|
||||
auto existing_user = storage_.Get(kUserPrefix + username);
|
||||
if (!existing_user) return std::nullopt;
|
||||
|
||||
auto user = User::Deserialize(ParseJson(*existing_user));
|
||||
LinkUser(user);
|
||||
return user;
|
||||
}
|
||||
|
||||
void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
|
||||
DisableIfModuleUsed();
|
||||
bool success = false;
|
||||
if (const auto *role = user.role(); role != nullptr) {
|
||||
success = storage_.PutMultiple(
|
||||
@@ -334,10 +333,6 @@ void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
|
||||
if (!success) {
|
||||
throw AuthException("Couldn't save user '{}'!", user.username());
|
||||
}
|
||||
|
||||
// Durability updated -> new epoch
|
||||
UpdateEpoch();
|
||||
|
||||
// All changes to the user end up calling this function, so no need to add a delta anywhere else
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
@@ -347,7 +342,6 @@ void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
|
||||
}
|
||||
|
||||
void Auth::UpdatePassword(auth::User &user, const std::optional<std::string> &password) {
|
||||
DisableIfModuleUsed();
|
||||
// Check if null
|
||||
if (!password) {
|
||||
if (!config_.password_permit_null) {
|
||||
@@ -379,7 +373,6 @@ void Auth::UpdatePassword(auth::User &user, const std::optional<std::string> &pa
|
||||
|
||||
std::optional<User> Auth::AddUser(const std::string &username, const std::optional<std::string> &password,
|
||||
system::Transaction *system_tx) {
|
||||
DisableIfModuleUsed();
|
||||
if (!NameRegexMatch(username)) {
|
||||
throw AuthException("Invalid user name.");
|
||||
}
|
||||
@@ -394,17 +387,12 @@ std::optional<User> Auth::AddUser(const std::string &username, const std::option
|
||||
}
|
||||
|
||||
bool Auth::RemoveUser(const std::string &username_orig, system::Transaction *system_tx) {
|
||||
DisableIfModuleUsed();
|
||||
auto username = utils::ToLowerCase(username_orig);
|
||||
if (!storage_.Get(kUserPrefix + username)) return false;
|
||||
std::vector<std::string> keys({kLinkPrefix + username, kUserPrefix + username});
|
||||
if (!storage_.DeleteMultiple(keys)) {
|
||||
throw AuthException("Couldn't remove user '{}'!", username);
|
||||
}
|
||||
|
||||
// Durability updated -> new epoch
|
||||
UpdateEpoch();
|
||||
|
||||
// Handling drop user delta
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
@@ -419,12 +407,9 @@ std::vector<auth::User> Auth::AllUsers() const {
|
||||
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
|
||||
auto username = it->first.substr(kUserPrefix.size());
|
||||
if (username != utils::ToLowerCase(username)) continue;
|
||||
try {
|
||||
User user = auth::User::Deserialize(ParseJson(it->second)); // Will throw on failure
|
||||
LinkUser(user);
|
||||
ret.emplace_back(std::move(user));
|
||||
} catch (AuthException &) {
|
||||
continue;
|
||||
auto user = GetUser(username);
|
||||
if (user) {
|
||||
ret.push_back(std::move(*user));
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
@@ -435,12 +420,9 @@ std::vector<std::string> Auth::AllUsernames() const {
|
||||
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
|
||||
auto username = it->first.substr(kUserPrefix.size());
|
||||
if (username != utils::ToLowerCase(username)) continue;
|
||||
try {
|
||||
// Check if serialized correctly
|
||||
memgraph::auth::User::Deserialize(ParseJson(it->second)); // Will throw on failure
|
||||
ret.emplace_back(std::move(username));
|
||||
} catch (AuthException &) {
|
||||
continue;
|
||||
auto user = GetUser(username);
|
||||
if (user) {
|
||||
ret.push_back(username);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
@@ -448,24 +430,25 @@ std::vector<std::string> Auth::AllUsernames() const {
|
||||
|
||||
bool Auth::HasUsers() const { return storage_.begin(kUserPrefix) != storage_.end(kUserPrefix); }
|
||||
|
||||
bool Auth::AccessControlled() const { return HasUsers() || module_.IsUsed(); }
|
||||
|
||||
std::optional<Role> Auth::GetRole(const std::string &rolename_orig) const {
|
||||
auto rolename = utils::ToLowerCase(rolename_orig);
|
||||
auto existing_role = storage_.Get(kRolePrefix + rolename);
|
||||
if (!existing_role) return std::nullopt;
|
||||
|
||||
return Role::Deserialize(ParseJson(*existing_role));
|
||||
nlohmann::json data;
|
||||
try {
|
||||
data = nlohmann::json::parse(*existing_role);
|
||||
} catch (const nlohmann::json::parse_error &e) {
|
||||
throw AuthException("Couldn't load role data!");
|
||||
}
|
||||
|
||||
return Role::Deserialize(data);
|
||||
}
|
||||
|
||||
void Auth::SaveRole(const Role &role, system::Transaction *system_tx) {
|
||||
if (!storage_.Put(kRolePrefix + role.rolename(), role.Serialize().dump())) {
|
||||
throw AuthException("Couldn't save role '{}'!", role.rolename());
|
||||
}
|
||||
|
||||
// Durability updated -> new epoch
|
||||
UpdateEpoch();
|
||||
|
||||
// All changes to the role end up calling this function, so no need to add a delta anywhere else
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
@@ -498,10 +481,6 @@ bool Auth::RemoveRole(const std::string &rolename_orig, system::Transaction *sys
|
||||
if (!storage_.DeleteMultiple(keys)) {
|
||||
throw AuthException("Couldn't remove role '{}'!", rolename);
|
||||
}
|
||||
|
||||
// Durability updated -> new epoch
|
||||
UpdateEpoch();
|
||||
|
||||
// Handling drop role delta
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
@@ -516,8 +495,11 @@ std::vector<auth::Role> Auth::AllRoles() const {
|
||||
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
|
||||
auto rolename = it->first.substr(kRolePrefix.size());
|
||||
if (rolename != utils::ToLowerCase(rolename)) continue;
|
||||
Role role = memgraph::auth::Role::Deserialize(ParseJson(it->second)); // Will throw on failure
|
||||
ret.emplace_back(std::move(role));
|
||||
if (auto role = GetRole(rolename)) {
|
||||
ret.push_back(*role);
|
||||
} else {
|
||||
throw AuthException("Couldn't load role '{}'!", rolename);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@@ -527,19 +509,14 @@ std::vector<std::string> Auth::AllRolenames() const {
|
||||
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
|
||||
auto rolename = it->first.substr(kRolePrefix.size());
|
||||
if (rolename != utils::ToLowerCase(rolename)) continue;
|
||||
try {
|
||||
// Check that the data is serialized correctly
|
||||
memgraph::auth::Role::Deserialize(ParseJson(it->second));
|
||||
ret.emplace_back(std::move(rolename));
|
||||
} catch (AuthException &) {
|
||||
continue;
|
||||
if (auto role = GetRole(rolename)) {
|
||||
ret.push_back(rolename);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig) const {
|
||||
DisableIfModuleUsed();
|
||||
const auto rolename = utils::ToLowerCase(rolename_orig);
|
||||
std::vector<auth::User> ret;
|
||||
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix); ++it) {
|
||||
@@ -558,176 +535,51 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
Auth::Result Auth::GrantDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
using enum Auth::Result;
|
||||
if (module_.IsUsed()) {
|
||||
if (auto role = GetRole(name)) {
|
||||
GrantDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
return NO_ROLE;
|
||||
}
|
||||
bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
if (auto user = GetUser(name)) {
|
||||
GrantDatabase(db, *user, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
if (auto role = GetRole(name)) {
|
||||
GrantDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
return NO_USER_ROLE;
|
||||
}
|
||||
|
||||
void Auth::GrantDatabase(const std::string &db, User &user, system::Transaction *system_tx) {
|
||||
if (db == kAllDatabases) {
|
||||
user.db_access().GrantAll();
|
||||
} else {
|
||||
user.db_access().Grant(db);
|
||||
}
|
||||
SaveUser(user, system_tx);
|
||||
}
|
||||
|
||||
void Auth::GrantDatabase(const std::string &db, Role &role, system::Transaction *system_tx) {
|
||||
if (db == kAllDatabases) {
|
||||
role.db_access().GrantAll();
|
||||
} else {
|
||||
role.db_access().Grant(db);
|
||||
}
|
||||
SaveRole(role, system_tx);
|
||||
}
|
||||
|
||||
Auth::Result Auth::DenyDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
using enum Auth::Result;
|
||||
if (module_.IsUsed()) {
|
||||
if (auto role = GetRole(name)) {
|
||||
DenyDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
if (db == kAllDatabases) {
|
||||
user->db_access().GrantAll();
|
||||
} else {
|
||||
user->db_access().Add(db);
|
||||
}
|
||||
return NO_ROLE;
|
||||
SaveUser(*user, system_tx);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
if (auto user = GetUser(name)) {
|
||||
DenyDatabase(db, *user, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
if (auto role = GetRole(name)) {
|
||||
DenyDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
return NO_USER_ROLE;
|
||||
}
|
||||
|
||||
void Auth::DenyDatabase(const std::string &db, User &user, system::Transaction *system_tx) {
|
||||
if (db == kAllDatabases) {
|
||||
user.db_access().DenyAll();
|
||||
} else {
|
||||
user.db_access().Deny(db);
|
||||
}
|
||||
SaveUser(user, system_tx);
|
||||
}
|
||||
|
||||
void Auth::DenyDatabase(const std::string &db, Role &role, system::Transaction *system_tx) {
|
||||
if (db == kAllDatabases) {
|
||||
role.db_access().DenyAll();
|
||||
} else {
|
||||
role.db_access().Deny(db);
|
||||
}
|
||||
SaveRole(role, system_tx);
|
||||
}
|
||||
|
||||
Auth::Result Auth::RevokeDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
using enum Auth::Result;
|
||||
if (module_.IsUsed()) {
|
||||
if (auto role = GetRole(name)) {
|
||||
RevokeDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
if (db == kAllDatabases) {
|
||||
user->db_access().DenyAll();
|
||||
} else {
|
||||
user->db_access().Remove(db);
|
||||
}
|
||||
return NO_ROLE;
|
||||
SaveUser(*user, system_tx);
|
||||
return true;
|
||||
}
|
||||
if (auto user = GetUser(name)) {
|
||||
RevokeDatabase(db, *user, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
if (auto role = GetRole(name)) {
|
||||
RevokeDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
return NO_USER_ROLE;
|
||||
}
|
||||
|
||||
void Auth::RevokeDatabase(const std::string &db, User &user, system::Transaction *system_tx) {
|
||||
if (db == kAllDatabases) {
|
||||
user.db_access().RevokeAll();
|
||||
} else {
|
||||
user.db_access().Revoke(db);
|
||||
}
|
||||
SaveUser(user, system_tx);
|
||||
}
|
||||
|
||||
void Auth::RevokeDatabase(const std::string &db, Role &role, system::Transaction *system_tx) {
|
||||
if (db == kAllDatabases) {
|
||||
role.db_access().RevokeAll();
|
||||
} else {
|
||||
role.db_access().Revoke(db);
|
||||
}
|
||||
SaveRole(role, system_tx);
|
||||
return false;
|
||||
}
|
||||
|
||||
void Auth::DeleteDatabase(const std::string &db, system::Transaction *system_tx) {
|
||||
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
|
||||
auto username = it->first.substr(kUserPrefix.size());
|
||||
try {
|
||||
User user = auth::User::Deserialize(ParseJson(it->second));
|
||||
LinkUser(user);
|
||||
user.db_access().Revoke(db);
|
||||
SaveUser(user, system_tx);
|
||||
} catch (AuthException &) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
|
||||
auto rolename = it->first.substr(kRolePrefix.size());
|
||||
try {
|
||||
auto role = memgraph::auth::Role::Deserialize(ParseJson(it->second));
|
||||
role.db_access().Revoke(db);
|
||||
SaveRole(role, system_tx);
|
||||
} catch (AuthException &) {
|
||||
continue;
|
||||
if (auto user = GetUser(username)) {
|
||||
user->db_access().Delete(db);
|
||||
SaveUser(*user, system_tx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Auth::Result Auth::SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx) {
|
||||
using enum Auth::Result;
|
||||
if (module_.IsUsed()) {
|
||||
if (auto role = GetRole(name)) {
|
||||
SetMainDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
return NO_ROLE;
|
||||
}
|
||||
bool Auth::SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx) {
|
||||
if (auto user = GetUser(name)) {
|
||||
SetMainDatabase(db, *user, system_tx);
|
||||
return SUCCESS;
|
||||
if (!user->db_access().SetDefault(db)) {
|
||||
throw AuthException("Couldn't set default database '{}' for user '{}'!", db, name);
|
||||
}
|
||||
SaveUser(*user, system_tx);
|
||||
return true;
|
||||
}
|
||||
if (auto role = GetRole(name)) {
|
||||
SetMainDatabase(db, *role, system_tx);
|
||||
return SUCCESS;
|
||||
}
|
||||
return NO_USER_ROLE;
|
||||
}
|
||||
|
||||
void Auth::SetMainDatabase(std::string_view db, User &user, system::Transaction *system_tx) {
|
||||
if (!user.db_access().SetMain(db)) {
|
||||
throw AuthException("Couldn't set default database '{}' for '{}'!", db, user.username());
|
||||
}
|
||||
SaveUser(user, system_tx);
|
||||
}
|
||||
|
||||
void Auth::SetMainDatabase(std::string_view db, Role &role, system::Transaction *system_tx) {
|
||||
if (!role.db_access().SetMain(db)) {
|
||||
throw AuthException("Couldn't set default database '{}' for '{}'!", db, role.rolename());
|
||||
}
|
||||
SaveRole(role, system_tx);
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -29,18 +29,6 @@ using SynchedAuth = memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph
|
||||
|
||||
static const constexpr char *const kAllDatabases = "*";
|
||||
|
||||
struct RoleWUsername : Role {
|
||||
template <typename... Args>
|
||||
RoleWUsername(std::string_view username, Args &&...args) : Role{std::forward<Args>(args)...}, username_{username} {}
|
||||
|
||||
std::string username() { return username_; }
|
||||
const std::string &username() const { return username_; }
|
||||
|
||||
private:
|
||||
std::string username_;
|
||||
};
|
||||
using UserOrRole = std::variant<User, RoleWUsername>;
|
||||
|
||||
/**
|
||||
* This class serves as the main Authentication/Authorization storage.
|
||||
* It provides functions for managing Users, Roles, Permissions and FineGrainedAccessPermissions.
|
||||
@@ -73,25 +61,6 @@ class Auth final {
|
||||
std::regex password_regex{password_regex_str};
|
||||
};
|
||||
|
||||
struct Epoch {
|
||||
Epoch() : epoch_{0} {}
|
||||
Epoch(unsigned e) : epoch_{e} {}
|
||||
|
||||
Epoch operator++() { return ++epoch_; }
|
||||
bool operator==(const Epoch &rhs) const = default;
|
||||
|
||||
private:
|
||||
unsigned epoch_;
|
||||
};
|
||||
|
||||
static const Epoch kStartEpoch;
|
||||
|
||||
enum class Result {
|
||||
SUCCESS,
|
||||
NO_USER_ROLE,
|
||||
NO_ROLE,
|
||||
};
|
||||
|
||||
explicit Auth(std::string storage_directory, Config config);
|
||||
|
||||
/**
|
||||
@@ -120,7 +89,7 @@ class Auth final {
|
||||
* @return a user when the username and password match, nullopt otherwise
|
||||
* @throw AuthException if unable to authenticate for whatever reason.
|
||||
*/
|
||||
std::optional<UserOrRole> Authenticate(const std::string &username, const std::string &password);
|
||||
std::optional<User> Authenticate(const std::string &username, const std::string &password);
|
||||
|
||||
/**
|
||||
* Gets a user from the storage.
|
||||
@@ -132,8 +101,6 @@ class Auth final {
|
||||
*/
|
||||
std::optional<User> GetUser(const std::string &username) const;
|
||||
|
||||
void LinkUser(User &user) const;
|
||||
|
||||
/**
|
||||
* Saves a user object to the storage.
|
||||
*
|
||||
@@ -196,13 +163,6 @@ class Auth final {
|
||||
*/
|
||||
bool HasUsers() const;
|
||||
|
||||
/**
|
||||
* Returns whether the access is controlled by authentication/authorization.
|
||||
*
|
||||
* @return `true` if auth needs to run
|
||||
*/
|
||||
bool AccessControlled() const;
|
||||
|
||||
/**
|
||||
* Gets a role from the storage.
|
||||
*
|
||||
@@ -213,37 +173,6 @@ class Auth final {
|
||||
*/
|
||||
std::optional<Role> GetRole(const std::string &rolename) const;
|
||||
|
||||
std::optional<UserOrRole> GetUserOrRole(const std::optional<std::string> &username,
|
||||
const std::optional<std::string> &rolename) const {
|
||||
auto expect = [](bool condition, std::string &&msg) {
|
||||
if (!condition) throw AuthException(std::move(msg));
|
||||
};
|
||||
// Special case if we are using a module; we must find the specified role
|
||||
if (module_.IsUsed()) {
|
||||
expect(username && rolename, "When using a module, a role needs to be connected to a username.");
|
||||
const auto role = GetRole(*rolename);
|
||||
expect(role != std::nullopt, "No role named " + *rolename);
|
||||
return UserOrRole(auth::RoleWUsername{*username, *role});
|
||||
}
|
||||
|
||||
// First check if we need to find a role
|
||||
if (username && rolename) {
|
||||
const auto role = GetRole(*rolename);
|
||||
expect(role != std::nullopt, "No role named " + *rolename);
|
||||
return UserOrRole(auth::RoleWUsername{*username, *role});
|
||||
}
|
||||
|
||||
// We are only looking for a user
|
||||
if (username) {
|
||||
const auto user = GetUser(*username);
|
||||
expect(user != std::nullopt, "No user named " + *username);
|
||||
return *user;
|
||||
}
|
||||
|
||||
// No user or role
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a role object to the storage.
|
||||
*
|
||||
@@ -300,6 +229,16 @@ class Auth final {
|
||||
std::vector<User> AllUsersForRole(const std::string &rolename) const;
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
/**
|
||||
* @brief Revoke access to individual database for a user.
|
||||
*
|
||||
* @param db name of the database to revoke
|
||||
* @param name user's username
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
bool RevokeDatabaseFromUser(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Grant access to individual database for a user.
|
||||
*
|
||||
@@ -308,33 +247,7 @@ class Auth final {
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
Result GrantDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
void GrantDatabase(const std::string &db, User &user, system::Transaction *system_tx = nullptr);
|
||||
void GrantDatabase(const std::string &db, Role &role, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Revoke access to individual database for a user.
|
||||
*
|
||||
* @param db name of the database to revoke
|
||||
* @param name user's username
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
Result DenyDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
void DenyDatabase(const std::string &db, User &user, system::Transaction *system_tx = nullptr);
|
||||
void DenyDatabase(const std::string &db, Role &role, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Revoke access to individual database for a user.
|
||||
*
|
||||
* @param db name of the database to revoke
|
||||
* @param name user's username
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
Result RevokeDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
void RevokeDatabase(const std::string &db, User &user, system::Transaction *system_tx = nullptr);
|
||||
void RevokeDatabase(const std::string &db, Role &role, system::Transaction *system_tx = nullptr);
|
||||
bool GrantDatabaseToUser(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Delete a database from all users.
|
||||
@@ -352,17 +265,9 @@ class Auth final {
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
Result SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
void SetMainDatabase(std::string_view db, User &user, system::Transaction *system_tx = nullptr);
|
||||
void SetMainDatabase(std::string_view db, Role &role, system::Transaction *system_tx = nullptr);
|
||||
bool SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
#endif
|
||||
|
||||
bool UpToDate(Epoch &e) const {
|
||||
bool res = e == epoch_;
|
||||
e = epoch_;
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief
|
||||
@@ -373,18 +278,11 @@ class Auth final {
|
||||
*/
|
||||
bool NameRegexMatch(const std::string &user_or_role) const;
|
||||
|
||||
void UpdateEpoch() { ++epoch_; }
|
||||
|
||||
void DisableIfModuleUsed() const {
|
||||
if (module_.IsUsed()) throw AuthException("Operation not permited when using an authentication module.");
|
||||
}
|
||||
|
||||
// Even though the `kvstore::KVStore` class is guaranteed to be thread-safe,
|
||||
// Auth is not thread-safe because modifying users and roles might require
|
||||
// more than one operation on the storage.
|
||||
kvstore::KVStore storage_;
|
||||
auth::Module module_;
|
||||
Config config_;
|
||||
Epoch epoch_{kStartEpoch};
|
||||
};
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -8,12 +8,10 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <json/json.hpp>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include <json/json.hpp>
|
||||
|
||||
namespace memgraph::auth {
|
||||
/// Need to be stable, auth durability depends on this
|
||||
enum class PasswordHashAlgorithm : uint8_t { BCRYPT = 0, SHA256 = 1, SHA256_MULTIPLE = 2 };
|
||||
|
||||
@@ -425,11 +425,10 @@ Role::Role(const std::string &rolename, const Permissions &permissions)
|
||||
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
|
||||
#ifdef MG_ENTERPRISE
|
||||
Role::Role(const std::string &rolename, const Permissions &permissions,
|
||||
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
|
||||
FineGrainedAccessHandler fine_grained_access_handler)
|
||||
: rolename_(utils::ToLowerCase(rolename)),
|
||||
permissions_(permissions),
|
||||
fine_grained_access_handler_(std::move(fine_grained_access_handler)),
|
||||
db_access_(std::move(db_access)) {}
|
||||
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
|
||||
#endif
|
||||
|
||||
const std::string &Role::rolename() const { return rolename_; }
|
||||
@@ -455,10 +454,8 @@ nlohmann::json Role::Serialize() const {
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
data[kFineGrainedAccessHandler] = fine_grained_access_handler_.Serialize();
|
||||
data[kDatabases] = db_access_.Serialize();
|
||||
} else {
|
||||
data[kFineGrainedAccessHandler] = {};
|
||||
data[kDatabases] = {};
|
||||
}
|
||||
#endif
|
||||
return data;
|
||||
@@ -474,21 +471,12 @@ Role Role::Deserialize(const nlohmann::json &data) {
|
||||
auto permissions = Permissions::Deserialize(data[kPermissions]);
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
Databases db_access;
|
||||
if (data[kDatabases].is_structured()) {
|
||||
db_access = Databases::Deserialize(data[kDatabases]);
|
||||
} else {
|
||||
// Back-compatibility
|
||||
spdlog::warn("Role without specified database access. Given access to the default database.");
|
||||
db_access.Grant(dbms::kDefaultDB);
|
||||
db_access.SetMain(dbms::kDefaultDB);
|
||||
}
|
||||
FineGrainedAccessHandler fine_grained_access_handler;
|
||||
// We can have an empty fine_grained if the user was created without a valid license
|
||||
if (data[kFineGrainedAccessHandler].is_object()) {
|
||||
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
|
||||
}
|
||||
return {data[kRoleName], permissions, std::move(fine_grained_access_handler), std::move(db_access)};
|
||||
return {data[kRoleName], permissions, std::move(fine_grained_access_handler)};
|
||||
}
|
||||
#endif
|
||||
return {data[kRoleName], permissions};
|
||||
@@ -505,7 +493,7 @@ bool operator==(const Role &first, const Role &second) {
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void Databases::Grant(std::string_view db) {
|
||||
void Databases::Add(std::string_view db) {
|
||||
if (allow_all_) {
|
||||
grants_dbs_.clear();
|
||||
allow_all_ = false;
|
||||
@@ -514,19 +502,19 @@ void Databases::Grant(std::string_view db) {
|
||||
denies_dbs_.erase(std::string{db}); // TODO: C++23 use transparent key compare
|
||||
}
|
||||
|
||||
void Databases::Deny(const std::string &db) {
|
||||
void Databases::Remove(const std::string &db) {
|
||||
denies_dbs_.emplace(db);
|
||||
grants_dbs_.erase(db);
|
||||
}
|
||||
|
||||
void Databases::Revoke(const std::string &db) {
|
||||
void Databases::Delete(const std::string &db) {
|
||||
denies_dbs_.erase(db);
|
||||
if (!allow_all_) {
|
||||
grants_dbs_.erase(db);
|
||||
}
|
||||
// Reset if default deleted
|
||||
if (main_db_ == db) {
|
||||
main_db_ = "";
|
||||
if (default_db_ == db) {
|
||||
default_db_ = "";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,16 +530,9 @@ void Databases::DenyAll() {
|
||||
denies_dbs_.clear();
|
||||
}
|
||||
|
||||
void Databases::RevokeAll() {
|
||||
allow_all_ = false;
|
||||
grants_dbs_.clear();
|
||||
denies_dbs_.clear();
|
||||
main_db_ = "";
|
||||
}
|
||||
|
||||
bool Databases::SetMain(std::string_view db) {
|
||||
bool Databases::SetDefault(std::string_view db) {
|
||||
if (!Contains(db)) return false;
|
||||
main_db_ = db;
|
||||
default_db_ = db;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -559,11 +540,11 @@ bool Databases::SetMain(std::string_view db) {
|
||||
return !denies_dbs_.contains(db) && (allow_all_ || grants_dbs_.contains(db));
|
||||
}
|
||||
|
||||
const std::string &Databases::GetMain() const {
|
||||
if (!Contains(main_db_)) {
|
||||
throw AuthException("No access to the set default database \"{}\".", main_db_);
|
||||
const std::string &Databases::GetDefault() const {
|
||||
if (!Contains(default_db_)) {
|
||||
throw AuthException("No access to the set default database \"{}\".", default_db_);
|
||||
}
|
||||
return main_db_;
|
||||
return default_db_;
|
||||
}
|
||||
|
||||
nlohmann::json Databases::Serialize() const {
|
||||
@@ -571,7 +552,7 @@ nlohmann::json Databases::Serialize() const {
|
||||
data[kGrants] = grants_dbs_;
|
||||
data[kDenies] = denies_dbs_;
|
||||
data[kAllowAll] = allow_all_;
|
||||
data[kDefault] = main_db_;
|
||||
data[kDefault] = default_db_;
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -738,16 +719,15 @@ User User::Deserialize(const nlohmann::json &data) {
|
||||
} else {
|
||||
// Back-compatibility
|
||||
spdlog::warn("User without specified database access. Given access to the default database.");
|
||||
db_access.Grant(dbms::kDefaultDB);
|
||||
db_access.SetMain(dbms::kDefaultDB);
|
||||
db_access.Add(dbms::kDefaultDB);
|
||||
db_access.SetDefault(dbms::kDefaultDB);
|
||||
}
|
||||
FineGrainedAccessHandler fine_grained_access_handler;
|
||||
// We can have an empty fine_grained if the user was created without a valid license
|
||||
if (data[kFineGrainedAccessHandler].is_object()) {
|
||||
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
|
||||
}
|
||||
return {data[kUsername], std::move(password_hash), permissions, std::move(fine_grained_access_handler),
|
||||
std::move(db_access)};
|
||||
return {data[kUsername], std::move(password_hash), permissions, std::move(fine_grained_access_handler), db_access};
|
||||
}
|
||||
#endif
|
||||
return {data[kUsername], std::move(password_hash), permissions};
|
||||
|
||||
@@ -205,96 +205,7 @@ class FineGrainedAccessHandler final {
|
||||
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
|
||||
#endif
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
class Databases final {
|
||||
public:
|
||||
Databases() : grants_dbs_{std::string{dbms::kDefaultDB}}, allow_all_(false), main_db_(dbms::kDefaultDB) {}
|
||||
|
||||
Databases(const Databases &) = default;
|
||||
Databases &operator=(const Databases &) = default;
|
||||
Databases(Databases &&) noexcept = default;
|
||||
Databases &operator=(Databases &&) noexcept = default;
|
||||
~Databases() = default;
|
||||
|
||||
/**
|
||||
* @brief Add database to the list of granted access. @note allow_all_ will be false after execution
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Grant(std::string_view db);
|
||||
|
||||
/**
|
||||
* @brief Remove database to the list of granted access.
|
||||
* @note if allow_all_ is set, the flag will remain set and the
|
||||
* database will be added to the set of denied databases.
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Deny(const std::string &db);
|
||||
|
||||
/**
|
||||
* @brief Called when database is dropped. Removes it from granted (if allow_all is false) and denied set.
|
||||
* @note allow_all_ is not changed
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Revoke(const std::string &db);
|
||||
|
||||
/**
|
||||
* @brief Set allow_all_ to true and clears grants and denied sets.
|
||||
*/
|
||||
void GrantAll();
|
||||
|
||||
/**
|
||||
* @brief Set allow_all_ to false and clears grants and denied sets.
|
||||
*/
|
||||
void DenyAll();
|
||||
|
||||
/**
|
||||
* @brief Set allow_all_ to false and clears grants and denied sets.
|
||||
*/
|
||||
void RevokeAll();
|
||||
|
||||
/**
|
||||
* @brief Set the default database.
|
||||
*/
|
||||
bool SetMain(std::string_view db);
|
||||
|
||||
/**
|
||||
* @brief Checks if access is grated to the database.
|
||||
*
|
||||
* @param db name of the database
|
||||
* @return true if allow_all and not denied or granted
|
||||
*/
|
||||
bool Contains(std::string_view db) const;
|
||||
bool Denies(std::string_view db_name) const { return denies_dbs_.contains(db_name); }
|
||||
bool Grants(std::string_view db_name) const { return allow_all_ || grants_dbs_.contains(db_name); }
|
||||
|
||||
bool GetAllowAll() const { return allow_all_; }
|
||||
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
|
||||
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
|
||||
const std::string &GetMain() const;
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
/// @throw AuthException if unable to deserialize.
|
||||
static Databases Deserialize(const nlohmann::json &data);
|
||||
|
||||
private:
|
||||
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
|
||||
std::string default_db = std::string{dbms::kDefaultDB})
|
||||
: grants_dbs_(std::move(grant)),
|
||||
denies_dbs_(std::move(deny)),
|
||||
allow_all_(allow_all),
|
||||
main_db_(std::move(default_db)) {}
|
||||
|
||||
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
|
||||
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
|
||||
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
|
||||
std::string main_db_; //!< user's default database
|
||||
};
|
||||
#endif
|
||||
|
||||
class Role {
|
||||
class Role final {
|
||||
public:
|
||||
Role() = default;
|
||||
|
||||
@@ -302,7 +213,7 @@ class Role {
|
||||
Role(const std::string &rolename, const Permissions &permissions);
|
||||
#ifdef MG_ENTERPRISE
|
||||
Role(const std::string &rolename, const Permissions &permissions,
|
||||
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
|
||||
FineGrainedAccessHandler fine_grained_access_handler);
|
||||
#endif
|
||||
Role(const Role &) = default;
|
||||
Role &operator=(const Role &) = default;
|
||||
@@ -313,23 +224,12 @@ class Role {
|
||||
const std::string &rolename() const;
|
||||
const Permissions &permissions() const;
|
||||
Permissions &permissions();
|
||||
Permissions GetPermissions() const { return permissions_; }
|
||||
#ifdef MG_ENTERPRISE
|
||||
const FineGrainedAccessHandler &fine_grained_access_handler() const;
|
||||
FineGrainedAccessHandler &fine_grained_access_handler();
|
||||
const FineGrainedAccessPermissions &GetFineGrainedAccessLabelPermissions() const;
|
||||
const FineGrainedAccessPermissions &GetFineGrainedAccessEdgeTypePermissions() const;
|
||||
#endif
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
Databases &db_access() { return db_access_; }
|
||||
const Databases &db_access() const { return db_access_; }
|
||||
|
||||
bool DeniesDB(std::string_view db_name) const { return db_access_.Denies(db_name); }
|
||||
bool GrantsDB(std::string_view db_name) const { return db_access_.Grants(db_name); }
|
||||
bool HasAccess(std::string_view db_name) const { return !DeniesDB(db_name) && GrantsDB(db_name); }
|
||||
#endif
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
|
||||
/// @throw AuthException if unable to deserialize.
|
||||
@@ -342,12 +242,93 @@ class Role {
|
||||
Permissions permissions_;
|
||||
#ifdef MG_ENTERPRISE
|
||||
FineGrainedAccessHandler fine_grained_access_handler_;
|
||||
Databases db_access_;
|
||||
#endif
|
||||
};
|
||||
|
||||
bool operator==(const Role &first, const Role &second);
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
class Databases final {
|
||||
public:
|
||||
Databases() : grants_dbs_{std::string{dbms::kDefaultDB}}, allow_all_(false), default_db_(dbms::kDefaultDB) {}
|
||||
|
||||
Databases(const Databases &) = default;
|
||||
Databases &operator=(const Databases &) = default;
|
||||
Databases(Databases &&) noexcept = default;
|
||||
Databases &operator=(Databases &&) noexcept = default;
|
||||
~Databases() = default;
|
||||
|
||||
/**
|
||||
* @brief Add database to the list of granted access. @note allow_all_ will be false after execution
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Add(std::string_view db);
|
||||
|
||||
/**
|
||||
* @brief Remove database to the list of granted access.
|
||||
* @note if allow_all_ is set, the flag will remain set and the
|
||||
* database will be added to the set of denied databases.
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Remove(const std::string &db);
|
||||
|
||||
/**
|
||||
* @brief Called when database is dropped. Removes it from granted (if allow_all is false) and denied set.
|
||||
* @note allow_all_ is not changed
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Delete(const std::string &db);
|
||||
|
||||
/**
|
||||
* @brief Set allow_all_ to true and clears grants and denied sets.
|
||||
*/
|
||||
void GrantAll();
|
||||
|
||||
/**
|
||||
* @brief Set allow_all_ to false and clears grants and denied sets.
|
||||
*/
|
||||
void DenyAll();
|
||||
|
||||
/**
|
||||
* @brief Set the default database.
|
||||
*/
|
||||
bool SetDefault(std::string_view db);
|
||||
|
||||
/**
|
||||
* @brief Checks if access is grated to the database.
|
||||
*
|
||||
* @param db name of the database
|
||||
* @return true if allow_all and not denied or granted
|
||||
*/
|
||||
bool Contains(std::string_view db) const;
|
||||
|
||||
bool GetAllowAll() const { return allow_all_; }
|
||||
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
|
||||
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
|
||||
const std::string &GetDefault() const;
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
/// @throw AuthException if unable to deserialize.
|
||||
static Databases Deserialize(const nlohmann::json &data);
|
||||
|
||||
private:
|
||||
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
|
||||
std::string default_db = std::string{dbms::kDefaultDB})
|
||||
: grants_dbs_(std::move(grant)),
|
||||
denies_dbs_(std::move(deny)),
|
||||
allow_all_(allow_all),
|
||||
default_db_(std::move(default_db)) {}
|
||||
|
||||
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
|
||||
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
|
||||
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
|
||||
std::string default_db_; //!< user's default database
|
||||
};
|
||||
#endif
|
||||
|
||||
// TODO (mferencevic): Implement password expiry.
|
||||
class User final {
|
||||
public:
|
||||
@@ -407,18 +388,6 @@ class User final {
|
||||
#ifdef MG_ENTERPRISE
|
||||
Databases &db_access() { return database_access_; }
|
||||
const Databases &db_access() const { return database_access_; }
|
||||
|
||||
bool DeniesDB(std::string_view db_name) const {
|
||||
bool denies = database_access_.Denies(db_name);
|
||||
if (role_) denies |= role_->DeniesDB(db_name);
|
||||
return denies;
|
||||
}
|
||||
bool GrantsDB(std::string_view db_name) const {
|
||||
bool grants = database_access_.Grants(db_name);
|
||||
if (role_) grants |= role_->GrantsDB(db_name);
|
||||
return grants;
|
||||
}
|
||||
bool HasAccess(std::string_view db_name) const { return !DeniesDB(db_name) && GrantsDB(db_name); }
|
||||
#endif
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
@@ -434,7 +403,7 @@ class User final {
|
||||
Permissions permissions_;
|
||||
#ifdef MG_ENTERPRISE
|
||||
FineGrainedAccessHandler fine_grained_access_handler_;
|
||||
Databases database_access_{};
|
||||
Databases database_access_;
|
||||
#endif
|
||||
std::optional<Role> role_;
|
||||
};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -403,7 +403,7 @@ nlohmann::json Module::Call(const nlohmann::json ¶ms, int timeout_millisec)
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool Module::IsUsed() const { return !module_executable_path_.empty(); }
|
||||
bool Module::IsUsed() { return !module_executable_path_.empty(); }
|
||||
|
||||
void Module::Shutdown() {
|
||||
if (pid_ == -1) return;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -49,7 +49,7 @@ class Module final {
|
||||
/// specified executable path and can thus be used.
|
||||
///
|
||||
/// @return boolean indicating whether the module can be used
|
||||
bool IsUsed() const;
|
||||
bool IsUsed();
|
||||
|
||||
~Module();
|
||||
|
||||
|
||||
@@ -17,15 +17,8 @@
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req) {
|
||||
spdlog::error(fmt::format("Received {} with main_id: {} != current_main_uuid: {}", rpc_req, std::string(main_req_id),
|
||||
current_main_uuid.has_value() ? std::string(current_main_uuid.value()) : ""));
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::UpdateAuthDataReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
@@ -33,12 +26,6 @@ void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
using memgraph::replication::UpdateAuthDataRes;
|
||||
UpdateAuthDataRes res(false);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::UpdateAuthDataReq::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
@@ -66,8 +53,7 @@ void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::DropAuthDataReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
@@ -75,12 +61,6 @@ void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_s
|
||||
using memgraph::replication::DropAuthDataRes;
|
||||
DropAuthDataRes res(false);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::DropAuthDataRes::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
@@ -175,14 +155,14 @@ void Register(replication::RoleReplicaData const &data, system::ReplicaHandlerAc
|
||||
auth::SynchedAuth &auth) {
|
||||
// NOTE: Register even without license as the user could add a license at run-time
|
||||
data.server->rpc_server_.Register<replication::UpdateAuthDataRpc>(
|
||||
[&data, system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received UpdateAuthDataRpc");
|
||||
UpdateAuthDataHandler(system_state_access, data.uuid_, auth, req_reader, res_builder);
|
||||
UpdateAuthDataHandler(system_state_access, auth, req_reader, res_builder);
|
||||
});
|
||||
data.server->rpc_server_.Register<replication::DropAuthDataRpc>(
|
||||
[&data, system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
[system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received DropAuthDataRpc");
|
||||
DropAuthDataHandler(system_state_access, data.uuid_, auth, req_reader, res_builder);
|
||||
DropAuthDataHandler(system_state_access, auth, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -17,16 +17,10 @@
|
||||
#include "system/state.hpp"
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req);
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void UpdateAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void UpdateAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
void DropAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
void DropAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
bool SystemRecoveryHandler(auth::SynchedAuth &auth, auth::Auth::Config auth_config,
|
||||
|
||||
@@ -18,9 +18,11 @@
|
||||
#include "utils/enum.hpp"
|
||||
|
||||
namespace memgraph::slk {
|
||||
// Serialize code for auth::Role
|
||||
void Save(const auth::Role &self, Builder *builder) { memgraph::slk::Save(self.Serialize().dump(), builder); }
|
||||
|
||||
// Serialize code for auth::Role
|
||||
void Save(const auth::Role &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.Serialize().dump(), builder);
|
||||
}
|
||||
namespace {
|
||||
auth::Role LoadAuthRole(memgraph::slk::Reader *reader) {
|
||||
std::string tmp;
|
||||
@@ -87,7 +89,6 @@ void Load(auth::Auth::Config *self, memgraph::slk::Reader *reader) {
|
||||
|
||||
// Serialize code for UpdateAuthDataReq
|
||||
void Save(const memgraph::replication::UpdateAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
memgraph::slk::Save(self.expected_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.new_group_timestamp, builder);
|
||||
@@ -95,7 +96,6 @@ void Save(const memgraph::replication::UpdateAuthDataReq &self, memgraph::slk::B
|
||||
memgraph::slk::Save(self.role, builder);
|
||||
}
|
||||
void Load(memgraph::replication::UpdateAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
memgraph::slk::Load(&self->expected_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->new_group_timestamp, reader);
|
||||
@@ -113,7 +113,6 @@ void Load(memgraph::replication::UpdateAuthDataRes *self, memgraph::slk::Reader
|
||||
|
||||
// Serialize code for DropAuthDataReq
|
||||
void Save(const memgraph::replication::DropAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
memgraph::slk::Save(self.expected_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.new_group_timestamp, builder);
|
||||
@@ -121,7 +120,6 @@ void Save(const memgraph::replication::DropAuthDataReq &self, memgraph::slk::Bui
|
||||
memgraph::slk::Save(self.name, builder);
|
||||
}
|
||||
void Load(memgraph::replication::DropAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
memgraph::slk::Load(&self->expected_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->new_group_timestamp, reader);
|
||||
|
||||
@@ -27,22 +27,17 @@ struct UpdateAuthDataReq {
|
||||
static void Load(UpdateAuthDataReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const UpdateAuthDataReq &self, memgraph::slk::Builder *builder);
|
||||
UpdateAuthDataReq() = default;
|
||||
UpdateAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
auth::User user)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
UpdateAuthDataReq(std::string epoch_id, uint64_t expected_ts, uint64_t new_ts, auth::User user)
|
||||
: epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
user{std::move(user)} {}
|
||||
UpdateAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
auth::Role role)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
UpdateAuthDataReq(std::string epoch_id, uint64_t expected_ts, uint64_t new_ts, auth::Role role)
|
||||
: epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
role{std::move(role)} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
@@ -74,16 +69,13 @@ struct DropAuthDataReq {
|
||||
|
||||
enum class DataType { USER, ROLE };
|
||||
|
||||
DropAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
DataType type, std::string_view name)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
DropAuthDataReq(std::string epoch_id, uint64_t expected_ts, uint64_t new_ts, DataType type, std::string_view name)
|
||||
: epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
type{type},
|
||||
name{name} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -15,9 +15,6 @@
|
||||
#include "communication/bolt/v1/value.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
#include "communication/bolt/v1/fmt.hpp"
|
||||
#include "io/network/fmt.hpp"
|
||||
|
||||
namespace {
|
||||
constexpr uint8_t kBoltV43Version[4] = {0x00, 0x00, 0x03, 0x04};
|
||||
constexpr uint8_t kEmptyBoltVersion[4] = {0x00, 0x00, 0x00, 0x00};
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#if FMT_VERSION > 90000
|
||||
#include <fmt/ostream.h>
|
||||
|
||||
#include "communication/bolt/v1/value.hpp"
|
||||
|
||||
template <>
|
||||
class fmt::formatter<memgraph::communication::bolt::Value> : public fmt::ostream_formatter {};
|
||||
|
||||
template <>
|
||||
class fmt::formatter<std::vector<memgraph::communication::bolt::Value>> : public fmt::ostream_formatter {};
|
||||
|
||||
template <>
|
||||
class fmt::formatter<std::map<std::string, memgraph::communication::bolt::Value>> : public fmt::ostream_formatter {};
|
||||
#endif
|
||||
@@ -1,20 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#if FMT_VERSION > 90000
|
||||
#include <fmt/ostream.h>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
|
||||
template <>
|
||||
class fmt::formatter<boost::asio::ip::tcp::endpoint> : public fmt::ostream_formatter {};
|
||||
#endif
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -21,7 +21,6 @@
|
||||
#include <boost/beast/core.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/fmt.hpp"
|
||||
#include "communication/http/session.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
@@ -83,7 +82,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TRequestHand
|
||||
return;
|
||||
}
|
||||
|
||||
spdlog::info("HTTP server is listening on {}", endpoint);
|
||||
spdlog::info("HTTP server is listening on {}:{}", endpoint.address(), endpoint.port());
|
||||
}
|
||||
|
||||
void DoAccept() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -23,7 +23,6 @@
|
||||
|
||||
#include "communication/session.hpp"
|
||||
#include "io/network/epoll.hpp"
|
||||
#include "io/network/fmt.hpp"
|
||||
#include "io/network/socket.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/signals.hpp"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -22,7 +22,6 @@
|
||||
|
||||
#include "communication/init.hpp"
|
||||
#include "communication/listener.hpp"
|
||||
#include "io/network/fmt.hpp"
|
||||
#include "io/network/socket.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/message.hpp"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -26,7 +26,6 @@
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/fmt.hpp"
|
||||
#include "communication/init.hpp"
|
||||
#include "communication/v2/listener.hpp"
|
||||
#include "communication/v2/pool.hpp"
|
||||
@@ -130,7 +129,7 @@ bool Server<TSession, TSessionContext>::Start() {
|
||||
listener_->Start();
|
||||
|
||||
spdlog::info("{} server is fully armed and operational", service_name_);
|
||||
spdlog::info("{} listening on {}", service_name_, endpoint_);
|
||||
spdlog::info("{} listening on {}", service_name_, endpoint_.address());
|
||||
context_thread_pool_.Run();
|
||||
|
||||
return true;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -47,7 +47,6 @@
|
||||
#include "communication/buffer.hpp"
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/exceptions.hpp"
|
||||
#include "communication/fmt.hpp"
|
||||
#include "dbms/global.hpp"
|
||||
#include "utils/event_counter.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -213,11 +212,14 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
|
||||
session_.Execute();
|
||||
DoRead();
|
||||
} catch (const SessionClosedException &e) {
|
||||
spdlog::info("{} client {} closed the connection.", service_name_, remote_endpoint_);
|
||||
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
DoClose();
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::error("Exception was thrown while processing event in {} session associated with {}", service_name_,
|
||||
remote_endpoint_);
|
||||
spdlog::error(
|
||||
"Exception was thrown while processing event in {} session "
|
||||
"associated with {}:{}",
|
||||
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
|
||||
spdlog::debug("Exception message: {}", e.what());
|
||||
DoClose();
|
||||
}
|
||||
@@ -374,7 +376,8 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
|
||||
socket.lowest_layer().non_blocking(false);
|
||||
});
|
||||
timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
|
||||
spdlog::info("Accepted a connection from {}: {}", service_name_, remote_endpoint_);
|
||||
spdlog::info("Accepted a connection from {}: {}:{}", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
}
|
||||
|
||||
void DoRead() {
|
||||
@@ -434,11 +437,14 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
|
||||
session_.Execute();
|
||||
DoRead();
|
||||
} catch (const SessionClosedException &e) {
|
||||
spdlog::info("{} client {} closed the connection.", service_name_, remote_endpoint_);
|
||||
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
DoShutdown();
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::error("Exception was thrown while processing event in {} session associated with {}", service_name_,
|
||||
remote_endpoint_);
|
||||
spdlog::error(
|
||||
"Exception was thrown while processing event in {} session "
|
||||
"associated with {}:{}",
|
||||
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
|
||||
spdlog::debug("Exception message: {}", e.what());
|
||||
DoShutdown();
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,44 +12,19 @@
|
||||
#include "communication/websocket/auth.hpp"
|
||||
|
||||
#include <string>
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
namespace memgraph::communication::websocket {
|
||||
|
||||
bool SafeAuth::Authenticate(const std::string &username, const std::string &password) const {
|
||||
user_or_role_ = auth_->Lock()->Authenticate(username, password);
|
||||
return user_or_role_.has_value();
|
||||
return auth_->Lock()->Authenticate(username, password).has_value();
|
||||
}
|
||||
|
||||
bool SafeAuth::HasPermission(const auth::Permission permission) const {
|
||||
auto locked_auth = auth_->ReadLock();
|
||||
// Update if cache invalidated
|
||||
if (!locked_auth->UpToDate(auth_epoch_) && user_or_role_) {
|
||||
bool success = true;
|
||||
std::visit(utils::Overloaded{[&](auth::User &user) {
|
||||
auto tmp = locked_auth->GetUser(user.username());
|
||||
if (!tmp) success = false;
|
||||
user = std::move(*tmp);
|
||||
},
|
||||
[&](auth::Role &role) {
|
||||
auto tmp = locked_auth->GetRole(role.rolename());
|
||||
if (!tmp) success = false;
|
||||
role = std::move(*tmp);
|
||||
}},
|
||||
*user_or_role_);
|
||||
// Missing user/role; delete from cache
|
||||
if (!success) user_or_role_.reset();
|
||||
bool SafeAuth::HasUserPermission(const std::string &username, const auth::Permission permission) const {
|
||||
if (const auto user = auth_->ReadLock()->GetUser(username); user) {
|
||||
return user->GetPermissions().Has(permission) == auth::PermissionLevel::GRANT;
|
||||
}
|
||||
// Check permissions
|
||||
if (user_or_role_) {
|
||||
return std::visit(utils::Overloaded{[&](auto &user_or_role) {
|
||||
return user_or_role.GetPermissions().Has(permission) == auth::PermissionLevel::GRANT;
|
||||
}},
|
||||
*user_or_role_);
|
||||
}
|
||||
// NOTE: websocket authenticates only if there is a user, so no need to check if access controlled
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SafeAuth::AccessControlled() const { return auth_->ReadLock()->AccessControlled(); }
|
||||
bool SafeAuth::HasAnyUsers() const { return auth_->ReadLock()->HasUsers(); }
|
||||
} // namespace memgraph::communication::websocket
|
||||
|
||||
@@ -21,9 +21,9 @@ class AuthenticationInterface {
|
||||
public:
|
||||
virtual bool Authenticate(const std::string &username, const std::string &password) const = 0;
|
||||
|
||||
virtual bool HasPermission(auth::Permission permission) const = 0;
|
||||
virtual bool HasUserPermission(const std::string &username, auth::Permission permission) const = 0;
|
||||
|
||||
virtual bool AccessControlled() const = 0;
|
||||
virtual bool HasAnyUsers() const = 0;
|
||||
};
|
||||
|
||||
class SafeAuth : public AuthenticationInterface {
|
||||
@@ -32,13 +32,11 @@ class SafeAuth : public AuthenticationInterface {
|
||||
|
||||
bool Authenticate(const std::string &username, const std::string &password) const override;
|
||||
|
||||
bool HasPermission(auth::Permission permission) const override;
|
||||
bool HasUserPermission(const std::string &username, auth::Permission permission) const override;
|
||||
|
||||
bool AccessControlled() const override;
|
||||
bool HasAnyUsers() const override;
|
||||
|
||||
private:
|
||||
auth::SynchedAuth *auth_;
|
||||
mutable std::optional<auth::UserOrRole> user_or_role_;
|
||||
mutable auth::Auth::Epoch auth_epoch_{};
|
||||
};
|
||||
} // namespace memgraph::communication::websocket
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -10,7 +10,6 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "communication/websocket/listener.hpp"
|
||||
#include "communication/fmt.hpp"
|
||||
|
||||
namespace memgraph::communication::websocket {
|
||||
namespace {
|
||||
@@ -62,7 +61,7 @@ Listener::Listener(boost::asio::io_context &ioc, ServerContext *context, tcp::en
|
||||
return;
|
||||
}
|
||||
|
||||
spdlog::info("WebSocket server is listening on {}", endpoint);
|
||||
spdlog::info("WebSocket server is listening on {}:{}", endpoint.address(), endpoint.port());
|
||||
}
|
||||
|
||||
void Listener::DoAccept() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -80,7 +80,7 @@ bool Session::Run() {
|
||||
return false;
|
||||
}
|
||||
|
||||
authenticated_ = !auth_.AccessControlled();
|
||||
authenticated_ = !auth_.HasAnyUsers();
|
||||
connected_.store(true, std::memory_order_relaxed);
|
||||
|
||||
// run on the strand
|
||||
@@ -162,7 +162,7 @@ utils::BasicResult<std::string> Session::Authorize(const nlohmann::json &creds)
|
||||
return {"Authentication failed!"};
|
||||
}
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (!auth_.HasPermission(auth::Permission::WEBSOCKET)) {
|
||||
if (!auth_.HasUserPermission(creds.at("username").get<std::string>(), auth::Permission::WEBSOCKET)) {
|
||||
return {"Authorization failed!"};
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -8,35 +8,21 @@ target_sources(mg-coordination
|
||||
include/coordination/coordinator_server.hpp
|
||||
include/coordination/coordinator_config.hpp
|
||||
include/coordination/coordinator_exceptions.hpp
|
||||
include/coordination/coordinator_slk.hpp
|
||||
include/coordination/coordinator_instance.hpp
|
||||
include/coordination/coordinator_slk.hpp
|
||||
include/coordination/coordinator_data.hpp
|
||||
include/coordination/constants.hpp
|
||||
include/coordination/coordinator_cluster_config.hpp
|
||||
include/coordination/coordinator_handlers.hpp
|
||||
include/coordination/instance_status.hpp
|
||||
include/coordination/replication_instance.hpp
|
||||
include/coordination/raft_state.hpp
|
||||
include/coordination/rpc_errors.hpp
|
||||
|
||||
include/nuraft/raft_log_action.hpp
|
||||
include/nuraft/coordinator_cluster_state.hpp
|
||||
include/nuraft/coordinator_log_store.hpp
|
||||
include/nuraft/coordinator_state_machine.hpp
|
||||
include/nuraft/coordinator_state_manager.hpp
|
||||
|
||||
PRIVATE
|
||||
coordinator_config.cpp
|
||||
coordinator_client.cpp
|
||||
coordinator_state.cpp
|
||||
coordinator_rpc.cpp
|
||||
coordinator_server.cpp
|
||||
coordinator_handlers.cpp
|
||||
coordinator_data.cpp
|
||||
coordinator_instance.cpp
|
||||
replication_instance.cpp
|
||||
raft_state.cpp
|
||||
|
||||
coordinator_log_store.cpp
|
||||
coordinator_state_machine.cpp
|
||||
coordinator_state_manager.cpp
|
||||
coordinator_cluster_state.cpp
|
||||
coordinator_handlers.cpp
|
||||
)
|
||||
target_include_directories(mg-coordination PUBLIC include)
|
||||
|
||||
|
||||
@@ -9,16 +9,13 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "replication_coordination_glue/messages.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
@@ -30,54 +27,35 @@ auto CreateClientContext(memgraph::coordination::CoordinatorClientConfig const &
|
||||
}
|
||||
} // namespace
|
||||
|
||||
CoordinatorClient::CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorClientConfig config,
|
||||
HealthCheckClientCallback succ_cb, HealthCheckClientCallback fail_cb)
|
||||
CoordinatorClient::CoordinatorClient(CoordinatorData *coord_data, CoordinatorClientConfig config,
|
||||
HealthCheckCallback succ_cb, HealthCheckCallback fail_cb)
|
||||
: rpc_context_{CreateClientContext(config)},
|
||||
rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port),
|
||||
&rpc_context_},
|
||||
config_{std::move(config)},
|
||||
coord_instance_{coord_instance},
|
||||
coord_data_{coord_data},
|
||||
succ_cb_{std::move(succ_cb)},
|
||||
fail_cb_{std::move(fail_cb)} {}
|
||||
|
||||
auto CoordinatorClient::InstanceName() const -> std::string { return config_.instance_name; }
|
||||
|
||||
auto CoordinatorClient::CoordinatorSocketAddress() const -> std::string { return config_.CoordinatorSocketAddress(); }
|
||||
auto CoordinatorClient::ReplicationSocketAddress() const -> std::string { return config_.ReplicationSocketAddress(); }
|
||||
|
||||
auto CoordinatorClient::InstanceDownTimeoutSec() const -> std::chrono::seconds {
|
||||
return config_.instance_down_timeout_sec;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::InstanceGetUUIDFrequencySec() const -> std::chrono::seconds {
|
||||
return config_.instance_get_uuid_frequency_sec;
|
||||
}
|
||||
auto CoordinatorClient::SocketAddress() const -> std::string { return rpc_client_.Endpoint().SocketAddress(); }
|
||||
|
||||
void CoordinatorClient::StartFrequentCheck() {
|
||||
if (instance_checker_.IsRunning()) {
|
||||
return;
|
||||
}
|
||||
|
||||
MG_ASSERT(config_.instance_health_check_frequency_sec > std::chrono::seconds(0),
|
||||
MG_ASSERT(config_.health_check_frequency_sec > std::chrono::seconds(0),
|
||||
"Health check frequency must be greater than 0");
|
||||
|
||||
instance_checker_.Run(
|
||||
config_.instance_name, config_.instance_health_check_frequency_sec,
|
||||
[this, instance_name = config_.instance_name] {
|
||||
config_.instance_name, config_.health_check_frequency_sec, [this, instance_name = config_.instance_name] {
|
||||
try {
|
||||
spdlog::trace("Sending frequent heartbeat to machine {} on {}", instance_name,
|
||||
config_.CoordinatorSocketAddress());
|
||||
rpc_client_.Endpoint().SocketAddress());
|
||||
{ // NOTE: This is intentionally scoped so that stream lock could get released.
|
||||
auto stream{rpc_client_.Stream<memgraph::replication_coordination_glue::FrequentHeartbeatRpc>()};
|
||||
stream.AwaitResponse();
|
||||
}
|
||||
// Subtle race condition:
|
||||
// acquiring of lock needs to happen before function call, as function callback can be changed
|
||||
// for instance after lock is already acquired
|
||||
// (failover case when instance is promoted to MAIN)
|
||||
succ_cb_(coord_instance_, instance_name);
|
||||
succ_cb_(coord_data_, instance_name);
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
fail_cb_(coord_instance_, instance_name);
|
||||
fail_cb_(coord_data_, instance_name);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -86,19 +64,23 @@ void CoordinatorClient::StopFrequentCheck() { instance_checker_.Stop(); }
|
||||
void CoordinatorClient::PauseFrequentCheck() { instance_checker_.Pause(); }
|
||||
void CoordinatorClient::ResumeFrequentCheck() { instance_checker_.Resume(); }
|
||||
|
||||
auto CoordinatorClient::SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCallback fail_cb) -> void {
|
||||
succ_cb_ = std::move(succ_cb);
|
||||
fail_cb_ = std::move(fail_cb);
|
||||
}
|
||||
|
||||
auto CoordinatorClient::ReplicationClientInfo() const -> ReplClientInfo { return config_.replication_client_info; }
|
||||
|
||||
auto CoordinatorClient::SendPromoteReplicaToMainRpc(const utils::UUID &uuid,
|
||||
ReplicationClientsInfo replication_clients_info) const -> bool {
|
||||
auto CoordinatorClient::SendPromoteReplicaToMainRpc(ReplicationClientsInfo replication_clients_info) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(uuid, std::move(replication_clients_info))};
|
||||
auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(std::move(replication_clients_info))};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful PromoteReplicaToMainRpc response!");
|
||||
spdlog::error("Failed to receive successful RPC failover response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("RPC error occurred while sending PromoteReplicaToMainRpc!");
|
||||
spdlog::error("RPC error occurred while sending failover RPC!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -119,71 +101,5 @@ auto CoordinatorClient::DemoteToReplica() const -> bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendSwapMainUUIDRpc(utils::UUID const &uuid) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<replication_coordination_glue::SwapMainUUIDRpc>(uuid)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC swapping of uuid response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occurred while sending swapping uuid RPC!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendUnregisterReplicaRpc(std::string_view instance_name) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<UnregisterReplicaRpc>(instance_name)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC response for unregistering replica!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("Failed to unregister replica!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendGetInstanceUUIDRpc() const
|
||||
-> utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>> {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<GetInstanceUUIDRpc>()};
|
||||
auto res = stream.AwaitResponse();
|
||||
return res.uuid;
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occured while sending GetInstance UUID RPC");
|
||||
return GetInstanceUUIDError::RPC_EXCEPTION;
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendEnableWritingOnMainRpc() const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<EnableWritingOnMainRpc>()};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC response for enabling writing on main!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("Failed to enable writing on main!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendGetInstanceTimestampsRpc() const
|
||||
-> utils::BasicResult<GetInstanceUUIDError, replication_coordination_glue::DatabaseHistories> {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<coordination::GetDatabaseHistoriesRpc>()};
|
||||
return stream.AwaitResponse().database_histories;
|
||||
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occured while sending GetInstance UUID RPC");
|
||||
return GetInstanceUUIDError::RPC_EXCEPTION;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_cluster_state.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
void to_json(nlohmann::json &j, InstanceState const &instance_state) {
|
||||
j = nlohmann::json{{"config", instance_state.config}, {"status", instance_state.status}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, InstanceState &instance_state) {
|
||||
j.at("config").get_to(instance_state.config);
|
||||
j.at("status").get_to(instance_state.status);
|
||||
}
|
||||
|
||||
CoordinatorClusterState::CoordinatorClusterState(std::map<std::string, InstanceState, std::less<>> instances)
|
||||
: instances_{std::move(instances)} {}
|
||||
|
||||
CoordinatorClusterState::CoordinatorClusterState(CoordinatorClusterState const &other) : instances_{other.instances_} {}
|
||||
|
||||
CoordinatorClusterState &CoordinatorClusterState::operator=(CoordinatorClusterState const &other) {
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
instances_ = other.instances_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
CoordinatorClusterState::CoordinatorClusterState(CoordinatorClusterState &&other) noexcept
|
||||
: instances_{std::move(other.instances_)} {}
|
||||
|
||||
CoordinatorClusterState &CoordinatorClusterState::operator=(CoordinatorClusterState &&other) noexcept {
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
instances_ = std::move(other.instances_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::MainExists() const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
return std::ranges::any_of(instances_,
|
||||
[](auto const &entry) { return entry.second.status == ReplicationRole::MAIN; });
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::IsMain(std::string_view instance_name) const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it = instances_.find(instance_name);
|
||||
return it != instances_.end() && it->second.status == ReplicationRole::MAIN;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::IsReplica(std::string_view instance_name) const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it = instances_.find(instance_name);
|
||||
return it != instances_.end() && it->second.status == ReplicationRole::REPLICA;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::InsertInstance(std::string instance_name, InstanceState instance_state) -> void {
|
||||
auto lock = std::lock_guard{log_lock_};
|
||||
instances_.insert_or_assign(std::move(instance_name), std::move(instance_state));
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::DoAction(TRaftLog log_entry, RaftLogAction log_action) -> void {
|
||||
auto lock = std::lock_guard{log_lock_};
|
||||
switch (log_action) {
|
||||
case RaftLogAction::REGISTER_REPLICATION_INSTANCE: {
|
||||
auto const &config = std::get<CoordinatorClientConfig>(log_entry);
|
||||
instances_[config.instance_name] = InstanceState{config, ReplicationRole::REPLICA};
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::UNREGISTER_REPLICATION_INSTANCE: {
|
||||
auto const instance_name = std::get<std::string>(log_entry);
|
||||
instances_.erase(instance_name);
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::SET_INSTANCE_AS_MAIN: {
|
||||
auto const instance_name = std::get<std::string>(log_entry);
|
||||
auto it = instances_.find(instance_name);
|
||||
MG_ASSERT(it != instances_.end(), "Instance does not exist as part of raft state!");
|
||||
it->second.status = ReplicationRole::MAIN;
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::SET_INSTANCE_AS_REPLICA: {
|
||||
auto const instance_name = std::get<std::string>(log_entry);
|
||||
auto it = instances_.find(instance_name);
|
||||
MG_ASSERT(it != instances_.end(), "Instance does not exist as part of raft state!");
|
||||
it->second.status = ReplicationRole::REPLICA;
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::UPDATE_UUID: {
|
||||
uuid_ = std::get<utils::UUID>(log_entry);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::Serialize(ptr<buffer> &data) -> void {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
|
||||
auto const log = nlohmann::json(instances_).dump();
|
||||
|
||||
data = buffer::alloc(sizeof(uint32_t) + log.size());
|
||||
buffer_serializer bs(data);
|
||||
bs.put_str(log);
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::Deserialize(buffer &data) -> CoordinatorClusterState {
|
||||
buffer_serializer bs(data);
|
||||
auto const j = nlohmann::json::parse(bs.get_str());
|
||||
auto instances = j.get<std::map<std::string, InstanceState, std::less<>>>();
|
||||
|
||||
return CoordinatorClusterState{std::move(instances)};
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::GetInstances() const -> std::vector<InstanceState> {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
return instances_ | ranges::views::values | ranges::to<std::vector<InstanceState>>;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::GetUUID() const -> utils::UUID { return uuid_; }
|
||||
|
||||
auto CoordinatorClusterState::FindCurrentMainInstanceName() const -> std::optional<std::string> {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it =
|
||||
std::ranges::find_if(instances_, [](auto const &entry) { return entry.second.status == ReplicationRole::MAIN; });
|
||||
if (it == instances_.end()) {
|
||||
return {};
|
||||
}
|
||||
return it->first;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,54 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
void to_json(nlohmann::json &j, ReplClientInfo const &config) {
|
||||
j = nlohmann::json{{"instance_name", config.instance_name},
|
||||
{"replication_mode", config.replication_mode},
|
||||
{"replication_ip_address", config.replication_ip_address},
|
||||
{"replication_port", config.replication_port}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, ReplClientInfo &config) {
|
||||
config.instance_name = j.at("instance_name").get<std::string>();
|
||||
config.replication_mode = j.at("replication_mode").get<replication_coordination_glue::ReplicationMode>();
|
||||
config.replication_ip_address = j.at("replication_ip_address").get<std::string>();
|
||||
config.replication_port = j.at("replication_port").get<uint16_t>();
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorClientConfig const &config) {
|
||||
j = nlohmann::json{{"instance_name", config.instance_name},
|
||||
{"ip_address", config.ip_address},
|
||||
{"port", config.port},
|
||||
{"instance_health_check_frequency_sec", config.instance_health_check_frequency_sec.count()},
|
||||
{"instance_down_timeout_sec", config.instance_down_timeout_sec.count()},
|
||||
{"instance_get_uuid_frequency_sec", config.instance_get_uuid_frequency_sec.count()},
|
||||
{"replication_client_info", config.replication_client_info}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, CoordinatorClientConfig &config) {
|
||||
config.instance_name = j.at("instance_name").get<std::string>();
|
||||
config.ip_address = j.at("ip_address").get<std::string>();
|
||||
config.port = j.at("port").get<uint16_t>();
|
||||
config.instance_health_check_frequency_sec =
|
||||
std::chrono::seconds{j.at("instance_health_check_frequency_sec").get<int>()};
|
||||
config.instance_down_timeout_sec = std::chrono::seconds{j.at("instance_down_timeout_sec").get<int>()};
|
||||
config.instance_get_uuid_frequency_sec = std::chrono::seconds{j.at("instance_get_uuid_frequency_sec").get<int>()};
|
||||
config.replication_client_info = j.at("replication_client_info").get<ReplClientInfo>();
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
199
src/coordination/coordinator_data.cpp
Normal file
199
src/coordination/coordinator_data.cpp
Normal file
@@ -0,0 +1,199 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_data.hpp"
|
||||
|
||||
#include <range/v3/view.hpp>
|
||||
#include <shared_mutex>
|
||||
#include "libnuraft/nuraft.hxx"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
CoordinatorData::CoordinatorData() {
|
||||
auto find_instance = [](CoordinatorData *coord_data, std::string_view instance_name) -> CoordinatorInstance & {
|
||||
auto instance = std::ranges::find_if(
|
||||
coord_data->registered_instances_,
|
||||
[instance_name](CoordinatorInstance const &instance) { return instance.InstanceName() == instance_name; });
|
||||
|
||||
MG_ASSERT(instance != coord_data->registered_instances_.end(), "Instance {} not found during callback!",
|
||||
instance_name);
|
||||
return *instance;
|
||||
};
|
||||
|
||||
replica_succ_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing replica successful callback", instance_name);
|
||||
find_instance(coord_data, instance_name).OnSuccessPing();
|
||||
};
|
||||
|
||||
replica_fail_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing replica failure callback", instance_name);
|
||||
find_instance(coord_data, instance_name).OnFailPing();
|
||||
};
|
||||
|
||||
main_succ_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing main successful callback", instance_name);
|
||||
|
||||
auto &instance = find_instance(coord_data, instance_name);
|
||||
|
||||
if (instance.IsAlive() || !coord_data->ClusterHasAliveMain_()) {
|
||||
instance.OnSuccessPing();
|
||||
return;
|
||||
}
|
||||
|
||||
bool const demoted = instance.DemoteToReplica(coord_data->replica_succ_cb_, coord_data->replica_fail_cb_);
|
||||
if (demoted) {
|
||||
instance.OnSuccessPing();
|
||||
spdlog::info("Instance {} demoted to replica", instance_name);
|
||||
} else {
|
||||
spdlog::error("Instance {} failed to become replica", instance_name);
|
||||
}
|
||||
};
|
||||
|
||||
main_fail_cb_ = [find_instance](CoordinatorData *coord_data, std::string_view instance_name) -> void {
|
||||
auto lock = std::lock_guard{coord_data->coord_data_lock_};
|
||||
spdlog::trace("Instance {} performing main failure callback", instance_name);
|
||||
find_instance(coord_data, instance_name).OnFailPing();
|
||||
|
||||
if (!coord_data->ClusterHasAliveMain_()) {
|
||||
spdlog::info("Cluster without main instance, trying automatic failover");
|
||||
coord_data->TryFailover();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
auto CoordinatorData::ClusterHasAliveMain_() const -> bool {
|
||||
auto const alive_main = [](CoordinatorInstance const &instance) { return instance.IsMain() && instance.IsAlive(); };
|
||||
return std::ranges::any_of(registered_instances_, alive_main);
|
||||
}
|
||||
|
||||
auto CoordinatorData::TryFailover() -> void {
|
||||
auto replica_instances = registered_instances_ | ranges::views::filter(&CoordinatorInstance::IsReplica);
|
||||
|
||||
auto chosen_replica_instance = std::ranges::find_if(replica_instances, &CoordinatorInstance::IsAlive);
|
||||
if (chosen_replica_instance == replica_instances.end()) {
|
||||
spdlog::warn("Failover failed since all replicas are down!");
|
||||
return;
|
||||
}
|
||||
|
||||
chosen_replica_instance->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&chosen_replica_instance] { chosen_replica_instance->ResumeFrequentCheck(); }};
|
||||
|
||||
std::vector<ReplClientInfo> repl_clients_info;
|
||||
repl_clients_info.reserve(std::ranges::distance(replica_instances));
|
||||
|
||||
auto const not_chosen_replica_instance = [&chosen_replica_instance](CoordinatorInstance const &instance) {
|
||||
return instance != *chosen_replica_instance;
|
||||
};
|
||||
|
||||
std::ranges::transform(registered_instances_ | ranges::views::filter(not_chosen_replica_instance),
|
||||
std::back_inserter(repl_clients_info),
|
||||
[](const CoordinatorInstance &instance) { return instance.ReplicationClientInfo(); });
|
||||
|
||||
if (!chosen_replica_instance->PromoteToMain(std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
spdlog::warn("Failover failed since promoting replica to main failed!");
|
||||
return;
|
||||
}
|
||||
spdlog::info("Failover successful! Instance {} promoted to main.", chosen_replica_instance->InstanceName());
|
||||
}
|
||||
|
||||
auto CoordinatorData::ShowInstances() const -> std::vector<CoordinatorInstanceStatus> {
|
||||
std::vector<CoordinatorInstanceStatus> instances_status;
|
||||
instances_status.reserve(registered_instances_.size());
|
||||
|
||||
auto const stringify_repl_role = [](CoordinatorInstance const &instance) -> std::string {
|
||||
if (!instance.IsAlive()) return "unknown";
|
||||
if (instance.IsMain()) return "main";
|
||||
return "replica";
|
||||
};
|
||||
|
||||
auto const instance_to_status =
|
||||
[&stringify_repl_role](CoordinatorInstance const &instance) -> CoordinatorInstanceStatus {
|
||||
return {.instance_name = instance.InstanceName(),
|
||||
.socket_address = instance.SocketAddress(),
|
||||
.replication_role = stringify_repl_role(instance),
|
||||
.is_alive = instance.IsAlive()};
|
||||
};
|
||||
|
||||
{
|
||||
auto lock = std::shared_lock{coord_data_lock_};
|
||||
std::ranges::transform(registered_instances_, std::back_inserter(instances_status), instance_to_status);
|
||||
}
|
||||
|
||||
return instances_status;
|
||||
}
|
||||
|
||||
auto CoordinatorData::SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_data_lock_};
|
||||
|
||||
auto const is_new_main = [&instance_name](CoordinatorInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
};
|
||||
auto new_main = std::ranges::find_if(registered_instances_, is_new_main);
|
||||
|
||||
if (new_main == registered_instances_.end()) {
|
||||
spdlog::error("Instance {} not registered. Please register it using REGISTER INSTANCE {}", instance_name,
|
||||
instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
ReplicationClientsInfo repl_clients_info;
|
||||
repl_clients_info.reserve(registered_instances_.size() - 1);
|
||||
|
||||
auto const is_not_new_main = [&instance_name](CoordinatorInstance const &instance) {
|
||||
return instance.InstanceName() != instance_name;
|
||||
};
|
||||
std::ranges::transform(registered_instances_ | ranges::views::filter(is_not_new_main),
|
||||
std::back_inserter(repl_clients_info),
|
||||
[](const CoordinatorInstance &instance) { return instance.ReplicationClientInfo(); });
|
||||
|
||||
if (!new_main->PromoteToMain(std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
|
||||
}
|
||||
|
||||
spdlog::info("Instance {} promoted to main", instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorData::RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_data_lock_};
|
||||
if (std::ranges::any_of(registered_instances_, [&config](CoordinatorInstance const &instance) {
|
||||
return instance.InstanceName() == config.instance_name;
|
||||
})) {
|
||||
return RegisterInstanceCoordinatorStatus::NAME_EXISTS;
|
||||
}
|
||||
|
||||
if (std::ranges::any_of(registered_instances_, [&config](CoordinatorInstance const &instance) {
|
||||
return instance.SocketAddress() == config.SocketAddress();
|
||||
})) {
|
||||
return RegisterInstanceCoordinatorStatus::ENDPOINT_EXISTS;
|
||||
}
|
||||
|
||||
try {
|
||||
registered_instances_.emplace_back(this, std::move(config), replica_succ_cb_, replica_fail_cb_);
|
||||
return RegisterInstanceCoordinatorStatus::SUCCESS;
|
||||
|
||||
} catch (CoordinatorRegisterInstanceException const &) {
|
||||
return RegisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -16,7 +16,6 @@
|
||||
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
#include "coordination/include/coordination/coordinator_server.hpp"
|
||||
#include "replication/state.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
@@ -33,64 +32,18 @@ void CoordinatorHandlers::Register(memgraph::coordination::CoordinatorServer &se
|
||||
spdlog::info("Received DemoteMainToReplicaRpc from coordinator server");
|
||||
CoordinatorHandlers::DemoteMainToReplicaHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<replication_coordination_glue::SwapMainUUIDRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received SwapMainUUIDRPC on coordinator server");
|
||||
CoordinatorHandlers::SwapMainUUIDHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::UnregisterReplicaRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received UnregisterReplicaRpc on coordinator server");
|
||||
CoordinatorHandlers::UnregisterReplicaHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::EnableWritingOnMainRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received EnableWritingOnMainRpc on coordinator server");
|
||||
CoordinatorHandlers::EnableWritingOnMainHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::GetInstanceUUIDRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received GetInstanceUUIDRpc on coordinator server");
|
||||
CoordinatorHandlers::GetInstanceUUIDHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::GetDatabaseHistoriesRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received GetDatabasesHistoryRpc on coordinator server");
|
||||
CoordinatorHandlers::GetDatabaseHistoriesHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::GetDatabaseHistoriesHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
|
||||
slk::Save(coordination::GetDatabaseHistoriesRes{replication_handler.GetDatabasesHistories()}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsReplica()) {
|
||||
spdlog::error("Setting main uuid must be performed on replica.");
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
replication_coordination_glue::SwapMainUUIDReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
spdlog::info(fmt::format("Set replica data UUID to main uuid {}", std::string(req.uuid)));
|
||||
std::get<memgraph::replication::RoleReplicaData>(replication_handler.GetReplState().ReplicationData()).uuid_ =
|
||||
req.uuid;
|
||||
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
spdlog::info("Executing DemoteMainToReplicaHandler");
|
||||
|
||||
if (!replication_handler.IsMain()) {
|
||||
spdlog::error("Setting to replica must be performed on main.");
|
||||
slk::Save(coordination::DemoteMainToReplicaRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
coordination::DemoteMainToReplicaReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
@@ -98,20 +51,13 @@ void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHan
|
||||
.ip_address = req.replication_client_info.replication_ip_address,
|
||||
.port = req.replication_client_info.replication_port};
|
||||
|
||||
if (!replication_handler.SetReplicationRoleReplica(clients_config, std::nullopt)) {
|
||||
if (!replication_handler.SetReplicationRoleReplica(clients_config)) {
|
||||
spdlog::error("Demoting main to replica failed!");
|
||||
slk::Save(coordination::DemoteMainToReplicaRes{false}, res_builder);
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
slk::Save(coordination::DemoteMainToReplicaRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::GetInstanceUUIDHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
|
||||
spdlog::info("Executing GetInstanceUUIDHandler");
|
||||
|
||||
slk::Save(coordination::GetInstanceUUIDRes{replication_handler.GetReplicaUUID()}, res_builder);
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHandler &replication_handler,
|
||||
@@ -121,17 +67,18 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
coordination::PromoteReplicaToMainReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
// This can fail because of disk. If it does, the cluster state could get inconsistent.
|
||||
// We don't handle disk issues.
|
||||
if (const bool success = replication_handler.DoReplicaToMainPromotion(req.main_uuid_); !success) {
|
||||
if (!replication_handler.DoReplicaToMainPromotion()) {
|
||||
spdlog::error("Promoting replica to main failed!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
coordination::PromoteReplicaToMainReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
auto const converter = [](const auto &repl_info_config) {
|
||||
return replication::ReplicationClientConfig{
|
||||
.name = repl_info_config.instance_name,
|
||||
@@ -162,68 +109,15 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
|
||||
spdlog::error("Registered replica could not be persisted!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
case memgraph::query::RegisterReplicaError::ERROR_ACCEPTING_MAIN:
|
||||
spdlog::error("Replica didn't accept change of main!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
case memgraph::query::RegisterReplicaError::CONNECTION_FAILED:
|
||||
// Connection failure is not a fatal error
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
spdlog::info("Promote replica to main was success {}", std::string(req.main_uuid_));
|
||||
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::UnregisterReplicaHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsMain()) {
|
||||
spdlog::error("Unregistering replica must be performed on main.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
coordination::UnregisterReplicaReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
auto res = replication_handler.UnregisterReplica(req.instance_name);
|
||||
switch (res) {
|
||||
using enum memgraph::query::UnregisterReplicaResult;
|
||||
case SUCCESS:
|
||||
slk::Save(coordination::UnregisterReplicaRes{true}, res_builder);
|
||||
break;
|
||||
case NOT_MAIN:
|
||||
spdlog::error("Unregistering replica must be performed on main.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
break;
|
||||
case CAN_NOT_UNREGISTER:
|
||||
spdlog::error("Could not unregister replica.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
break;
|
||||
case COULD_NOT_BE_PERSISTED:
|
||||
spdlog::error("Could not persist replica unregistration.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::EnableWritingOnMainHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsMain()) {
|
||||
spdlog::error("Enable writing on main must be performed on main!");
|
||||
slk::Save(coordination::EnableWritingOnMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!replication_handler.GetReplState().EnableWritingOnMain()) {
|
||||
spdlog::error("Enabling writing on main failed!");
|
||||
slk::Save(coordination::EnableWritingOnMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
slk::Save(coordination::EnableWritingOnMainRes{true}, res_builder);
|
||||
}
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
#endif
|
||||
|
||||
@@ -13,548 +13,71 @@
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "coordination/fmt.hpp"
|
||||
#include "dbms/constants.hpp"
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
#include "utils/counter.hpp"
|
||||
#include "utils/functional.hpp"
|
||||
#include "utils/resource_lock.hpp"
|
||||
|
||||
#include <range/v3/view.hpp>
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::ptr;
|
||||
using nuraft::srv_config;
|
||||
|
||||
CoordinatorInstance::CoordinatorInstance()
|
||||
: raft_state_(RaftState::MakeRaftState(
|
||||
[this]() {
|
||||
spdlog::info("Leader changed, starting all replication instances!");
|
||||
auto const instances = raft_state_.GetInstances();
|
||||
auto replicas = instances | ranges::views::filter([](auto const &instance) {
|
||||
return instance.status == ReplicationRole::REPLICA;
|
||||
});
|
||||
|
||||
std::ranges::for_each(replicas, [this](auto &replica) {
|
||||
spdlog::info("Started pinging replication instance {}", replica.config.instance_name);
|
||||
repl_instances_.emplace_back(this, replica.config, client_succ_cb_, client_fail_cb_,
|
||||
&CoordinatorInstance::ReplicaSuccessCallback,
|
||||
&CoordinatorInstance::ReplicaFailCallback);
|
||||
});
|
||||
|
||||
auto main = instances | ranges::views::filter(
|
||||
[](auto const &instance) { return instance.status == ReplicationRole::MAIN; });
|
||||
|
||||
std::ranges::for_each(main, [this](auto &main_instance) {
|
||||
spdlog::info("Started pinging main instance {}", main_instance.config.instance_name);
|
||||
repl_instances_.emplace_back(this, main_instance.config, client_succ_cb_, client_fail_cb_,
|
||||
&CoordinatorInstance::MainSuccessCallback,
|
||||
&CoordinatorInstance::MainFailCallback);
|
||||
});
|
||||
|
||||
std::ranges::for_each(repl_instances_, [this](auto &instance) {
|
||||
instance.SetNewMainUUID(raft_state_.GetUUID());
|
||||
instance.StartFrequentCheck();
|
||||
});
|
||||
},
|
||||
[this]() {
|
||||
spdlog::info("Leader changed, stopping all replication instances!");
|
||||
repl_instances_.clear();
|
||||
})) {
|
||||
client_succ_cb_ = [](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::lock_guard{self->coord_instance_lock_};
|
||||
auto &repl_instance = self->FindReplicationInstance(repl_instance_name);
|
||||
std::invoke(repl_instance.GetSuccessCallback(), self, repl_instance_name);
|
||||
};
|
||||
|
||||
client_fail_cb_ = [](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::lock_guard{self->coord_instance_lock_};
|
||||
auto &repl_instance = self->FindReplicationInstance(repl_instance_name);
|
||||
std::invoke(repl_instance.GetFailCallback(), self, repl_instance_name);
|
||||
};
|
||||
CoordinatorInstance::CoordinatorInstance(CoordinatorData *data, CoordinatorClientConfig config,
|
||||
HealthCheckCallback succ_cb, HealthCheckCallback fail_cb)
|
||||
: client_(data, std::move(config), std::move(succ_cb), std::move(fail_cb)),
|
||||
replication_role_(replication_coordination_glue::ReplicationRole::REPLICA),
|
||||
is_alive_(true) {
|
||||
if (!client_.DemoteToReplica()) {
|
||||
throw CoordinatorRegisterInstanceException("Failed to demote instance {} to replica", client_.InstanceName());
|
||||
}
|
||||
client_.StartFrequentCheck();
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::FindReplicationInstance(std::string_view replication_instance_name) -> ReplicationInstance & {
|
||||
auto repl_instance =
|
||||
std::ranges::find_if(repl_instances_, [replication_instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == replication_instance_name;
|
||||
});
|
||||
|
||||
MG_ASSERT(repl_instance != repl_instances_.end(), "Instance {} not found during callback!",
|
||||
replication_instance_name);
|
||||
return *repl_instance;
|
||||
auto CoordinatorInstance::OnSuccessPing() -> void {
|
||||
last_response_time_ = std::chrono::system_clock::now();
|
||||
is_alive_ = true;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::ShowInstances() const -> std::vector<InstanceStatus> {
|
||||
auto const coord_instance_to_status = [](ptr<srv_config> const &instance) -> InstanceStatus {
|
||||
return {.instance_name = "coordinator_" + std::to_string(instance->get_id()),
|
||||
.raft_socket_address = instance->get_endpoint(),
|
||||
.cluster_role = "coordinator",
|
||||
.health = "unknown"}; // TODO: (andi) Get this info from RAFT and test it or when we will move
|
||||
};
|
||||
auto instances_status = utils::fmap(raft_state_.GetAllCoordinators(), coord_instance_to_status);
|
||||
|
||||
if (raft_state_.IsLeader()) {
|
||||
auto const stringify_repl_role = [this](ReplicationInstance const &instance) -> std::string {
|
||||
if (!instance.IsAlive()) return "unknown";
|
||||
if (raft_state_.IsMain(instance.InstanceName())) return "main";
|
||||
return "replica";
|
||||
};
|
||||
|
||||
auto const stringify_repl_health = [](ReplicationInstance const &instance) -> std::string {
|
||||
return instance.IsAlive() ? "up" : "down";
|
||||
};
|
||||
|
||||
auto process_repl_instance_as_leader =
|
||||
[&stringify_repl_role, &stringify_repl_health](ReplicationInstance const &instance) -> InstanceStatus {
|
||||
return {.instance_name = instance.InstanceName(),
|
||||
.coord_socket_address = instance.CoordinatorSocketAddress(),
|
||||
.cluster_role = stringify_repl_role(instance),
|
||||
.health = stringify_repl_health(instance)};
|
||||
};
|
||||
|
||||
{
|
||||
auto lock = std::shared_lock{coord_instance_lock_};
|
||||
std::ranges::transform(repl_instances_, std::back_inserter(instances_status), process_repl_instance_as_leader);
|
||||
}
|
||||
} else {
|
||||
auto const stringify_inst_status = [](ReplicationRole status) -> std::string {
|
||||
return status == ReplicationRole::MAIN ? "main" : "replica";
|
||||
};
|
||||
|
||||
// TODO: (andi) Add capability that followers can also return socket addresses
|
||||
auto process_repl_instance_as_follower = [&stringify_inst_status](auto const &instance) -> InstanceStatus {
|
||||
return {.instance_name = instance.config.instance_name,
|
||||
.cluster_role = stringify_inst_status(instance.status),
|
||||
.health = "unknown"};
|
||||
};
|
||||
|
||||
std::ranges::transform(raft_state_.GetInstances(), std::back_inserter(instances_status),
|
||||
process_repl_instance_as_follower);
|
||||
}
|
||||
|
||||
return instances_status;
|
||||
auto CoordinatorInstance::OnFailPing() -> bool {
|
||||
is_alive_ =
|
||||
std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - last_response_time_).count() <
|
||||
CoordinatorClusterConfig::alive_response_time_difference_sec_;
|
||||
return is_alive_;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::TryFailover() -> void {
|
||||
auto const is_replica = [this](ReplicationInstance const &instance) { return IsReplica(instance.InstanceName()); };
|
||||
auto CoordinatorInstance::InstanceName() const -> std::string { return client_.InstanceName(); }
|
||||
auto CoordinatorInstance::SocketAddress() const -> std::string { return client_.SocketAddress(); }
|
||||
auto CoordinatorInstance::IsAlive() const -> bool { return is_alive_; }
|
||||
|
||||
auto alive_replicas =
|
||||
repl_instances_ | ranges::views::filter(is_replica) | ranges::views::filter(&ReplicationInstance::IsAlive);
|
||||
|
||||
if (ranges::empty(alive_replicas)) {
|
||||
spdlog::warn("Failover failed since all replicas are down!");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
spdlog::error("Failover failed since the instance is not the leader!");
|
||||
return;
|
||||
}
|
||||
|
||||
auto const get_ts = [](ReplicationInstance &replica) { return replica.GetClient().SendGetInstanceTimestampsRpc(); };
|
||||
|
||||
auto maybe_instance_db_histories = alive_replicas | ranges::views::transform(get_ts) | ranges::to<std::vector>();
|
||||
|
||||
auto const ts_has_error = [](auto const &res) -> bool { return res.HasError(); };
|
||||
|
||||
if (std::ranges::any_of(maybe_instance_db_histories, ts_has_error)) {
|
||||
spdlog::error("Aborting failover as at least one instance didn't provide per database history.");
|
||||
return;
|
||||
}
|
||||
|
||||
auto transform_to_pairs = ranges::views::transform([](auto const &zipped) {
|
||||
auto &[replica, res] = zipped;
|
||||
return std::make_pair(replica.InstanceName(), res.GetValue());
|
||||
});
|
||||
|
||||
auto instance_db_histories =
|
||||
ranges::views::zip(alive_replicas, maybe_instance_db_histories) | transform_to_pairs | ranges::to<std::vector>();
|
||||
|
||||
auto [most_up_to_date_instance, latest_epoch, latest_commit_timestamp] =
|
||||
ChooseMostUpToDateInstance(instance_db_histories);
|
||||
|
||||
spdlog::trace("The most up to date instance is {} with epoch {} and {} latest commit timestamp",
|
||||
most_up_to_date_instance, latest_epoch, latest_commit_timestamp); // NOLINT
|
||||
|
||||
auto *new_main = &FindReplicationInstance(most_up_to_date_instance);
|
||||
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
auto const is_not_new_main = [&new_main](ReplicationInstance &instance) {
|
||||
return instance.InstanceName() != new_main->InstanceName();
|
||||
};
|
||||
|
||||
auto const new_main_uuid = utils::UUID{};
|
||||
|
||||
auto const failed_to_swap = [&new_main_uuid](ReplicationInstance &instance) {
|
||||
return !instance.SendSwapAndUpdateUUID(new_main_uuid);
|
||||
};
|
||||
|
||||
// If for some replicas swap fails, for others on successful ping we will revert back on next change
|
||||
// or we will do failover first again and then it will be consistent again
|
||||
if (std::ranges::any_of(alive_replicas | ranges::views::filter(is_not_new_main), failed_to_swap)) {
|
||||
spdlog::error("Failed to swap uuid for all instances");
|
||||
return;
|
||||
}
|
||||
auto repl_clients_info = repl_instances_ | ranges::views::filter(is_not_new_main) |
|
||||
ranges::views::transform(&ReplicationInstance::ReplicationClientInfo) |
|
||||
ranges::to<ReplicationClientsInfo>();
|
||||
|
||||
if (!new_main->PromoteToMain(new_main_uuid, std::move(repl_clients_info), &CoordinatorInstance::MainSuccessCallback,
|
||||
&CoordinatorInstance::MainFailCallback)) {
|
||||
spdlog::warn("Failover failed since promoting replica to main failed!");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendUpdateUUIDLog(new_main_uuid)) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto const new_main_instance_name = new_main->InstanceName();
|
||||
|
||||
if (!raft_state_.AppendSetInstanceAsMainLog(new_main_instance_name)) {
|
||||
return;
|
||||
}
|
||||
|
||||
spdlog::info("Failover successful! Instance {} promoted to main.", new_main->InstanceName());
|
||||
auto CoordinatorInstance::IsReplica() const -> bool {
|
||||
return replication_role_ == replication_coordination_glue::ReplicationRole::REPLICA;
|
||||
}
|
||||
auto CoordinatorInstance::IsMain() const -> bool {
|
||||
return replication_role_ == replication_coordination_glue::ReplicationRole::MAIN;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::SetReplicationInstanceToMain(std::string_view instance_name)
|
||||
-> SetInstanceToMainCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
|
||||
if (raft_state_.MainExists()) {
|
||||
return SetInstanceToMainCoordinatorStatus::MAIN_ALREADY_EXISTS;
|
||||
auto CoordinatorInstance::PromoteToMain(ReplicationClientsInfo repl_clients_info, HealthCheckCallback main_succ_cb,
|
||||
HealthCheckCallback main_fail_cb) -> bool {
|
||||
if (!client_.SendPromoteReplicaToMainRpc(std::move(repl_clients_info))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return SetInstanceToMainCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
replication_role_ = replication_coordination_glue::ReplicationRole::MAIN;
|
||||
client_.SetCallbacks(std::move(main_succ_cb), std::move(main_fail_cb));
|
||||
|
||||
auto const is_new_main = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
};
|
||||
|
||||
auto new_main = std::ranges::find_if(repl_instances_, is_new_main);
|
||||
|
||||
if (new_main == repl_instances_.end()) {
|
||||
spdlog::error("Instance {} not registered. Please register it using REGISTER INSTANCE {}", instance_name,
|
||||
instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
auto const is_not_new_main = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() != instance_name;
|
||||
};
|
||||
|
||||
auto const new_main_uuid = utils::UUID{};
|
||||
|
||||
auto const failed_to_swap = [&new_main_uuid](ReplicationInstance &instance) {
|
||||
return !instance.SendSwapAndUpdateUUID(new_main_uuid);
|
||||
};
|
||||
|
||||
if (std::ranges::any_of(repl_instances_ | ranges::views::filter(is_not_new_main), failed_to_swap)) {
|
||||
spdlog::error("Failed to swap uuid for all instances");
|
||||
return SetInstanceToMainCoordinatorStatus::SWAP_UUID_FAILED;
|
||||
}
|
||||
|
||||
auto repl_clients_info = repl_instances_ | ranges::views::filter(is_not_new_main) |
|
||||
ranges::views::transform(&ReplicationInstance::ReplicationClientInfo) |
|
||||
ranges::to<ReplicationClientsInfo>();
|
||||
|
||||
if (!new_main->PromoteToMain(new_main_uuid, std::move(repl_clients_info), &CoordinatorInstance::MainSuccessCallback,
|
||||
&CoordinatorInstance::MainFailCallback)) {
|
||||
return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendUpdateUUIDLog(new_main_uuid)) {
|
||||
return SetInstanceToMainCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendSetInstanceAsMainLog(instance_name)) {
|
||||
return SetInstanceToMainCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
spdlog::info("Instance {} promoted to main on leader", instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::SUCCESS;
|
||||
return true;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, [instance_name = config.instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
})) {
|
||||
return RegisterInstanceCoordinatorStatus::NAME_EXISTS;
|
||||
auto CoordinatorInstance::DemoteToReplica(HealthCheckCallback replica_succ_cb, HealthCheckCallback replica_fail_cb)
|
||||
-> bool {
|
||||
if (!client_.DemoteToReplica()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, [&config](ReplicationInstance const &instance) {
|
||||
return instance.CoordinatorSocketAddress() == config.CoordinatorSocketAddress();
|
||||
})) {
|
||||
return RegisterInstanceCoordinatorStatus::COORD_ENDPOINT_EXISTS;
|
||||
}
|
||||
replication_role_ = replication_coordination_glue::ReplicationRole::REPLICA;
|
||||
client_.SetCallbacks(std::move(replica_succ_cb), std::move(replica_fail_cb));
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, [&config](ReplicationInstance const &instance) {
|
||||
return instance.ReplicationSocketAddress() == config.ReplicationSocketAddress();
|
||||
})) {
|
||||
return RegisterInstanceCoordinatorStatus::REPL_ENDPOINT_EXISTS;
|
||||
}
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return RegisterInstanceCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
|
||||
auto const undo_action_ = [this]() { repl_instances_.pop_back(); };
|
||||
|
||||
auto *new_instance = &repl_instances_.emplace_back(this, config, client_succ_cb_, client_fail_cb_,
|
||||
&CoordinatorInstance::ReplicaSuccessCallback,
|
||||
&CoordinatorInstance::ReplicaFailCallback);
|
||||
|
||||
if (!new_instance->SendDemoteToReplicaRpc()) {
|
||||
spdlog::error("Failed to send demote to replica rpc for instance {}", config.instance_name);
|
||||
undo_action_();
|
||||
return RegisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendRegisterReplicationInstanceLog(config)) {
|
||||
undo_action_();
|
||||
return RegisterInstanceCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
new_instance->StartFrequentCheck();
|
||||
|
||||
spdlog::info("Instance {} registered", config.instance_name);
|
||||
return RegisterInstanceCoordinatorStatus::SUCCESS;
|
||||
return true;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::UnregisterReplicationInstance(std::string_view instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
auto CoordinatorInstance::PauseFrequentCheck() -> void { client_.PauseFrequentCheck(); }
|
||||
auto CoordinatorInstance::ResumeFrequentCheck() -> void { client_.ResumeFrequentCheck(); }
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return UnregisterInstanceCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
|
||||
auto const name_matches = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
};
|
||||
|
||||
auto inst_to_remove = std::ranges::find_if(repl_instances_, name_matches);
|
||||
if (inst_to_remove == repl_instances_.end()) {
|
||||
return UnregisterInstanceCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
auto const is_main = [this](ReplicationInstance const &instance) {
|
||||
return IsMain(instance.InstanceName()) && instance.GetMainUUID() == raft_state_.GetUUID() && instance.IsAlive();
|
||||
};
|
||||
|
||||
if (is_main(*inst_to_remove)) {
|
||||
return UnregisterInstanceCoordinatorStatus::IS_MAIN;
|
||||
}
|
||||
|
||||
inst_to_remove->StopFrequentCheck();
|
||||
|
||||
auto curr_main = std::ranges::find_if(repl_instances_, is_main);
|
||||
|
||||
if (curr_main != repl_instances_.end() && curr_main->IsAlive()) {
|
||||
if (!curr_main->SendUnregisterReplicaRpc(instance_name)) {
|
||||
inst_to_remove->StartFrequentCheck();
|
||||
return UnregisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
std::erase_if(repl_instances_, name_matches);
|
||||
|
||||
if (!raft_state_.AppendUnregisterReplicationInstanceLog(instance_name)) {
|
||||
return UnregisterInstanceCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
return UnregisterInstanceCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port,
|
||||
std::string_view raft_address) -> void {
|
||||
raft_state_.AddCoordinatorInstance(raft_server_id, raft_port, raft_address);
|
||||
}
|
||||
|
||||
void CoordinatorInstance::MainFailCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing main fail callback", repl_instance_name);
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
repl_instance.OnFailPing();
|
||||
const auto &repl_instance_uuid = repl_instance.GetMainUUID();
|
||||
MG_ASSERT(repl_instance_uuid.has_value(), "Replication instance must have uuid set");
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
if (!repl_instance.IsAlive() && raft_state_.GetUUID() == repl_instance_uuid.value()) {
|
||||
spdlog::info("Cluster without main instance, trying automatic failover");
|
||||
TryFailover();
|
||||
}
|
||||
}
|
||||
|
||||
void CoordinatorInstance::MainSuccessCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing main successful callback", repl_instance_name);
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
if (repl_instance.IsAlive()) {
|
||||
repl_instance.OnSuccessPing();
|
||||
return;
|
||||
}
|
||||
|
||||
const auto &repl_instance_uuid = repl_instance.GetMainUUID();
|
||||
MG_ASSERT(repl_instance_uuid.has_value(), "Instance must have uuid set.");
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
if (raft_state_.GetUUID() == repl_instance_uuid.value()) {
|
||||
if (!repl_instance.EnableWritingOnMain()) {
|
||||
spdlog::error("Failed to enable writing on main instance {}", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
repl_instance.OnSuccessPing();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
spdlog::error("Demoting main instance {} to replica failed since the instance is not the leader!",
|
||||
repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
if (repl_instance.DemoteToReplica(&CoordinatorInstance::ReplicaSuccessCallback,
|
||||
&CoordinatorInstance::ReplicaFailCallback)) {
|
||||
repl_instance.OnSuccessPing();
|
||||
spdlog::info("Instance {} demoted to replica", repl_instance_name);
|
||||
} else {
|
||||
spdlog::error("Instance {} failed to become replica", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!repl_instance.SendSwapAndUpdateUUID(raft_state_.GetUUID())) {
|
||||
spdlog::error("Failed to swap uuid for demoted main instance {}", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendSetInstanceAsReplicaLog(repl_instance_name)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void CoordinatorInstance::ReplicaSuccessCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing replica successful callback", repl_instance_name);
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
if (!IsReplica(repl_instance_name)) {
|
||||
spdlog::error("Aborting replica callback since instance {} is not replica anymore", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
// We need to get replicas UUID from time to time to ensure replica is listening to correct main
|
||||
// and that it didn't go down for less time than we could notice
|
||||
// We need to get id of main replica is listening to
|
||||
// and swap if necessary
|
||||
if (!repl_instance.EnsureReplicaHasCorrectMainUUID(raft_state_.GetUUID())) {
|
||||
spdlog::error("Failed to swap uuid for replica instance {} which is alive", repl_instance.InstanceName());
|
||||
return;
|
||||
}
|
||||
|
||||
repl_instance.OnSuccessPing();
|
||||
}
|
||||
|
||||
void CoordinatorInstance::ReplicaFailCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing replica failure callback", repl_instance_name);
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
if (!IsReplica(repl_instance_name)) {
|
||||
spdlog::error("Aborting replica fail callback since instance {} is not replica anymore", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
repl_instance.OnFailPing();
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::ChooseMostUpToDateInstance(std::span<InstanceNameDbHistories> instance_database_histories)
|
||||
-> NewMainRes {
|
||||
std::optional<NewMainRes> new_main_res;
|
||||
std::for_each(
|
||||
instance_database_histories.begin(), instance_database_histories.end(),
|
||||
[&new_main_res](const InstanceNameDbHistories &instance_res_pair) {
|
||||
const auto &[instance_name, instance_db_histories] = instance_res_pair;
|
||||
|
||||
// Find default db for instance and its history
|
||||
auto default_db_history_data = std::ranges::find_if(
|
||||
instance_db_histories, [default_db = memgraph::dbms::kDefaultDB](
|
||||
const replication_coordination_glue::DatabaseHistory &db_timestamps) {
|
||||
return db_timestamps.name == default_db;
|
||||
});
|
||||
|
||||
std::ranges::for_each(
|
||||
instance_db_histories,
|
||||
[&instance_name = instance_name](const replication_coordination_glue::DatabaseHistory &db_history) {
|
||||
spdlog::debug("Instance {}: name {}, default db {}", instance_name, db_history.name,
|
||||
memgraph::dbms::kDefaultDB);
|
||||
});
|
||||
|
||||
MG_ASSERT(default_db_history_data != instance_db_histories.end(), "No history for instance");
|
||||
|
||||
const auto &instance_default_db_history = default_db_history_data->history;
|
||||
|
||||
std::ranges::for_each(instance_default_db_history | ranges::views::reverse,
|
||||
[&instance_name = instance_name](const auto &epoch_history_it) {
|
||||
spdlog::debug("Instance {}: epoch {}, last_commit_timestamp: {}", instance_name,
|
||||
std::get<0>(epoch_history_it), std::get<1>(epoch_history_it));
|
||||
});
|
||||
|
||||
// get latest epoch
|
||||
// get latest timestamp
|
||||
|
||||
if (!new_main_res) {
|
||||
const auto &[epoch, timestamp] = *instance_default_db_history.crbegin();
|
||||
new_main_res = std::make_optional<NewMainRes>({instance_name, epoch, timestamp});
|
||||
spdlog::debug("Currently the most up to date instance is {} with epoch {} and {} latest commit timestamp",
|
||||
instance_name, epoch, timestamp);
|
||||
return;
|
||||
}
|
||||
|
||||
bool found_same_point{false};
|
||||
std::string last_most_up_to_date_epoch{new_main_res->latest_epoch};
|
||||
for (auto [epoch, timestamp] : ranges::reverse_view(instance_default_db_history)) {
|
||||
if (new_main_res->latest_commit_timestamp < timestamp) {
|
||||
new_main_res = std::make_optional<NewMainRes>({instance_name, epoch, timestamp});
|
||||
spdlog::trace("Found the new most up to date instance {} with epoch {} and {} latest commit timestamp",
|
||||
instance_name, epoch, timestamp);
|
||||
}
|
||||
|
||||
// we found point at which they were same
|
||||
if (epoch == last_most_up_to_date_epoch) {
|
||||
found_same_point = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found_same_point) {
|
||||
spdlog::error("Didn't find same history epoch {} for instance {} and instance {}", last_most_up_to_date_epoch,
|
||||
new_main_res->most_up_to_date_instance, instance_name);
|
||||
}
|
||||
});
|
||||
|
||||
return std::move(*new_main_res);
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::IsMain(std::string_view instance_name) const -> bool {
|
||||
return raft_state_.IsMain(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::IsReplica(std::string_view instance_name) const -> bool {
|
||||
return raft_state_.IsReplica(instance_name);
|
||||
auto CoordinatorInstance::ReplicationClientInfo() const -> CoordinatorClientConfig::ReplicationClientInfo {
|
||||
return client_.ReplicationClientInfo();
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -1,193 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_log_store.hpp"
|
||||
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::cs_new;
|
||||
|
||||
namespace {
|
||||
|
||||
ptr<log_entry> MakeClone(const ptr<log_entry> &entry) {
|
||||
return cs_new<log_entry>(entry->get_term(), buffer::clone(entry->get_buf()), entry->get_val_type(),
|
||||
entry->get_timestamp());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
CoordinatorLogStore::CoordinatorLogStore() : start_idx_(1) {
|
||||
ptr<buffer> buf = buffer::alloc(sizeof(uint64_t));
|
||||
logs_[0] = cs_new<log_entry>(0, buf);
|
||||
}
|
||||
|
||||
CoordinatorLogStore::~CoordinatorLogStore() {}
|
||||
|
||||
auto CoordinatorLogStore::FindOrDefault_(uint64_t index) const -> ptr<log_entry> {
|
||||
auto entry = logs_.find(index);
|
||||
if (entry == logs_.end()) {
|
||||
entry = logs_.find(0);
|
||||
}
|
||||
return entry->second;
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::next_slot() const {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
return start_idx_ + logs_.size() - 1;
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::start_index() const { return start_idx_; }
|
||||
|
||||
ptr<log_entry> CoordinatorLogStore::last_entry() const {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
|
||||
uint64_t const last_idx = start_idx_ + logs_.size() - 1;
|
||||
auto const last_src = FindOrDefault_(last_idx - 1);
|
||||
|
||||
return MakeClone(last_src);
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::append(ptr<log_entry> &entry) {
|
||||
ptr<log_entry> clone = MakeClone(entry);
|
||||
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
uint64_t next_slot = start_idx_ + logs_.size() - 1;
|
||||
logs_[next_slot] = clone;
|
||||
|
||||
return next_slot;
|
||||
}
|
||||
|
||||
// TODO: (andi) I think this is used for resolving conflicts inside NuRaft, check...
|
||||
// different compared to in_memory_log_store.cxx
|
||||
void CoordinatorLogStore::write_at(uint64_t index, ptr<log_entry> &entry) {
|
||||
ptr<log_entry> clone = MakeClone(entry);
|
||||
|
||||
// Discard all logs equal to or greater than `index.
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
auto itr = logs_.lower_bound(index);
|
||||
while (itr != logs_.end()) {
|
||||
itr = logs_.erase(itr);
|
||||
}
|
||||
logs_[index] = clone;
|
||||
}
|
||||
|
||||
ptr<std::vector<ptr<log_entry>>> CoordinatorLogStore::log_entries(uint64_t start, uint64_t end) {
|
||||
auto ret = cs_new<std::vector<ptr<log_entry>>>();
|
||||
ret->resize(end - start);
|
||||
|
||||
for (uint64_t i = start, curr_index = 0; i < end; i++, curr_index++) {
|
||||
ptr<log_entry> src = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
if (auto const entry = logs_.find(i); entry != logs_.end()) {
|
||||
src = entry->second;
|
||||
} else {
|
||||
throw RaftCouldNotFindEntryException("Could not find entry at index {}", i);
|
||||
}
|
||||
}
|
||||
(*ret)[curr_index] = MakeClone(src);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
ptr<log_entry> CoordinatorLogStore::entry_at(uint64_t index) {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
ptr<log_entry> src = FindOrDefault_(index);
|
||||
return MakeClone(src);
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::term_at(uint64_t index) {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
return FindOrDefault_(index)->get_term();
|
||||
}
|
||||
|
||||
ptr<buffer> CoordinatorLogStore::pack(uint64_t index, int32 cnt) {
|
||||
std::vector<ptr<buffer>> logs;
|
||||
|
||||
size_t size_total = 0;
|
||||
uint64_t const end_index = index + cnt;
|
||||
for (uint64_t i = index; i < end_index; ++i) {
|
||||
ptr<log_entry> le = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
le = logs_[i];
|
||||
}
|
||||
MG_ASSERT(le.get(), "Could not find log entry at index {}", i);
|
||||
auto buf = le->serialize();
|
||||
size_total += buf->size();
|
||||
logs.push_back(buf);
|
||||
}
|
||||
|
||||
auto buf_out = buffer::alloc(sizeof(int32) + cnt * sizeof(int32) + size_total);
|
||||
buf_out->pos(0);
|
||||
buf_out->put((int32)cnt);
|
||||
|
||||
for (auto &entry : logs) {
|
||||
buf_out->put(static_cast<int32>(entry->size()));
|
||||
buf_out->put(*entry);
|
||||
}
|
||||
return buf_out;
|
||||
}
|
||||
|
||||
void CoordinatorLogStore::apply_pack(uint64_t index, buffer &pack) {
|
||||
pack.pos(0);
|
||||
int32 const num_logs = pack.get_int();
|
||||
|
||||
for (int32 i = 0; i < num_logs; ++i) {
|
||||
uint64_t cur_idx = index + i;
|
||||
int32 buf_size = pack.get_int();
|
||||
|
||||
ptr<buffer> buf_local = buffer::alloc(buf_size);
|
||||
pack.get(buf_local);
|
||||
|
||||
ptr<log_entry> le = log_entry::deserialize(*buf_local);
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
logs_[cur_idx] = le;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
auto const entry = logs_.upper_bound(0);
|
||||
if (entry != logs_.end()) {
|
||||
start_idx_ = entry->first;
|
||||
} else {
|
||||
start_idx_ = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: Remove all logs up to given 'last_log_index' (inclusive).
|
||||
bool CoordinatorLogStore::compact(uint64_t last_log_index) {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
for (uint64_t ii = start_idx_; ii <= last_log_index; ++ii) {
|
||||
auto const entry = logs_.find(ii);
|
||||
if (entry != logs_.end()) {
|
||||
logs_.erase(entry);
|
||||
}
|
||||
}
|
||||
|
||||
if (start_idx_ <= last_log_index) {
|
||||
start_idx_ = last_log_index + 1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CoordinatorLogStore::flush() { return true; }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -52,69 +52,6 @@ void DemoteMainToReplicaRes::Load(DemoteMainToReplicaRes *self, memgraph::slk::R
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void UnregisterReplicaReq::Save(UnregisterReplicaReq const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void UnregisterReplicaReq::Load(UnregisterReplicaReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void UnregisterReplicaRes::Save(UnregisterReplicaRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void UnregisterReplicaRes::Load(UnregisterReplicaRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void EnableWritingOnMainRes::Save(EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void EnableWritingOnMainRes::Load(EnableWritingOnMainRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void EnableWritingOnMainReq::Save(EnableWritingOnMainReq const & /*self*/, memgraph::slk::Builder * /*builder*/) {}
|
||||
|
||||
void EnableWritingOnMainReq::Load(EnableWritingOnMainReq * /*self*/, memgraph::slk::Reader * /*reader*/) {}
|
||||
|
||||
// GetInstanceUUID
|
||||
void GetInstanceUUIDReq::Save(const GetInstanceUUIDReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void GetInstanceUUIDReq::Load(GetInstanceUUIDReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void GetInstanceUUIDRes::Save(const GetInstanceUUIDRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void GetInstanceUUIDRes::Load(GetInstanceUUIDRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
// GetDatabaseHistoriesRpc
|
||||
|
||||
void GetDatabaseHistoriesReq::Save(const GetDatabaseHistoriesReq & /*self*/, memgraph::slk::Builder * /*builder*/) {
|
||||
/* nothing to serialize */
|
||||
}
|
||||
|
||||
void GetDatabaseHistoriesReq::Load(GetDatabaseHistoriesReq * /*self*/, memgraph::slk::Reader * /*reader*/) {
|
||||
/* nothing to serialize */
|
||||
}
|
||||
|
||||
void GetDatabaseHistoriesRes::Save(const GetDatabaseHistoriesRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void GetDatabaseHistoriesRes::Load(GetDatabaseHistoriesRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
} // namespace coordination
|
||||
|
||||
constexpr utils::TypeInfo coordination::PromoteReplicaToMainReq::kType{utils::TypeId::COORD_FAILOVER_REQ,
|
||||
@@ -127,37 +64,10 @@ constexpr utils::TypeInfo coordination::DemoteMainToReplicaReq::kType{utils::Typ
|
||||
"CoordDemoteToReplicaReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::DemoteMainToReplicaRes::kType{utils::TypeId::COORD_SET_REPL_MAIN_RES,
|
||||
|
||||
"CoordDemoteToReplicaRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::UnregisterReplicaReq::kType{utils::TypeId::COORD_UNREGISTER_REPLICA_REQ,
|
||||
"UnregisterReplicaReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::UnregisterReplicaRes::kType{utils::TypeId::COORD_UNREGISTER_REPLICA_RES,
|
||||
"UnregisterReplicaRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::EnableWritingOnMainReq::kType{utils::TypeId::COORD_ENABLE_WRITING_ON_MAIN_REQ,
|
||||
"CoordEnableWritingOnMainReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::EnableWritingOnMainRes::kType{utils::TypeId::COORD_ENABLE_WRITING_ON_MAIN_RES,
|
||||
"CoordEnableWritingOnMainRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::GetInstanceUUIDReq::kType{utils::TypeId::COORD_GET_UUID_REQ, "CoordGetUUIDReq",
|
||||
nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::GetInstanceUUIDRes::kType{utils::TypeId::COORD_GET_UUID_RES, "CoordGetUUIDRes",
|
||||
nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::GetDatabaseHistoriesReq::kType{utils::TypeId::COORD_GET_INSTANCE_DATABASES_REQ,
|
||||
"GetInstanceDatabasesReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::GetDatabaseHistoriesRes::kType{utils::TypeId::COORD_GET_INSTANCE_DATABASES_RES,
|
||||
"GetInstanceDatabasesRes", nullptr};
|
||||
|
||||
namespace slk {
|
||||
|
||||
// PromoteReplicaToMainRpc
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
@@ -167,16 +77,13 @@ void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::
|
||||
}
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid_, builder);
|
||||
memgraph::slk::Save(self.replication_clients_info, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid_, reader);
|
||||
memgraph::slk::Load(&self->replication_clients_info, reader);
|
||||
}
|
||||
|
||||
// DemoteMainToReplicaRpc
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.replication_client_info, builder);
|
||||
}
|
||||
@@ -193,60 +100,6 @@ void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::R
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
// UnregisterReplicaRpc
|
||||
|
||||
void Save(memgraph::coordination::UnregisterReplicaReq const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.instance_name, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::UnregisterReplicaReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->instance_name, reader);
|
||||
}
|
||||
|
||||
void Save(memgraph::coordination::UnregisterReplicaRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::UnregisterReplicaRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
void Save(memgraph::coordination::EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::EnableWritingOnMainRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
// GetInstanceUUIDRpc
|
||||
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDReq & /*self*/, memgraph::slk::Builder * /*builder*/) {
|
||||
/* nothing to serialize*/
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::GetInstanceUUIDReq * /*self*/, memgraph::slk::Reader * /*reader*/) {
|
||||
/* nothing to serialize*/
|
||||
}
|
||||
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::GetInstanceUUIDRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
}
|
||||
|
||||
// GetInstanceTimestampsReq
|
||||
|
||||
void Save(const memgraph::coordination::GetDatabaseHistoriesRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.database_histories, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::GetDatabaseHistoriesRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->database_histories, reader);
|
||||
}
|
||||
|
||||
} // namespace slk
|
||||
|
||||
} // namespace memgraph
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "replication_coordination_glue/messages.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
namespace memgraph::coordination {
|
||||
|
||||
CoordinatorState::CoordinatorState() {
|
||||
MG_ASSERT(!(FLAGS_raft_server_id && FLAGS_coordinator_server_port),
|
||||
MG_ASSERT(!(FLAGS_coordinator && FLAGS_coordinator_server_port),
|
||||
"Instance cannot be a coordinator and have registered coordinator server.");
|
||||
|
||||
spdlog::info("Executing coordinator constructor");
|
||||
@@ -41,55 +41,37 @@ CoordinatorState::CoordinatorState() {
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorState::RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
auto CoordinatorState::RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return RegisterInstanceCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[config](CoordinatorInstance &coordinator_instance) {
|
||||
return coordinator_instance.RegisterReplicationInstance(config);
|
||||
}},
|
||||
memgraph::utils::Overloaded{
|
||||
[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return RegisterInstanceCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[config](CoordinatorData &coordinator_data) { return coordinator_data.RegisterInstance(config); }},
|
||||
data_);
|
||||
}
|
||||
|
||||
auto CoordinatorState::UnregisterReplicationInstance(std::string_view instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot unregister instance since variant holds wrong alternative");
|
||||
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return UnregisterInstanceCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[&instance_name](CoordinatorInstance &coordinator_instance) {
|
||||
return coordinator_instance.UnregisterReplicationInstance(instance_name);
|
||||
}},
|
||||
data_);
|
||||
}
|
||||
|
||||
auto CoordinatorState::SetReplicationInstanceToMain(std::string_view instance_name)
|
||||
-> SetInstanceToMainCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
auto CoordinatorState::SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return SetInstanceToMainCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[&instance_name](CoordinatorInstance &coordinator_instance) {
|
||||
return coordinator_instance.SetReplicationInstanceToMain(instance_name);
|
||||
[&instance_name](CoordinatorData &coordinator_data) {
|
||||
return coordinator_data.SetInstanceToMain(instance_name);
|
||||
}},
|
||||
data_);
|
||||
}
|
||||
|
||||
auto CoordinatorState::ShowInstances() const -> std::vector<InstanceStatus> {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
auto CoordinatorState::ShowInstances() const -> std::vector<CoordinatorInstanceStatus> {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorData>(data_),
|
||||
"Can't call show instances on data_, as variant holds wrong alternative");
|
||||
return std::get<CoordinatorInstance>(data_).ShowInstances();
|
||||
return std::get<CoordinatorData>(data_).ShowInstances();
|
||||
}
|
||||
|
||||
auto CoordinatorState::GetCoordinatorServer() const -> CoordinatorServer & {
|
||||
@@ -97,13 +79,5 @@ auto CoordinatorState::GetCoordinatorServer() const -> CoordinatorServer & {
|
||||
"Cannot get coordinator server since variant holds wrong alternative");
|
||||
return *std::get<CoordinatorMainReplicaData>(data_).coordinator_server_;
|
||||
}
|
||||
|
||||
auto CoordinatorState::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port,
|
||||
std::string_view raft_address) -> void {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
return std::get<CoordinatorInstance>(data_).AddCoordinatorInstance(raft_server_id, raft_port, raft_address);
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -1,215 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace {
|
||||
constexpr int MAX_SNAPSHOTS = 3;
|
||||
} // namespace
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
auto CoordinatorStateMachine::FindCurrentMainInstanceName() const -> std::optional<std::string> {
|
||||
return cluster_state_.FindCurrentMainInstanceName();
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::MainExists() const -> bool { return cluster_state_.MainExists(); }
|
||||
|
||||
auto CoordinatorStateMachine::IsMain(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.IsMain(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::IsReplica(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.IsReplica(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::CreateLog(nlohmann::json &&log) -> ptr<buffer> {
|
||||
auto const log_dump = log.dump();
|
||||
ptr<buffer> log_buf = buffer::alloc(sizeof(uint32_t) + log_dump.size());
|
||||
buffer_serializer bs(log_buf);
|
||||
bs.put_str(log_dump);
|
||||
return log_buf;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeRegisterInstance(CoordinatorClientConfig const &config) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::REGISTER_REPLICATION_INSTANCE}, {"info", config}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeUnregisterInstance(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::UNREGISTER_REPLICATION_INSTANCE}, {"info", instance_name}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeSetInstanceAsMain(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::SET_INSTANCE_AS_MAIN}, {"info", instance_name}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeSetInstanceAsReplica(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::SET_INSTANCE_AS_REPLICA}, {"info", instance_name}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeUpdateUUID(utils::UUID const &uuid) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::UPDATE_UUID}, {"info", uuid}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::DecodeLog(buffer &data) -> std::pair<TRaftLog, RaftLogAction> {
|
||||
buffer_serializer bs(data);
|
||||
auto const json = nlohmann::json::parse(bs.get_str());
|
||||
|
||||
auto const action = json["action"].get<RaftLogAction>();
|
||||
auto const &info = json["info"];
|
||||
|
||||
switch (action) {
|
||||
case RaftLogAction::REGISTER_REPLICATION_INSTANCE:
|
||||
return {info.get<CoordinatorClientConfig>(), action};
|
||||
case RaftLogAction::UPDATE_UUID:
|
||||
return {info.get<utils::UUID>(), action};
|
||||
case RaftLogAction::UNREGISTER_REPLICATION_INSTANCE:
|
||||
case RaftLogAction::SET_INSTANCE_AS_MAIN:
|
||||
[[fallthrough]];
|
||||
case RaftLogAction::SET_INSTANCE_AS_REPLICA:
|
||||
return {info.get<std::string>(), action};
|
||||
}
|
||||
throw std::runtime_error("Unknown action");
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::pre_commit(ulong const /*log_idx*/, buffer & /*data*/) -> ptr<buffer> { return nullptr; }
|
||||
|
||||
auto CoordinatorStateMachine::commit(ulong const log_idx, buffer &data) -> ptr<buffer> {
|
||||
spdlog::debug("Commit: log_idx={}, data.size()={}", log_idx, data.size());
|
||||
auto const [parsed_data, log_action] = DecodeLog(data);
|
||||
cluster_state_.DoAction(parsed_data, log_action);
|
||||
last_committed_idx_ = log_idx;
|
||||
|
||||
// Return raft log number
|
||||
ptr<buffer> ret = buffer::alloc(sizeof(log_idx));
|
||||
buffer_serializer bs_ret(ret);
|
||||
bs_ret.put_u64(log_idx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::commit_config(ulong const log_idx, ptr<cluster_config> & /*new_conf*/) -> void {
|
||||
last_committed_idx_ = log_idx;
|
||||
spdlog::debug("Commit config: log_idx={}", log_idx);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::rollback(ulong const log_idx, buffer &data) -> void {
|
||||
// NOTE: Nothing since we don't do anything in pre_commit
|
||||
spdlog::debug("Rollback: log_idx={}, data.size()={}", log_idx, data.size());
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::read_logical_snp_obj(snapshot &snapshot, void *& /*user_snp_ctx*/, ulong obj_id,
|
||||
ptr<buffer> &data_out, bool &is_last_obj) -> int {
|
||||
spdlog::debug("read logical snapshot object, obj_id: {}", obj_id);
|
||||
|
||||
ptr<SnapshotCtx> ctx = nullptr;
|
||||
{
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
auto entry = snapshots_.find(snapshot.get_last_log_idx());
|
||||
if (entry == snapshots_.end()) {
|
||||
data_out = nullptr;
|
||||
is_last_obj = true;
|
||||
return 0;
|
||||
}
|
||||
ctx = entry->second;
|
||||
}
|
||||
|
||||
if (obj_id == 0) {
|
||||
// Object ID == 0: first object, put dummy data.
|
||||
data_out = buffer::alloc(sizeof(int32));
|
||||
buffer_serializer bs(data_out);
|
||||
bs.put_i32(0);
|
||||
is_last_obj = false;
|
||||
} else {
|
||||
// Object ID > 0: second object, put actual value.
|
||||
ctx->cluster_state_.Serialize(data_out);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::save_logical_snp_obj(snapshot &snapshot, ulong &obj_id, buffer &data, bool is_first_obj,
|
||||
bool is_last_obj) -> void {
|
||||
spdlog::debug("save logical snapshot object, obj_id: {}, is_first_obj: {}, is_last_obj: {}", obj_id, is_first_obj,
|
||||
is_last_obj);
|
||||
|
||||
if (obj_id == 0) {
|
||||
ptr<buffer> snp_buf = snapshot.serialize();
|
||||
auto ss = snapshot::deserialize(*snp_buf);
|
||||
create_snapshot_internal(ss);
|
||||
} else {
|
||||
auto cluster_state = CoordinatorClusterState::Deserialize(data);
|
||||
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
auto entry = snapshots_.find(snapshot.get_last_log_idx());
|
||||
DMG_ASSERT(entry != snapshots_.end());
|
||||
entry->second->cluster_state_ = cluster_state;
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::apply_snapshot(snapshot &s) -> bool {
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
spdlog::debug("apply snapshot, last_log_idx: {}", s.get_last_log_idx());
|
||||
|
||||
auto entry = snapshots_.find(s.get_last_log_idx());
|
||||
if (entry == snapshots_.end()) return false;
|
||||
|
||||
cluster_state_ = entry->second->cluster_state_;
|
||||
return true;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::free_user_snp_ctx(void *&user_snp_ctx) -> void {}
|
||||
|
||||
auto CoordinatorStateMachine::last_snapshot() -> ptr<snapshot> {
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
spdlog::debug("last_snapshot");
|
||||
auto entry = snapshots_.rbegin();
|
||||
if (entry == snapshots_.rend()) return nullptr;
|
||||
|
||||
ptr<SnapshotCtx> ctx = entry->second;
|
||||
return ctx->snapshot_;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::last_commit_index() -> ulong { return last_committed_idx_; }
|
||||
|
||||
auto CoordinatorStateMachine::create_snapshot(snapshot &s, async_result<bool>::handler_type &when_done) -> void {
|
||||
spdlog::debug("create_snapshot, last_log_idx: {}", s.get_last_log_idx());
|
||||
ptr<buffer> snp_buf = s.serialize();
|
||||
ptr<snapshot> ss = snapshot::deserialize(*snp_buf);
|
||||
create_snapshot_internal(ss);
|
||||
|
||||
ptr<std::exception> except(nullptr);
|
||||
bool ret = true;
|
||||
when_done(ret, except);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::create_snapshot_internal(ptr<snapshot> snapshot) -> void {
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
spdlog::debug("create_snapshot_internal, last_log_idx: {}", snapshot->get_last_log_idx());
|
||||
|
||||
auto ctx = cs_new<SnapshotCtx>(snapshot, cluster_state_);
|
||||
snapshots_[snapshot->get_last_log_idx()] = ctx;
|
||||
|
||||
while (snapshots_.size() > MAX_SNAPSHOTS) {
|
||||
snapshots_.erase(snapshots_.begin());
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::GetInstances() const -> std::vector<InstanceState> {
|
||||
return cluster_state_.GetInstances();
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::GetUUID() const -> utils::UUID { return cluster_state_.GetUUID(); }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::cluster_config;
|
||||
using nuraft::cs_new;
|
||||
using nuraft::srv_config;
|
||||
using nuraft::srv_state;
|
||||
using nuraft::state_mgr;
|
||||
|
||||
CoordinatorStateManager::CoordinatorStateManager(int srv_id, std::string const &endpoint)
|
||||
: my_id_(srv_id), my_endpoint_(endpoint), cur_log_store_(cs_new<CoordinatorLogStore>()) {
|
||||
my_srv_config_ = cs_new<srv_config>(srv_id, endpoint);
|
||||
|
||||
// Initial cluster config: contains only one server (myself).
|
||||
cluster_config_ = cs_new<cluster_config>();
|
||||
cluster_config_->get_servers().push_back(my_srv_config_);
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::load_config() -> ptr<cluster_config> {
|
||||
// Just return in-memory data in this example.
|
||||
// May require reading from disk here, if it has been written to disk.
|
||||
return cluster_config_;
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::save_config(cluster_config const &config) -> void {
|
||||
// Just keep in memory in this example.
|
||||
// Need to write to disk here, if want to make it durable.
|
||||
ptr<buffer> buf = config.serialize();
|
||||
cluster_config_ = cluster_config::deserialize(*buf);
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::save_state(srv_state const &state) -> void {
|
||||
// Just keep in memory in this example.
|
||||
// Need to write to disk here, if want to make it durable.
|
||||
ptr<buffer> buf = state.serialize();
|
||||
saved_state_ = srv_state::deserialize(*buf);
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::read_state() -> ptr<srv_state> {
|
||||
// Just return in-memory data in this example.
|
||||
// May require reading from disk here, if it has been written to disk.
|
||||
return saved_state_;
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::load_log_store() -> ptr<log_store> { return cur_log_store_; }
|
||||
|
||||
auto CoordinatorStateManager::server_id() -> int32 { return my_id_; }
|
||||
|
||||
auto CoordinatorStateManager::system_exit(int const exit_code) -> void {}
|
||||
|
||||
auto CoordinatorStateManager::GetSrvConfig() const -> ptr<srv_config> { return my_srv_config_; }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,60 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#if FMT_VERSION > 90000
|
||||
#include <fmt/ostream.h>
|
||||
#include <string>
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
inline std::string ToString(const nuraft::cmd_result_code &code) {
|
||||
switch (code) {
|
||||
case nuraft::cmd_result_code::OK:
|
||||
return "OK";
|
||||
case nuraft::cmd_result_code::FAILED:
|
||||
return "FAILED";
|
||||
case nuraft::cmd_result_code::RESULT_NOT_EXIST_YET:
|
||||
return "RESULT_NOT_EXIST_YET";
|
||||
case nuraft::cmd_result_code::TERM_MISMATCH:
|
||||
return "TERM_MISMATCH";
|
||||
case nuraft::cmd_result_code::SERVER_IS_LEAVING:
|
||||
return "SERVER_IS_LEAVING";
|
||||
case nuraft::cmd_result_code::CANNOT_REMOVE_LEADER:
|
||||
return "CANNOT_REMOVE_LEADER";
|
||||
case nuraft::cmd_result_code::SERVER_NOT_FOUND:
|
||||
return "SERVER_NOT_FOUND";
|
||||
case nuraft::cmd_result_code::SERVER_IS_JOINING:
|
||||
return "SERVER_IS_JOINING";
|
||||
case nuraft::cmd_result_code::CONFIG_CHANGING:
|
||||
return "CONFIG_CHANGING";
|
||||
case nuraft::cmd_result_code::SERVER_ALREADY_EXISTS:
|
||||
return "SERVER_ALREADY_EXISTS";
|
||||
case nuraft::cmd_result_code::BAD_REQUEST:
|
||||
return "BAD_REQUEST";
|
||||
case nuraft::cmd_result_code::NOT_LEADER:
|
||||
return "NOT_LEADER";
|
||||
case nuraft::cmd_result_code::TIMEOUT:
|
||||
return "TIMEOUT";
|
||||
case nuraft::cmd_result_code::CANCELLED:
|
||||
return "CANCELLED";
|
||||
}
|
||||
LOG_FATAL("ToString of a nuraft::cmd_result_code -> check missing switch case");
|
||||
}
|
||||
inline std::ostream &operator<<(std::ostream &os, const nuraft::cmd_result_code &code) {
|
||||
os << ToString(code);
|
||||
return os;
|
||||
}
|
||||
template <>
|
||||
class fmt::formatter<nuraft::cmd_result_code> : public fmt::ostream_formatter {};
|
||||
#endif
|
||||
@@ -9,7 +9,14 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace memgraph::coordination {
|
||||
enum class GetInstanceUUIDError { NO_RESPONSE, RPC_EXCEPTION };
|
||||
enum class GetInstanceTimestampsError { NO_RESPONSE, RPC_EXCEPTION };
|
||||
|
||||
#ifdef MG_EXPERIMENTAL_HIGH_AVAILABILITY
|
||||
constexpr bool allow_ha = true;
|
||||
#else
|
||||
constexpr bool allow_ha = false;
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
@@ -14,23 +14,19 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "rpc/client.hpp"
|
||||
#include "rpc_errors.hpp"
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/scheduler.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
using HealthCheckClientCallback = std::function<void(CoordinatorInstance *, std::string_view)>;
|
||||
class CoordinatorData;
|
||||
using HealthCheckCallback = std::function<void(CoordinatorData *, std::string_view)>;
|
||||
using ReplicationClientsInfo = std::vector<ReplClientInfo>;
|
||||
|
||||
class CoordinatorClient {
|
||||
public:
|
||||
explicit CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorClientConfig config,
|
||||
HealthCheckClientCallback succ_cb, HealthCheckClientCallback fail_cb);
|
||||
explicit CoordinatorClient(CoordinatorData *coord_data_, CoordinatorClientConfig config, HealthCheckCallback succ_cb,
|
||||
HealthCheckCallback fail_cb);
|
||||
|
||||
~CoordinatorClient() = default;
|
||||
|
||||
@@ -46,32 +42,14 @@ class CoordinatorClient {
|
||||
void ResumeFrequentCheck();
|
||||
|
||||
auto InstanceName() const -> std::string;
|
||||
auto CoordinatorSocketAddress() const -> std::string;
|
||||
auto ReplicationSocketAddress() const -> std::string;
|
||||
auto SocketAddress() const -> std::string;
|
||||
|
||||
[[nodiscard]] auto SendPromoteReplicaToMainRpc(ReplicationClientsInfo replication_clients_info) const -> bool;
|
||||
[[nodiscard]] auto DemoteToReplica() const -> bool;
|
||||
|
||||
auto SendPromoteReplicaToMainRpc(utils::UUID const &uuid, ReplicationClientsInfo replication_clients_info) const
|
||||
-> bool;
|
||||
|
||||
auto SendSwapMainUUIDRpc(utils::UUID const &uuid) const -> bool;
|
||||
|
||||
auto SendUnregisterReplicaRpc(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto SendEnableWritingOnMainRpc() const -> bool;
|
||||
|
||||
auto SendGetInstanceUUIDRpc() const -> memgraph::utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>>;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto SendGetInstanceTimestampsRpc() const
|
||||
-> utils::BasicResult<GetInstanceUUIDError, replication_coordination_glue::DatabaseHistories>;
|
||||
|
||||
auto RpcClient() -> rpc::Client & { return rpc_client_; }
|
||||
|
||||
auto InstanceDownTimeoutSec() const -> std::chrono::seconds;
|
||||
|
||||
auto InstanceGetUUIDFrequencySec() const -> std::chrono::seconds;
|
||||
auto SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCallback fail_cb) -> void;
|
||||
|
||||
friend bool operator==(CoordinatorClient const &first, CoordinatorClient const &second) {
|
||||
return first.config_ == second.config_;
|
||||
@@ -80,13 +58,14 @@ class CoordinatorClient {
|
||||
private:
|
||||
utils::Scheduler instance_checker_;
|
||||
|
||||
// TODO: (andi) Pimpl?
|
||||
communication::ClientContext rpc_context_;
|
||||
mutable rpc::Client rpc_client_;
|
||||
|
||||
CoordinatorClientConfig config_;
|
||||
CoordinatorInstance *coord_instance_;
|
||||
HealthCheckClientCallback succ_cb_;
|
||||
HealthCheckClientCallback fail_cb_;
|
||||
CoordinatorData *coord_data_;
|
||||
HealthCheckCallback succ_cb_;
|
||||
HealthCheckCallback fail_cb_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -11,11 +11,12 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#if FMT_VERSION > 90000
|
||||
#include <fmt/ostream.h>
|
||||
#ifdef MG_ENTERPRISE
|
||||
namespace memgraph::coordination {
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
struct CoordinatorClusterConfig {
|
||||
static constexpr int alive_response_time_difference_sec_{5};
|
||||
};
|
||||
|
||||
template <>
|
||||
class fmt::formatter<memgraph::io::network::Endpoint> : public fmt::ostream_formatter {};
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -14,16 +14,12 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "replication_coordination_glue/mode.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include "json/json.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
|
||||
@@ -32,15 +28,9 @@ struct CoordinatorClientConfig {
|
||||
std::string instance_name;
|
||||
std::string ip_address;
|
||||
uint16_t port{};
|
||||
std::chrono::seconds instance_health_check_frequency_sec{1};
|
||||
std::chrono::seconds instance_down_timeout_sec{5};
|
||||
std::chrono::seconds instance_get_uuid_frequency_sec{10};
|
||||
std::chrono::seconds health_check_frequency_sec{1};
|
||||
|
||||
auto CoordinatorSocketAddress() const -> std::string { return fmt::format("{}:{}", ip_address, port); }
|
||||
auto ReplicationSocketAddress() const -> std::string {
|
||||
return fmt::format("{}:{}", replication_client_info.replication_ip_address,
|
||||
replication_client_info.replication_port);
|
||||
}
|
||||
auto SocketAddress() const -> std::string { return ip_address + ":" + std::to_string(port); }
|
||||
|
||||
struct ReplicationClientInfo {
|
||||
std::string instance_name;
|
||||
@@ -83,11 +73,5 @@ struct CoordinatorServerConfig {
|
||||
friend bool operator==(CoordinatorServerConfig const &, CoordinatorServerConfig const &) = default;
|
||||
};
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorClientConfig const &config);
|
||||
void from_json(nlohmann::json const &j, CoordinatorClientConfig &config);
|
||||
|
||||
void to_json(nlohmann::json &j, ReplClientInfo const &config);
|
||||
void from_json(nlohmann::json const &j, ReplClientInfo &config);
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
52
src/coordination/include/coordination/coordinator_data.hpp
Normal file
52
src/coordination/include/coordination/coordinator_data.hpp
Normal file
@@ -0,0 +1,52 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
#include "coordination/coordinator_instance_status.hpp"
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "utils/rw_lock.hpp"
|
||||
#include "utils/thread_pool.hpp"
|
||||
|
||||
#include <list>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
class CoordinatorData {
|
||||
public:
|
||||
CoordinatorData();
|
||||
|
||||
[[nodiscard]] auto RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
|
||||
|
||||
[[nodiscard]] auto SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
|
||||
|
||||
auto TryFailover() -> void;
|
||||
|
||||
auto ShowInstances() const -> std::vector<CoordinatorInstanceStatus>;
|
||||
|
||||
private:
|
||||
auto ClusterHasAliveMain_() const -> bool;
|
||||
|
||||
mutable utils::RWLock coord_data_lock_{utils::RWLock::Priority::READ};
|
||||
HealthCheckCallback main_succ_cb_, main_fail_cb_, replica_succ_cb_, replica_fail_cb_;
|
||||
// NOTE: Must be std::list because we rely on pointer stability
|
||||
std::list<CoordinatorInstance> registered_instances_;
|
||||
};
|
||||
|
||||
struct CoordinatorMainReplicaData {
|
||||
std::unique_ptr<CoordinatorServer> coordinator_server_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -28,71 +28,5 @@ class CoordinatorRegisterInstanceException final : public utils::BasicException
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(CoordinatorRegisterInstanceException)
|
||||
};
|
||||
|
||||
class RaftServerStartException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftServerStartException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftServerStartException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftServerStartException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftServerStartException)
|
||||
};
|
||||
|
||||
class RaftAddServerException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftAddServerException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftAddServerException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftAddServerException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftAddServerException)
|
||||
};
|
||||
|
||||
class RaftBecomeLeaderException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftBecomeLeaderException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftBecomeLeaderException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftBecomeLeaderException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftBecomeLeaderException)
|
||||
};
|
||||
|
||||
class RaftCouldNotFindEntryException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftCouldNotFindEntryException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftCouldNotFindEntryException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftCouldNotFindEntryException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftCouldNotFindEntryException)
|
||||
};
|
||||
|
||||
class RaftCouldNotParseFlagsException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftCouldNotParseFlagsException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftCouldNotParseFlagsException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftCouldNotParseFlagsException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftCouldNotParseFlagsException)
|
||||
};
|
||||
|
||||
class InvalidRaftLogActionException final : public utils::BasicException {
|
||||
public:
|
||||
explicit InvalidRaftLogActionException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit InvalidRaftLogActionException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: InvalidRaftLogActionException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(InvalidRaftLogActionException)
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -31,19 +31,6 @@ class CoordinatorHandlers {
|
||||
slk::Builder *res_builder);
|
||||
static void DemoteMainToReplicaHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
static void SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
|
||||
static void UnregisterReplicaHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
static void EnableWritingOnMainHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
|
||||
static void GetInstanceUUIDHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
|
||||
static void GetDatabaseHistoriesHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -13,69 +13,55 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "coordination/instance_status.hpp"
|
||||
#include "coordination/raft_state.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "coordination/replication_instance.hpp"
|
||||
#include "utils/resource_lock.hpp"
|
||||
#include "utils/rw_lock.hpp"
|
||||
#include "utils/thread_pool.hpp"
|
||||
|
||||
#include <list>
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
#include "coordination/coordinator_cluster_config.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
struct NewMainRes {
|
||||
std::string most_up_to_date_instance;
|
||||
std::string latest_epoch;
|
||||
uint64_t latest_commit_timestamp;
|
||||
};
|
||||
using InstanceNameDbHistories = std::pair<std::string, replication_coordination_glue::DatabaseHistories>;
|
||||
class CoordinatorData;
|
||||
|
||||
class CoordinatorInstance {
|
||||
public:
|
||||
CoordinatorInstance();
|
||||
CoordinatorInstance(CoordinatorData *data, CoordinatorClientConfig config, HealthCheckCallback succ_cb,
|
||||
HealthCheckCallback fail_cb);
|
||||
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto UnregisterReplicationInstance(std::string_view instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus;
|
||||
CoordinatorInstance(CoordinatorInstance const &other) = delete;
|
||||
CoordinatorInstance &operator=(CoordinatorInstance const &other) = delete;
|
||||
CoordinatorInstance(CoordinatorInstance &&other) noexcept = delete;
|
||||
CoordinatorInstance &operator=(CoordinatorInstance &&other) noexcept = delete;
|
||||
~CoordinatorInstance() = default;
|
||||
|
||||
[[nodiscard]] auto SetReplicationInstanceToMain(std::string_view instance_name) -> SetInstanceToMainCoordinatorStatus;
|
||||
auto OnSuccessPing() -> void;
|
||||
auto OnFailPing() -> bool;
|
||||
|
||||
auto ShowInstances() const -> std::vector<InstanceStatus>;
|
||||
auto IsAlive() const -> bool;
|
||||
|
||||
auto TryFailover() -> void;
|
||||
auto InstanceName() const -> std::string;
|
||||
auto SocketAddress() const -> std::string;
|
||||
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
auto IsReplica() const -> bool;
|
||||
auto IsMain() const -> bool;
|
||||
|
||||
static auto ChooseMostUpToDateInstance(std::span<InstanceNameDbHistories> histories) -> NewMainRes;
|
||||
auto PromoteToMain(ReplicationClientsInfo repl_clients_info, HealthCheckCallback main_succ_cb,
|
||||
HealthCheckCallback main_fail_cb) -> bool;
|
||||
auto DemoteToReplica(HealthCheckCallback replica_succ_cb, HealthCheckCallback replica_fail_cb) -> bool;
|
||||
|
||||
auto PauseFrequentCheck() -> void;
|
||||
auto ResumeFrequentCheck() -> void;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
private:
|
||||
HealthCheckClientCallback client_succ_cb_, client_fail_cb_;
|
||||
CoordinatorClient client_;
|
||||
replication_coordination_glue::ReplicationRole replication_role_;
|
||||
std::chrono::system_clock::time_point last_response_time_{};
|
||||
bool is_alive_{false};
|
||||
|
||||
auto OnRaftCommitCallback(TRaftLog const &log_entry, RaftLogAction log_action) -> void;
|
||||
|
||||
auto FindReplicationInstance(std::string_view replication_instance_name) -> ReplicationInstance &;
|
||||
|
||||
void MainFailCallback(std::string_view);
|
||||
|
||||
void MainSuccessCallback(std::string_view);
|
||||
|
||||
void ReplicaSuccessCallback(std::string_view);
|
||||
|
||||
void ReplicaFailCallback(std::string_view);
|
||||
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
// NOTE: Must be std::list because we rely on pointer stability.
|
||||
// Leader and followers should both have same view on repl_instances_
|
||||
std::list<ReplicationInstance> repl_instances_;
|
||||
mutable utils::ResourceLock coord_instance_lock_{};
|
||||
|
||||
RaftState raft_state_;
|
||||
friend bool operator==(CoordinatorInstance const &first, CoordinatorInstance const &second) {
|
||||
return first.client_ == second.client_ && first.replication_role_ == second.replication_role_;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -19,14 +19,11 @@
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
// TODO: (andi) For phase IV. Some instances won't have raft_socket_address, coord_socket_address, replication_role and
|
||||
// cluster role... At the end, all instances will have everything.
|
||||
struct InstanceStatus {
|
||||
struct CoordinatorInstanceStatus {
|
||||
std::string instance_name;
|
||||
std::string raft_socket_address;
|
||||
std::string coord_socket_address;
|
||||
std::string cluster_role;
|
||||
std::string health;
|
||||
std::string socket_address;
|
||||
std::string replication_role;
|
||||
bool is_alive;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
@@ -11,11 +11,9 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "rpc/messages.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
|
||||
@@ -28,13 +26,10 @@ struct PromoteReplicaToMainReq {
|
||||
static void Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit PromoteReplicaToMainReq(const utils::UUID &uuid,
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
|
||||
: main_uuid_(uuid), replication_clients_info(std::move(replication_clients_info)) {}
|
||||
explicit PromoteReplicaToMainReq(std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
|
||||
: replication_clients_info(std::move(replication_clients_info)) {}
|
||||
PromoteReplicaToMainReq() = default;
|
||||
|
||||
// get uuid here
|
||||
utils::UUID main_uuid_;
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info;
|
||||
};
|
||||
|
||||
@@ -83,148 +78,27 @@ struct DemoteMainToReplicaRes {
|
||||
|
||||
using DemoteMainToReplicaRpc = rpc::RequestResponse<DemoteMainToReplicaReq, DemoteMainToReplicaRes>;
|
||||
|
||||
struct UnregisterReplicaReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(UnregisterReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(UnregisterReplicaReq const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit UnregisterReplicaReq(std::string_view inst_name) : instance_name(inst_name) {}
|
||||
|
||||
UnregisterReplicaReq() = default;
|
||||
|
||||
std::string instance_name;
|
||||
};
|
||||
|
||||
struct UnregisterReplicaRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(UnregisterReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const UnregisterReplicaRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit UnregisterReplicaRes(bool success) : success(success) {}
|
||||
UnregisterReplicaRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using UnregisterReplicaRpc = rpc::RequestResponse<UnregisterReplicaReq, UnregisterReplicaRes>;
|
||||
|
||||
struct EnableWritingOnMainReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(EnableWritingOnMainReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(EnableWritingOnMainReq const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
EnableWritingOnMainReq() = default;
|
||||
};
|
||||
|
||||
struct EnableWritingOnMainRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(EnableWritingOnMainRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit EnableWritingOnMainRes(bool success) : success(success) {}
|
||||
EnableWritingOnMainRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using EnableWritingOnMainRpc = rpc::RequestResponse<EnableWritingOnMainReq, EnableWritingOnMainRes>;
|
||||
|
||||
struct GetInstanceUUIDReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(GetInstanceUUIDReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const GetInstanceUUIDReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
GetInstanceUUIDReq() = default;
|
||||
};
|
||||
|
||||
struct GetInstanceUUIDRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(GetInstanceUUIDRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const GetInstanceUUIDRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit GetInstanceUUIDRes(std::optional<utils::UUID> uuid) : uuid(uuid) {}
|
||||
GetInstanceUUIDRes() = default;
|
||||
|
||||
std::optional<utils::UUID> uuid;
|
||||
};
|
||||
|
||||
using GetInstanceUUIDRpc = rpc::RequestResponse<GetInstanceUUIDReq, GetInstanceUUIDRes>;
|
||||
|
||||
struct GetDatabaseHistoriesReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(GetDatabaseHistoriesReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const GetDatabaseHistoriesReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
GetDatabaseHistoriesReq() = default;
|
||||
};
|
||||
|
||||
struct GetDatabaseHistoriesRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(GetDatabaseHistoriesRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const GetDatabaseHistoriesRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit GetDatabaseHistoriesRes(const replication_coordination_glue::DatabaseHistories &database_histories)
|
||||
: database_histories(database_histories) {}
|
||||
GetDatabaseHistoriesRes() = default;
|
||||
|
||||
replication_coordination_glue::DatabaseHistories database_histories;
|
||||
};
|
||||
|
||||
using GetDatabaseHistoriesRpc = rpc::RequestResponse<GetDatabaseHistoriesReq, GetDatabaseHistoriesRes>;
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
// SLK serialization declarations
|
||||
namespace memgraph::slk {
|
||||
|
||||
// PromoteReplicaToMainRpc
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// DemoteMainToReplicaRpc
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// GetInstanceUUIDRpc
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDReq &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::GetInstanceUUIDReq *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::GetInstanceUUIDRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// UnregisterReplicaRpc
|
||||
void Save(memgraph::coordination::UnregisterReplicaRes const &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::UnregisterReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(memgraph::coordination::UnregisterReplicaReq const &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::UnregisterReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// EnableWritingOnMainRpc
|
||||
void Save(memgraph::coordination::EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::EnableWritingOnMainRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// GetDatabaseHistoriesRpc
|
||||
void Save(const memgraph::coordination::GetDatabaseHistoriesRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::GetDatabaseHistoriesRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
} // namespace memgraph::slk
|
||||
|
||||
#endif
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
|
||||
@@ -35,18 +34,5 @@ inline void Load(ReplicationClientInfo *obj, Reader *reader) {
|
||||
Load(&obj->replication_ip_address, reader);
|
||||
Load(&obj->replication_port, reader);
|
||||
}
|
||||
|
||||
inline void Save(const replication_coordination_glue::DatabaseHistory &obj, Builder *builder) {
|
||||
Save(obj.db_uuid, builder);
|
||||
Save(obj.history, builder);
|
||||
Save(obj.name, builder);
|
||||
}
|
||||
|
||||
inline void Load(replication_coordination_glue::DatabaseHistory *obj, Reader *reader) {
|
||||
Load(&obj->db_uuid, reader);
|
||||
Load(&obj->history, reader);
|
||||
Load(&obj->name, reader);
|
||||
}
|
||||
|
||||
} // namespace memgraph::slk
|
||||
#endif
|
||||
|
||||
@@ -13,9 +13,9 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
#include "coordination/coordinator_data.hpp"
|
||||
#include "coordination/coordinator_instance_status.hpp"
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "coordination/instance_status.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
|
||||
#include <variant>
|
||||
@@ -33,26 +33,17 @@ class CoordinatorState {
|
||||
CoordinatorState(CoordinatorState &&) noexcept = delete;
|
||||
CoordinatorState &operator=(CoordinatorState &&) noexcept = delete;
|
||||
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto UnregisterReplicationInstance(std::string_view instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto RegisterInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
|
||||
|
||||
[[nodiscard]] auto SetReplicationInstanceToMain(std::string_view instance_name) -> SetInstanceToMainCoordinatorStatus;
|
||||
[[nodiscard]] auto SetInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
|
||||
|
||||
auto ShowInstances() const -> std::vector<InstanceStatus>;
|
||||
auto ShowInstances() const -> std::vector<CoordinatorInstanceStatus>;
|
||||
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
|
||||
// NOTE: The client code must check that the server exists before calling this method.
|
||||
// The client code must check that the server exists before calling this method.
|
||||
auto GetCoordinatorServer() const -> CoordinatorServer &;
|
||||
|
||||
private:
|
||||
struct CoordinatorMainReplicaData {
|
||||
std::unique_ptr<CoordinatorServer> coordinator_server_;
|
||||
};
|
||||
|
||||
std::variant<CoordinatorInstance, CoordinatorMainReplicaData> data_;
|
||||
std::variant<CoordinatorData, CoordinatorMainReplicaData> data_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <flags/replication.hpp>
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
struct CoordinatorClientConfig;
|
||||
|
||||
using BecomeLeaderCb = std::function<void()>;
|
||||
using BecomeFollowerCb = std::function<void()>;
|
||||
|
||||
using nuraft::buffer;
|
||||
using nuraft::logger;
|
||||
using nuraft::ptr;
|
||||
using nuraft::raft_launcher;
|
||||
using nuraft::raft_server;
|
||||
using nuraft::srv_config;
|
||||
using nuraft::state_machine;
|
||||
using nuraft::state_mgr;
|
||||
using raft_result = nuraft::cmd_result<ptr<buffer>>;
|
||||
|
||||
class RaftState {
|
||||
private:
|
||||
explicit RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t raft_server_id,
|
||||
uint32_t raft_port, std::string raft_address);
|
||||
|
||||
auto InitRaftServer() -> void;
|
||||
|
||||
public:
|
||||
RaftState() = delete;
|
||||
RaftState(RaftState const &other) = default;
|
||||
RaftState &operator=(RaftState const &other) = default;
|
||||
RaftState(RaftState &&other) noexcept = default;
|
||||
RaftState &operator=(RaftState &&other) noexcept = default;
|
||||
~RaftState();
|
||||
|
||||
static auto MakeRaftState(BecomeLeaderCb &&become_leader_cb, BecomeFollowerCb &&become_follower_cb) -> RaftState;
|
||||
|
||||
auto InstanceName() const -> std::string;
|
||||
auto RaftSocketAddress() const -> std::string;
|
||||
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
auto GetAllCoordinators() const -> std::vector<ptr<srv_config>>;
|
||||
|
||||
auto RequestLeadership() -> bool;
|
||||
auto IsLeader() const -> bool;
|
||||
|
||||
auto FindCurrentMainInstanceName() const -> std::optional<std::string>;
|
||||
auto MainExists() const -> bool;
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto AppendRegisterReplicationInstanceLog(CoordinatorClientConfig const &config) -> bool;
|
||||
auto AppendUnregisterReplicationInstanceLog(std::string_view instance_name) -> bool;
|
||||
auto AppendSetInstanceAsMainLog(std::string_view instance_name) -> bool;
|
||||
auto AppendSetInstanceAsReplicaLog(std::string_view instance_name) -> bool;
|
||||
auto AppendUpdateUUIDLog(utils::UUID const &uuid) -> bool;
|
||||
|
||||
auto GetInstances() const -> std::vector<InstanceState>;
|
||||
auto GetUUID() const -> utils::UUID;
|
||||
|
||||
private:
|
||||
// TODO: (andi) I think variables below can be abstracted/clean them.
|
||||
io::network::Endpoint raft_endpoint_;
|
||||
uint32_t raft_server_id_;
|
||||
|
||||
ptr<CoordinatorStateMachine> state_machine_;
|
||||
ptr<CoordinatorStateManager> state_manager_;
|
||||
ptr<raft_server> raft_server_;
|
||||
ptr<logger> logger_;
|
||||
raft_launcher launcher_;
|
||||
|
||||
BecomeLeaderCb become_leader_cb_;
|
||||
BecomeFollowerCb become_follower_cb_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -19,34 +19,17 @@ namespace memgraph::coordination {
|
||||
|
||||
enum class RegisterInstanceCoordinatorStatus : uint8_t {
|
||||
NAME_EXISTS,
|
||||
COORD_ENDPOINT_EXISTS,
|
||||
REPL_ENDPOINT_EXISTS,
|
||||
ENDPOINT_EXISTS,
|
||||
NOT_COORDINATOR,
|
||||
NOT_LEADER,
|
||||
RPC_FAILED,
|
||||
RAFT_LOG_ERROR,
|
||||
SUCCESS
|
||||
};
|
||||
|
||||
enum class UnregisterInstanceCoordinatorStatus : uint8_t {
|
||||
NO_INSTANCE_WITH_NAME,
|
||||
IS_MAIN,
|
||||
NOT_COORDINATOR,
|
||||
RPC_FAILED,
|
||||
NOT_LEADER,
|
||||
RAFT_LOG_ERROR,
|
||||
SUCCESS,
|
||||
};
|
||||
|
||||
enum class SetInstanceToMainCoordinatorStatus : uint8_t {
|
||||
NO_INSTANCE_WITH_NAME,
|
||||
MAIN_ALREADY_EXISTS,
|
||||
NOT_COORDINATOR,
|
||||
NOT_LEADER,
|
||||
RAFT_LOG_ERROR,
|
||||
COULD_NOT_PROMOTE_TO_MAIN,
|
||||
SWAP_UUID_FAILED,
|
||||
SUCCESS,
|
||||
COULD_NOT_PROMOTE_TO_MAIN,
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
|
||||
#include "utils/resource_lock.hpp"
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
class ReplicationInstance;
|
||||
|
||||
using HealthCheckInstanceCallback = void (CoordinatorInstance::*)(std::string_view);
|
||||
|
||||
class ReplicationInstance {
|
||||
public:
|
||||
ReplicationInstance(CoordinatorInstance *peer, CoordinatorClientConfig config, HealthCheckClientCallback succ_cb,
|
||||
HealthCheckClientCallback fail_cb, HealthCheckInstanceCallback succ_instance_cb,
|
||||
HealthCheckInstanceCallback fail_instance_cb);
|
||||
|
||||
ReplicationInstance(ReplicationInstance const &other) = delete;
|
||||
ReplicationInstance &operator=(ReplicationInstance const &other) = delete;
|
||||
ReplicationInstance(ReplicationInstance &&other) noexcept = delete;
|
||||
ReplicationInstance &operator=(ReplicationInstance &&other) noexcept = delete;
|
||||
~ReplicationInstance() = default;
|
||||
|
||||
auto OnSuccessPing() -> void;
|
||||
auto OnFailPing() -> bool;
|
||||
auto IsReadyForUUIDPing() -> bool;
|
||||
|
||||
void UpdateReplicaLastResponseUUID();
|
||||
|
||||
auto IsAlive() const -> bool;
|
||||
|
||||
auto InstanceName() const -> std::string;
|
||||
auto CoordinatorSocketAddress() const -> std::string;
|
||||
auto ReplicationSocketAddress() const -> std::string;
|
||||
|
||||
auto PromoteToMain(utils::UUID const &uuid, ReplicationClientsInfo repl_clients_info,
|
||||
HealthCheckInstanceCallback main_succ_cb, HealthCheckInstanceCallback main_fail_cb) -> bool;
|
||||
|
||||
auto SendDemoteToReplicaRpc() -> bool;
|
||||
|
||||
auto DemoteToReplica(HealthCheckInstanceCallback replica_succ_cb, HealthCheckInstanceCallback replica_fail_cb)
|
||||
-> bool;
|
||||
|
||||
auto StartFrequentCheck() -> void;
|
||||
auto StopFrequentCheck() -> void;
|
||||
auto PauseFrequentCheck() -> void;
|
||||
auto ResumeFrequentCheck() -> void;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto EnsureReplicaHasCorrectMainUUID(utils::UUID const &curr_main_uuid) -> bool;
|
||||
|
||||
auto SendSwapAndUpdateUUID(utils::UUID const &new_main_uuid) -> bool;
|
||||
auto SendUnregisterReplicaRpc(std::string_view instance_name) -> bool;
|
||||
|
||||
auto SendGetInstanceUUID() -> utils::BasicResult<coordination::GetInstanceUUIDError, std::optional<utils::UUID>>;
|
||||
auto GetClient() -> CoordinatorClient &;
|
||||
|
||||
auto EnableWritingOnMain() -> bool;
|
||||
|
||||
auto SetNewMainUUID(utils::UUID const &main_uuid) -> void;
|
||||
auto ResetMainUUID() -> void;
|
||||
auto GetMainUUID() const -> std::optional<utils::UUID> const &;
|
||||
|
||||
auto GetSuccessCallback() -> HealthCheckInstanceCallback &;
|
||||
auto GetFailCallback() -> HealthCheckInstanceCallback &;
|
||||
|
||||
private:
|
||||
CoordinatorClient client_;
|
||||
std::chrono::system_clock::time_point last_response_time_{};
|
||||
bool is_alive_{false};
|
||||
std::chrono::system_clock::time_point last_check_of_uuid_{};
|
||||
|
||||
// for replica this is main uuid of current main
|
||||
// for "main" main this same as in CoordinatorData
|
||||
// it is set to nullopt when replica is down
|
||||
// TLDR; when replica is down and comes back up we reset uuid of main replica is listening to
|
||||
// so we need to send swap uuid again
|
||||
std::optional<utils::UUID> main_uuid_;
|
||||
|
||||
HealthCheckInstanceCallback succ_cb_;
|
||||
HealthCheckInstanceCallback fail_cb_;
|
||||
|
||||
friend bool operator==(ReplicationInstance const &first, ReplicationInstance const &second) {
|
||||
return first.client_ == second.client_ && first.last_response_time_ == second.last_response_time_ &&
|
||||
first.is_alive_ == second.is_alive_ && first.main_uuid_ == second.main_uuid_;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,92 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "nuraft/raft_log_action.hpp"
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
#include "utils/resource_lock.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
#include <range/v3/view.hpp>
|
||||
#include "json/json.hpp"
|
||||
|
||||
#include <map>
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
#include <variant>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using replication_coordination_glue::ReplicationRole;
|
||||
|
||||
struct InstanceState {
|
||||
CoordinatorClientConfig config;
|
||||
ReplicationRole status;
|
||||
|
||||
friend auto operator==(InstanceState const &lhs, InstanceState const &rhs) -> bool {
|
||||
return lhs.config == rhs.config && lhs.status == rhs.status;
|
||||
}
|
||||
};
|
||||
|
||||
void to_json(nlohmann::json &j, InstanceState const &instance_state);
|
||||
void from_json(nlohmann::json const &j, InstanceState &instance_state);
|
||||
|
||||
using TRaftLog = std::variant<CoordinatorClientConfig, std::string, utils::UUID>;
|
||||
|
||||
using nuraft::buffer;
|
||||
using nuraft::buffer_serializer;
|
||||
using nuraft::ptr;
|
||||
|
||||
class CoordinatorClusterState {
|
||||
public:
|
||||
CoordinatorClusterState() = default;
|
||||
explicit CoordinatorClusterState(std::map<std::string, InstanceState, std::less<>> instances);
|
||||
|
||||
CoordinatorClusterState(CoordinatorClusterState const &);
|
||||
CoordinatorClusterState &operator=(CoordinatorClusterState const &);
|
||||
|
||||
CoordinatorClusterState(CoordinatorClusterState &&other) noexcept;
|
||||
CoordinatorClusterState &operator=(CoordinatorClusterState &&other) noexcept;
|
||||
~CoordinatorClusterState() = default;
|
||||
|
||||
auto FindCurrentMainInstanceName() const -> std::optional<std::string>;
|
||||
|
||||
auto MainExists() const -> bool;
|
||||
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto InsertInstance(std::string instance_name, InstanceState instance_state) -> void;
|
||||
|
||||
auto DoAction(TRaftLog log_entry, RaftLogAction log_action) -> void;
|
||||
|
||||
auto Serialize(ptr<buffer> &data) -> void;
|
||||
|
||||
static auto Deserialize(buffer &data) -> CoordinatorClusterState;
|
||||
|
||||
auto GetInstances() const -> std::vector<InstanceState>;
|
||||
|
||||
auto GetUUID() const -> utils::UUID;
|
||||
|
||||
private:
|
||||
std::map<std::string, InstanceState, std::less<>> instances_{};
|
||||
utils::UUID uuid_{};
|
||||
mutable utils::ResourceLock log_lock_{};
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,70 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::buffer;
|
||||
using nuraft::int32;
|
||||
using nuraft::int64;
|
||||
using nuraft::log_entry;
|
||||
using nuraft::log_store;
|
||||
using nuraft::ptr;
|
||||
using nuraft::raft_server;
|
||||
|
||||
class CoordinatorLogStore : public log_store {
|
||||
public:
|
||||
CoordinatorLogStore();
|
||||
CoordinatorLogStore(CoordinatorLogStore const &) = delete;
|
||||
CoordinatorLogStore &operator=(CoordinatorLogStore const &) = delete;
|
||||
CoordinatorLogStore(CoordinatorLogStore &&) = delete;
|
||||
CoordinatorLogStore &operator=(CoordinatorLogStore &&) = delete;
|
||||
~CoordinatorLogStore() override;
|
||||
|
||||
ulong next_slot() const override;
|
||||
|
||||
ulong start_index() const override;
|
||||
|
||||
ptr<log_entry> last_entry() const override;
|
||||
|
||||
ulong append(ptr<log_entry> &entry) override;
|
||||
|
||||
void write_at(ulong index, ptr<log_entry> &entry) override;
|
||||
|
||||
ptr<std::vector<ptr<log_entry>>> log_entries(ulong start, ulong end) override;
|
||||
|
||||
ptr<log_entry> entry_at(ulong index) override;
|
||||
|
||||
ulong term_at(ulong index) override;
|
||||
|
||||
ptr<buffer> pack(ulong index, int32 cnt) override;
|
||||
|
||||
void apply_pack(ulong index, buffer &pack) override;
|
||||
|
||||
bool compact(ulong last_log_index) override;
|
||||
|
||||
bool flush() override;
|
||||
|
||||
private:
|
||||
auto FindOrDefault_(ulong index) const -> ptr<log_entry>;
|
||||
|
||||
std::map<ulong, ptr<log_entry>> logs_;
|
||||
mutable std::mutex logs_lock_;
|
||||
std::atomic<ulong> start_idx_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,108 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "nuraft/coordinator_cluster_state.hpp"
|
||||
#include "nuraft/raft_log_action.hpp"
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
#include <variant>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::async_result;
|
||||
using nuraft::buffer;
|
||||
using nuraft::buffer_serializer;
|
||||
using nuraft::cluster_config;
|
||||
using nuraft::int32;
|
||||
using nuraft::ptr;
|
||||
using nuraft::snapshot;
|
||||
using nuraft::state_machine;
|
||||
|
||||
class CoordinatorStateMachine : public state_machine {
|
||||
public:
|
||||
CoordinatorStateMachine() = default;
|
||||
CoordinatorStateMachine(CoordinatorStateMachine const &) = delete;
|
||||
CoordinatorStateMachine &operator=(CoordinatorStateMachine const &) = delete;
|
||||
CoordinatorStateMachine(CoordinatorStateMachine &&) = delete;
|
||||
CoordinatorStateMachine &operator=(CoordinatorStateMachine &&) = delete;
|
||||
~CoordinatorStateMachine() override {}
|
||||
|
||||
auto FindCurrentMainInstanceName() const -> std::optional<std::string>;
|
||||
auto MainExists() const -> bool;
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
static auto CreateLog(nlohmann::json &&log) -> ptr<buffer>;
|
||||
static auto SerializeRegisterInstance(CoordinatorClientConfig const &config) -> ptr<buffer>;
|
||||
static auto SerializeUnregisterInstance(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeSetInstanceAsMain(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeSetInstanceAsReplica(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeUpdateUUID(utils::UUID const &uuid) -> ptr<buffer>;
|
||||
|
||||
static auto DecodeLog(buffer &data) -> std::pair<TRaftLog, RaftLogAction>;
|
||||
|
||||
auto pre_commit(ulong log_idx, buffer &data) -> ptr<buffer> override;
|
||||
|
||||
auto commit(ulong log_idx, buffer &data) -> ptr<buffer> override;
|
||||
|
||||
auto commit_config(ulong log_idx, ptr<cluster_config> & /*new_conf*/) -> void override;
|
||||
|
||||
auto rollback(ulong log_idx, buffer &data) -> void override;
|
||||
|
||||
auto read_logical_snp_obj(snapshot & /*snapshot*/, void *& /*user_snp_ctx*/, ulong /*obj_id*/, ptr<buffer> &data_out,
|
||||
bool &is_last_obj) -> int override;
|
||||
|
||||
auto save_logical_snp_obj(snapshot &s, ulong &obj_id, buffer & /*data*/, bool /*is_first_obj*/, bool /*is_last_obj*/)
|
||||
-> void override;
|
||||
|
||||
auto apply_snapshot(snapshot &s) -> bool override;
|
||||
|
||||
auto free_user_snp_ctx(void *&user_snp_ctx) -> void override;
|
||||
|
||||
auto last_snapshot() -> ptr<snapshot> override;
|
||||
|
||||
auto last_commit_index() -> ulong override;
|
||||
|
||||
auto create_snapshot(snapshot &s, async_result<bool>::handler_type &when_done) -> void override;
|
||||
|
||||
auto GetInstances() const -> std::vector<InstanceState>;
|
||||
auto GetUUID() const -> utils::UUID;
|
||||
|
||||
private:
|
||||
struct SnapshotCtx {
|
||||
SnapshotCtx(ptr<snapshot> &snapshot, CoordinatorClusterState const &cluster_state)
|
||||
: snapshot_(snapshot), cluster_state_(cluster_state) {}
|
||||
|
||||
ptr<snapshot> snapshot_;
|
||||
CoordinatorClusterState cluster_state_;
|
||||
};
|
||||
|
||||
auto create_snapshot_internal(ptr<snapshot> snapshot) -> void;
|
||||
|
||||
CoordinatorClusterState cluster_state_;
|
||||
std::atomic<uint64_t> last_committed_idx_{0};
|
||||
|
||||
std::map<uint64_t, ptr<SnapshotCtx>> snapshots_;
|
||||
std::mutex snapshots_lock_;
|
||||
|
||||
ptr<snapshot> last_snapshot_;
|
||||
std::mutex last_snapshot_lock_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,66 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_log_store.hpp"
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::cluster_config;
|
||||
using nuraft::cs_new;
|
||||
using nuraft::srv_config;
|
||||
using nuraft::srv_state;
|
||||
using nuraft::state_mgr;
|
||||
|
||||
class CoordinatorStateManager : public state_mgr {
|
||||
public:
|
||||
explicit CoordinatorStateManager(int srv_id, std::string const &endpoint);
|
||||
|
||||
CoordinatorStateManager(CoordinatorStateManager const &) = delete;
|
||||
CoordinatorStateManager &operator=(CoordinatorStateManager const &) = delete;
|
||||
CoordinatorStateManager(CoordinatorStateManager &&) = delete;
|
||||
CoordinatorStateManager &operator=(CoordinatorStateManager &&) = delete;
|
||||
|
||||
~CoordinatorStateManager() override = default;
|
||||
|
||||
auto load_config() -> ptr<cluster_config> override;
|
||||
|
||||
auto save_config(cluster_config const &config) -> void override;
|
||||
|
||||
auto save_state(srv_state const &state) -> void override;
|
||||
|
||||
auto read_state() -> ptr<srv_state> override;
|
||||
|
||||
auto load_log_store() -> ptr<log_store> override;
|
||||
|
||||
auto server_id() -> int32 override;
|
||||
|
||||
auto system_exit(int exit_code) -> void override;
|
||||
|
||||
auto GetSrvConfig() const -> ptr<srv_config>;
|
||||
|
||||
private:
|
||||
int my_id_;
|
||||
std::string my_endpoint_;
|
||||
ptr<CoordinatorLogStore> cur_log_store_;
|
||||
ptr<srv_config> my_srv_config_;
|
||||
ptr<cluster_config> cluster_config_;
|
||||
ptr<srv_state> saved_state_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,42 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
#include "json/json.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
enum class RaftLogAction : uint8_t {
|
||||
REGISTER_REPLICATION_INSTANCE,
|
||||
UNREGISTER_REPLICATION_INSTANCE,
|
||||
SET_INSTANCE_AS_MAIN,
|
||||
SET_INSTANCE_AS_REPLICA,
|
||||
UPDATE_UUID
|
||||
};
|
||||
|
||||
NLOHMANN_JSON_SERIALIZE_ENUM(RaftLogAction, {
|
||||
{RaftLogAction::REGISTER_REPLICATION_INSTANCE, "register"},
|
||||
{RaftLogAction::UNREGISTER_REPLICATION_INSTANCE, "unregister"},
|
||||
{RaftLogAction::SET_INSTANCE_AS_MAIN, "promote"},
|
||||
{RaftLogAction::SET_INSTANCE_AS_REPLICA, "demote"},
|
||||
{RaftLogAction::UPDATE_UUID, "update_uuid"},
|
||||
})
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,279 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
#include <chrono>
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "coordination/raft_state.hpp"
|
||||
#include "utils/counter.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::asio_service;
|
||||
using nuraft::cb_func;
|
||||
using nuraft::CbReturnCode;
|
||||
using nuraft::cmd_result;
|
||||
using nuraft::cs_new;
|
||||
using nuraft::ptr;
|
||||
using nuraft::raft_params;
|
||||
using nuraft::raft_server;
|
||||
using nuraft::srv_config;
|
||||
using raft_result = cmd_result<ptr<buffer>>;
|
||||
|
||||
RaftState::RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t raft_server_id,
|
||||
uint32_t raft_port, std::string raft_address)
|
||||
: raft_endpoint_(raft_address, raft_port),
|
||||
raft_server_id_(raft_server_id),
|
||||
state_machine_(cs_new<CoordinatorStateMachine>()),
|
||||
state_manager_(cs_new<CoordinatorStateManager>(raft_server_id_, raft_endpoint_.SocketAddress())),
|
||||
logger_(nullptr),
|
||||
become_leader_cb_(std::move(become_leader_cb)),
|
||||
become_follower_cb_(std::move(become_follower_cb)) {}
|
||||
|
||||
auto RaftState::InitRaftServer() -> void {
|
||||
asio_service::options asio_opts;
|
||||
asio_opts.thread_pool_size_ = 1;
|
||||
|
||||
raft_params params;
|
||||
params.heart_beat_interval_ = 100;
|
||||
params.election_timeout_lower_bound_ = 200;
|
||||
params.election_timeout_upper_bound_ = 400;
|
||||
params.reserved_log_items_ = 5;
|
||||
params.snapshot_distance_ = 5;
|
||||
params.client_req_timeout_ = 3000;
|
||||
params.return_method_ = raft_params::blocking;
|
||||
|
||||
// If the leader doesn't receive any response from quorum nodes
|
||||
// in 200ms, it will step down.
|
||||
// This allows us to achieve strong consistency even if network partition
|
||||
// happens between the current leader and followers.
|
||||
// The value must be <= election_timeout_lower_bound_ so that cluster can never
|
||||
// have multiple leaders.
|
||||
params.leadership_expiry_ = 200;
|
||||
|
||||
raft_server::init_options init_opts;
|
||||
init_opts.raft_callback_ = [this](cb_func::Type event_type, cb_func::Param *param) -> nuraft::CbReturnCode {
|
||||
if (event_type == cb_func::BecomeLeader) {
|
||||
spdlog::info("Node {} became leader", param->leaderId);
|
||||
become_leader_cb_();
|
||||
} else if (event_type == cb_func::BecomeFollower) {
|
||||
spdlog::info("Node {} became follower", param->myId);
|
||||
become_follower_cb_();
|
||||
}
|
||||
return CbReturnCode::Ok;
|
||||
};
|
||||
|
||||
raft_launcher launcher;
|
||||
|
||||
raft_server_ =
|
||||
launcher.init(state_machine_, state_manager_, logger_, raft_endpoint_.port, asio_opts, params, init_opts);
|
||||
|
||||
if (!raft_server_) {
|
||||
throw RaftServerStartException("Failed to launch raft server on {}", raft_endpoint_.SocketAddress());
|
||||
}
|
||||
|
||||
auto maybe_stop = utils::ResettableCounter<20>();
|
||||
do {
|
||||
if (raft_server_->is_initialized()) {
|
||||
return;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(250));
|
||||
} while (!maybe_stop());
|
||||
|
||||
throw RaftServerStartException("Failed to initialize raft server on {}", raft_endpoint_.SocketAddress());
|
||||
}
|
||||
|
||||
auto RaftState::MakeRaftState(BecomeLeaderCb &&become_leader_cb, BecomeFollowerCb &&become_follower_cb) -> RaftState {
|
||||
uint32_t raft_server_id = FLAGS_raft_server_id;
|
||||
uint32_t raft_port = FLAGS_raft_server_port;
|
||||
|
||||
auto raft_state =
|
||||
RaftState(std::move(become_leader_cb), std::move(become_follower_cb), raft_server_id, raft_port, "127.0.0.1");
|
||||
|
||||
raft_state.InitRaftServer();
|
||||
return raft_state;
|
||||
}
|
||||
|
||||
RaftState::~RaftState() { launcher_.shutdown(); }
|
||||
|
||||
auto RaftState::InstanceName() const -> std::string {
|
||||
return fmt::format("coordinator_{}", std::to_string(raft_server_id_));
|
||||
}
|
||||
|
||||
auto RaftState::RaftSocketAddress() const -> std::string { return raft_endpoint_.SocketAddress(); }
|
||||
|
||||
auto RaftState::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address)
|
||||
-> void {
|
||||
auto const endpoint = fmt::format("{}:{}", raft_address, raft_port);
|
||||
srv_config const srv_config_to_add(static_cast<int>(raft_server_id), endpoint);
|
||||
|
||||
auto cmd_result = raft_server_->add_srv(srv_config_to_add);
|
||||
|
||||
if (cmd_result->get_result_code() == nuraft::cmd_result_code::OK) {
|
||||
spdlog::info("Request to add server {} to the cluster accepted", endpoint);
|
||||
} else {
|
||||
throw RaftAddServerException("Failed to accept request to add server {} to the cluster with error code {}",
|
||||
endpoint, cmd_result->get_result_code());
|
||||
}
|
||||
|
||||
// Waiting for server to join
|
||||
constexpr int max_tries{10};
|
||||
auto maybe_stop = utils::ResettableCounter<max_tries>();
|
||||
constexpr int waiting_period{200};
|
||||
bool added{false};
|
||||
while (!maybe_stop()) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(waiting_period));
|
||||
const auto server_config = raft_server_->get_srv_config(static_cast<nuraft::int32>(raft_server_id));
|
||||
if (server_config) {
|
||||
spdlog::trace("Server with id {} added to cluster", raft_server_id);
|
||||
added = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!added) {
|
||||
throw RaftAddServerException("Failed to add server {} to the cluster in {}ms", endpoint,
|
||||
max_tries * waiting_period);
|
||||
}
|
||||
}
|
||||
|
||||
auto RaftState::GetAllCoordinators() const -> std::vector<ptr<srv_config>> {
|
||||
std::vector<ptr<srv_config>> all_srv_configs;
|
||||
raft_server_->get_srv_config_all(all_srv_configs);
|
||||
return all_srv_configs;
|
||||
}
|
||||
|
||||
auto RaftState::IsLeader() const -> bool { return raft_server_->is_leader(); }
|
||||
|
||||
auto RaftState::RequestLeadership() -> bool { return raft_server_->is_leader() || raft_server_->request_leadership(); }
|
||||
|
||||
auto RaftState::AppendRegisterReplicationInstanceLog(CoordinatorClientConfig const &config) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeRegisterInstance(config);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for registering instance {}. Most likely the reason is that the instance is not "
|
||||
"the "
|
||||
"leader.",
|
||||
config.instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
spdlog::info("Request for registering instance {} accepted", config.instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to register instance {} with error code {}", config.instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendUnregisterReplicationInstanceLog(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeUnregisterInstance(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for unregistering instance {}. Most likely the reason is that the instance is not "
|
||||
"the leader.",
|
||||
instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
spdlog::info("Request for unregistering instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to unregister instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendSetInstanceAsMainLog(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeSetInstanceAsMain(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for promoting instance {}. Most likely the reason is that the instance is not "
|
||||
"the leader.",
|
||||
instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
spdlog::info("Request for promoting instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendSetInstanceAsReplicaLog(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeSetInstanceAsReplica(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for demoting instance {}. Most likely the reason is that the instance is not "
|
||||
"the leader.",
|
||||
instance_name);
|
||||
return false;
|
||||
}
|
||||
spdlog::info("Request for demoting instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendUpdateUUIDLog(utils::UUID const &uuid) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeUpdateUUID(uuid);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for updating UUID. Most likely the reason is that the instance is not "
|
||||
"the leader.");
|
||||
return false;
|
||||
}
|
||||
spdlog::info("Request for updating UUID accepted");
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to update UUID with error code {}", res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::FindCurrentMainInstanceName() const -> std::optional<std::string> {
|
||||
return state_machine_->FindCurrentMainInstanceName();
|
||||
}
|
||||
|
||||
auto RaftState::MainExists() const -> bool { return state_machine_->MainExists(); }
|
||||
|
||||
auto RaftState::IsMain(std::string_view instance_name) const -> bool { return state_machine_->IsMain(instance_name); }
|
||||
|
||||
auto RaftState::IsReplica(std::string_view instance_name) const -> bool {
|
||||
return state_machine_->IsReplica(instance_name);
|
||||
}
|
||||
|
||||
auto RaftState::GetInstances() const -> std::vector<InstanceState> { return state_machine_->GetInstances(); }
|
||||
|
||||
auto RaftState::GetUUID() const -> utils::UUID { return state_machine_->GetUUID(); }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -1,137 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/replication_instance.hpp"
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
ReplicationInstance::ReplicationInstance(CoordinatorInstance *peer, CoordinatorClientConfig config,
|
||||
HealthCheckClientCallback succ_cb, HealthCheckClientCallback fail_cb,
|
||||
HealthCheckInstanceCallback succ_instance_cb,
|
||||
HealthCheckInstanceCallback fail_instance_cb)
|
||||
: client_(peer, std::move(config), std::move(succ_cb), std::move(fail_cb)),
|
||||
succ_cb_(succ_instance_cb),
|
||||
fail_cb_(fail_instance_cb) {}
|
||||
|
||||
auto ReplicationInstance::OnSuccessPing() -> void {
|
||||
last_response_time_ = std::chrono::system_clock::now();
|
||||
is_alive_ = true;
|
||||
}
|
||||
|
||||
auto ReplicationInstance::OnFailPing() -> bool {
|
||||
auto elapsed_time = std::chrono::system_clock::now() - last_response_time_;
|
||||
is_alive_ = elapsed_time < client_.InstanceDownTimeoutSec();
|
||||
return is_alive_;
|
||||
}
|
||||
|
||||
auto ReplicationInstance::IsReadyForUUIDPing() -> bool {
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - last_check_of_uuid_) >
|
||||
client_.InstanceGetUUIDFrequencySec();
|
||||
}
|
||||
|
||||
auto ReplicationInstance::InstanceName() const -> std::string { return client_.InstanceName(); }
|
||||
auto ReplicationInstance::CoordinatorSocketAddress() const -> std::string { return client_.CoordinatorSocketAddress(); }
|
||||
auto ReplicationInstance::ReplicationSocketAddress() const -> std::string { return client_.ReplicationSocketAddress(); }
|
||||
auto ReplicationInstance::IsAlive() const -> bool { return is_alive_; }
|
||||
|
||||
auto ReplicationInstance::PromoteToMain(utils::UUID const &new_uuid, ReplicationClientsInfo repl_clients_info,
|
||||
HealthCheckInstanceCallback main_succ_cb,
|
||||
HealthCheckInstanceCallback main_fail_cb) -> bool {
|
||||
if (!client_.SendPromoteReplicaToMainRpc(new_uuid, std::move(repl_clients_info))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
main_uuid_ = new_uuid;
|
||||
succ_cb_ = main_succ_cb;
|
||||
fail_cb_ = main_fail_cb;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto ReplicationInstance::SendDemoteToReplicaRpc() -> bool { return client_.DemoteToReplica(); }
|
||||
|
||||
auto ReplicationInstance::DemoteToReplica(HealthCheckInstanceCallback replica_succ_cb,
|
||||
HealthCheckInstanceCallback replica_fail_cb) -> bool {
|
||||
if (!client_.DemoteToReplica()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
succ_cb_ = replica_succ_cb;
|
||||
fail_cb_ = replica_fail_cb;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto ReplicationInstance::StartFrequentCheck() -> void { client_.StartFrequentCheck(); }
|
||||
auto ReplicationInstance::StopFrequentCheck() -> void { client_.StopFrequentCheck(); }
|
||||
auto ReplicationInstance::PauseFrequentCheck() -> void { client_.PauseFrequentCheck(); }
|
||||
auto ReplicationInstance::ResumeFrequentCheck() -> void { client_.ResumeFrequentCheck(); }
|
||||
|
||||
auto ReplicationInstance::ReplicationClientInfo() const -> CoordinatorClientConfig::ReplicationClientInfo {
|
||||
return client_.ReplicationClientInfo();
|
||||
}
|
||||
|
||||
auto ReplicationInstance::GetSuccessCallback() -> HealthCheckInstanceCallback & { return succ_cb_; }
|
||||
auto ReplicationInstance::GetFailCallback() -> HealthCheckInstanceCallback & { return fail_cb_; }
|
||||
|
||||
auto ReplicationInstance::GetClient() -> CoordinatorClient & { return client_; }
|
||||
|
||||
auto ReplicationInstance::SetNewMainUUID(utils::UUID const &main_uuid) -> void { main_uuid_ = main_uuid; }
|
||||
auto ReplicationInstance::GetMainUUID() const -> std::optional<utils::UUID> const & { return main_uuid_; }
|
||||
|
||||
auto ReplicationInstance::EnsureReplicaHasCorrectMainUUID(utils::UUID const &curr_main_uuid) -> bool {
|
||||
if (!IsReadyForUUIDPing()) {
|
||||
return true;
|
||||
}
|
||||
auto res = SendGetInstanceUUID();
|
||||
if (res.HasError()) {
|
||||
return false;
|
||||
}
|
||||
UpdateReplicaLastResponseUUID();
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
if (res.GetValue().has_value() && res.GetValue().value() == curr_main_uuid) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return SendSwapAndUpdateUUID(curr_main_uuid);
|
||||
}
|
||||
|
||||
auto ReplicationInstance::SendSwapAndUpdateUUID(utils::UUID const &new_main_uuid) -> bool {
|
||||
if (!replication_coordination_glue::SendSwapMainUUIDRpc(client_.RpcClient(), new_main_uuid)) {
|
||||
return false;
|
||||
}
|
||||
SetNewMainUUID(new_main_uuid);
|
||||
return true;
|
||||
}
|
||||
|
||||
auto ReplicationInstance::SendUnregisterReplicaRpc(std::string_view instance_name) -> bool {
|
||||
return client_.SendUnregisterReplicaRpc(instance_name);
|
||||
}
|
||||
|
||||
auto ReplicationInstance::EnableWritingOnMain() -> bool { return client_.SendEnableWritingOnMainRpc(); }
|
||||
|
||||
auto ReplicationInstance::SendGetInstanceUUID()
|
||||
-> utils::BasicResult<coordination::GetInstanceUUIDError, std::optional<utils::UUID>> {
|
||||
return client_.SendGetInstanceUUIDRpc();
|
||||
}
|
||||
|
||||
void ReplicationInstance::UpdateReplicaLastResponseUUID() { last_check_of_uuid_ = std::chrono::system_clock::now(); }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -16,4 +16,10 @@ namespace memgraph::dbms {
|
||||
constexpr std::string_view kDefaultDB = "memgraph"; //!< Name of the default database
|
||||
constexpr std::string_view kMultiTenantDir = "databases"; //!< Name of the multi-tenant directory
|
||||
|
||||
#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY
|
||||
constexpr bool allow_mt_repl = true;
|
||||
#else
|
||||
constexpr bool allow_mt_repl = false;
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user