Compare commits
137 Commits
add-new-os
...
cpp23
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c66eb5c207 | ||
|
|
47cdca4f6e | ||
|
|
bae3e8a6d3 | ||
|
|
f3574012c5 | ||
|
|
33c400fcc1 | ||
|
|
5ac938a6c9 | ||
|
|
3e3224f0a2 | ||
|
|
bfc756c092 | ||
|
|
5f2e3f01d0 | ||
|
|
2c774ff09b | ||
|
|
20b47845f0 | ||
|
|
fb281459b9 | ||
|
|
3a7e62f72c | ||
|
|
f48151576b | ||
|
|
4a7c7f0898 | ||
|
|
7688a1b068 | ||
|
|
4f4a569c72 | ||
|
|
a511e63c7a | ||
|
|
0133673f1d | ||
|
|
786cdea260 | ||
|
|
54f78f9217 | ||
|
|
dcdbd0a19a | ||
|
|
cf80687d1d | ||
|
|
2fa8e00124 | ||
|
|
c15b62a88d | ||
|
|
4ef6a1f9c3 | ||
|
|
7ead00f23e | ||
|
|
c46dad18fe | ||
|
|
cb7b88ad92 | ||
|
|
b443934b68 | ||
|
|
6ab4235cc9 | ||
|
|
a9ef28c68e | ||
|
|
79361e9205 | ||
|
|
97b1e67d80 | ||
|
|
78a88737f8 | ||
|
|
ff44d68843 | ||
|
|
f1484240a0 | ||
|
|
1c95c3dc59 | ||
|
|
9f7118d893 | ||
|
|
38ade99652 | ||
|
|
6706ebfa2b | ||
|
|
e7f6a5f4f4 | ||
|
|
071df2f439 | ||
|
|
7f10636470 | ||
|
|
76589903a4 | ||
|
|
a8b625d861 | ||
|
|
9c89fce249 | ||
|
|
5e5f215be4 | ||
|
|
23dff58d22 | ||
|
|
0a7a7bc0d1 | ||
|
|
c772cab766 | ||
|
|
2e4d27c59a | ||
|
|
31f15b3651 | ||
|
|
b3d0c2ccc2 | ||
|
|
d4bcdb77ad | ||
|
|
1ba2f4e619 | ||
|
|
4bb9238679 | ||
|
|
bd11266f82 | ||
|
|
57e40a2b18 | ||
|
|
0fb8e4116f | ||
|
|
7128e1cea8 | ||
|
|
4788a633a6 | ||
|
|
ce2705d012 | ||
|
|
686fadf072 | ||
|
|
9e76021b94 | ||
|
|
cd37de481e | ||
|
|
f11b3c6d9d | ||
|
|
4ef86efb6f | ||
|
|
04fb92dce8 | ||
|
|
71e76cc980 | ||
|
|
cb4d4db813 | ||
|
|
39ee248d34 | ||
|
|
b35df12c1a | ||
|
|
21bbc196ae | ||
|
|
375c3c5ddd | ||
|
|
340057f959 | ||
|
|
e56e516f94 | ||
|
|
7a9c4f5ec4 | ||
|
|
eceed274d9 | ||
|
|
74fa6d21f6 | ||
|
|
d836b38a8b | ||
|
|
eeb9671bac | ||
|
|
e716c90031 | ||
|
|
9690682bc2 | ||
|
|
64e5428d94 | ||
|
|
66e86c060f | ||
|
|
953a8f5340 | ||
|
|
31efe28878 | ||
|
|
0fb3ae2d56 | ||
|
|
46bfeb0023 | ||
|
|
d58a464141 | ||
|
|
997779fe07 | ||
|
|
3ccd78ac71 | ||
|
|
14f92b4a0f | ||
|
|
9f555cf93d | ||
|
|
7fc9b89634 | ||
|
|
ed5bdb841f | ||
|
|
b74aee186e | ||
|
|
ac0c4193b0 | ||
|
|
9868dee73b | ||
|
|
bb2a7b8f21 | ||
|
|
6680f99e4d | ||
|
|
72d47fc3bf | ||
|
|
3cf13e2deb | ||
|
|
7f5a55f1b2 | ||
|
|
08acde3973 | ||
|
|
70db2fca56 | ||
|
|
8b9e1fa08b | ||
|
|
e4f94c15c6 | ||
|
|
0ce6dcf194 | ||
|
|
1d90b60f56 | ||
|
|
108d83cc8c | ||
|
|
eff857447a | ||
|
|
d03fafcef6 | ||
|
|
c31a7f9648 | ||
|
|
6053a91ef8 | ||
|
|
645568a75b | ||
|
|
d3f4c35362 | ||
|
|
c037cddb0e | ||
|
|
ced08fd7bc | ||
|
|
a370b09d12 | ||
|
|
1527bdf435 | ||
|
|
9cc060c4b0 | ||
|
|
e671a0737e | ||
|
|
11be3972c4 | ||
|
|
fdab42a023 | ||
|
|
e5b2c19ea2 | ||
|
|
e907817854 | ||
|
|
0756cd6898 | ||
|
|
38ad5e2146 | ||
|
|
3c413a7e50 | ||
|
|
df3274d78f | ||
|
|
2946d74fdd | ||
|
|
eb4ebab438 | ||
|
|
260d60451d | ||
|
|
c4d9116c9c | ||
|
|
50c485fe40 |
@@ -2,7 +2,7 @@
|
||||
BasedOnStyle: Google
|
||||
---
|
||||
Language: Cpp
|
||||
Standard: "c++20"
|
||||
Standard: c++20
|
||||
UseTab: Never
|
||||
DerivePointerAlignment: false
|
||||
PointerAlignment: Right
|
||||
|
||||
@@ -6,6 +6,7 @@ Checks: '*,
|
||||
-altera-unroll-loops,
|
||||
-android-*,
|
||||
-cert-err58-cpp,
|
||||
-cppcoreguidelines-avoid-do-while,
|
||||
-cppcoreguidelines-avoid-c-arrays,
|
||||
-cppcoreguidelines-avoid-goto,
|
||||
-cppcoreguidelines-avoid-magic-numbers,
|
||||
@@ -60,6 +61,7 @@ Checks: '*,
|
||||
-readability-implicit-bool-conversion,
|
||||
-readability-magic-numbers,
|
||||
-readability-named-parameter,
|
||||
-readability-identifier-length,
|
||||
-misc-no-recursion,
|
||||
-concurrency-mt-unsafe,
|
||||
-bugprone-easily-swappable-parameters'
|
||||
|
||||
1
.github/ISSUE_TEMPLATE/bug_report.md
vendored
1
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@@ -3,7 +3,6 @@ name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ""
|
||||
labels: bug
|
||||
assignees: gitbuda
|
||||
---
|
||||
|
||||
**Memgraph version**
|
||||
|
||||
2
.github/workflows/daily_benchmark.yaml
vendored
2
.github/workflows/daily_benchmark.yaml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
102
.github/workflows/diff.yaml
vendored
102
.github/workflows/diff.yaml
vendored
@@ -27,7 +27,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -96,7 +96,7 @@ jobs:
|
||||
|
||||
- name: Python code analysis
|
||||
run: |
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only --diff-filter=d)
|
||||
for file in ${CHANGED_FILES}; do
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
@@ -137,9 +137,9 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage"
|
||||
name: "Code coverage(Code analysis)"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
- name: Run clang-tidy
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -208,9 +208,9 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage"
|
||||
name: "Code coverage(Debug build)"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
release_build:
|
||||
@@ -223,16 +223,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Check e2e service dependencies
|
||||
run: |
|
||||
cd tests/e2e
|
||||
./dependency_check.sh
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
@@ -255,7 +251,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
@@ -271,6 +267,13 @@ jobs:
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
cd tests
|
||||
@@ -279,6 +282,13 @@ jobs:
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
@@ -312,16 +322,63 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
name: "Test data(Release build)"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
experimental_build_ha:
|
||||
name: "High availability build"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
./init
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
|
||||
make -j$THREADS
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
- name: Run e2e tests
|
||||
if: false
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh "Coordinator"
|
||||
./run.sh "Client initiated failover"
|
||||
./run.sh "Uninitialized cluster"
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data(High availability build)"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
@@ -337,7 +394,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -354,13 +411,18 @@ jobs:
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Refresh Jepsen Cluster
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh cluster-refresh
|
||||
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
@@ -376,7 +438,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
2
.github/workflows/full_clang_tidy.yaml
vendored
2
.github/workflows/full_clang_tidy.yaml
vendored
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
name: Package All
|
||||
name: Package memgraph
|
||||
|
||||
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
|
||||
|
||||
@@ -6,91 +6,169 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
memgraph_version:
|
||||
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
|
||||
description: "Memgraph version to upload as. Leave this field empty if you don't want to upload binaries to S3. Format: 'X.Y.Z'"
|
||||
required: false
|
||||
build_type:
|
||||
type: choice
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
description: "Memgraph Build type. Default value is Release"
|
||||
default: 'Release'
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
target_os:
|
||||
type: choice
|
||||
description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS. Default is Ubuntu 22.04"
|
||||
default: 'ubuntu-22_04'
|
||||
options:
|
||||
- all
|
||||
- amzn-2
|
||||
- centos-7
|
||||
- centos-9
|
||||
- debian-10
|
||||
- debian-11
|
||||
- debian-11-arm
|
||||
- debian-11-platform
|
||||
- docker
|
||||
- fedora-36
|
||||
- ubuntu-18_04
|
||||
- ubuntu-20_04
|
||||
- ubuntu-22_04
|
||||
- ubuntu-22_04-arm
|
||||
|
||||
jobs:
|
||||
centos-7:
|
||||
amzn-2:
|
||||
if: ${{ github.event.inputs.target_os == 'amzn-2' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: amzn-2
|
||||
path: build/output/amzn-2/memgraph*.rpm
|
||||
|
||||
centos-7:
|
||||
if: ${{ github.event.inputs.target_os == 'centos-7' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: centos-7
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
centos-9:
|
||||
if: ${{ github.event.inputs.target_os == 'centos-9' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: centos-9
|
||||
path: build/output/centos-9/memgraph*.rpm
|
||||
|
||||
debian-10:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-10
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
debian-11:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
docker:
|
||||
debian-11-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11-platform' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
docker:
|
||||
if: ${{ github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -99,143 +177,97 @@ jobs:
|
||||
./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker
|
||||
./run.sh docker
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: docker
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
ubuntu-1804:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-18.04
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-20.04
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
ubuntu-2204:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
fedora-36:
|
||||
if: ${{ github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fedora-36
|
||||
path: build/output/fedora-36/memgraph*.rpm
|
||||
|
||||
amzn-2:
|
||||
ubuntu-18_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
|
||||
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: amzn-2
|
||||
path: build/output/amzn-2/memgraph*.rpm
|
||||
name: ubuntu-18.04
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
debian-11-arm:
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
ubuntu-20_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
|
||||
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
name: ubuntu-20.04
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
ubuntu-2204-arm:
|
||||
ubuntu-22_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
ubuntu-22_04-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04-aarch64
|
||||
path: build/output/ubuntu-22.04-arm/memgraph*.deb
|
||||
@@ -243,11 +275,11 @@ jobs:
|
||||
upload-to-s3:
|
||||
# only run upload if we specified version. Allows for runs without upload
|
||||
if: "${{ github.event.inputs.memgraph_version != '' }}"
|
||||
needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm]
|
||||
needs: [amzn-2, centos-7, centos-9, debian-10, debian-11, debian-11-arm, debian-11-platform, docker, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04, ubuntu-22_04-arm]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
# name: # if name input parameter is not provided, all artifacts are downloaded
|
||||
# and put in directories named after each one.
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
318
.github/workflows/release_centos8.yaml
vendored
318
.github/workflows/release_centos8.yaml
vendored
@@ -1,318 +0,0 @@
|
||||
name: Release CentOS 8
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
type: choice
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, Linux, X64, CentOS8]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 960
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build community binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build community binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
|
||||
coverage_build:
|
||||
name: "Coverage build"
|
||||
runs-on: [self-hosted, Linux, X64, CentOS8]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build coverage binaries.
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
|
||||
- name: Compute code coverage
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Compute code coverage.
|
||||
cd tools/github
|
||||
./coverage_convert
|
||||
|
||||
# Package code coverage.
|
||||
cd generated
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, Linux, X64, CentOS8]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build debug binaries.
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run leftover CTest tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run leftover CTest tests (all except unit and benchmark tests).
|
||||
cd build
|
||||
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
|
||||
|
||||
- name: Run drivers tests
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run cppcheck and clang-format.
|
||||
cd tools/github
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, Linux, X64, CentOS8]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 960
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Create enterprise RPM package
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
cd build
|
||||
|
||||
# create mgconsole
|
||||
# we use the -B to force the build
|
||||
make -j$THREADS -B mgconsole
|
||||
|
||||
# Create enterprise RPM package.
|
||||
mkdir output && cd output
|
||||
cpack -G RPM --config ../CPackConfig.cmake
|
||||
rpmlint memgraph*.rpm
|
||||
|
||||
- name: Save enterprise RPM package
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Enterprise RPM package"
|
||||
path: build/output/memgraph*.rpm
|
||||
|
||||
- name: Run micro benchmark tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run micro benchmark tests.
|
||||
cd build
|
||||
# The `eval` benchmark needs a large stack limit.
|
||||
ulimit -s 262144
|
||||
ctest -R memgraph__benchmark -V
|
||||
|
||||
- name: Run macro benchmark tests
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Run parallel macro benchmark tests
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QueryParallelSuite MemgraphRunner \
|
||||
--groups aggregation_parallel create_parallel bfs_parallel \
|
||||
--num-database-workers 9 --num-clients-workers 30 \
|
||||
--no-strict
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration
|
||||
|
||||
- name: Run stress test (SSL)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run stress test (large)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --large-dataset
|
||||
|
||||
- name: Run durability test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 5
|
||||
|
||||
- name: Run durability test (large)
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 20
|
||||
237
.github/workflows/release_debian10.yaml
vendored
237
.github/workflows/release_debian10.yaml
vendored
@@ -1,4 +1,7 @@
|
||||
name: Release Debian 10
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -10,23 +13,30 @@ on:
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
|
||||
push:
|
||||
branches:
|
||||
- "release/**"
|
||||
tags:
|
||||
- "v*.*.*-rc*"
|
||||
- "v*.*-rc*"
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
|
||||
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 960
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -40,10 +50,6 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build community binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
|
||||
@@ -65,10 +71,11 @@ jobs:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -110,22 +117,19 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage"
|
||||
name: "Code coverage(Coverage build)"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -157,10 +161,6 @@ jobs:
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -171,23 +171,49 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage"
|
||||
name: "Code coverage(Debug build)"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
debug_integration_test:
|
||||
name: "Debug integration tests"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build debug binaries.
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 960
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -201,10 +227,6 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
@@ -226,11 +248,60 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
|
||||
release_benchmark_tests:
|
||||
name: "Release Benchmark Tests"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run micro benchmark tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -257,29 +328,37 @@ jobs:
|
||||
--num-database-workers 9 --num-clients-workers 30 \
|
||||
--no-strict
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
release_e2e_test:
|
||||
name: "Release End-to-end Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
timeout-minutes: 60
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run unit tests
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Run unit tests.
|
||||
# Build release binaries
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
@@ -289,6 +368,40 @@ jobs:
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: always()
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
|
||||
release_durability_stress_tests:
|
||||
name: "Release durability and stress tests"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10]
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
@@ -299,11 +412,6 @@ jobs:
|
||||
cd tests/stress
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run stress test (large)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --large-dataset
|
||||
|
||||
- name: Run durability test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
@@ -319,15 +427,11 @@ jobs:
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -340,22 +444,23 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build only memgraph release binary.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Refresh Jepsen Cluster
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh cluster-refresh
|
||||
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
|
||||
2
.github/workflows/release_docker.yaml
vendored
2
.github/workflows/release_docker.yaml
vendored
@@ -19,7 +19,7 @@ jobs:
|
||||
DOCKER_REPOSITORY_NAME: memgraph
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2
|
||||
|
||||
224
.github/workflows/release_ubuntu2004.yaml
vendored
224
.github/workflows/release_ubuntu2004.yaml
vendored
@@ -1,4 +1,7 @@
|
||||
name: Release Ubuntu 20.04
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@@ -10,23 +13,30 @@ on:
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
|
||||
push:
|
||||
branches:
|
||||
- "release/**"
|
||||
tags:
|
||||
- "v*.*.*-rc*"
|
||||
- "v*.*-rc*"
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
|
||||
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 960
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -40,10 +50,6 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build community binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
|
||||
@@ -61,14 +67,11 @@ jobs:
|
||||
coverage_build:
|
||||
name: "Coverage build"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -110,22 +113,19 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage"
|
||||
name: "Code coverage(Coverage build)"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -157,10 +157,6 @@ jobs:
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -171,23 +167,49 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Code coverage"
|
||||
name: "Code coverage(Debug build)"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
debug_integration_test:
|
||||
name: "Debug integration tests"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build debug binaries.
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
tests/integration/run.sh
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
timeout-minutes: 960
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -201,10 +223,6 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Set default build_type to Release
|
||||
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
|
||||
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
@@ -226,11 +244,60 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
|
||||
release_benchmark_tests:
|
||||
name: "Release Benchmark Tests"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run micro benchmark tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -257,29 +324,37 @@ jobs:
|
||||
--num-database-workers 9 --num-clients-workers 30 \
|
||||
--no-strict
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
release_e2e_test:
|
||||
name: "Release End-to-end Test"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
timeout-minutes: 60
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Run unit tests
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Run unit tests.
|
||||
# Build release binaries
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
cd ../pulsar
|
||||
docker-compose up -d
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
@@ -289,6 +364,40 @@ jobs:
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: always()
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
cd ../pulsar
|
||||
docker-compose down
|
||||
|
||||
release_durability_stress_tests:
|
||||
name: "Release durability and stress tests"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
timeout-minutes: 60
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
@@ -299,11 +408,6 @@ jobs:
|
||||
cd tests/stress
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run stress test (large)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --large-dataset
|
||||
|
||||
- name: Run durability test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
|
||||
68
.github/workflows/stress_test_large.yaml
vendored
Normal file
68
.github/workflows/stress_test_large.yaml
vendored
Normal file
@@ -0,0 +1,68 @@
|
||||
name: Stress test large
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref_name }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
type: choice
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*-rc*"
|
||||
- "v*.*-rc*"
|
||||
schedule:
|
||||
- cron: "0 22 * * *"
|
||||
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
|
||||
|
||||
jobs:
|
||||
stress_test_large:
|
||||
name: "Stress test large"
|
||||
timeout-minutes: 720
|
||||
strategy:
|
||||
matrix:
|
||||
os: [Debian10, Ubuntu20.04]
|
||||
extra: [BigMemory, Gen8]
|
||||
exclude:
|
||||
- os: Debian10
|
||||
extra: Gen8
|
||||
- os: Ubuntu20.04
|
||||
extra: BigMemory
|
||||
runs-on: [self-hosted, Linux, X64, "${{ matrix.os }}", "${{ matrix.extra }}"]
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run stress test (large)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --large-dataset
|
||||
2
.github/workflows/upload_to_s3.yaml
vendored
2
.github/workflows/upload_to_s3.yaml
vendored
@@ -15,7 +15,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: dawidd6/action-download-artifact@v2
|
||||
uses: dawidd6/action-download-artifact@v4
|
||||
with:
|
||||
workflow: package_all.yaml
|
||||
workflow_conclusion: success
|
||||
|
||||
@@ -17,9 +17,10 @@ repos:
|
||||
name: isort (python)
|
||||
args: ["--profile", "black"]
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v13.0.0
|
||||
rev: v17.0.6
|
||||
hooks:
|
||||
- id: clang-format
|
||||
types_or: [c++, c]
|
||||
# - repo: local
|
||||
# hooks:
|
||||
# - id: clang-tidy
|
||||
|
||||
22
.sonarcloud.properties
Normal file
22
.sonarcloud.properties
Normal file
@@ -0,0 +1,22 @@
|
||||
# Path to sources
|
||||
sonar.sources = .
|
||||
# sonar.exclusions=
|
||||
sonar.inclusions=src,include,query_modules
|
||||
|
||||
# Path to tests
|
||||
sonar.tests = tests/
|
||||
# sonar.test.exclusions=
|
||||
# sonar.test.inclusions=
|
||||
|
||||
# Source encoding
|
||||
# sonar.sourceEncoding=
|
||||
|
||||
# Exclusions for copy-paste detection
|
||||
# sonar.cpd.exclusions=
|
||||
|
||||
# Python version (for python projects only)
|
||||
# sonar.python.version=
|
||||
|
||||
# C++ standard version (for C++ projects only)
|
||||
# If not specified, it defaults to the latest supported standard
|
||||
# sonar.cfamily.reportingCppStandardOverride=c++98|c++11|c++14|c++17|c++20
|
||||
32
ADRs/001_tantivy.md
Normal file
32
ADRs/001_tantivy.md
Normal file
@@ -0,0 +1,32 @@
|
||||
# Tantivy ADR
|
||||
|
||||
**Author**
|
||||
Marko Budiselic (github.com/gitbuda)
|
||||
|
||||
**Status**
|
||||
APPROVED
|
||||
|
||||
**Date**
|
||||
January 5, 2024
|
||||
|
||||
**Problem**
|
||||
|
||||
For some of Memgraph workloads, text search is a required feature. We don't
|
||||
want to build a new text search engine because that's not Memgraph's core
|
||||
value.
|
||||
|
||||
**Criteria**
|
||||
|
||||
- easy integration with our C++ codebase
|
||||
- ability to operate in-memory and on-disk
|
||||
- sufficient features (regex, full-text search, fuzzy search, aggregations over
|
||||
text data)
|
||||
- production-ready
|
||||
|
||||
**Decision**
|
||||
|
||||
All known C++ libraries are not production-ready. Recent Rust libraries, in
|
||||
particular [Tantivy](https://github.com/quickwit-oss/tantivy), seem to provide
|
||||
much more features, it is production ready. The way how we'll integrate Tantivy
|
||||
into the current Memgraph codebase is via
|
||||
[cxx](https://github.com/dtolnay/cxx). **We select Tantivy.**
|
||||
34
ADRs/002_nuraft.md
Normal file
34
ADRs/002_nuraft.md
Normal file
@@ -0,0 +1,34 @@
|
||||
# NuRaft ADR
|
||||
|
||||
**Author**
|
||||
Marko Budiselic (github.com/gitbuda)
|
||||
|
||||
**Status**
|
||||
PROPOSED
|
||||
|
||||
**Date**
|
||||
January 10, 2024
|
||||
|
||||
**Problem**
|
||||
|
||||
In order to enhance Memgraph to have High Availability features as requested by
|
||||
customers, we want to have reliable coordinators backed by RAFT consensus algorithm. Implementing
|
||||
RAFT to be correct and performant is a very challenging task. Skillful Memgraph
|
||||
engineers already tried 3 times and failed to deliver in a reasonable timeframe
|
||||
all three times (approximately 4 person-weeks of engineering work each time).
|
||||
|
||||
**Criteria**
|
||||
|
||||
- easy integration with our C++ codebase
|
||||
- heavily tested in production environments
|
||||
- implementation of performance optimizations on top of the canonical Raft
|
||||
implementation
|
||||
|
||||
**Decision**
|
||||
|
||||
There are a few, robust C++ implementations of Raft but as a part of other
|
||||
projects or bigger libraries. **We select
|
||||
[NuRaft](https://github.com/eBay/NuRaft)** because it focuses on delivering
|
||||
Raft without bloatware, and it's used by
|
||||
[Clickhouse](https://github.com/ClickHouse/ClickHouse) (an comparable peer to
|
||||
Memgraph, a very well-established product).
|
||||
38
ADRs/003_rocksdb.md
Normal file
38
ADRs/003_rocksdb.md
Normal file
@@ -0,0 +1,38 @@
|
||||
# RocksDB ADR
|
||||
|
||||
**Author**
|
||||
Marko Budiselic (github.com/gitbuda)
|
||||
|
||||
**Status**
|
||||
ACCEPTED
|
||||
|
||||
**Date**
|
||||
January 23, 2024
|
||||
|
||||
**Problem**
|
||||
|
||||
Interacting with data (reads and writes) on disk in a concurrent, safe, and
|
||||
fast way is a challenging task. Implementing all low-level primitives to
|
||||
interact with various disk hardware efficiently consumes significant
|
||||
engineering people. Whenever Memgraph has to store data on disk (or any
|
||||
other colder than RAM storage system), the problem is how to do that in the
|
||||
least amount of development time while satisfying all functional requirements
|
||||
(often performance).
|
||||
|
||||
**Criteria**
|
||||
|
||||
- working efficiently in a highly concurrent environment
|
||||
- easy integration with Memgraph's C++ codebase
|
||||
- providing low-level key-value API
|
||||
- heavily tested in production environments
|
||||
- providing abstractions for the storage hardware (even for cloud-based
|
||||
storages like S3)
|
||||
|
||||
**Decision**
|
||||
|
||||
There are a few robust key-value stores, but finding one that is
|
||||
production-ready and compatible with Memgraph's C++ codebase is challenging.
|
||||
**We select [RocksDB](https://github.com/facebook/rocksdb)** because it
|
||||
delivers robust API to manage data on disk; it's battle-tested in many
|
||||
production environments (many databases systems are embedding RocksDB), and
|
||||
it's the most compatible one.
|
||||
67
ADRs/README.md
Normal file
67
ADRs/README.md
Normal file
@@ -0,0 +1,67 @@
|
||||
# Architecture Decision Records
|
||||
|
||||
Also known as ADRs. This practice has become widespread in many
|
||||
high performing engineering teams. It is a technique for communicating
|
||||
between software engineers. ADRs provide a clear and documented
|
||||
history of architectural choices, ensuring that everyone on the
|
||||
team is on the same page. This improves communication and reduces
|
||||
misunderstandings. The act of recording decisions encourages
|
||||
thoughtful consideration before making choices. This can lead to
|
||||
more robust and better-informed architectural decisions.
|
||||
|
||||
Links must be created, pointing both to and from the Github Issues
|
||||
and/or the Notion Program Management "Initiative" database.
|
||||
|
||||
ADRs are complimentary to any tech specs that get written while
|
||||
designing a solution. ADRs are very short and to the point, while
|
||||
tech specs will include diagrams and can be quite verbose.
|
||||
|
||||
## HOWTO
|
||||
|
||||
Each ADR will be assigned a monotonically increasing unique numeric
|
||||
identifier, which will be zero-padded to 3 digits. Each ADR will
|
||||
be in a single markdown file containing no more than one page of
|
||||
text, and the filename will start with that unique identifier,
|
||||
followed by a camel case phrase summarizing the problem. For
|
||||
example: `001_architecture_decision_records.md` or
|
||||
`002_big_integration_cap_theorem.md`.
|
||||
|
||||
We want to use an ADR when:
|
||||
1. Significant Impact: This includes choices that affect scalability, performance, or fundamental design principles.
|
||||
1. Long-Term Ramifications: When a decision is expected to have long-term ramifications or is difficult to reverse.
|
||||
1. Architectural Principles: ADRs are suitable for documenting decisions related to architectural principles, frameworks, or patterns that shape the system's structure.
|
||||
1. Controversial Choices: When a decision is likely to be controversial or may require justification in the future.
|
||||
|
||||
The most senior engineer on a project will evaluate and decide
|
||||
whether or not an ADR is needed.
|
||||
|
||||
## Do
|
||||
|
||||
1. Keep them brief and concise.
|
||||
1. Explain the trade-offs.
|
||||
1. Each ADR should be about one AD, not multiple ADs
|
||||
1. Don't alter existing information in an ADR. Instead, amend the ADR by adding new information, or supersede the ADR by creating a new ADR.
|
||||
1. Explain your organization's situation and business priorities.
|
||||
1. Include rationale and considerations based on social and skills makeups of your teams.
|
||||
1. Include pros and cons that are relevant, and describe them in terms that align with your needs and goals.
|
||||
1. Explain what follows from making the decision. This can include the effects, outcomes, outputs, follow ups, and more.
|
||||
|
||||
## Don't
|
||||
|
||||
1. Try to guess what the executive leader wants, and then attempt to please them. Be objective.
|
||||
1. Try to solve everything all at once. A pretty good solution now is MUCH BETTER than a perfect solution later. Carpe diem!
|
||||
1. Hide any doubts or unanswered questions.
|
||||
1. Make it a sales pitch. Everything has upsides and downsides - be authentic and honest about them.
|
||||
1. Perform merely a superficial investigation. If an ADR doesn't call for some deep thinking, then it probably shouldn't exist.
|
||||
1. Ignore the long-term costs such as performance, tech debt or hardware and maintenance.
|
||||
1. Get tunnel vision where creative or surprising approaches are not explored.
|
||||
|
||||
# Template - use the format below for each new ADR
|
||||
|
||||
1. **Author** - who has written the ADR
|
||||
1. **Status** - one of: PROPOSED, ACCEPTED, REJECTED, SUPERSEDED-BY or DEPRECATED
|
||||
1. **Date** - when the status was most recently updated
|
||||
1. **Problem** - a concise paragraph explaining the context
|
||||
1. **Criteria** - a list of the two or three metrics by which the solution was evaluated, and their relative weights (importance)
|
||||
1. **Decision** - what was chosen as the way forward, and what the consequences are of the decision
|
||||
|
||||
@@ -189,7 +189,7 @@ add_custom_target(clean_all
|
||||
# is easier debugging of compilation and linker flags.
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_CXX_STANDARD 23)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
# c99-designator is disabled because of required mixture of designated and
|
||||
# non-designated initializers in Python Query Module code (`py_module.cpp`).
|
||||
@@ -271,6 +271,17 @@ endif()
|
||||
set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
|
||||
add_subdirectory(libs EXCLUDE_FROM_ALL)
|
||||
|
||||
option(MG_EXPERIMENTAL_HIGH_AVAILABILITY "Feature flag for experimental high availability" OFF)
|
||||
|
||||
if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
set(MG_EXPERIMENTAL_HIGH_AVAILABILITY OFF)
|
||||
message(FATAL_ERROR "MG_EXPERIMENTAL_HIGH_AVAILABILITY can only be used with enterpise version of the code.")
|
||||
endif ()
|
||||
|
||||
if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
add_compile_definitions(MG_EXPERIMENTAL_HIGH_AVAILABILITY)
|
||||
endif ()
|
||||
|
||||
# Optional subproject configuration -------------------------------------------
|
||||
option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
|
||||
option(TOOLS "Build tools binaries" ON)
|
||||
@@ -279,6 +290,8 @@ option(ASAN "Build with Address Sanitizer. To get a reasonable performance optio
|
||||
option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
|
||||
option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
|
||||
|
||||
# Build feature flags
|
||||
|
||||
if (TEST_COVERAGE)
|
||||
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
if (NOT lower_build_type STREQUAL "debug")
|
||||
@@ -292,7 +305,7 @@ if (MG_ENTERPRISE)
|
||||
add_definitions(-DMG_ENTERPRISE)
|
||||
endif()
|
||||
|
||||
set(ENABLE_JEMALLOC ON)
|
||||
option(ENABLE_JEMALLOC "Use jemalloc" ON)
|
||||
|
||||
if (ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
|
||||
@@ -111,6 +111,14 @@ modifications:
|
||||
value: "false"
|
||||
override: true
|
||||
|
||||
- name: "storage_parallel_schema_recovery"
|
||||
value: "false"
|
||||
override: true
|
||||
|
||||
- name: "storage_enable_schema_metadata"
|
||||
value: "false"
|
||||
override: true
|
||||
|
||||
- name: "query_callable_mappings_path"
|
||||
value: "/etc/memgraph/apoc_compatibility_mappings.json"
|
||||
override: true
|
||||
|
||||
@@ -234,7 +234,57 @@ inline mgp_type *type_duration() { return MgInvoke<mgp_type *>(mgp_type_duration
|
||||
|
||||
inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke<mgp_type *>(mgp_type_nullable, type); }
|
||||
|
||||
inline bool create_label_index(mgp_graph *graph, const char *label) {
|
||||
return MgInvoke<int>(mgp_create_label_index, graph, label);
|
||||
}
|
||||
|
||||
inline bool drop_label_index(mgp_graph *graph, const char *label) {
|
||||
return MgInvoke<int>(mgp_drop_label_index, graph, label);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_label_indices(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_label_indices, graph, memory);
|
||||
}
|
||||
|
||||
inline bool create_label_property_index(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_create_label_property_index, graph, label, property);
|
||||
}
|
||||
|
||||
inline bool drop_label_property_index(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_drop_label_property_index, graph, label, property);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_label_property_indices, graph, memory);
|
||||
}
|
||||
|
||||
inline bool create_existence_constraint(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_create_existence_constraint, graph, label, property);
|
||||
}
|
||||
|
||||
inline bool drop_existence_constraint(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_drop_existence_constraint, graph, label, property);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_existence_constraints, graph, memory);
|
||||
}
|
||||
|
||||
inline bool create_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) {
|
||||
return MgInvoke<int>(mgp_create_unique_constraint, memgraph_graph, label, properties);
|
||||
}
|
||||
|
||||
inline bool drop_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) {
|
||||
return MgInvoke<int>(mgp_drop_unique_constraint, memgraph_graph, label, properties);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_unique_constraints, graph, memory);
|
||||
}
|
||||
|
||||
// mgp_graph
|
||||
|
||||
inline bool graph_is_transactional(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_transactional, graph); }
|
||||
|
||||
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
|
||||
|
||||
@@ -328,6 +378,8 @@ inline mgp_list *list_copy(mgp_list *list, mgp_memory *memory) {
|
||||
|
||||
inline void list_destroy(mgp_list *list) { mgp_list_destroy(list); }
|
||||
|
||||
inline bool list_contains_deleted(mgp_list *list) { return MgInvoke<int>(mgp_list_contains_deleted, list); }
|
||||
|
||||
inline void list_append(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append, list, val); }
|
||||
|
||||
inline void list_append_extend(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append_extend, list, val); }
|
||||
@@ -346,6 +398,8 @@ inline mgp_map *map_copy(mgp_map *map, mgp_memory *memory) { return MgInvoke<mgp
|
||||
|
||||
inline void map_destroy(mgp_map *map) { mgp_map_destroy(map); }
|
||||
|
||||
inline bool map_contains_deleted(mgp_map *map) { return MgInvoke<int>(mgp_map_contains_deleted, map); }
|
||||
|
||||
inline void map_insert(mgp_map *map, const char *key, mgp_value *value) {
|
||||
MgInvokeVoid(mgp_map_insert, map, key, value);
|
||||
}
|
||||
@@ -394,6 +448,8 @@ inline mgp_vertex *vertex_copy(mgp_vertex *v, mgp_memory *memory) {
|
||||
|
||||
inline void vertex_destroy(mgp_vertex *v) { mgp_vertex_destroy(v); }
|
||||
|
||||
inline bool vertex_is_deleted(mgp_vertex *v) { return MgInvoke<int>(mgp_vertex_is_deleted, v); }
|
||||
|
||||
inline bool vertex_equal(mgp_vertex *v1, mgp_vertex *v2) { return MgInvoke<int>(mgp_vertex_equal, v1, v2); }
|
||||
|
||||
inline size_t vertex_labels_count(mgp_vertex *v) { return MgInvoke<size_t>(mgp_vertex_labels_count, v); }
|
||||
@@ -446,6 +502,8 @@ inline mgp_edge *edge_copy(mgp_edge *e, mgp_memory *memory) { return MgInvoke<mg
|
||||
|
||||
inline void edge_destroy(mgp_edge *e) { mgp_edge_destroy(e); }
|
||||
|
||||
inline bool edge_is_deleted(mgp_edge *e) { return MgInvoke<int>(mgp_edge_is_deleted, e); }
|
||||
|
||||
inline bool edge_equal(mgp_edge *e1, mgp_edge *e2) { return MgInvoke<int>(mgp_edge_equal, e1, e2); }
|
||||
|
||||
inline mgp_edge_type edge_get_type(mgp_edge *e) { return MgInvoke<mgp_edge_type>(mgp_edge_get_type, e); }
|
||||
@@ -482,6 +540,8 @@ inline mgp_path *path_copy(mgp_path *path, mgp_memory *memory) {
|
||||
|
||||
inline void path_destroy(mgp_path *path) { mgp_path_destroy(path); }
|
||||
|
||||
inline bool path_contains_deleted(mgp_path *path) { return MgInvoke<int>(mgp_path_contains_deleted, path); }
|
||||
|
||||
inline void path_expand(mgp_path *path, mgp_edge *edge) { MgInvokeVoid(mgp_path_expand, path, edge); }
|
||||
|
||||
inline void path_pop(mgp_path *path) { MgInvokeVoid(mgp_path_pop, path); }
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -25,7 +25,7 @@ namespace mg_exception {
|
||||
template <typename FirstArg, typename... Args>
|
||||
std::string StringSerialize(FirstArg &&firstArg, Args &&...args) {
|
||||
std::stringstream stream;
|
||||
stream << firstArg;
|
||||
stream << std::forward<FirstArg>(firstArg);
|
||||
((stream << " " << args), ...);
|
||||
return stream.str();
|
||||
}
|
||||
|
||||
@@ -429,6 +429,9 @@ enum mgp_error mgp_list_copy(struct mgp_list *list, struct mgp_memory *memory, s
|
||||
/// Free the memory used by the given mgp_list and contained elements.
|
||||
void mgp_list_destroy(struct mgp_list *list);
|
||||
|
||||
/// Return whether the given mgp_list contains any deleted values.
|
||||
enum mgp_error mgp_list_contains_deleted(struct mgp_list *list, int *result);
|
||||
|
||||
/// Append a copy of mgp_value to mgp_list if capacity allows.
|
||||
/// The list copies the given value and therefore does not take ownership of the
|
||||
/// original value. You still need to call mgp_value_destroy to free the
|
||||
@@ -469,6 +472,9 @@ enum mgp_error mgp_map_copy(struct mgp_map *map, struct mgp_memory *memory, stru
|
||||
/// Free the memory used by the given mgp_map and contained items.
|
||||
void mgp_map_destroy(struct mgp_map *map);
|
||||
|
||||
/// Return whether the given mgp_map contains any deleted values.
|
||||
enum mgp_error mgp_map_contains_deleted(struct mgp_map *map, int *result);
|
||||
|
||||
/// Insert a new mapping from a NULL terminated character string to a value.
|
||||
/// If a mapping with the same key already exists, it is *not* replaced.
|
||||
/// In case of insertion, both the string and the value are copied into the map.
|
||||
@@ -552,6 +558,9 @@ enum mgp_error mgp_path_copy(struct mgp_path *path, struct mgp_memory *memory, s
|
||||
/// Free the memory used by the given mgp_path and contained vertices and edges.
|
||||
void mgp_path_destroy(struct mgp_path *path);
|
||||
|
||||
/// Return whether the given mgp_path contains any deleted values.
|
||||
enum mgp_error mgp_path_contains_deleted(struct mgp_path *path, int *result);
|
||||
|
||||
/// Append an edge continuing from the last vertex on the path.
|
||||
/// The edge is copied into the path. Therefore, the path does not take
|
||||
/// ownership of the original edge, so you still need to free the edge memory
|
||||
@@ -725,6 +734,9 @@ enum mgp_error mgp_vertex_copy(struct mgp_vertex *v, struct mgp_memory *memory,
|
||||
/// Free the memory used by a mgp_vertex.
|
||||
void mgp_vertex_destroy(struct mgp_vertex *v);
|
||||
|
||||
/// Return whether the given mgp_vertex is deleted.
|
||||
enum mgp_error mgp_vertex_is_deleted(struct mgp_vertex *v, int *result);
|
||||
|
||||
/// Result is non-zero if given vertices are equal, otherwise 0.
|
||||
enum mgp_error mgp_vertex_equal(struct mgp_vertex *v1, struct mgp_vertex *v2, int *result);
|
||||
|
||||
@@ -819,6 +831,9 @@ enum mgp_error mgp_edge_copy(struct mgp_edge *e, struct mgp_memory *memory, stru
|
||||
/// Free the memory used by a mgp_edge.
|
||||
void mgp_edge_destroy(struct mgp_edge *e);
|
||||
|
||||
/// Return whether the given mgp_edge is deleted.
|
||||
enum mgp_error mgp_edge_is_deleted(struct mgp_edge *e, int *result);
|
||||
|
||||
/// Result is non-zero if given edges are equal, otherwise 0.
|
||||
enum mgp_error mgp_edge_equal(struct mgp_edge *e1, struct mgp_edge *e2, int *result);
|
||||
|
||||
@@ -876,12 +891,77 @@ enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *m
|
||||
enum mgp_error mgp_graph_get_vertex_by_id(struct mgp_graph *g, struct mgp_vertex_id id, struct mgp_memory *memory,
|
||||
struct mgp_vertex **result);
|
||||
|
||||
/// Creates label index for given label.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if label index already exists, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_label_index(struct mgp_graph *graph, const char *label, int *result);
|
||||
|
||||
/// Drop label index.
|
||||
enum mgp_error mgp_drop_label_index(struct mgp_graph *graph, const char *label, int *result);
|
||||
|
||||
/// List all label indices.
|
||||
enum mgp_error mgp_list_all_label_indices(struct mgp_graph *graph, struct mgp_memory *memory, struct mgp_list **result);
|
||||
|
||||
/// Creates label-property index for given label and propery.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if label property index already exists, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_label_property_index(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// Drops label-property index for given label and propery.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if dropping label property index failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_drop_label_property_index(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// List all label+property indices.
|
||||
enum mgp_error mgp_list_all_label_property_indices(struct mgp_graph *graph, struct mgp_memory *memory,
|
||||
struct mgp_list **result);
|
||||
|
||||
/// Creates existence constraint for given label and property.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if creating existence constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_existence_constraint(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// Drops existence constraint for given label and property.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if dropping existence constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_drop_existence_constraint(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// List all existence constraints.
|
||||
enum mgp_error mgp_list_all_existence_constraints(struct mgp_graph *graph, struct mgp_memory *memory,
|
||||
struct mgp_list **result);
|
||||
|
||||
/// Creates unique constraint for given label and properties.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if creating unique constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties,
|
||||
int *result);
|
||||
|
||||
/// Drops unique constraint for given label and properties.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if dropping unique constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_drop_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties,
|
||||
int *result);
|
||||
|
||||
/// List all unique constraints
|
||||
enum mgp_error mgp_list_all_unique_constraints(struct mgp_graph *graph, struct mgp_memory *memory,
|
||||
struct mgp_list **result);
|
||||
|
||||
/// Result is non-zero if the graph can be modified.
|
||||
/// If a graph is immutable, then vertices cannot be created or deleted, and all of the returned vertices will be
|
||||
/// immutable also. The same applies for edges.
|
||||
/// Current implementation always returns without errors.
|
||||
enum mgp_error mgp_graph_is_mutable(struct mgp_graph *graph, int *result);
|
||||
|
||||
/// Result is non-zero if the graph is in transactional storage mode.
|
||||
/// If a graph is not in transactional mode (i.e. analytical mode), then vertices and edges can be missing
|
||||
/// because changes from other transactions are visible.
|
||||
/// Current implementation always returns without errors.
|
||||
enum mgp_error mgp_graph_is_transactional(struct mgp_graph *graph, int *result);
|
||||
|
||||
/// Add a new vertex to the graph.
|
||||
/// Resulting vertex must be freed using mgp_vertex_destroy.
|
||||
/// Return mgp_error::MGP_ERROR_IMMUTABLE_OBJECT if `graph` is immutable.
|
||||
|
||||
403
include/mgp.hpp
403
include/mgp.hpp
File diff suppressed because it is too large
Load Diff
2
init
2
init
@@ -120,7 +120,7 @@ fi
|
||||
# develop on them -> pre-commit hook not required -> we can use latest
|
||||
# packages.
|
||||
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ] && [ "${DISTRO}" != "amzn-2" ]; then
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pip install pre-commit==3.5.*
|
||||
python3 -m pre_commit install
|
||||
# Install py format tools for usage during the development.
|
||||
echo "Install black formatter"
|
||||
|
||||
@@ -294,3 +294,10 @@ set_path_external_library(jemalloc STATIC
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/include/)
|
||||
|
||||
import_header_library(rangev3 ${CMAKE_CURRENT_SOURCE_DIR}/rangev3/include)
|
||||
|
||||
# Setup NuRaft
|
||||
import_external_library(nuraft STATIC
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/nuraft/lib/libnuraft.a
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/nuraft/include/)
|
||||
find_package(OpenSSL REQUIRED)
|
||||
target_link_libraries(nuraft INTERFACE ${OPENSSL_LIBRARIES})
|
||||
|
||||
@@ -123,9 +123,10 @@ declare -A primary_urls=(
|
||||
["pulsar"]="http://$local_cache_host/git/pulsar.git"
|
||||
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
|
||||
["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
|
||||
["absl"]="https://$local_cache_host/git/abseil-cpp.git"
|
||||
["jemalloc"]="https://$local_cache_host/git/jemalloc.git"
|
||||
["range-v3"]="https://$local_cache_host/git/ericniebler/range-v3.git"
|
||||
["absl"]="http://$local_cache_host/git/abseil-cpp.git"
|
||||
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
|
||||
["range-v3"]="http://$local_cache_host/git/range-v3.git"
|
||||
["nuraft"]="http://$local_cache_host/git/NuRaft.git"
|
||||
)
|
||||
|
||||
# The goal of secondary urls is to have links to the "source of truth" of
|
||||
@@ -155,6 +156,7 @@ declare -A secondary_urls=(
|
||||
["absl"]="https://github.com/abseil/abseil-cpp.git"
|
||||
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
|
||||
["range-v3"]="https://github.com/ericniebler/range-v3.git"
|
||||
["nuraft"]="https://github.com/eBay/NuRaft.git"
|
||||
)
|
||||
|
||||
# antlr
|
||||
@@ -259,7 +261,7 @@ repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" "
|
||||
|
||||
# jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756
|
||||
JEMALLOC_COMMIT_VERSION="5.2.1"
|
||||
repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
|
||||
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
|
||||
|
||||
# this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR
|
||||
pushd jemalloc
|
||||
@@ -268,6 +270,8 @@ pushd jemalloc
|
||||
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
|
||||
./configure \
|
||||
--disable-cxx \
|
||||
--with-lg-page=12 \
|
||||
--with-lg-hugepage=21 \
|
||||
--enable-shared=no --prefix=$working_dir \
|
||||
--with-malloc-conf="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
|
||||
@@ -277,3 +281,10 @@ popd
|
||||
#range-v3 release-0.12.0
|
||||
range_v3_ref="release-0.12.0"
|
||||
repo_clone_try_double "${primary_urls[range-v3]}" "${secondary_urls[range-v3]}" "rangev3" "$range_v3_ref"
|
||||
|
||||
# NuRaft
|
||||
nuraft_tag="v2.1.0"
|
||||
repo_clone_try_double "${primary_urls[nuraft]}" "${secondary_urls[nuraft]}" "nuraft" "$nuraft_tag" true
|
||||
pushd nuraft
|
||||
./prepare.sh
|
||||
popd
|
||||
|
||||
@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
|
||||
3. using the Licensed Work to create a work or solution
|
||||
which competes (or might reasonably be expected to
|
||||
compete) with the Licensed Work.
|
||||
CHANGE DATE: 2027-30-10
|
||||
CHANGE DATE: 2028-21-01
|
||||
CHANGE LICENSE: Apache License, Version 2.0
|
||||
|
||||
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.
|
||||
|
||||
@@ -13,6 +13,7 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
add_library(example_c SHARED example.c)
|
||||
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(example_c PRIVATE -Wall)
|
||||
target_link_libraries(example_c PRIVATE -static-libgcc -static-libstdc++)
|
||||
# Strip C example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET example_c POST_BUILD
|
||||
@@ -28,6 +29,7 @@ install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
|
||||
add_library(example_cpp SHARED example.cpp)
|
||||
target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(example_cpp PRIVATE -Wall)
|
||||
target_link_libraries(example_cpp PRIVATE -static-libgcc -static-libstdc++)
|
||||
# Strip C++ example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET example_cpp POST_BUILD
|
||||
@@ -43,6 +45,7 @@ install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
|
||||
add_library(schema SHARED schema.cpp)
|
||||
target_include_directories(schema PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(schema PRIVATE -Wall)
|
||||
target_link_libraries(schema PRIVATE -static-libgcc -static-libstdc++)
|
||||
# Strip C++ example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET schema POST_BUILD
|
||||
|
||||
@@ -10,18 +10,33 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <mgp.hpp>
|
||||
#include "utils/string.hpp"
|
||||
|
||||
#include <optional>
|
||||
|
||||
namespace Schema {
|
||||
|
||||
/*NodeTypeProperties and RelTypeProperties constants*/
|
||||
constexpr std::string_view kStatusKept = "Kept";
|
||||
constexpr std::string_view kStatusCreated = "Created";
|
||||
constexpr std::string_view kStatusDropped = "Dropped";
|
||||
constexpr std::string_view kReturnNodeType = "nodeType";
|
||||
constexpr std::string_view kProcedureNodeType = "node_type_properties";
|
||||
constexpr std::string_view kProcedureRelType = "rel_type_properties";
|
||||
constexpr std::string_view kProcedureAssert = "assert";
|
||||
constexpr std::string_view kReturnLabels = "nodeLabels";
|
||||
constexpr std::string_view kReturnRelType = "relType";
|
||||
constexpr std::string_view kReturnPropertyName = "propertyName";
|
||||
constexpr std::string_view kReturnPropertyType = "propertyTypes";
|
||||
constexpr std::string_view kReturnMandatory = "mandatory";
|
||||
constexpr std::string_view kReturnLabel = "label";
|
||||
constexpr std::string_view kReturnKey = "key";
|
||||
constexpr std::string_view kReturnKeys = "keys";
|
||||
constexpr std::string_view kReturnUnique = "unique";
|
||||
constexpr std::string_view kReturnAction = "action";
|
||||
constexpr std::string_view kParameterIndices = "indices";
|
||||
constexpr std::string_view kParameterUniqueConstraints = "unique_constraints";
|
||||
constexpr std::string_view kParameterExistenceConstraints = "existence_constraints";
|
||||
constexpr std::string_view kParameterDropExisting = "drop_existing";
|
||||
|
||||
std::string TypeOf(const mgp::Type &type);
|
||||
|
||||
@@ -35,6 +50,7 @@ void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, con
|
||||
|
||||
void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
} // namespace Schema
|
||||
|
||||
/*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this
|
||||
@@ -92,31 +108,83 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t
|
||||
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
|
||||
}
|
||||
|
||||
void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
struct Property {
|
||||
std::string name;
|
||||
mgp::Value value;
|
||||
|
||||
Property(const std::string &name, mgp::Value &&value) : name(name), value(std::move(value)) {}
|
||||
};
|
||||
|
||||
struct LabelsHash {
|
||||
std::size_t operator()(const std::set<std::string> &set) const {
|
||||
std::size_t seed = set.size();
|
||||
for (const auto &i : set) {
|
||||
seed ^= std::hash<std::string>{}(i) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
|
||||
struct LabelsComparator {
|
||||
bool operator()(const std::set<std::string> &lhs, const std::set<std::string> &rhs) const { return lhs == rhs; }
|
||||
};
|
||||
|
||||
struct PropertyComparator {
|
||||
bool operator()(const Property &lhs, const Property &rhs) const { return lhs.name < rhs.name; }
|
||||
};
|
||||
|
||||
struct PropertyInfo {
|
||||
std::set<Property, PropertyComparator> properties;
|
||||
bool mandatory;
|
||||
};
|
||||
|
||||
void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result,
|
||||
mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
;
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
const mgp::Graph graph = mgp::Graph(memgraph_graph);
|
||||
for (auto node : graph.Nodes()) {
|
||||
std::string type = "";
|
||||
mgp::List labels = mgp::List();
|
||||
std::unordered_map<std::set<std::string>, PropertyInfo, LabelsHash, LabelsComparator> node_types_properties;
|
||||
|
||||
for (auto node : mgp::Graph(memgraph_graph).Nodes()) {
|
||||
std::set<std::string> labels_set = {};
|
||||
for (auto label : node.Labels()) {
|
||||
labels.AppendExtend(mgp::Value(label));
|
||||
type += ":`" + std::string(label) + "`";
|
||||
labels_set.emplace(label);
|
||||
}
|
||||
|
||||
if (node.Properties().size() == 0) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
|
||||
if (node_types_properties.find(labels_set) == node_types_properties.end()) {
|
||||
node_types_properties[labels_set] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
|
||||
}
|
||||
|
||||
if (node.Properties().empty()) {
|
||||
node_types_properties[labels_set].mandatory = false; // if there is node with no property, it is not mandatory
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &property_info = node_types_properties.at(labels_set);
|
||||
for (auto &[key, prop] : node.Properties()) {
|
||||
auto property_type = mgp::List();
|
||||
property_info.properties.emplace(key, std::move(prop));
|
||||
if (property_info.mandatory) {
|
||||
property_info.mandatory =
|
||||
property_info.properties.size() == 1; // if there is only one property, it is mandatory
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &[labels, property_info] : node_types_properties) {
|
||||
std::string label_type;
|
||||
mgp::List labels_list = mgp::List();
|
||||
for (auto const &label : labels) {
|
||||
label_type += ":`" + std::string(label) + "`";
|
||||
labels_list.AppendExtend(mgp::Value(label));
|
||||
}
|
||||
for (auto const &prop : property_info.properties) {
|
||||
auto record = record_factory.NewRecord();
|
||||
property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
|
||||
ProcessPropertiesNode<mgp::List>(record, type, labels, key, property_type, true);
|
||||
ProcessPropertiesNode(record, label_type, labels_list, prop.name, TypeOf(prop.value.Type()),
|
||||
property_info.mandatory);
|
||||
}
|
||||
if (property_info.properties.empty()) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesNode<std::string>(record, label_type, labels_list, "", "", false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,26 +194,43 @@ void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_r
|
||||
}
|
||||
}
|
||||
|
||||
void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
;
|
||||
|
||||
std::unordered_map<std::string, PropertyInfo> rel_types_properties;
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
const mgp::Graph graph = mgp::Graph(memgraph_graph);
|
||||
|
||||
for (auto rel : graph.Relationships()) {
|
||||
std::string type = ":`" + std::string(rel.Type()) + "`";
|
||||
if (rel.Properties().size() == 0) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesRel<std::string>(record, type, "", "", false);
|
||||
std::string rel_type = std::string(rel.Type());
|
||||
if (rel_types_properties.find(rel_type) == rel_types_properties.end()) {
|
||||
rel_types_properties[rel_type] = PropertyInfo{std::set<Property, PropertyComparator>(), true};
|
||||
}
|
||||
|
||||
if (rel.Properties().empty()) {
|
||||
rel_types_properties[rel_type].mandatory = false; // if there is rel with no property, it is not mandatory
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &property_info = rel_types_properties.at(rel_type);
|
||||
for (auto &[key, prop] : rel.Properties()) {
|
||||
auto property_type = mgp::List();
|
||||
property_info.properties.emplace(key, std::move(prop));
|
||||
if (property_info.mandatory) {
|
||||
property_info.mandatory =
|
||||
property_info.properties.size() == 1; // if there is only one property, it is mandatory
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &[type, property_info] : rel_types_properties) {
|
||||
std::string type_str = ":`" + std::string(type) + "`";
|
||||
for (auto const &prop : property_info.properties) {
|
||||
auto record = record_factory.NewRecord();
|
||||
property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
|
||||
ProcessPropertiesRel<mgp::List>(record, type, key, property_type, true);
|
||||
ProcessPropertiesRel(record, type_str, prop.name, TypeOf(prop.value.Type()), property_info.mandatory);
|
||||
}
|
||||
if (property_info.properties.empty()) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesRel<std::string>(record, type_str, "", "", false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,29 +240,435 @@ void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_re
|
||||
}
|
||||
}
|
||||
|
||||
void InsertRecordForLabelIndex(const auto &record_factory, const std::string_view label,
|
||||
const std::string_view status) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
|
||||
record.Insert(std::string(Schema::kReturnKey).c_str(), "");
|
||||
record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List());
|
||||
record.Insert(std::string(Schema::kReturnUnique).c_str(), false);
|
||||
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
|
||||
}
|
||||
|
||||
void InsertRecordForUniqueConstraint(const auto &record_factory, const std::string_view label,
|
||||
const mgp::List &properties, const std::string_view status) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
|
||||
record.Insert(std::string(Schema::kReturnKey).c_str(), properties.ToString());
|
||||
record.Insert(std::string(Schema::kReturnKeys).c_str(), properties);
|
||||
record.Insert(std::string(Schema::kReturnUnique).c_str(), true);
|
||||
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
|
||||
}
|
||||
|
||||
void InsertRecordForLabelPropertyIndexAndExistenceConstraint(const auto &record_factory, const std::string_view label,
|
||||
const std::string_view property,
|
||||
const std::string_view status) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
|
||||
record.Insert(std::string(Schema::kReturnKey).c_str(), property);
|
||||
record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List({mgp::Value(property)}));
|
||||
record.Insert(std::string(Schema::kReturnUnique).c_str(), false);
|
||||
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
|
||||
}
|
||||
|
||||
void ProcessCreatingLabelIndex(const std::string_view label, const std::set<std::string_view> &existing_label_indices,
|
||||
mgp_graph *memgraph_graph, const auto &record_factory) {
|
||||
if (existing_label_indices.contains(label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusKept);
|
||||
} else if (mgp::CreateLabelIndex(memgraph_graph, label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TFunc>
|
||||
void ProcessCreatingLabelPropertyIndexAndExistenceConstraint(const std::string_view label,
|
||||
const std::string_view property,
|
||||
const std::set<std::string_view> &existing_collection,
|
||||
const TFunc &func_creation, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory) {
|
||||
const auto label_property_search_key = std::string(label) + ":" + std::string(property);
|
||||
if (existing_collection.contains(label_property_search_key)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusKept);
|
||||
} else if (func_creation(memgraph_graph, label, property)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusCreated);
|
||||
}
|
||||
}
|
||||
|
||||
/// We collect properties for which index was created.
|
||||
using AssertedIndices = std::set<std::string, std::less<>>;
|
||||
AssertedIndices CreateIndicesForLabel(const std::string_view label, const mgp::Value &properties_val,
|
||||
mgp_graph *memgraph_graph, const auto &record_factory,
|
||||
const std::set<std::string_view> &existing_label_indices,
|
||||
const std::set<std::string_view> &existing_label_property_indices) {
|
||||
AssertedIndices asserted_indices;
|
||||
if (!properties_val.IsList()) {
|
||||
return {};
|
||||
}
|
||||
if (const auto properties = properties_val.ValueList();
|
||||
properties.Empty() && mgp::CreateLabelIndex(memgraph_graph, label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated);
|
||||
asserted_indices.emplace("");
|
||||
} else {
|
||||
std::for_each(properties.begin(), properties.end(),
|
||||
[&label, &existing_label_indices, &existing_label_property_indices, &memgraph_graph, &record_factory,
|
||||
&asserted_indices](const mgp::Value &property) {
|
||||
if (!property.IsString()) {
|
||||
return;
|
||||
}
|
||||
const auto property_str = property.ValueString();
|
||||
if (property_str.empty()) {
|
||||
ProcessCreatingLabelIndex(label, existing_label_indices, memgraph_graph, record_factory);
|
||||
asserted_indices.emplace("");
|
||||
} else {
|
||||
ProcessCreatingLabelPropertyIndexAndExistenceConstraint(
|
||||
label, property_str, existing_label_property_indices, mgp::CreateLabelPropertyIndex,
|
||||
memgraph_graph, record_factory);
|
||||
asserted_indices.emplace(property_str);
|
||||
}
|
||||
});
|
||||
}
|
||||
return asserted_indices;
|
||||
}
|
||||
|
||||
void ProcessIndices(const mgp::Map &indices_map, mgp_graph *memgraph_graph, const auto &record_factory,
|
||||
bool drop_existing) {
|
||||
auto mgp_existing_label_indices = mgp::ListAllLabelIndices(memgraph_graph);
|
||||
auto mgp_existing_label_property_indices = mgp::ListAllLabelPropertyIndices(memgraph_graph);
|
||||
|
||||
std::set<std::string_view> existing_label_indices;
|
||||
std::transform(mgp_existing_label_indices.begin(), mgp_existing_label_indices.end(),
|
||||
std::inserter(existing_label_indices, existing_label_indices.begin()),
|
||||
[](const mgp::Value &index) { return index.ValueString(); });
|
||||
|
||||
std::set<std::string_view> existing_label_property_indices;
|
||||
std::transform(mgp_existing_label_property_indices.begin(), mgp_existing_label_property_indices.end(),
|
||||
std::inserter(existing_label_property_indices, existing_label_property_indices.begin()),
|
||||
[](const mgp::Value &index) { return index.ValueString(); });
|
||||
|
||||
std::set<std::string> asserted_label_indices;
|
||||
std::set<std::string> asserted_label_property_indices;
|
||||
|
||||
auto merge_label_property = [](const std::string &label, const std::string &property) {
|
||||
return label + ":" + property;
|
||||
};
|
||||
|
||||
for (const auto &index : indices_map) {
|
||||
const std::string_view label = index.key;
|
||||
const mgp::Value &properties_val = index.value;
|
||||
|
||||
AssertedIndices asserted_indices_new = CreateIndicesForLabel(
|
||||
label, properties_val, memgraph_graph, record_factory, existing_label_indices, existing_label_property_indices);
|
||||
|
||||
if (!drop_existing) {
|
||||
continue;
|
||||
}
|
||||
std::ranges::for_each(asserted_indices_new, [&asserted_label_indices, &asserted_label_property_indices, label,
|
||||
&merge_label_property](const std::string &property) {
|
||||
if (property.empty()) {
|
||||
asserted_label_indices.emplace(label);
|
||||
} else {
|
||||
asserted_label_property_indices.emplace(merge_label_property(std::string(label), property));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (!drop_existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::set<std::string_view> label_indices_to_drop;
|
||||
std::ranges::set_difference(existing_label_indices, asserted_label_indices,
|
||||
std::inserter(label_indices_to_drop, label_indices_to_drop.begin()));
|
||||
|
||||
std::ranges::for_each(label_indices_to_drop, [memgraph_graph, &record_factory](const std::string_view label) {
|
||||
if (mgp::DropLabelIndex(memgraph_graph, label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
|
||||
std::set<std::string_view> label_property_indices_to_drop;
|
||||
std::ranges::set_difference(existing_label_property_indices, asserted_label_property_indices,
|
||||
std::inserter(label_property_indices_to_drop, label_property_indices_to_drop.begin()));
|
||||
|
||||
auto decouple_label_property = [](const std::string_view label_property) {
|
||||
const auto label_size = label_property.find(':');
|
||||
const auto label = std::string(label_property.substr(0, label_size));
|
||||
const auto property = std::string(label_property.substr(label_size + 1));
|
||||
return std::make_pair(label, property);
|
||||
};
|
||||
|
||||
std::ranges::for_each(label_property_indices_to_drop, [memgraph_graph, &record_factory, decouple_label_property](
|
||||
const std::string_view label_property) {
|
||||
const auto [label, property] = decouple_label_property(label_property);
|
||||
if (mgp::DropLabelPropertyIndex(memgraph_graph, label, property)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
using ExistenceConstraintsStorage = std::set<std::string_view>;
|
||||
|
||||
ExistenceConstraintsStorage CreateExistenceConstraintsForLabel(
|
||||
const std::string_view label, const mgp::Value &properties_val, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory, const std::set<std::string_view> &existing_existence_constraints) {
|
||||
ExistenceConstraintsStorage asserted_existence_constraints;
|
||||
if (!properties_val.IsList()) {
|
||||
return asserted_existence_constraints;
|
||||
}
|
||||
|
||||
auto validate_property = [](const mgp::Value &property) -> bool {
|
||||
return property.IsString() && !property.ValueString().empty();
|
||||
};
|
||||
|
||||
const auto &properties = properties_val.ValueList();
|
||||
std::for_each(properties.begin(), properties.end(),
|
||||
[&label, &existing_existence_constraints, &asserted_existence_constraints, &memgraph_graph,
|
||||
&record_factory, &validate_property](const mgp::Value &property) {
|
||||
if (!validate_property(property)) {
|
||||
return;
|
||||
}
|
||||
const std::string_view property_str = property.ValueString();
|
||||
asserted_existence_constraints.emplace(property_str);
|
||||
ProcessCreatingLabelPropertyIndexAndExistenceConstraint(
|
||||
label, property_str, existing_existence_constraints, mgp::CreateExistenceConstraint,
|
||||
memgraph_graph, record_factory);
|
||||
});
|
||||
return asserted_existence_constraints;
|
||||
}
|
||||
|
||||
void ProcessExistenceConstraints(const mgp::Map &existence_constraints_map, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory, bool drop_existing) {
|
||||
auto mgp_existing_existence_constraints = mgp::ListAllExistenceConstraints(memgraph_graph);
|
||||
std::set<std::string_view> existing_existence_constraints;
|
||||
std::transform(mgp_existing_existence_constraints.begin(), mgp_existing_existence_constraints.end(),
|
||||
std::inserter(existing_existence_constraints, existing_existence_constraints.begin()),
|
||||
[](const mgp::Value &constraint) { return constraint.ValueString(); });
|
||||
|
||||
auto merge_label_property = [](const std::string_view label, const std::string_view property) {
|
||||
auto str = std::string(label) + ":";
|
||||
str += property;
|
||||
return str;
|
||||
};
|
||||
|
||||
ExistenceConstraintsStorage asserted_existence_constraints;
|
||||
|
||||
for (const auto &existing_constraint : existence_constraints_map) {
|
||||
const std::string_view label = existing_constraint.key;
|
||||
const mgp::Value &properties_val = existing_constraint.value;
|
||||
auto asserted_existence_constraints_new = CreateExistenceConstraintsForLabel(
|
||||
label, properties_val, memgraph_graph, record_factory, existing_existence_constraints);
|
||||
if (!drop_existing) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::ranges::for_each(asserted_existence_constraints_new, [&asserted_existence_constraints, &merge_label_property,
|
||||
label](const std::string_view property) {
|
||||
asserted_existence_constraints.emplace(merge_label_property(label, property));
|
||||
});
|
||||
}
|
||||
|
||||
if (!drop_existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::set<std::string_view> existence_constraints_to_drop;
|
||||
std::ranges::set_difference(existing_existence_constraints, asserted_existence_constraints,
|
||||
std::inserter(existence_constraints_to_drop, existence_constraints_to_drop.begin()));
|
||||
|
||||
auto decouple_label_property = [](const std::string_view label_property) {
|
||||
const auto label_size = label_property.find(':');
|
||||
const auto label = std::string(label_property.substr(0, label_size));
|
||||
const auto property = std::string(label_property.substr(label_size + 1));
|
||||
return std::make_pair(label, property);
|
||||
};
|
||||
|
||||
std::ranges::for_each(existence_constraints_to_drop, [&](const std::string_view label_property) {
|
||||
const auto [label, property] = decouple_label_property(label_property);
|
||||
if (mgp::DropExistenceConstraint(memgraph_graph, label, property)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
using AssertedUniqueConstraintsStorage = std::set<std::set<std::string_view>>;
|
||||
AssertedUniqueConstraintsStorage CreateUniqueConstraintsForLabel(
|
||||
const std::string_view label, const mgp::Value &unique_props_nested,
|
||||
const std::map<std::string_view, AssertedUniqueConstraintsStorage> &existing_unique_constraints,
|
||||
mgp_graph *memgraph_graph, const auto &record_factory) {
|
||||
AssertedUniqueConstraintsStorage asserted_unique_constraints;
|
||||
if (!unique_props_nested.IsList()) {
|
||||
return asserted_unique_constraints;
|
||||
}
|
||||
|
||||
auto validate_unique_constraint_props = [](const mgp::Value &properties) -> bool {
|
||||
if (!properties.IsList()) {
|
||||
return false;
|
||||
}
|
||||
const auto &properties_list = properties.ValueList();
|
||||
if (properties_list.Empty()) {
|
||||
return false;
|
||||
}
|
||||
return std::all_of(properties_list.begin(), properties_list.end(), [](const mgp::Value &property) {
|
||||
return property.IsString() && !property.ValueString().empty();
|
||||
});
|
||||
};
|
||||
|
||||
auto unique_constraint_exists =
|
||||
[](const std::string_view label, const std::set<std::string_view> &properties,
|
||||
const std::map<std::string_view, AssertedUniqueConstraintsStorage> &existing_unique_constraints) -> bool {
|
||||
auto iter = existing_unique_constraints.find(label);
|
||||
if (iter == existing_unique_constraints.end()) {
|
||||
return false;
|
||||
}
|
||||
return iter->second.find(properties) != iter->second.end();
|
||||
};
|
||||
|
||||
for (const auto unique_props_nested_list = unique_props_nested.ValueList();
|
||||
const auto &properties : unique_props_nested_list) {
|
||||
if (!validate_unique_constraint_props(properties)) {
|
||||
continue;
|
||||
}
|
||||
const auto properties_list = properties.ValueList();
|
||||
std::set<std::string_view> properties_coll;
|
||||
std::transform(properties_list.begin(), properties_list.end(),
|
||||
std::inserter(properties_coll, properties_coll.begin()),
|
||||
[](const mgp::Value &property) { return property.ValueString(); });
|
||||
|
||||
if (unique_constraint_exists(label, properties_coll, existing_unique_constraints)) {
|
||||
InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusKept);
|
||||
} else if (mgp::CreateUniqueConstraint(memgraph_graph, label, properties.ptr())) {
|
||||
InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusCreated);
|
||||
}
|
||||
asserted_unique_constraints.emplace(std::move(properties_coll));
|
||||
}
|
||||
return asserted_unique_constraints;
|
||||
}
|
||||
|
||||
void ProcessUniqueConstraints(const mgp::Map &unique_constraints_map, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory, bool drop_existing) {
|
||||
auto mgp_existing_unique_constraints = mgp::ListAllUniqueConstraints(memgraph_graph);
|
||||
// label-unique_constraints pair
|
||||
std::map<std::string_view, AssertedUniqueConstraintsStorage> existing_unique_constraints;
|
||||
for (const auto &constraint : mgp_existing_unique_constraints) {
|
||||
auto constraint_list = constraint.ValueList();
|
||||
std::set<std::string_view> properties;
|
||||
for (int i = 1; i < constraint_list.Size(); i++) {
|
||||
properties.emplace(constraint_list[i].ValueString());
|
||||
}
|
||||
const std::string_view label = constraint_list[0].ValueString();
|
||||
auto [it, inserted] = existing_unique_constraints.try_emplace(label, AssertedUniqueConstraintsStorage{properties});
|
||||
if (!inserted) {
|
||||
it->second.emplace(std::move(properties));
|
||||
}
|
||||
}
|
||||
|
||||
std::map<std::string_view, AssertedUniqueConstraintsStorage> asserted_unique_constraints;
|
||||
|
||||
for (const auto &[label, unique_props_nested] : unique_constraints_map) {
|
||||
auto asserted_unique_constraints_new = CreateUniqueConstraintsForLabel(
|
||||
label, unique_props_nested, existing_unique_constraints, memgraph_graph, record_factory);
|
||||
if (drop_existing) {
|
||||
asserted_unique_constraints.emplace(label, std::move(asserted_unique_constraints_new));
|
||||
}
|
||||
}
|
||||
|
||||
if (!drop_existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string_view, std::set<std::string_view>>> unique_constraints_to_drop;
|
||||
|
||||
// Check for each label for we found existing constraint in the DB whether it was asserted.
|
||||
// If no unique constraint was found with label, we can drop all unique constraints for this label. (if branch)
|
||||
// If some unique constraint was found with label, we can drop only those unique constraints that were not asserted.
|
||||
// (else branch.)
|
||||
std::ranges::for_each(existing_unique_constraints, [&asserted_unique_constraints, &unique_constraints_to_drop](
|
||||
const auto &existing_label_unique_constraints) {
|
||||
const auto &label = existing_label_unique_constraints.first;
|
||||
const auto &existing_unique_constraints_for_label = existing_label_unique_constraints.second;
|
||||
const auto &asserted_unique_constraints_for_label = asserted_unique_constraints.find(label);
|
||||
if (asserted_unique_constraints_for_label == asserted_unique_constraints.end()) {
|
||||
std::ranges::for_each(
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.begin()),
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.end()),
|
||||
[&unique_constraints_to_drop, &label](std::set<std::string_view> existing_unique_constraint_for_label) {
|
||||
unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label));
|
||||
});
|
||||
} else {
|
||||
const auto &asserted_unique_constraints_for_label_coll = asserted_unique_constraints_for_label->second;
|
||||
std::ranges::for_each(
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.begin()),
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.end()),
|
||||
[&unique_constraints_to_drop, &label, &asserted_unique_constraints_for_label_coll](
|
||||
std::set<std::string_view> existing_unique_constraint_for_label) {
|
||||
if (!asserted_unique_constraints_for_label_coll.contains(existing_unique_constraint_for_label)) {
|
||||
unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
std::ranges::for_each(
|
||||
unique_constraints_to_drop, [memgraph_graph, &record_factory](const auto &label_unique_constraint) {
|
||||
const auto &[label, unique_constraint] = label_unique_constraint;
|
||||
|
||||
auto unique_constraint_list = mgp::List();
|
||||
std::ranges::for_each(unique_constraint, [&unique_constraint_list](const std::string_view &property) {
|
||||
unique_constraint_list.AppendExtend(mgp::Value(property));
|
||||
});
|
||||
|
||||
if (mgp::DropUniqueConstraint(memgraph_graph, label, mgp::Value(unique_constraint_list).ptr())) {
|
||||
InsertRecordForUniqueConstraint(record_factory, label, unique_constraint_list, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
auto arguments = mgp::List(args);
|
||||
auto indices_map = arguments[0].ValueMap();
|
||||
auto unique_constraints_map = arguments[1].ValueMap();
|
||||
auto existence_constraints_map = arguments[2].ValueMap();
|
||||
auto drop_existing = arguments[3].ValueBool();
|
||||
|
||||
ProcessIndices(indices_map, memgraph_graph, record_factory, drop_existing);
|
||||
ProcessExistenceConstraints(existence_constraints_map, memgraph_graph, record_factory, drop_existing);
|
||||
ProcessUniqueConstraints(unique_constraints_map, memgraph_graph, record_factory, drop_existing);
|
||||
}
|
||||
|
||||
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
|
||||
try {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
;
|
||||
|
||||
AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read,
|
||||
{},
|
||||
{mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
|
||||
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
|
||||
AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {},
|
||||
{mgp::Return(Schema::kReturnNodeType, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnLabels, {mgp::Type::List, mgp::Type::String}),
|
||||
mgp::Return(Schema::kReturnPropertyName, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any),
|
||||
mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)},
|
||||
module, memory);
|
||||
|
||||
AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read,
|
||||
{},
|
||||
{mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
|
||||
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
|
||||
AddProcedure(Schema::RelTypeProperties, Schema::kProcedureRelType, mgp::ProcedureType::Read, {},
|
||||
{mgp::Return(Schema::kReturnRelType, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnPropertyName, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any),
|
||||
mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)},
|
||||
module, memory);
|
||||
|
||||
AddProcedure(
|
||||
Schema::Assert, Schema::kProcedureAssert, mgp::ProcedureType::Read,
|
||||
{
|
||||
mgp::Parameter(Schema::kParameterIndices, {mgp::Type::Map, mgp::Type::Any}),
|
||||
mgp::Parameter(Schema::kParameterUniqueConstraints, {mgp::Type::Map, mgp::Type::Any}),
|
||||
mgp::Parameter(Schema::kParameterExistenceConstraints, {mgp::Type::Map, mgp::Type::Any},
|
||||
mgp::Value(mgp::Map{})),
|
||||
mgp::Parameter(Schema::kParameterDropExisting, mgp::Type::Bool, mgp::Value(true)),
|
||||
},
|
||||
{mgp::Return(Schema::kReturnLabel, mgp::Type::String), mgp::Return(Schema::kReturnKey, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnKeys, {mgp::Type::List, mgp::Type::String}),
|
||||
mgp::Return(Schema::kReturnUnique, mgp::Type::Bool), mgp::Return(Schema::kReturnAction, mgp::Type::String)},
|
||||
module, memory);
|
||||
} catch (const std::exception &e) {
|
||||
std::cerr << "Error while initializing query module: " << e.what() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
@@ -104,7 +104,9 @@ def retry(retry_limit, timeout=100):
|
||||
except Exception:
|
||||
time.sleep(timeout)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return inner_func
|
||||
|
||||
|
||||
@@ -200,19 +202,19 @@ if args.version:
|
||||
try:
|
||||
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
|
||||
if current_branch != "master":
|
||||
branches = get_output("git", "branch")
|
||||
if "master" in branches:
|
||||
branches = get_output("git", "branch", "-r", "--list", "origin/master")
|
||||
if "origin/master" in branches:
|
||||
# If master is present locally, the fetch is allowed to fail
|
||||
# because this script will still be able to compare against the
|
||||
# master branch.
|
||||
try:
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
get_output("git", "fetch", "origin", "master")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# If master is not present locally, the fetch command has to
|
||||
# succeed because something else will fail otherwise.
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
get_output("git", "fetch", "origin", "master")
|
||||
except Exception:
|
||||
print("Fatal error while ensuring local master branch.")
|
||||
sys.exit(1)
|
||||
@@ -232,7 +234,7 @@ for branch in branches:
|
||||
match = branch_regex.match(branch)
|
||||
if match is not None:
|
||||
version = tuple(map(int, match.group(1).split(".")))
|
||||
master_branch_merge = get_output("git", "merge-base", "master", branch)
|
||||
master_branch_merge = get_output("git", "merge-base", "origin/master", branch)
|
||||
versions.append((version, branch, master_branch_merge))
|
||||
versions.sort(reverse=True)
|
||||
|
||||
@@ -243,7 +245,7 @@ current_version = None
|
||||
for version in versions:
|
||||
version_tuple, branch, master_branch_merge = version
|
||||
current_branch_merge = get_output("git", "merge-base", current_hash, branch)
|
||||
master_current_merge = get_output("git", "merge-base", current_hash, "master")
|
||||
master_current_merge = get_output("git", "merge-base", current_hash, "origin/master")
|
||||
# The first check checks whether this commit is a child of `master` and
|
||||
# the version branch was created before us.
|
||||
# The second check checks whether this commit is a child of the version
|
||||
|
||||
@@ -22,6 +22,10 @@ add_subdirectory(dbms)
|
||||
add_subdirectory(flags)
|
||||
add_subdirectory(distributed)
|
||||
add_subdirectory(replication)
|
||||
add_subdirectory(replication_handler)
|
||||
add_subdirectory(coordination)
|
||||
add_subdirectory(replication_coordination_glue)
|
||||
add_subdirectory(system)
|
||||
|
||||
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
|
||||
@@ -41,10 +45,10 @@ set(mg_single_node_v2_sources
|
||||
add_executable(memgraph ${mg_single_node_v2_sources})
|
||||
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(memgraph stdc++fs Threads::Threads
|
||||
mg-telemetry mg-communication mg-communication-metrics mg-memory mg-utils mg-license mg-settings mg-glue mg-flags)
|
||||
mg-telemetry mg-communication mg-communication-metrics mg-memory mg-utils mg-license mg-settings mg-glue mg-flags mg::system mg::replication_handler)
|
||||
|
||||
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
|
||||
# should be dynamically exported, so that `dlopen` can correctly link the
|
||||
# should be dynamically exported, so that `dlopen` can correctly link th
|
||||
# symbols in custom procedure module libraries.
|
||||
target_link_libraries(memgraph "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
|
||||
set_target_properties(memgraph PROPERTIES
|
||||
@@ -53,7 +57,10 @@ set_target_properties(memgraph PROPERTIES
|
||||
OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
|
||||
|
||||
# Output the executable in main binary dir.
|
||||
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}
|
||||
|
||||
POSITION_INDEPENDENT_CODE ON
|
||||
)
|
||||
|
||||
# Create symlink to the built executable.
|
||||
add_custom_command(TARGET memgraph POST_BUILD
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <json/json.hpp>
|
||||
#include <utility>
|
||||
|
||||
#include "storage/v2/temporal.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -87,8 +88,8 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
|
||||
: storage_directory_(storage_directory),
|
||||
Log::Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
|
||||
: storage_directory_(std::move(storage_directory)),
|
||||
buffer_size_(buffer_size),
|
||||
buffer_flush_interval_millis_(buffer_flush_interval_millis),
|
||||
started_(false) {}
|
||||
|
||||
@@ -36,7 +36,7 @@ class Log {
|
||||
};
|
||||
|
||||
public:
|
||||
Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
|
||||
Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
|
||||
|
||||
~Log();
|
||||
|
||||
|
||||
@@ -2,7 +2,9 @@ set(auth_src_files
|
||||
auth.cpp
|
||||
crypto.cpp
|
||||
models.cpp
|
||||
module.cpp)
|
||||
module.cpp
|
||||
rpc.cpp
|
||||
replication_handlers.cpp)
|
||||
|
||||
find_package(Seccomp REQUIRED)
|
||||
find_package(fmt REQUIRED)
|
||||
@@ -11,7 +13,7 @@ find_package(gflags REQUIRED)
|
||||
|
||||
add_library(mg-auth STATIC ${auth_src_files})
|
||||
target_link_libraries(mg-auth json libbcrypt gflags fmt::fmt)
|
||||
target_link_libraries(mg-auth mg-utils mg-kvstore mg-license )
|
||||
target_link_libraries(mg-auth mg-utils mg-kvstore mg-license mg::system mg-replication)
|
||||
|
||||
target_link_libraries(mg-auth ${Seccomp_LIBRARIES})
|
||||
target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -8,17 +8,18 @@
|
||||
|
||||
#include "auth/auth.hpp"
|
||||
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "auth/crypto.hpp"
|
||||
#include "auth/exceptions.hpp"
|
||||
#include "auth/rpc.hpp"
|
||||
#include "license/license.hpp"
|
||||
#include "system/transaction.hpp"
|
||||
#include "utils/flag_validation.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/settings.hpp"
|
||||
#include "utils/string.hpp"
|
||||
@@ -43,9 +44,89 @@ DEFINE_VALIDATED_int32(auth_module_timeout_ms, 10000,
|
||||
FLAG_IN_RANGE(100, 1800000));
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
namespace {
|
||||
#ifdef MG_ENTERPRISE
|
||||
/**
|
||||
* REPLICATION SYSTEM ACTION IMPLEMENTATIONS
|
||||
*/
|
||||
struct UpdateAuthData : memgraph::system::ISystemAction {
|
||||
explicit UpdateAuthData(User user) : user_{std::move(user)}, role_{std::nullopt} {}
|
||||
explicit UpdateAuthData(Role role) : user_{std::nullopt}, role_{std::move(role)} {}
|
||||
|
||||
void DoDurability() override { /* Done during Auth execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const replication::UpdateAuthDataRes &response) { return response.success; };
|
||||
if (user_) {
|
||||
return client.SteamAndFinalizeDelta<replication::UpdateAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
*user_);
|
||||
}
|
||||
if (role_) {
|
||||
return client.SteamAndFinalizeDelta<replication::UpdateAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
*role_);
|
||||
}
|
||||
// Should never get here
|
||||
MG_ASSERT(false, "Trying to update auth data that is not a user nor a role");
|
||||
return {};
|
||||
}
|
||||
|
||||
void PostReplication(replication::RoleMainData &mainData) const override {}
|
||||
|
||||
private:
|
||||
std::optional<User> user_;
|
||||
std::optional<Role> role_;
|
||||
};
|
||||
|
||||
struct DropAuthData : memgraph::system::ISystemAction {
|
||||
enum class AuthDataType { USER, ROLE };
|
||||
|
||||
explicit DropAuthData(AuthDataType type, std::string_view name) : type_{type}, name_{name} {}
|
||||
|
||||
void DoDurability() override { /* Done during Auth execution */
|
||||
}
|
||||
|
||||
bool DoReplication(replication::ReplicationClient &client, const utils::UUID &main_uuid,
|
||||
replication::ReplicationEpoch const &epoch,
|
||||
memgraph::system::Transaction const &txn) const override {
|
||||
auto check_response = [](const replication::DropAuthDataRes &response) { return response.success; };
|
||||
|
||||
memgraph::replication::DropAuthDataReq::DataType type{};
|
||||
switch (type_) {
|
||||
case AuthDataType::USER:
|
||||
type = memgraph::replication::DropAuthDataReq::DataType::USER;
|
||||
break;
|
||||
case AuthDataType::ROLE:
|
||||
type = memgraph::replication::DropAuthDataReq::DataType::ROLE;
|
||||
break;
|
||||
}
|
||||
return client.SteamAndFinalizeDelta<replication::DropAuthDataRpc>(
|
||||
check_response, main_uuid, std::string{epoch.id()}, txn.last_committed_system_timestamp(), txn.timestamp(),
|
||||
type, name_);
|
||||
}
|
||||
void PostReplication(replication::RoleMainData &mainData) const override {}
|
||||
|
||||
private:
|
||||
AuthDataType type_;
|
||||
std::string name_;
|
||||
};
|
||||
#endif
|
||||
|
||||
/**
|
||||
* CONSTANTS
|
||||
*/
|
||||
const std::string kUserPrefix = "user:";
|
||||
const std::string kRolePrefix = "role:";
|
||||
const std::string kLinkPrefix = "link:";
|
||||
const std::string kVersion = "version";
|
||||
|
||||
static constexpr auto kVersionV1 = "V1";
|
||||
} // namespace
|
||||
|
||||
/**
|
||||
* All data stored in the `Auth` storage is stored in an underlying
|
||||
@@ -64,7 +145,59 @@ const std::string kLinkPrefix = "link:";
|
||||
* key="link:<username>", value="<rolename>"
|
||||
*/
|
||||
|
||||
Auth::Auth(const std::string &storage_directory) : storage_(storage_directory), module_(FLAGS_auth_module_executable) {}
|
||||
namespace {
|
||||
void MigrateVersions(kvstore::KVStore &store) {
|
||||
static constexpr auto kPasswordHashV0V1 = "password_hash";
|
||||
auto version_str = store.Get(kVersion);
|
||||
|
||||
if (!version_str) {
|
||||
using namespace std::string_literals;
|
||||
|
||||
// pre versioning, add version to the store
|
||||
auto puts = std::map<std::string, std::string>{{kVersion, kVersionV1}};
|
||||
|
||||
// also add hash kind into durability
|
||||
|
||||
auto it = store.begin(kUserPrefix);
|
||||
auto const e = store.end(kUserPrefix);
|
||||
|
||||
if (it != e) {
|
||||
const auto hash_algo = CurrentHashAlgorithm();
|
||||
spdlog::info("Updating auth durability, assuming previously stored as {}", AsString(hash_algo));
|
||||
|
||||
for (; it != e; ++it) {
|
||||
auto const &[key, value] = *it;
|
||||
try {
|
||||
auto user_data = nlohmann::json::parse(value);
|
||||
|
||||
auto password_hash = user_data[kPasswordHashV0V1];
|
||||
if (!password_hash.is_string()) {
|
||||
throw AuthException("Couldn't load user data!");
|
||||
}
|
||||
// upgrade the password_hash to include the hash algortihm
|
||||
if (password_hash.empty()) {
|
||||
user_data[kPasswordHashV0V1] = nullptr;
|
||||
} else {
|
||||
user_data[kPasswordHashV0V1] = HashedPassword{hash_algo, password_hash};
|
||||
}
|
||||
puts.emplace(key, user_data.dump());
|
||||
} catch (const nlohmann::json::parse_error &e) {
|
||||
throw AuthException("Couldn't load user data!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Perform migration to V1
|
||||
store.PutMultiple(puts);
|
||||
version_str = kVersionV1;
|
||||
}
|
||||
}
|
||||
}; // namespace
|
||||
|
||||
Auth::Auth(std::string storage_directory, Config config)
|
||||
: storage_(std::move(storage_directory)), module_(FLAGS_auth_module_executable), config_{std::move(config)} {
|
||||
MigrateVersions(storage_);
|
||||
}
|
||||
|
||||
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
|
||||
if (module_.IsUsed()) {
|
||||
@@ -95,6 +228,12 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
|
||||
// Authenticate the user.
|
||||
if (!is_authenticated) return std::nullopt;
|
||||
|
||||
/**
|
||||
* TODO
|
||||
* The auth module should not update auth data.
|
||||
* There is now way to replicate it and we should not be storing sensitive data if we don't have to.
|
||||
*/
|
||||
|
||||
// Find or create the user and return it.
|
||||
auto user = GetUser(username);
|
||||
if (!user) {
|
||||
@@ -113,7 +252,7 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
|
||||
return std::nullopt;
|
||||
}
|
||||
} else {
|
||||
user->UpdatePassword(password);
|
||||
UpdatePassword(*user, password);
|
||||
}
|
||||
if (FLAGS_auth_module_manage_roles) {
|
||||
if (!rolename.empty()) {
|
||||
@@ -155,6 +294,10 @@ std::optional<User> Auth::Authenticate(const std::string &username, const std::s
|
||||
username, "https://memgr.ph/auth"));
|
||||
return std::nullopt;
|
||||
}
|
||||
if (user->UpgradeHash(password)) {
|
||||
SaveUser(*user);
|
||||
}
|
||||
|
||||
return user;
|
||||
}
|
||||
}
|
||||
@@ -183,7 +326,7 @@ std::optional<User> Auth::GetUser(const std::string &username_orig) const {
|
||||
return user;
|
||||
}
|
||||
|
||||
void Auth::SaveUser(const User &user) {
|
||||
void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
|
||||
bool success = false;
|
||||
if (const auto *role = user.role(); role != nullptr) {
|
||||
success = storage_.PutMultiple(
|
||||
@@ -195,26 +338,72 @@ void Auth::SaveUser(const User &user) {
|
||||
if (!success) {
|
||||
throw AuthException("Couldn't save user '{}'!", user.username());
|
||||
}
|
||||
// All changes to the user end up calling this function, so no need to add a delta anywhere else
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
system_tx->AddAction<UpdateAuthData>(user);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<User> Auth::AddUser(const std::string &username, const std::optional<std::string> &password) {
|
||||
void Auth::UpdatePassword(auth::User &user, const std::optional<std::string> &password) {
|
||||
// Check if null
|
||||
if (!password) {
|
||||
if (!config_.password_permit_null) {
|
||||
throw AuthException("Null passwords aren't permitted!");
|
||||
}
|
||||
} else {
|
||||
// Check if compliant with our filter
|
||||
if (config_.custom_password_regex) {
|
||||
if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
|
||||
license_check_result.HasError()) {
|
||||
throw AuthException(
|
||||
"Custom password regex is a Memgraph Enterprise feature. Please set the config "
|
||||
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
|
||||
glue::kDefaultPasswordRegex,
|
||||
license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
|
||||
}
|
||||
}
|
||||
if (!std::regex_match(*password, config_.password_regex)) {
|
||||
throw AuthException(
|
||||
"The user password doesn't conform to the required strength! Regex: "
|
||||
"\"{}\"",
|
||||
config_.password_regex_str);
|
||||
}
|
||||
}
|
||||
|
||||
// All checks passed; update
|
||||
user.UpdatePassword(password);
|
||||
}
|
||||
|
||||
std::optional<User> Auth::AddUser(const std::string &username, const std::optional<std::string> &password,
|
||||
system::Transaction *system_tx) {
|
||||
if (!NameRegexMatch(username)) {
|
||||
throw AuthException("Invalid user name.");
|
||||
}
|
||||
auto existing_user = GetUser(username);
|
||||
if (existing_user) return std::nullopt;
|
||||
auto existing_role = GetRole(username);
|
||||
if (existing_role) return std::nullopt;
|
||||
auto new_user = User(username);
|
||||
new_user.UpdatePassword(password);
|
||||
SaveUser(new_user);
|
||||
UpdatePassword(new_user, password);
|
||||
SaveUser(new_user, system_tx);
|
||||
return new_user;
|
||||
}
|
||||
|
||||
bool Auth::RemoveUser(const std::string &username_orig) {
|
||||
bool Auth::RemoveUser(const std::string &username_orig, system::Transaction *system_tx) {
|
||||
auto username = utils::ToLowerCase(username_orig);
|
||||
if (!storage_.Get(kUserPrefix + username)) return false;
|
||||
std::vector<std::string> keys({kLinkPrefix + username, kUserPrefix + username});
|
||||
if (!storage_.DeleteMultiple(keys)) {
|
||||
throw AuthException("Couldn't remove user '{}'!", username);
|
||||
}
|
||||
// Handling drop user delta
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
system_tx->AddAction<DropAuthData>(DropAuthData::AuthDataType::USER, username);
|
||||
#endif
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -231,6 +420,19 @@ std::vector<auth::User> Auth::AllUsers() const {
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<std::string> Auth::AllUsernames() const {
|
||||
std::vector<std::string> ret;
|
||||
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
|
||||
auto username = it->first.substr(kUserPrefix.size());
|
||||
if (username != utils::ToLowerCase(username)) continue;
|
||||
auto user = GetUser(username);
|
||||
if (user) {
|
||||
ret.push_back(username);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool Auth::HasUsers() const { return storage_.begin(kUserPrefix) != storage_.end(kUserPrefix); }
|
||||
|
||||
std::optional<Role> Auth::GetRole(const std::string &rolename_orig) const {
|
||||
@@ -248,23 +450,30 @@ std::optional<Role> Auth::GetRole(const std::string &rolename_orig) const {
|
||||
return Role::Deserialize(data);
|
||||
}
|
||||
|
||||
void Auth::SaveRole(const Role &role) {
|
||||
void Auth::SaveRole(const Role &role, system::Transaction *system_tx) {
|
||||
if (!storage_.Put(kRolePrefix + role.rolename(), role.Serialize().dump())) {
|
||||
throw AuthException("Couldn't save role '{}'!", role.rolename());
|
||||
}
|
||||
// All changes to the role end up calling this function, so no need to add a delta anywhere else
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
system_tx->AddAction<UpdateAuthData>(role);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<Role> Auth::AddRole(const std::string &rolename) {
|
||||
auto existing_role = GetRole(rolename);
|
||||
if (existing_role) return std::nullopt;
|
||||
auto existing_user = GetUser(rolename);
|
||||
if (existing_user) return std::nullopt;
|
||||
std::optional<Role> Auth::AddRole(const std::string &rolename, system::Transaction *system_tx) {
|
||||
if (!NameRegexMatch(rolename)) {
|
||||
throw AuthException("Invalid role name.");
|
||||
}
|
||||
if (auto existing_role = GetRole(rolename)) return std::nullopt;
|
||||
if (auto existing_user = GetUser(rolename)) return std::nullopt;
|
||||
auto new_role = Role(rolename);
|
||||
SaveRole(new_role);
|
||||
SaveRole(new_role, system_tx);
|
||||
return new_role;
|
||||
}
|
||||
|
||||
bool Auth::RemoveRole(const std::string &rolename_orig) {
|
||||
bool Auth::RemoveRole(const std::string &rolename_orig, system::Transaction *system_tx) {
|
||||
auto rolename = utils::ToLowerCase(rolename_orig);
|
||||
if (!storage_.Get(kRolePrefix + rolename)) return false;
|
||||
std::vector<std::string> keys;
|
||||
@@ -277,6 +486,12 @@ bool Auth::RemoveRole(const std::string &rolename_orig) {
|
||||
if (!storage_.DeleteMultiple(keys)) {
|
||||
throw AuthException("Couldn't remove role '{}'!", rolename);
|
||||
}
|
||||
// Handling drop role delta
|
||||
if (system_tx) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
system_tx->AddAction<DropAuthData>(DropAuthData::AuthDataType::ROLE, rolename);
|
||||
#endif
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -285,8 +500,7 @@ std::vector<auth::Role> Auth::AllRoles() const {
|
||||
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
|
||||
auto rolename = it->first.substr(kRolePrefix.size());
|
||||
if (rolename != utils::ToLowerCase(rolename)) continue;
|
||||
auto role = GetRole(rolename);
|
||||
if (role) {
|
||||
if (auto role = GetRole(rolename)) {
|
||||
ret.push_back(*role);
|
||||
} else {
|
||||
throw AuthException("Couldn't load role '{}'!", rolename);
|
||||
@@ -295,16 +509,27 @@ std::vector<auth::Role> Auth::AllRoles() const {
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<std::string> Auth::AllRolenames() const {
|
||||
std::vector<std::string> ret;
|
||||
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
|
||||
auto rolename = it->first.substr(kRolePrefix.size());
|
||||
if (rolename != utils::ToLowerCase(rolename)) continue;
|
||||
if (auto role = GetRole(rolename)) {
|
||||
ret.push_back(rolename);
|
||||
}
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig) const {
|
||||
auto rolename = utils::ToLowerCase(rolename_orig);
|
||||
const auto rolename = utils::ToLowerCase(rolename_orig);
|
||||
std::vector<auth::User> ret;
|
||||
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix); ++it) {
|
||||
auto username = it->first.substr(kLinkPrefix.size());
|
||||
if (username != utils::ToLowerCase(username)) continue;
|
||||
if (it->second != utils::ToLowerCase(it->second)) continue;
|
||||
if (it->second == rolename) {
|
||||
auto user = GetUser(username);
|
||||
if (user) {
|
||||
if (auto user = GetUser(username)) {
|
||||
ret.push_back(std::move(*user));
|
||||
} else {
|
||||
throw AuthException("Couldn't load user '{}'!", username);
|
||||
@@ -315,56 +540,67 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name) {
|
||||
auto user = GetUser(name);
|
||||
if (user) {
|
||||
bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
if (auto user = GetUser(name)) {
|
||||
if (db == kAllDatabases) {
|
||||
user->db_access().GrantAll();
|
||||
} else {
|
||||
user->db_access().Add(db);
|
||||
}
|
||||
SaveUser(*user);
|
||||
SaveUser(*user, system_tx);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name) {
|
||||
auto user = GetUser(name);
|
||||
if (user) {
|
||||
bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name, system::Transaction *system_tx) {
|
||||
if (auto user = GetUser(name)) {
|
||||
if (db == kAllDatabases) {
|
||||
user->db_access().DenyAll();
|
||||
} else {
|
||||
user->db_access().Remove(db);
|
||||
}
|
||||
SaveUser(*user);
|
||||
SaveUser(*user, system_tx);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void Auth::DeleteDatabase(const std::string &db) {
|
||||
void Auth::DeleteDatabase(const std::string &db, system::Transaction *system_tx) {
|
||||
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
|
||||
auto username = it->first.substr(kUserPrefix.size());
|
||||
auto user = GetUser(username);
|
||||
if (user) {
|
||||
if (auto user = GetUser(username)) {
|
||||
user->db_access().Delete(db);
|
||||
SaveUser(*user);
|
||||
SaveUser(*user, system_tx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Auth::SetMainDatabase(const std::string &db, const std::string &name) {
|
||||
auto user = GetUser(name);
|
||||
if (user) {
|
||||
bool Auth::SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx) {
|
||||
if (auto user = GetUser(name)) {
|
||||
if (!user->db_access().SetDefault(db)) {
|
||||
throw AuthException("Couldn't set default database '{}' for user '{}'!", db, name);
|
||||
}
|
||||
SaveUser(*user);
|
||||
SaveUser(*user, system_tx);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool Auth::NameRegexMatch(const std::string &user_or_role) const {
|
||||
if (config_.custom_name_regex) {
|
||||
if (const auto license_check_result =
|
||||
memgraph::license::global_license_checker.IsEnterpriseValid(memgraph::utils::global_settings);
|
||||
license_check_result.HasError()) {
|
||||
throw memgraph::auth::AuthException(
|
||||
"Custom user/role regex is a Memgraph Enterprise feature. Please set the config "
|
||||
"(\"--auth-user-or-role-name-regex\") to its default value (\"{}\") or remove the flag.\n{}",
|
||||
glue::kDefaultUserRoleRegex,
|
||||
memgraph::license::LicenseCheckErrorToString(license_check_result.GetError(), "user/role regex"));
|
||||
}
|
||||
}
|
||||
return std::regex_match(user_or_role, config_.name_regex);
|
||||
}
|
||||
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -10,16 +10,23 @@
|
||||
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <regex>
|
||||
#include <vector>
|
||||
|
||||
#include "auth/exceptions.hpp"
|
||||
#include "auth/models.hpp"
|
||||
#include "auth/module.hpp"
|
||||
#include "glue/auth_global.hpp"
|
||||
#include "kvstore/kvstore.hpp"
|
||||
#include "system/action.hpp"
|
||||
#include "utils/settings.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
class Auth;
|
||||
using SynchedAuth = memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock>;
|
||||
|
||||
static const constexpr char *const kAllDatabases = "*";
|
||||
|
||||
/**
|
||||
@@ -31,7 +38,47 @@ static const constexpr char *const kAllDatabases = "*";
|
||||
*/
|
||||
class Auth final {
|
||||
public:
|
||||
explicit Auth(const std::string &storage_directory);
|
||||
struct Config {
|
||||
Config() {}
|
||||
Config(std::string name_regex, std::string password_regex, bool password_permit_null)
|
||||
: name_regex_str{std::move(name_regex)},
|
||||
password_regex_str{std::move(password_regex)},
|
||||
password_permit_null{password_permit_null},
|
||||
custom_name_regex{name_regex_str != glue::kDefaultUserRoleRegex},
|
||||
name_regex{name_regex_str},
|
||||
custom_password_regex{password_regex_str != glue::kDefaultPasswordRegex},
|
||||
password_regex{password_regex_str} {}
|
||||
|
||||
std::string name_regex_str{glue::kDefaultUserRoleRegex};
|
||||
std::string password_regex_str{glue::kDefaultPasswordRegex};
|
||||
bool password_permit_null{true};
|
||||
|
||||
private:
|
||||
friend class Auth;
|
||||
bool custom_name_regex{false};
|
||||
std::regex name_regex{name_regex_str};
|
||||
bool custom_password_regex{false};
|
||||
std::regex password_regex{password_regex_str};
|
||||
};
|
||||
|
||||
explicit Auth(std::string storage_directory, Config config);
|
||||
|
||||
/**
|
||||
* @brief Set the Config object
|
||||
*
|
||||
* @param config
|
||||
*/
|
||||
void SetConfig(Config config) {
|
||||
// NOTE: The Auth class itself is not thread-safe, higher-level code needs to synchronize it when using it.
|
||||
config_ = std::move(config);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief
|
||||
*
|
||||
* @return Config
|
||||
*/
|
||||
Config GetConfig() const { return config_; }
|
||||
|
||||
/**
|
||||
* Authenticates a user using his username and password.
|
||||
@@ -61,7 +108,7 @@ class Auth final {
|
||||
*
|
||||
* @throw AuthException if unable to save the user.
|
||||
*/
|
||||
void SaveUser(const User &user);
|
||||
void SaveUser(const User &user, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* Creates a user if the user doesn't exist.
|
||||
@@ -72,7 +119,8 @@ class Auth final {
|
||||
* @return a user when the user is created, nullopt if the user exists
|
||||
* @throw AuthException if unable to save the user.
|
||||
*/
|
||||
std::optional<User> AddUser(const std::string &username, const std::optional<std::string> &password = std::nullopt);
|
||||
std::optional<User> AddUser(const std::string &username, const std::optional<std::string> &password = std::nullopt,
|
||||
system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* Removes a user from the storage.
|
||||
@@ -83,7 +131,15 @@ class Auth final {
|
||||
* doesn't exist
|
||||
* @throw AuthException if unable to remove the user.
|
||||
*/
|
||||
bool RemoveUser(const std::string &username);
|
||||
bool RemoveUser(const std::string &username, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief
|
||||
*
|
||||
* @param user
|
||||
* @param password
|
||||
*/
|
||||
void UpdatePassword(auth::User &user, const std::optional<std::string> &password);
|
||||
|
||||
/**
|
||||
* Gets all users from the storage.
|
||||
@@ -93,6 +149,13 @@ class Auth final {
|
||||
*/
|
||||
std::vector<User> AllUsers() const;
|
||||
|
||||
/**
|
||||
* @brief
|
||||
*
|
||||
* @return std::vector<std::string>
|
||||
*/
|
||||
std::vector<std::string> AllUsernames() const;
|
||||
|
||||
/**
|
||||
* Returns whether there are users in the storage.
|
||||
*
|
||||
@@ -117,7 +180,7 @@ class Auth final {
|
||||
*
|
||||
* @throw AuthException if unable to save the role.
|
||||
*/
|
||||
void SaveRole(const Role &role);
|
||||
void SaveRole(const Role &role, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* Creates a role if the role doesn't exist.
|
||||
@@ -127,7 +190,7 @@ class Auth final {
|
||||
* @return a role when the role is created, nullopt if the role exists
|
||||
* @throw AuthException if unable to save the role.
|
||||
*/
|
||||
std::optional<Role> AddRole(const std::string &rolename);
|
||||
std::optional<Role> AddRole(const std::string &rolename, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* Removes a role from the storage.
|
||||
@@ -138,7 +201,7 @@ class Auth final {
|
||||
* doesn't exist
|
||||
* @throw AuthException if unable to remove the role.
|
||||
*/
|
||||
bool RemoveRole(const std::string &rolename);
|
||||
bool RemoveRole(const std::string &rolename, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* Gets all roles from the storage.
|
||||
@@ -148,6 +211,13 @@ class Auth final {
|
||||
*/
|
||||
std::vector<Role> AllRoles() const;
|
||||
|
||||
/**
|
||||
* @brief
|
||||
*
|
||||
* @return std::vector<std::string>
|
||||
*/
|
||||
std::vector<std::string> AllRolenames() const;
|
||||
|
||||
/**
|
||||
* Gets all users for a role from the storage.
|
||||
*
|
||||
@@ -167,7 +237,7 @@ class Auth final {
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
bool RevokeDatabaseFromUser(const std::string &db, const std::string &name);
|
||||
bool RevokeDatabaseFromUser(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Grant access to individual database for a user.
|
||||
@@ -177,7 +247,7 @@ class Auth final {
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
bool GrantDatabaseToUser(const std::string &db, const std::string &name);
|
||||
bool GrantDatabaseToUser(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Delete a database from all users.
|
||||
@@ -185,7 +255,7 @@ class Auth final {
|
||||
* @param db name of the database to delete
|
||||
* @throw AuthException if unable to read data
|
||||
*/
|
||||
void DeleteDatabase(const std::string &db);
|
||||
void DeleteDatabase(const std::string &db, system::Transaction *system_tx = nullptr);
|
||||
|
||||
/**
|
||||
* @brief Set main database for an individual user.
|
||||
@@ -195,14 +265,24 @@ class Auth final {
|
||||
* @return true on success
|
||||
* @throw AuthException if unable to find or update the user
|
||||
*/
|
||||
bool SetMainDatabase(const std::string &db, const std::string &name);
|
||||
bool SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx = nullptr);
|
||||
#endif
|
||||
|
||||
private:
|
||||
/**
|
||||
* @brief
|
||||
*
|
||||
* @param user_or_role
|
||||
* @return true
|
||||
* @return false
|
||||
*/
|
||||
bool NameRegexMatch(const std::string &user_or_role) const;
|
||||
|
||||
// Even though the `kvstore::KVStore` class is guaranteed to be thread-safe,
|
||||
// Auth is not thread-safe because modifying users and roles might require
|
||||
// more than one operation on the storage.
|
||||
kvstore::KVStore storage_;
|
||||
auth::Module module_;
|
||||
Config config_;
|
||||
};
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -22,10 +22,14 @@
|
||||
|
||||
namespace {
|
||||
using namespace std::literals;
|
||||
inline constexpr std::array password_encryption_mappings{
|
||||
std::pair{"bcrypt"sv, memgraph::auth::PasswordEncryptionAlgorithm::BCRYPT},
|
||||
std::pair{"sha256"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256},
|
||||
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256_MULTIPLE}};
|
||||
|
||||
constexpr auto kHashAlgo = "hash_algo";
|
||||
constexpr auto kPasswordHash = "password_hash";
|
||||
|
||||
inline constexpr std::array password_hash_mappings{
|
||||
std::pair{"bcrypt"sv, memgraph::auth::PasswordHashAlgorithm::BCRYPT},
|
||||
std::pair{"sha256"sv, memgraph::auth::PasswordHashAlgorithm::SHA256},
|
||||
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordHashAlgorithm::SHA256_MULTIPLE}};
|
||||
|
||||
inline constexpr uint64_t ONE_SHA_ITERATION = 1;
|
||||
inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
|
||||
@@ -35,7 +39,7 @@ inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
|
||||
DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
|
||||
"The password encryption algorithm used for authentication.", {
|
||||
if (const auto result =
|
||||
memgraph::utils::IsValidEnumValueString(value, password_encryption_mappings);
|
||||
memgraph::utils::IsValidEnumValueString(value, password_hash_mappings);
|
||||
result.HasError()) {
|
||||
const auto error = result.GetError();
|
||||
switch (error) {
|
||||
@@ -45,7 +49,7 @@ DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
|
||||
}
|
||||
case memgraph::utils::ValidationError::InvalidValue: {
|
||||
std::cout << "Invalid value for password encryption algorithm. Allowed values: "
|
||||
<< memgraph::utils::GetAllowedEnumValuesString(password_encryption_mappings)
|
||||
<< memgraph::utils::GetAllowedEnumValuesString(password_hash_mappings)
|
||||
<< std::endl;
|
||||
break;
|
||||
}
|
||||
@@ -58,7 +62,7 @@ DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
|
||||
|
||||
namespace memgraph::auth {
|
||||
namespace BCrypt {
|
||||
std::string EncryptPassword(const std::string &password) {
|
||||
std::string HashPassword(const std::string &password) {
|
||||
char salt[BCRYPT_HASHSIZE];
|
||||
char hash[BCRYPT_HASHSIZE];
|
||||
|
||||
@@ -86,16 +90,30 @@ bool VerifyPassword(const std::string &password, const std::string &hash) {
|
||||
} // namespace BCrypt
|
||||
|
||||
namespace SHA {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr auto SHA_LENGTH = 64U;
|
||||
constexpr auto SALT_SIZE = 16U;
|
||||
constexpr auto SALT_SIZE_DURABLE = SALT_SIZE * 2;
|
||||
|
||||
#if OPENSSL_VERSION_MAJOR >= 3
|
||||
std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t number_of_iterations) {
|
||||
std::string HashPasswordOpenSSL3(std::string_view password, const uint64_t number_of_iterations,
|
||||
std::string_view salt) {
|
||||
unsigned char hash[SHA256_DIGEST_LENGTH];
|
||||
|
||||
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
|
||||
EVP_MD *md = EVP_MD_fetch(nullptr, "SHA2-256", nullptr);
|
||||
|
||||
EVP_DigestInit_ex(ctx, md, nullptr);
|
||||
|
||||
if (!salt.empty()) {
|
||||
DMG_ASSERT(salt.size() == SALT_SIZE);
|
||||
EVP_DigestUpdate(ctx, salt.data(), salt.size());
|
||||
}
|
||||
|
||||
for (auto i = 0; i < number_of_iterations; i++) {
|
||||
EVP_DigestUpdate(ctx, password.c_str(), password.size());
|
||||
EVP_DigestUpdate(ctx, password.data(), password.size());
|
||||
}
|
||||
EVP_DigestFinal_ex(ctx, hash, nullptr);
|
||||
|
||||
@@ -103,6 +121,11 @@ std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t
|
||||
EVP_MD_CTX_free(ctx);
|
||||
|
||||
std::stringstream result_stream;
|
||||
|
||||
for (unsigned char salt_char : salt) {
|
||||
result_stream << std::hex << std::setw(2) << std::setfill('0') << (((unsigned int)salt_char) & 0xFFU);
|
||||
}
|
||||
|
||||
for (auto hash_char : hash) {
|
||||
result_stream << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
|
||||
}
|
||||
@@ -110,17 +133,27 @@ std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t
|
||||
return result_stream.str();
|
||||
}
|
||||
#else
|
||||
std::string EncryptPasswordOpenSSL1_1(const std::string &password, const uint64_t number_of_iterations) {
|
||||
std::string HashPasswordOpenSSL1_1(std::string_view password, const uint64_t number_of_iterations,
|
||||
std::string_view salt) {
|
||||
unsigned char hash[SHA256_DIGEST_LENGTH];
|
||||
|
||||
SHA256_CTX sha256;
|
||||
SHA256_Init(&sha256);
|
||||
|
||||
if (!salt.empty()) {
|
||||
DMG_ASSERT(salt.size() == SALT_SIZE);
|
||||
SHA256_Update(&sha256, salt.data(), salt.size());
|
||||
}
|
||||
|
||||
for (auto i = 0; i < number_of_iterations; i++) {
|
||||
SHA256_Update(&sha256, password.c_str(), password.size());
|
||||
SHA256_Update(&sha256, password.data(), password.size());
|
||||
}
|
||||
SHA256_Final(hash, &sha256);
|
||||
|
||||
std::stringstream ss;
|
||||
for (unsigned char salt_char : salt) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << (((unsigned int)salt_char) & 0xFFU);
|
||||
}
|
||||
for (auto hash_char : hash) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
|
||||
}
|
||||
@@ -129,55 +162,144 @@ std::string EncryptPasswordOpenSSL1_1(const std::string &password, const uint64_
|
||||
}
|
||||
#endif
|
||||
|
||||
std::string EncryptPassword(const std::string &password, const uint64_t number_of_iterations) {
|
||||
std::string HashPassword(std::string_view password, const uint64_t number_of_iterations, std::string_view salt) {
|
||||
#if OPENSSL_VERSION_MAJOR >= 3
|
||||
return EncryptPasswordOpenSSL3(password, number_of_iterations);
|
||||
return HashPasswordOpenSSL3(password, number_of_iterations, salt);
|
||||
#else
|
||||
return EncryptPasswordOpenSSL1_1(password, number_of_iterations);
|
||||
return HashPasswordOpenSSL1_1(password, number_of_iterations, salt);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash, const uint64_t number_of_iterations) {
|
||||
auto password_hash = EncryptPassword(password, number_of_iterations);
|
||||
auto ExtractSalt(std::string_view salt_durable) -> std::array<char, SALT_SIZE> {
|
||||
static_assert(SALT_SIZE_DURABLE % 2 == 0);
|
||||
static_assert(SALT_SIZE_DURABLE / 2 == SALT_SIZE);
|
||||
|
||||
MG_ASSERT(salt_durable.size() == SALT_SIZE_DURABLE);
|
||||
auto const *b = salt_durable.cbegin();
|
||||
auto const *const e = salt_durable.cend();
|
||||
|
||||
auto salt = std::array<char, SALT_SIZE>{};
|
||||
auto *inserter = salt.begin();
|
||||
|
||||
auto const toval = [](char a) -> uint8_t {
|
||||
if ('0' <= a && a <= '9') {
|
||||
return a - '0';
|
||||
}
|
||||
if ('a' <= a && a <= 'f') {
|
||||
return 10 + (a - 'a');
|
||||
}
|
||||
MG_ASSERT(false, "Currupt hash, can't extract salt");
|
||||
__builtin_unreachable();
|
||||
};
|
||||
|
||||
for (; b != e; b += 2, ++inserter) {
|
||||
*inserter = static_cast<char>(static_cast<uint8_t>(toval(b[0]) << 4U) | toval(b[1]));
|
||||
}
|
||||
return salt;
|
||||
}
|
||||
|
||||
bool IsSalted(std::string_view hash) { return hash.size() == SHA_LENGTH + SALT_SIZE_DURABLE; }
|
||||
|
||||
bool VerifyPassword(std::string_view password, std::string_view hash, const uint64_t number_of_iterations) {
|
||||
auto password_hash = std::invoke([&] {
|
||||
if (hash.size() == SHA_LENGTH) [[unlikely]] {
|
||||
// Just SHA256
|
||||
return HashPassword(password, number_of_iterations, {});
|
||||
} else {
|
||||
// SHA256 + SALT
|
||||
MG_ASSERT(IsSalted(hash));
|
||||
auto const salt_durable = std::string_view{hash.data(), SALT_SIZE_DURABLE};
|
||||
std::array<char, SALT_SIZE> salt = ExtractSalt(salt_durable);
|
||||
return HashPassword(password, number_of_iterations, {salt.data(), salt.size()});
|
||||
}
|
||||
});
|
||||
return password_hash == hash;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace SHA
|
||||
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash) {
|
||||
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
|
||||
FLAGS_password_encryption_algorithm, password_encryption_mappings);
|
||||
HashedPassword HashPassword(const std::string &password, std::optional<PasswordHashAlgorithm> override_algo) {
|
||||
auto const hash_algo = override_algo.value_or(CurrentHashAlgorithm());
|
||||
auto password_hash = std::invoke([&] {
|
||||
switch (hash_algo) {
|
||||
case PasswordHashAlgorithm::BCRYPT: {
|
||||
return BCrypt::HashPassword(password);
|
||||
}
|
||||
case PasswordHashAlgorithm::SHA256:
|
||||
case PasswordHashAlgorithm::SHA256_MULTIPLE: {
|
||||
auto gen = std::mt19937(std::random_device{}());
|
||||
auto salt = std::array<char, SHA::SALT_SIZE>{};
|
||||
auto dis = std::uniform_int_distribution<unsigned char>(0, 255);
|
||||
std::generate(salt.begin(), salt.end(), [&]() { return dis(gen); });
|
||||
auto iterations = (hash_algo == PasswordHashAlgorithm::SHA256) ? ONE_SHA_ITERATION : MULTIPLE_SHA_ITERATIONS;
|
||||
return SHA::HashPassword(password, iterations, {salt.data(), salt.size()});
|
||||
}
|
||||
}
|
||||
});
|
||||
return HashedPassword{hash_algo, std::move(password_hash)};
|
||||
};
|
||||
|
||||
if (!password_encryption_algorithm.has_value()) {
|
||||
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
|
||||
namespace {
|
||||
|
||||
auto InternalParseHashAlgorithm(std::string_view algo) -> PasswordHashAlgorithm {
|
||||
auto maybe_parsed = utils::StringToEnum<PasswordHashAlgorithm>(algo, password_hash_mappings);
|
||||
if (!maybe_parsed) {
|
||||
throw AuthException("Invalid password encryption '{}'!", algo);
|
||||
}
|
||||
|
||||
switch (password_encryption_algorithm.value()) {
|
||||
case PasswordEncryptionAlgorithm::BCRYPT:
|
||||
return BCrypt::VerifyPassword(password, hash);
|
||||
case PasswordEncryptionAlgorithm::SHA256:
|
||||
return SHA::VerifyPassword(password, hash, ONE_SHA_ITERATION);
|
||||
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
|
||||
return SHA::VerifyPassword(password, hash, MULTIPLE_SHA_ITERATIONS);
|
||||
}
|
||||
|
||||
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
|
||||
return *maybe_parsed;
|
||||
}
|
||||
|
||||
std::string EncryptPassword(const std::string &password) {
|
||||
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
|
||||
FLAGS_password_encryption_algorithm, password_encryption_mappings);
|
||||
PasswordHashAlgorithm &InternalCurrentHashAlgorithm() {
|
||||
static auto current = PasswordHashAlgorithm::BCRYPT;
|
||||
static std::once_flag flag;
|
||||
std::call_once(flag, [] { current = InternalParseHashAlgorithm(FLAGS_password_encryption_algorithm); });
|
||||
return current;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
if (!password_encryption_algorithm.has_value()) {
|
||||
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
|
||||
auto CurrentHashAlgorithm() -> PasswordHashAlgorithm { return InternalCurrentHashAlgorithm(); }
|
||||
|
||||
void SetHashAlgorithm(std::string_view algo) {
|
||||
auto ¤t = InternalCurrentHashAlgorithm();
|
||||
current = InternalParseHashAlgorithm(algo);
|
||||
}
|
||||
|
||||
auto AsString(PasswordHashAlgorithm hash_algo) -> std::string_view {
|
||||
return *utils::EnumToString<PasswordHashAlgorithm>(hash_algo, password_hash_mappings);
|
||||
}
|
||||
|
||||
bool HashedPassword::VerifyPassword(const std::string &password) {
|
||||
switch (hash_algo) {
|
||||
case PasswordHashAlgorithm::BCRYPT:
|
||||
return BCrypt::VerifyPassword(password, password_hash);
|
||||
case PasswordHashAlgorithm::SHA256:
|
||||
return SHA::VerifyPassword(password, password_hash, ONE_SHA_ITERATION);
|
||||
case PasswordHashAlgorithm::SHA256_MULTIPLE:
|
||||
return SHA::VerifyPassword(password, password_hash, MULTIPLE_SHA_ITERATIONS);
|
||||
}
|
||||
}
|
||||
|
||||
switch (password_encryption_algorithm.value()) {
|
||||
case PasswordEncryptionAlgorithm::BCRYPT:
|
||||
return BCrypt::EncryptPassword(password);
|
||||
case PasswordEncryptionAlgorithm::SHA256:
|
||||
return SHA::EncryptPassword(password, ONE_SHA_ITERATION);
|
||||
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
|
||||
return SHA::EncryptPassword(password, MULTIPLE_SHA_ITERATIONS);
|
||||
void to_json(nlohmann::json &j, const HashedPassword &p) {
|
||||
j = nlohmann::json{{kHashAlgo, p.hash_algo}, {kPasswordHash, p.password_hash}};
|
||||
}
|
||||
|
||||
void from_json(const nlohmann::json &j, HashedPassword &p) {
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
|
||||
PasswordHashAlgorithm hash_algo;
|
||||
j.at(kHashAlgo).get_to(hash_algo);
|
||||
auto password_hash = j.value(kPasswordHash, std::string());
|
||||
p = HashedPassword{hash_algo, std::move(password_hash)};
|
||||
}
|
||||
|
||||
bool HashedPassword::IsSalted() const {
|
||||
switch (hash_algo) {
|
||||
case PasswordHashAlgorithm::BCRYPT:
|
||||
return true;
|
||||
case PasswordHashAlgorithm::SHA256:
|
||||
case PasswordHashAlgorithm::SHA256_MULTIPLE:
|
||||
return SHA::IsSalted(password_hash);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -8,14 +8,45 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <json/json.hpp>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace memgraph::auth {
|
||||
enum class PasswordEncryptionAlgorithm : uint8_t { BCRYPT, SHA256, SHA256_MULTIPLE };
|
||||
/// Need to be stable, auth durability depends on this
|
||||
enum class PasswordHashAlgorithm : uint8_t { BCRYPT = 0, SHA256 = 1, SHA256_MULTIPLE = 2 };
|
||||
|
||||
/// @throw AuthException if unable to encrypt the password.
|
||||
std::string EncryptPassword(const std::string &password);
|
||||
void SetHashAlgorithm(std::string_view algo);
|
||||
|
||||
/// @throw AuthException if unable to verify the password.
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash);
|
||||
auto CurrentHashAlgorithm() -> PasswordHashAlgorithm;
|
||||
|
||||
auto AsString(PasswordHashAlgorithm hash_algo) -> std::string_view;
|
||||
|
||||
struct HashedPassword {
|
||||
HashedPassword() = default;
|
||||
HashedPassword(PasswordHashAlgorithm hash_algo, std::string password_hash)
|
||||
: hash_algo{hash_algo}, password_hash{std::move(password_hash)} {}
|
||||
HashedPassword(HashedPassword const &) = default;
|
||||
HashedPassword(HashedPassword &&) = default;
|
||||
HashedPassword &operator=(HashedPassword const &) = default;
|
||||
HashedPassword &operator=(HashedPassword &&) = default;
|
||||
|
||||
friend bool operator==(HashedPassword const &, HashedPassword const &) = default;
|
||||
|
||||
bool VerifyPassword(const std::string &password);
|
||||
|
||||
bool IsSalted() const;
|
||||
|
||||
auto HashAlgo() const -> PasswordHashAlgorithm { return hash_algo; }
|
||||
|
||||
friend void to_json(nlohmann::json &j, const HashedPassword &p);
|
||||
friend void from_json(const nlohmann::json &j, HashedPassword &p);
|
||||
|
||||
private:
|
||||
PasswordHashAlgorithm hash_algo{PasswordHashAlgorithm::BCRYPT};
|
||||
std::string password_hash{};
|
||||
};
|
||||
|
||||
/// @throw AuthException if unable to hash the password.
|
||||
HashedPassword HashPassword(const std::string &password, std::optional<PasswordHashAlgorithm> override_algo = {});
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -9,7 +9,7 @@
|
||||
#include "auth/models.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <regex>
|
||||
#include <utility>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
@@ -20,22 +20,26 @@
|
||||
#include "query/constants.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
#include "utils/cast.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/settings.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(auth_password_permit_null, true, "Set to false to disable null passwords.");
|
||||
|
||||
inline constexpr std::string_view default_password_regex = ".+";
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
|
||||
"The regular expression that should be used to match the entire "
|
||||
"entered password to ensure its strength.");
|
||||
|
||||
namespace memgraph::auth {
|
||||
namespace {
|
||||
|
||||
constexpr auto kRoleName = "rolename";
|
||||
constexpr auto kPermissions = "permissions";
|
||||
constexpr auto kGrants = "grants";
|
||||
constexpr auto kDenies = "denies";
|
||||
constexpr auto kUsername = "username";
|
||||
constexpr auto kPasswordHash = "password_hash";
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
constexpr auto kGlobalPermission = "global_permission";
|
||||
constexpr auto kFineGrainedAccessHandler = "fine_grained_access_handler";
|
||||
constexpr auto kAllowAll = "allow_all";
|
||||
constexpr auto kDefault = "default";
|
||||
constexpr auto kDatabases = "databases";
|
||||
#endif
|
||||
|
||||
// Constant list of all available permissions.
|
||||
const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
|
||||
Permission::CREATE,
|
||||
@@ -61,7 +65,8 @@ const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
|
||||
Permission::TRANSACTION_MANAGEMENT,
|
||||
Permission::STORAGE_MODE,
|
||||
Permission::MULTI_DATABASE_EDIT,
|
||||
Permission::MULTI_DATABASE_USE};
|
||||
Permission::MULTI_DATABASE_USE,
|
||||
Permission::COORDINATOR};
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -117,6 +122,8 @@ std::string PermissionToString(Permission permission) {
|
||||
return "MULTI_DATABASE_EDIT";
|
||||
case Permission::MULTI_DATABASE_USE:
|
||||
return "MULTI_DATABASE_USE";
|
||||
case Permission::COORDINATOR:
|
||||
return "COORDINATOR";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,8 +248,9 @@ std::vector<Permission> Permissions::GetDenies() const {
|
||||
|
||||
nlohmann::json Permissions::Serialize() const {
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["grants"] = grants_;
|
||||
data["denies"] = denies_;
|
||||
|
||||
data[kGrants] = grants_;
|
||||
data[kDenies] = denies_;
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -250,10 +258,10 @@ Permissions Permissions::Deserialize(const nlohmann::json &data) {
|
||||
if (!data.is_object()) {
|
||||
throw AuthException("Couldn't load permissions data!");
|
||||
}
|
||||
if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) {
|
||||
if (!data[kGrants].is_number_unsigned() || !data[kDenies].is_number_unsigned()) {
|
||||
throw AuthException("Couldn't load permissions data!");
|
||||
}
|
||||
return Permissions{data["grants"], data["denies"]};
|
||||
return Permissions{data[kGrants], data[kDenies]};
|
||||
}
|
||||
|
||||
uint64_t Permissions::grants() const { return grants_; }
|
||||
@@ -315,8 +323,8 @@ nlohmann::json FineGrainedAccessPermissions::Serialize() const {
|
||||
return {};
|
||||
}
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["permissions"] = permissions_;
|
||||
data["global_permission"] = global_permission_.has_value() ? global_permission_.value() : -1;
|
||||
data[kPermissions] = permissions_;
|
||||
data[kGlobalPermission] = global_permission_.has_value() ? global_permission_.value() : -1;
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -329,13 +337,13 @@ FineGrainedAccessPermissions FineGrainedAccessPermissions::Deserialize(const nlo
|
||||
}
|
||||
std::optional<uint64_t> global_permission;
|
||||
|
||||
if (data["global_permission"].empty() || data["global_permission"] == -1) {
|
||||
if (data[kGlobalPermission].empty() || data[kGlobalPermission] == -1) {
|
||||
global_permission = std::nullopt;
|
||||
} else {
|
||||
global_permission = data["global_permission"];
|
||||
global_permission = data[kGlobalPermission];
|
||||
}
|
||||
|
||||
return FineGrainedAccessPermissions(data["permissions"], global_permission);
|
||||
return FineGrainedAccessPermissions(data[kPermissions], global_permission);
|
||||
}
|
||||
|
||||
const std::unordered_map<std::string, uint64_t> &FineGrainedAccessPermissions::GetPermissions() const {
|
||||
@@ -441,13 +449,13 @@ const FineGrainedAccessPermissions &Role::GetFineGrainedAccessEdgeTypePermission
|
||||
|
||||
nlohmann::json Role::Serialize() const {
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["rolename"] = rolename_;
|
||||
data["permissions"] = permissions_.Serialize();
|
||||
data[kRoleName] = rolename_;
|
||||
data[kPermissions] = permissions_.Serialize();
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
|
||||
data[kFineGrainedAccessHandler] = fine_grained_access_handler_.Serialize();
|
||||
} else {
|
||||
data["fine_grained_access_handler"] = {};
|
||||
data[kFineGrainedAccessHandler] = {};
|
||||
}
|
||||
#endif
|
||||
return data;
|
||||
@@ -457,21 +465,21 @@ Role Role::Deserialize(const nlohmann::json &data) {
|
||||
if (!data.is_object()) {
|
||||
throw AuthException("Couldn't load role data!");
|
||||
}
|
||||
if (!data["rolename"].is_string() || !data["permissions"].is_object()) {
|
||||
if (!data[kRoleName].is_string() || !data[kPermissions].is_object()) {
|
||||
throw AuthException("Couldn't load role data!");
|
||||
}
|
||||
auto permissions = Permissions::Deserialize(data["permissions"]);
|
||||
auto permissions = Permissions::Deserialize(data[kPermissions]);
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
FineGrainedAccessHandler fine_grained_access_handler;
|
||||
// We can have an empty fine_grained if the user was created without a valid license
|
||||
if (data["fine_grained_access_handler"].is_object()) {
|
||||
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
|
||||
if (data[kFineGrainedAccessHandler].is_object()) {
|
||||
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
|
||||
}
|
||||
return {data["rolename"], permissions, std::move(fine_grained_access_handler)};
|
||||
return {data[kRoleName], permissions, std::move(fine_grained_access_handler)};
|
||||
}
|
||||
#endif
|
||||
return {data["rolename"], permissions};
|
||||
return {data[kRoleName], permissions};
|
||||
}
|
||||
|
||||
bool operator==(const Role &first, const Role &second) {
|
||||
@@ -485,13 +493,13 @@ bool operator==(const Role &first, const Role &second) {
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void Databases::Add(const std::string &db) {
|
||||
void Databases::Add(std::string_view db) {
|
||||
if (allow_all_) {
|
||||
grants_dbs_.clear();
|
||||
allow_all_ = false;
|
||||
}
|
||||
grants_dbs_.emplace(db);
|
||||
denies_dbs_.erase(db);
|
||||
denies_dbs_.erase(std::string{db}); // TODO: C++23 use transparent key compare
|
||||
}
|
||||
|
||||
void Databases::Remove(const std::string &db) {
|
||||
@@ -522,13 +530,13 @@ void Databases::DenyAll() {
|
||||
denies_dbs_.clear();
|
||||
}
|
||||
|
||||
bool Databases::SetDefault(const std::string &db) {
|
||||
bool Databases::SetDefault(std::string_view db) {
|
||||
if (!Contains(db)) return false;
|
||||
default_db_ = db;
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] bool Databases::Contains(const std::string &db) const {
|
||||
[[nodiscard]] bool Databases::Contains(std::string_view db) const {
|
||||
return !denies_dbs_.contains(db) && (allow_all_ || grants_dbs_.contains(db));
|
||||
}
|
||||
|
||||
@@ -541,10 +549,10 @@ const std::string &Databases::GetDefault() const {
|
||||
|
||||
nlohmann::json Databases::Serialize() const {
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["grants"] = grants_dbs_;
|
||||
data["denies"] = denies_dbs_;
|
||||
data["allow_all"] = allow_all_;
|
||||
data["default"] = default_db_;
|
||||
data[kGrants] = grants_dbs_;
|
||||
data[kDenies] = denies_dbs_;
|
||||
data[kAllowAll] = allow_all_;
|
||||
data[kDefault] = default_db_;
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -552,63 +560,41 @@ Databases Databases::Deserialize(const nlohmann::json &data) {
|
||||
if (!data.is_object()) {
|
||||
throw AuthException("Couldn't load database data!");
|
||||
}
|
||||
if (!data["grants"].is_structured() || !data["denies"].is_structured() || !data["allow_all"].is_boolean() ||
|
||||
!data["default"].is_string()) {
|
||||
if (!data[kGrants].is_structured() || !data[kDenies].is_structured() || !data[kAllowAll].is_boolean() ||
|
||||
!data[kDefault].is_string()) {
|
||||
throw AuthException("Couldn't load database data!");
|
||||
}
|
||||
return {data["allow_all"], data["grants"], data["denies"], data["default"]};
|
||||
return {data[kAllowAll], data[kGrants], data[kDenies], data[kDefault]};
|
||||
}
|
||||
#endif
|
||||
|
||||
User::User() {}
|
||||
User::User() = default;
|
||||
|
||||
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
|
||||
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
|
||||
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
|
||||
User::User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions)
|
||||
: username_(utils::ToLowerCase(username)), password_hash_(std::move(password_hash)), permissions_(permissions) {}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
|
||||
User::User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions,
|
||||
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
|
||||
: username_(utils::ToLowerCase(username)),
|
||||
password_hash_(password_hash),
|
||||
password_hash_(std::move(password_hash)),
|
||||
permissions_(permissions),
|
||||
fine_grained_access_handler_(std::move(fine_grained_access_handler)),
|
||||
database_access_(db_access) {}
|
||||
database_access_(std::move(db_access)) {}
|
||||
#endif
|
||||
|
||||
bool User::CheckPassword(const std::string &password) {
|
||||
if (password_hash_.empty()) return true;
|
||||
return VerifyPassword(password, password_hash_);
|
||||
return password_hash_ ? password_hash_->VerifyPassword(password) : true;
|
||||
}
|
||||
|
||||
void User::UpdatePassword(const std::optional<std::string> &password) {
|
||||
void User::UpdatePassword(const std::optional<std::string> &password,
|
||||
std::optional<PasswordHashAlgorithm> algo_override) {
|
||||
if (!password) {
|
||||
if (!FLAGS_auth_password_permit_null) {
|
||||
throw AuthException("Null passwords aren't permitted!");
|
||||
}
|
||||
password_hash_ = "";
|
||||
password_hash_.reset();
|
||||
return;
|
||||
}
|
||||
|
||||
if (FLAGS_auth_password_strength_regex != default_password_regex) {
|
||||
if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
|
||||
license_check_result.HasError()) {
|
||||
throw AuthException(
|
||||
"Custom password regex is a Memgraph Enterprise feature. Please set the config "
|
||||
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
|
||||
default_password_regex,
|
||||
license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
|
||||
}
|
||||
}
|
||||
std::regex re(FLAGS_auth_password_strength_regex);
|
||||
if (!std::regex_match(*password, re)) {
|
||||
throw AuthException(
|
||||
"The user password doesn't conform to the required strength! Regex: "
|
||||
"\"{}\"",
|
||||
FLAGS_auth_password_strength_regex);
|
||||
}
|
||||
|
||||
password_hash_ = EncryptPassword(*password);
|
||||
password_hash_ = HashPassword(*password, algo_override);
|
||||
}
|
||||
|
||||
void User::SetRole(const Role &role) { role_.emplace(role); }
|
||||
@@ -625,27 +611,49 @@ Permissions User::GetPermissions() const {
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
FineGrainedAccessPermissions User::GetFineGrainedAccessLabelPermissions() const {
|
||||
return Merge(GetUserFineGrainedAccessLabelPermissions(), GetRoleFineGrainedAccessLabelPermissions());
|
||||
}
|
||||
|
||||
FineGrainedAccessPermissions User::GetFineGrainedAccessEdgeTypePermissions() const {
|
||||
return Merge(GetUserFineGrainedAccessEdgeTypePermissions(), GetRoleFineGrainedAccessEdgeTypePermissions());
|
||||
}
|
||||
|
||||
FineGrainedAccessPermissions User::GetUserFineGrainedAccessEdgeTypePermissions() const {
|
||||
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
return FineGrainedAccessPermissions{};
|
||||
}
|
||||
|
||||
if (role_) {
|
||||
return Merge(role()->fine_grained_access_handler().label_permissions(),
|
||||
fine_grained_access_handler_.label_permissions());
|
||||
return fine_grained_access_handler_.edge_type_permissions();
|
||||
}
|
||||
|
||||
FineGrainedAccessPermissions User::GetUserFineGrainedAccessLabelPermissions() const {
|
||||
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
return FineGrainedAccessPermissions{};
|
||||
}
|
||||
|
||||
return fine_grained_access_handler_.label_permissions();
|
||||
}
|
||||
|
||||
FineGrainedAccessPermissions User::GetFineGrainedAccessEdgeTypePermissions() const {
|
||||
FineGrainedAccessPermissions User::GetRoleFineGrainedAccessEdgeTypePermissions() const {
|
||||
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
return FineGrainedAccessPermissions{};
|
||||
}
|
||||
|
||||
if (role_) {
|
||||
return Merge(role()->fine_grained_access_handler().edge_type_permissions(),
|
||||
fine_grained_access_handler_.edge_type_permissions());
|
||||
return role()->fine_grained_access_handler().edge_type_permissions();
|
||||
}
|
||||
return fine_grained_access_handler_.edge_type_permissions();
|
||||
return FineGrainedAccessPermissions{};
|
||||
}
|
||||
|
||||
FineGrainedAccessPermissions User::GetRoleFineGrainedAccessLabelPermissions() const {
|
||||
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
return FineGrainedAccessPermissions{};
|
||||
}
|
||||
|
||||
if (role_) {
|
||||
return role()->fine_grained_access_handler().label_permissions();
|
||||
}
|
||||
return FineGrainedAccessPermissions{};
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -667,16 +675,20 @@ const Role *User::role() const {
|
||||
|
||||
nlohmann::json User::Serialize() const {
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["username"] = username_;
|
||||
data["password_hash"] = password_hash_;
|
||||
data["permissions"] = permissions_.Serialize();
|
||||
data[kUsername] = username_;
|
||||
if (password_hash_.has_value()) {
|
||||
data[kPasswordHash] = *password_hash_;
|
||||
} else {
|
||||
data[kPasswordHash] = nullptr;
|
||||
}
|
||||
data[kPermissions] = permissions_.Serialize();
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
|
||||
data["databases"] = database_access_.Serialize();
|
||||
data[kFineGrainedAccessHandler] = fine_grained_access_handler_.Serialize();
|
||||
data[kDatabases] = database_access_.Serialize();
|
||||
} else {
|
||||
data["fine_grained_access_handler"] = {};
|
||||
data["databases"] = {};
|
||||
data[kFineGrainedAccessHandler] = {};
|
||||
data[kDatabases] = {};
|
||||
}
|
||||
#endif
|
||||
// The role shouldn't be serialized here, it is stored as a foreign key.
|
||||
@@ -687,15 +699,23 @@ User User::Deserialize(const nlohmann::json &data) {
|
||||
if (!data.is_object()) {
|
||||
throw AuthException("Couldn't load user data!");
|
||||
}
|
||||
if (!data["username"].is_string() || !data["password_hash"].is_string() || !data["permissions"].is_object()) {
|
||||
auto password_hash_json = data[kPasswordHash];
|
||||
if (!data[kUsername].is_string() || !(password_hash_json.is_object() || password_hash_json.is_null()) ||
|
||||
!data[kPermissions].is_object()) {
|
||||
throw AuthException("Couldn't load user data!");
|
||||
}
|
||||
auto permissions = Permissions::Deserialize(data["permissions"]);
|
||||
|
||||
std::optional<HashedPassword> password_hash{};
|
||||
if (password_hash_json.is_object()) {
|
||||
password_hash = password_hash_json.get<HashedPassword>();
|
||||
}
|
||||
|
||||
auto permissions = Permissions::Deserialize(data[kPermissions]);
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
Databases db_access;
|
||||
if (data["databases"].is_structured()) {
|
||||
db_access = Databases::Deserialize(data["databases"]);
|
||||
if (data[kDatabases].is_structured()) {
|
||||
db_access = Databases::Deserialize(data[kDatabases]);
|
||||
} else {
|
||||
// Back-compatibility
|
||||
spdlog::warn("User without specified database access. Given access to the default database.");
|
||||
@@ -704,13 +724,13 @@ User User::Deserialize(const nlohmann::json &data) {
|
||||
}
|
||||
FineGrainedAccessHandler fine_grained_access_handler;
|
||||
// We can have an empty fine_grained if the user was created without a valid license
|
||||
if (data["fine_grained_access_handler"].is_object()) {
|
||||
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
|
||||
if (data[kFineGrainedAccessHandler].is_object()) {
|
||||
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
|
||||
}
|
||||
return {data["username"], data["password_hash"], permissions, std::move(fine_grained_access_handler), db_access};
|
||||
return {data[kUsername], std::move(password_hash), permissions, std::move(fine_grained_access_handler), db_access};
|
||||
}
|
||||
#endif
|
||||
return {data["username"], data["password_hash"], permissions};
|
||||
return {data[kUsername], std::move(password_hash), permissions};
|
||||
}
|
||||
|
||||
bool operator==(const User &first, const User &second) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -14,6 +14,8 @@
|
||||
#include <unordered_map>
|
||||
|
||||
#include <json/json.hpp>
|
||||
#include <utility>
|
||||
#include "crypto.hpp"
|
||||
#include "dbms/constants.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
@@ -47,6 +49,7 @@ enum class Permission : uint64_t {
|
||||
STORAGE_MODE = 1U << 22U,
|
||||
MULTI_DATABASE_EDIT = 1U << 23U,
|
||||
MULTI_DATABASE_USE = 1U << 24U,
|
||||
COORDINATOR = 1U << 25U,
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
@@ -204,6 +207,8 @@ bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHa
|
||||
|
||||
class Role final {
|
||||
public:
|
||||
Role() = default;
|
||||
|
||||
explicit Role(const std::string &rolename);
|
||||
Role(const std::string &rolename, const Permissions &permissions);
|
||||
#ifdef MG_ENTERPRISE
|
||||
@@ -245,7 +250,7 @@ bool operator==(const Role &first, const Role &second);
|
||||
#ifdef MG_ENTERPRISE
|
||||
class Databases final {
|
||||
public:
|
||||
Databases() : grants_dbs_({dbms::kDefaultDB}), allow_all_(false), default_db_(dbms::kDefaultDB) {}
|
||||
Databases() : grants_dbs_{std::string{dbms::kDefaultDB}}, allow_all_(false), default_db_(dbms::kDefaultDB) {}
|
||||
|
||||
Databases(const Databases &) = default;
|
||||
Databases &operator=(const Databases &) = default;
|
||||
@@ -258,7 +263,7 @@ class Databases final {
|
||||
*
|
||||
* @param db name of the database to grant access to
|
||||
*/
|
||||
void Add(const std::string &db);
|
||||
void Add(std::string_view db);
|
||||
|
||||
/**
|
||||
* @brief Remove database to the list of granted access.
|
||||
@@ -290,7 +295,7 @@ class Databases final {
|
||||
/**
|
||||
* @brief Set the default database.
|
||||
*/
|
||||
bool SetDefault(const std::string &db);
|
||||
bool SetDefault(std::string_view db);
|
||||
|
||||
/**
|
||||
* @brief Checks if access is grated to the database.
|
||||
@@ -298,11 +303,11 @@ class Databases final {
|
||||
* @param db name of the database
|
||||
* @return true if allow_all and not denied or granted
|
||||
*/
|
||||
bool Contains(const std::string &db) const;
|
||||
bool Contains(std::string_view db) const;
|
||||
|
||||
bool GetAllowAll() const { return allow_all_; }
|
||||
const std::set<std::string> &GetGrants() const { return grants_dbs_; }
|
||||
const std::set<std::string> &GetDenies() const { return denies_dbs_; }
|
||||
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
|
||||
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
|
||||
const std::string &GetDefault() const;
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
@@ -310,14 +315,17 @@ class Databases final {
|
||||
static Databases Deserialize(const nlohmann::json &data);
|
||||
|
||||
private:
|
||||
Databases(bool allow_all, std::set<std::string> grant, std::set<std::string> deny,
|
||||
const std::string &default_db = dbms::kDefaultDB)
|
||||
: grants_dbs_(grant), denies_dbs_(deny), allow_all_(allow_all), default_db_(default_db) {}
|
||||
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
|
||||
std::string default_db = std::string{dbms::kDefaultDB})
|
||||
: grants_dbs_(std::move(grant)),
|
||||
denies_dbs_(std::move(deny)),
|
||||
allow_all_(allow_all),
|
||||
default_db_(std::move(default_db)) {}
|
||||
|
||||
std::set<std::string> grants_dbs_; //!< set of databases with granted access
|
||||
std::set<std::string> denies_dbs_; //!< set of databases with denied access
|
||||
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
|
||||
std::string default_db_; //!< user's default database
|
||||
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
|
||||
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
|
||||
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
|
||||
std::string default_db_; //!< user's default database
|
||||
};
|
||||
#endif
|
||||
|
||||
@@ -327,9 +335,9 @@ class User final {
|
||||
User();
|
||||
|
||||
explicit User(const std::string &username);
|
||||
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
|
||||
User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions);
|
||||
#ifdef MG_ENTERPRISE
|
||||
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
|
||||
User(const std::string &username, std::optional<HashedPassword> password_hash, const Permissions &permissions,
|
||||
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
|
||||
#endif
|
||||
User(const User &) = default;
|
||||
@@ -341,8 +349,18 @@ class User final {
|
||||
/// @throw AuthException if unable to verify the password.
|
||||
bool CheckPassword(const std::string &password);
|
||||
|
||||
bool UpgradeHash(const std::string password) {
|
||||
if (!password_hash_) return false;
|
||||
if (password_hash_->IsSalted()) return false;
|
||||
|
||||
auto const algo = password_hash_->HashAlgo();
|
||||
UpdatePassword(password, algo);
|
||||
return true;
|
||||
}
|
||||
|
||||
/// @throw AuthException if unable to set the password.
|
||||
void UpdatePassword(const std::optional<std::string> &password = std::nullopt);
|
||||
void UpdatePassword(const std::optional<std::string> &password = {},
|
||||
std::optional<PasswordHashAlgorithm> algo_override = std::nullopt);
|
||||
|
||||
void SetRole(const Role &role);
|
||||
|
||||
@@ -353,6 +371,10 @@ class User final {
|
||||
#ifdef MG_ENTERPRISE
|
||||
FineGrainedAccessPermissions GetFineGrainedAccessLabelPermissions() const;
|
||||
FineGrainedAccessPermissions GetFineGrainedAccessEdgeTypePermissions() const;
|
||||
FineGrainedAccessPermissions GetUserFineGrainedAccessLabelPermissions() const;
|
||||
FineGrainedAccessPermissions GetUserFineGrainedAccessEdgeTypePermissions() const;
|
||||
FineGrainedAccessPermissions GetRoleFineGrainedAccessLabelPermissions() const;
|
||||
FineGrainedAccessPermissions GetRoleFineGrainedAccessEdgeTypePermissions() const;
|
||||
const FineGrainedAccessHandler &fine_grained_access_handler() const;
|
||||
FineGrainedAccessHandler &fine_grained_access_handler();
|
||||
#endif
|
||||
@@ -377,7 +399,7 @@ class User final {
|
||||
|
||||
private:
|
||||
std::string username_;
|
||||
std::string password_hash_;
|
||||
std::optional<HashedPassword> password_hash_;
|
||||
Permissions permissions_;
|
||||
#ifdef MG_ENTERPRISE
|
||||
FineGrainedAccessHandler fine_grained_access_handler_;
|
||||
|
||||
190
src/auth/replication_handlers.cpp
Normal file
190
src/auth/replication_handlers.cpp
Normal file
@@ -0,0 +1,190 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "auth/replication_handlers.hpp"
|
||||
|
||||
#include "auth/auth.hpp"
|
||||
#include "auth/rpc.hpp"
|
||||
#include "license/license.hpp"
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req) {
|
||||
spdlog::error(fmt::format("Received {} with main_id: {} != current_main_uuid: {}", rpc_req, std::string(main_req_id),
|
||||
current_main_uuid.has_value() ? std::string(current_main_uuid.value()) : ""));
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void UpdateAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::UpdateAuthDataReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
|
||||
using memgraph::replication::UpdateAuthDataRes;
|
||||
UpdateAuthDataRes res(false);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::UpdateAuthDataReq::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
// what we have so far.
|
||||
|
||||
if (req.expected_group_timestamp != system_state_access.LastCommitedTS()) {
|
||||
spdlog::debug("UpdateAuthDataHandler: bad expected timestamp {},{}", req.expected_group_timestamp,
|
||||
system_state_access.LastCommitedTS());
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Update
|
||||
if (req.user) auth->SaveUser(*req.user);
|
||||
if (req.role) auth->SaveRole(*req.role);
|
||||
// Success
|
||||
system_state_access.SetLastCommitedTS(req.new_group_timestamp);
|
||||
res = UpdateAuthDataRes(true);
|
||||
spdlog::debug("UpdateAuthDataHandler: SUCCESS updated LCTS to {}", req.new_group_timestamp);
|
||||
} catch (const auth::AuthException & /* not used */) {
|
||||
// Failure
|
||||
}
|
||||
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void DropAuthDataHandler(memgraph::system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::DropAuthDataReq req;
|
||||
memgraph::slk::Load(&req, req_reader);
|
||||
|
||||
using memgraph::replication::DropAuthDataRes;
|
||||
DropAuthDataRes res(false);
|
||||
|
||||
if (!current_main_uuid.has_value() || req.main_uuid != current_main_uuid) [[unlikely]] {
|
||||
LogWrongMain(current_main_uuid, req.main_uuid, replication::DropAuthDataRes::kType.name);
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
// Note: No need to check epoch, recovery mechanism is done by a full uptodate snapshot
|
||||
// of the set of databases. Hence no history exists to maintain regarding epoch change.
|
||||
// If MAIN has changed we need to check this new group_timestamp is consistent with
|
||||
// what we have so far.
|
||||
|
||||
if (req.expected_group_timestamp != system_state_access.LastCommitedTS()) {
|
||||
spdlog::debug("DropAuthDataHandler: bad expected timestamp {},{}", req.expected_group_timestamp,
|
||||
system_state_access.LastCommitedTS());
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Remove
|
||||
switch (req.type) {
|
||||
case replication::DropAuthDataReq::DataType::USER:
|
||||
auth->RemoveUser(req.name);
|
||||
break;
|
||||
case replication::DropAuthDataReq::DataType::ROLE:
|
||||
auth->RemoveRole(req.name);
|
||||
break;
|
||||
}
|
||||
// Success
|
||||
system_state_access.SetLastCommitedTS(req.new_group_timestamp);
|
||||
res = DropAuthDataRes(true);
|
||||
spdlog::debug("DropAuthDataHandler: SUCCESS updated LCTS to {}", req.new_group_timestamp);
|
||||
} catch (const auth::AuthException & /* not used */) {
|
||||
// Failure
|
||||
}
|
||||
|
||||
memgraph::slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
bool SystemRecoveryHandler(auth::SynchedAuth &auth, auth::Auth::Config auth_config,
|
||||
const std::vector<auth::User> &users, const std::vector<auth::Role> &roles) {
|
||||
return auth.WithLock([&](auto &locked_auth) {
|
||||
// Update config
|
||||
locked_auth.SetConfig(std::move(auth_config));
|
||||
// Get all current users
|
||||
auto old_users = locked_auth.AllUsernames();
|
||||
// Save incoming users
|
||||
for (const auto &user : users) {
|
||||
// Missing users
|
||||
try {
|
||||
locked_auth.SaveUser(user);
|
||||
} catch (const auth::AuthException &) {
|
||||
spdlog::debug("SystemRecoveryHandler: Failed to save user");
|
||||
return false;
|
||||
}
|
||||
const auto it = std::find(old_users.begin(), old_users.end(), user.username());
|
||||
if (it != old_users.end()) old_users.erase(it);
|
||||
}
|
||||
// Delete all the leftover users
|
||||
for (const auto &user : old_users) {
|
||||
if (!locked_auth.RemoveUser(user)) {
|
||||
spdlog::debug("SystemRecoveryHandler: Failed to remove user \"{}\".", user);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Roles are only supported with a license
|
||||
if (license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
// Get all current roles
|
||||
auto old_roles = locked_auth.AllRolenames();
|
||||
// Save incoming users
|
||||
for (const auto &role : roles) {
|
||||
// Missing users
|
||||
try {
|
||||
locked_auth.SaveRole(role);
|
||||
} catch (const auth::AuthException &) {
|
||||
spdlog::debug("SystemRecoveryHandler: Failed to save user");
|
||||
return false;
|
||||
}
|
||||
const auto it = std::find(old_roles.begin(), old_roles.end(), role.rolename());
|
||||
if (it != old_roles.end()) old_roles.erase(it);
|
||||
}
|
||||
// Delete all the leftover users
|
||||
for (const auto &role : old_roles) {
|
||||
if (!locked_auth.RemoveRole(role)) {
|
||||
spdlog::debug("SystemRecoveryHandler: Failed to remove user \"{}\".", role);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Success
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
void Register(replication::RoleReplicaData const &data, system::ReplicaHandlerAccessToState &system_state_access,
|
||||
auth::SynchedAuth &auth) {
|
||||
// NOTE: Register even without license as the user could add a license at run-time
|
||||
data.server->rpc_server_.Register<replication::UpdateAuthDataRpc>(
|
||||
[&data, system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received UpdateAuthDataRpc");
|
||||
UpdateAuthDataHandler(system_state_access, data.uuid_, auth, req_reader, res_builder);
|
||||
});
|
||||
data.server->rpc_server_.Register<replication::DropAuthDataRpc>(
|
||||
[&data, system_state_access, &auth](auto *req_reader, auto *res_builder) mutable {
|
||||
spdlog::debug("Received DropAuthDataRpc");
|
||||
DropAuthDataHandler(system_state_access, data.uuid_, auth, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::auth
|
||||
37
src/auth/replication_handlers.hpp
Normal file
37
src/auth/replication_handlers.hpp
Normal file
@@ -0,0 +1,37 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "auth/auth.hpp"
|
||||
#include "replication/state.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
#include "system/state.hpp"
|
||||
|
||||
namespace memgraph::auth {
|
||||
|
||||
void LogWrongMain(const std::optional<utils::UUID> ¤t_main_uuid, const utils::UUID &main_req_id,
|
||||
std::string_view rpc_req);
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
void UpdateAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
void DropAuthDataHandler(system::ReplicaHandlerAccessToState &system_state_access,
|
||||
const std::optional<utils::UUID> ¤t_main_uuid, auth::SynchedAuth &auth,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
bool SystemRecoveryHandler(auth::SynchedAuth &auth, auth::Auth::Config auth_config,
|
||||
const std::vector<auth::User> &users, const std::vector<auth::Role> &roles);
|
||||
void Register(replication::RoleReplicaData const &data, system::ReplicaHandlerAccessToState &system_state_access,
|
||||
auth::SynchedAuth &auth);
|
||||
#endif
|
||||
} // namespace memgraph::auth
|
||||
182
src/auth/rpc.cpp
Normal file
182
src/auth/rpc.cpp
Normal file
@@ -0,0 +1,182 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "auth/rpc.hpp"
|
||||
|
||||
#include <json/json.hpp>
|
||||
#include "auth/auth.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
#include "utils/enum.hpp"
|
||||
|
||||
namespace memgraph::slk {
|
||||
|
||||
// Serialize code for auth::Role
|
||||
void Save(const auth::Role &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.Serialize().dump(), builder);
|
||||
}
|
||||
namespace {
|
||||
auth::Role LoadAuthRole(memgraph::slk::Reader *reader) {
|
||||
std::string tmp;
|
||||
memgraph::slk::Load(&tmp, reader);
|
||||
const auto json = nlohmann::json::parse(tmp);
|
||||
return memgraph::auth::Role::Deserialize(json);
|
||||
}
|
||||
} // namespace
|
||||
// Deserialize code for auth::Role
|
||||
void Load(auth::Role *self, memgraph::slk::Reader *reader) { *self = LoadAuthRole(reader); }
|
||||
// Special case for optional<Role>
|
||||
template <>
|
||||
inline void Load<auth::Role>(std::optional<auth::Role> *obj, Reader *reader) {
|
||||
bool exists = false;
|
||||
Load(&exists, reader);
|
||||
if (exists) {
|
||||
obj->emplace(LoadAuthRole(reader));
|
||||
} else {
|
||||
*obj = std::nullopt;
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize code for auth::User
|
||||
void Save(const auth::User &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.Serialize().dump(), builder);
|
||||
std::optional<auth::Role> role{};
|
||||
if (const auto *role_ptr = self.role(); role_ptr) {
|
||||
role.emplace(*role_ptr);
|
||||
}
|
||||
memgraph::slk::Save(role, builder);
|
||||
}
|
||||
// Deserialize code for auth::User
|
||||
void Load(auth::User *self, memgraph::slk::Reader *reader) {
|
||||
std::string tmp;
|
||||
memgraph::slk::Load(&tmp, reader);
|
||||
const auto json = nlohmann::json::parse(tmp);
|
||||
*self = memgraph::auth::User::Deserialize(json);
|
||||
std::optional<auth::Role> role{};
|
||||
memgraph::slk::Load(&role, reader);
|
||||
if (role)
|
||||
self->SetRole(*role);
|
||||
else
|
||||
self->ClearRole();
|
||||
}
|
||||
|
||||
// Serialize code for auth::Auth::Config
|
||||
void Save(const auth::Auth::Config &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.name_regex_str, builder);
|
||||
memgraph::slk::Save(self.password_regex_str, builder);
|
||||
memgraph::slk::Save(self.password_permit_null, builder);
|
||||
}
|
||||
// Deserialize code for auth::Auth::Config
|
||||
void Load(auth::Auth::Config *self, memgraph::slk::Reader *reader) {
|
||||
std::string name_regex_str{};
|
||||
std::string password_regex_str{};
|
||||
bool password_permit_null{};
|
||||
|
||||
memgraph::slk::Load(&name_regex_str, reader);
|
||||
memgraph::slk::Load(&password_regex_str, reader);
|
||||
memgraph::slk::Load(&password_permit_null, reader);
|
||||
|
||||
*self = auth::Auth::Config{std::move(name_regex_str), std::move(password_regex_str), password_permit_null};
|
||||
}
|
||||
|
||||
// Serialize code for UpdateAuthDataReq
|
||||
void Save(const memgraph::replication::UpdateAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
memgraph::slk::Save(self.expected_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.new_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.user, builder);
|
||||
memgraph::slk::Save(self.role, builder);
|
||||
}
|
||||
void Load(memgraph::replication::UpdateAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
memgraph::slk::Load(&self->expected_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->new_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->user, reader);
|
||||
memgraph::slk::Load(&self->role, reader);
|
||||
}
|
||||
|
||||
// Serialize code for UpdateAuthDataRes
|
||||
void Save(const memgraph::replication::UpdateAuthDataRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
void Load(memgraph::replication::UpdateAuthDataRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
// Serialize code for DropAuthDataReq
|
||||
void Save(const memgraph::replication::DropAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid, builder);
|
||||
memgraph::slk::Save(self.epoch_id, builder);
|
||||
memgraph::slk::Save(self.expected_group_timestamp, builder);
|
||||
memgraph::slk::Save(self.new_group_timestamp, builder);
|
||||
memgraph::slk::Save(utils::EnumToNum<2, uint8_t>(self.type), builder);
|
||||
memgraph::slk::Save(self.name, builder);
|
||||
}
|
||||
void Load(memgraph::replication::DropAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid, reader);
|
||||
memgraph::slk::Load(&self->epoch_id, reader);
|
||||
memgraph::slk::Load(&self->expected_group_timestamp, reader);
|
||||
memgraph::slk::Load(&self->new_group_timestamp, reader);
|
||||
uint8_t type_tmp = 0;
|
||||
memgraph::slk::Load(&type_tmp, reader);
|
||||
if (!utils::NumToEnum<2>(type_tmp, self->type)) {
|
||||
throw SlkReaderException("Unexpected result line:{}!", __LINE__);
|
||||
}
|
||||
memgraph::slk::Load(&self->name, reader);
|
||||
}
|
||||
|
||||
// Serialize code for DropAuthDataRes
|
||||
void Save(const memgraph::replication::DropAuthDataRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
void Load(memgraph::replication::DropAuthDataRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
} // namespace memgraph::slk
|
||||
|
||||
namespace memgraph::replication {
|
||||
|
||||
constexpr utils::TypeInfo UpdateAuthDataReq::kType{utils::TypeId::REP_UPDATE_AUTH_DATA_REQ, "UpdateAuthDataReq",
|
||||
nullptr};
|
||||
|
||||
constexpr utils::TypeInfo UpdateAuthDataRes::kType{utils::TypeId::REP_UPDATE_AUTH_DATA_RES, "UpdateAuthDataRes",
|
||||
nullptr};
|
||||
|
||||
constexpr utils::TypeInfo DropAuthDataReq::kType{utils::TypeId::REP_DROP_AUTH_DATA_REQ, "DropAuthDataReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo DropAuthDataRes::kType{utils::TypeId::REP_DROP_AUTH_DATA_RES, "DropAuthDataRes", nullptr};
|
||||
|
||||
void UpdateAuthDataReq::Save(const UpdateAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
void UpdateAuthDataReq::Load(UpdateAuthDataReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
void UpdateAuthDataRes::Save(const UpdateAuthDataRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
void UpdateAuthDataRes::Load(UpdateAuthDataRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void DropAuthDataReq::Save(const DropAuthDataReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
void DropAuthDataReq::Load(DropAuthDataReq *self, memgraph::slk::Reader *reader) { memgraph::slk::Load(self, reader); }
|
||||
void DropAuthDataRes::Save(const DropAuthDataRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
void DropAuthDataRes::Load(DropAuthDataRes *self, memgraph::slk::Reader *reader) { memgraph::slk::Load(self, reader); }
|
||||
|
||||
} // namespace memgraph::replication
|
||||
127
src/auth/rpc.hpp
Normal file
127
src/auth/rpc.hpp
Normal file
@@ -0,0 +1,127 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "auth/auth.hpp"
|
||||
#include "auth/models.hpp"
|
||||
#include "rpc/messages.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
|
||||
namespace memgraph::replication {
|
||||
|
||||
struct UpdateAuthDataReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(UpdateAuthDataReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const UpdateAuthDataReq &self, memgraph::slk::Builder *builder);
|
||||
UpdateAuthDataReq() = default;
|
||||
UpdateAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
auth::User user)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
user{std::move(user)} {}
|
||||
UpdateAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
auth::Role role)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
role{std::move(role)} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
std::optional<auth::User> user;
|
||||
std::optional<auth::Role> role;
|
||||
};
|
||||
|
||||
struct UpdateAuthDataRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(UpdateAuthDataRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const UpdateAuthDataRes &self, memgraph::slk::Builder *builder);
|
||||
UpdateAuthDataRes() = default;
|
||||
explicit UpdateAuthDataRes(bool success) : success{success} {}
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using UpdateAuthDataRpc = rpc::RequestResponse<UpdateAuthDataReq, UpdateAuthDataRes>;
|
||||
|
||||
struct DropAuthDataReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(DropAuthDataReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const DropAuthDataReq &self, memgraph::slk::Builder *builder);
|
||||
DropAuthDataReq() = default;
|
||||
|
||||
enum class DataType { USER, ROLE };
|
||||
|
||||
DropAuthDataReq(const utils::UUID &main_uuid, std::string epoch_id, uint64_t expected_ts, uint64_t new_ts,
|
||||
DataType type, std::string_view name)
|
||||
: main_uuid(main_uuid),
|
||||
epoch_id{std::move(epoch_id)},
|
||||
expected_group_timestamp{expected_ts},
|
||||
new_group_timestamp{new_ts},
|
||||
type{type},
|
||||
name{name} {}
|
||||
|
||||
utils::UUID main_uuid;
|
||||
std::string epoch_id;
|
||||
uint64_t expected_group_timestamp;
|
||||
uint64_t new_group_timestamp;
|
||||
DataType type;
|
||||
std::string name;
|
||||
};
|
||||
|
||||
struct DropAuthDataRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(DropAuthDataRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const DropAuthDataRes &self, memgraph::slk::Builder *builder);
|
||||
DropAuthDataRes() = default;
|
||||
explicit DropAuthDataRes(bool success) : success{success} {}
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using DropAuthDataRpc = rpc::RequestResponse<DropAuthDataReq, DropAuthDataRes>;
|
||||
|
||||
} // namespace memgraph::replication
|
||||
|
||||
namespace memgraph::slk {
|
||||
|
||||
void Save(const auth::Role &self, memgraph::slk::Builder *builder);
|
||||
void Load(auth::Role *self, memgraph::slk::Reader *reader);
|
||||
void Save(const auth::User &self, memgraph::slk::Builder *builder);
|
||||
void Load(auth::User *self, memgraph::slk::Reader *reader);
|
||||
void Save(const auth::Auth::Config &self, memgraph::slk::Builder *builder);
|
||||
void Load(auth::Auth::Config *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Save(const memgraph::replication::UpdateAuthDataRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::replication::UpdateAuthDataRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::replication::UpdateAuthDataReq & /*self*/, memgraph::slk::Builder * /*builder*/);
|
||||
void Load(memgraph::replication::UpdateAuthDataReq * /*self*/, memgraph::slk::Reader * /*reader*/);
|
||||
void Save(const memgraph::replication::DropAuthDataRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::replication::DropAuthDataRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::replication::DropAuthDataReq & /*self*/, memgraph::slk::Builder * /*builder*/);
|
||||
void Load(memgraph::replication::DropAuthDataReq * /*self*/, memgraph::slk::Reader * /*reader*/);
|
||||
} // namespace memgraph::slk
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "communication/bolt/v1/codes.hpp"
|
||||
@@ -34,8 +35,8 @@ class FailureResponseException : public utils::BasicException {
|
||||
|
||||
explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {}
|
||||
|
||||
FailureResponseException(const std::string &code, const std::string &message)
|
||||
: utils::BasicException{message}, code_{code} {}
|
||||
FailureResponseException(std::string code, const std::string &message)
|
||||
: utils::BasicException{message}, code_{std::move(code)} {}
|
||||
|
||||
const std::string &code() const { return code_; }
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -51,7 +51,7 @@ enum class ChunkState : uint8_t {
|
||||
template <typename TBuffer>
|
||||
class ChunkedDecoderBuffer {
|
||||
public:
|
||||
ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
|
||||
explicit ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
|
||||
|
||||
/**
|
||||
* Reads data from the internal buffer.
|
||||
|
||||
@@ -401,11 +401,11 @@ class Decoder {
|
||||
}
|
||||
auto &labels = dv.ValueList();
|
||||
vertex.labels.reserve(labels.size());
|
||||
for (size_t i = 0; i < labels.size(); ++i) {
|
||||
if (labels[i].type() != Value::Type::String) {
|
||||
for (auto &label : labels) {
|
||||
if (label.type() != Value::Type::String) {
|
||||
return false;
|
||||
}
|
||||
vertex.labels.emplace_back(std::move(labels[i].ValueString()));
|
||||
vertex.labels.emplace_back(std::move(label.ValueString()));
|
||||
}
|
||||
|
||||
// read properties
|
||||
|
||||
@@ -111,12 +111,12 @@ class BaseEncoder {
|
||||
|
||||
void WriteList(const std::vector<Value> &value) {
|
||||
WriteTypeSize(value.size(), MarkerList);
|
||||
for (auto &x : value) WriteValue(x);
|
||||
for (const auto &x : value) WriteValue(x);
|
||||
}
|
||||
|
||||
void WriteMap(const std::map<std::string, Value> &value) {
|
||||
WriteTypeSize(value.size(), MarkerMap);
|
||||
for (auto &x : value) {
|
||||
for (const auto &x : value) {
|
||||
WriteString(x.first);
|
||||
WriteValue(x.second);
|
||||
}
|
||||
@@ -205,11 +205,11 @@ class BaseEncoder {
|
||||
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
|
||||
WriteRAW(utils::UnderlyingCast(Signature::Path));
|
||||
WriteTypeSize(path.vertices.size(), MarkerList);
|
||||
for (auto &v : path.vertices) WriteVertex(v);
|
||||
for (const auto &v : path.vertices) WriteVertex(v);
|
||||
WriteTypeSize(path.edges.size(), MarkerList);
|
||||
for (auto &e : path.edges) WriteEdge(e);
|
||||
for (const auto &e : path.edges) WriteEdge(e);
|
||||
WriteTypeSize(path.indices.size(), MarkerList);
|
||||
for (auto &i : path.indices) WriteInt(i);
|
||||
for (const auto &i : path.indices) WriteInt(i);
|
||||
}
|
||||
|
||||
void WriteDate(const utils::Date &date) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -48,7 +48,7 @@ namespace memgraph::communication::bolt {
|
||||
template <class TOutputStream>
|
||||
class ChunkedEncoderBuffer {
|
||||
public:
|
||||
ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
|
||||
explicit ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
|
||||
|
||||
/**
|
||||
* Writes n values into the buffer. If n is bigger than whole chunk size
|
||||
|
||||
@@ -39,7 +39,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
|
||||
using BaseEncoder<Buffer>::buffer_;
|
||||
|
||||
public:
|
||||
ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
explicit ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
|
||||
using BaseEncoder<Buffer>::UpdateVersion;
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class Encoder : private BaseEncoder<Buffer> {
|
||||
using BaseEncoder<Buffer>::buffer_;
|
||||
|
||||
public:
|
||||
Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
explicit Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
|
||||
using BaseEncoder<Buffer>::UpdateVersion;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -93,7 +93,7 @@ State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<i
|
||||
return State::Close;
|
||||
}
|
||||
|
||||
if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
|
||||
if (summary.contains("has_more") && summary.at("has_more").ValueBool()) {
|
||||
return State::Result;
|
||||
}
|
||||
|
||||
@@ -148,13 +148,13 @@ State HandlePullDiscardV4(TSession &session, const State state, const Marker mar
|
||||
spdlog::trace("Couldn't read extra field!");
|
||||
}
|
||||
const auto &extra_map = extra.ValueMap();
|
||||
if (extra_map.count("n")) {
|
||||
if (extra_map.contains("n")) {
|
||||
if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
|
||||
n = n_value;
|
||||
}
|
||||
}
|
||||
|
||||
if (extra_map.count("qid")) {
|
||||
if (extra_map.contains("qid")) {
|
||||
if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
|
||||
qid = qid_value;
|
||||
}
|
||||
@@ -170,6 +170,7 @@ inline State HandleFailure(TSession &session, const std::exception &e) {
|
||||
spdlog::trace("Error trace: {}", p->trace());
|
||||
}
|
||||
session.encoder_buffer_.Clear();
|
||||
|
||||
auto code_message = ExceptionToErrorMessage(e);
|
||||
bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
|
||||
if (!fail_sent) {
|
||||
@@ -367,14 +368,16 @@ State HandleReset(TSession &session, const Marker marker) {
|
||||
return State::Close;
|
||||
}
|
||||
|
||||
if (!session.encoder_.MessageSuccess()) {
|
||||
spdlog::trace("Couldn't send success message!");
|
||||
return State::Close;
|
||||
try {
|
||||
session.Abort();
|
||||
if (!session.encoder_.MessageSuccess({})) {
|
||||
spdlog::trace("Couldn't send success message!");
|
||||
return State::Close;
|
||||
}
|
||||
return State::Idle;
|
||||
} catch (const std::exception &e) {
|
||||
return HandleFailure(session, e);
|
||||
}
|
||||
|
||||
session.Abort();
|
||||
|
||||
return State::Idle;
|
||||
}
|
||||
|
||||
template <typename TSession>
|
||||
@@ -397,19 +400,17 @@ State HandleBegin(TSession &session, const State state, const Marker marker) {
|
||||
|
||||
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
|
||||
|
||||
if (!session.encoder_.MessageSuccess({})) {
|
||||
spdlog::trace("Couldn't send success message!");
|
||||
return State::Close;
|
||||
}
|
||||
|
||||
try {
|
||||
session.Configure(extra.ValueMap());
|
||||
session.BeginTransaction(extra.ValueMap());
|
||||
if (!session.encoder_.MessageSuccess({})) {
|
||||
spdlog::trace("Couldn't send success message!");
|
||||
return State::Close;
|
||||
}
|
||||
return State::Idle;
|
||||
} catch (const std::exception &e) {
|
||||
return HandleFailure(session, e);
|
||||
}
|
||||
|
||||
return State::Idle;
|
||||
}
|
||||
|
||||
template <typename TSession>
|
||||
@@ -427,11 +428,11 @@ State HandleCommit(TSession &session, const State state, const Marker marker) {
|
||||
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
|
||||
|
||||
try {
|
||||
session.CommitTransaction();
|
||||
if (!session.encoder_.MessageSuccess({})) {
|
||||
spdlog::trace("Couldn't send success message!");
|
||||
return State::Close;
|
||||
}
|
||||
session.CommitTransaction();
|
||||
return State::Idle;
|
||||
} catch (const std::exception &e) {
|
||||
return HandleFailure(session, e);
|
||||
@@ -453,11 +454,11 @@ State HandleRollback(TSession &session, const State state, const Marker marker)
|
||||
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
|
||||
|
||||
try {
|
||||
session.RollbackTransaction();
|
||||
if (!session.encoder_.MessageSuccess({})) {
|
||||
spdlog::trace("Couldn't send success message!");
|
||||
return State::Close;
|
||||
}
|
||||
session.RollbackTransaction();
|
||||
return State::Idle;
|
||||
} catch (const std::exception &e) {
|
||||
return HandleFailure(session, e);
|
||||
|
||||
@@ -42,11 +42,11 @@ std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
|
||||
std::string username;
|
||||
std::string password;
|
||||
if (data["scheme"].ValueString() == "basic") {
|
||||
if (!data.count("principal")) { // Special case principal = ""
|
||||
if (!data.contains("principal")) { // Special case principal = ""
|
||||
spdlog::warn("The client didn't supply the principal field! Trying with \"\"...");
|
||||
data["principal"] = "";
|
||||
}
|
||||
if (!data.count("credentials")) { // Special case credentials = ""
|
||||
if (!data.contains("credentials")) { // Special case credentials = ""
|
||||
spdlog::warn("The client didn't supply the credentials field! Trying with \"\"...");
|
||||
data["credentials"] = "";
|
||||
}
|
||||
@@ -118,7 +118,7 @@ std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
|
||||
}
|
||||
|
||||
auto &data = metadata.ValueMap();
|
||||
if (!data.count("user_agent")) {
|
||||
if (!data.contains("user_agent")) {
|
||||
spdlog::warn("The client didn't supply the user agent!");
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -142,7 +142,7 @@ std::optional<Value> GetInitDataV5(TSession &session, const Marker marker) {
|
||||
}
|
||||
|
||||
const auto &data = metadata.ValueMap();
|
||||
if (!data.count("user_agent")) {
|
||||
if (!data.contains("user_agent")) {
|
||||
spdlog::warn("The client didn't supply the user agent!");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ struct UnboundedEdge {
|
||||
* The decoder writes data into this structure.
|
||||
*/
|
||||
struct Path {
|
||||
Path() {}
|
||||
Path() = default;
|
||||
|
||||
Path(const std::vector<Vertex> &vertices, const std::vector<Edge> &edges) {
|
||||
// Helper function. Looks for the given element in the collection. If found,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,6 +12,7 @@
|
||||
#include "communication/client.hpp"
|
||||
|
||||
#include "communication/helpers.hpp"
|
||||
#include "io/network/network_error.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::communication {
|
||||
@@ -25,7 +26,9 @@ Client::~Client() {
|
||||
|
||||
bool Client::Connect(const io::network::Endpoint &endpoint) {
|
||||
// Try to establish a socket connection.
|
||||
if (!socket_.Connect(endpoint)) return false;
|
||||
if (!socket_.Connect(endpoint)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Enable TCP keep alive for all connections.
|
||||
// Because we manually always set the `have_more` flag to the socket
|
||||
|
||||
@@ -132,7 +132,7 @@ class Client final {
|
||||
*/
|
||||
class ClientInputStream final {
|
||||
public:
|
||||
ClientInputStream(Client &client);
|
||||
explicit ClientInputStream(Client &client);
|
||||
|
||||
ClientInputStream(const ClientInputStream &) = delete;
|
||||
ClientInputStream(ClientInputStream &&) = delete;
|
||||
@@ -156,7 +156,7 @@ class ClientInputStream final {
|
||||
*/
|
||||
class ClientOutputStream final {
|
||||
public:
|
||||
ClientOutputStream(Client &client);
|
||||
explicit ClientOutputStream(Client &client);
|
||||
|
||||
ClientOutputStream(const ClientOutputStream &) = delete;
|
||||
ClientOutputStream(ClientOutputStream &&) = delete;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -34,7 +34,7 @@ ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) {
|
||||
}
|
||||
|
||||
ClientContext::ClientContext(const std::string &key_file, const std::string &cert_file) : ClientContext(true) {
|
||||
if (key_file != "" && cert_file != "") {
|
||||
if (!key_file.empty() && !cert_file.empty()) {
|
||||
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1,
|
||||
"Couldn't load client certificate from file: {}", cert_file);
|
||||
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1,
|
||||
@@ -124,7 +124,7 @@ ServerContext &ServerContext::operator=(ServerContext &&other) noexcept {
|
||||
return *this;
|
||||
}
|
||||
|
||||
ServerContext::~ServerContext() {}
|
||||
ServerContext::~ServerContext() = default;
|
||||
|
||||
SSL_CTX *ServerContext::context() {
|
||||
MG_ASSERT(ctx_);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
namespace memgraph::communication {
|
||||
|
||||
const std::string SslGetLastError() {
|
||||
std::string SslGetLastError() {
|
||||
char buff[2048];
|
||||
auto err = ERR_get_error();
|
||||
ERR_error_string_n(err, buff, sizeof(buff));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,6 +18,6 @@ namespace memgraph::communication {
|
||||
/**
|
||||
* This function reads and returns a string describing the last OpenSSL error.
|
||||
*/
|
||||
const std::string SslGetLastError();
|
||||
std::string SslGetLastError();
|
||||
|
||||
} // namespace memgraph::communication
|
||||
|
||||
@@ -38,7 +38,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TRequestHand
|
||||
Listener(Listener &&) = delete;
|
||||
Listener &operator=(const Listener &) = delete;
|
||||
Listener &operator=(Listener &&) = delete;
|
||||
~Listener() {}
|
||||
~Listener() = default;
|
||||
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Listener> Create(Args &&...args) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -107,7 +107,7 @@ class Session : public std::enable_shared_from_this<Session<TRequestHandler, TSe
|
||||
void DoRead() {
|
||||
req_ = {};
|
||||
|
||||
ExecuteForStream([this](auto &&stream) {
|
||||
ExecuteForStream([this](auto &stream) {
|
||||
boost::beast::get_lowest_layer(stream).expires_after(std::chrono::seconds(kSSLExpirySeconds));
|
||||
|
||||
boost::beast::http::async_read(
|
||||
@@ -129,7 +129,7 @@ class Session : public std::enable_shared_from_this<Session<TRequestHandler, TSe
|
||||
}
|
||||
|
||||
auto async_write = [this](boost::beast::http::response<boost::beast::http::string_body> msg) {
|
||||
ExecuteForStream([this, &msg](auto &&stream) {
|
||||
ExecuteForStream([this, &msg](auto &stream) {
|
||||
// The lifetime of the message has to extend
|
||||
// for the duration of the async operation so
|
||||
// we use a shared_ptr to manage it.
|
||||
@@ -171,7 +171,7 @@ class Session : public std::enable_shared_from_this<Session<TRequestHandler, TSe
|
||||
}
|
||||
|
||||
auto GetExecutor() {
|
||||
return std::visit(utils::Overloaded{[](auto &&stream) { return stream.get_executor(); }}, stream_);
|
||||
return std::visit(utils::Overloaded{[](auto &stream) { return stream.get_executor(); }}, stream_);
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
@@ -51,13 +52,13 @@ class Listener final {
|
||||
using SessionHandler = Session<TSession, TSessionContext>;
|
||||
|
||||
public:
|
||||
Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, const std::string &service_name,
|
||||
Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, std::string service_name,
|
||||
size_t workers_count)
|
||||
: data_(data),
|
||||
alive_(false),
|
||||
context_(context),
|
||||
inactivity_timeout_sec_(inactivity_timeout_sec),
|
||||
service_name_(service_name),
|
||||
service_name_(std::move(service_name)),
|
||||
workers_count_(workers_count) {}
|
||||
|
||||
~Listener() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -44,7 +44,7 @@ class ResultStreamFaker {
|
||||
std::vector<memgraph::communication::bolt::Value> bvalues;
|
||||
bvalues.reserve(values.size());
|
||||
for (const auto &value : values) {
|
||||
auto maybe_value = memgraph::glue::ToBoltValue(value, *store_, memgraph::storage::View::NEW);
|
||||
auto maybe_value = memgraph::glue::ToBoltValue(value, store_, memgraph::storage::View::NEW);
|
||||
MG_ASSERT(maybe_value.HasValue());
|
||||
bvalues.push_back(std::move(*maybe_value));
|
||||
}
|
||||
@@ -56,7 +56,7 @@ class ResultStreamFaker {
|
||||
void Summary(const std::map<std::string, memgraph::query::TypedValue> &summary) {
|
||||
std::map<std::string, memgraph::communication::bolt::Value> bsummary;
|
||||
for (const auto &item : summary) {
|
||||
auto maybe_value = memgraph::glue::ToBoltValue(item.second, *store_, memgraph::storage::View::NEW);
|
||||
auto maybe_value = memgraph::glue::ToBoltValue(item.second, store_, memgraph::storage::View::NEW);
|
||||
MG_ASSERT(maybe_value.HasValue());
|
||||
bsummary.insert({item.first, std::move(*maybe_value)});
|
||||
}
|
||||
@@ -80,7 +80,7 @@ class ResultStreamFaker {
|
||||
std::transform(header.begin(), header.end(), column_widths.begin(), [](const auto &s) { return s.size(); });
|
||||
|
||||
// convert all the results into strings, and track max column width
|
||||
auto &results_data = results.GetResults();
|
||||
const auto &results_data = results.GetResults();
|
||||
std::vector<std::vector<std::string>> result_strings(results_data.size(),
|
||||
std::vector<std::string>(column_widths.size()));
|
||||
for (int row_ind = 0; row_ind < static_cast<int>(results_data.size()); ++row_ind) {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
#include <openssl/bio.h>
|
||||
#include <openssl/err.h>
|
||||
@@ -51,7 +52,8 @@ using InputStream = Buffer::ReadEnd;
|
||||
*/
|
||||
class OutputStream final {
|
||||
public:
|
||||
OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function) : write_function_(write_function) {}
|
||||
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
|
||||
: write_function_(std::move(write_function)) {}
|
||||
|
||||
OutputStream(const OutputStream &) = delete;
|
||||
OutputStream(OutputStream &&) = delete;
|
||||
|
||||
@@ -47,7 +47,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
|
||||
Listener(Listener &&) = delete;
|
||||
Listener &operator=(const Listener &) = delete;
|
||||
Listener &operator=(Listener &&) = delete;
|
||||
~Listener() {}
|
||||
~Listener() = default;
|
||||
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Listener> Create(Args &&...args) {
|
||||
|
||||
@@ -76,7 +76,7 @@ using tcp = boost::asio::ip::tcp;
|
||||
class OutputStream final {
|
||||
public:
|
||||
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
|
||||
: write_function_(write_function) {}
|
||||
: write_function_(std::move(write_function)) {}
|
||||
|
||||
OutputStream(const OutputStream &) = delete;
|
||||
OutputStream(OutputStream &&) = delete;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,8 +14,6 @@
|
||||
#include <string>
|
||||
|
||||
#include "auth/auth.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::communication::websocket {
|
||||
|
||||
@@ -30,7 +28,7 @@ class AuthenticationInterface {
|
||||
|
||||
class SafeAuth : public AuthenticationInterface {
|
||||
public:
|
||||
explicit SafeAuth(utils::Synchronized<auth::Auth, utils::WritePrioritizedRWLock> *auth) : auth_{auth} {}
|
||||
explicit SafeAuth(auth::SynchedAuth *auth) : auth_{auth} {}
|
||||
|
||||
bool Authenticate(const std::string &username, const std::string &password) const override;
|
||||
|
||||
@@ -39,6 +37,6 @@ class SafeAuth : public AuthenticationInterface {
|
||||
bool HasAnyUsers() const override;
|
||||
|
||||
private:
|
||||
utils::Synchronized<auth::Auth, utils::WritePrioritizedRWLock> *auth_;
|
||||
auth::SynchedAuth *auth_;
|
||||
};
|
||||
} // namespace memgraph::communication::websocket
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -16,6 +16,7 @@
|
||||
#include <spdlog/sinks/base_sink.h>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <utility>
|
||||
|
||||
#include "communication/websocket/listener.hpp"
|
||||
#include "io/network/endpoint.hpp"
|
||||
@@ -45,7 +46,7 @@ class Server final {
|
||||
|
||||
class LoggingSink : public spdlog::sinks::base_sink<std::mutex> {
|
||||
public:
|
||||
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(listener) {}
|
||||
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(std::move(listener)) {}
|
||||
|
||||
private:
|
||||
void sink_it_(const spdlog::details::log_msg &msg) override;
|
||||
|
||||
42
src/coordination/CMakeLists.txt
Normal file
42
src/coordination/CMakeLists.txt
Normal file
@@ -0,0 +1,42 @@
|
||||
add_library(mg-coordination STATIC)
|
||||
add_library(mg::coordination ALIAS mg-coordination)
|
||||
target_sources(mg-coordination
|
||||
PUBLIC
|
||||
include/coordination/coordinator_client.hpp
|
||||
include/coordination/coordinator_state.hpp
|
||||
include/coordination/coordinator_rpc.hpp
|
||||
include/coordination/coordinator_server.hpp
|
||||
include/coordination/coordinator_config.hpp
|
||||
include/coordination/coordinator_exceptions.hpp
|
||||
include/coordination/coordinator_slk.hpp
|
||||
include/coordination/coordinator_instance.hpp
|
||||
include/coordination/coordinator_handlers.hpp
|
||||
include/coordination/constants.hpp
|
||||
include/coordination/instance_status.hpp
|
||||
include/coordination/replication_instance.hpp
|
||||
include/coordination/raft_state.hpp
|
||||
include/coordination/rpc_errors.hpp
|
||||
|
||||
include/nuraft/coordinator_log_store.hpp
|
||||
include/nuraft/coordinator_state_machine.hpp
|
||||
include/nuraft/coordinator_state_manager.hpp
|
||||
|
||||
PRIVATE
|
||||
coordinator_client.cpp
|
||||
coordinator_state.cpp
|
||||
coordinator_rpc.cpp
|
||||
coordinator_server.cpp
|
||||
coordinator_handlers.cpp
|
||||
coordinator_instance.cpp
|
||||
replication_instance.cpp
|
||||
raft_state.cpp
|
||||
|
||||
coordinator_log_store.cpp
|
||||
coordinator_state_machine.cpp
|
||||
coordinator_state_manager.cpp
|
||||
)
|
||||
target_include_directories(mg-coordination PUBLIC include)
|
||||
|
||||
target_link_libraries(mg-coordination
|
||||
PUBLIC mg::utils mg::rpc mg::slk mg::io mg::repl_coord_glue lib::rangev3 nuraft mg-replication_handler
|
||||
)
|
||||
175
src/coordination/coordinator_client.cpp
Normal file
175
src/coordination/coordinator_client.cpp
Normal file
@@ -0,0 +1,175 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
#include "replication_coordination_glue/messages.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
namespace {
|
||||
auto CreateClientContext(memgraph::coordination::CoordinatorClientConfig const &config)
|
||||
-> communication::ClientContext {
|
||||
return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file}
|
||||
: communication::ClientContext{};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
CoordinatorClient::CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorClientConfig config,
|
||||
HealthCheckCallback succ_cb, HealthCheckCallback fail_cb)
|
||||
: rpc_context_{CreateClientContext(config)},
|
||||
rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port),
|
||||
&rpc_context_},
|
||||
config_{std::move(config)},
|
||||
coord_instance_{coord_instance},
|
||||
succ_cb_{std::move(succ_cb)},
|
||||
fail_cb_{std::move(fail_cb)} {}
|
||||
|
||||
auto CoordinatorClient::InstanceName() const -> std::string { return config_.instance_name; }
|
||||
auto CoordinatorClient::SocketAddress() const -> std::string { return rpc_client_.Endpoint().SocketAddress(); }
|
||||
|
||||
auto CoordinatorClient::InstanceDownTimeoutSec() const -> std::chrono::seconds {
|
||||
return config_.instance_down_timeout_sec;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::InstanceGetUUIDFrequencySec() const -> std::chrono::seconds {
|
||||
return config_.instance_get_uuid_frequency_sec;
|
||||
}
|
||||
|
||||
void CoordinatorClient::StartFrequentCheck() {
|
||||
if (instance_checker_.IsRunning()) {
|
||||
return;
|
||||
}
|
||||
|
||||
MG_ASSERT(config_.instance_health_check_frequency_sec > std::chrono::seconds(0),
|
||||
"Health check frequency must be greater than 0");
|
||||
|
||||
instance_checker_.Run(
|
||||
config_.instance_name, config_.instance_health_check_frequency_sec,
|
||||
[this, instance_name = config_.instance_name] {
|
||||
try {
|
||||
spdlog::trace("Sending frequent heartbeat to machine {} on {}", instance_name,
|
||||
rpc_client_.Endpoint().SocketAddress());
|
||||
{ // NOTE: This is intentionally scoped so that stream lock could get released.
|
||||
auto stream{rpc_client_.Stream<memgraph::replication_coordination_glue::FrequentHeartbeatRpc>()};
|
||||
stream.AwaitResponse();
|
||||
}
|
||||
succ_cb_(coord_instance_, instance_name);
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
fail_cb_(coord_instance_, instance_name);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void CoordinatorClient::StopFrequentCheck() { instance_checker_.Stop(); }
|
||||
void CoordinatorClient::PauseFrequentCheck() { instance_checker_.Pause(); }
|
||||
void CoordinatorClient::ResumeFrequentCheck() { instance_checker_.Resume(); }
|
||||
|
||||
auto CoordinatorClient::SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCallback fail_cb) -> void {
|
||||
succ_cb_ = std::move(succ_cb);
|
||||
fail_cb_ = std::move(fail_cb);
|
||||
}
|
||||
|
||||
auto CoordinatorClient::ReplicationClientInfo() const -> ReplClientInfo { return config_.replication_client_info; }
|
||||
|
||||
auto CoordinatorClient::SendPromoteReplicaToMainRpc(const utils::UUID &uuid,
|
||||
ReplicationClientsInfo replication_clients_info) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<PromoteReplicaToMainRpc>(uuid, std::move(replication_clients_info))};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful PromoteReplicaToMainRpc response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("RPC error occurred while sending PromoteReplicaToMainRpc!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::DemoteToReplica() const -> bool {
|
||||
auto const &instance_name = config_.instance_name;
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<DemoteMainToReplicaRpc>(config_.replication_client_info)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC response for setting instance {} to replica!", instance_name);
|
||||
return false;
|
||||
}
|
||||
spdlog::info("Sent request RPC from coordinator to instance to set it as replica!");
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("Failed to set instance {} to replica!", instance_name);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendSwapMainUUIDRpc(const utils::UUID &uuid) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<replication_coordination_glue::SwapMainUUIDRpc>(uuid)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC swapping of uuid response!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occurred while sending swapping uuid RPC!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendUnregisterReplicaRpc(std::string const &instance_name) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<UnregisterReplicaRpc>(instance_name)};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC response for unregistering replica!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("Failed to unregister replica!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendGetInstanceUUIDRpc() const
|
||||
-> utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>> {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<GetInstanceUUIDRpc>()};
|
||||
auto res = stream.AwaitResponse();
|
||||
return res.uuid;
|
||||
} catch (const rpc::RpcFailedException &) {
|
||||
spdlog::error("RPC error occured while sending GetInstance UUID RPC");
|
||||
return GetInstanceUUIDError::RPC_EXCEPTION;
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorClient::SendEnableWritingOnMainRpc() const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<EnableWritingOnMainRpc>()};
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC response for enabling writing on main!");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (rpc::RpcFailedException const &) {
|
||||
spdlog::error("Failed to enable writing on main!");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
218
src/coordination/coordinator_handlers.cpp
Normal file
218
src/coordination/coordinator_handlers.cpp
Normal file
@@ -0,0 +1,218 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
#include "coordination/coordinator_handlers.hpp"
|
||||
|
||||
#include <range/v3/view.hpp>
|
||||
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
#include "coordination/include/coordination/coordinator_server.hpp"
|
||||
#include "replication/state.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
void CoordinatorHandlers::Register(memgraph::coordination::CoordinatorServer &server,
|
||||
replication::ReplicationHandler &replication_handler) {
|
||||
server.Register<coordination::PromoteReplicaToMainRpc>(
|
||||
[&](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received PromoteReplicaToMainRpc");
|
||||
CoordinatorHandlers::PromoteReplicaToMainHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::DemoteMainToReplicaRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received DemoteMainToReplicaRpc from coordinator server");
|
||||
CoordinatorHandlers::DemoteMainToReplicaHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<replication_coordination_glue::SwapMainUUIDRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received SwapMainUUIDRPC on coordinator server");
|
||||
CoordinatorHandlers::SwapMainUUIDHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::UnregisterReplicaRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received UnregisterReplicaRpc on coordinator server");
|
||||
CoordinatorHandlers::UnregisterReplicaHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::EnableWritingOnMainRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received EnableWritingOnMainRpc on coordinator server");
|
||||
CoordinatorHandlers::EnableWritingOnMainHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
server.Register<coordination::GetInstanceUUIDRpc>(
|
||||
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
|
||||
spdlog::info("Received GetInstanceUUIDRpc on coordinator server");
|
||||
CoordinatorHandlers::GetInstanceUUIDHandler(replication_handler, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsReplica()) {
|
||||
spdlog::error("Setting main uuid must be performed on replica.");
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
replication_coordination_glue::SwapMainUUIDReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
spdlog::info(fmt::format("Set replica data UUID to main uuid {}", std::string(req.uuid)));
|
||||
std::get<memgraph::replication::RoleReplicaData>(replication_handler.GetReplState().ReplicationData()).uuid_ =
|
||||
req.uuid;
|
||||
|
||||
slk::Save(replication_coordination_glue::SwapMainUUIDRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
spdlog::info("Executing DemoteMainToReplicaHandler");
|
||||
|
||||
coordination::DemoteMainToReplicaReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
const replication::ReplicationServerConfig clients_config{
|
||||
.ip_address = req.replication_client_info.replication_ip_address,
|
||||
.port = req.replication_client_info.replication_port};
|
||||
|
||||
if (!replication_handler.SetReplicationRoleReplica(clients_config, std::nullopt)) {
|
||||
spdlog::error("Demoting main to replica failed!");
|
||||
slk::Save(coordination::DemoteMainToReplicaRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
slk::Save(coordination::DemoteMainToReplicaRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::GetInstanceUUIDHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
|
||||
spdlog::info("Executing GetInstanceUUIDHandler");
|
||||
|
||||
slk::Save(coordination::GetInstanceUUIDRes{replication_handler.GetReplicaUUID()}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsReplica()) {
|
||||
spdlog::error("Failover must be performed on replica!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
coordination::PromoteReplicaToMainReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
// This can fail because of disk. If it does, the cluster state could get inconsistent.
|
||||
// We don't handle disk issues.
|
||||
if (const bool success = replication_handler.DoReplicaToMainPromotion(req.main_uuid_); !success) {
|
||||
spdlog::error("Promoting replica to main failed!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
auto const converter = [](const auto &repl_info_config) {
|
||||
return replication::ReplicationClientConfig{
|
||||
.name = repl_info_config.instance_name,
|
||||
.mode = repl_info_config.replication_mode,
|
||||
.ip_address = repl_info_config.replication_ip_address,
|
||||
.port = repl_info_config.replication_port,
|
||||
};
|
||||
};
|
||||
|
||||
// registering replicas
|
||||
for (auto const &config : req.replication_clients_info | ranges::views::transform(converter)) {
|
||||
auto instance_client = replication_handler.RegisterReplica(config, false);
|
||||
if (instance_client.HasError()) {
|
||||
using enum memgraph::replication::RegisterReplicaError;
|
||||
switch (instance_client.GetError()) {
|
||||
// Can't happen, checked on the coordinator side
|
||||
case memgraph::query::RegisterReplicaError::NAME_EXISTS:
|
||||
spdlog::error("Replica with the same name already exists!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
// Can't happen, checked on the coordinator side
|
||||
case memgraph::query::RegisterReplicaError::ENDPOINT_EXISTS:
|
||||
spdlog::error("Replica with the same endpoint already exists!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
// We don't handle disk issues
|
||||
case memgraph::query::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
|
||||
spdlog::error("Registered replica could not be persisted!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
case memgraph::query::RegisterReplicaError::ERROR_ACCEPTING_MAIN:
|
||||
spdlog::error("Replica didn't accept change of main!");
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
|
||||
return;
|
||||
case memgraph::query::RegisterReplicaError::CONNECTION_FAILED:
|
||||
// Connection failure is not a fatal error
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
spdlog::info("Promote replica to main was success {}", std::string(req.main_uuid_));
|
||||
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::UnregisterReplicaHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsMain()) {
|
||||
spdlog::error("Unregistering replica must be performed on main.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
coordination::UnregisterReplicaReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
auto res = replication_handler.UnregisterReplica(req.instance_name);
|
||||
switch (res) {
|
||||
using enum memgraph::query::UnregisterReplicaResult;
|
||||
case SUCCESS:
|
||||
slk::Save(coordination::UnregisterReplicaRes{true}, res_builder);
|
||||
break;
|
||||
case NOT_MAIN:
|
||||
spdlog::error("Unregistering replica must be performed on main.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
break;
|
||||
case CAN_NOT_UNREGISTER:
|
||||
spdlog::error("Could not unregister replica.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
break;
|
||||
case COULD_NOT_BE_PERSISTED:
|
||||
spdlog::error("Could not persist replica unregistration.");
|
||||
slk::Save(coordination::UnregisterReplicaRes{false}, res_builder);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void CoordinatorHandlers::EnableWritingOnMainHandler(replication::ReplicationHandler &replication_handler,
|
||||
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
|
||||
if (!replication_handler.IsMain()) {
|
||||
spdlog::error("Enable writing on main must be performed on main!");
|
||||
slk::Save(coordination::EnableWritingOnMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!replication_handler.GetReplState().EnableWritingOnMain()) {
|
||||
spdlog::error("Enabling writing on main failed!");
|
||||
slk::Save(coordination::EnableWritingOnMainRes{false}, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
slk::Save(coordination::EnableWritingOnMainRes{true}, res_builder);
|
||||
}
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
#endif
|
||||
348
src/coordination/coordinator_instance.cpp
Normal file
348
src/coordination/coordinator_instance.cpp
Normal file
@@ -0,0 +1,348 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
#include "utils/counter.hpp"
|
||||
#include "utils/functional.hpp"
|
||||
|
||||
#include <range/v3/view.hpp>
|
||||
#include <shared_mutex>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::ptr;
|
||||
using nuraft::srv_config;
|
||||
|
||||
CoordinatorInstance::CoordinatorInstance()
|
||||
: raft_state_(RaftState::MakeRaftState(
|
||||
[this] { std::ranges::for_each(repl_instances_, &ReplicationInstance::StartFrequentCheck); },
|
||||
[this] { std::ranges::for_each(repl_instances_, &ReplicationInstance::StopFrequentCheck); })) {
|
||||
auto find_repl_instance = [](CoordinatorInstance *self,
|
||||
std::string_view repl_instance_name) -> ReplicationInstance & {
|
||||
auto repl_instance =
|
||||
std::ranges::find_if(self->repl_instances_, [repl_instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == repl_instance_name;
|
||||
});
|
||||
|
||||
MG_ASSERT(repl_instance != self->repl_instances_.end(), "Instance {} not found during callback!",
|
||||
repl_instance_name);
|
||||
return *repl_instance;
|
||||
};
|
||||
|
||||
replica_succ_cb_ = [find_repl_instance](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::lock_guard{self->coord_instance_lock_};
|
||||
spdlog::trace("Instance {} performing replica successful callback", repl_instance_name);
|
||||
auto &repl_instance = find_repl_instance(self, repl_instance_name);
|
||||
|
||||
// We need to get replicas UUID from time to time to ensure replica is listening to correct main
|
||||
// and that it didn't go down for less time than we could notice
|
||||
// We need to get id of main replica is listening to
|
||||
// and swap if necessary
|
||||
if (!repl_instance.EnsureReplicaHasCorrectMainUUID(self->GetMainUUID())) {
|
||||
spdlog::error("Failed to swap uuid for replica instance {} which is alive", repl_instance.InstanceName());
|
||||
return;
|
||||
}
|
||||
|
||||
repl_instance.OnSuccessPing();
|
||||
};
|
||||
|
||||
replica_fail_cb_ = [find_repl_instance](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::lock_guard{self->coord_instance_lock_};
|
||||
spdlog::trace("Instance {} performing replica failure callback", repl_instance_name);
|
||||
auto &repl_instance = find_repl_instance(self, repl_instance_name);
|
||||
repl_instance.OnFailPing();
|
||||
};
|
||||
|
||||
main_succ_cb_ = [find_repl_instance](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::lock_guard{self->coord_instance_lock_};
|
||||
spdlog::trace("Instance {} performing main successful callback", repl_instance_name);
|
||||
|
||||
auto &repl_instance = find_repl_instance(self, repl_instance_name);
|
||||
|
||||
if (repl_instance.IsAlive()) {
|
||||
repl_instance.OnSuccessPing();
|
||||
return;
|
||||
}
|
||||
|
||||
const auto &repl_instance_uuid = repl_instance.GetMainUUID();
|
||||
MG_ASSERT(repl_instance_uuid.has_value(), "Instance must have uuid set.");
|
||||
|
||||
auto const curr_main_uuid = self->GetMainUUID();
|
||||
if (curr_main_uuid == repl_instance_uuid.value()) {
|
||||
if (!repl_instance.EnableWritingOnMain()) {
|
||||
spdlog::error("Failed to enable writing on main instance {}", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
repl_instance.OnSuccessPing();
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO(antoniof) make demoteToReplica idempotent since main can be demoted to replica but
|
||||
// swapUUID can fail
|
||||
if (repl_instance.DemoteToReplica(self->replica_succ_cb_, self->replica_fail_cb_)) {
|
||||
repl_instance.OnSuccessPing();
|
||||
spdlog::info("Instance {} demoted to replica", repl_instance_name);
|
||||
} else {
|
||||
spdlog::error("Instance {} failed to become replica", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!repl_instance.SendSwapAndUpdateUUID(curr_main_uuid)) {
|
||||
spdlog::error(fmt::format("Failed to swap uuid for demoted main instance {}", repl_instance.InstanceName()));
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
main_fail_cb_ = [find_repl_instance](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::lock_guard{self->coord_instance_lock_};
|
||||
spdlog::trace("Instance {} performing main failure callback", repl_instance_name);
|
||||
auto &repl_instance = find_repl_instance(self, repl_instance_name);
|
||||
repl_instance.OnFailPing();
|
||||
const auto &repl_instance_uuid = repl_instance.GetMainUUID();
|
||||
MG_ASSERT(repl_instance_uuid.has_value(), "Instance must have uuid set");
|
||||
|
||||
if (!repl_instance.IsAlive() && self->GetMainUUID() == repl_instance_uuid.value()) {
|
||||
spdlog::info("Cluster without main instance, trying automatic failover");
|
||||
self->TryFailover(); // TODO: (andi) Initiate failover
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::ShowInstances() const -> std::vector<InstanceStatus> {
|
||||
auto const coord_instances = raft_state_.GetAllCoordinators();
|
||||
|
||||
auto const stringify_repl_role = [](ReplicationInstance const &instance) -> std::string {
|
||||
if (!instance.IsAlive()) return "unknown";
|
||||
if (instance.IsMain()) return "main";
|
||||
return "replica";
|
||||
};
|
||||
|
||||
auto const repl_instance_to_status = [&stringify_repl_role](ReplicationInstance const &instance) -> InstanceStatus {
|
||||
return {.instance_name = instance.InstanceName(),
|
||||
.coord_socket_address = instance.SocketAddress(),
|
||||
.cluster_role = stringify_repl_role(instance),
|
||||
.is_alive = instance.IsAlive()};
|
||||
};
|
||||
|
||||
auto const coord_instance_to_status = [](ptr<srv_config> const &instance) -> InstanceStatus {
|
||||
return {.instance_name = "coordinator_" + std::to_string(instance->get_id()),
|
||||
.raft_socket_address = instance->get_endpoint(),
|
||||
.cluster_role = "coordinator",
|
||||
.is_alive = true}; // TODO: (andi) Get this info from RAFT and test it or when we will move
|
||||
// CoordinatorState to every instance, we can be smarter about this using our RPC.
|
||||
};
|
||||
|
||||
auto instances_status = utils::fmap(coord_instance_to_status, coord_instances);
|
||||
{
|
||||
auto lock = std::shared_lock{coord_instance_lock_};
|
||||
std::ranges::transform(repl_instances_, std::back_inserter(instances_status), repl_instance_to_status);
|
||||
}
|
||||
|
||||
return instances_status;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::TryFailover() -> void {
|
||||
auto alive_replicas = repl_instances_ | ranges::views::filter(&ReplicationInstance::IsReplica) |
|
||||
ranges::views::filter(&ReplicationInstance::IsAlive);
|
||||
|
||||
if (ranges::empty(alive_replicas)) {
|
||||
spdlog::warn("Failover failed since all replicas are down!");
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: Smarter choice
|
||||
auto new_main = ranges::begin(alive_replicas);
|
||||
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
auto const is_not_new_main = [&new_main](ReplicationInstance &instance) {
|
||||
return instance.InstanceName() != new_main->InstanceName();
|
||||
};
|
||||
|
||||
auto const new_main_uuid = utils::UUID{};
|
||||
// If for some replicas swap fails, for others on successful ping we will revert back on next change
|
||||
// or we will do failover first again and then it will be consistent again
|
||||
for (auto &other_replica_instance : alive_replicas | ranges::views::filter(is_not_new_main)) {
|
||||
if (!other_replica_instance.SendSwapAndUpdateUUID(new_main_uuid)) {
|
||||
spdlog::error(fmt::format("Failed to swap uuid for instance {} which is alive, aborting failover",
|
||||
other_replica_instance.InstanceName()));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: (andi) fmap compliant
|
||||
ReplicationClientsInfo repl_clients_info;
|
||||
repl_clients_info.reserve(repl_instances_.size() - 1);
|
||||
std::ranges::transform(repl_instances_ | ranges::views::filter(is_not_new_main),
|
||||
std::back_inserter(repl_clients_info), &ReplicationInstance::ReplicationClientInfo);
|
||||
|
||||
if (!new_main->PromoteToMain(new_main_uuid, std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
spdlog::warn("Failover failed since promoting replica to main failed!");
|
||||
return;
|
||||
}
|
||||
// TODO: (andi) This should be replicated across all coordinator instances with Raft log
|
||||
SetMainUUID(new_main_uuid);
|
||||
spdlog::info("Failover successful! Instance {} promoted to main.", new_main->InstanceName());
|
||||
}
|
||||
|
||||
// TODO: (andi) Make sure you cannot put coordinator instance to the main
|
||||
auto CoordinatorInstance::SetReplicationInstanceToMain(std::string instance_name)
|
||||
-> SetInstanceToMainCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, &ReplicationInstance::IsMain)) {
|
||||
return SetInstanceToMainCoordinatorStatus::MAIN_ALREADY_EXISTS;
|
||||
}
|
||||
|
||||
auto const is_new_main = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
};
|
||||
auto new_main = std::ranges::find_if(repl_instances_, is_new_main);
|
||||
|
||||
if (new_main == repl_instances_.end()) {
|
||||
spdlog::error("Instance {} not registered. Please register it using REGISTER INSTANCE {}", instance_name,
|
||||
instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
auto const is_not_new_main = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() != instance_name;
|
||||
};
|
||||
|
||||
auto const new_main_uuid = utils::UUID{};
|
||||
|
||||
for (auto &other_instance : repl_instances_ | ranges::views::filter(is_not_new_main)) {
|
||||
if (!other_instance.SendSwapAndUpdateUUID(new_main_uuid)) {
|
||||
spdlog::error(
|
||||
fmt::format("Failed to swap uuid for instance {}, aborting failover", other_instance.InstanceName()));
|
||||
return SetInstanceToMainCoordinatorStatus::SWAP_UUID_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
ReplicationClientsInfo repl_clients_info;
|
||||
repl_clients_info.reserve(repl_instances_.size() - 1);
|
||||
std::ranges::transform(repl_instances_ | ranges::views::filter(is_not_new_main),
|
||||
std::back_inserter(repl_clients_info), &ReplicationInstance::ReplicationClientInfo);
|
||||
|
||||
if (!new_main->PromoteToMain(new_main_uuid, std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
|
||||
return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
|
||||
}
|
||||
|
||||
// TODO: (andi) This should be replicated across all coordinator instances with Raft log
|
||||
SetMainUUID(new_main_uuid);
|
||||
spdlog::info("Instance {} promoted to main", instance_name);
|
||||
return SetInstanceToMainCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::RegisterReplicationInstance(CoordinatorClientConfig config)
|
||||
-> RegisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
|
||||
auto instance_name = config.instance_name;
|
||||
|
||||
auto const name_matches = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
};
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, name_matches)) {
|
||||
return RegisterInstanceCoordinatorStatus::NAME_EXISTS;
|
||||
}
|
||||
|
||||
auto const socket_address_matches = [&config](ReplicationInstance const &instance) {
|
||||
return instance.SocketAddress() == config.SocketAddress();
|
||||
};
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, socket_address_matches)) {
|
||||
return RegisterInstanceCoordinatorStatus::ENDPOINT_EXISTS;
|
||||
}
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return RegisterInstanceCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
|
||||
auto const res = raft_state_.AppendRegisterReplicationInstance(instance_name);
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for registering instance {}. Most likely the reason is that the instance is not "
|
||||
"the "
|
||||
"leader.",
|
||||
config.instance_name);
|
||||
return RegisterInstanceCoordinatorStatus::RAFT_COULD_NOT_ACCEPT;
|
||||
}
|
||||
|
||||
spdlog::info("Request for registering instance {} accepted", instance_name);
|
||||
try {
|
||||
repl_instances_.emplace_back(this, std::move(config), replica_succ_cb_, replica_fail_cb_);
|
||||
} catch (CoordinatorRegisterInstanceException const &) {
|
||||
return RegisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
}
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to register instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return RegisterInstanceCoordinatorStatus::RAFT_COULD_NOT_APPEND;
|
||||
}
|
||||
|
||||
spdlog::info("Instance {} registered", instance_name);
|
||||
return RegisterInstanceCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::UnregisterReplicationInstance(std::string instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
|
||||
auto const name_matches = [&instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
};
|
||||
|
||||
auto inst_to_remove = std::ranges::find_if(repl_instances_, name_matches);
|
||||
if (inst_to_remove == repl_instances_.end()) {
|
||||
return UnregisterInstanceCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
if (inst_to_remove->IsMain() && inst_to_remove->IsAlive()) {
|
||||
return UnregisterInstanceCoordinatorStatus::IS_MAIN;
|
||||
}
|
||||
|
||||
inst_to_remove->StopFrequentCheck();
|
||||
auto curr_main = std::ranges::find_if(repl_instances_, &ReplicationInstance::IsMain);
|
||||
MG_ASSERT(curr_main != repl_instances_.end(), "There must be a main instance when unregistering a replica");
|
||||
if (!curr_main->SendUnregisterReplicaRpc(instance_name)) {
|
||||
inst_to_remove->StartFrequentCheck();
|
||||
return UnregisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
}
|
||||
std::erase_if(repl_instances_, name_matches);
|
||||
|
||||
return UnregisterInstanceCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string raft_address)
|
||||
-> void {
|
||||
raft_state_.AddCoordinatorInstance(raft_server_id, raft_port, std::move(raft_address));
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::GetMainUUID() const -> utils::UUID { return main_uuid_; }
|
||||
|
||||
// TODO: (andi) Add to the RAFT log.
|
||||
auto CoordinatorInstance::SetMainUUID(utils::UUID new_uuid) -> void { main_uuid_ = new_uuid; }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
201
src/coordination/coordinator_log_store.cpp
Normal file
201
src/coordination/coordinator_log_store.cpp
Normal file
@@ -0,0 +1,201 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_log_store.hpp"
|
||||
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::cs_new;
|
||||
|
||||
namespace {
|
||||
|
||||
ptr<log_entry> MakeClone(const ptr<log_entry> &entry) {
|
||||
return cs_new<log_entry>(entry->get_term(), buffer::clone(entry->get_buf()), entry->get_val_type(),
|
||||
entry->get_timestamp());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
CoordinatorLogStore::CoordinatorLogStore() : start_idx_(1) {
|
||||
ptr<buffer> buf = buffer::alloc(sizeof(uint64_t));
|
||||
logs_[0] = cs_new<log_entry>(0, buf);
|
||||
}
|
||||
|
||||
CoordinatorLogStore::~CoordinatorLogStore() {}
|
||||
|
||||
auto CoordinatorLogStore::FindOrDefault_(uint64_t index) const -> ptr<log_entry> {
|
||||
auto entry = logs_.find(index);
|
||||
if (entry == logs_.end()) {
|
||||
entry = logs_.find(0);
|
||||
}
|
||||
return entry->second;
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::next_slot() const {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
return start_idx_ + logs_.size() - 1;
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::start_index() const { return start_idx_; }
|
||||
|
||||
ptr<log_entry> CoordinatorLogStore::last_entry() const {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
|
||||
uint64_t const last_idx = start_idx_ + logs_.size() - 1;
|
||||
auto const last_src = FindOrDefault_(last_idx - 1);
|
||||
|
||||
return MakeClone(last_src);
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::append(ptr<log_entry> &entry) {
|
||||
ptr<log_entry> clone = MakeClone(entry);
|
||||
uint64_t next_slot{0};
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
next_slot = start_idx_ + logs_.size() - 1;
|
||||
logs_[next_slot] = clone;
|
||||
}
|
||||
return next_slot;
|
||||
}
|
||||
|
||||
void CoordinatorLogStore::write_at(uint64_t index, ptr<log_entry> &entry) {
|
||||
ptr<log_entry> clone = MakeClone(entry);
|
||||
|
||||
// Discard all logs equal to or greater than `index.
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
auto itr = logs_.lower_bound(index);
|
||||
while (itr != logs_.end()) {
|
||||
itr = logs_.erase(itr);
|
||||
}
|
||||
logs_[index] = clone;
|
||||
}
|
||||
}
|
||||
|
||||
ptr<std::vector<ptr<log_entry>>> CoordinatorLogStore::log_entries(uint64_t start, uint64_t end) {
|
||||
auto ret = cs_new<std::vector<ptr<log_entry>>>();
|
||||
ret->resize(end - start);
|
||||
|
||||
for (uint64_t i = start, curr_index = 0; i < end; ++i, ++curr_index) {
|
||||
ptr<log_entry> src = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
if (auto const entry = logs_.find(i); entry != logs_.end()) {
|
||||
src = entry->second;
|
||||
} else {
|
||||
throw RaftCouldNotFindEntryException("Could not find entry at index {}", i);
|
||||
}
|
||||
}
|
||||
(*ret)[curr_index] = MakeClone(src);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
ptr<log_entry> CoordinatorLogStore::entry_at(uint64_t index) {
|
||||
ptr<log_entry> src = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
src = FindOrDefault_(index);
|
||||
}
|
||||
return MakeClone(src);
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::term_at(uint64_t index) {
|
||||
uint64_t term = 0;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
term = FindOrDefault_(index)->get_term();
|
||||
}
|
||||
return term;
|
||||
}
|
||||
|
||||
ptr<buffer> CoordinatorLogStore::pack(uint64_t index, int32 cnt) {
|
||||
std::vector<ptr<buffer>> logs;
|
||||
|
||||
size_t size_total = 0;
|
||||
uint64_t const end_index = index + cnt;
|
||||
for (uint64_t i = index; i < end_index; ++i) {
|
||||
ptr<log_entry> le = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
le = logs_[i];
|
||||
}
|
||||
MG_ASSERT(le.get(), "Could not find log entry at index {}", i);
|
||||
auto buf = le->serialize();
|
||||
size_total += buf->size();
|
||||
logs.push_back(buf);
|
||||
}
|
||||
|
||||
auto buf_out = buffer::alloc(sizeof(int32) + cnt * sizeof(int32) + size_total);
|
||||
buf_out->pos(0);
|
||||
buf_out->put((int32)cnt);
|
||||
|
||||
for (auto &entry : logs) {
|
||||
buf_out->put(static_cast<int32>(entry->size()));
|
||||
buf_out->put(*entry);
|
||||
}
|
||||
return buf_out;
|
||||
}
|
||||
|
||||
void CoordinatorLogStore::apply_pack(uint64_t index, buffer &pack) {
|
||||
pack.pos(0);
|
||||
int32 const num_logs = pack.get_int();
|
||||
|
||||
for (int32 i = 0; i < num_logs; ++i) {
|
||||
uint64_t cur_idx = index + i;
|
||||
int32 buf_size = pack.get_int();
|
||||
|
||||
ptr<buffer> buf_local = buffer::alloc(buf_size);
|
||||
pack.get(buf_local);
|
||||
|
||||
ptr<log_entry> le = log_entry::deserialize(*buf_local);
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
logs_[cur_idx] = le;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
auto const entry = logs_.upper_bound(0);
|
||||
if (entry != logs_.end()) {
|
||||
start_idx_ = entry->first;
|
||||
} else {
|
||||
start_idx_ = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// NOTE: Remove all logs up to given 'last_log_index' (inclusive).
|
||||
bool CoordinatorLogStore::compact(uint64_t last_log_index) {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
for (uint64_t ii = start_idx_; ii <= last_log_index; ++ii) {
|
||||
auto const entry = logs_.find(ii);
|
||||
if (entry != logs_.end()) {
|
||||
logs_.erase(entry);
|
||||
}
|
||||
}
|
||||
|
||||
if (start_idx_ <= last_log_index) {
|
||||
start_idx_ = last_log_index + 1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool CoordinatorLogStore::flush() { return true; }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
220
src/coordination/coordinator_rpc.cpp
Normal file
220
src/coordination/coordinator_rpc.cpp
Normal file
@@ -0,0 +1,220 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
|
||||
#include "coordination/coordinator_slk.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
|
||||
namespace memgraph {
|
||||
|
||||
namespace coordination {
|
||||
|
||||
void PromoteReplicaToMainReq::Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void PromoteReplicaToMainReq::Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void PromoteReplicaToMainRes::Save(const PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void PromoteReplicaToMainRes::Load(PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void DemoteMainToReplicaReq::Save(const DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void DemoteMainToReplicaReq::Load(DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void DemoteMainToReplicaRes::Save(const DemoteMainToReplicaRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void DemoteMainToReplicaRes::Load(DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void UnregisterReplicaReq::Save(UnregisterReplicaReq const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void UnregisterReplicaReq::Load(UnregisterReplicaReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void UnregisterReplicaRes::Save(UnregisterReplicaRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void UnregisterReplicaRes::Load(UnregisterReplicaRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void EnableWritingOnMainRes::Save(EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void EnableWritingOnMainRes::Load(EnableWritingOnMainRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void EnableWritingOnMainReq::Save(EnableWritingOnMainReq const &self, memgraph::slk::Builder *builder) {}
|
||||
|
||||
void EnableWritingOnMainReq::Load(EnableWritingOnMainReq *self, memgraph::slk::Reader *reader) {}
|
||||
|
||||
// GetInstanceUUID
|
||||
void GetInstanceUUIDReq::Save(const GetInstanceUUIDReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void GetInstanceUUIDReq::Load(GetInstanceUUIDReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
void GetInstanceUUIDRes::Save(const GetInstanceUUIDRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self, builder);
|
||||
}
|
||||
|
||||
void GetInstanceUUIDRes::Load(GetInstanceUUIDRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(self, reader);
|
||||
}
|
||||
|
||||
} // namespace coordination
|
||||
|
||||
constexpr utils::TypeInfo coordination::PromoteReplicaToMainReq::kType{utils::TypeId::COORD_FAILOVER_REQ,
|
||||
"CoordPromoteReplicaToMainReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::PromoteReplicaToMainRes::kType{utils::TypeId::COORD_FAILOVER_RES,
|
||||
"CoordPromoteReplicaToMainRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::DemoteMainToReplicaReq::kType{utils::TypeId::COORD_SET_REPL_MAIN_REQ,
|
||||
"CoordDemoteToReplicaReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::DemoteMainToReplicaRes::kType{utils::TypeId::COORD_SET_REPL_MAIN_RES,
|
||||
|
||||
"CoordDemoteToReplicaRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::UnregisterReplicaReq::kType{utils::TypeId::COORD_UNREGISTER_REPLICA_REQ,
|
||||
"UnregisterReplicaReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::UnregisterReplicaRes::kType{utils::TypeId::COORD_UNREGISTER_REPLICA_RES,
|
||||
"UnregisterReplicaRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::EnableWritingOnMainReq::kType{utils::TypeId::COORD_ENABLE_WRITING_ON_MAIN_REQ,
|
||||
"CoordEnableWritingOnMainReq", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::EnableWritingOnMainRes::kType{utils::TypeId::COORD_ENABLE_WRITING_ON_MAIN_RES,
|
||||
"CoordEnableWritingOnMainRes", nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::GetInstanceUUIDReq::kType{utils::TypeId::COORD_GET_UUID_REQ, "CoordGetUUIDReq",
|
||||
nullptr};
|
||||
|
||||
constexpr utils::TypeInfo coordination::GetInstanceUUIDRes::kType{utils::TypeId::COORD_GET_UUID_RES, "CoordGetUUIDRes",
|
||||
nullptr};
|
||||
|
||||
namespace slk {
|
||||
|
||||
// PromoteReplicaToMainRpc
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.main_uuid_, builder);
|
||||
memgraph::slk::Save(self.replication_clients_info, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->main_uuid_, reader);
|
||||
memgraph::slk::Load(&self->replication_clients_info, reader);
|
||||
}
|
||||
|
||||
// DemoteMainToReplicaRpc
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.replication_client_info, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->replication_client_info, reader);
|
||||
}
|
||||
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
// UnregisterReplicaRpc
|
||||
|
||||
void Save(memgraph::coordination::UnregisterReplicaReq const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.instance_name, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::UnregisterReplicaReq *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->instance_name, reader);
|
||||
}
|
||||
|
||||
void Save(memgraph::coordination::UnregisterReplicaRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::UnregisterReplicaRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
void Save(memgraph::coordination::EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.success, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::EnableWritingOnMainRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->success, reader);
|
||||
}
|
||||
|
||||
// GetInstanceUUIDRpc
|
||||
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDReq & /*self*/, memgraph::slk::Builder * /*builder*/) {
|
||||
/* nothing to serialize*/
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::GetInstanceUUIDReq * /*self*/, memgraph::slk::Reader * /*reader*/) {
|
||||
/* nothing to serialize*/
|
||||
}
|
||||
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDRes &self, memgraph::slk::Builder *builder) {
|
||||
memgraph::slk::Save(self.uuid, builder);
|
||||
}
|
||||
|
||||
void Load(memgraph::coordination::GetInstanceUUIDRes *self, memgraph::slk::Reader *reader) {
|
||||
memgraph::slk::Load(&self->uuid, reader);
|
||||
}
|
||||
|
||||
} // namespace slk
|
||||
|
||||
} // namespace memgraph
|
||||
|
||||
#endif
|
||||
57
src/coordination/coordinator_server.cpp
Normal file
57
src/coordination/coordinator_server.cpp
Normal file
@@ -0,0 +1,57 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "replication_coordination_glue/handler.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
namespace {
|
||||
|
||||
auto CreateServerContext(const memgraph::coordination::CoordinatorServerConfig &config)
|
||||
-> communication::ServerContext {
|
||||
return (config.ssl) ? communication::ServerContext{config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file,
|
||||
config.ssl->verify_peer}
|
||||
: communication::ServerContext{};
|
||||
}
|
||||
|
||||
// NOTE: The coordinator server doesn't more than 1 processing thread - each replica can
|
||||
// have only a single coordinator server. Also, the single-threaded guarantee
|
||||
// simplifies the rest of the implementation.
|
||||
constexpr auto kCoordinatorServerThreads = 1;
|
||||
|
||||
} // namespace
|
||||
|
||||
CoordinatorServer::CoordinatorServer(const CoordinatorServerConfig &config)
|
||||
: rpc_server_context_{CreateServerContext(config)},
|
||||
rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_,
|
||||
kCoordinatorServerThreads} {
|
||||
rpc_server_.Register<replication_coordination_glue::FrequentHeartbeatRpc>([](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received FrequentHeartbeatRpc on coordinator server");
|
||||
replication_coordination_glue::FrequentHeartbeatHandler(req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
|
||||
CoordinatorServer::~CoordinatorServer() {
|
||||
if (rpc_server_.IsRunning()) {
|
||||
auto const &endpoint = rpc_server_.endpoint();
|
||||
spdlog::trace("Closing coordinator server on {}:{}", endpoint.address, endpoint.port);
|
||||
rpc_server_.Shutdown();
|
||||
}
|
||||
rpc_server_.AwaitShutdown();
|
||||
}
|
||||
|
||||
bool CoordinatorServer::Start() { return rpc_server_.Start(); }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
107
src/coordination/coordinator_state.cpp
Normal file
107
src/coordination/coordinator_state.cpp
Normal file
@@ -0,0 +1,107 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_state.hpp"
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "flags/replication.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
CoordinatorState::CoordinatorState() {
|
||||
MG_ASSERT(!(FLAGS_raft_server_id && FLAGS_coordinator_server_port),
|
||||
"Instance cannot be a coordinator and have registered coordinator server.");
|
||||
|
||||
spdlog::info("Executing coordinator constructor");
|
||||
if (FLAGS_coordinator_server_port) {
|
||||
spdlog::info("Coordinator server port set");
|
||||
auto const config = CoordinatorServerConfig{
|
||||
.ip_address = kDefaultReplicationServerIp,
|
||||
.port = static_cast<uint16_t>(FLAGS_coordinator_server_port),
|
||||
};
|
||||
spdlog::info("Executing coordinator constructor main replica");
|
||||
|
||||
data_ = CoordinatorMainReplicaData{.coordinator_server_ = std::make_unique<CoordinatorServer>(config)};
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorState::RegisterReplicationInstance(CoordinatorClientConfig config)
|
||||
-> RegisterInstanceCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return RegisterInstanceCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[config](CoordinatorInstance &coordinator_instance) {
|
||||
return coordinator_instance.RegisterReplicationInstance(config);
|
||||
}},
|
||||
data_);
|
||||
}
|
||||
|
||||
auto CoordinatorState::UnregisterReplicationInstance(std::string instance_name) -> UnregisterInstanceCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot unregister instance since variant holds wrong alternative");
|
||||
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return UnregisterInstanceCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[&instance_name](CoordinatorInstance &coordinator_instance) {
|
||||
return coordinator_instance.UnregisterReplicationInstance(instance_name);
|
||||
}},
|
||||
data_);
|
||||
}
|
||||
|
||||
auto CoordinatorState::SetReplicationInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{[](const CoordinatorMainReplicaData & /*coordinator_main_replica_data*/) {
|
||||
return SetInstanceToMainCoordinatorStatus::NOT_COORDINATOR;
|
||||
},
|
||||
[&instance_name](CoordinatorInstance &coordinator_instance) {
|
||||
return coordinator_instance.SetReplicationInstanceToMain(instance_name);
|
||||
}},
|
||||
data_);
|
||||
}
|
||||
|
||||
auto CoordinatorState::ShowInstances() const -> std::vector<InstanceStatus> {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Can't call show instances on data_, as variant holds wrong alternative");
|
||||
return std::get<CoordinatorInstance>(data_).ShowInstances();
|
||||
}
|
||||
|
||||
auto CoordinatorState::GetCoordinatorServer() const -> CoordinatorServer & {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorMainReplicaData>(data_),
|
||||
"Cannot get coordinator server since variant holds wrong alternative");
|
||||
return *std::get<CoordinatorMainReplicaData>(data_).coordinator_server_;
|
||||
}
|
||||
|
||||
auto CoordinatorState::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string raft_address)
|
||||
-> void {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
return std::get<CoordinatorInstance>(data_).AddCoordinatorInstance(raft_server_id, raft_port, raft_address);
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
111
src/coordination/coordinator_state_machine.cpp
Normal file
111
src/coordination/coordinator_state_machine.cpp
Normal file
@@ -0,0 +1,111 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
auto CoordinatorStateMachine::EncodeRegisterReplicationInstance(const std::string &name) -> ptr<buffer> {
|
||||
std::string str_log = name + "_replica";
|
||||
ptr<buffer> log = buffer::alloc(sizeof(uint32_t) + str_log.size());
|
||||
buffer_serializer bs(log);
|
||||
bs.put_str(str_log);
|
||||
return log;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::DecodeRegisterReplicationInstance(buffer &data) -> std::string {
|
||||
buffer_serializer bs(data);
|
||||
return bs.get_str();
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::pre_commit(ulong const log_idx, buffer &data) -> ptr<buffer> {
|
||||
buffer_serializer bs(data);
|
||||
std::string str = bs.get_str();
|
||||
|
||||
spdlog::info("pre_commit {} : {}", log_idx, str);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::commit(ulong const log_idx, buffer &data) -> ptr<buffer> {
|
||||
buffer_serializer bs(data);
|
||||
std::string str = bs.get_str();
|
||||
|
||||
spdlog::info("commit {} : {}", log_idx, str);
|
||||
|
||||
last_committed_idx_ = log_idx;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::commit_config(ulong const log_idx, ptr<cluster_config> & /*new_conf*/) -> void {
|
||||
last_committed_idx_ = log_idx;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::rollback(ulong const log_idx, buffer &data) -> void {
|
||||
buffer_serializer bs(data);
|
||||
std::string str = bs.get_str();
|
||||
|
||||
spdlog::info("rollback {} : {}", log_idx, str);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::read_logical_snp_obj(snapshot & /*snapshot*/, void *& /*user_snp_ctx*/, ulong /*obj_id*/,
|
||||
ptr<buffer> &data_out, bool &is_last_obj) -> int {
|
||||
// Put dummy data.
|
||||
data_out = buffer::alloc(sizeof(int32));
|
||||
buffer_serializer bs(data_out);
|
||||
bs.put_i32(0);
|
||||
|
||||
is_last_obj = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::save_logical_snp_obj(snapshot &s, ulong &obj_id, buffer & /*data*/, bool /*is_first_obj*/,
|
||||
bool /*is_last_obj*/) -> void {
|
||||
spdlog::info("save snapshot {} term {} object ID", s.get_last_log_idx(), s.get_last_log_term(), obj_id);
|
||||
// Request next object.
|
||||
obj_id++;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::apply_snapshot(snapshot &s) -> bool {
|
||||
spdlog::info("apply snapshot {} term {}", s.get_last_log_idx(), s.get_last_log_term());
|
||||
{
|
||||
auto lock = std::lock_guard{last_snapshot_lock_};
|
||||
ptr<buffer> snp_buf = s.serialize();
|
||||
last_snapshot_ = snapshot::deserialize(*snp_buf);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::free_user_snp_ctx(void *&user_snp_ctx) -> void {}
|
||||
|
||||
auto CoordinatorStateMachine::last_snapshot() -> ptr<snapshot> {
|
||||
auto lock = std::lock_guard{last_snapshot_lock_};
|
||||
return last_snapshot_;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::last_commit_index() -> ulong { return last_committed_idx_; }
|
||||
|
||||
auto CoordinatorStateMachine::create_snapshot(snapshot &s, async_result<bool>::handler_type &when_done) -> void {
|
||||
spdlog::info("create snapshot {} term {}", s.get_last_log_idx(), s.get_last_log_term());
|
||||
// Clone snapshot from `s`.
|
||||
{
|
||||
auto lock = std::lock_guard{last_snapshot_lock_};
|
||||
ptr<buffer> snp_buf = s.serialize();
|
||||
last_snapshot_ = snapshot::deserialize(*snp_buf);
|
||||
}
|
||||
ptr<std::exception> except(nullptr);
|
||||
bool ret = true;
|
||||
when_done(ret, except);
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
68
src/coordination/coordinator_state_manager.cpp
Normal file
68
src/coordination/coordinator_state_manager.cpp
Normal file
@@ -0,0 +1,68 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::cluster_config;
|
||||
using nuraft::cs_new;
|
||||
using nuraft::srv_config;
|
||||
using nuraft::srv_state;
|
||||
using nuraft::state_mgr;
|
||||
|
||||
CoordinatorStateManager::CoordinatorStateManager(int srv_id, std::string const &endpoint)
|
||||
: my_id_(srv_id), my_endpoint_(endpoint), cur_log_store_(cs_new<CoordinatorLogStore>()) {
|
||||
my_srv_config_ = cs_new<srv_config>(srv_id, endpoint);
|
||||
|
||||
// Initial cluster config: contains only one server (myself).
|
||||
cluster_config_ = cs_new<cluster_config>();
|
||||
cluster_config_->get_servers().push_back(my_srv_config_);
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::load_config() -> ptr<cluster_config> {
|
||||
// Just return in-memory data in this example.
|
||||
// May require reading from disk here, if it has been written to disk.
|
||||
return cluster_config_;
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::save_config(cluster_config const &config) -> void {
|
||||
// Just keep in memory in this example.
|
||||
// Need to write to disk here, if want to make it durable.
|
||||
ptr<buffer> buf = config.serialize();
|
||||
cluster_config_ = cluster_config::deserialize(*buf);
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::save_state(srv_state const &state) -> void {
|
||||
// Just keep in memory in this example.
|
||||
// Need to write to disk here, if want to make it durable.
|
||||
ptr<buffer> buf = state.serialize();
|
||||
saved_state_ = srv_state::deserialize(*buf);
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::read_state() -> ptr<srv_state> {
|
||||
// Just return in-memory data in this example.
|
||||
// May require reading from disk here, if it has been written to disk.
|
||||
return saved_state_;
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::load_log_store() -> ptr<log_store> { return cur_log_store_; }
|
||||
|
||||
auto CoordinatorStateManager::server_id() -> int32 { return my_id_; }
|
||||
|
||||
auto CoordinatorStateManager::system_exit(int const exit_code) -> void {}
|
||||
|
||||
auto CoordinatorStateManager::GetSrvConfig() const -> ptr<srv_config> { return my_srv_config_; }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
22
src/coordination/include/coordination/constants.hpp
Normal file
22
src/coordination/include/coordination/constants.hpp
Normal file
@@ -0,0 +1,22 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
#ifdef MG_EXPERIMENTAL_HIGH_AVAILABILITY
|
||||
constexpr bool allow_ha = true;
|
||||
#else
|
||||
constexpr bool allow_ha = false;
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
90
src/coordination/include/coordination/coordinator_client.hpp
Normal file
90
src/coordination/include/coordination/coordinator_client.hpp
Normal file
@@ -0,0 +1,90 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "rpc/client.hpp"
|
||||
#include "rpc_errors.hpp"
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/scheduler.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
using HealthCheckCallback = std::function<void(CoordinatorInstance *, std::string_view)>;
|
||||
using ReplicationClientsInfo = std::vector<ReplClientInfo>;
|
||||
|
||||
class CoordinatorClient {
|
||||
public:
|
||||
explicit CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorClientConfig config,
|
||||
HealthCheckCallback succ_cb, HealthCheckCallback fail_cb);
|
||||
|
||||
~CoordinatorClient() = default;
|
||||
|
||||
CoordinatorClient(CoordinatorClient &) = delete;
|
||||
CoordinatorClient &operator=(CoordinatorClient const &) = delete;
|
||||
|
||||
CoordinatorClient(CoordinatorClient &&) noexcept = delete;
|
||||
CoordinatorClient &operator=(CoordinatorClient &&) noexcept = delete;
|
||||
|
||||
void StartFrequentCheck();
|
||||
void StopFrequentCheck();
|
||||
void PauseFrequentCheck();
|
||||
void ResumeFrequentCheck();
|
||||
|
||||
auto InstanceName() const -> std::string;
|
||||
auto SocketAddress() const -> std::string;
|
||||
|
||||
[[nodiscard]] auto DemoteToReplica() const -> bool;
|
||||
|
||||
auto SendPromoteReplicaToMainRpc(const utils::UUID &uuid, ReplicationClientsInfo replication_clients_info) const
|
||||
-> bool;
|
||||
|
||||
auto SendSwapMainUUIDRpc(const utils::UUID &uuid) const -> bool;
|
||||
|
||||
auto SendUnregisterReplicaRpc(std::string const &instance_name) const -> bool;
|
||||
|
||||
auto SendEnableWritingOnMainRpc() const -> bool;
|
||||
|
||||
auto SendGetInstanceUUIDRpc() const -> memgraph::utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>>;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCallback fail_cb) -> void;
|
||||
|
||||
auto RpcClient() -> rpc::Client & { return rpc_client_; }
|
||||
|
||||
auto InstanceDownTimeoutSec() const -> std::chrono::seconds;
|
||||
|
||||
auto InstanceGetUUIDFrequencySec() const -> std::chrono::seconds;
|
||||
|
||||
friend bool operator==(CoordinatorClient const &first, CoordinatorClient const &second) {
|
||||
return first.config_ == second.config_;
|
||||
}
|
||||
|
||||
private:
|
||||
utils::Scheduler instance_checker_;
|
||||
|
||||
communication::ClientContext rpc_context_;
|
||||
mutable rpc::Client rpc_client_;
|
||||
|
||||
CoordinatorClientConfig config_;
|
||||
CoordinatorInstance *coord_instance_;
|
||||
HealthCheckCallback succ_cb_;
|
||||
HealthCheckCallback fail_cb_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
79
src/coordination/include/coordination/coordinator_config.hpp
Normal file
79
src/coordination/include/coordination/coordinator_config.hpp
Normal file
@@ -0,0 +1,79 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "replication_coordination_glue/mode.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
|
||||
|
||||
struct CoordinatorClientConfig {
|
||||
std::string instance_name;
|
||||
std::string ip_address;
|
||||
uint16_t port{};
|
||||
std::chrono::seconds instance_health_check_frequency_sec{1};
|
||||
std::chrono::seconds instance_down_timeout_sec{5};
|
||||
std::chrono::seconds instance_get_uuid_frequency_sec{10};
|
||||
|
||||
auto SocketAddress() const -> std::string { return ip_address + ":" + std::to_string(port); }
|
||||
|
||||
struct ReplicationClientInfo {
|
||||
std::string instance_name;
|
||||
replication_coordination_glue::ReplicationMode replication_mode{};
|
||||
std::string replication_ip_address;
|
||||
uint16_t replication_port{};
|
||||
|
||||
friend bool operator==(ReplicationClientInfo const &, ReplicationClientInfo const &) = default;
|
||||
};
|
||||
|
||||
ReplicationClientInfo replication_client_info;
|
||||
|
||||
struct SSL {
|
||||
std::string key_file;
|
||||
std::string cert_file;
|
||||
|
||||
friend bool operator==(const SSL &, const SSL &) = default;
|
||||
};
|
||||
|
||||
std::optional<SSL> ssl;
|
||||
|
||||
friend bool operator==(CoordinatorClientConfig const &, CoordinatorClientConfig const &) = default;
|
||||
};
|
||||
|
||||
using ReplClientInfo = CoordinatorClientConfig::ReplicationClientInfo;
|
||||
|
||||
struct CoordinatorServerConfig {
|
||||
std::string ip_address;
|
||||
uint16_t port{};
|
||||
struct SSL {
|
||||
std::string key_file;
|
||||
std::string cert_file;
|
||||
std::string ca_file;
|
||||
bool verify_peer{};
|
||||
friend bool operator==(SSL const &, SSL const &) = default;
|
||||
};
|
||||
|
||||
std::optional<SSL> ssl;
|
||||
|
||||
friend bool operator==(CoordinatorServerConfig const &, CoordinatorServerConfig const &) = default;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -0,0 +1,87 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
class CoordinatorRegisterInstanceException final : public utils::BasicException {
|
||||
public:
|
||||
explicit CoordinatorRegisterInstanceException(const std::string_view what) noexcept
|
||||
: BasicException("Failed to create instance: " + std::string(what)) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit CoordinatorRegisterInstanceException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: CoordinatorRegisterInstanceException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(CoordinatorRegisterInstanceException)
|
||||
};
|
||||
|
||||
class RaftServerStartException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftServerStartException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftServerStartException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftServerStartException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftServerStartException)
|
||||
};
|
||||
|
||||
class RaftAddServerException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftAddServerException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftAddServerException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftAddServerException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftAddServerException)
|
||||
};
|
||||
|
||||
class RaftBecomeLeaderException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftBecomeLeaderException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftBecomeLeaderException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftBecomeLeaderException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftBecomeLeaderException)
|
||||
};
|
||||
|
||||
class RaftCouldNotFindEntryException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftCouldNotFindEntryException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftCouldNotFindEntryException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftCouldNotFindEntryException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftCouldNotFindEntryException)
|
||||
};
|
||||
|
||||
class RaftCouldNotParseFlagsException final : public utils::BasicException {
|
||||
public:
|
||||
explicit RaftCouldNotParseFlagsException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit RaftCouldNotParseFlagsException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: RaftCouldNotParseFlagsException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(RaftCouldNotParseFlagsException)
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "replication_handler/replication_handler.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
class DbmsHandler;
|
||||
|
||||
class CoordinatorHandlers {
|
||||
public:
|
||||
static void Register(memgraph::coordination::CoordinatorServer &server,
|
||||
replication::ReplicationHandler &replication_handler);
|
||||
|
||||
private:
|
||||
static void PromoteReplicaToMainHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
static void DemoteMainToReplicaHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
static void SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
|
||||
static void UnregisterReplicaHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
static void EnableWritingOnMainHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
|
||||
static void GetInstanceUUIDHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder);
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,60 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "coordination/instance_status.hpp"
|
||||
#include "coordination/raft_state.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "coordination/replication_instance.hpp"
|
||||
#include "utils/rw_lock.hpp"
|
||||
#include "utils/thread_pool.hpp"
|
||||
|
||||
#include <list>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance {
|
||||
public:
|
||||
CoordinatorInstance();
|
||||
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto UnregisterReplicationInstance(std::string instance_name) -> UnregisterInstanceCoordinatorStatus;
|
||||
|
||||
[[nodiscard]] auto SetReplicationInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
|
||||
|
||||
auto ShowInstances() const -> std::vector<InstanceStatus>;
|
||||
|
||||
auto TryFailover() -> void;
|
||||
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string raft_address) -> void;
|
||||
|
||||
auto GetMainUUID() const -> utils::UUID;
|
||||
|
||||
auto SetMainUUID(utils::UUID new_uuid) -> void;
|
||||
|
||||
private:
|
||||
HealthCheckCallback main_succ_cb_, main_fail_cb_, replica_succ_cb_, replica_fail_cb_;
|
||||
|
||||
// NOTE: Must be std::list because we rely on pointer stability
|
||||
std::list<ReplicationInstance> repl_instances_;
|
||||
mutable utils::RWLock coord_instance_lock_{utils::RWLock::Priority::READ};
|
||||
|
||||
utils::UUID main_uuid_;
|
||||
|
||||
RaftState raft_state_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
197
src/coordination/include/coordination/coordinator_rpc.hpp
Normal file
197
src/coordination/include/coordination/coordinator_rpc.hpp
Normal file
@@ -0,0 +1,197 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "rpc/messages.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
struct PromoteReplicaToMainReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit PromoteReplicaToMainReq(const utils::UUID &uuid,
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
|
||||
: main_uuid_(uuid), replication_clients_info(std::move(replication_clients_info)) {}
|
||||
PromoteReplicaToMainReq() = default;
|
||||
|
||||
// get uuid here
|
||||
utils::UUID main_uuid_;
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info;
|
||||
};
|
||||
|
||||
struct PromoteReplicaToMainRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit PromoteReplicaToMainRes(bool success) : success(success) {}
|
||||
PromoteReplicaToMainRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using PromoteReplicaToMainRpc = rpc::RequestResponse<PromoteReplicaToMainReq, PromoteReplicaToMainRes>;
|
||||
|
||||
struct DemoteMainToReplicaReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit DemoteMainToReplicaReq(CoordinatorClientConfig::ReplicationClientInfo replication_client_info)
|
||||
: replication_client_info(std::move(replication_client_info)) {}
|
||||
|
||||
DemoteMainToReplicaReq() = default;
|
||||
|
||||
CoordinatorClientConfig::ReplicationClientInfo replication_client_info;
|
||||
};
|
||||
|
||||
struct DemoteMainToReplicaRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const DemoteMainToReplicaRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit DemoteMainToReplicaRes(bool success) : success(success) {}
|
||||
DemoteMainToReplicaRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using DemoteMainToReplicaRpc = rpc::RequestResponse<DemoteMainToReplicaReq, DemoteMainToReplicaRes>;
|
||||
|
||||
struct UnregisterReplicaReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(UnregisterReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(UnregisterReplicaReq const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit UnregisterReplicaReq(std::string instance_name) : instance_name(std::move(instance_name)) {}
|
||||
|
||||
UnregisterReplicaReq() = default;
|
||||
|
||||
std::string instance_name;
|
||||
};
|
||||
|
||||
struct UnregisterReplicaRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(UnregisterReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const UnregisterReplicaRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit UnregisterReplicaRes(bool success) : success(success) {}
|
||||
UnregisterReplicaRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using UnregisterReplicaRpc = rpc::RequestResponse<UnregisterReplicaReq, UnregisterReplicaRes>;
|
||||
|
||||
struct EnableWritingOnMainReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(EnableWritingOnMainReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(EnableWritingOnMainReq const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
EnableWritingOnMainReq() = default;
|
||||
};
|
||||
|
||||
struct EnableWritingOnMainRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(EnableWritingOnMainRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit EnableWritingOnMainRes(bool success) : success(success) {}
|
||||
EnableWritingOnMainRes() = default;
|
||||
|
||||
bool success;
|
||||
};
|
||||
|
||||
using EnableWritingOnMainRpc = rpc::RequestResponse<EnableWritingOnMainReq, EnableWritingOnMainRes>;
|
||||
|
||||
struct GetInstanceUUIDReq {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(GetInstanceUUIDReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const GetInstanceUUIDReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
GetInstanceUUIDReq() = default;
|
||||
};
|
||||
|
||||
struct GetInstanceUUIDRes {
|
||||
static const utils::TypeInfo kType;
|
||||
static const utils::TypeInfo &GetTypeInfo() { return kType; }
|
||||
|
||||
static void Load(GetInstanceUUIDRes *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const GetInstanceUUIDRes &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit GetInstanceUUIDRes(std::optional<utils::UUID> uuid) : uuid(uuid) {}
|
||||
GetInstanceUUIDRes() = default;
|
||||
|
||||
std::optional<utils::UUID> uuid;
|
||||
};
|
||||
|
||||
using GetInstanceUUIDRpc = rpc::RequestResponse<GetInstanceUUIDReq, GetInstanceUUIDRes>;
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
// SLK serialization declarations
|
||||
namespace memgraph::slk {
|
||||
|
||||
// PromoteReplicaToMainRpc
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::coordination::PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// DemoteMainToReplicaRpc
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
// GetInstanceUUIDRpc
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDReq &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::GetInstanceUUIDReq *self, memgraph::slk::Reader *reader);
|
||||
void Save(const memgraph::coordination::GetInstanceUUIDRes &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::GetInstanceUUIDRes *self, memgraph::slk::Reader *reader);
|
||||
// UnregisterReplicaRpc
|
||||
void Save(memgraph::coordination::UnregisterReplicaRes const &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::UnregisterReplicaRes *self, memgraph::slk::Reader *reader);
|
||||
void Save(memgraph::coordination::UnregisterReplicaReq const &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::UnregisterReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
|
||||
void Save(memgraph::coordination::EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder);
|
||||
void Load(memgraph::coordination::EnableWritingOnMainRes *self, memgraph::slk::Reader *reader);
|
||||
|
||||
} // namespace memgraph::slk
|
||||
|
||||
#endif
|
||||
44
src/coordination/include/coordination/coordinator_server.hpp
Normal file
44
src/coordination/include/coordination/coordinator_server.hpp
Normal file
@@ -0,0 +1,44 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "rpc/server.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorServer {
|
||||
public:
|
||||
explicit CoordinatorServer(const CoordinatorServerConfig &config);
|
||||
CoordinatorServer(const CoordinatorServer &) = delete;
|
||||
CoordinatorServer(CoordinatorServer &&) = delete;
|
||||
CoordinatorServer &operator=(const CoordinatorServer &) = delete;
|
||||
CoordinatorServer &operator=(CoordinatorServer &&) = delete;
|
||||
|
||||
virtual ~CoordinatorServer();
|
||||
|
||||
bool Start();
|
||||
|
||||
template <typename TRequestResponse, typename F>
|
||||
void Register(F &&callback) {
|
||||
rpc_server_.Register<TRequestResponse>(std::forward<F>(callback));
|
||||
}
|
||||
|
||||
private:
|
||||
communication::ServerContext rpc_server_context_;
|
||||
rpc::Server rpc_server_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
38
src/coordination/include/coordination/coordinator_slk.hpp
Normal file
38
src/coordination/include/coordination/coordinator_slk.hpp
Normal file
@@ -0,0 +1,38 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
|
||||
namespace memgraph::slk {
|
||||
|
||||
using ReplicationClientInfo = coordination::CoordinatorClientConfig::ReplicationClientInfo;
|
||||
|
||||
inline void Save(const ReplicationClientInfo &obj, Builder *builder) {
|
||||
Save(obj.instance_name, builder);
|
||||
Save(obj.replication_mode, builder);
|
||||
Save(obj.replication_ip_address, builder);
|
||||
Save(obj.replication_port, builder);
|
||||
}
|
||||
|
||||
inline void Load(ReplicationClientInfo *obj, Reader *reader) {
|
||||
Load(&obj->instance_name, reader);
|
||||
Load(&obj->replication_mode, reader);
|
||||
Load(&obj->replication_ip_address, reader);
|
||||
Load(&obj->replication_port, reader);
|
||||
}
|
||||
} // namespace memgraph::slk
|
||||
#endif
|
||||
57
src/coordination/include/coordination/coordinator_state.hpp
Normal file
57
src/coordination/include/coordination/coordinator_state.hpp
Normal file
@@ -0,0 +1,57 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
#include "coordination/coordinator_server.hpp"
|
||||
#include "coordination/instance_status.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
|
||||
#include <variant>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorState {
|
||||
public:
|
||||
CoordinatorState();
|
||||
~CoordinatorState() = default;
|
||||
|
||||
CoordinatorState(CoordinatorState const &) = delete;
|
||||
CoordinatorState &operator=(CoordinatorState const &) = delete;
|
||||
|
||||
CoordinatorState(CoordinatorState &&) noexcept = delete;
|
||||
CoordinatorState &operator=(CoordinatorState &&) noexcept = delete;
|
||||
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorClientConfig config) -> RegisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto UnregisterReplicationInstance(std::string instance_name) -> UnregisterInstanceCoordinatorStatus;
|
||||
|
||||
[[nodiscard]] auto SetReplicationInstanceToMain(std::string instance_name) -> SetInstanceToMainCoordinatorStatus;
|
||||
|
||||
auto ShowInstances() const -> std::vector<InstanceStatus>;
|
||||
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string raft_address) -> void;
|
||||
|
||||
// NOTE: The client code must check that the server exists before calling this method.
|
||||
auto GetCoordinatorServer() const -> CoordinatorServer &;
|
||||
|
||||
private:
|
||||
struct CoordinatorMainReplicaData {
|
||||
std::unique_ptr<CoordinatorServer> coordinator_server_;
|
||||
};
|
||||
|
||||
std::variant<CoordinatorInstance, CoordinatorMainReplicaData> data_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
34
src/coordination/include/coordination/instance_status.hpp
Normal file
34
src/coordination/include/coordination/instance_status.hpp
Normal file
@@ -0,0 +1,34 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
|
||||
#include <string_view>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
// TODO: (andi) For phase IV. Some instances won't have raft_socket_address, coord_socket_address, replication_role and
|
||||
// cluster role... At the end, all instances will have everything.
|
||||
struct InstanceStatus {
|
||||
std::string instance_name;
|
||||
std::string raft_socket_address;
|
||||
std::string coord_socket_address;
|
||||
std::string cluster_role;
|
||||
bool is_alive;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
#endif
|
||||
79
src/coordination/include/coordination/raft_state.hpp
Normal file
79
src/coordination/include/coordination/raft_state.hpp
Normal file
@@ -0,0 +1,79 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <flags/replication.hpp>
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using BecomeLeaderCb = std::function<void()>;
|
||||
using BecomeFollowerCb = std::function<void()>;
|
||||
|
||||
using nuraft::buffer;
|
||||
using nuraft::logger;
|
||||
using nuraft::ptr;
|
||||
using nuraft::raft_launcher;
|
||||
using nuraft::raft_server;
|
||||
using nuraft::srv_config;
|
||||
using nuraft::state_machine;
|
||||
using nuraft::state_mgr;
|
||||
using raft_result = nuraft::cmd_result<ptr<buffer>>;
|
||||
|
||||
class RaftState {
|
||||
private:
|
||||
explicit RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t raft_server_id,
|
||||
uint32_t raft_port, std::string raft_address);
|
||||
|
||||
auto InitRaftServer() -> void;
|
||||
|
||||
public:
|
||||
RaftState() = delete;
|
||||
RaftState(RaftState const &other) = default;
|
||||
RaftState &operator=(RaftState const &other) = default;
|
||||
RaftState(RaftState &&other) noexcept = default;
|
||||
RaftState &operator=(RaftState &&other) noexcept = default;
|
||||
~RaftState();
|
||||
|
||||
static auto MakeRaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb) -> RaftState;
|
||||
|
||||
auto InstanceName() const -> std::string;
|
||||
auto RaftSocketAddress() const -> std::string;
|
||||
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string raft_address) -> void;
|
||||
auto GetAllCoordinators() const -> std::vector<ptr<srv_config>>;
|
||||
|
||||
auto RequestLeadership() -> bool;
|
||||
auto IsLeader() const -> bool;
|
||||
|
||||
auto AppendRegisterReplicationInstance(std::string const &instance) -> ptr<raft_result>;
|
||||
|
||||
// TODO: (andi) I think variables below can be abstracted
|
||||
uint32_t raft_server_id_;
|
||||
uint32_t raft_port_;
|
||||
std::string raft_address_;
|
||||
|
||||
ptr<state_machine> state_machine_;
|
||||
ptr<state_mgr> state_manager_;
|
||||
ptr<raft_server> raft_server_;
|
||||
ptr<logger> logger_;
|
||||
raft_launcher launcher_;
|
||||
|
||||
BecomeLeaderCb become_leader_cb_;
|
||||
BecomeFollowerCb become_follower_cb_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -0,0 +1,50 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
enum class RegisterInstanceCoordinatorStatus : uint8_t {
|
||||
NAME_EXISTS,
|
||||
ENDPOINT_EXISTS,
|
||||
NOT_COORDINATOR,
|
||||
RPC_FAILED,
|
||||
NOT_LEADER,
|
||||
RAFT_COULD_NOT_ACCEPT,
|
||||
RAFT_COULD_NOT_APPEND,
|
||||
SUCCESS
|
||||
};
|
||||
|
||||
enum class UnregisterInstanceCoordinatorStatus : uint8_t {
|
||||
NO_INSTANCE_WITH_NAME,
|
||||
IS_MAIN,
|
||||
NOT_COORDINATOR,
|
||||
NOT_LEADER,
|
||||
RPC_FAILED,
|
||||
SUCCESS,
|
||||
};
|
||||
|
||||
enum class SetInstanceToMainCoordinatorStatus : uint8_t {
|
||||
NO_INSTANCE_WITH_NAME,
|
||||
MAIN_ALREADY_EXISTS,
|
||||
NOT_COORDINATOR,
|
||||
SUCCESS,
|
||||
COULD_NOT_PROMOTE_TO_MAIN,
|
||||
SWAP_UUID_FAILED
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -0,0 +1,99 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
|
||||
class ReplicationInstance {
|
||||
public:
|
||||
ReplicationInstance(CoordinatorInstance *peer, CoordinatorClientConfig config, HealthCheckCallback succ_cb,
|
||||
HealthCheckCallback fail_cb);
|
||||
|
||||
ReplicationInstance(ReplicationInstance const &other) = delete;
|
||||
ReplicationInstance &operator=(ReplicationInstance const &other) = delete;
|
||||
ReplicationInstance(ReplicationInstance &&other) noexcept = delete;
|
||||
ReplicationInstance &operator=(ReplicationInstance &&other) noexcept = delete;
|
||||
~ReplicationInstance() = default;
|
||||
|
||||
auto OnSuccessPing() -> void;
|
||||
auto OnFailPing() -> bool;
|
||||
auto IsReadyForUUIDPing() -> bool;
|
||||
|
||||
void UpdateReplicaLastResponseUUID();
|
||||
|
||||
auto IsAlive() const -> bool;
|
||||
|
||||
auto InstanceName() const -> std::string;
|
||||
auto SocketAddress() const -> std::string;
|
||||
|
||||
auto IsReplica() const -> bool;
|
||||
auto IsMain() const -> bool;
|
||||
|
||||
auto PromoteToMain(utils::UUID uuid, ReplicationClientsInfo repl_clients_info, HealthCheckCallback main_succ_cb,
|
||||
HealthCheckCallback main_fail_cb) -> bool;
|
||||
auto DemoteToReplica(HealthCheckCallback replica_succ_cb, HealthCheckCallback replica_fail_cb) -> bool;
|
||||
|
||||
auto StartFrequentCheck() -> void;
|
||||
auto StopFrequentCheck() -> void;
|
||||
auto PauseFrequentCheck() -> void;
|
||||
auto ResumeFrequentCheck() -> void;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto EnsureReplicaHasCorrectMainUUID(utils::UUID const &curr_main_uuid) -> bool;
|
||||
|
||||
auto SendSwapAndUpdateUUID(const utils::UUID &new_main_uuid) -> bool;
|
||||
auto SendUnregisterReplicaRpc(std::string const &instance_name) -> bool;
|
||||
|
||||
|
||||
auto SendGetInstanceUUID() -> utils::BasicResult<coordination::GetInstanceUUIDError, std::optional<utils::UUID>>;
|
||||
auto GetClient() -> CoordinatorClient &;
|
||||
|
||||
auto EnableWritingOnMain() -> bool;
|
||||
|
||||
auto SetNewMainUUID(utils::UUID const &main_uuid) -> void;
|
||||
auto ResetMainUUID() -> void;
|
||||
auto GetMainUUID() const -> const std::optional<utils::UUID> &;
|
||||
|
||||
private:
|
||||
CoordinatorClient client_;
|
||||
replication_coordination_glue::ReplicationRole replication_role_;
|
||||
std::chrono::system_clock::time_point last_response_time_{};
|
||||
bool is_alive_{false};
|
||||
std::chrono::system_clock::time_point last_check_of_uuid_{};
|
||||
|
||||
// for replica this is main uuid of current main
|
||||
// for "main" main this same as in CoordinatorData
|
||||
// it is set to nullopt when replica is down
|
||||
// TLDR; when replica is down and comes back up we reset uuid of main replica is listening to
|
||||
// so we need to send swap uuid again
|
||||
std::optional<utils::UUID> main_uuid_;
|
||||
|
||||
friend bool operator==(ReplicationInstance const &first, ReplicationInstance const &second) {
|
||||
return first.client_ == second.client_ && first.replication_role_ == second.replication_role_;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
14
src/coordination/include/coordination/rpc_errors.hpp
Normal file
14
src/coordination/include/coordination/rpc_errors.hpp
Normal file
@@ -0,0 +1,14 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
namespace memgraph::coordination {
|
||||
enum class GetInstanceUUIDError { NO_RESPONSE, RPC_EXCEPTION };
|
||||
} // namespace memgraph::coordination
|
||||
70
src/coordination/include/nuraft/coordinator_log_store.hpp
Normal file
70
src/coordination/include/nuraft/coordinator_log_store.hpp
Normal file
@@ -0,0 +1,70 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::buffer;
|
||||
using nuraft::int32;
|
||||
using nuraft::int64;
|
||||
using nuraft::log_entry;
|
||||
using nuraft::log_store;
|
||||
using nuraft::ptr;
|
||||
using nuraft::raft_server;
|
||||
|
||||
class CoordinatorLogStore : public log_store {
|
||||
public:
|
||||
CoordinatorLogStore();
|
||||
CoordinatorLogStore(CoordinatorLogStore const &) = delete;
|
||||
CoordinatorLogStore &operator=(CoordinatorLogStore const &) = delete;
|
||||
CoordinatorLogStore(CoordinatorLogStore &&) = delete;
|
||||
CoordinatorLogStore &operator=(CoordinatorLogStore &&) = delete;
|
||||
~CoordinatorLogStore() override;
|
||||
|
||||
ulong next_slot() const override;
|
||||
|
||||
ulong start_index() const override;
|
||||
|
||||
ptr<log_entry> last_entry() const override;
|
||||
|
||||
ulong append(ptr<log_entry> &entry) override;
|
||||
|
||||
void write_at(ulong index, ptr<log_entry> &entry) override;
|
||||
|
||||
ptr<std::vector<ptr<log_entry>>> log_entries(ulong start, ulong end) override;
|
||||
|
||||
ptr<log_entry> entry_at(ulong index) override;
|
||||
|
||||
ulong term_at(ulong index) override;
|
||||
|
||||
ptr<buffer> pack(ulong index, int32 cnt) override;
|
||||
|
||||
void apply_pack(ulong index, buffer &pack) override;
|
||||
|
||||
bool compact(ulong last_log_index) override;
|
||||
|
||||
bool flush() override;
|
||||
|
||||
private:
|
||||
auto FindOrDefault_(ulong index) const -> ptr<log_entry>;
|
||||
|
||||
std::map<ulong, ptr<log_entry>> logs_;
|
||||
mutable std::mutex logs_lock_;
|
||||
std::atomic<ulong> start_idx_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::async_result;
|
||||
using nuraft::buffer;
|
||||
using nuraft::buffer_serializer;
|
||||
using nuraft::cluster_config;
|
||||
using nuraft::int32;
|
||||
using nuraft::ptr;
|
||||
using nuraft::snapshot;
|
||||
using nuraft::state_machine;
|
||||
|
||||
class CoordinatorStateMachine : public state_machine {
|
||||
public:
|
||||
CoordinatorStateMachine() = default;
|
||||
CoordinatorStateMachine(CoordinatorStateMachine const &) = delete;
|
||||
CoordinatorStateMachine &operator=(CoordinatorStateMachine const &) = delete;
|
||||
CoordinatorStateMachine(CoordinatorStateMachine &&) = delete;
|
||||
CoordinatorStateMachine &operator=(CoordinatorStateMachine &&) = delete;
|
||||
~CoordinatorStateMachine() override {}
|
||||
|
||||
static auto EncodeRegisterReplicationInstance(const std::string &name) -> ptr<buffer>;
|
||||
|
||||
static auto DecodeRegisterReplicationInstance(buffer &data) -> std::string;
|
||||
|
||||
auto pre_commit(ulong log_idx, buffer &data) -> ptr<buffer> override;
|
||||
|
||||
auto commit(ulong log_idx, buffer &data) -> ptr<buffer> override;
|
||||
|
||||
auto commit_config(ulong log_idx, ptr<cluster_config> & /*new_conf*/) -> void override;
|
||||
|
||||
auto rollback(ulong log_idx, buffer &data) -> void override;
|
||||
|
||||
auto read_logical_snp_obj(snapshot & /*snapshot*/, void *& /*user_snp_ctx*/, ulong /*obj_id*/, ptr<buffer> &data_out,
|
||||
bool &is_last_obj) -> int override;
|
||||
|
||||
auto save_logical_snp_obj(snapshot &s, ulong &obj_id, buffer & /*data*/, bool /*is_first_obj*/, bool /*is_last_obj*/)
|
||||
-> void override;
|
||||
|
||||
auto apply_snapshot(snapshot &s) -> bool override;
|
||||
|
||||
auto free_user_snp_ctx(void *&user_snp_ctx) -> void override;
|
||||
|
||||
auto last_snapshot() -> ptr<snapshot> override;
|
||||
|
||||
auto last_commit_index() -> ulong override;
|
||||
|
||||
auto create_snapshot(snapshot &s, async_result<bool>::handler_type &when_done) -> void override;
|
||||
|
||||
private:
|
||||
std::atomic<uint64_t> last_committed_idx_{0};
|
||||
|
||||
ptr<snapshot> last_snapshot_;
|
||||
|
||||
std::mutex last_snapshot_lock_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -0,0 +1,66 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "nuraft/coordinator_log_store.hpp"
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using nuraft::cluster_config;
|
||||
using nuraft::cs_new;
|
||||
using nuraft::srv_config;
|
||||
using nuraft::srv_state;
|
||||
using nuraft::state_mgr;
|
||||
|
||||
class CoordinatorStateManager : public state_mgr {
|
||||
public:
|
||||
explicit CoordinatorStateManager(int srv_id, std::string const &endpoint);
|
||||
|
||||
CoordinatorStateManager(CoordinatorStateManager const &) = delete;
|
||||
CoordinatorStateManager &operator=(CoordinatorStateManager const &) = delete;
|
||||
CoordinatorStateManager(CoordinatorStateManager &&) = delete;
|
||||
CoordinatorStateManager &operator=(CoordinatorStateManager &&) = delete;
|
||||
|
||||
~CoordinatorStateManager() override = default;
|
||||
|
||||
auto load_config() -> ptr<cluster_config> override;
|
||||
|
||||
auto save_config(cluster_config const &config) -> void override;
|
||||
|
||||
auto save_state(srv_state const &state) -> void override;
|
||||
|
||||
auto read_state() -> ptr<srv_state> override;
|
||||
|
||||
auto load_log_store() -> ptr<log_store> override;
|
||||
|
||||
auto server_id() -> int32 override;
|
||||
|
||||
auto system_exit(int exit_code) -> void override;
|
||||
|
||||
auto GetSrvConfig() const -> ptr<srv_config>;
|
||||
|
||||
private:
|
||||
int my_id_;
|
||||
std::string my_endpoint_;
|
||||
ptr<CoordinatorLogStore> cur_log_store_;
|
||||
ptr<srv_config> my_srv_config_;
|
||||
ptr<cluster_config> cluster_config_;
|
||||
ptr<srv_state> saved_state_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user