Compare commits
2 Commits
run-packag
...
temporal-m
| Author | SHA1 | Date | |
|---|---|---|---|
|
cfc0cfa121 | ||
|
c899350009 |
@@ -64,8 +64,8 @@ Checks: '*,
|
||||
-readability-identifier-length,
|
||||
-misc-no-recursion,
|
||||
-concurrency-mt-unsafe,
|
||||
-bugprone-easily-swappable-parameters,
|
||||
-bugprone-unchecked-optional-access'
|
||||
-bugprone-easily-swappable-parameters'
|
||||
|
||||
WarningsAsErrors: ''
|
||||
HeaderFilterRegex: 'src/.*'
|
||||
AnalyzeTemporaryDtors: false
|
||||
|
||||
22
.github/pull_request_template.md
vendored
22
.github/pull_request_template.md
vendored
@@ -1,28 +1,14 @@
|
||||
### Description
|
||||
|
||||
Please briefly explain the changes you made here.
|
||||
|
||||
|
||||
Please delete either the [master < EPIC] or [master < Task] part, depending on what are your needs.
|
||||
|
||||
[master < Epic] PR
|
||||
- [ ] Check, and update documentation if necessary
|
||||
- [ ] Write E2E tests
|
||||
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
|
||||
- [ ] Provide the full content or a guide for the final git message
|
||||
- [FINAL GIT MESSAGE]
|
||||
|
||||
[master < Task] PR
|
||||
- [ ] Check, and update documentation if necessary
|
||||
- [ ] Provide the full content or a guide for the final git message
|
||||
- **[FINAL GIT MESSAGE]**
|
||||
|
||||
|
||||
### Documentation checklist
|
||||
- [ ] Add the documentation label tag
|
||||
- [ ] Add the bug / feature label tag
|
||||
- [ ] Add the milestone for which this feature is intended
|
||||
- If not known, set for a later milestone
|
||||
- [ ] Write a release note, including added/changed clauses
|
||||
- **[Release note text]**
|
||||
- [ ] Link the documentation PR here
|
||||
- **[Documentation PR link]**
|
||||
To keep docs changelog up to date, one more thing to do:
|
||||
- [ ] Write a release note here, including added/changed clauses
|
||||
- [ ] Tag someone from docs team in the comments
|
||||
|
||||
600
.github/workflows/diff.yaml
vendored
600
.github/workflows/diff.yaml
vendored
@@ -19,16 +19,11 @@ on:
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, Linux, X64, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
OS: debian-11
|
||||
TOOLCHAIN: v5
|
||||
ARCH: amd
|
||||
BUILD_TYPE: RelWithDebInfo
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
@@ -38,62 +33,35 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Spin up mgbuild container
|
||||
- name: Build community binaries
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
run
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
- name: Build community binary
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph --community
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build community binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMG_ENTERPRISE=OFF ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph unit
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
- name: Stop mgbuild container
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
stop --remove
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
code_analysis:
|
||||
name: "Code analysis"
|
||||
runs-on: [self-hosted, Linux, X64, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
OS: debian-11
|
||||
TOOLCHAIN: v5
|
||||
ARCH: amd
|
||||
BUILD_TYPE: Debug
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
@@ -103,20 +71,6 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Spin up mgbuild container
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
run
|
||||
|
||||
# This is also needed if we want do to comparison against other branches
|
||||
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
|
||||
- name: Fetch all history for all tags and branches
|
||||
@@ -124,13 +78,11 @@ jobs:
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph --init-only
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Set base branch
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
@@ -144,43 +96,45 @@ jobs:
|
||||
|
||||
- name: Python code analysis
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph code-analysis --base-branch "${{ env.BASE_BRANCH }}"
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only --diff-filter=d)
|
||||
for file in ${CHANGED_FILES}; do
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
python3 -m black --check --diff ${file}
|
||||
python3 -m isort --profile black --check-only --diff ${file}
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build combined ASAN, UBSAN and coverage binary
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph --coverage --asan --ubsan
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph unit-coverage
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
|
||||
cd build
|
||||
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
|
||||
|
||||
- name: Compute code coverage
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph code-coverage
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Compute code coverage.
|
||||
cd tools/github
|
||||
./coverage_convert
|
||||
|
||||
# Package code coverage.
|
||||
cd generated
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -190,36 +144,21 @@ jobs:
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph clang-tidy --base-branch "${{ env.BASE_BRANCH }}"
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
- name: Stop mgbuild container
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
stop --remove
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, Linux, X64, DockerMgBuild]
|
||||
timeout-minutes: 100
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
OS: debian-11
|
||||
TOOLCHAIN: v5
|
||||
ARCH: amd
|
||||
BUILD_TYPE: Debug
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
@@ -229,84 +168,44 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Spin up mgbuild container
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
run
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
- name: Build debug binary
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build debug binaries.
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run leftover CTest tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph leftover-CTest
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run leftover CTest tests (all except unit and benchmark tests).
|
||||
cd build
|
||||
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
|
||||
|
||||
- name: Run drivers tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph drivers
|
||||
|
||||
- name: Run HA driver tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph drivers-high-availability
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph integration
|
||||
tests/integration/run.sh
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph cppcheck-and-clang-format
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run cppcheck and clang-format.
|
||||
cd tools/github
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -314,27 +213,13 @@ jobs:
|
||||
name: "Code coverage(Debug build)"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
- name: Stop mgbuild container
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
stop --remove
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, Linux, X64, DockerMgBuild]
|
||||
timeout-minutes: 100
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
OS: debian-11
|
||||
TOOLCHAIN: v5
|
||||
ARCH: amd
|
||||
BUILD_TYPE: Release
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
@@ -344,39 +229,26 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Spin up mgbuild container
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
run
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
- name: Build release binary
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=Release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph gql-behave
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
cd gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v4
|
||||
@@ -388,19 +260,14 @@ jobs:
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--threads $THREADS \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph unit
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
# This step will be skipped because the e2e stream tests have been disabled
|
||||
# We need to fix this as soon as possible
|
||||
- name: Ensure Kafka and Pulsar are up
|
||||
if: false
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose up -d
|
||||
@@ -409,17 +276,13 @@ jobs:
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph e2e
|
||||
cd tests
|
||||
./setup.sh /opt/toolchain-v4/activate
|
||||
source ve3/bin/activate_e2e
|
||||
cd e2e
|
||||
./run.sh
|
||||
|
||||
# Same as two steps prior
|
||||
- name: Ensure Kafka and Pulsar are down
|
||||
if: false
|
||||
run: |
|
||||
cd tests/e2e/streams/kafka
|
||||
docker-compose down
|
||||
@@ -428,92 +291,59 @@ jobs:
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph stress-plain
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
./continuous_integration
|
||||
|
||||
- name: Run stress test (SSL)
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph stress-ssl
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run durability test
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph durability
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 5
|
||||
|
||||
- name: Create enterprise DEB package
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
package-memgraph
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
cd build
|
||||
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
copy --package
|
||||
# create mgconsole
|
||||
# we use the -B to force the build
|
||||
make -j$THREADS -B mgconsole
|
||||
|
||||
# Create enterprise DEB package.
|
||||
mkdir output && cd output
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/${{ env.OS }}/memgraph*.deb
|
||||
|
||||
- name: Copy build logs
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
copy --build-logs
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data(Release build)"
|
||||
path: build/logs
|
||||
|
||||
- name: Stop mgbuild container
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
stop --remove
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, DockerMgBuild]
|
||||
timeout-minutes: 80
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
#continue-on-error: true
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
OS: debian-12
|
||||
TOOLCHAIN: v5
|
||||
ARCH: amd
|
||||
BUILD_TYPE: RelWithDebInfo
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
@@ -523,37 +353,16 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Spin up mgbuild container
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
run
|
||||
|
||||
- name: Build release binary
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph
|
||||
|
||||
- name: Copy memgraph binary
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
copy --binary
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Refresh Jepsen Cluster
|
||||
run: |
|
||||
@@ -567,32 +376,18 @@ jobs:
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
- name: Stop mgbuild container
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
stop --remove
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, DockerMgBuild, Gen7]
|
||||
timeout-minutes: 60
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
OS: debian-11
|
||||
TOOLCHAIN: v5
|
||||
ARCH: amd
|
||||
BUILD_TYPE: Release
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
@@ -602,39 +397,25 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Spin up mgbuild container
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
run
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
- name: Build release binary
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--build-type $BUILD_TYPE \
|
||||
--threads $THREADS \
|
||||
build-memgraph
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph macro-benchmark
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Get branch name (merge)
|
||||
if: github.event_name != 'pull_request'
|
||||
@@ -648,49 +429,30 @@ jobs:
|
||||
|
||||
- name: Upload macro benchmark results
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph upload-to-bench-graph \
|
||||
--benchmark-name "macro_benchmark" \
|
||||
--benchmark-results "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id ${{ github.run_id }} \
|
||||
--github-run-number ${{ github.run_number }} \
|
||||
--head-branch-name ${{ env.BRANCH_NAME }}
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
# TODO (andi) No need for path flags and for --disk-storage and --in-memory-analytical
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph mgbench
|
||||
cd tests/mgbench
|
||||
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
--enterprise-license $MEMGRAPH_ENTERPRISE_LICENSE \
|
||||
--organization-name $MEMGRAPH_ORGANIZATION_NAME \
|
||||
test-memgraph upload-to-bench-graph \
|
||||
--benchmark-name "mgbench" \
|
||||
--benchmark-results "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
- name: Stop mgbuild container
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain $TOOLCHAIN \
|
||||
--os $OS \
|
||||
--arch $ARCH \
|
||||
stop --remove
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
623
.github/workflows/package_memgraph.yaml
vendored
623
.github/workflows/package_memgraph.yaml
vendored
@@ -1,19 +1,24 @@
|
||||
name: Package memgraph
|
||||
|
||||
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
memgraph_version:
|
||||
description: "Memgraph version to upload as. Leave this field empty if you don't want to upload binaries to S3. Format: 'X.Y.Z'"
|
||||
required: false
|
||||
build_type:
|
||||
type: choice
|
||||
description: "Memgraph Build type."
|
||||
description: "Memgraph Build type. Default value is Release"
|
||||
default: 'Release'
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
target_os:
|
||||
type: choice
|
||||
description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS."
|
||||
default: 'debian-12'
|
||||
description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS. Default is Ubuntu 22.04"
|
||||
default: 'ubuntu-22_04'
|
||||
options:
|
||||
- all
|
||||
- amzn-2
|
||||
@@ -23,392 +28,268 @@ on:
|
||||
- debian-11
|
||||
- debian-11-arm
|
||||
- debian-11-platform
|
||||
- debian-12
|
||||
- debian-12-arm
|
||||
- docker
|
||||
- docker-arm
|
||||
- fedora-36
|
||||
- fedora-38
|
||||
- fedora-39
|
||||
- rocky-9.3
|
||||
- ubuntu-18.04
|
||||
- ubuntu-20.04
|
||||
- ubuntu-22.04
|
||||
- ubuntu-22.04-arm
|
||||
toolchain:
|
||||
type: choice
|
||||
description: "Toolchain version with which memgraph will be packaged."
|
||||
default: 'v5'
|
||||
options:
|
||||
- v4
|
||||
- v5
|
||||
push_to_s3:
|
||||
type: boolean
|
||||
description: "Push final packages to an S3 bucket?"
|
||||
default: false
|
||||
s3_bucket:
|
||||
type: choice
|
||||
description: "Name of the target S3 bucket. Select 'download.memgraph.io' for official and 'deps.memgraph.io' for unofficial packages."
|
||||
default: 'download.memgraph.io'
|
||||
options:
|
||||
- download.memgraph.com
|
||||
- deps.memgraph.io
|
||||
s3_region:
|
||||
type: choice
|
||||
description: "Region for the target S3 bucket."
|
||||
options:
|
||||
- eu-west-1
|
||||
s3_dest_dir:
|
||||
type: string
|
||||
description: "Target dir path in S3 bucket. For official packages use format 'memgraph/X.Y.Z/'. For unofficial packages use format 'memgraph-unofficial/<SOMETHING>/'."
|
||||
default: ''
|
||||
- ubuntu-18_04
|
||||
- ubuntu-20_04
|
||||
- ubuntu-22_04
|
||||
- ubuntu-22_04-arm
|
||||
|
||||
jobs:
|
||||
AmazonLinux2:
|
||||
amzn-2:
|
||||
if: ${{ github.event.inputs.target_os == 'amzn-2' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "amzn-2"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: amzn-2
|
||||
path: build/output/amzn-2/memgraph*.rpm
|
||||
|
||||
CentOS7:
|
||||
centos-7:
|
||||
if: ${{ github.event.inputs.target_os == 'centos-7' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "centos-7"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: centos-7
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
CentOS9:
|
||||
centos-9:
|
||||
if: ${{ github.event.inputs.target_os == 'centos-9' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "centos-9"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: centos-9
|
||||
path: build/output/centos-9/memgraph*.rpm
|
||||
|
||||
Debian10:
|
||||
if: ${{ (github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "debian-10"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
debian-10:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-10
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
Debian11:
|
||||
debian-11:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "debian-11"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
Debian11-platform:
|
||||
debian-11-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11-platform' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "debian-11"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
additional_build_args: "--for-platform"
|
||||
artifact_name: "debian-11-platform"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
Debian11-arm:
|
||||
if: ${{ (github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "debian-11-arm"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "arm"
|
||||
runner_arch_label: "ARM64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 120
|
||||
artifact_name: "debian-11-aarch64"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
docker:
|
||||
if: ${{ github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
cd release/package
|
||||
./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker
|
||||
./run.sh docker
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: docker
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
Debian12:
|
||||
if: ${{ (github.event.inputs.target_os == 'debian-12' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "debian-12"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
fedora-36:
|
||||
if: ${{ github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fedora-36
|
||||
path: build/output/fedora-36/memgraph*.rpm
|
||||
|
||||
Debian12-arm:
|
||||
if: ${{ (github.event.inputs.target_os == 'debian-12-arm' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "debian-12-arm"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "arm"
|
||||
runner_arch_label: "ARM64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 120
|
||||
artifact_name: "debian-12-aarch64"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
ubuntu-18_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-18.04
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
Docker-v4:
|
||||
if: ${{ (github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
|
||||
uses: ./.github/workflows/reusable_docker_package.yaml
|
||||
with:
|
||||
os: "debian-11"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
additional_build_args: "--for-docker"
|
||||
artifact_name: "docker"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
ubuntu-20_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-20.04
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
Docker-v5:
|
||||
if: ${{ (github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_docker_package.yaml
|
||||
with:
|
||||
os: "debian-12"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
additional_build_args: "--for-docker"
|
||||
artifact_name: "docker"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
ubuntu-22_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
Docker-v4-arm:
|
||||
if: ${{ (github.event.inputs.target_os == 'docker-arm' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
|
||||
uses: ./.github/workflows/reusable_docker_package.yaml
|
||||
with:
|
||||
os: "debian-11-arm"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "arm"
|
||||
runner_arch_label: "ARM64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 120
|
||||
additional_build_args: "--for-docker"
|
||||
artifact_name: "docker-aarch64"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
ubuntu-22_04-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ubuntu-22.04-aarch64
|
||||
path: build/output/ubuntu-22.04-arm/memgraph*.deb
|
||||
|
||||
Docker-v5-arm:
|
||||
if: ${{ (github.event.inputs.target_os == 'docker-arm' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_docker_package.yaml
|
||||
with:
|
||||
os: "debian-12-arm"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "arm"
|
||||
runner_arch_label: "ARM64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 120
|
||||
additional_build_args: "--for-docker"
|
||||
artifact_name: "docker-aarch64"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Fedora36:
|
||||
if: ${{ (github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "fedora-36"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Fedora38:
|
||||
if: ${{ (github.event.inputs.target_os == 'fedora-38' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "fedora-38"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Fedora39:
|
||||
if: ${{ (github.event.inputs.target_os == 'fedora-39' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "fedora-39"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
RockyLinux9_3:
|
||||
if: ${{ (github.event.inputs.target_os == 'rocky-9.3' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v5' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "rocky-9.3"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Ubuntu18_04:
|
||||
if: ${{ (github.event.inputs.target_os == 'ubuntu-18.04' || github.event.inputs.target_os == 'all') && github.event.inputs.toolchain == 'v4' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "ubuntu-18.04"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Ubuntu20_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-20.04' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "ubuntu-20.04"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Ubuntu22_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22.04' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "ubuntu-22.04"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "amd"
|
||||
runner_arch_label: "X64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 60
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
|
||||
Ubuntu22_04-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22.04-arm' || github.event.inputs.target_os == 'all' }}
|
||||
uses: ./.github/workflows/reusable_package.yaml
|
||||
with:
|
||||
os: "ubuntu-22.04-arm"
|
||||
toolchain: ${{ github.event.inputs.toolchain }}
|
||||
arch: "arm"
|
||||
runner_arch_label: "ARM64"
|
||||
build_type: ${{ github.event.inputs.build_type }}
|
||||
timeout_minutes: 120
|
||||
artifact_name: "ubuntu-22.04-aarch64"
|
||||
push_to_s3: ${{ github.event.inputs.push_to_s3 }}
|
||||
s3_bucket: ${{ github.event.inputs.s3_bucket }}
|
||||
s3_region: ${{ github.event.inputs.s3_region }}
|
||||
s3_dest_dir: ${{ github.event.inputs.s3_dest_dir }}
|
||||
secrets: inherit
|
||||
upload-to-s3:
|
||||
# only run upload if we specified version. Allows for runs without upload
|
||||
if: "${{ github.event.inputs.memgraph_version != '' }}"
|
||||
needs: [amzn-2, centos-7, centos-9, debian-10, debian-11, debian-11-arm, debian-11-platform, docker, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04, ubuntu-22_04-arm]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
# name: # if name input parameter is not provided, all artifacts are downloaded
|
||||
# and put in directories named after each one.
|
||||
path: build/output/release
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "download.memgraph.com"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output/release"
|
||||
DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/"
|
||||
|
||||
161
.github/workflows/reusable_docker_package.yaml
vendored
161
.github/workflows/reusable_docker_package.yaml
vendored
@@ -1,161 +0,0 @@
|
||||
name: Reusable docker package make and upload
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
os:
|
||||
type: string
|
||||
description: "Target os. Default value is debian-12."
|
||||
default: 'debian-12'
|
||||
arch:
|
||||
type: string
|
||||
description: "Target architecture. Default value is amd."
|
||||
default: 'amd'
|
||||
runner_arch_label:
|
||||
type: string
|
||||
description: "Runner architecture label. Amd is built on X64, arm on ARM64. Default value is X64."
|
||||
default: 'X64'
|
||||
toolchain:
|
||||
type: string
|
||||
description: "Toolchain version. Default value is v5."
|
||||
default: 'v5'
|
||||
build_type:
|
||||
type: string
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
artifact_name:
|
||||
type: string
|
||||
description: "Unique artifact name, pass it in to override the default. Default value is docker."
|
||||
default: ''
|
||||
additional_build_args:
|
||||
type: string
|
||||
description: "Additional build flags (--for-platform, --for-docker ...). Default value is empty."
|
||||
default: ''
|
||||
timeout_minutes:
|
||||
type: number
|
||||
description: "Timeout in minutes for the job. Default value is 60."
|
||||
default: 60
|
||||
push_to_s3:
|
||||
type: string
|
||||
description: "Should the final package be pushed to an S3 bucket. Default value is false"
|
||||
default: false
|
||||
s3_bucket:
|
||||
type: string
|
||||
description: "Name of the target S3 bucket."
|
||||
default: ''
|
||||
s3_region:
|
||||
type: string
|
||||
description: "Region for the target S3 bucket."
|
||||
default: 'eu-west-1'
|
||||
s3_dest_dir:
|
||||
type: string
|
||||
description: "Target dir path in S3 bucket."
|
||||
default: ''
|
||||
|
||||
jobs:
|
||||
package:
|
||||
runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
|
||||
timeout-minutes: ${{ inputs.timeout_minutes }}
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos
|
||||
if: ${{ inputs.arch == 'arm' }}
|
||||
run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db
|
||||
|
||||
- name: "Spin up mgbuild container"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
run
|
||||
|
||||
- name: "Build Memgraph binaries"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
--build-type ${{ inputs.build_type }} \
|
||||
build-memgraph ${{ inputs.additional_build_args }}
|
||||
|
||||
- name: "Make package"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
--build-type ${{ inputs.build_type }} \
|
||||
package-memgraph
|
||||
|
||||
- name: "Copy package"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
copy --package
|
||||
|
||||
- name: "Create docker package"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
package-docker
|
||||
|
||||
- name: "Upload docker package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ inputs.artifact_name }}
|
||||
path: "build/output/docker/${{ inputs.arch }}/${{ inputs.toolchain }}/memgraph*"
|
||||
|
||||
- name: "Stop mgbuild container"
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
stop --remove
|
||||
|
||||
upload-to-s3:
|
||||
if: ${{ inputs.push_to_s3 == 'true' }}
|
||||
needs: [package]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Validate dest_dir
|
||||
if: ${{ inputs.s3_bucket == 'download.memgraph.com' }}
|
||||
run: |
|
||||
if ! [[ "${{ inputs.s3_dest_dir }}" =~ memgraph/[0-9]\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo -e "Error: ${{ inputs.s3_dest_dir }} is not a valid official format. When pushing to 'download.memgraph.com' the dest dir has to match format 'memgraph/X.Y.Z'!"
|
||||
exit 1
|
||||
else
|
||||
echo -e "Passed: Dest dir ${{ inputs.s3_dest_dir }} has a valid official format."
|
||||
fi
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ inputs.artifact_name }}
|
||||
path: "build/output/release"
|
||||
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: ${{ inputs.s3_bucket }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: ${{ inputs.s3_region }}
|
||||
SOURCE_DIR: "build/output/release"
|
||||
DEST_DIR: "${{ inputs.s3_dest_dir }}/${{ inputs.artifact_name }}/"
|
||||
156
.github/workflows/reusable_package.yaml
vendored
156
.github/workflows/reusable_package.yaml
vendored
@@ -1,156 +0,0 @@
|
||||
name: Reusable package make and upload
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
os:
|
||||
type: string
|
||||
description: "Target os. Default value is debian-12."
|
||||
default: 'debian-12'
|
||||
arch:
|
||||
type: string
|
||||
description: "Target architecture. Default value is amd."
|
||||
default: 'amd'
|
||||
runner_arch_label:
|
||||
type: string
|
||||
description: "Runner architecture label. Amd is built on X64, arm on ARM64. Default value is X64."
|
||||
default: 'X64'
|
||||
toolchain:
|
||||
type: string
|
||||
description: "Toolchain version. Default value is v5."
|
||||
default: 'v5'
|
||||
build_type:
|
||||
type: string
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
default: 'Release'
|
||||
artifact_name:
|
||||
type: string
|
||||
description: "Unique artifact name, pass it in to override the default. Default value is os name."
|
||||
default: ''
|
||||
additional_build_args:
|
||||
type: string
|
||||
description: "Additional build flags (--for-platform, --for-docker ...). Default value is empty."
|
||||
default: ''
|
||||
timeout_minutes:
|
||||
type: number
|
||||
description: "Timeout in minutes for the job. Default value is 60."
|
||||
default: 60
|
||||
push_to_s3:
|
||||
type: string
|
||||
description: "Should the final package be pushed to an S3 bucket. Default value is false"
|
||||
default: false
|
||||
s3_bucket:
|
||||
type: string
|
||||
description: "Name of the target S3 bucket."
|
||||
default: ''
|
||||
s3_region:
|
||||
type: string
|
||||
description: "Region for the target S3 bucket."
|
||||
default: 'eu-west-1'
|
||||
s3_dest_dir:
|
||||
type: string
|
||||
description: "Target dir path in S3 bucket."
|
||||
default: ''
|
||||
|
||||
env:
|
||||
ARTIFACT_NAME: ${{ inputs.artifact_name || inputs.os }}
|
||||
|
||||
jobs:
|
||||
package:
|
||||
runs-on: [self-hosted, DockerMgBuild, "${{ inputs.runner_arch_label }}"]
|
||||
timeout-minutes: ${{ inputs.timeout_minutes }}
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: "Unlock keychain for current session" # Required beacuse ARM builds run on macos
|
||||
if: ${{ inputs.arch == 'arm' }}
|
||||
run: security -v unlock-keychain -p ${{ secrets.SELF_HOSTED_RUNNER_PASSWORD }} ~/Library/Keychains/login.keychain-db
|
||||
|
||||
- name: "Spin up mgbuild container"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
run
|
||||
|
||||
- name: "Build Memgraph binaries"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
--build-type ${{ inputs.build_type }} \
|
||||
build-memgraph ${{ inputs.additional_build_args }}
|
||||
|
||||
- name: "Make package"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
--build-type ${{ inputs.build_type }} \
|
||||
package-memgraph
|
||||
|
||||
- name: "Copy package"
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
copy --package
|
||||
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ env.ARTIFACT_NAME }}
|
||||
path: "build/output/${{ inputs.os }}/memgraph*"
|
||||
|
||||
- name: "Stop mgbuild container"
|
||||
if: always()
|
||||
run: |
|
||||
./release/package/mgbuild.sh \
|
||||
--toolchain ${{ inputs.toolchain }} \
|
||||
--os ${{ inputs.os }} \
|
||||
--arch ${{ inputs.arch }} \
|
||||
stop --remove
|
||||
|
||||
upload-to-s3:
|
||||
if: ${{ inputs.push_to_s3 == 'true' }}
|
||||
needs: [package]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Validate dest_dir
|
||||
if: ${{ inputs.s3_bucket == 'download.memgraph.com' }}
|
||||
run: |
|
||||
if ! [[ "${{ inputs.s3_dest_dir }}" =~ memgraph/[0-9]\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo -e "Error: ${{ inputs.s3_dest_dir }} is not a valid official format. When pushing to 'download.memgraph.com' the dest dir has to match format 'memgraph/X.Y.Z'!"
|
||||
exit 1
|
||||
else
|
||||
echo -e "Passed: Dest dir ${{ inputs.s3_dest_dir }} has a valid official format."
|
||||
fi
|
||||
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ env.ARTIFACT_NAME }}
|
||||
path: "build/output/release"
|
||||
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: ${{ inputs.s3_bucket }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: ${{ inputs.s3_region }}
|
||||
SOURCE_DIR: "build/output/release"
|
||||
DEST_DIR: "${{ inputs.s3_dest_dir }}/${{ env.ARTIFACT_NAME }}/"
|
||||
@@ -300,19 +300,6 @@ endif()
|
||||
|
||||
option(ENABLE_JEMALLOC "Use jemalloc" ON)
|
||||
|
||||
option(MG_MEMORY_PROFILE "If build should be setup for memory profiling" OFF)
|
||||
if (MG_MEMORY_PROFILE AND ENABLE_JEMALLOC)
|
||||
message(STATUS "Jemalloc has been disabled because MG_MEMORY_PROFILE is enabled")
|
||||
set(ENABLE_JEMALLOC OFF)
|
||||
endif ()
|
||||
if (MG_MEMORY_PROFILE AND ASAN)
|
||||
message(STATUS "ASAN has been disabled because MG_MEMORY_PROFILE is enabled")
|
||||
set(ASAN OFF)
|
||||
endif ()
|
||||
if (MG_MEMORY_PROFILE)
|
||||
add_compile_definitions(MG_MEMORY_PROFILE)
|
||||
endif ()
|
||||
|
||||
if (ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
set(ENABLE_JEMALLOC OFF)
|
||||
@@ -337,8 +324,6 @@ if (ASAN)
|
||||
endif()
|
||||
|
||||
if (TSAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
set(ENABLE_JEMALLOC OFF)
|
||||
# ThreadSanitizer generally requires all code to be compiled with -fsanitize=thread.
|
||||
# If some code (e.g. dynamic libraries) is not compiled with the flag, it can
|
||||
# lead to false positive race reports, false negative race reports and/or
|
||||
@@ -354,7 +339,7 @@ if (TSAN)
|
||||
# By default ThreadSanitizer uses addr2line utility to symbolize reports.
|
||||
# llvm-symbolizer is faster, consumes less memory and produces much better
|
||||
# reports. To use it set runtime flag:
|
||||
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
|
||||
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
|
||||
# For more runtime flags see: https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
|
||||
endif()
|
||||
|
||||
|
||||
@@ -46,14 +46,14 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat-traditional # tests are using nc to wait for memgraph
|
||||
netcat # tests are using nc to wait for memgraph
|
||||
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless custom-maven3.9.3 # for driver tests
|
||||
dotnet-sdk-8.0 golang custom-golang1.18.9 nodejs npm
|
||||
dotnet-sdk-7.0 golang custom-golang1.18.9 nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
libsasl2-dev
|
||||
@@ -86,12 +86,6 @@ check() {
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-8.0 ]; then
|
||||
# if ! /root/.dotnet/dotnet --list-sdks | grep -q "8\.0."; then
|
||||
# missing="$pkg $missing"
|
||||
# fi
|
||||
continue
|
||||
fi
|
||||
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
@@ -123,11 +117,13 @@ install() {
|
||||
install_custom_golang "1.18.9"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-8.0 ]; then
|
||||
wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh
|
||||
chmod +x ./dotnet-install.sh
|
||||
./dotnet-install.sh --channel 8.0
|
||||
rm dotnet-install.sh
|
||||
if [ "$pkg" == dotnet-sdk-7.0 ]; then
|
||||
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https dotnet-sdk-7.0
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
apt install -y "$pkg"
|
||||
|
||||
@@ -53,7 +53,7 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless custom-maven3.9.3 # for driver tests
|
||||
dotnet-sdk-8.0 golang custom-golang1.18.9 nodejs npm
|
||||
dotnet-sdk-7.0 golang custom-golang1.18.9 nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
libsasl2-dev
|
||||
@@ -119,12 +119,12 @@ install() {
|
||||
install_custom_golang "1.18.9"
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-8.0 ]; then
|
||||
if [ "$pkg" == dotnet-sdk-7.0 ]; then
|
||||
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https dotnet-sdk-8.0
|
||||
apt-get install -y apt-transport-https dotnet-sdk-7.0
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -45,7 +45,7 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel openssl
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
|
||||
libcurl-devel # mg-requests
|
||||
|
||||
@@ -45,7 +45,7 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel openssl
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
|
||||
libcurl-devel # mg-requests
|
||||
|
||||
@@ -59,7 +59,7 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
doxygen graphviz # source documentation generators
|
||||
which nodejs golang custom-golang1.18.9 # for driver tests
|
||||
zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.3 # for driver tests
|
||||
cl-asdf common-lisp-controller sbcl # for custom Lisp C++ preprocessing
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
cyrus-sasl-devel
|
||||
@@ -162,30 +162,6 @@ install() {
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == doxygen ]; then
|
||||
if ! dnf list installed doxygen >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://dl.rockylinux.org/pub/rocky/9/CRB/x86_64/os/Packages/d/doxygen-1.9.1-11.el9.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == cl-asdf ]; then
|
||||
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.sysadmins.ws/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == common-lisp-controller ]; then
|
||||
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.sysadmins.ws/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.sysadmins.ws/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == PyYAML ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user PyYAML
|
||||
|
||||
@@ -5,21 +5,17 @@ IFS=' '
|
||||
# NOTE: docker_image_name could be local image build based on release/package images.
|
||||
# NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space.
|
||||
# "docker_container_type script_name docker_image_name"
|
||||
# docker_container_type OPTIONS:
|
||||
# * mgrun -> running plain/empty operating system for the purposes of testing native memgraph package
|
||||
# * mgbuild -> running the builder container to build memgraph inside it -> it's possible create builder images using release/package/run.sh
|
||||
OPERATING_SYSTEMS=(
|
||||
# "mgrun amzn-2 amazonlinux:2"
|
||||
# "mgrun centos-7 centos:7"
|
||||
# "mgrun centos-9 dokken/centos-stream-9"
|
||||
# "mgrun debian-10 debian:10"
|
||||
# "mgrun debian-11 debian:11"
|
||||
# "mgrun fedora-36 fedora:36"
|
||||
# "mgrun fedora-39 fedora:39"
|
||||
# "mgrun ubuntu-18.04 ubuntu:18.04"
|
||||
# "mgrun ubuntu-20.04 ubuntu:20.04"
|
||||
# "mgrun ubuntu-22.04 ubuntu:22.04"
|
||||
# "mgbuild debian-12 memgraph/memgraph-builder:v5_debian-12"
|
||||
"mgrun amzn-2 amazonlinux:2"
|
||||
"mgrun centos-7 centos:7"
|
||||
"mgrun centos-9 dokken/centos-stream-9"
|
||||
"mgrun debian-10 debian:10"
|
||||
"mgrun debian-11 debian:11"
|
||||
"mgrun fedora-36 fedora:36"
|
||||
"mgrun ubuntu-18.04 ubuntu:18.04"
|
||||
"mgrun ubuntu-20.04 ubuntu:20.04"
|
||||
"mgrun ubuntu-22.04 ubuntu:22.04"
|
||||
# "mgbuild centos-7 package-mgbuild_centos-7"
|
||||
)
|
||||
|
||||
if [ ! "$(docker info)" ]; then
|
||||
@@ -37,24 +33,14 @@ print_help () {
|
||||
# NOTE: This is an idempotent operation!
|
||||
# TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function.
|
||||
docker_run () {
|
||||
cnt_type="$1"
|
||||
if [[ "$cnt_type" != "mgbuild" && "$cnt_type" != "mgrun" ]]; then
|
||||
echo "ERROR: Wrong docker_container_type -> valid options are mgbuild, mgrun"
|
||||
exit 1
|
||||
fi
|
||||
cnt_name="$2"
|
||||
cnt_image="$3"
|
||||
cnt_name="$1"
|
||||
cnt_image="$2"
|
||||
if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then
|
||||
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
|
||||
echo "Cleanup of the old exited container..."
|
||||
docker rm $cnt_name
|
||||
fi
|
||||
if [[ "$cnt_type" == "mgbuild" ]]; then
|
||||
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image"
|
||||
fi
|
||||
if [[ "$cnt_type" == "mgrun" ]]; then
|
||||
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
|
||||
fi
|
||||
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
|
||||
fi
|
||||
echo "The $cnt_image container is active under $cnt_name name!"
|
||||
}
|
||||
@@ -69,9 +55,9 @@ docker_stop_and_rm () {
|
||||
cnt_name="$1"
|
||||
if [ "$(docker ps -q -f name=$cnt_name)" ]; then
|
||||
docker stop "$1"
|
||||
fi
|
||||
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
|
||||
docker rm "$1"
|
||||
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
|
||||
docker rm "$1"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -85,7 +71,7 @@ start_all () {
|
||||
docker_name="${docker_container_type}_$script_name"
|
||||
echo ""
|
||||
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
|
||||
docker_run "$docker_container_type" "$docker_name" "$docker_image"
|
||||
docker_run "$docker_name" "$docker_image"
|
||||
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS"
|
||||
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
|
||||
echo ""
|
||||
|
||||
@@ -20,18 +20,14 @@ if [ ! -f "$INPUT" ]; then
|
||||
fi
|
||||
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create constraints manually if needed${COLOR_NULL}"
|
||||
|
||||
echo 'CREATE INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' > "$OUTPUT"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create consraints manually if needed${COLOR_NULL}"
|
||||
|
||||
sed -e 's/^:begin/BEGIN/g; s/^BEGIN$/BEGIN;/g;' \
|
||||
-e 's/^:commit/COMMIT/g; s/^COMMIT$/COMMIT;/g;' \
|
||||
-e '/^CALL/d; /^SCHEMA AWAIT/d;' \
|
||||
-e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
|
||||
-e 's/) ON (n./(/g;' \
|
||||
-e '/^CREATE CONSTRAINT/d; /^DROP CONSTRAINT/d;' "$INPUT" >> "$OUTPUT"
|
||||
|
||||
echo 'DROP INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' >> "$OUTPUT"
|
||||
-e '/^CREATE CONSTRAINT/d; /^DROP CONSTRAINT/d;' "$INPUT" > "$OUTPUT"
|
||||
|
||||
echo ""
|
||||
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher file under $OUTPUT"
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
COLOR_ORANGE="\e[38;5;208m"
|
||||
COLOR_GREEN="\e[38;5;35m"
|
||||
COLOR_RED="\e[0;31m"
|
||||
COLOR_NULL="\e[0m"
|
||||
|
||||
print_help() {
|
||||
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 input_file_schema_path input_file_nodes_path input_file_relationships_path input_file_cleanup_path output_file_path"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$#" -ne 5 ]; then
|
||||
print_help
|
||||
fi
|
||||
INPUT_SCHEMA="$1"
|
||||
INPUT_NODES="$2"
|
||||
INPUT_RELATIONSHIPS="$3"
|
||||
INPUT_CLEANUP="$4"
|
||||
OUTPUT="$5"
|
||||
|
||||
if [ ! -f "$INPUT_SCHEMA" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_NODES" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_RELATIONSHIPS" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_CLEANUP" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create constraints manually if needed${COLOR_NULL}"
|
||||
|
||||
|
||||
echo 'CREATE INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' > "$OUTPUT"
|
||||
|
||||
sed -e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
|
||||
-e 's/) ON (n./(/g;' \
|
||||
-e '/^CREATE CONSTRAINT/d' $INPUT_SCHEMA >> "$OUTPUT"
|
||||
|
||||
cat "$INPUT_NODES" >> "$OUTPUT"
|
||||
cat "$INPUT_RELATIONSHIPS" >> "$OUTPUT"
|
||||
|
||||
sed -e '/^DROP CONSTRAINT/d' "$INPUT_CLEANUP" >> "$OUTPUT"
|
||||
|
||||
echo 'DROP INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' >> "$OUTPUT"
|
||||
|
||||
echo ""
|
||||
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher file under $OUTPUT"
|
||||
echo ""
|
||||
echo "Please import data by executing => \`cat $OUTPUT | mgconsole\`"
|
||||
@@ -1,64 +0,0 @@
|
||||
#!/bin/bash -e
|
||||
COLOR_ORANGE="\e[38;5;208m"
|
||||
COLOR_GREEN="\e[38;5;35m"
|
||||
COLOR_RED="\e[0;31m"
|
||||
COLOR_NULL="\e[0m"
|
||||
|
||||
print_help() {
|
||||
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 input_file_schema_path input_file_nodes_path input_file_relationships_path input_file_cleanup_path output_file_schema_path output_file_nodes_path output_file_relationships_path output_file_cleanup_path"
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "$#" -ne 8 ]; then
|
||||
print_help
|
||||
fi
|
||||
INPUT_SCHEMA="$1"
|
||||
INPUT_NODES="$2"
|
||||
INPUT_RELATIONSHIPS="$3"
|
||||
INPUT_CLEANUP="$4"
|
||||
OUTPUT_SCHEMA="$5"
|
||||
OUTPUT_NODES="$6"
|
||||
OUTPUT_RELATIONSHIPS="$7"
|
||||
OUTPUT_CLEANUP="$8"
|
||||
|
||||
if [ ! -f "$INPUT_SCHEMA" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_NODES" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_RELATIONSHIPS" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
if [ ! -f "$INPUT_CLEANUP" ]; then
|
||||
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
|
||||
print_help
|
||||
fi
|
||||
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
|
||||
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create constraints manually if needed${COLOR_NULL}"
|
||||
|
||||
|
||||
echo 'CREATE INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' > "$OUTPUT_SCHEMA"
|
||||
|
||||
sed -e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
|
||||
-e 's/) ON (n./(/g;' \
|
||||
-e '/^CREATE CONSTRAINT/d' $INPUT_SCHEMA >> "$OUTPUT_SCHEMA"
|
||||
|
||||
cat "$INPUT_NODES" > "$OUTPUT_NODES"
|
||||
cat "$INPUT_RELATIONSHIPS" > "$OUTPUT_RELATIONSHIPS"
|
||||
|
||||
sed -e '/^DROP CONSTRAINT/d' "$INPUT_CLEANUP" >> "$OUTPUT_CLEANUP"
|
||||
|
||||
echo 'DROP INDEX ON :`UNIQUE IMPORT LABEL`(`UNIQUE IMPORT ID`);' >> "$OUTPUT_CLEANUP"
|
||||
|
||||
echo ""
|
||||
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher files under $OUTPUT_SCHEMA, $OUTPUT_NODES, $OUTPUT_RELATIONSHIPS and $OUTPUT_CLEANUP"
|
||||
echo ""
|
||||
echo "Please import data by executing => \`cat $OUTPUT_SCHEMA | mgconsole\`, \`cat $OUTPUT_NODES | mgconsole\`, \`cat $OUTPUT_RELATIONSHIPS | mgconsole\` and \`cat $OUTPUT_CLEANUP | mgconsole\`"
|
||||
@@ -326,21 +326,6 @@ inline mgp_vertex *graph_get_vertex_by_id(mgp_graph *g, mgp_vertex_id id, mgp_me
|
||||
return MgInvoke<mgp_vertex *>(mgp_graph_get_vertex_by_id, g, id, memory);
|
||||
}
|
||||
|
||||
inline bool graph_has_text_index(mgp_graph *graph, const char *index_name) {
|
||||
return MgInvoke<int>(mgp_graph_has_text_index, graph, index_name);
|
||||
}
|
||||
|
||||
inline mgp_map *graph_search_text_index(mgp_graph *graph, const char *index_name, const char *search_query,
|
||||
text_search_mode search_mode, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_map *>(mgp_graph_search_text_index, graph, index_name, search_query, search_mode, memory);
|
||||
}
|
||||
|
||||
inline mgp_map *graph_aggregate_over_text_index(mgp_graph *graph, const char *index_name, const char *search_query,
|
||||
const char *aggregation_query, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_map *>(mgp_graph_aggregate_over_text_index, graph, index_name, search_query, aggregation_query,
|
||||
memory);
|
||||
}
|
||||
|
||||
inline mgp_vertices_iterator *graph_iter_vertices(mgp_graph *g, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_vertices_iterator *>(mgp_graph_iter_vertices, g, memory);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -891,36 +891,6 @@ enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *m
|
||||
enum mgp_error mgp_graph_get_vertex_by_id(struct mgp_graph *g, struct mgp_vertex_id id, struct mgp_memory *memory,
|
||||
struct mgp_vertex **result);
|
||||
|
||||
/// Result is non-zero if the index with the given name exists.
|
||||
/// The current implementation always returns without errors.
|
||||
enum mgp_error mgp_graph_has_text_index(struct mgp_graph *graph, const char *index_name, int *result);
|
||||
|
||||
/// Available modes of searching text indices.
|
||||
MGP_ENUM_CLASS text_search_mode{
|
||||
SPECIFIED_PROPERTIES,
|
||||
REGEX,
|
||||
ALL_PROPERTIES,
|
||||
};
|
||||
|
||||
/// Search the named text index for the given query. The result is a map with the "search_results" and "error_msg" keys.
|
||||
/// The "search_results" key contains the vertices whose text-indexed properties match the given query.
|
||||
/// In case of a Tantivy error, the "search_results" key is absent, and "error_msg" contains the error message.
|
||||
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if there’s an allocation error while constructing the results map.
|
||||
/// Return mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS if the same key is being created in the results map more than once.
|
||||
enum mgp_error mgp_graph_search_text_index(struct mgp_graph *graph, const char *index_name, const char *search_query,
|
||||
enum text_search_mode search_mode, struct mgp_memory *memory,
|
||||
struct mgp_map **result);
|
||||
|
||||
/// Aggregate over the results of a search over the named text index. The result is a map with the "aggregation_results"
|
||||
/// and "error_msg" keys.
|
||||
/// The "aggregation_results" key contains the vertices whose text-indexed properties match the given query.
|
||||
/// In case of a Tantivy error, the "aggregation_results" key is absent, and "error_msg" contains the error message.
|
||||
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if there’s an allocation error while constructing the results map.
|
||||
/// Return mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS if the same key is being created in the results map more than once.
|
||||
enum mgp_error mgp_graph_aggregate_over_text_index(struct mgp_graph *graph, const char *index_name,
|
||||
const char *search_query, const char *aggregation_query,
|
||||
struct mgp_memory *memory, struct mgp_map **result);
|
||||
|
||||
/// Creates label index for given label.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if label index already exists, result will be 0, otherwise 1.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -32,15 +32,6 @@
|
||||
|
||||
namespace mgp {
|
||||
|
||||
class TextSearchException : public std::exception {
|
||||
public:
|
||||
explicit TextSearchException(std::string message) : message_(std::move(message)) {}
|
||||
const char *what() const noexcept override { return message_.c_str(); }
|
||||
|
||||
private:
|
||||
std::string message_;
|
||||
};
|
||||
|
||||
class IndexException : public std::exception {
|
||||
public:
|
||||
explicit IndexException(std::string message) : message_(std::move(message)) {}
|
||||
@@ -4315,12 +4306,12 @@ inline void AddParamsReturnsToProc(mgp_proc *proc, std::vector<Parameter> ¶m
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline bool CreateLabelIndex(mgp_graph *memgraph_graph, const std::string_view label) {
|
||||
return create_label_index(memgraph_graph, label.data());
|
||||
inline bool CreateLabelIndex(mgp_graph *memgaph_graph, const std::string_view label) {
|
||||
return create_label_index(memgaph_graph, label.data());
|
||||
}
|
||||
|
||||
inline bool DropLabelIndex(mgp_graph *memgraph_graph, const std::string_view label) {
|
||||
return drop_label_index(memgraph_graph, label.data());
|
||||
inline bool DropLabelIndex(mgp_graph *memgaph_graph, const std::string_view label) {
|
||||
return drop_label_index(memgaph_graph, label.data());
|
||||
}
|
||||
|
||||
inline List ListAllLabelIndices(mgp_graph *memgraph_graph) {
|
||||
@@ -4331,14 +4322,14 @@ inline List ListAllLabelIndices(mgp_graph *memgraph_graph) {
|
||||
return List(label_indices);
|
||||
}
|
||||
|
||||
inline bool CreateLabelPropertyIndex(mgp_graph *memgraph_graph, const std::string_view label,
|
||||
inline bool CreateLabelPropertyIndex(mgp_graph *memgaph_graph, const std::string_view label,
|
||||
const std::string_view property) {
|
||||
return create_label_property_index(memgraph_graph, label.data(), property.data());
|
||||
return create_label_property_index(memgaph_graph, label.data(), property.data());
|
||||
}
|
||||
|
||||
inline bool DropLabelPropertyIndex(mgp_graph *memgraph_graph, const std::string_view label,
|
||||
inline bool DropLabelPropertyIndex(mgp_graph *memgaph_graph, const std::string_view label,
|
||||
const std::string_view property) {
|
||||
return drop_label_property_index(memgraph_graph, label.data(), property.data());
|
||||
return drop_label_property_index(memgaph_graph, label.data(), property.data());
|
||||
}
|
||||
|
||||
inline List ListAllLabelPropertyIndices(mgp_graph *memgraph_graph) {
|
||||
@@ -4349,58 +4340,6 @@ inline List ListAllLabelPropertyIndices(mgp_graph *memgraph_graph) {
|
||||
return List(label_property_indices);
|
||||
}
|
||||
|
||||
namespace {
|
||||
constexpr std::string_view kErrorMsgKey = "error_msg";
|
||||
constexpr std::string_view kSearchResultsKey = "search_results";
|
||||
constexpr std::string_view kAggregationResultsKey = "aggregation_results";
|
||||
} // namespace
|
||||
|
||||
inline List SearchTextIndex(mgp_graph *memgraph_graph, std::string_view index_name, std::string_view search_query,
|
||||
text_search_mode search_mode) {
|
||||
auto results_or_error = Map(mgp::MemHandlerCallback(graph_search_text_index, memgraph_graph, index_name.data(),
|
||||
search_query.data(), search_mode));
|
||||
if (results_or_error.KeyExists(kErrorMsgKey)) {
|
||||
if (!results_or_error.At(kErrorMsgKey).IsString()) {
|
||||
throw TextSearchException{"The error message is not a string!"};
|
||||
}
|
||||
throw TextSearchException(results_or_error.At(kErrorMsgKey).ValueString().data());
|
||||
}
|
||||
|
||||
if (!results_or_error.KeyExists(kSearchResultsKey)) {
|
||||
throw TextSearchException{"Incomplete text index search results!"};
|
||||
}
|
||||
|
||||
if (!results_or_error.At(kSearchResultsKey).IsList()) {
|
||||
throw TextSearchException{"Text index search results have wrong type!"};
|
||||
}
|
||||
|
||||
return results_or_error.At(kSearchResultsKey).ValueList();
|
||||
}
|
||||
|
||||
inline std::string_view AggregateOverTextIndex(mgp_graph *memgraph_graph, std::string_view index_name,
|
||||
std::string_view search_query, std::string_view aggregation_query) {
|
||||
auto results_or_error =
|
||||
Map(mgp::MemHandlerCallback(graph_aggregate_over_text_index, memgraph_graph, index_name.data(),
|
||||
search_query.data(), aggregation_query.data()));
|
||||
|
||||
if (results_or_error.KeyExists(kErrorMsgKey)) {
|
||||
if (!results_or_error.At(kErrorMsgKey).IsString()) {
|
||||
throw TextSearchException{"The error message is not a string!"};
|
||||
}
|
||||
throw TextSearchException(results_or_error.At(kErrorMsgKey).ValueString().data());
|
||||
}
|
||||
|
||||
if (!results_or_error.KeyExists(kAggregationResultsKey)) {
|
||||
throw TextSearchException{"Incomplete text index aggregation results!"};
|
||||
}
|
||||
|
||||
if (!results_or_error.At(kAggregationResultsKey).IsString()) {
|
||||
throw TextSearchException{"Text index aggregation results have wrong type!"};
|
||||
}
|
||||
|
||||
return results_or_error.At(kAggregationResultsKey).ValueString();
|
||||
}
|
||||
|
||||
inline bool CreateExistenceConstraint(mgp_graph *memgraph_graph, const std::string_view label,
|
||||
const std::string_view property) {
|
||||
return create_existence_constraint(memgraph_graph, label.data(), property.data());
|
||||
|
||||
41
init
41
init
@@ -14,7 +14,6 @@ function print_help () {
|
||||
echo "Optional arguments:"
|
||||
echo -e " -h\tdisplay this help and exit"
|
||||
echo -e " --without-libs-setup\tskip the step for setting up libs"
|
||||
echo -e " --ci\tscript is being run inside ci"
|
||||
}
|
||||
|
||||
function setup_virtualenv () {
|
||||
@@ -36,7 +35,6 @@ function setup_virtualenv () {
|
||||
}
|
||||
|
||||
setup_libs=true
|
||||
ci=false
|
||||
if [[ $# -eq 1 && "$1" == "-h" ]]; then
|
||||
print_help
|
||||
exit 0
|
||||
@@ -47,10 +45,6 @@ else
|
||||
shift
|
||||
setup_libs=false
|
||||
;;
|
||||
--ci)
|
||||
shift
|
||||
ci=true
|
||||
;;
|
||||
*)
|
||||
# unknown option
|
||||
echo "Invalid argument provided: $1"
|
||||
@@ -82,20 +76,11 @@ if [[ "$setup_libs" == "true" ]]; then
|
||||
fi
|
||||
|
||||
# Fix for centos 7 during release
|
||||
if [[ "$ci" == "false" ]]; then
|
||||
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then
|
||||
if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then
|
||||
python3 -m pip uninstall -y virtualenv
|
||||
fi
|
||||
python3 -m pip install virtualenv
|
||||
fi
|
||||
else
|
||||
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "amzn-2" ]; then
|
||||
if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then
|
||||
python3 -m pip uninstall -y virtualenv
|
||||
fi
|
||||
python3 -m pip install --user virtualenv
|
||||
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then
|
||||
if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then
|
||||
python3 -m pip uninstall -y virtualenv
|
||||
fi
|
||||
python3 -m pip install virtualenv
|
||||
fi
|
||||
|
||||
# setup gql_behave dependencies
|
||||
@@ -134,16 +119,14 @@ fi
|
||||
# Install precommit hook except on old operating systems because we don't
|
||||
# develop on them -> pre-commit hook not required -> we can use latest
|
||||
# packages.
|
||||
if [[ "$ci" == "false" ]]; then
|
||||
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ] && [ "${DISTRO}" != "amzn-2" ]; then
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pre_commit install
|
||||
# Install py format tools for usage during the development.
|
||||
echo "Install black formatter"
|
||||
python3 -m pip install black==23.1.*
|
||||
echo "Install isort"
|
||||
python3 -m pip install isort==5.12.*
|
||||
fi
|
||||
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ] && [ "${DISTRO}" != "amzn-2" ]; then
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pre_commit install
|
||||
# Install py format tools for usage during the development.
|
||||
echo "Install black formatter"
|
||||
python3 -m pip install black==23.1.*
|
||||
echo "Install isort"
|
||||
python3 -m pip install isort==5.12.*
|
||||
fi
|
||||
|
||||
# Link `include/mgp.py` with `release/mgp/mgp.py`
|
||||
|
||||
@@ -35,18 +35,17 @@ endfunction(import_header_library)
|
||||
|
||||
function(import_library name type location include_dir)
|
||||
add_library(${name} ${type} IMPORTED GLOBAL)
|
||||
# https://cmake.org/cmake/help/latest/command/function.html#arguments
|
||||
if(ARGN)
|
||||
|
||||
if(${ARGN})
|
||||
# Optional argument is the name of the external project that we need to
|
||||
# depend on.
|
||||
list(GET ARGN 0 dependency_name)
|
||||
add_dependencies(${name} ${dependency_name})
|
||||
message(STATUS "ADDED DEPENDENCY ${name} -> ${dependency_name}")
|
||||
add_dependencies(${name} ${ARGN0})
|
||||
else()
|
||||
add_dependencies(${name} ${name}-proj)
|
||||
message(STATUS "ADDED DEPENDENCY ${name} -> ${name}-proj")
|
||||
endif()
|
||||
|
||||
set_property(TARGET ${name} PROPERTY IMPORTED_LOCATION ${location})
|
||||
|
||||
# We need to create the include directory first in order to be able to add it
|
||||
# as an include directory. The header files in the include directory will be
|
||||
# generated later during the build process.
|
||||
@@ -61,12 +60,12 @@ endfunction(import_library)
|
||||
# done in SOURCE_DIR, which defaults to ${CMAKE_CURRENT_SOURCE_DIR}/${name}.
|
||||
# You can pass additional arguments via CMAKE_ARGS. Dependencies and
|
||||
# installation can be set as in regular ExternalProject_Add, via DEPENDS and
|
||||
# INSTALL_COMMAND arguments. It's also possible to pass BUILD_BYPRODUCTS.
|
||||
# INSTALL_COMMAND arguments.
|
||||
function(add_external_project name)
|
||||
set(options NO_C_COMPILER)
|
||||
set(one_value_kwargs SOURCE_DIR BUILD_IN_SOURCE)
|
||||
set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
|
||||
CONFIGURE_COMMAND BUILD_BYPRODUCTS)
|
||||
CONFIGURE_COMMAND)
|
||||
cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
|
||||
set(source_dir ${CMAKE_CURRENT_SOURCE_DIR}/${name})
|
||||
|
||||
@@ -93,8 +92,7 @@ function(add_external_project name)
|
||||
-DCMAKE_INSTALL_PREFIX=${source_dir}
|
||||
${KW_CMAKE_ARGS}
|
||||
INSTALL_COMMAND ${KW_INSTALL_COMMAND}
|
||||
BUILD_COMMAND ${KW_BUILD_COMMAND}
|
||||
BUILD_BYPRODUCTS ${KW_BUILD_BYPRODUCTS})
|
||||
BUILD_COMMAND ${KW_BUILD_COMMAND})
|
||||
endfunction(add_external_project)
|
||||
|
||||
# Calls `add_external_project`, sets NAME_LIBRARY, NAME_INCLUDE_DIR variables
|
||||
@@ -138,6 +136,7 @@ import_external_library(antlr4 STATIC
|
||||
import_external_library(benchmark STATIC
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/${CMAKE_INSTALL_LIBDIR}/libbenchmark.a
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/include
|
||||
|
||||
# Skip testing. The tests don't compile with Clang 8.
|
||||
CMAKE_ARGS -DBENCHMARK_ENABLE_TESTING=OFF)
|
||||
|
||||
@@ -151,8 +150,7 @@ mark_as_advanced(RC_ENABLE_GTEST RC_ENABLE_GMOCK)
|
||||
add_subdirectory(rapidcheck EXCLUDE_FROM_ALL)
|
||||
|
||||
# setup google test
|
||||
add_external_project(gtest SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest
|
||||
CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=lib)
|
||||
add_external_project(gtest SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest)
|
||||
set(GTEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest/include
|
||||
CACHE PATH "Path to gtest and gmock include directory" FORCE)
|
||||
set(GMOCK_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgmock.a
|
||||
@@ -303,17 +301,3 @@ import_external_library(nuraft STATIC
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/nuraft/include/)
|
||||
find_package(OpenSSL REQUIRED)
|
||||
target_link_libraries(nuraft INTERFACE ${OPENSSL_LIBRARIES})
|
||||
|
||||
# Setup mgcxx (provides: text search)
|
||||
set(MGCXX_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/mgcxx/include
|
||||
CACHE PATH "Path to mgcxx include directory" FORCE)
|
||||
set(TANTIVY_TEXT_SEARCH_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/mgcxx/lib/libtantivy_text_search.a
|
||||
CACHE FILEPATH "Path to mgcxx tantivy library (internal one)" FORCE)
|
||||
set(MGCXX_TEXT_SEARCH_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/mgcxx/lib/libmgcxx_text_search.a
|
||||
CACHE FILEPATH "Path to mgcxx text search library" FORCE)
|
||||
add_external_project(mgcxx
|
||||
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/mgcxx
|
||||
CMAKE_ARGS -DENABLE_TESTS=OFF)
|
||||
mark_as_advanced(MGCXX_INCLUDE_DIR TANTIVY_TEXT_SEARCH_LIBRARY MGCXX_TEXT_SEARCH_LIBRARY)
|
||||
import_library(tantivy_text_search STATIC ${TANTIVY_TEXT_SEARCH_LIBRARY} ${MGCXX_INCLUDE_DIR} mgcxx-proj)
|
||||
import_library(mgcxx_text_search STATIC ${MGCXX_TEXT_SEARCH_LIBRARY} ${MGCXX_INCLUDE_DIR} mgcxx-proj)
|
||||
|
||||
@@ -127,8 +127,6 @@ declare -A primary_urls=(
|
||||
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
|
||||
["range-v3"]="http://$local_cache_host/git/range-v3.git"
|
||||
["nuraft"]="http://$local_cache_host/git/NuRaft.git"
|
||||
["asio"]="http://$local_cache_host/git/asio.git"
|
||||
["mgcxx"]="http://$local_cache_host/git/mgcxx.git"
|
||||
)
|
||||
|
||||
# The goal of secondary urls is to have links to the "source of truth" of
|
||||
@@ -159,8 +157,6 @@ declare -A secondary_urls=(
|
||||
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
|
||||
["range-v3"]="https://github.com/ericniebler/range-v3.git"
|
||||
["nuraft"]="https://github.com/eBay/NuRaft.git"
|
||||
["asio"]="https://github.com/chriskohlhoff/asio.git"
|
||||
["mgcxx"]="http://github.com/memgraph/mgcxx.git"
|
||||
)
|
||||
|
||||
# antlr
|
||||
@@ -184,7 +180,7 @@ benchmark_tag="v1.6.0"
|
||||
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
|
||||
|
||||
# google test
|
||||
googletest_tag="v1.14.0"
|
||||
googletest_tag="release-1.8.0"
|
||||
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
|
||||
|
||||
# libbcrypt
|
||||
@@ -270,13 +266,13 @@ repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}"
|
||||
pushd jemalloc
|
||||
|
||||
./autogen.sh
|
||||
MALLOC_CONF="background_thread:true,retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
|
||||
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
|
||||
./configure \
|
||||
--disable-cxx \
|
||||
--with-lg-page=12 \
|
||||
--with-lg-hugepage=21 \
|
||||
--enable-shared=no --prefix=$working_dir \
|
||||
--with-malloc-conf="background_thread:true,retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
--with-malloc-conf="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
|
||||
make -j$CPUS install
|
||||
popd
|
||||
@@ -290,11 +286,5 @@ nuraft_tag="v2.1.0"
|
||||
repo_clone_try_double "${primary_urls[nuraft]}" "${secondary_urls[nuraft]}" "nuraft" "$nuraft_tag" true
|
||||
pushd nuraft
|
||||
git apply ../nuraft2.1.0.patch
|
||||
asio_tag="asio-1-29-0"
|
||||
repo_clone_try_double "${primary_urls[asio]}" "${secondary_urls[asio]}" "asio" "$asio_tag" true
|
||||
./prepare.sh
|
||||
popd
|
||||
|
||||
# mgcxx (text search)
|
||||
mgcxx_tag="v0.0.4"
|
||||
repo_clone_try_double "${primary_urls[mgcxx]}" "${secondary_urls[mgcxx]}" "mgcxx" "$mgcxx_tag" true
|
||||
|
||||
@@ -6,8 +6,6 @@ project(memgraph_query_modules)
|
||||
|
||||
disallow_in_source_build()
|
||||
|
||||
find_package(fmt REQUIRED)
|
||||
|
||||
# Everything that is installed here, should be under the "query_modules" component.
|
||||
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules")
|
||||
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
@@ -60,22 +58,6 @@ install(PROGRAMS $<TARGET_FILE:schema>
|
||||
# Also install the source of the example, so user can read it.
|
||||
install(FILES schema.cpp DESTINATION lib/memgraph/query_modules/src)
|
||||
|
||||
add_library(text SHARED text_search_module.cpp)
|
||||
target_include_directories(text PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(text PRIVATE -Wall)
|
||||
target_link_libraries(text PRIVATE -static-libgcc -static-libstdc++ fmt::fmt)
|
||||
# Strip C++ example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET text POST_BUILD
|
||||
COMMAND strip -s $<TARGET_FILE:text>
|
||||
COMMENT "Stripping symbols and sections from the C++ text_search module")
|
||||
endif()
|
||||
install(PROGRAMS $<TARGET_FILE:text>
|
||||
DESTINATION lib/memgraph/query_modules
|
||||
RENAME text.so)
|
||||
# Also install the source of the example, so user can read it.
|
||||
install(FILES text_search_module.cpp DESTINATION lib/memgraph/query_modules/src)
|
||||
|
||||
# Install the Python example and modules
|
||||
install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
|
||||
install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include <mgp.hpp>
|
||||
|
||||
namespace TextSearch {
|
||||
constexpr std::string_view kProcedureSearch = "search";
|
||||
constexpr std::string_view kProcedureRegexSearch = "regex_search";
|
||||
constexpr std::string_view kProcedureSearchAllProperties = "search_all";
|
||||
constexpr std::string_view kProcedureAggregate = "aggregate";
|
||||
constexpr std::string_view kParameterIndexName = "index_name";
|
||||
constexpr std::string_view kParameterSearchQuery = "search_query";
|
||||
constexpr std::string_view kParameterAggregationQuery = "aggregation_query";
|
||||
constexpr std::string_view kReturnNode = "node";
|
||||
constexpr std::string_view kReturnAggregation = "aggregation";
|
||||
const std::string kSearchAllPrefix = "all";
|
||||
|
||||
void Search(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void RegexSearch(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void SearchAllProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void Aggregate(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
} // namespace TextSearch
|
||||
|
||||
void TextSearch::Search(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
auto arguments = mgp::List(args);
|
||||
|
||||
try {
|
||||
const auto *index_name = arguments[0].ValueString().data();
|
||||
const auto *search_query = arguments[1].ValueString().data();
|
||||
for (const auto &node :
|
||||
mgp::SearchTextIndex(memgraph_graph, index_name, search_query, text_search_mode::SPECIFIED_PROPERTIES)) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(TextSearch::kReturnNode.data(), node.ValueNode());
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
record_factory.SetErrorMessage(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void TextSearch::RegexSearch(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
auto arguments = mgp::List(args);
|
||||
|
||||
try {
|
||||
const auto *index_name = arguments[0].ValueString().data();
|
||||
const auto *search_query = arguments[1].ValueString().data();
|
||||
for (const auto &node : mgp::SearchTextIndex(memgraph_graph, index_name, search_query, text_search_mode::REGEX)) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(TextSearch::kReturnNode.data(), node.ValueNode());
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
record_factory.SetErrorMessage(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void TextSearch::SearchAllProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result,
|
||||
mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
auto arguments = mgp::List(args);
|
||||
|
||||
try {
|
||||
const auto *index_name = arguments[0].ValueString().data();
|
||||
const auto *search_query = fmt::format("{}:{}", kSearchAllPrefix, arguments[1].ValueString()).data();
|
||||
for (const auto &node :
|
||||
mgp::SearchTextIndex(memgraph_graph, index_name, search_query, text_search_mode::ALL_PROPERTIES)) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(TextSearch::kReturnNode.data(), node.ValueNode());
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
record_factory.SetErrorMessage(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void TextSearch::Aggregate(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
auto arguments = mgp::List(args);
|
||||
|
||||
try {
|
||||
const auto *index_name = arguments[0].ValueString().data();
|
||||
const auto *search_query = arguments[1].ValueString().data();
|
||||
const auto *aggregation_query = arguments[2].ValueString().data();
|
||||
const auto aggregation_result =
|
||||
mgp::AggregateOverTextIndex(memgraph_graph, index_name, search_query, aggregation_query);
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(TextSearch::kReturnAggregation.data(), aggregation_result.data());
|
||||
} catch (const std::exception &e) {
|
||||
record_factory.SetErrorMessage(e.what());
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
|
||||
try {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
|
||||
AddProcedure(TextSearch::Search, TextSearch::kProcedureSearch, mgp::ProcedureType::Read,
|
||||
{
|
||||
mgp::Parameter(TextSearch::kParameterIndexName, mgp::Type::String),
|
||||
mgp::Parameter(TextSearch::kParameterSearchQuery, mgp::Type::String),
|
||||
},
|
||||
{mgp::Return(TextSearch::kReturnNode, mgp::Type::Node)}, module, memory);
|
||||
|
||||
AddProcedure(TextSearch::RegexSearch, TextSearch::kProcedureRegexSearch, mgp::ProcedureType::Read,
|
||||
{
|
||||
mgp::Parameter(TextSearch::kParameterIndexName, mgp::Type::String),
|
||||
mgp::Parameter(TextSearch::kParameterSearchQuery, mgp::Type::String),
|
||||
},
|
||||
{mgp::Return(TextSearch::kReturnNode, mgp::Type::Node)}, module, memory);
|
||||
|
||||
AddProcedure(TextSearch::SearchAllProperties, TextSearch::kProcedureSearchAllProperties, mgp::ProcedureType::Read,
|
||||
{
|
||||
mgp::Parameter(TextSearch::kParameterIndexName, mgp::Type::String),
|
||||
mgp::Parameter(TextSearch::kParameterSearchQuery, mgp::Type::String),
|
||||
},
|
||||
{mgp::Return(TextSearch::kReturnNode, mgp::Type::Node)}, module, memory);
|
||||
|
||||
AddProcedure(TextSearch::Aggregate, TextSearch::kProcedureAggregate, mgp::ProcedureType::Read,
|
||||
{
|
||||
mgp::Parameter(TextSearch::kParameterIndexName, mgp::Type::String),
|
||||
mgp::Parameter(TextSearch::kParameterSearchQuery, mgp::Type::String),
|
||||
mgp::Parameter(TextSearch::kParameterAggregationQuery, mgp::Type::String),
|
||||
},
|
||||
{mgp::Return(TextSearch::kReturnAggregation, mgp::Type::String)}, module, memory);
|
||||
} catch (const std::exception &e) {
|
||||
std::cerr << "Error while initializing query module: " << e.what() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern "C" int mgp_shutdown_module() { return 0; }
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM debian:11
|
||||
FROM debian:bullseye
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG BINARY_NAME
|
||||
@@ -13,40 +13,17 @@ working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
|
||||
latest_image=""
|
||||
tag_latest=""
|
||||
toolchain_version="v5"
|
||||
package_path=""
|
||||
arch=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--latest)
|
||||
latest_image="memgraph:latest"
|
||||
tag_latest="-t memgraph:latest"
|
||||
shift
|
||||
;;
|
||||
--package-path)
|
||||
package_path=$2
|
||||
shift 2
|
||||
;;
|
||||
--toolchain)
|
||||
toolchain_version=$2
|
||||
shift 2
|
||||
;;
|
||||
--arch)
|
||||
arch=$2
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
print_help
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo -e "Error: Unknown option '$1'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
|
||||
latest_image="memgraph:latest"
|
||||
tag_latest="-t memgraph:latest"
|
||||
shift
|
||||
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
|
||||
print_help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$package_path" == "" || ! -f "$package_path" ]]; then
|
||||
package_path="$1"
|
||||
if [[ ! -f "$package_path" ]]; then
|
||||
echo "File '$package_path' does not exist!"
|
||||
exit 1
|
||||
fi
|
||||
@@ -61,13 +38,13 @@ if [[ "$extension" == "deb" ]]; then
|
||||
# Extract version and offering from deb name.
|
||||
package_name=`echo $(basename "$package_path") | sed 's/.deb$//'`
|
||||
version=`echo ${package_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
|
||||
dockerfile_path="${working_dir}/${toolchain_version}_deb.dockerfile"
|
||||
dockerfile_path="${working_dir}/memgraph_deb.dockerfile"
|
||||
elif [[ "$extension" == "rpm" ]]; then
|
||||
# Extract version and offering from deb name.
|
||||
package_name=`echo $(basename "$package_path") | sed 's/.rpm$//'`
|
||||
version=`echo ${package_name} | cut -d '-' -f 2 | rev | cut -d '-' -f 2- | rev`
|
||||
version=${version%_1}
|
||||
dockerfile_path="${working_dir}/${toolchain_version}_rpm.dockerfile"
|
||||
dockerfile_path="${working_dir}/memgraph_rpm.dockerfile"
|
||||
else
|
||||
echo "Invalid file sent as the package"
|
||||
print_help
|
||||
@@ -85,5 +62,4 @@ docker build --pull -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
|
||||
--build-arg TARGETARCH="" .
|
||||
docker save ${image_name} ${latest_image} | gzip > ${image_package_name}
|
||||
rm "${package_name}.${extension}"
|
||||
docker image rm $image_name
|
||||
echo "Built Docker image at '${working_dir}/${image_package_name}'"
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
FROM debian:12
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG BINARY_NAME
|
||||
ARG EXTENSION
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
openssl libcurl4 libssl3 libseccomp2 python3 libpython3.11 python3-pip \
|
||||
--no-install-recommends \
|
||||
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
RUN pip3 install --break-system-packages networkx==3.2.1 numpy==1.26.4 scipy==1.12.0
|
||||
|
||||
COPY "${BINARY_NAME}${TARGETARCH}.${EXTENSION}" /
|
||||
|
||||
# Install memgraph package
|
||||
RUN dpkg -i "${BINARY_NAME}${TARGETARCH}.deb"
|
||||
|
||||
# Memgraph listens for Bolt Protocol on this port by default.
|
||||
EXPOSE 7687
|
||||
# Snapshots and logging volumes
|
||||
VOLUME /var/log/memgraph
|
||||
VOLUME /var/lib/memgraph
|
||||
# Configuration volume
|
||||
VOLUME /etc/memgraph
|
||||
|
||||
USER memgraph
|
||||
WORKDIR /usr/lib/memgraph
|
||||
|
||||
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
|
||||
CMD [""]
|
||||
@@ -1,100 +0,0 @@
|
||||
version: "3"
|
||||
services:
|
||||
mgbuild_v4_amzn-2:
|
||||
image: "memgraph/mgbuild:v4_amzn-2"
|
||||
build:
|
||||
context: amzn-2
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_amzn-2"
|
||||
|
||||
mgbuild_v4_centos-7:
|
||||
image: "memgraph/mgbuild:v4_centos-7"
|
||||
build:
|
||||
context: centos-7
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_centos-7"
|
||||
|
||||
mgbuild_v4_centos-9:
|
||||
image: "memgraph/mgbuild:v4_centos-9"
|
||||
build:
|
||||
context: centos-9
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_centos-9"
|
||||
|
||||
mgbuild_v4_debian-10:
|
||||
image: "memgraph/mgbuild:v4_debian-10"
|
||||
build:
|
||||
context: debian-10
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_debian-10"
|
||||
|
||||
mgbuild_v4_debian-11:
|
||||
image: "memgraph/mgbuild:v4_debian-11"
|
||||
build:
|
||||
context: debian-11
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_debian-11"
|
||||
|
||||
mgbuild_v4_fedora-36:
|
||||
image: "memgraph/mgbuild:v4_fedora-36"
|
||||
build:
|
||||
context: fedora-36
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_fedora-36"
|
||||
|
||||
mgbuild_v4_ubuntu-18.04:
|
||||
image: "memgraph/mgbuild:v4_ubuntu-18.04"
|
||||
build:
|
||||
context: ubuntu-18.04
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_ubuntu-18.04"
|
||||
|
||||
mgbuild_v4_ubuntu-20.04:
|
||||
image: "memgraph/mgbuild:v4_ubuntu-20.04"
|
||||
build:
|
||||
context: ubuntu-20.04
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_ubuntu-20.04"
|
||||
|
||||
mgbuild_v4_ubuntu-22.04:
|
||||
image: "memgraph/mgbuild:v4_ubuntu-22.04"
|
||||
build:
|
||||
context: ubuntu-22.04
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_ubuntu-22.04"
|
||||
@@ -1,111 +0,0 @@
|
||||
version: "3"
|
||||
services:
|
||||
mgbuild_v5_amzn-2:
|
||||
image: "memgraph/mgbuild:v5_amzn-2"
|
||||
build:
|
||||
context: amzn-2
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_amzn-2"
|
||||
|
||||
mgbuild_v5_centos-7:
|
||||
image: "memgraph/mgbuild:v5_centos-7"
|
||||
build:
|
||||
context: centos-7
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_centos-7"
|
||||
|
||||
mgbuild_v5_centos-9:
|
||||
image: "memgraph/mgbuild:v5_centos-9"
|
||||
build:
|
||||
context: centos-9
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_centos-9"
|
||||
|
||||
mgbuild_v5_debian-11:
|
||||
image: "memgraph/mgbuild:v5_debian-11"
|
||||
build:
|
||||
context: debian-11
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_debian-11"
|
||||
|
||||
mgbuild_v5_debian-12:
|
||||
image: "memgraph/mgbuild:v5_debian-12"
|
||||
build:
|
||||
context: debian-12
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_debian-12"
|
||||
|
||||
mgbuild_v5_fedora-38:
|
||||
image: "memgraph/mgbuild:v5_fedora-38"
|
||||
build:
|
||||
context: fedora-38
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_fedora-38"
|
||||
|
||||
mgbuild_v5_fedora-39:
|
||||
image: "memgraph/mgbuild:v5_fedora-39"
|
||||
build:
|
||||
context: fedora-39
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_fedora-39"
|
||||
|
||||
mgbuild_v5_rocky-9.3:
|
||||
image: "memgraph/mgbuild:v5_rocky-9.3"
|
||||
build:
|
||||
context: rocky-9.3
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_rocky-9.3"
|
||||
|
||||
mgbuild_v5_ubuntu-20.04:
|
||||
image: "memgraph/mgbuild:v5_ubuntu-20.04"
|
||||
build:
|
||||
context: ubuntu-20.04
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_ubuntu-20.04"
|
||||
|
||||
mgbuild_v5_ubuntu-22.04:
|
||||
image: "memgraph/mgbuild:v5_ubuntu-22.04"
|
||||
build:
|
||||
context: ubuntu-22.04
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_ubuntu-22.04"
|
||||
@@ -7,32 +7,9 @@ RUN yum -y update \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
# Download and install toolchain
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/amzn-2.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/amzn-2.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
# Install PyYAML (only for amzn-2, centos-7, cento-9 and rocky-9)
|
||||
RUN pip3 install --user PyYAML
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
mgbuild_v4_debian-11-arm:
|
||||
image: "memgraph/mgbuild:v4_debian-11-arm"
|
||||
build:
|
||||
context: debian-11-arm
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_debian-11-arm"
|
||||
|
||||
mgbuild_v4_ubuntu-22.04-arm:
|
||||
image: "memgraph/mgbuild:v4_ubuntu-22.04-arm"
|
||||
build:
|
||||
context: ubuntu-22.04-arm
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v4"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v4_ubuntu-22.04-arm"
|
||||
@@ -1,24 +0,0 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
mgbuild_v5_debian-12-arm:
|
||||
image: "memgraph/mgbuild:v5_debian-12-arm"
|
||||
build:
|
||||
context: debian-12-arm
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_debian-12-arm"
|
||||
|
||||
mgbuild_v5_ubuntu-22.04-arm:
|
||||
image: "memgraph/mgbuild:v5_ubuntu-22.04-arm"
|
||||
build:
|
||||
context: ubuntu-22.04-arm
|
||||
args:
|
||||
TOOLCHAIN_VERSION: "v5"
|
||||
extra_hosts:
|
||||
- "mgdeps-cache:10.42.16.10"
|
||||
- "bench-graph-api:10.42.16.10"
|
||||
container_name: "mgbuild_v5_ubuntu-22.04-arm"
|
||||
11
release/package/arm-builders.yml
Normal file
11
release/package/arm-builders.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
version: "3"
|
||||
|
||||
services:
|
||||
debian-11-arm:
|
||||
build:
|
||||
context: debian-11-arm
|
||||
container_name: "mgbuild_debian-11-arm"
|
||||
ubuntu-2204-arm:
|
||||
build:
|
||||
context: ubuntu-22.04-arm
|
||||
container_name: "mgbuild_ubuntu-22.04-arm"
|
||||
@@ -7,31 +7,9 @@ RUN yum -y update \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/centos-7.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/centos-7.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
# Install PyYAML (only for amzn-2, centos-7, cento-9 and rocky-9)
|
||||
RUN pip3 install --user PyYAML
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -7,31 +7,9 @@ RUN yum -y update \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/centos-9.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/centos-9.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
# Install PyYAML (only for amzn-2, centos-7, cento-9 and rocky-9)
|
||||
RUN pip3 install --user PyYAML
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/debian-10.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/debian-10.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/debian-11-arm.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/debian-11-arm.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/debian-11.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/debian-11.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
FROM debian:12
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-arm64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-arm64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-arm64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-arm64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/debian-12-arm.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/debian-12-arm.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -1,37 +0,0 @@
|
||||
FROM debian:12
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-debian-12-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/debian-12.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/debian-12.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
38
release/package/docker-compose.yml
Normal file
38
release/package/docker-compose.yml
Normal file
@@ -0,0 +1,38 @@
|
||||
version: "3"
|
||||
services:
|
||||
mgbuild_centos-7:
|
||||
build:
|
||||
context: centos-7
|
||||
container_name: "mgbuild_centos-7"
|
||||
mgbuild_centos-9:
|
||||
build:
|
||||
context: centos-9
|
||||
container_name: "mgbuild_centos-9"
|
||||
mgbuild_debian-10:
|
||||
build:
|
||||
context: debian-10
|
||||
container_name: "mgbuild_debian-10"
|
||||
mgbuild_debian-11:
|
||||
build:
|
||||
context: debian-11
|
||||
container_name: "mgbuild_debian-11"
|
||||
mgbuild_ubuntu-18.04:
|
||||
build:
|
||||
context: ubuntu-18.04
|
||||
container_name: "mgbuild_ubuntu-18.04"
|
||||
mgbuild_ubuntu-20.04:
|
||||
build:
|
||||
context: ubuntu-20.04
|
||||
container_name: "mgbuild_ubuntu-20.04"
|
||||
mgbuild_ubuntu-22.04:
|
||||
build:
|
||||
context: ubuntu-22.04
|
||||
container_name: "mgbuild_ubuntu-22.04"
|
||||
mgbuild_fedora-36:
|
||||
build:
|
||||
context: fedora-36
|
||||
container_name: "mgbuild_fedora-36"
|
||||
mgbuild_amzn-2:
|
||||
build:
|
||||
context: amzn-2
|
||||
container_name: "mgbuild_amzn-2"
|
||||
@@ -8,28 +8,9 @@ RUN yum -y update \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/fedora-36.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/fedora-36.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
FROM fedora:38
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
RUN yum -y update \
|
||||
&& yum install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-38-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-38-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-38-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-38-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/fedora-38.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/fedora-38.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -1,35 +0,0 @@
|
||||
FROM fedora:39
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
RUN yum -y update \
|
||||
&& yum install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-39-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-39-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-39-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-fedora-39-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/fedora-39.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/fedora-39.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -1,735 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -Eeuo pipefail
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
SCRIPT_NAME=${0##*/}
|
||||
PROJECT_ROOT="$SCRIPT_DIR/../.."
|
||||
MGBUILD_HOME_DIR="/home/mg"
|
||||
MGBUILD_ROOT_DIR="$MGBUILD_HOME_DIR/memgraph"
|
||||
|
||||
DEFAULT_TOOLCHAIN="v5"
|
||||
SUPPORTED_TOOLCHAINS=(
|
||||
v4 v5
|
||||
)
|
||||
DEFAULT_OS="all"
|
||||
SUPPORTED_OS=(
|
||||
all
|
||||
amzn-2
|
||||
centos-7 centos-9
|
||||
debian-10 debian-11 debian-11-arm debian-12 debian-12-arm
|
||||
fedora-36 fedora-38 fedora-39
|
||||
rocky-9.3
|
||||
ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
|
||||
)
|
||||
SUPPORTED_OS_V4=(
|
||||
amzn-2
|
||||
centos-7 centos-9
|
||||
debian-10 debian-11 debian-11-arm
|
||||
fedora-36
|
||||
ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
|
||||
)
|
||||
SUPPORTED_OS_V5=(
|
||||
amzn-2
|
||||
centos-7 centos-9
|
||||
debian-11 debian-11-arm debian-12 debian-12-arm
|
||||
fedora-38 fedora-39
|
||||
rocky-9.3
|
||||
ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
|
||||
)
|
||||
DEFAULT_BUILD_TYPE="Release"
|
||||
SUPPORTED_BUILD_TYPES=(
|
||||
Debug
|
||||
Release
|
||||
RelWithDebInfo
|
||||
)
|
||||
DEFAULT_ARCH="amd"
|
||||
SUPPORTED_ARCHS=(
|
||||
amd
|
||||
arm
|
||||
)
|
||||
SUPPORTED_TESTS=(
|
||||
clang-tidy cppcheck-and-clang-format code-analysis
|
||||
code-coverage drivers drivers-high-availability durability e2e gql-behave
|
||||
integration leftover-CTest macro-benchmark
|
||||
mgbench stress-plain stress-ssl
|
||||
unit unit-coverage upload-to-bench-graph
|
||||
|
||||
)
|
||||
DEFAULT_THREADS=0
|
||||
DEFAULT_ENTERPRISE_LICENSE=""
|
||||
DEFAULT_ORGANIZATION_NAME="memgraph"
|
||||
|
||||
print_help () {
|
||||
echo -e "\nUsage: $SCRIPT_NAME [GLOBAL OPTIONS] COMMAND [COMMAND OPTIONS]"
|
||||
echo -e "\nInteract with mgbuild containers"
|
||||
|
||||
echo -e "\nCommands:"
|
||||
echo -e " build [OPTIONS] Build mgbuild image"
|
||||
echo -e " build-memgraph [OPTIONS] Build memgraph binary inside mgbuild container"
|
||||
echo -e " copy OPTIONS Copy an artifact from mgbuild container to host"
|
||||
echo -e " package-memgraph Create memgraph package from built binary inside mgbuild container"
|
||||
echo -e " package-docker Create memgraph docker image and pack it as .tar.gz"
|
||||
echo -e " pull Pull mgbuild image from dockerhub"
|
||||
echo -e " push [OPTIONS] Push mgbuild image to dockerhub"
|
||||
echo -e " run [OPTIONS] Run mgbuild container"
|
||||
echo -e " stop [OPTIONS] Stop mgbuild container"
|
||||
echo -e " test-memgraph TEST Run a selected test TEST (see supported tests below) inside mgbuild container"
|
||||
|
||||
echo -e "\nSupported tests:"
|
||||
echo -e " \"${SUPPORTED_TESTS[*]}\""
|
||||
|
||||
echo -e "\nGlobal options:"
|
||||
echo -e " --arch string Specify target architecture (\"${SUPPORTED_ARCHS[*]}\") (default \"$DEFAULT_ARCH\")"
|
||||
echo -e " --build-type string Specify build type (\"${SUPPORTED_BUILD_TYPES[*]}\") (default \"$DEFAULT_BUILD_TYPE\")"
|
||||
echo -e " --enterprise-license string Specify the enterprise license (default \"\")"
|
||||
echo -e " --organization-name string Specify the organization name (default \"memgraph\")"
|
||||
echo -e " --os string Specify operating system (\"${SUPPORTED_OS[*]}\") (default \"$DEFAULT_OS\")"
|
||||
echo -e " --threads int Specify the number of threads a command will use (default \"\$(nproc)\" for container)"
|
||||
echo -e " --toolchain string Specify toolchain version (\"${SUPPORTED_TOOLCHAINS[*]}\") (default \"$DEFAULT_TOOLCHAIN\")"
|
||||
|
||||
echo -e "\nbuild options:"
|
||||
echo -e " --git-ref string Specify git ref from which the environment deps will be installed (default \"master\")"
|
||||
|
||||
echo -e "\nbuild-memgraph options:"
|
||||
echo -e " --asan Build with ASAN"
|
||||
echo -e " --community Build community version"
|
||||
echo -e " --coverage Build with code coverage"
|
||||
echo -e " --for-docker Add flag -DMG_TELEMETRY_ID_OVERRIDE=DOCKER to cmake"
|
||||
echo -e " --for-platform Add flag -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM to cmake"
|
||||
echo -e " --init-only Only run init script"
|
||||
echo -e " --no-copy Don't copy the memgraph repo from host."
|
||||
echo -e " Use this option with caution, be sure that memgraph source code is in correct location inside mgbuild container"
|
||||
echo -e " --ubsan Build with UBSAN"
|
||||
|
||||
echo -e "\ncopy options:"
|
||||
echo -e " --binary Copy memgraph binary from mgbuild container to host"
|
||||
echo -e " --build-logs Copy build logs from mgbuild container to host"
|
||||
echo -e " --package Copy memgraph package from mgbuild container to host"
|
||||
|
||||
echo -e "\npush options:"
|
||||
echo -e " -p, --password string Specify password for docker login (default empty)"
|
||||
echo -e " -u, --username string Specify username for docker login (default empty)"
|
||||
|
||||
echo -e "\nrun options:"
|
||||
echo -e " --pull Pull the mgbuild image before running"
|
||||
|
||||
echo -e "\nstop options:"
|
||||
echo -e " --remove Remove the stopped mgbuild container"
|
||||
|
||||
echo -e "\nToolchain v4 supported OSs:"
|
||||
echo -e " \"${SUPPORTED_OS_V4[*]}\""
|
||||
|
||||
echo -e "\nToolchain v5 supported OSs:"
|
||||
echo -e " \"${SUPPORTED_OS_V5[*]}\""
|
||||
|
||||
echo -e "\nExample usage:"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd build --git-ref my-special-branch"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd run"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd --build-type RelWithDebInfo build-memgraph --community"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd --build-type RelWithDebInfo test-memgraph unit"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd package"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd copy --package"
|
||||
echo -e " $SCRIPT_NAME --os debian-12 --toolchain v5 --arch amd stop --remove"
|
||||
}
|
||||
|
||||
check_support() {
|
||||
local is_supported=false
|
||||
case "$1" in
|
||||
arch)
|
||||
for e in "${SUPPORTED_ARCHS[@]}"; do
|
||||
if [[ "$e" == "$2" ]]; then
|
||||
is_supported=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_supported" == false ]]; then
|
||||
echo -e "Error: Architecture $2 isn't supported!\nChoose from ${SUPPORTED_ARCHS[*]}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
build_type)
|
||||
for e in "${SUPPORTED_BUILD_TYPES[@]}"; do
|
||||
if [[ "$e" == "$2" ]]; then
|
||||
is_supported=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_supported" == false ]]; then
|
||||
echo -e "Error: Build type $2 isn't supported!\nChoose from ${SUPPORTED_BUILD_TYPES[*]}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
os)
|
||||
for e in "${SUPPORTED_OS[@]}"; do
|
||||
if [[ "$e" == "$2" ]]; then
|
||||
is_supported=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_supported" == false ]]; then
|
||||
echo -e "Error: OS $2 isn't supported!\nChoose from ${SUPPORTED_OS[*]}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
toolchain)
|
||||
for e in "${SUPPORTED_TOOLCHAINS[@]}"; do
|
||||
if [[ "$e" == "$2" ]]; then
|
||||
is_supported=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_supported" == false ]]; then
|
||||
echo -e "TError: oolchain version $2 isn't supported!\nChoose from ${SUPPORTED_TOOLCHAINS[*]}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
os_toolchain_combo)
|
||||
if [[ "$3" == "v4" ]]; then
|
||||
local SUPPORTED_OS_TOOLCHAIN=("${SUPPORTED_OS_V4[@]}")
|
||||
elif [[ "$3" == "v5" ]]; then
|
||||
local SUPPORTED_OS_TOOLCHAIN=("${SUPPORTED_OS_V5[@]}")
|
||||
else
|
||||
echo -e "Error: $3 isn't a supported toolchain_version!\nChoose from ${SUPPORTED_TOOLCHAINS[*]}"
|
||||
exit 1
|
||||
fi
|
||||
for e in "${SUPPORTED_OS_TOOLCHAIN[@]}"; do
|
||||
if [[ "$e" == "$2" ]]; then
|
||||
is_supported=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_supported" == false ]]; then
|
||||
echo -e "Error: Toolchain version $3 doesn't support OS $2!\nChoose from ${SUPPORTED_OS_TOOLCHAIN[*]}"
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo -e "Error: This function can only check arch, build_type, os, toolchain version and os toolchain combination"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
##################################################
|
||||
######## BUILD, COPY AND PACKAGE MEMGRAPH ########
|
||||
##################################################
|
||||
build_memgraph () {
|
||||
local build_container="mgbuild_${toolchain_version}_${os}"
|
||||
local ACTIVATE_TOOLCHAIN="source /opt/toolchain-${toolchain_version}/activate"
|
||||
local ACTIVATE_CARGO="source $MGBUILD_HOME_DIR/.cargo/env"
|
||||
local container_build_dir="$MGBUILD_ROOT_DIR/build"
|
||||
local container_output_dir="$container_build_dir/output"
|
||||
local arm_flag=""
|
||||
if [[ "$arch" == "arm" ]] || [[ "$os" =~ "-arm" ]]; then
|
||||
arm_flag="-DMG_ARCH="ARM64""
|
||||
fi
|
||||
local build_type_flag="-DCMAKE_BUILD_TYPE=$build_type"
|
||||
local telemetry_id_override_flag=""
|
||||
local community_flag=""
|
||||
local coverage_flag=""
|
||||
local asan_flag=""
|
||||
local ubsan_flag=""
|
||||
local init_only=false
|
||||
local for_docker=false
|
||||
local for_platform=false
|
||||
local copy_from_host=true
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--community)
|
||||
community_flag="-DMG_ENTERPRISE=OFF"
|
||||
shift 1
|
||||
;;
|
||||
--init-only)
|
||||
init_only=true
|
||||
shift 1
|
||||
;;
|
||||
--for-docker)
|
||||
for_docker=true
|
||||
if [[ "$for_platform" == "true" ]]; then
|
||||
echo "Error: Cannot combine --for-docker and --for-platform flags"
|
||||
exit 1
|
||||
fi
|
||||
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER "
|
||||
shift 1
|
||||
;;
|
||||
--for-platform)
|
||||
for_platform=true
|
||||
if [[ "$for_docker" == "true" ]]; then
|
||||
echo "Error: Cannot combine --for-docker and --for-platform flags"
|
||||
exit 1
|
||||
fi
|
||||
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM "
|
||||
shift 1
|
||||
;;
|
||||
--coverage)
|
||||
coverage_flag="-DTEST_COVERAGE=ON"
|
||||
shift 1
|
||||
;;
|
||||
--asan)
|
||||
asan_flag="-DASAN=ON"
|
||||
shift 1
|
||||
;;
|
||||
--ubsan)
|
||||
ubsan_flag="-DUBSAN=ON"
|
||||
shift 1
|
||||
;;
|
||||
--no-copy)
|
||||
copy_from_host=false
|
||||
shift 1
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown flag '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "Initializing deps ..."
|
||||
# If master is not the current branch, fetch it, because the get_version
|
||||
# script depends on it. If we are on master, the fetch command is going to
|
||||
# fail so that's why there is the explicit check.
|
||||
# Required here because Docker build container can't access remote.
|
||||
cd "$PROJECT_ROOT"
|
||||
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
|
||||
git fetch origin master:master
|
||||
fi
|
||||
|
||||
if [[ "$copy_from_host" == "true" ]]; then
|
||||
# Ensure we have a clean build directory
|
||||
docker exec -u mg "$build_container" bash -c "rm -rf $MGBUILD_ROOT_DIR && mkdir -p $MGBUILD_ROOT_DIR"
|
||||
echo "Copying project files..."
|
||||
docker cp "$PROJECT_ROOT/." "$build_container:$MGBUILD_ROOT_DIR/"
|
||||
# Change ownership of copied files so the mg user inside container can access them
|
||||
docker exec -u root $build_container bash -c "chown -R mg:mg $MGBUILD_ROOT_DIR"
|
||||
fi
|
||||
|
||||
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
|
||||
docker exec -u root "$build_container" bash -c "$MGBUILD_ROOT_DIR/environment/os/$os.sh check TOOLCHAIN_RUN_DEPS || $MGBUILD_ROOT_DIR/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
|
||||
docker exec -u root "$build_container" bash -c "$MGBUILD_ROOT_DIR/environment/os/$os.sh check MEMGRAPH_BUILD_DEPS || $MGBUILD_ROOT_DIR/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
|
||||
|
||||
echo "Building targeted package..."
|
||||
# Fix issue with git marking directory as not safe
|
||||
docker exec -u mg "$build_container" bash -c "cd $MGBUILD_ROOT_DIR && git config --global --add safe.directory '*'"
|
||||
docker exec -u mg "$build_container" bash -c "cd $MGBUILD_ROOT_DIR && $ACTIVATE_TOOLCHAIN && ./init --ci"
|
||||
if [[ "$init_only" == "true" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Building Memgraph for $os on $build_container..."
|
||||
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
|
||||
# Fix cmake failing locally if remote is clone via ssh
|
||||
docker exec -u mg "$build_container" bash -c "cd $MGBUILD_ROOT_DIR && git remote set-url origin https://github.com/memgraph/memgraph.git"
|
||||
|
||||
# Define cmake command
|
||||
local cmake_cmd="cmake $build_type_flag $arm_flag $community_flag $telemetry_id_override_flag $coverage_flag $asan_flag $ubsan_flag .."
|
||||
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && $ACTIVATE_CARGO && $cmake_cmd"
|
||||
# ' is used instead of " because we need to run make within the allowed
|
||||
# container resources.
|
||||
# Default value for $threads is 0 instead of $(nproc) because macos
|
||||
# doesn't support the nproc command.
|
||||
# 0 is set for default value and checked here because mgbuild containers
|
||||
# support nproc
|
||||
# shellcheck disable=SC2016
|
||||
if [[ "$threads" == 0 ]]; then
|
||||
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && $ACTIVATE_CARGO "'&& make -j$(nproc)'
|
||||
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && $ACTIVATE_CARGO "'&& make -j$(nproc) -B mgconsole'
|
||||
else
|
||||
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && $ACTIVATE_CARGO "'&& make -j$threads'
|
||||
docker exec -u mg "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && $ACTIVATE_CARGO "'&& make -j$threads -B mgconsole'
|
||||
fi
|
||||
}
|
||||
|
||||
package_memgraph() {
|
||||
local ACTIVATE_TOOLCHAIN="source /opt/toolchain-${toolchain_version}/activate"
|
||||
local build_container="mgbuild_${toolchain_version}_${os}"
|
||||
local container_output_dir="$MGBUILD_ROOT_DIR/build/output"
|
||||
local package_command=""
|
||||
if [[ "$os" =~ ^"centos".* ]] || [[ "$os" =~ ^"fedora".* ]] || [[ "$os" =~ ^"amzn".* ]] || [[ "$os" =~ ^"rocky".* ]]; then
|
||||
docker exec -u root "$build_container" bash -c "yum -y update"
|
||||
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
|
||||
fi
|
||||
if [[ "$os" =~ ^"debian".* ]]; then
|
||||
docker exec -u root "$build_container" bash -c "apt --allow-releaseinfo-change -y update"
|
||||
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
|
||||
fi
|
||||
if [[ "$os" =~ ^"ubuntu".* ]]; then
|
||||
docker exec -u root "$build_container" bash -c "apt update"
|
||||
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
|
||||
fi
|
||||
docker exec -u mg "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
|
||||
}
|
||||
|
||||
package_docker() {
|
||||
if [[ "$toolchain_version" == "v4" ]]; then
|
||||
if [[ "$os" != "debian-11" && "$os" != "debian-11-arm" ]]; then
|
||||
echo -e "Error: When passing '--toolchain v4' the 'docker' command accepts only '--os debian-11' and '--os debian-11-arm'"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if [[ "$os" != "debian-12" && "$os" != "debian-12-arm" ]]; then
|
||||
echo -e "Error: When passing '--toolchain v5' the 'docker' command accepts only '--os debian-12' and '--os debian-12-arm'"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
local package_dir="$PROJECT_ROOT/build/output/$os"
|
||||
# shellcheck disable=SC2012
|
||||
local last_package_name=$(cd $package_dir && ls -t memgraph* | head -1)
|
||||
local docker_build_folder="$PROJECT_ROOT/release/docker"
|
||||
cd "$docker_build_folder"
|
||||
./package_docker --latest --package-path "$package_dir/$last_package_name" --toolchain $toolchain_version --arch "${arch}64"
|
||||
# shellcheck disable=SC2012
|
||||
local docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
|
||||
local docker_host_folder="$PROJECT_ROOT/build/output/docker/${arch}/${toolchain_version}"
|
||||
local docker_host_image_path="$docker_host_folder/$docker_image_name"
|
||||
mkdir -p "$docker_host_folder"
|
||||
cp "$docker_build_folder/$docker_image_name" "$docker_host_folder"
|
||||
echo "Docker images saved to $docker_host_image_path."
|
||||
}
|
||||
|
||||
copy_memgraph() {
|
||||
local build_container="mgbuild_${toolchain_version}_${os}"
|
||||
case "$1" in
|
||||
--binary)
|
||||
echo "Copying memgraph binary to host..."
|
||||
local container_output_path="$MGBUILD_ROOT_DIR/build/memgraph"
|
||||
local host_output_path="$PROJECT_ROOT/build/memgraph"
|
||||
mkdir -p "$PROJECT_ROOT/build"
|
||||
docker cp -L $build_container:$container_output_path $host_output_path
|
||||
echo "Binary saved to $host_output_path"
|
||||
;;
|
||||
--build-logs)
|
||||
echo "Copying memgraph build logs to host..."
|
||||
local container_output_path="$MGBUILD_ROOT_DIR/build/logs"
|
||||
local host_output_path="$PROJECT_ROOT/build/logs"
|
||||
mkdir -p "$PROJECT_ROOT/build"
|
||||
docker cp -L $build_container:$container_output_path $host_output_path
|
||||
echo "Build logs saved to $host_output_path"
|
||||
;;
|
||||
--package)
|
||||
echo "Copying memgraph package to host..."
|
||||
local container_output_dir="$MGBUILD_ROOT_DIR/build/output"
|
||||
local host_output_dir="$PROJECT_ROOT/build/output/$os"
|
||||
local last_package_name=$(docker exec -u mg "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
|
||||
mkdir -p "$host_output_dir"
|
||||
docker cp "$build_container:$container_output_dir/$last_package_name" "$host_output_dir/$last_package_name"
|
||||
echo "Package saved to $host_output_dir/$last_package_name"
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown flag '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
##################################################
|
||||
##################### TESTS ######################
|
||||
##################################################
|
||||
test_memgraph() {
|
||||
local ACTIVATE_TOOLCHAIN="source /opt/toolchain-${toolchain_version}/activate"
|
||||
local ACTIVATE_VENV="./setup.sh /opt/toolchain-${toolchain_version}/activate"
|
||||
local ACTIVATE_CARGO="source $MGBUILD_HOME_DIR/.cargo/env"
|
||||
local EXPORT_LICENSE="export MEMGRAPH_ENTERPRISE_LICENSE=$enterprise_license"
|
||||
local EXPORT_ORG_NAME="export MEMGRAPH_ORGANIZATION_NAME=$organization_name"
|
||||
local BUILD_DIR="$MGBUILD_ROOT_DIR/build"
|
||||
local build_container="mgbuild_${toolchain_version}_${os}"
|
||||
echo "Running $1 test on $build_container..."
|
||||
|
||||
case "$1" in
|
||||
unit)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $BUILD_DIR && $ACTIVATE_TOOLCHAIN "'&& ctest -R memgraph__unit --output-on-failure -j$threads'
|
||||
;;
|
||||
unit-coverage)
|
||||
local setup_lsan_ubsan="export LSAN_OPTIONS=suppressions=$BUILD_DIR/../tools/lsan.supp && export UBSAN_OPTIONS=halt_on_error=1"
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $BUILD_DIR && $ACTIVATE_TOOLCHAIN && $setup_lsan_ubsan "'&& ctest -R memgraph__unit --output-on-failure -j2'
|
||||
;;
|
||||
leftover-CTest)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $BUILD_DIR && $ACTIVATE_TOOLCHAIN "'&& ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure'
|
||||
;;
|
||||
drivers)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR "'&& ./tests/drivers/run.sh'
|
||||
;;
|
||||
drivers-high-availability)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR "'&& ./tests/drivers/run_cluster.sh'
|
||||
;;
|
||||
integration)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR "'&& tests/integration/run.sh'
|
||||
;;
|
||||
cppcheck-and-clang-format)
|
||||
local test_output_path="$MGBUILD_ROOT_DIR/tools/github/cppcheck_and_clang_format.txt"
|
||||
local test_output_host_dest="$PROJECT_ROOT/tools/github/cppcheck_and_clang_format.txt"
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tools/github && $ACTIVATE_TOOLCHAIN "'&& ./cppcheck_and_clang_format diff'
|
||||
docker cp $build_container:$test_output_path $test_output_host_dest
|
||||
;;
|
||||
stress-plain)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& ./continuous_integration'
|
||||
;;
|
||||
stress-ssl)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& ./continuous_integration --use-ssl'
|
||||
;;
|
||||
durability)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/stress && source ve3/bin/activate "'&& python3 durability --num-steps 5'
|
||||
;;
|
||||
gql-behave)
|
||||
local test_output_dir="$MGBUILD_ROOT_DIR/tests/gql_behave"
|
||||
local test_output_host_dest="$PROJECT_ROOT/tests/gql_behave"
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests && $ACTIVATE_VENV && cd $MGBUILD_ROOT_DIR/tests/gql_behave "'&& ./continuous_integration'
|
||||
docker cp $build_container:$test_output_dir/gql_behave_status.csv $test_output_host_dest/gql_behave_status.csv
|
||||
docker cp $build_container:$test_output_dir/gql_behave_status.html $test_output_host_dest/gql_behave_status.html
|
||||
;;
|
||||
macro-benchmark)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && export USER=mg && export LANG=$(echo $LANG) && cd $MGBUILD_ROOT_DIR/tests/macro_benchmark "'&& ./harness QuerySuite MemgraphRunner --groups aggregation 1000_create unwind_create dense_expand match --no-strict'
|
||||
;;
|
||||
mgbench)
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/mgbench "'&& ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*'
|
||||
;;
|
||||
upload-to-bench-graph)
|
||||
shift 1
|
||||
local SETUP_PASSED_ARGS="export PASSED_ARGS=\"$@\""
|
||||
local SETUP_VE3_ENV="virtualenv -p python3 ve3 && source ve3/bin/activate && pip install -r requirements.txt"
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tools/bench-graph-client && $SETUP_VE3_ENV && $SETUP_PASSED_ARGS "'&& ./main.py $PASSED_ARGS'
|
||||
;;
|
||||
code-analysis)
|
||||
shift 1
|
||||
local SETUP_PASSED_ARGS="export PASSED_ARGS=\"$@\""
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && cd $MGBUILD_ROOT_DIR/tests/code_analysis && $SETUP_PASSED_ARGS "'&& ./python_code_analysis.sh $PASSED_ARGS'
|
||||
;;
|
||||
code-coverage)
|
||||
local test_output_path="$MGBUILD_ROOT_DIR/tools/github/generated/code_coverage.tar.gz"
|
||||
local test_output_host_dest="$PROJECT_ROOT/tools/github/generated/code_coverage.tar.gz"
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && $ACTIVATE_TOOLCHAIN && cd $MGBUILD_ROOT_DIR/tools/github "'&& ./coverage_convert'
|
||||
docker exec -u mg $build_container bash -c "cd $MGBUILD_ROOT_DIR/tools/github/generated && tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu"
|
||||
mkdir -p $PROJECT_ROOT/tools/github/generated
|
||||
docker cp $build_container:$test_output_path $test_output_host_dest
|
||||
;;
|
||||
clang-tidy)
|
||||
shift 1
|
||||
local SETUP_PASSED_ARGS="export PASSED_ARGS=\"$@\""
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && export THREADS=$threads && $ACTIVATE_TOOLCHAIN && cd $MGBUILD_ROOT_DIR/tests/code_analysis && $SETUP_PASSED_ARGS "'&& ./clang_tidy.sh $PASSED_ARGS'
|
||||
;;
|
||||
e2e)
|
||||
# local kafka_container="kafka_kafka_1"
|
||||
# local kafka_hostname="kafka"
|
||||
# local pulsar_container="pulsar_pulsar_1"
|
||||
# local pulsar_hostname="pulsar"
|
||||
# local setup_hostnames="export KAFKA_HOSTNAME=$kafka_hostname && PULSAR_HOSTNAME=$pulsar_hostname"
|
||||
# local build_container_network=$(docker inspect $build_container --format='{{ .HostConfig.NetworkMode }}')
|
||||
# docker network connect --alias $kafka_hostname $build_container_network $kafka_container > /dev/null 2>&1 || echo "Kafka container already inside correct network or something went wrong ..."
|
||||
# docker network connect --alias $pulsar_hostname $build_container_network $pulsar_container > /dev/null 2>&1 || echo "Kafka container already inside correct network or something went wrong ..."
|
||||
docker exec -u mg $build_container bash -c "pip install --user networkx && pip3 install --user networkx"
|
||||
docker exec -u mg $build_container bash -c "$EXPORT_LICENSE && $EXPORT_ORG_NAME && $ACTIVATE_CARGO && cd $MGBUILD_ROOT_DIR/tests && $ACTIVATE_VENV && source ve3/bin/activate_e2e && cd $MGBUILD_ROOT_DIR/tests/e2e "'&& ./run.sh'
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown test '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
|
||||
##################################################
|
||||
################### PARSE ARGS ###################
|
||||
##################################################
|
||||
if [ "$#" -eq 0 ] || [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
|
||||
print_help
|
||||
exit 0
|
||||
fi
|
||||
arch=$DEFAULT_ARCH
|
||||
build_type=$DEFAULT_BUILD_TYPE
|
||||
enterprise_license=$DEFAULT_ENTERPRISE_LICENSE
|
||||
organization_name=$DEFAULT_ORGANIZATION_NAME
|
||||
os=$DEFAULT_OS
|
||||
threads=$DEFAULT_THREADS
|
||||
toolchain_version=$DEFAULT_TOOLCHAIN
|
||||
command=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--arch)
|
||||
arch=$2
|
||||
check_support arch $arch
|
||||
shift 2
|
||||
;;
|
||||
--build-type)
|
||||
build_type=$2
|
||||
check_support build_type $build_type
|
||||
shift 2
|
||||
;;
|
||||
--enterprise-license)
|
||||
enterprise_license=$2
|
||||
shift 2
|
||||
;;
|
||||
--organization-name)
|
||||
organization_name=$2
|
||||
shift 2
|
||||
;;
|
||||
--os)
|
||||
os=$2
|
||||
check_support os $os
|
||||
shift 2
|
||||
;;
|
||||
--threads)
|
||||
threads=$2
|
||||
shift 2
|
||||
;;
|
||||
--toolchain)
|
||||
toolchain_version=$2
|
||||
check_support toolchain $toolchain_version
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
if [[ "$1" =~ ^--.* ]]; then
|
||||
echo -e "Error: Unknown option '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
else
|
||||
command=$1
|
||||
shift 1
|
||||
break
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ "$os" != "all" ]]; then
|
||||
check_support os_toolchain_combo $os $toolchain_version
|
||||
fi
|
||||
|
||||
if [[ "$command" == "" ]]; then
|
||||
echo -e "Error: Command not provided, please provide command"
|
||||
print_help
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if docker compose version > /dev/null 2>&1; then
|
||||
docker_compose_cmd="docker compose"
|
||||
elif which docker-compose > /dev/null 2>&1; then
|
||||
docker_compose_cmd="docker-compose"
|
||||
else
|
||||
echo -e "Missing command: There has to be installed either 'docker-compose' or 'docker compose'"
|
||||
exit 1
|
||||
fi
|
||||
echo "Using $docker_compose_cmd"
|
||||
|
||||
##################################################
|
||||
################# PARSE COMMAND ##################
|
||||
##################################################
|
||||
case $command in
|
||||
build)
|
||||
cd $SCRIPT_DIR
|
||||
git_ref_flag=""
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--git-ref)
|
||||
git_ref_flag="--build-arg GIT_REF=$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown flag '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ "$os" == "all" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml build $git_ref_flag
|
||||
else
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml build $git_ref_flag mgbuild_${toolchain_version}_${os}
|
||||
fi
|
||||
;;
|
||||
run)
|
||||
cd $SCRIPT_DIR
|
||||
pull=false
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--pull)
|
||||
pull=true
|
||||
shift 1
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown flag '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ "$os" == "all" ]]; then
|
||||
if [[ "$pull" == "true" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml pull --ignore-pull-failures
|
||||
elif [[ "$docker_compose_cmd" == "docker compose" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml pull --ignore-pull-failures --policy missing
|
||||
fi
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml up -d
|
||||
else
|
||||
if [[ "$pull" == "true" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml pull mgbuild_${toolchain_version}_${os}
|
||||
elif ! docker image inspect memgraph/mgbuild:${toolchain_version}_${os} > /dev/null 2>&1; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml pull --ignore-pull-failures mgbuild_${toolchain_version}_${os}
|
||||
fi
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml up -d mgbuild_${toolchain_version}_${os}
|
||||
fi
|
||||
;;
|
||||
stop)
|
||||
cd $SCRIPT_DIR
|
||||
remove=false
|
||||
while [[ "$#" -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--remove)
|
||||
remove=true
|
||||
shift 1
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown flag '$1'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ "$os" == "all" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml down
|
||||
else
|
||||
docker stop mgbuild_${toolchain_version}_${os}
|
||||
if [[ "$remove" == "true" ]]; then
|
||||
docker rm mgbuild_${toolchain_version}_${os}
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
pull)
|
||||
cd $SCRIPT_DIR
|
||||
if [[ "$os" == "all" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml pull --ignore-pull-failures
|
||||
else
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml pull mgbuild_${toolchain_version}_${os}
|
||||
fi
|
||||
;;
|
||||
push)
|
||||
docker login $@
|
||||
cd $SCRIPT_DIR
|
||||
if [[ "$os" == "all" ]]; then
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml push --ignore-push-failures
|
||||
else
|
||||
$docker_compose_cmd -f ${arch}-builders-${toolchain_version}.yml push mgbuild_${toolchain_version}_${os}
|
||||
fi
|
||||
;;
|
||||
build-memgraph)
|
||||
build_memgraph $@
|
||||
;;
|
||||
package-memgraph)
|
||||
package_memgraph
|
||||
;;
|
||||
test-memgraph)
|
||||
test_memgraph $@
|
||||
;;
|
||||
copy)
|
||||
copy_memgraph $@
|
||||
;;
|
||||
package-docker)
|
||||
package_docker
|
||||
;;
|
||||
*)
|
||||
echo "Error: Unknown command '$command'"
|
||||
print_help
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1,38 +0,0 @@
|
||||
FROM rockylinux:9.3
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
RUN yum -y update \
|
||||
&& yum install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-rocky-9.3-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-rocky-9.3-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-rocky-9.3-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-rocky-9.3-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/rocky-9.3.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/rocky-9.3.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
|
||||
# Install PyYAML (only for amzn-2, centos-7, cento-9 and rocky-9.3)
|
||||
RUN pip3 install --user PyYAML
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
208
release/package/run.sh
Executable file
208
release/package/run.sh
Executable file
@@ -0,0 +1,208 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
SUPPORTED_OS=(
|
||||
centos-7 centos-9
|
||||
debian-10 debian-11 debian-11-arm
|
||||
ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
|
||||
fedora-36
|
||||
amzn-2
|
||||
)
|
||||
|
||||
SUPPORTED_BUILD_TYPES=(
|
||||
Debug
|
||||
Release
|
||||
RelWithDebInfo
|
||||
)
|
||||
|
||||
PROJECT_ROOT="$SCRIPT_DIR/../.."
|
||||
TOOLCHAIN_VERSION="toolchain-v4"
|
||||
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
|
||||
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
|
||||
|
||||
print_help () {
|
||||
# TODO(gitbuda): Update the release/package/run.sh help
|
||||
echo "$0 init|package|docker|test {os} {build_type} [--for-docker|--for-platform]"
|
||||
echo ""
|
||||
echo " OSs: ${SUPPORTED_OS[*]}"
|
||||
echo " Build types: ${SUPPORTED_BUILD_TYPES[*]}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
make_package () {
|
||||
os="$1"
|
||||
build_type="$2"
|
||||
|
||||
build_container="mgbuild_$os"
|
||||
echo "Building Memgraph for $os on $build_container..."
|
||||
|
||||
package_command=""
|
||||
if [[ "$os" =~ ^"centos".* ]] || [[ "$os" =~ ^"fedora".* ]] || [[ "$os" =~ ^"amzn".* ]]; then
|
||||
docker exec "$build_container" bash -c "yum -y update"
|
||||
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
|
||||
fi
|
||||
if [[ "$os" =~ ^"debian".* ]]; then
|
||||
docker exec "$build_container" bash -c "apt --allow-releaseinfo-change -y update"
|
||||
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
|
||||
fi
|
||||
if [[ "$os" =~ ^"ubuntu".* ]]; then
|
||||
docker exec "$build_container" bash -c "apt update"
|
||||
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
|
||||
fi
|
||||
telemetry_id_override_flag=""
|
||||
if [[ "$#" -gt 2 ]]; then
|
||||
if [[ "$3" == "--for-docker" ]]; then
|
||||
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER "
|
||||
elif [[ "$3" == "--for-platform" ]]; then
|
||||
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM"
|
||||
else
|
||||
print_help
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Copying project files..."
|
||||
# If master is not the current branch, fetch it, because the get_version
|
||||
# script depends on it. If we are on master, the fetch command is going to
|
||||
# fail so that's why there is the explicit check.
|
||||
# Required here because Docker build container can't access remote.
|
||||
cd "$PROJECT_ROOT"
|
||||
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
|
||||
git fetch origin master:master
|
||||
fi
|
||||
|
||||
# Ensure we have a clean build directory
|
||||
docker exec "$build_container" rm -rf /memgraph
|
||||
|
||||
docker exec "$build_container" mkdir -p /memgraph
|
||||
# TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI.
|
||||
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
|
||||
|
||||
container_build_dir="/memgraph/build"
|
||||
container_output_dir="$container_build_dir/output"
|
||||
|
||||
# TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
|
||||
# image build phase, but that is not easy at this point because the
|
||||
# environment/os/{os}.sh does not come within the toolchain package. When
|
||||
# migrating to the next version of toolchain do that, and remove the
|
||||
# TOOLCHAIN_RUN_DEPS installation from here.
|
||||
# TODO(gitbuda): On the other side, having this here allows updating deps
|
||||
# wihout reruning the build containers.
|
||||
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
|
||||
|
||||
echo "Building targeted package..."
|
||||
# Fix issue with git marking directory as not safe
|
||||
docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'"
|
||||
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
|
||||
# TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX
|
||||
if [[ "$os" =~ "-arm" ]]; then
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=$build_type -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
|
||||
else
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=$build_type $telemetry_id_override_flag .."
|
||||
fi
|
||||
# ' is used instead of " because we need to run make within the allowed
|
||||
# container resources.
|
||||
# shellcheck disable=SC2016
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc)'
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc) -B mgconsole'
|
||||
docker exec "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
|
||||
|
||||
echo "Copying targeted package to host..."
|
||||
last_package_name=$(docker exec "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
|
||||
# The operating system folder is introduced because multiple different
|
||||
# packages could be preserved during the same build "session".
|
||||
mkdir -p "$HOST_OUTPUT_DIR/$os"
|
||||
package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
|
||||
docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
|
||||
echo "Package saved to $package_host_destination."
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
init)
|
||||
cd "$SCRIPT_DIR"
|
||||
if ! which "docker-compose" >/dev/null; then
|
||||
docker_compose_cmd="docker compose"
|
||||
else
|
||||
docker_compose_cmd="docker-compose"
|
||||
fi
|
||||
$docker_compose_cmd build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
|
||||
$docker_compose_cmd up -d
|
||||
;;
|
||||
|
||||
docker)
|
||||
# NOTE: Docker is build on top of Debian 11 package.
|
||||
based_on_os="debian-11"
|
||||
# shellcheck disable=SC2012
|
||||
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
|
||||
docker_build_folder="$PROJECT_ROOT/release/docker"
|
||||
cd "$docker_build_folder"
|
||||
./package_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
|
||||
# shellcheck disable=SC2012
|
||||
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
|
||||
docker_host_folder="$HOST_OUTPUT_DIR/docker"
|
||||
docker_host_image_path="$docker_host_folder/$docker_image_name"
|
||||
mkdir -p "$docker_host_folder"
|
||||
cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
|
||||
echo "Docker images saved to $docker_host_image_path."
|
||||
;;
|
||||
|
||||
package)
|
||||
shift 1
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
print_help
|
||||
fi
|
||||
os="$1"
|
||||
build_type="$2"
|
||||
shift 2
|
||||
is_os_ok=false
|
||||
for supported_os in "${SUPPORTED_OS[@]}"; do
|
||||
if [[ "$supported_os" == "${os}" ]]; then
|
||||
is_os_ok=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
is_build_type_ok=false
|
||||
for supported_build_type in "${SUPPORTED_BUILD_TYPES[@]}"; do
|
||||
if [[ "$supported_build_type" == "${build_type}" ]]; then
|
||||
is_build_type_ok=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$is_os_ok" == true && "$is_build_type_ok" == true ]]; then
|
||||
make_package "$os" "$build_type" "$@"
|
||||
else
|
||||
if [[ "$is_os_ok" == false ]]; then
|
||||
echo "Unsupported OS: $os"
|
||||
elif [[ "$is_build_type_ok" == false ]]; then
|
||||
echo "Unsupported build type: $build_type"
|
||||
fi
|
||||
print_help
|
||||
fi
|
||||
;;
|
||||
|
||||
build)
|
||||
shift 1
|
||||
if [[ "$#" -ne 2 ]]; then
|
||||
print_help
|
||||
fi
|
||||
# in the vX format, e.g. v5
|
||||
toolchain_version="$1"
|
||||
# a name of the os folder, e.g. ubuntu-22.04-arm
|
||||
os="$2"
|
||||
cd "$SCRIPT_DIR/$os"
|
||||
docker build -f Dockerfile --build-arg TOOLCHAIN_VERSION="toolchain-$toolchain_version" -t "memgraph/memgraph-builder:${toolchain_version}_$os" .
|
||||
;;
|
||||
|
||||
test)
|
||||
echo "TODO(gitbuda): Test all packages on mgtest containers."
|
||||
;;
|
||||
|
||||
*)
|
||||
print_help
|
||||
;;
|
||||
esac
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/ubuntu-18.04.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/ubuntu-18.04.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/ubuntu-20.04.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/ubuntu-20.04.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git --branch ${GIT_REF} clone https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/ubuntu-22.04-arm.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/ubuntu-22.04-arm.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,28 +10,9 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-${TOOLCHAIN_VERSION}/toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
-O toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
&& tar xzvf toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt \
|
||||
&& rm toolchain-${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz
|
||||
|
||||
# Install toolchain run deps and memgraph build deps
|
||||
ARG GIT_REF=master
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN git clone --branch ${GIT_REF} https://github.com/memgraph/memgraph.git \
|
||||
&& cd memgraph \
|
||||
&& ./environment/os/ubuntu-22.04.sh install TOOLCHAIN_RUN_DEPS \
|
||||
&& ./environment/os/ubuntu-22.04.sh install MEMGRAPH_BUILD_DEPS \
|
||||
&& cd .. && rm -rf memgraph
|
||||
|
||||
# Create mg user and set as default
|
||||
RUN useradd -m -s /bin/bash mg
|
||||
USER mg
|
||||
|
||||
# Install rust
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
|
||||
# Fix node
|
||||
RUN curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt \
|
||||
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -42,10 +42,6 @@ BuildRequires: systemd
|
||||
%{nil}
|
||||
%endif
|
||||
|
||||
%if 0%{?fedora} && 0%{?fedora} > 37
|
||||
%global __brp_check_rpaths %{nil}
|
||||
%endif
|
||||
|
||||
%if 0%{?fedora} && 0%{?fedora} < 35
|
||||
%global __os_install_post \
|
||||
/usr/lib/rpm/brp-compress \
|
||||
|
||||
@@ -45,7 +45,7 @@ set(mg_single_node_v2_sources
|
||||
add_executable(memgraph ${mg_single_node_v2_sources})
|
||||
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(memgraph stdc++fs Threads::Threads
|
||||
mg-telemetry mgcxx_text_search tantivy_text_search mg-communication mg-communication-metrics mg-memory mg-utils mg-license mg-settings mg-glue mg-flags mg::system mg::replication_handler)
|
||||
mg-telemetry mg-communication mg-communication-metrics mg-memory mg-utils mg-license mg-settings mg-glue mg-flags mg::system mg::replication_handler)
|
||||
|
||||
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
|
||||
# should be dynamically exported, so that `dlopen` can correctly link th
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -88,12 +88,6 @@ class Session {
|
||||
|
||||
virtual void Configure(const std::map<std::string, memgraph::communication::bolt::Value> &run_time_info) = 0;
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
virtual auto Route(std::map<std::string, Value> const &routing,
|
||||
std::vector<memgraph::communication::bolt::Value> const &bookmarks,
|
||||
std::map<std::string, Value> const &extra) -> std::map<std::string, Value> = 0;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Put results of the processed query in the `encoder`.
|
||||
*
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -79,9 +79,9 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
|
||||
}
|
||||
case Signature::Route: {
|
||||
if constexpr (bolt_minor >= 3) {
|
||||
return HandleRoute<TSession>(session, marker);
|
||||
if (signature == Signature::Route) return HandleRoute<TSession>(session, marker);
|
||||
} else {
|
||||
spdlog::trace("Supported only in bolt versions >= 4.3");
|
||||
spdlog::trace("Supported only in bolt v4.3");
|
||||
return State::Close;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -478,6 +478,9 @@ State HandleGoodbye() {
|
||||
|
||||
template <typename TSession>
|
||||
State HandleRoute(TSession &session, const Marker marker) {
|
||||
// Route message is not implemented since it is Neo4j specific, therefore we will receive it and inform user that
|
||||
// there is no implementation. Before that, we have to read out the fields from the buffer to leave it in a clean
|
||||
// state.
|
||||
if (marker != Marker::TinyStruct3) {
|
||||
spdlog::trace("Expected TinyStruct3 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
|
||||
return State::Close;
|
||||
@@ -493,27 +496,11 @@ State HandleRoute(TSession &session, const Marker marker) {
|
||||
spdlog::trace("Couldn't read bookmarks field!");
|
||||
return State::Close;
|
||||
}
|
||||
|
||||
// TODO: (andi) Fix Bolt versions
|
||||
Value db;
|
||||
if (!session.decoder_.ReadValue(&db)) {
|
||||
spdlog::trace("Couldn't read db field!");
|
||||
return State::Close;
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
try {
|
||||
auto res = session.Route(routing.ValueMap(), bookmarks.ValueList(), {});
|
||||
if (!session.encoder_.MessageSuccess(std::move(res))) {
|
||||
spdlog::trace("Couldn't send result of routing!");
|
||||
return State::Close;
|
||||
}
|
||||
return State::Idle;
|
||||
} catch (const std::exception &e) {
|
||||
return HandleFailure(session, e);
|
||||
}
|
||||
|
||||
#else
|
||||
session.encoder_buffer_.Clear();
|
||||
bool fail_sent =
|
||||
session.encoder_.MessageFailure({{"code", "66"}, {"message", "Route message is not supported in Memgraph!"}});
|
||||
@@ -522,7 +509,6 @@ State HandleRoute(TSession &session, const Marker marker) {
|
||||
return State::Close;
|
||||
}
|
||||
return State::Error;
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename TSession>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -6,7 +6,7 @@ target_sources(mg-coordination
|
||||
include/coordination/coordinator_state.hpp
|
||||
include/coordination/coordinator_rpc.hpp
|
||||
include/coordination/coordinator_server.hpp
|
||||
include/coordination/coordinator_communication_config.hpp
|
||||
include/coordination/coordinator_config.hpp
|
||||
include/coordination/coordinator_exceptions.hpp
|
||||
include/coordination/coordinator_slk.hpp
|
||||
include/coordination/coordinator_instance.hpp
|
||||
@@ -23,7 +23,7 @@ target_sources(mg-coordination
|
||||
include/nuraft/coordinator_state_manager.hpp
|
||||
|
||||
PRIVATE
|
||||
coordinator_communication_config.cpp
|
||||
coordinator_config.cpp
|
||||
coordinator_client.cpp
|
||||
coordinator_state.cpp
|
||||
coordinator_rpc.cpp
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
#include "coordination/coordinator_client.hpp"
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/coordinator_rpc.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "replication_coordination_glue/messages.hpp"
|
||||
@@ -23,17 +23,18 @@
|
||||
namespace memgraph::coordination {
|
||||
|
||||
namespace {
|
||||
auto CreateClientContext(memgraph::coordination::CoordinatorToReplicaConfig const &config)
|
||||
auto CreateClientContext(memgraph::coordination::CoordinatorClientConfig const &config)
|
||||
-> communication::ClientContext {
|
||||
return (config.ssl) ? communication::ClientContext{config.ssl->key_file, config.ssl->cert_file}
|
||||
: communication::ClientContext{};
|
||||
}
|
||||
} // namespace
|
||||
|
||||
CoordinatorClient::CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorToReplicaConfig config,
|
||||
CoordinatorClient::CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorClientConfig config,
|
||||
HealthCheckClientCallback succ_cb, HealthCheckClientCallback fail_cb)
|
||||
: rpc_context_{CreateClientContext(config)},
|
||||
rpc_client_{config.mgt_server, &rpc_context_},
|
||||
rpc_client_{io::network::Endpoint(io::network::Endpoint::needs_resolving, config.ip_address, config.port),
|
||||
&rpc_context_},
|
||||
config_{std::move(config)},
|
||||
coord_instance_{coord_instance},
|
||||
succ_cb_{std::move(succ_cb)},
|
||||
@@ -85,9 +86,7 @@ void CoordinatorClient::StopFrequentCheck() { instance_checker_.Stop(); }
|
||||
void CoordinatorClient::PauseFrequentCheck() { instance_checker_.Pause(); }
|
||||
void CoordinatorClient::ResumeFrequentCheck() { instance_checker_.Resume(); }
|
||||
|
||||
auto CoordinatorClient::ReplicationClientInfo() const -> coordination::ReplicationClientInfo {
|
||||
return config_.replication_client_info;
|
||||
}
|
||||
auto CoordinatorClient::ReplicationClientInfo() const -> ReplClientInfo { return config_.replication_client_info; }
|
||||
|
||||
auto CoordinatorClient::SendPromoteReplicaToMainRpc(const utils::UUID &uuid,
|
||||
ReplicationClientsInfo replication_clients_info) const -> bool {
|
||||
@@ -136,7 +135,8 @@ auto CoordinatorClient::SendSwapMainUUIDRpc(utils::UUID const &uuid) const -> bo
|
||||
|
||||
auto CoordinatorClient::SendUnregisterReplicaRpc(std::string_view instance_name) const -> bool {
|
||||
try {
|
||||
auto stream{rpc_client_.Stream<UnregisterReplicaRpc>(instance_name)};
|
||||
auto stream{rpc_client_.Stream<UnregisterReplicaRpc>(
|
||||
std::string(instance_name))}; // TODO: (andi) Try to change to stream string_view and do just one copy later
|
||||
if (!stream.AwaitResponse().success) {
|
||||
spdlog::error("Failed to receive successful RPC response for unregistering replica!");
|
||||
return false;
|
||||
|
||||
@@ -18,214 +18,148 @@
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
void to_json(nlohmann::json &j, ReplicationInstanceState const &instance_state) {
|
||||
j = nlohmann::json{
|
||||
{"config", instance_state.config}, {"status", instance_state.status}, {"uuid", instance_state.instance_uuid}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, ReplicationInstanceState &instance_state) {
|
||||
j.at("config").get_to(instance_state.config);
|
||||
j.at("status").get_to(instance_state.status);
|
||||
j.at("uuid").get_to(instance_state.instance_uuid);
|
||||
}
|
||||
|
||||
CoordinatorClusterState::CoordinatorClusterState(std::map<std::string, ReplicationInstanceState, std::less<>> instances,
|
||||
utils::UUID const ¤t_main_uuid, bool is_lock_opened)
|
||||
: repl_instances_{std::move(instances)}, current_main_uuid_(current_main_uuid), is_lock_opened_(is_lock_opened) {}
|
||||
using replication_coordination_glue::ReplicationRole;
|
||||
|
||||
CoordinatorClusterState::CoordinatorClusterState(CoordinatorClusterState const &other)
|
||||
: repl_instances_{other.repl_instances_},
|
||||
current_main_uuid_(other.current_main_uuid_),
|
||||
is_lock_opened_(other.is_lock_opened_) {}
|
||||
: instance_roles_{other.instance_roles_} {}
|
||||
|
||||
CoordinatorClusterState &CoordinatorClusterState::operator=(CoordinatorClusterState const &other) {
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
repl_instances_ = other.repl_instances_;
|
||||
current_main_uuid_ = other.current_main_uuid_;
|
||||
is_lock_opened_ = other.is_lock_opened_;
|
||||
instance_roles_ = other.instance_roles_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
CoordinatorClusterState::CoordinatorClusterState(CoordinatorClusterState &&other) noexcept
|
||||
: repl_instances_{std::move(other.repl_instances_)},
|
||||
current_main_uuid_(other.current_main_uuid_),
|
||||
is_lock_opened_(other.is_lock_opened_) {}
|
||||
: instance_roles_{std::move(other.instance_roles_)} {}
|
||||
|
||||
CoordinatorClusterState &CoordinatorClusterState::operator=(CoordinatorClusterState &&other) noexcept {
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
repl_instances_ = std::move(other.repl_instances_);
|
||||
current_main_uuid_ = other.current_main_uuid_;
|
||||
is_lock_opened_ = other.is_lock_opened_;
|
||||
instance_roles_ = std::move(other.instance_roles_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::MainExists() const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
return std::ranges::any_of(repl_instances_,
|
||||
[](auto const &entry) { return entry.second.status == ReplicationRole::MAIN; });
|
||||
return std::ranges::any_of(instance_roles_,
|
||||
[](auto const &entry) { return entry.second.role == ReplicationRole::MAIN; });
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::HasMainState(std::string_view instance_name) const -> bool {
|
||||
auto CoordinatorClusterState::IsMain(std::string_view instance_name) const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it = repl_instances_.find(instance_name);
|
||||
return it != repl_instances_.end() && it->second.status == ReplicationRole::MAIN;
|
||||
auto const it = instance_roles_.find(instance_name);
|
||||
return it != instance_roles_.end() && it->second.role == ReplicationRole::MAIN;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::HasReplicaState(std::string_view instance_name) const -> bool {
|
||||
auto CoordinatorClusterState::IsReplica(std::string_view instance_name) const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it = repl_instances_.find(instance_name);
|
||||
return it != repl_instances_.end() && it->second.status == ReplicationRole::REPLICA;
|
||||
auto const it = instance_roles_.find(instance_name);
|
||||
return it != instance_roles_.end() && it->second.role == ReplicationRole::REPLICA;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::IsCurrentMain(std::string_view instance_name) const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it = repl_instances_.find(instance_name);
|
||||
return it != repl_instances_.end() && it->second.status == ReplicationRole::MAIN &&
|
||||
it->second.instance_uuid == current_main_uuid_;
|
||||
auto CoordinatorClusterState::InsertInstance(std::string_view instance_name, ReplicationRole role) -> void {
|
||||
auto lock = std::unique_lock{log_lock_};
|
||||
instance_roles_[instance_name.data()].role = role;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::DoAction(TRaftLog log_entry, RaftLogAction log_action) -> void {
|
||||
auto lock = std::lock_guard{log_lock_};
|
||||
auto lock = std::unique_lock{log_lock_};
|
||||
switch (log_action) {
|
||||
// end of OPEN_LOCK_REGISTER_REPLICATION_INSTANCE
|
||||
case RaftLogAction::REGISTER_REPLICATION_INSTANCE: {
|
||||
auto const &config = std::get<CoordinatorToReplicaConfig>(log_entry);
|
||||
spdlog::trace("DoAction: register replication instance {}", config.instance_name);
|
||||
// Setting instance uuid to random, if registration fails, we are still in random state
|
||||
repl_instances_.emplace(config.instance_name,
|
||||
ReplicationInstanceState{config, ReplicationRole::REPLICA, utils::UUID{}});
|
||||
is_lock_opened_ = false;
|
||||
auto const &config = std::get<CoordinatorClientConfig>(log_entry);
|
||||
instance_roles_[config.instance_name] = InstanceState{config, ReplicationRole::REPLICA};
|
||||
break;
|
||||
}
|
||||
// end of OPEN_LOCK_UNREGISTER_REPLICATION_INSTANCE
|
||||
case RaftLogAction::UNREGISTER_REPLICATION_INSTANCE: {
|
||||
auto const instance_name = std::get<std::string>(log_entry);
|
||||
spdlog::trace("DoAction: unregister replication instance {}", instance_name);
|
||||
repl_instances_.erase(instance_name);
|
||||
is_lock_opened_ = false;
|
||||
instance_roles_.erase(instance_name);
|
||||
break;
|
||||
}
|
||||
// end of OPEN_LOCK_SET_INSTANCE_AS_MAIN and OPEN_LOCK_FAILOVER
|
||||
case RaftLogAction::SET_INSTANCE_AS_MAIN: {
|
||||
auto const instance_uuid_change = std::get<InstanceUUIDUpdate>(log_entry);
|
||||
auto it = repl_instances_.find(instance_uuid_change.instance_name);
|
||||
MG_ASSERT(it != repl_instances_.end(), "Instance does not exist as part of raft state!");
|
||||
it->second.status = ReplicationRole::MAIN;
|
||||
it->second.instance_uuid = instance_uuid_change.uuid;
|
||||
is_lock_opened_ = false;
|
||||
spdlog::trace("DoAction: set replication instance {} as main with uuid {}", instance_uuid_change.instance_name,
|
||||
std::string{instance_uuid_change.uuid});
|
||||
auto const instance_name = std::get<std::string>(log_entry);
|
||||
auto it = instance_roles_.find(instance_name);
|
||||
MG_ASSERT(it != instance_roles_.end(), "Instance does not exist as part of raft state!");
|
||||
it->second.role = ReplicationRole::MAIN;
|
||||
break;
|
||||
}
|
||||
// end of OPEN_LOCK_SET_INSTANCE_AS_REPLICA
|
||||
case RaftLogAction::SET_INSTANCE_AS_REPLICA: {
|
||||
auto const instance_name = std::get<std::string>(log_entry);
|
||||
auto it = repl_instances_.find(instance_name);
|
||||
MG_ASSERT(it != repl_instances_.end(), "Instance does not exist as part of raft state!");
|
||||
it->second.status = ReplicationRole::REPLICA;
|
||||
is_lock_opened_ = false;
|
||||
spdlog::trace("DoAction: set replication instance {} as replica", instance_name);
|
||||
auto it = instance_roles_.find(instance_name);
|
||||
MG_ASSERT(it != instance_roles_.end(), "Instance does not exist as part of raft state!");
|
||||
it->second.role = ReplicationRole::REPLICA;
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::UPDATE_UUID_OF_NEW_MAIN: {
|
||||
current_main_uuid_ = std::get<utils::UUID>(log_entry);
|
||||
spdlog::trace("DoAction: update uuid of new main {}", std::string{current_main_uuid_});
|
||||
case RaftLogAction::UPDATE_UUID: {
|
||||
uuid_ = std::get<utils::UUID>(log_entry);
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::UPDATE_UUID_FOR_INSTANCE: {
|
||||
auto const instance_uuid_change = std::get<InstanceUUIDUpdate>(log_entry);
|
||||
auto it = repl_instances_.find(instance_uuid_change.instance_name);
|
||||
MG_ASSERT(it != repl_instances_.end(), "Instance doesn't exist as part of RAFT state");
|
||||
it->second.instance_uuid = instance_uuid_change.uuid;
|
||||
spdlog::trace("DoAction: update uuid for instance {} to {}", instance_uuid_change.instance_name,
|
||||
std::string{instance_uuid_change.uuid});
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::ADD_COORDINATOR_INSTANCE: {
|
||||
auto const &config = std::get<CoordinatorToCoordinatorConfig>(log_entry);
|
||||
coordinators_.emplace_back(CoordinatorInstanceState{config});
|
||||
spdlog::trace("DoAction: add coordinator instance {}", config.coordinator_server_id);
|
||||
break;
|
||||
}
|
||||
case RaftLogAction::OPEN_LOCK_REGISTER_REPLICATION_INSTANCE: {
|
||||
is_lock_opened_ = true;
|
||||
spdlog::trace("DoAction: open lock register");
|
||||
break;
|
||||
// TODO(antoniofilipovic) save what we are doing to be able to undo....
|
||||
}
|
||||
case RaftLogAction::OPEN_LOCK_UNREGISTER_REPLICATION_INSTANCE: {
|
||||
is_lock_opened_ = true;
|
||||
spdlog::trace("DoAction: open lock unregister");
|
||||
break;
|
||||
// TODO(antoniofilipovic) save what we are doing
|
||||
}
|
||||
case RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_MAIN: {
|
||||
is_lock_opened_ = true;
|
||||
spdlog::trace("DoAction: open lock set instance as main");
|
||||
break;
|
||||
// TODO(antoniofilipovic) save what we are doing
|
||||
}
|
||||
case RaftLogAction::OPEN_LOCK_FAILOVER: {
|
||||
is_lock_opened_ = true;
|
||||
spdlog::trace("DoAction: open lock failover");
|
||||
break;
|
||||
// TODO(antoniofilipovic) save what we are doing
|
||||
}
|
||||
case RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_REPLICA: {
|
||||
is_lock_opened_ = true;
|
||||
spdlog::trace("DoAction: open lock set instance as replica");
|
||||
break;
|
||||
// TODO(antoniofilipovic) save what we need to undo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: (andi) Improve based on Gareth's comments
|
||||
auto CoordinatorClusterState::Serialize(ptr<buffer> &data) -> void {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
nlohmann::json j = {{"repl_instances", repl_instances_},
|
||||
{"is_lock_opened", is_lock_opened_},
|
||||
{"current_main_uuid", current_main_uuid_}};
|
||||
auto const log = j.dump();
|
||||
data = buffer::alloc(sizeof(uint32_t) + log.size());
|
||||
auto const role_to_string = [](auto const &role) -> std::string_view {
|
||||
switch (role) {
|
||||
case ReplicationRole::MAIN:
|
||||
return "main";
|
||||
case ReplicationRole::REPLICA:
|
||||
return "replica";
|
||||
}
|
||||
};
|
||||
|
||||
auto const entry_to_string = [&role_to_string](auto const &entry) {
|
||||
return fmt::format("{}_{}", entry.first, role_to_string(entry.second.role));
|
||||
};
|
||||
|
||||
auto instances_str_view = instance_roles_ | ranges::views::transform(entry_to_string);
|
||||
uint32_t size =
|
||||
std::accumulate(instances_str_view.begin(), instances_str_view.end(), 0,
|
||||
[](uint32_t acc, auto const &entry) { return acc + sizeof(uint32_t) + entry.size(); });
|
||||
|
||||
data = buffer::alloc(size);
|
||||
buffer_serializer bs(data);
|
||||
bs.put_str(log);
|
||||
std::for_each(instances_str_view.begin(), instances_str_view.end(), [&bs](auto const &entry) { bs.put_str(entry); });
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::Deserialize(buffer &data) -> CoordinatorClusterState {
|
||||
auto const str_to_role = [](auto const &str) -> ReplicationRole {
|
||||
if (str == "main") {
|
||||
return ReplicationRole::MAIN;
|
||||
}
|
||||
return ReplicationRole::REPLICA;
|
||||
};
|
||||
|
||||
CoordinatorClusterState cluster_state;
|
||||
buffer_serializer bs(data);
|
||||
auto const j = nlohmann::json::parse(bs.get_str());
|
||||
auto instances = j["repl_instances"].get<std::map<std::string, ReplicationInstanceState, std::less<>>>();
|
||||
auto current_main_uuid = j["current_main_uuid"].get<utils::UUID>();
|
||||
bool is_lock_opened = j["is_lock_opened"].get<int>();
|
||||
return CoordinatorClusterState{std::move(instances), current_main_uuid, is_lock_opened};
|
||||
while (bs.size() > 0) {
|
||||
auto const entry = bs.get_str();
|
||||
auto const first_dash = entry.find('_');
|
||||
auto const instance_name = entry.substr(0, first_dash);
|
||||
auto const role_str = entry.substr(first_dash + 1);
|
||||
cluster_state.InsertInstance(instance_name, str_to_role(role_str));
|
||||
}
|
||||
return cluster_state;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::GetReplicationInstances() const -> std::vector<ReplicationInstanceState> {
|
||||
auto CoordinatorClusterState::GetInstances() const -> std::vector<InstanceState> {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
return repl_instances_ | ranges::views::values | ranges::to<std::vector<ReplicationInstanceState>>;
|
||||
return instance_roles_ | ranges::views::values | ranges::to<std::vector<InstanceState>>;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::GetCurrentMainUUID() const -> utils::UUID { return current_main_uuid_; }
|
||||
auto CoordinatorClusterState::GetUUID() const -> utils::UUID { return uuid_; }
|
||||
|
||||
auto CoordinatorClusterState::GetInstanceUUID(std::string_view instance_name) const -> utils::UUID {
|
||||
auto CoordinatorClusterState::FindCurrentMainInstanceName() const -> std::optional<std::string> {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
auto const it = repl_instances_.find(instance_name);
|
||||
MG_ASSERT(it != repl_instances_.end(), "Instance with that name doesn't exist.");
|
||||
return it->second.instance_uuid;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::GetCoordinatorInstances() const -> std::vector<CoordinatorInstanceState> {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
return coordinators_;
|
||||
}
|
||||
|
||||
auto CoordinatorClusterState::IsLockOpened() const -> bool {
|
||||
auto lock = std::shared_lock{log_lock_};
|
||||
return is_lock_opened_;
|
||||
auto const it = std::ranges::find_if(instance_roles_,
|
||||
[](auto const &entry) { return entry.second.role == ReplicationRole::MAIN; });
|
||||
if (it == instance_roles_.end()) {
|
||||
return {};
|
||||
}
|
||||
return it->first;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -11,62 +11,43 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorToCoordinatorConfig const &config) {
|
||||
j = nlohmann::json{{"coordinator_server_id", config.coordinator_server_id},
|
||||
{"coordinator_server", config.coordinator_server},
|
||||
{"bolt_server", config.bolt_server}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, CoordinatorToCoordinatorConfig &config) {
|
||||
config.coordinator_server_id = j.at("coordinator_server_id").get<uint32_t>();
|
||||
config.coordinator_server = j.at("coordinator_server").get<io::network::Endpoint>();
|
||||
config.bolt_server = j.at("bolt_server").get<io::network::Endpoint>();
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json &j, ReplicationClientInfo const &config) {
|
||||
void to_json(nlohmann::json &j, ReplClientInfo const &config) {
|
||||
j = nlohmann::json{{"instance_name", config.instance_name},
|
||||
{"replication_mode", config.replication_mode},
|
||||
{"replication_server", config.replication_server}};
|
||||
{"replication_ip_address", config.replication_ip_address},
|
||||
{"replication_port", config.replication_port}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, ReplicationClientInfo &config) {
|
||||
void from_json(nlohmann::json const &j, ReplClientInfo &config) {
|
||||
config.instance_name = j.at("instance_name").get<std::string>();
|
||||
config.replication_mode = j.at("replication_mode").get<replication_coordination_glue::ReplicationMode>();
|
||||
config.replication_server = j.at("replication_server").get<io::network::Endpoint>();
|
||||
config.replication_ip_address = j.at("replication_ip_address").get<std::string>();
|
||||
config.replication_port = j.at("replication_port").get<uint16_t>();
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorToReplicaConfig const &config) {
|
||||
void to_json(nlohmann::json &j, CoordinatorClientConfig const &config) {
|
||||
j = nlohmann::json{{"instance_name", config.instance_name},
|
||||
{"mgt_server", config.mgt_server},
|
||||
{"bolt_server", config.bolt_server},
|
||||
{"ip_address", config.ip_address},
|
||||
{"port", config.port},
|
||||
{"instance_health_check_frequency_sec", config.instance_health_check_frequency_sec.count()},
|
||||
{"instance_down_timeout_sec", config.instance_down_timeout_sec.count()},
|
||||
{"instance_get_uuid_frequency_sec", config.instance_get_uuid_frequency_sec.count()},
|
||||
{"replication_client_info", config.replication_client_info}};
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, CoordinatorToReplicaConfig &config) {
|
||||
void from_json(nlohmann::json const &j, CoordinatorClientConfig &config) {
|
||||
config.instance_name = j.at("instance_name").get<std::string>();
|
||||
config.mgt_server = j.at("mgt_server").get<io::network::Endpoint>();
|
||||
config.bolt_server = j.at("bolt_server").get<io::network::Endpoint>();
|
||||
config.ip_address = j.at("ip_address").get<std::string>();
|
||||
config.port = j.at("port").get<uint16_t>();
|
||||
config.instance_health_check_frequency_sec =
|
||||
std::chrono::seconds{j.at("instance_health_check_frequency_sec").get<int>()};
|
||||
config.instance_down_timeout_sec = std::chrono::seconds{j.at("instance_down_timeout_sec").get<int>()};
|
||||
config.instance_get_uuid_frequency_sec = std::chrono::seconds{j.at("instance_get_uuid_frequency_sec").get<int>()};
|
||||
config.replication_client_info = j.at("replication_client_info").get<ReplicationClientInfo>();
|
||||
}
|
||||
|
||||
void from_json(nlohmann::json const &j, InstanceUUIDUpdate &instance_uuid_change) {
|
||||
instance_uuid_change.uuid = j.at("uuid").get<utils::UUID>();
|
||||
instance_uuid_change.instance_name = j.at("instance_name").get<std::string>();
|
||||
}
|
||||
|
||||
void to_json(nlohmann::json &j, InstanceUUIDUpdate const &instance_uuid_change) {
|
||||
j = nlohmann::json{{"instance_name", instance_uuid_change.instance_name}, {"uuid", instance_uuid_change.uuid}};
|
||||
config.replication_client_info = j.at("replication_client_info").get<ReplClientInfo>();
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
@@ -95,8 +95,8 @@ void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHan
|
||||
slk::Load(&req, req_reader);
|
||||
|
||||
const replication::ReplicationServerConfig clients_config{
|
||||
.ip_address = req.replication_client_info.replication_server.address,
|
||||
.port = req.replication_client_info.replication_server.port};
|
||||
.ip_address = req.replication_client_info.replication_ip_address,
|
||||
.port = req.replication_client_info.replication_port};
|
||||
|
||||
if (!replication_handler.SetReplicationRoleReplica(clients_config, std::nullopt)) {
|
||||
spdlog::error("Demoting main to replica failed!");
|
||||
@@ -136,8 +136,8 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
|
||||
return replication::ReplicationClientConfig{
|
||||
.name = repl_info_config.instance_name,
|
||||
.mode = repl_info_config.replication_mode,
|
||||
.ip_address = repl_info_config.replication_server.address,
|
||||
.port = repl_info_config.replication_server.port,
|
||||
.ip_address = repl_info_config.replication_ip_address,
|
||||
.port = repl_info_config.replication_port,
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include "coordination/coordinator_instance.hpp"
|
||||
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "coordination/fmt.hpp"
|
||||
#include "dbms/constants.hpp"
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
@@ -30,60 +31,51 @@ using nuraft::ptr;
|
||||
using nuraft::srv_config;
|
||||
|
||||
CoordinatorInstance::CoordinatorInstance()
|
||||
: thread_pool_{1},
|
||||
raft_state_(RaftState::MakeRaftState(
|
||||
: raft_state_(RaftState::MakeRaftState(
|
||||
[this]() {
|
||||
spdlog::info("Leader changed, starting all replication instances!");
|
||||
auto const instances = raft_state_.GetReplicationInstances();
|
||||
auto const instances = raft_state_.GetInstances();
|
||||
auto replicas = instances | ranges::views::filter([](auto const &instance) {
|
||||
return instance.status == ReplicationRole::REPLICA;
|
||||
return instance.role == ReplicationRole::REPLICA;
|
||||
});
|
||||
|
||||
std::ranges::for_each(replicas, [this](auto &replica) {
|
||||
spdlog::info("Started pinging replication instance {}", replica.config.instance_name);
|
||||
spdlog::info("Starting replication instance {}", replica.config.instance_name);
|
||||
repl_instances_.emplace_back(this, replica.config, client_succ_cb_, client_fail_cb_,
|
||||
&CoordinatorInstance::ReplicaSuccessCallback,
|
||||
&CoordinatorInstance::ReplicaFailCallback);
|
||||
});
|
||||
|
||||
auto main = instances | ranges::views::filter(
|
||||
[](auto const &instance) { return instance.status == ReplicationRole::MAIN; });
|
||||
[](auto const &instance) { return instance.role == ReplicationRole::MAIN; });
|
||||
|
||||
// TODO: (andi) Add support for this
|
||||
// MG_ASSERT(std::ranges::distance(main) == 1, "There should be exactly one main instance");
|
||||
|
||||
std::ranges::for_each(main, [this](auto &main_instance) {
|
||||
spdlog::info("Started pinging main instance {}", main_instance.config.instance_name);
|
||||
spdlog::info("Starting main instance {}", main_instance.config.instance_name);
|
||||
repl_instances_.emplace_back(this, main_instance.config, client_succ_cb_, client_fail_cb_,
|
||||
&CoordinatorInstance::MainSuccessCallback,
|
||||
&CoordinatorInstance::MainFailCallback);
|
||||
});
|
||||
|
||||
std::ranges::for_each(repl_instances_, [](auto &instance) { instance.StartFrequentCheck(); });
|
||||
std::ranges::for_each(repl_instances_, [this](auto &instance) {
|
||||
instance.SetNewMainUUID(raft_state_.GetUUID()); // TODO: (andi) Rename
|
||||
instance.StartFrequentCheck();
|
||||
});
|
||||
},
|
||||
[this]() {
|
||||
thread_pool_.AddTask([this]() {
|
||||
spdlog::info("Leader changed, trying to stop all replication instances frequent checks!");
|
||||
// We need to stop checks before taking a lock because deadlock can happen if instances waits
|
||||
// to take a lock in frequent check, and this thread already has a lock and waits for instance to
|
||||
// be done with frequent check
|
||||
for (auto &repl_instance : repl_instances_) {
|
||||
repl_instance.StopFrequentCheck();
|
||||
}
|
||||
auto lock = std::unique_lock{coord_instance_lock_};
|
||||
repl_instances_.clear();
|
||||
spdlog::info("Stopped all replication instance frequent checks.");
|
||||
});
|
||||
spdlog::info("Leader changed, stopping all replication instances!");
|
||||
repl_instances_.clear();
|
||||
})) {
|
||||
client_succ_cb_ = [](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::unique_lock{self->coord_instance_lock_};
|
||||
// when coordinator is becoming follower it will want to stop all threads doing frequent checks
|
||||
// Thread can get stuck here waiting for lock so we need to frequently check if we are in shutdown state
|
||||
|
||||
auto &repl_instance = self->FindReplicationInstance(repl_instance_name);
|
||||
std::invoke(repl_instance.GetSuccessCallback(), self, repl_instance_name);
|
||||
};
|
||||
|
||||
client_fail_cb_ = [](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
|
||||
auto lock = std::unique_lock{self->coord_instance_lock_};
|
||||
|
||||
auto &repl_instance = self->FindReplicationInstance(repl_instance_name);
|
||||
std::invoke(repl_instance.GetFailCallback(), self, repl_instance_name);
|
||||
};
|
||||
@@ -106,13 +98,14 @@ auto CoordinatorInstance::ShowInstances() const -> std::vector<InstanceStatus> {
|
||||
.raft_socket_address = instance->get_endpoint(),
|
||||
.cluster_role = "coordinator",
|
||||
.health = "unknown"}; // TODO: (andi) Get this info from RAFT and test it or when we will move
|
||||
// CoordinatorState to every instance, we can be smarter about this using our RPC.
|
||||
};
|
||||
auto instances_status = utils::fmap(raft_state_.GetAllCoordinators(), coord_instance_to_status);
|
||||
|
||||
if (raft_state_.IsLeader()) {
|
||||
auto const stringify_repl_role = [this](ReplicationInstance const &instance) -> std::string {
|
||||
if (!instance.IsAlive()) return "unknown";
|
||||
if (raft_state_.IsCurrentMain(instance.InstanceName())) return "main";
|
||||
if (raft_state_.IsMain(instance.InstanceName())) return "main";
|
||||
return "replica";
|
||||
};
|
||||
|
||||
@@ -133,36 +126,26 @@ auto CoordinatorInstance::ShowInstances() const -> std::vector<InstanceStatus> {
|
||||
std::ranges::transform(repl_instances_, std::back_inserter(instances_status), process_repl_instance_as_leader);
|
||||
}
|
||||
} else {
|
||||
auto const stringify_inst_status = [raft_state_ptr = &raft_state_](
|
||||
utils::UUID const &main_uuid,
|
||||
ReplicationInstanceState const &instance) -> std::string {
|
||||
if (raft_state_ptr->IsCurrentMain(instance.config.instance_name)) {
|
||||
return "main";
|
||||
}
|
||||
if (raft_state_ptr->HasMainState(instance.config.instance_name)) {
|
||||
return "unknown";
|
||||
}
|
||||
return "replica";
|
||||
auto const stringify_repl_role = [](ReplicationRole role) -> std::string {
|
||||
return role == ReplicationRole::MAIN ? "main" : "replica";
|
||||
};
|
||||
|
||||
// TODO: (andi) Add capability that followers can also return socket addresses
|
||||
auto process_repl_instance_as_follower =
|
||||
[this, &stringify_inst_status](ReplicationInstanceState const &instance) -> InstanceStatus {
|
||||
auto process_repl_instance_as_follower = [&stringify_repl_role](auto const &instance) -> InstanceStatus {
|
||||
return {.instance_name = instance.config.instance_name,
|
||||
.cluster_role = stringify_inst_status(raft_state_.GetCurrentMainUUID(), instance),
|
||||
.cluster_role = stringify_repl_role(instance.role),
|
||||
.health = "unknown"};
|
||||
};
|
||||
|
||||
std::ranges::transform(raft_state_.GetReplicationInstances(), std::back_inserter(instances_status),
|
||||
std::ranges::transform(raft_state_.GetInstances(), std::back_inserter(instances_status),
|
||||
process_repl_instance_as_follower);
|
||||
}
|
||||
|
||||
return instances_status;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::TryFailover() -> void {
|
||||
auto const is_replica = [this](ReplicationInstance const &instance) {
|
||||
return HasReplicaState(instance.InstanceName());
|
||||
};
|
||||
auto const is_replica = [this](ReplicationInstance const &instance) { return IsReplica(instance.InstanceName()); };
|
||||
|
||||
auto alive_replicas =
|
||||
repl_instances_ | ranges::views::filter(is_replica) | ranges::views::filter(&ReplicationInstance::IsAlive);
|
||||
@@ -172,6 +155,11 @@ auto CoordinatorInstance::TryFailover() -> void {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
spdlog::error("Failover failed since the instance is not the leader!");
|
||||
return;
|
||||
}
|
||||
|
||||
auto const get_ts = [](ReplicationInstance &replica) { return replica.GetClient().SendGetInstanceTimestampsRpc(); };
|
||||
|
||||
auto maybe_instance_db_histories = alive_replicas | ranges::views::transform(get_ts) | ranges::to<std::vector>();
|
||||
@@ -199,10 +187,6 @@ auto CoordinatorInstance::TryFailover() -> void {
|
||||
|
||||
auto *new_main = &FindReplicationInstance(most_up_to_date_instance);
|
||||
|
||||
if (!raft_state_.AppendOpenLockFailover(most_up_to_date_instance)) {
|
||||
spdlog::error("Aborting failover as instance is not anymore leader.");
|
||||
return;
|
||||
}
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
@@ -212,18 +196,16 @@ auto CoordinatorInstance::TryFailover() -> void {
|
||||
|
||||
auto const new_main_uuid = utils::UUID{};
|
||||
|
||||
auto const failed_to_swap = [this, &new_main_uuid](ReplicationInstance &instance) {
|
||||
return !instance.SendSwapAndUpdateUUID(new_main_uuid) ||
|
||||
!raft_state_.AppendUpdateUUIDForInstanceLog(instance.InstanceName(), new_main_uuid);
|
||||
auto const failed_to_swap = [&new_main_uuid](ReplicationInstance &instance) {
|
||||
return !instance.SendSwapAndUpdateUUID(new_main_uuid);
|
||||
};
|
||||
|
||||
// If for some replicas swap fails, for others on successful ping we will revert back on next change
|
||||
// or we will do failover first again and then it will be consistent again
|
||||
if (std::ranges::any_of(alive_replicas | ranges::views::filter(is_not_new_main), failed_to_swap)) {
|
||||
spdlog::error("Aborting failover. Failed to swap uuid for all alive instances.");
|
||||
spdlog::error("Failed to swap uuid for all instances");
|
||||
return;
|
||||
}
|
||||
|
||||
auto repl_clients_info = repl_instances_ | ranges::views::filter(is_not_new_main) |
|
||||
ranges::views::transform(&ReplicationInstance::ReplicationClientInfo) |
|
||||
ranges::to<ReplicationClientsInfo>();
|
||||
@@ -234,36 +216,27 @@ auto CoordinatorInstance::TryFailover() -> void {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendUpdateUUIDForNewMainLog(new_main_uuid)) {
|
||||
if (!raft_state_.AppendUpdateUUIDLog(new_main_uuid)) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto const new_main_instance_name = new_main->InstanceName();
|
||||
|
||||
if (!raft_state_.AppendSetInstanceAsMainLog(new_main_instance_name, new_main_uuid)) {
|
||||
if (!raft_state_.AppendSetInstanceAsMainLog(new_main_instance_name)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!new_main->EnableWritingOnMain()) {
|
||||
spdlog::error("Failover successful but couldn't enable writing on instance.");
|
||||
}
|
||||
|
||||
spdlog::info("Failover successful! Instance {} promoted to main.", new_main->InstanceName());
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::SetReplicationInstanceToMain(std::string_view instance_name)
|
||||
-> SetInstanceToMainCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
return SetInstanceToMainCoordinatorStatus::LOCK_OPENED;
|
||||
}
|
||||
|
||||
if (raft_state_.MainExists()) {
|
||||
return SetInstanceToMainCoordinatorStatus::MAIN_ALREADY_EXISTS;
|
||||
}
|
||||
|
||||
// TODO(antoniofilipovic) Check if request leadership can cause problems due to changing of leadership while other
|
||||
// doing failover
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return SetInstanceToMainCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
@@ -280,10 +253,6 @@ auto CoordinatorInstance::SetReplicationInstanceToMain(std::string_view instance
|
||||
return SetInstanceToMainCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendOpenLockSetInstanceToMain(instance_name)) {
|
||||
return SetInstanceToMainCoordinatorStatus::OPEN_LOCK;
|
||||
}
|
||||
|
||||
new_main->PauseFrequentCheck();
|
||||
utils::OnScopeExit scope_exit{[&new_main] { new_main->ResumeFrequentCheck(); }};
|
||||
|
||||
@@ -293,13 +262,12 @@ auto CoordinatorInstance::SetReplicationInstanceToMain(std::string_view instance
|
||||
|
||||
auto const new_main_uuid = utils::UUID{};
|
||||
|
||||
auto const failed_to_swap = [this, &new_main_uuid](ReplicationInstance &instance) {
|
||||
return !instance.SendSwapAndUpdateUUID(new_main_uuid) ||
|
||||
!raft_state_.AppendUpdateUUIDForInstanceLog(instance.InstanceName(), new_main_uuid);
|
||||
auto const failed_to_swap = [&new_main_uuid](ReplicationInstance &instance) {
|
||||
return !instance.SendSwapAndUpdateUUID(new_main_uuid);
|
||||
};
|
||||
|
||||
if (std::ranges::any_of(repl_instances_ | ranges::views::filter(is_not_new_main), failed_to_swap)) {
|
||||
spdlog::error("Failed to swap uuid for all currently alive instances.");
|
||||
spdlog::error("Failed to swap uuid for all instances");
|
||||
return SetInstanceToMainCoordinatorStatus::SWAP_UUID_FAILED;
|
||||
}
|
||||
|
||||
@@ -311,28 +279,22 @@ auto CoordinatorInstance::SetReplicationInstanceToMain(std::string_view instance
|
||||
&CoordinatorInstance::MainFailCallback)) {
|
||||
return SetInstanceToMainCoordinatorStatus::COULD_NOT_PROMOTE_TO_MAIN;
|
||||
}
|
||||
if (!raft_state_.AppendUpdateUUIDForNewMainLog(new_main_uuid)) {
|
||||
|
||||
if (!raft_state_.AppendUpdateUUIDLog(new_main_uuid)) {
|
||||
return SetInstanceToMainCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendSetInstanceAsMainLog(instance_name, new_main_uuid)) {
|
||||
if (!raft_state_.AppendSetInstanceAsMainLog(instance_name)) {
|
||||
return SetInstanceToMainCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
spdlog::info("Instance {} promoted to main on leader", instance_name);
|
||||
|
||||
if (!new_main->EnableWritingOnMain()) {
|
||||
return SetInstanceToMainCoordinatorStatus::ENABLE_WRITING_FAILED;
|
||||
}
|
||||
return SetInstanceToMainCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::RegisterReplicationInstance(CoordinatorToReplicaConfig const &config)
|
||||
auto CoordinatorInstance::RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
return RegisterInstanceCoordinatorStatus::LOCK_OPENED;
|
||||
}
|
||||
|
||||
if (std::ranges::any_of(repl_instances_, [instance_name = config.instance_name](ReplicationInstance const &instance) {
|
||||
return instance.InstanceName() == instance_name;
|
||||
@@ -352,27 +314,24 @@ auto CoordinatorInstance::RegisterReplicationInstance(CoordinatorToReplicaConfig
|
||||
return RegisterInstanceCoordinatorStatus::REPL_ENDPOINT_EXISTS;
|
||||
}
|
||||
|
||||
// TODO(antoniofilipovic) Check if this is an issue
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return RegisterInstanceCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendOpenLockRegister(config)) {
|
||||
return RegisterInstanceCoordinatorStatus::OPEN_LOCK;
|
||||
}
|
||||
|
||||
auto *new_instance = &repl_instances_.emplace_back(this, config, client_succ_cb_, client_fail_cb_,
|
||||
&CoordinatorInstance::ReplicaSuccessCallback,
|
||||
&CoordinatorInstance::ReplicaFailCallback);
|
||||
|
||||
if (!new_instance->SendDemoteToReplicaRpc()) {
|
||||
spdlog::error("Failed to send demote to replica rpc for instance {}", config.instance_name);
|
||||
repl_instances_.pop_back();
|
||||
return RegisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendRegisterReplicationInstanceLog(config)) {
|
||||
return RegisterInstanceCoordinatorStatus::RAFT_LOG_ERROR;
|
||||
}
|
||||
|
||||
new_instance->StartFrequentCheck();
|
||||
|
||||
spdlog::info("Instance {} registered", config.instance_name);
|
||||
@@ -383,11 +342,6 @@ auto CoordinatorInstance::UnregisterReplicationInstance(std::string_view instanc
|
||||
-> UnregisterInstanceCoordinatorStatus {
|
||||
auto lock = std::lock_guard{coord_instance_lock_};
|
||||
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
return UnregisterInstanceCoordinatorStatus::LOCK_OPENED;
|
||||
}
|
||||
|
||||
// TODO(antoniofilipovic) Check if this is an issue
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
return UnregisterInstanceCoordinatorStatus::NOT_LEADER;
|
||||
}
|
||||
@@ -401,23 +355,19 @@ auto CoordinatorInstance::UnregisterReplicationInstance(std::string_view instanc
|
||||
return UnregisterInstanceCoordinatorStatus::NO_INSTANCE_WITH_NAME;
|
||||
}
|
||||
|
||||
auto const is_current_main = [this](ReplicationInstance const &instance) {
|
||||
return raft_state_.IsCurrentMain(instance.InstanceName()) && instance.IsAlive();
|
||||
};
|
||||
// TODO: (andi) Change so that RaftLogState is the central place for asking who is main...
|
||||
|
||||
if (is_current_main(*inst_to_remove)) {
|
||||
auto const is_main = [this](ReplicationInstance const &instance) { return IsMain(instance.InstanceName()); };
|
||||
|
||||
if (is_main(*inst_to_remove) && inst_to_remove->IsAlive()) {
|
||||
return UnregisterInstanceCoordinatorStatus::IS_MAIN;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendOpenLockUnregister(instance_name)) {
|
||||
return UnregisterInstanceCoordinatorStatus::OPEN_LOCK;
|
||||
}
|
||||
|
||||
inst_to_remove->StopFrequentCheck();
|
||||
|
||||
auto curr_main = std::ranges::find_if(repl_instances_, is_current_main);
|
||||
auto curr_main = std::ranges::find_if(repl_instances_, is_main);
|
||||
|
||||
if (curr_main != repl_instances_.end()) {
|
||||
if (curr_main != repl_instances_.end() && curr_main->IsAlive()) {
|
||||
if (!curr_main->SendUnregisterReplicaRpc(instance_name)) {
|
||||
inst_to_remove->StartFrequentCheck();
|
||||
return UnregisterInstanceCoordinatorStatus::RPC_FAILED;
|
||||
@@ -433,25 +383,20 @@ auto CoordinatorInstance::UnregisterReplicationInstance(std::string_view instanc
|
||||
return UnregisterInstanceCoordinatorStatus::SUCCESS;
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void {
|
||||
raft_state_.AddCoordinatorInstance(config);
|
||||
// NOTE: We ignore error we added coordinator instance to networking stuff but not in raft log.
|
||||
if (!raft_state_.AppendAddCoordinatorInstanceLog(config)) {
|
||||
spdlog::error("Failed to append add coordinator instance log");
|
||||
}
|
||||
auto CoordinatorInstance::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port,
|
||||
std::string_view raft_address) -> void {
|
||||
raft_state_.AddCoordinatorInstance(raft_server_id, raft_port, raft_address);
|
||||
}
|
||||
|
||||
void CoordinatorInstance::MainFailCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing main fail callback", repl_instance_name);
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
spdlog::error("Returning from main fail callback as the last action didn't successfully finish");
|
||||
}
|
||||
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
repl_instance.OnFailPing();
|
||||
const auto &repl_instance_uuid = repl_instance.GetMainUUID();
|
||||
MG_ASSERT(repl_instance_uuid.has_value(), "Replication instance must have uuid set");
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
if (!repl_instance.IsAlive() && raft_state_.IsCurrentMain(repl_instance_name)) {
|
||||
if (!repl_instance.IsAlive() && raft_state_.GetUUID() == repl_instance_uuid.value()) {
|
||||
spdlog::info("Cluster without main instance, trying automatic failover");
|
||||
TryFailover();
|
||||
}
|
||||
@@ -459,12 +404,6 @@ void CoordinatorInstance::MainFailCallback(std::string_view repl_instance_name)
|
||||
|
||||
void CoordinatorInstance::MainSuccessCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing main successful callback", repl_instance_name);
|
||||
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
spdlog::error("Stopping main successful callback as the last action didn't successfully finish");
|
||||
return;
|
||||
}
|
||||
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
if (repl_instance.IsAlive()) {
|
||||
@@ -472,8 +411,11 @@ void CoordinatorInstance::MainSuccessCallback(std::string_view repl_instance_nam
|
||||
return;
|
||||
}
|
||||
|
||||
const auto &repl_instance_uuid = repl_instance.GetMainUUID();
|
||||
MG_ASSERT(repl_instance_uuid.has_value(), "Instance must have uuid set.");
|
||||
|
||||
// NOLINTNEXTLINE
|
||||
if (raft_state_.IsCurrentMain(repl_instance.InstanceName())) {
|
||||
if (raft_state_.GetUUID() == repl_instance_uuid.value()) {
|
||||
if (!repl_instance.EnableWritingOnMain()) {
|
||||
spdlog::error("Failed to enable writing on main instance {}", repl_instance_name);
|
||||
return;
|
||||
@@ -483,8 +425,9 @@ void CoordinatorInstance::MainSuccessCallback(std::string_view repl_instance_nam
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendOpenLockSetInstanceToReplica(repl_instance.InstanceName())) {
|
||||
spdlog::error("Failed to open lock for demoting OLD MAIN {} to REPLICA", repl_instance_name);
|
||||
if (!raft_state_.RequestLeadership()) {
|
||||
spdlog::error("Demoting main instance {} to replica failed since the instance is not the leader!",
|
||||
repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -497,38 +440,29 @@ void CoordinatorInstance::MainSuccessCallback(std::string_view repl_instance_nam
|
||||
return;
|
||||
}
|
||||
|
||||
if (!repl_instance.SendSwapAndUpdateUUID(raft_state_.GetCurrentMainUUID())) {
|
||||
if (!repl_instance.SendSwapAndUpdateUUID(raft_state_.GetUUID())) {
|
||||
spdlog::error("Failed to swap uuid for demoted main instance {}", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendUpdateUUIDForInstanceLog(repl_instance_name, raft_state_.GetCurrentMainUUID())) {
|
||||
spdlog::error("Failed to update log of changing instance uuid {} to {}", repl_instance_name,
|
||||
std::string{raft_state_.GetCurrentMainUUID()});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!raft_state_.AppendSetInstanceAsReplicaLog(repl_instance_name)) {
|
||||
spdlog::error("Failed to append log that OLD MAIN was demoted to REPLICA {}", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void CoordinatorInstance::ReplicaSuccessCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing replica successful callback", repl_instance_name);
|
||||
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
spdlog::error("Stopping main successful callback as the last action didn't successfully finish");
|
||||
return;
|
||||
}
|
||||
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
if (!IsReplica(repl_instance_name)) {
|
||||
spdlog::error("Aborting replica callback since instance {} is not replica anymore", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
// We need to get replicas UUID from time to time to ensure replica is listening to correct main
|
||||
// and that it didn't go down for less time than we could notice
|
||||
// We need to get id of main replica is listening to
|
||||
// and swap if necessary
|
||||
if (!repl_instance.EnsureReplicaHasCorrectMainUUID(raft_state_.GetCurrentMainUUID())) {
|
||||
if (!repl_instance.EnsureReplicaHasCorrectMainUUID(raft_state_.GetUUID())) {
|
||||
spdlog::error("Failed to swap uuid for replica instance {} which is alive", repl_instance.InstanceName());
|
||||
return;
|
||||
}
|
||||
@@ -538,14 +472,13 @@ void CoordinatorInstance::ReplicaSuccessCallback(std::string_view repl_instance_
|
||||
|
||||
void CoordinatorInstance::ReplicaFailCallback(std::string_view repl_instance_name) {
|
||||
spdlog::trace("Instance {} performing replica failure callback", repl_instance_name);
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
if (raft_state_.IsLockOpened()) {
|
||||
spdlog::error("Stopping main successful callback as the last action didn't successfully finish.");
|
||||
if (!IsReplica(repl_instance_name)) {
|
||||
spdlog::error("Aborting replica fail callback since instance {} is not replica anymore", repl_instance_name);
|
||||
return;
|
||||
}
|
||||
|
||||
auto &repl_instance = FindReplicationInstance(repl_instance_name);
|
||||
|
||||
repl_instance.OnFailPing();
|
||||
}
|
||||
|
||||
@@ -617,63 +550,12 @@ auto CoordinatorInstance::ChooseMostUpToDateInstance(std::span<InstanceNameDbHis
|
||||
return std::move(*new_main_res);
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::HasMainState(std::string_view instance_name) const -> bool {
|
||||
return raft_state_.HasMainState(instance_name);
|
||||
auto CoordinatorInstance::IsMain(std::string_view instance_name) const -> bool {
|
||||
return raft_state_.IsMain(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::HasReplicaState(std::string_view instance_name) const -> bool {
|
||||
return raft_state_.HasReplicaState(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorInstance::GetRoutingTable(std::map<std::string, std::string> const &routing) -> RoutingTable {
|
||||
auto res = RoutingTable{};
|
||||
|
||||
auto const repl_instance_to_bolt = [](ReplicationInstanceState const &instance) {
|
||||
return instance.config.BoltSocketAddress();
|
||||
};
|
||||
|
||||
// TODO: (andi) This is wrong check, Fico will correct in #1819.
|
||||
auto const is_instance_main = [&](ReplicationInstanceState const &instance) {
|
||||
return instance.status == ReplicationRole::MAIN;
|
||||
};
|
||||
|
||||
auto const is_instance_replica = [&](ReplicationInstanceState const &instance) {
|
||||
return instance.status == ReplicationRole::REPLICA;
|
||||
};
|
||||
|
||||
auto const &raft_log_repl_instances = raft_state_.GetReplicationInstances();
|
||||
|
||||
auto bolt_mains = raft_log_repl_instances | ranges::views::filter(is_instance_main) |
|
||||
ranges::views::transform(repl_instance_to_bolt) | ranges::to<std::vector>();
|
||||
MG_ASSERT(bolt_mains.size() <= 1, "There can be at most one main instance active!");
|
||||
|
||||
if (!std::ranges::empty(bolt_mains)) {
|
||||
res.emplace_back(std::move(bolt_mains), "WRITE");
|
||||
}
|
||||
|
||||
auto bolt_replicas = raft_log_repl_instances | ranges::views::filter(is_instance_replica) |
|
||||
ranges::views::transform(repl_instance_to_bolt) | ranges::to<std::vector>();
|
||||
if (!std::ranges::empty(bolt_replicas)) {
|
||||
res.emplace_back(std::move(bolt_replicas), "READ");
|
||||
}
|
||||
|
||||
auto const coord_instance_to_bolt = [](CoordinatorInstanceState const &instance) {
|
||||
return instance.config.bolt_server.SocketAddress();
|
||||
};
|
||||
|
||||
auto const &raft_log_coord_instances = raft_state_.GetCoordinatorInstances();
|
||||
auto bolt_coords =
|
||||
raft_log_coord_instances | ranges::views::transform(coord_instance_to_bolt) | ranges::to<std::vector>();
|
||||
|
||||
auto const &local_bolt_coord = routing.find("address");
|
||||
if (local_bolt_coord == routing.end()) {
|
||||
throw InvalidRoutingTableException("No bolt address found in routing table for the current coordinator!");
|
||||
}
|
||||
|
||||
bolt_coords.push_back(local_bolt_coord->second);
|
||||
res.emplace_back(std::move(bolt_coords), "ROUTE");
|
||||
|
||||
return res;
|
||||
auto CoordinatorInstance::IsReplica(std::string_view instance_name) const -> bool {
|
||||
return raft_state_.IsReplica(instance_name);
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -62,33 +62,34 @@ ptr<log_entry> CoordinatorLogStore::last_entry() const {
|
||||
|
||||
uint64_t CoordinatorLogStore::append(ptr<log_entry> &entry) {
|
||||
ptr<log_entry> clone = MakeClone(entry);
|
||||
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
uint64_t next_slot = start_idx_ + logs_.size() - 1;
|
||||
logs_[next_slot] = clone;
|
||||
|
||||
uint64_t next_slot{0};
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
next_slot = start_idx_ + logs_.size() - 1;
|
||||
logs_[next_slot] = clone;
|
||||
}
|
||||
return next_slot;
|
||||
}
|
||||
|
||||
// TODO: (andi) I think this is used for resolving conflicts inside NuRaft, check...
|
||||
// different compared to in_memory_log_store.cxx
|
||||
void CoordinatorLogStore::write_at(uint64_t index, ptr<log_entry> &entry) {
|
||||
ptr<log_entry> clone = MakeClone(entry);
|
||||
|
||||
// Discard all logs equal to or greater than `index.
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
auto itr = logs_.lower_bound(index);
|
||||
while (itr != logs_.end()) {
|
||||
itr = logs_.erase(itr);
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
auto itr = logs_.lower_bound(index);
|
||||
while (itr != logs_.end()) {
|
||||
itr = logs_.erase(itr);
|
||||
}
|
||||
logs_[index] = clone;
|
||||
}
|
||||
logs_[index] = clone;
|
||||
}
|
||||
|
||||
ptr<std::vector<ptr<log_entry>>> CoordinatorLogStore::log_entries(uint64_t start, uint64_t end) {
|
||||
auto ret = cs_new<std::vector<ptr<log_entry>>>();
|
||||
ret->resize(end - start);
|
||||
|
||||
for (uint64_t i = start, curr_index = 0; i < end; i++, curr_index++) {
|
||||
for (uint64_t i = start, curr_index = 0; i < end; ++i, ++curr_index) {
|
||||
ptr<log_entry> src = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
@@ -104,14 +105,21 @@ ptr<std::vector<ptr<log_entry>>> CoordinatorLogStore::log_entries(uint64_t start
|
||||
}
|
||||
|
||||
ptr<log_entry> CoordinatorLogStore::entry_at(uint64_t index) {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
ptr<log_entry> src = FindOrDefault_(index);
|
||||
ptr<log_entry> src = nullptr;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
src = FindOrDefault_(index);
|
||||
}
|
||||
return MakeClone(src);
|
||||
}
|
||||
|
||||
uint64_t CoordinatorLogStore::term_at(uint64_t index) {
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
return FindOrDefault_(index)->get_term();
|
||||
uint64_t term = 0;
|
||||
{
|
||||
auto lock = std::lock_guard{logs_lock_};
|
||||
term = FindOrDefault_(index)->get_term();
|
||||
}
|
||||
return term;
|
||||
}
|
||||
|
||||
ptr<buffer> CoordinatorLogStore::pack(uint64_t index, int32 cnt) {
|
||||
|
||||
@@ -18,7 +18,8 @@ namespace memgraph::coordination {
|
||||
|
||||
namespace {
|
||||
|
||||
auto CreateServerContext(const memgraph::coordination::ManagementServerConfig &config) -> communication::ServerContext {
|
||||
auto CreateServerContext(const memgraph::coordination::CoordinatorServerConfig &config)
|
||||
-> communication::ServerContext {
|
||||
return (config.ssl) ? communication::ServerContext{config.ssl->key_file, config.ssl->cert_file, config.ssl->ca_file,
|
||||
config.ssl->verify_peer}
|
||||
: communication::ServerContext{};
|
||||
@@ -31,7 +32,7 @@ constexpr auto kCoordinatorServerThreads = 1;
|
||||
|
||||
} // namespace
|
||||
|
||||
CoordinatorServer::CoordinatorServer(const ManagementServerConfig &config)
|
||||
CoordinatorServer::CoordinatorServer(const CoordinatorServerConfig &config)
|
||||
: rpc_server_context_{CreateServerContext(config)},
|
||||
rpc_server_{io::network::Endpoint{config.ip_address, config.port}, &rpc_server_context_,
|
||||
kCoordinatorServerThreads} {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
#include "coordination/coordinator_state.hpp"
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
#include "flags/replication.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
@@ -25,15 +25,15 @@
|
||||
namespace memgraph::coordination {
|
||||
|
||||
CoordinatorState::CoordinatorState() {
|
||||
MG_ASSERT(!(FLAGS_coordinator_id && FLAGS_management_port),
|
||||
MG_ASSERT(!(FLAGS_raft_server_id && FLAGS_coordinator_server_port),
|
||||
"Instance cannot be a coordinator and have registered coordinator server.");
|
||||
|
||||
spdlog::info("Executing coordinator constructor");
|
||||
if (FLAGS_management_port) {
|
||||
if (FLAGS_coordinator_server_port) {
|
||||
spdlog::info("Coordinator server port set");
|
||||
auto const config = ManagementServerConfig{
|
||||
auto const config = CoordinatorServerConfig{
|
||||
.ip_address = kDefaultReplicationServerIp,
|
||||
.port = static_cast<uint16_t>(FLAGS_management_port),
|
||||
.port = static_cast<uint16_t>(FLAGS_coordinator_server_port),
|
||||
};
|
||||
spdlog::info("Executing coordinator constructor main replica");
|
||||
|
||||
@@ -41,7 +41,7 @@ CoordinatorState::CoordinatorState() {
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorState::RegisterReplicationInstance(CoordinatorToReplicaConfig const &config)
|
||||
auto CoordinatorState::RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
@@ -98,16 +98,11 @@ auto CoordinatorState::GetCoordinatorServer() const -> CoordinatorServer & {
|
||||
return *std::get<CoordinatorMainReplicaData>(data_).coordinator_server_;
|
||||
}
|
||||
|
||||
auto CoordinatorState::AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void {
|
||||
auto CoordinatorState::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port,
|
||||
std::string_view raft_address) -> void {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot register replica since variant holds wrong alternative");
|
||||
return std::get<CoordinatorInstance>(data_).AddCoordinatorInstance(config);
|
||||
}
|
||||
|
||||
auto CoordinatorState::GetRoutingTable(std::map<std::string, std::string> const &routing) -> RoutingTable {
|
||||
MG_ASSERT(std::holds_alternative<CoordinatorInstance>(data_),
|
||||
"Coordinator cannot get routing table since variant holds wrong alternative");
|
||||
return std::get<CoordinatorInstance>(data_).GetRoutingTable(routing);
|
||||
return std::get<CoordinatorInstance>(data_).AddCoordinatorInstance(raft_server_id, raft_port, raft_address);
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -14,20 +14,20 @@
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace {
|
||||
constexpr int MAX_SNAPSHOTS = 3;
|
||||
} // namespace
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
auto CoordinatorStateMachine::FindCurrentMainInstanceName() const -> std::optional<std::string> {
|
||||
return cluster_state_.FindCurrentMainInstanceName();
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::MainExists() const -> bool { return cluster_state_.MainExists(); }
|
||||
|
||||
auto CoordinatorStateMachine::HasMainState(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.HasMainState(instance_name);
|
||||
auto CoordinatorStateMachine::IsMain(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.IsMain(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::HasReplicaState(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.HasReplicaState(instance_name);
|
||||
auto CoordinatorStateMachine::IsReplica(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.IsReplica(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::CreateLog(nlohmann::json &&log) -> ptr<buffer> {
|
||||
@@ -38,24 +38,7 @@ auto CoordinatorStateMachine::CreateLog(nlohmann::json &&log) -> ptr<buffer> {
|
||||
return log_buf;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeOpenLockRegister(CoordinatorToReplicaConfig const &config) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::OPEN_LOCK_REGISTER_REPLICATION_INSTANCE}, {"info", config}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeOpenLockUnregister(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog(
|
||||
{{"action", RaftLogAction::OPEN_LOCK_UNREGISTER_REPLICATION_INSTANCE}, {"info", std::string{instance_name}}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeOpenLockFailover(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::OPEN_LOCK_FAILOVER}, {"info", std::string(instance_name)}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeOpenLockSetInstanceAsMain(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_MAIN}, {"info", std::string(instance_name)}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeRegisterInstance(CoordinatorToReplicaConfig const &config) -> ptr<buffer> {
|
||||
auto CoordinatorStateMachine::SerializeRegisterInstance(CoordinatorClientConfig const &config) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::REGISTER_REPLICATION_INSTANCE}, {"info", config}});
|
||||
}
|
||||
|
||||
@@ -63,65 +46,35 @@ auto CoordinatorStateMachine::SerializeUnregisterInstance(std::string_view insta
|
||||
return CreateLog({{"action", RaftLogAction::UNREGISTER_REPLICATION_INSTANCE}, {"info", instance_name}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeSetInstanceAsMain(InstanceUUIDUpdate const &instance_uuid_change)
|
||||
-> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::SET_INSTANCE_AS_MAIN}, {"info", instance_uuid_change}});
|
||||
auto CoordinatorStateMachine::SerializeSetInstanceAsMain(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::SET_INSTANCE_AS_MAIN}, {"info", instance_name}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeSetInstanceAsReplica(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::SET_INSTANCE_AS_REPLICA}, {"info", instance_name}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeUpdateUUIDForNewMain(utils::UUID const &uuid) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::UPDATE_UUID_OF_NEW_MAIN}, {"info", uuid}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeUpdateUUIDForInstance(InstanceUUIDUpdate const &instance_uuid_change)
|
||||
-> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::UPDATE_UUID_FOR_INSTANCE}, {"info", instance_uuid_change}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeAddCoordinatorInstance(CoordinatorToCoordinatorConfig const &config)
|
||||
-> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::ADD_COORDINATOR_INSTANCE}, {"info", config}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::SerializeOpenLockSetInstanceAsReplica(std::string_view instance_name) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_REPLICA}, {"info", instance_name}});
|
||||
auto CoordinatorStateMachine::SerializeUpdateUUID(utils::UUID const &uuid) -> ptr<buffer> {
|
||||
return CreateLog({{"action", RaftLogAction::UPDATE_UUID}, {"info", uuid}});
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::DecodeLog(buffer &data) -> std::pair<TRaftLog, RaftLogAction> {
|
||||
buffer_serializer bs(data);
|
||||
auto const json = nlohmann::json::parse(bs.get_str());
|
||||
|
||||
auto const action = json["action"].get<RaftLogAction>();
|
||||
auto const &info = json["info"];
|
||||
|
||||
switch (action) {
|
||||
case RaftLogAction::OPEN_LOCK_REGISTER_REPLICATION_INSTANCE: {
|
||||
return {info.get<CoordinatorToReplicaConfig>(), action};
|
||||
}
|
||||
case RaftLogAction::OPEN_LOCK_UNREGISTER_REPLICATION_INSTANCE:
|
||||
[[fallthrough]];
|
||||
case RaftLogAction::OPEN_LOCK_FAILOVER:
|
||||
[[fallthrough]];
|
||||
case RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_MAIN:
|
||||
[[fallthrough]];
|
||||
case RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_REPLICA: {
|
||||
return {info.get<std::string>(), action};
|
||||
}
|
||||
case RaftLogAction::REGISTER_REPLICATION_INSTANCE:
|
||||
return {info.get<CoordinatorToReplicaConfig>(), action};
|
||||
case RaftLogAction::UPDATE_UUID_OF_NEW_MAIN:
|
||||
return {info.get<CoordinatorClientConfig>(), action};
|
||||
case RaftLogAction::UPDATE_UUID:
|
||||
return {info.get<utils::UUID>(), action};
|
||||
case RaftLogAction::UPDATE_UUID_FOR_INSTANCE:
|
||||
case RaftLogAction::SET_INSTANCE_AS_MAIN:
|
||||
return {info.get<InstanceUUIDUpdate>(), action};
|
||||
case RaftLogAction::UNREGISTER_REPLICATION_INSTANCE:
|
||||
case RaftLogAction::SET_INSTANCE_AS_MAIN:
|
||||
[[fallthrough]];
|
||||
case RaftLogAction::SET_INSTANCE_AS_REPLICA:
|
||||
return {info.get<std::string>(), action};
|
||||
case RaftLogAction::ADD_COORDINATOR_INSTANCE:
|
||||
return {info.get<CoordinatorToCoordinatorConfig>(), action};
|
||||
}
|
||||
throw std::runtime_error("Unknown action");
|
||||
}
|
||||
@@ -129,7 +82,6 @@ auto CoordinatorStateMachine::DecodeLog(buffer &data) -> std::pair<TRaftLog, Raf
|
||||
auto CoordinatorStateMachine::pre_commit(ulong const /*log_idx*/, buffer & /*data*/) -> ptr<buffer> { return nullptr; }
|
||||
|
||||
auto CoordinatorStateMachine::commit(ulong const log_idx, buffer &data) -> ptr<buffer> {
|
||||
spdlog::debug("Commit: log_idx={}, data.size()={}", log_idx, data.size());
|
||||
auto const [parsed_data, log_action] = DecodeLog(data);
|
||||
cluster_state_.DoAction(parsed_data, log_action);
|
||||
last_committed_idx_ = log_idx;
|
||||
@@ -143,17 +95,15 @@ auto CoordinatorStateMachine::commit(ulong const log_idx, buffer &data) -> ptr<b
|
||||
|
||||
auto CoordinatorStateMachine::commit_config(ulong const log_idx, ptr<cluster_config> & /*new_conf*/) -> void {
|
||||
last_committed_idx_ = log_idx;
|
||||
spdlog::debug("Commit config: log_idx={}", log_idx);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::rollback(ulong const log_idx, buffer &data) -> void {
|
||||
// NOTE: Nothing since we don't do anything in pre_commit
|
||||
spdlog::debug("Rollback: log_idx={}, data.size()={}", log_idx, data.size());
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::read_logical_snp_obj(snapshot &snapshot, void *& /*user_snp_ctx*/, ulong obj_id,
|
||||
ptr<buffer> &data_out, bool &is_last_obj) -> int {
|
||||
spdlog::debug("read logical snapshot object, obj_id: {}", obj_id);
|
||||
spdlog::info("read logical snapshot object, obj_id: {}", obj_id);
|
||||
|
||||
ptr<SnapshotCtx> ctx = nullptr;
|
||||
{
|
||||
@@ -166,45 +116,29 @@ auto CoordinatorStateMachine::read_logical_snp_obj(snapshot &snapshot, void *& /
|
||||
}
|
||||
ctx = entry->second;
|
||||
}
|
||||
|
||||
if (obj_id == 0) {
|
||||
// Object ID == 0: first object, put dummy data.
|
||||
data_out = buffer::alloc(sizeof(int32));
|
||||
buffer_serializer bs(data_out);
|
||||
bs.put_i32(0);
|
||||
is_last_obj = false;
|
||||
} else {
|
||||
// Object ID > 0: second object, put actual value.
|
||||
ctx->cluster_state_.Serialize(data_out);
|
||||
is_last_obj = true;
|
||||
}
|
||||
|
||||
ctx->cluster_state_.Serialize(data_out);
|
||||
is_last_obj = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::save_logical_snp_obj(snapshot &snapshot, ulong &obj_id, buffer &data, bool is_first_obj,
|
||||
bool is_last_obj) -> void {
|
||||
spdlog::debug("save logical snapshot object, obj_id: {}, is_first_obj: {}, is_last_obj: {}", obj_id, is_first_obj,
|
||||
is_last_obj);
|
||||
spdlog::info("save logical snapshot object, obj_id: {}, is_first_obj: {}, is_last_obj: {}", obj_id, is_first_obj,
|
||||
is_last_obj);
|
||||
|
||||
if (obj_id == 0) {
|
||||
ptr<buffer> snp_buf = snapshot.serialize();
|
||||
auto ss = snapshot::deserialize(*snp_buf);
|
||||
create_snapshot_internal(ss);
|
||||
} else {
|
||||
auto cluster_state = CoordinatorClusterState::Deserialize(data);
|
||||
buffer_serializer bs(data);
|
||||
auto cluster_state = CoordinatorClusterState::Deserialize(data);
|
||||
|
||||
{
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
auto entry = snapshots_.find(snapshot.get_last_log_idx());
|
||||
DMG_ASSERT(entry != snapshots_.end());
|
||||
entry->second->cluster_state_ = cluster_state;
|
||||
}
|
||||
obj_id++;
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::apply_snapshot(snapshot &s) -> bool {
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
spdlog::debug("apply snapshot, last_log_idx: {}", s.get_last_log_idx());
|
||||
|
||||
auto entry = snapshots_.find(s.get_last_log_idx());
|
||||
if (entry == snapshots_.end()) return false;
|
||||
@@ -217,7 +151,6 @@ auto CoordinatorStateMachine::free_user_snp_ctx(void *&user_snp_ctx) -> void {}
|
||||
|
||||
auto CoordinatorStateMachine::last_snapshot() -> ptr<snapshot> {
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
spdlog::debug("last_snapshot");
|
||||
auto entry = snapshots_.rbegin();
|
||||
if (entry == snapshots_.rend()) return nullptr;
|
||||
|
||||
@@ -228,7 +161,6 @@ auto CoordinatorStateMachine::last_snapshot() -> ptr<snapshot> {
|
||||
auto CoordinatorStateMachine::last_commit_index() -> ulong { return last_committed_idx_; }
|
||||
|
||||
auto CoordinatorStateMachine::create_snapshot(snapshot &s, async_result<bool>::handler_type &when_done) -> void {
|
||||
spdlog::debug("create_snapshot, last_log_idx: {}", s.get_last_log_idx());
|
||||
ptr<buffer> snp_buf = s.serialize();
|
||||
ptr<snapshot> ss = snapshot::deserialize(*snp_buf);
|
||||
create_snapshot_internal(ss);
|
||||
@@ -240,34 +172,21 @@ auto CoordinatorStateMachine::create_snapshot(snapshot &s, async_result<bool>::h
|
||||
|
||||
auto CoordinatorStateMachine::create_snapshot_internal(ptr<snapshot> snapshot) -> void {
|
||||
auto ll = std::lock_guard{snapshots_lock_};
|
||||
spdlog::debug("create_snapshot_internal, last_log_idx: {}", snapshot->get_last_log_idx());
|
||||
|
||||
auto ctx = cs_new<SnapshotCtx>(snapshot, cluster_state_);
|
||||
snapshots_[snapshot->get_last_log_idx()] = ctx;
|
||||
|
||||
constexpr int MAX_SNAPSHOTS = 3;
|
||||
while (snapshots_.size() > MAX_SNAPSHOTS) {
|
||||
snapshots_.erase(snapshots_.begin());
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::GetReplicationInstances() const -> std::vector<ReplicationInstanceState> {
|
||||
return cluster_state_.GetReplicationInstances();
|
||||
auto CoordinatorStateMachine::GetInstances() const -> std::vector<InstanceState> {
|
||||
return cluster_state_.GetInstances();
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::GetCurrentMainUUID() const -> utils::UUID { return cluster_state_.GetCurrentMainUUID(); }
|
||||
|
||||
auto CoordinatorStateMachine::IsCurrentMain(std::string_view instance_name) const -> bool {
|
||||
return cluster_state_.IsCurrentMain(instance_name);
|
||||
}
|
||||
auto CoordinatorStateMachine::GetCoordinatorInstances() const -> std::vector<CoordinatorInstanceState> {
|
||||
return cluster_state_.GetCoordinatorInstances();
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::GetInstanceUUID(std::string_view instance_name) const -> utils::UUID {
|
||||
return cluster_state_.GetInstanceUUID(instance_name);
|
||||
}
|
||||
|
||||
auto CoordinatorStateMachine::IsLockOpened() const -> bool { return cluster_state_.IsLockOpened(); }
|
||||
auto CoordinatorStateMachine::GetUUID() const -> utils::UUID { return cluster_state_.GetUUID(); }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -33,7 +33,6 @@ CoordinatorStateManager::CoordinatorStateManager(int srv_id, std::string const &
|
||||
auto CoordinatorStateManager::load_config() -> ptr<cluster_config> {
|
||||
// Just return in-memory data in this example.
|
||||
// May require reading from disk here, if it has been written to disk.
|
||||
spdlog::trace("Loading cluster config");
|
||||
return cluster_config_;
|
||||
}
|
||||
|
||||
@@ -42,11 +41,6 @@ auto CoordinatorStateManager::save_config(cluster_config const &config) -> void
|
||||
// Need to write to disk here, if want to make it durable.
|
||||
ptr<buffer> buf = config.serialize();
|
||||
cluster_config_ = cluster_config::deserialize(*buf);
|
||||
spdlog::info("Saving cluster config.");
|
||||
auto servers = cluster_config_->get_servers();
|
||||
for (auto const &server : servers) {
|
||||
spdlog::trace("Server id: {}, endpoint: {}", server->get_id(), server->get_endpoint());
|
||||
}
|
||||
}
|
||||
|
||||
auto CoordinatorStateManager::save_state(srv_state const &state) -> void {
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "rpc/client.hpp"
|
||||
#include "rpc_errors.hpp"
|
||||
@@ -25,11 +25,11 @@ namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
using HealthCheckClientCallback = std::function<void(CoordinatorInstance *, std::string_view)>;
|
||||
using ReplicationClientsInfo = std::vector<ReplicationClientInfo>;
|
||||
using ReplicationClientsInfo = std::vector<ReplClientInfo>;
|
||||
|
||||
class CoordinatorClient {
|
||||
public:
|
||||
explicit CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorToReplicaConfig config,
|
||||
explicit CoordinatorClient(CoordinatorInstance *coord_instance, CoordinatorClientConfig config,
|
||||
HealthCheckClientCallback succ_cb, HealthCheckClientCallback fail_cb);
|
||||
|
||||
~CoordinatorClient() = default;
|
||||
@@ -62,7 +62,7 @@ class CoordinatorClient {
|
||||
|
||||
auto SendGetInstanceUUIDRpc() const -> memgraph::utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>>;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplicationClientInfo;
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto SendGetInstanceTimestampsRpc() const
|
||||
-> utils::BasicResult<GetInstanceUUIDError, replication_coordination_glue::DatabaseHistories>;
|
||||
@@ -83,7 +83,7 @@ class CoordinatorClient {
|
||||
communication::ClientContext rpc_context_;
|
||||
mutable rpc::Client rpc_client_;
|
||||
|
||||
CoordinatorToReplicaConfig config_;
|
||||
CoordinatorClientConfig config_;
|
||||
CoordinatorInstance *coord_instance_;
|
||||
HealthCheckClientCallback succ_cb_;
|
||||
HealthCheckClientCallback fail_cb_;
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "replication_coordination_glue/mode.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include "json/json.hpp"
|
||||
#include "utils/uuid.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
|
||||
|
||||
struct ReplicationClientInfo {
|
||||
std::string instance_name{};
|
||||
replication_coordination_glue::ReplicationMode replication_mode{};
|
||||
io::network::Endpoint replication_server;
|
||||
|
||||
friend bool operator==(ReplicationClientInfo const &, ReplicationClientInfo const &) = default;
|
||||
};
|
||||
|
||||
struct CoordinatorToReplicaConfig {
|
||||
auto BoltSocketAddress() const -> std::string { return bolt_server.SocketAddress(); }
|
||||
auto CoordinatorSocketAddress() const -> std::string { return mgt_server.SocketAddress(); }
|
||||
auto ReplicationSocketAddress() const -> std::string {
|
||||
return replication_client_info.replication_server.SocketAddress();
|
||||
}
|
||||
|
||||
std::string instance_name{};
|
||||
io::network::Endpoint mgt_server;
|
||||
io::network::Endpoint bolt_server;
|
||||
ReplicationClientInfo replication_client_info;
|
||||
|
||||
std::chrono::seconds instance_health_check_frequency_sec{1};
|
||||
std::chrono::seconds instance_down_timeout_sec{5};
|
||||
std::chrono::seconds instance_get_uuid_frequency_sec{10};
|
||||
|
||||
struct SSL {
|
||||
std::string key_file;
|
||||
std::string cert_file;
|
||||
friend bool operator==(const SSL &, const SSL &) = default;
|
||||
};
|
||||
|
||||
std::optional<SSL> ssl;
|
||||
|
||||
friend bool operator==(CoordinatorToReplicaConfig const &, CoordinatorToReplicaConfig const &) = default;
|
||||
};
|
||||
|
||||
struct CoordinatorToCoordinatorConfig {
|
||||
uint32_t coordinator_server_id{0};
|
||||
io::network::Endpoint bolt_server;
|
||||
io::network::Endpoint coordinator_server;
|
||||
|
||||
friend bool operator==(CoordinatorToCoordinatorConfig const &, CoordinatorToCoordinatorConfig const &) = default;
|
||||
};
|
||||
|
||||
struct ManagementServerConfig {
|
||||
std::string ip_address;
|
||||
uint16_t port{};
|
||||
struct SSL {
|
||||
std::string key_file;
|
||||
std::string cert_file;
|
||||
std::string ca_file;
|
||||
bool verify_peer{};
|
||||
friend bool operator==(SSL const &, SSL const &) = default;
|
||||
};
|
||||
|
||||
std::optional<SSL> ssl;
|
||||
|
||||
friend bool operator==(ManagementServerConfig const &, ManagementServerConfig const &) = default;
|
||||
};
|
||||
|
||||
struct InstanceUUIDUpdate {
|
||||
std::string instance_name;
|
||||
memgraph::utils::UUID uuid;
|
||||
};
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorToReplicaConfig const &config);
|
||||
void from_json(nlohmann::json const &j, CoordinatorToReplicaConfig &config);
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorToCoordinatorConfig const &config);
|
||||
void from_json(nlohmann::json const &j, CoordinatorToCoordinatorConfig &config);
|
||||
|
||||
void to_json(nlohmann::json &j, ReplicationClientInfo const &config);
|
||||
void from_json(nlohmann::json const &j, ReplicationClientInfo &config);
|
||||
|
||||
void to_json(nlohmann::json &j, InstanceUUIDUpdate const &config);
|
||||
void from_json(nlohmann::json const &j, InstanceUUIDUpdate &config);
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
93
src/coordination/include/coordination/coordinator_config.hpp
Normal file
93
src/coordination/include/coordination/coordinator_config.hpp
Normal file
@@ -0,0 +1,93 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "replication_coordination_glue/mode.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include "json/json.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
|
||||
|
||||
struct CoordinatorClientConfig {
|
||||
std::string instance_name;
|
||||
std::string ip_address;
|
||||
uint16_t port{};
|
||||
std::chrono::seconds instance_health_check_frequency_sec{1};
|
||||
std::chrono::seconds instance_down_timeout_sec{5};
|
||||
std::chrono::seconds instance_get_uuid_frequency_sec{10};
|
||||
|
||||
auto CoordinatorSocketAddress() const -> std::string { return fmt::format("{}:{}", ip_address, port); }
|
||||
auto ReplicationSocketAddress() const -> std::string {
|
||||
return fmt::format("{}:{}", replication_client_info.replication_ip_address,
|
||||
replication_client_info.replication_port);
|
||||
}
|
||||
|
||||
struct ReplicationClientInfo {
|
||||
std::string instance_name;
|
||||
replication_coordination_glue::ReplicationMode replication_mode{};
|
||||
std::string replication_ip_address;
|
||||
uint16_t replication_port{};
|
||||
|
||||
friend bool operator==(ReplicationClientInfo const &, ReplicationClientInfo const &) = default;
|
||||
};
|
||||
|
||||
ReplicationClientInfo replication_client_info;
|
||||
|
||||
struct SSL {
|
||||
std::string key_file;
|
||||
std::string cert_file;
|
||||
|
||||
friend bool operator==(const SSL &, const SSL &) = default;
|
||||
};
|
||||
|
||||
std::optional<SSL> ssl;
|
||||
|
||||
friend bool operator==(CoordinatorClientConfig const &, CoordinatorClientConfig const &) = default;
|
||||
};
|
||||
|
||||
using ReplClientInfo = CoordinatorClientConfig::ReplicationClientInfo;
|
||||
|
||||
struct CoordinatorServerConfig {
|
||||
std::string ip_address;
|
||||
uint16_t port{};
|
||||
struct SSL {
|
||||
std::string key_file;
|
||||
std::string cert_file;
|
||||
std::string ca_file;
|
||||
bool verify_peer{};
|
||||
friend bool operator==(SSL const &, SSL const &) = default;
|
||||
};
|
||||
|
||||
std::optional<SSL> ssl;
|
||||
|
||||
friend bool operator==(CoordinatorServerConfig const &, CoordinatorServerConfig const &) = default;
|
||||
};
|
||||
|
||||
void to_json(nlohmann::json &j, CoordinatorClientConfig const &config);
|
||||
void from_json(nlohmann::json const &j, CoordinatorClientConfig &config);
|
||||
|
||||
void to_json(nlohmann::json &j, ReplClientInfo const &config);
|
||||
void from_json(nlohmann::json const &j, ReplClientInfo &config);
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
@@ -94,16 +94,5 @@ class InvalidRaftLogActionException final : public utils::BasicException {
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(InvalidRaftLogActionException)
|
||||
};
|
||||
|
||||
class InvalidRoutingTableException final : public utils::BasicException {
|
||||
public:
|
||||
explicit InvalidRoutingTableException(std::string_view what) noexcept : BasicException(what) {}
|
||||
|
||||
template <class... Args>
|
||||
explicit InvalidRoutingTableException(fmt::format_string<Args...> fmt, Args &&...args) noexcept
|
||||
: InvalidRoutingTableException(fmt::format(fmt, std::forward<Args>(args)...)) {}
|
||||
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(InvalidRoutingTableException)
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -26,8 +26,6 @@
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
using RoutingTable = std::vector<std::pair<std::vector<std::string>, std::string>>;
|
||||
|
||||
struct NewMainRes {
|
||||
std::string most_up_to_date_instance;
|
||||
std::string latest_epoch;
|
||||
@@ -38,14 +36,8 @@ using InstanceNameDbHistories = std::pair<std::string, replication_coordination_
|
||||
class CoordinatorInstance {
|
||||
public:
|
||||
CoordinatorInstance();
|
||||
CoordinatorInstance(CoordinatorInstance const &) = delete;
|
||||
CoordinatorInstance &operator=(CoordinatorInstance const &) = delete;
|
||||
CoordinatorInstance(CoordinatorInstance &&) noexcept = delete;
|
||||
CoordinatorInstance &operator=(CoordinatorInstance &&) noexcept = delete;
|
||||
|
||||
~CoordinatorInstance() = default;
|
||||
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorToReplicaConfig const &config)
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto UnregisterReplicationInstance(std::string_view instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus;
|
||||
@@ -56,17 +48,15 @@ class CoordinatorInstance {
|
||||
|
||||
auto TryFailover() -> void;
|
||||
|
||||
auto AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void;
|
||||
|
||||
auto GetRoutingTable(std::map<std::string, std::string> const &routing) -> RoutingTable;
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
|
||||
static auto ChooseMostUpToDateInstance(std::span<InstanceNameDbHistories> histories) -> NewMainRes;
|
||||
|
||||
auto HasMainState(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto HasReplicaState(std::string_view instance_name) const -> bool;
|
||||
|
||||
private:
|
||||
HealthCheckClientCallback client_succ_cb_, client_fail_cb_;
|
||||
|
||||
auto OnRaftCommitCallback(TRaftLog const &log_entry, RaftLogAction log_action) -> void;
|
||||
|
||||
auto FindReplicationInstance(std::string_view replication_instance_name) -> ReplicationInstance &;
|
||||
|
||||
void MainFailCallback(std::string_view);
|
||||
@@ -77,14 +67,14 @@ class CoordinatorInstance {
|
||||
|
||||
void ReplicaFailCallback(std::string_view);
|
||||
|
||||
HealthCheckClientCallback client_succ_cb_, client_fail_cb_;
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
// NOTE: Must be std::list because we rely on pointer stability.
|
||||
// Leader and followers should both have same view on repl_instances_
|
||||
std::list<ReplicationInstance> repl_instances_;
|
||||
mutable utils::ResourceLock coord_instance_lock_{};
|
||||
|
||||
// Thread pool needs to be constructed before raft state as raft state can call thread pool
|
||||
utils::ThreadPool thread_pool_;
|
||||
|
||||
RaftState raft_state_;
|
||||
};
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
#include "utils/uuid.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "rpc/messages.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
@@ -28,13 +28,14 @@ struct PromoteReplicaToMainReq {
|
||||
static void Load(PromoteReplicaToMainReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const PromoteReplicaToMainReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit PromoteReplicaToMainReq(const utils::UUID &uuid, std::vector<ReplicationClientInfo> replication_clients_info)
|
||||
explicit PromoteReplicaToMainReq(const utils::UUID &uuid,
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info)
|
||||
: main_uuid_(uuid), replication_clients_info(std::move(replication_clients_info)) {}
|
||||
PromoteReplicaToMainReq() = default;
|
||||
|
||||
// get uuid here
|
||||
utils::UUID main_uuid_;
|
||||
std::vector<ReplicationClientInfo> replication_clients_info;
|
||||
std::vector<CoordinatorClientConfig::ReplicationClientInfo> replication_clients_info;
|
||||
};
|
||||
|
||||
struct PromoteReplicaToMainRes {
|
||||
@@ -59,12 +60,12 @@ struct DemoteMainToReplicaReq {
|
||||
static void Load(DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(const DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit DemoteMainToReplicaReq(ReplicationClientInfo replication_client_info)
|
||||
explicit DemoteMainToReplicaReq(CoordinatorClientConfig::ReplicationClientInfo replication_client_info)
|
||||
: replication_client_info(std::move(replication_client_info)) {}
|
||||
|
||||
DemoteMainToReplicaReq() = default;
|
||||
|
||||
ReplicationClientInfo replication_client_info;
|
||||
CoordinatorClientConfig::ReplicationClientInfo replication_client_info;
|
||||
};
|
||||
|
||||
struct DemoteMainToReplicaRes {
|
||||
@@ -89,7 +90,7 @@ struct UnregisterReplicaReq {
|
||||
static void Load(UnregisterReplicaReq *self, memgraph::slk::Reader *reader);
|
||||
static void Save(UnregisterReplicaReq const &self, memgraph::slk::Builder *builder);
|
||||
|
||||
explicit UnregisterReplicaReq(std::string_view inst_name) : instance_name(inst_name) {}
|
||||
explicit UnregisterReplicaReq(std::string instance_name) : instance_name(std::move(instance_name)) {}
|
||||
|
||||
UnregisterReplicaReq() = default;
|
||||
|
||||
|
||||
@@ -13,14 +13,14 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "rpc/server.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorServer {
|
||||
public:
|
||||
explicit CoordinatorServer(const ManagementServerConfig &config);
|
||||
explicit CoordinatorServer(const CoordinatorServerConfig &config);
|
||||
CoordinatorServer(const CoordinatorServer &) = delete;
|
||||
CoordinatorServer(CoordinatorServer &&) = delete;
|
||||
CoordinatorServer &operator=(const CoordinatorServer &) = delete;
|
||||
|
||||
@@ -13,37 +13,27 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "replication_coordination_glue/common.hpp"
|
||||
#include "slk/serialization.hpp"
|
||||
#include "slk/streams.hpp"
|
||||
|
||||
namespace memgraph::slk {
|
||||
|
||||
using ReplicationClientInfo = coordination::ReplicationClientInfo;
|
||||
using ReplicationClientInfo = coordination::CoordinatorClientConfig::ReplicationClientInfo;
|
||||
|
||||
inline void Save(io::network::Endpoint const &obj, Builder *builder) {
|
||||
Save(obj.address, builder);
|
||||
Save(obj.port, builder);
|
||||
Save(obj.family, builder);
|
||||
}
|
||||
|
||||
inline void Load(io::network::Endpoint *obj, Reader *reader) {
|
||||
Load(&obj->address, reader);
|
||||
Load(&obj->port, reader);
|
||||
Load(&obj->family, reader);
|
||||
}
|
||||
|
||||
inline void Save(ReplicationClientInfo const &obj, Builder *builder) {
|
||||
inline void Save(const ReplicationClientInfo &obj, Builder *builder) {
|
||||
Save(obj.instance_name, builder);
|
||||
Save(obj.replication_mode, builder);
|
||||
Save(obj.replication_server, builder);
|
||||
Save(obj.replication_ip_address, builder);
|
||||
Save(obj.replication_port, builder);
|
||||
}
|
||||
|
||||
inline void Load(ReplicationClientInfo *obj, Reader *reader) {
|
||||
Load(&obj->instance_name, reader);
|
||||
Load(&obj->replication_mode, reader);
|
||||
Load(&obj->replication_server, reader);
|
||||
Load(&obj->replication_ip_address, reader);
|
||||
Load(&obj->replication_port, reader);
|
||||
}
|
||||
|
||||
inline void Save(const replication_coordination_glue::DatabaseHistory &obj, Builder *builder) {
|
||||
|
||||
@@ -33,7 +33,7 @@ class CoordinatorState {
|
||||
CoordinatorState(CoordinatorState &&) noexcept = delete;
|
||||
CoordinatorState &operator=(CoordinatorState &&) noexcept = delete;
|
||||
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorToReplicaConfig const &config)
|
||||
[[nodiscard]] auto RegisterReplicationInstance(CoordinatorClientConfig const &config)
|
||||
-> RegisterInstanceCoordinatorStatus;
|
||||
[[nodiscard]] auto UnregisterReplicationInstance(std::string_view instance_name)
|
||||
-> UnregisterInstanceCoordinatorStatus;
|
||||
@@ -42,13 +42,11 @@ class CoordinatorState {
|
||||
|
||||
auto ShowInstances() const -> std::vector<InstanceStatus>;
|
||||
|
||||
auto AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void;
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
|
||||
// NOTE: The client code must check that the server exists before calling this method.
|
||||
auto GetCoordinatorServer() const -> CoordinatorServer &;
|
||||
|
||||
auto GetRoutingTable(std::map<std::string, std::string> const &routing) -> RoutingTable;
|
||||
|
||||
private:
|
||||
struct CoordinatorMainReplicaData {
|
||||
std::unique_ptr<CoordinatorServer> coordinator_server_;
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include <flags/replication.hpp>
|
||||
#include "io/network/endpoint.hpp"
|
||||
#include "nuraft/coordinator_state_machine.hpp"
|
||||
#include "nuraft/coordinator_state_manager.hpp"
|
||||
|
||||
@@ -23,7 +22,7 @@
|
||||
namespace memgraph::coordination {
|
||||
|
||||
class CoordinatorInstance;
|
||||
struct CoordinatorToReplicaConfig;
|
||||
struct CoordinatorClientConfig;
|
||||
|
||||
using BecomeLeaderCb = std::function<void()>;
|
||||
using BecomeFollowerCb = std::function<void()>;
|
||||
@@ -40,7 +39,7 @@ using raft_result = nuraft::cmd_result<ptr<buffer>>;
|
||||
|
||||
class RaftState {
|
||||
private:
|
||||
explicit RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t coordinator_id,
|
||||
explicit RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t raft_server_id,
|
||||
uint32_t raft_port, std::string raft_address);
|
||||
|
||||
auto InitRaftServer() -> void;
|
||||
@@ -58,43 +57,31 @@ class RaftState {
|
||||
auto InstanceName() const -> std::string;
|
||||
auto RaftSocketAddress() const -> std::string;
|
||||
|
||||
auto AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void;
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
auto GetAllCoordinators() const -> std::vector<ptr<srv_config>>;
|
||||
|
||||
auto RequestLeadership() -> bool;
|
||||
auto IsLeader() const -> bool;
|
||||
|
||||
auto AppendRegisterReplicationInstanceLog(CoordinatorToReplicaConfig const &config) -> bool;
|
||||
auto AppendUnregisterReplicationInstanceLog(std::string_view instance_name) -> bool;
|
||||
auto AppendSetInstanceAsMainLog(std::string_view instance_name, utils::UUID const &uuid) -> bool;
|
||||
auto AppendSetInstanceAsReplicaLog(std::string_view instance_name) -> bool;
|
||||
auto AppendUpdateUUIDForNewMainLog(utils::UUID const &uuid) -> bool;
|
||||
auto AppendUpdateUUIDForInstanceLog(std::string_view instance_name, utils::UUID const &uuid) -> bool;
|
||||
auto AppendOpenLockRegister(CoordinatorToReplicaConfig const &) -> bool;
|
||||
auto AppendOpenLockUnregister(std::string_view) -> bool;
|
||||
auto AppendOpenLockFailover(std::string_view instance_name) -> bool;
|
||||
auto AppendOpenLockSetInstanceToMain(std::string_view instance_name) -> bool;
|
||||
auto AppendOpenLockSetInstanceToReplica(std::string_view instance_name) -> bool;
|
||||
auto AppendAddCoordinatorInstanceLog(CoordinatorToCoordinatorConfig const &config) -> bool;
|
||||
|
||||
auto GetReplicationInstances() const -> std::vector<ReplicationInstanceState>;
|
||||
// TODO: (andi) Do we need then GetAllCoordinators?
|
||||
auto GetCoordinatorInstances() const -> std::vector<CoordinatorInstanceState>;
|
||||
|
||||
auto FindCurrentMainInstanceName() const -> std::optional<std::string>;
|
||||
auto MainExists() const -> bool;
|
||||
auto HasMainState(std::string_view instance_name) const -> bool;
|
||||
auto HasReplicaState(std::string_view instance_name) const -> bool;
|
||||
auto IsCurrentMain(std::string_view instance_name) const -> bool;
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto GetCurrentMainUUID() const -> utils::UUID;
|
||||
auto GetInstanceUUID(std::string_view) const -> utils::UUID;
|
||||
auto AppendRegisterReplicationInstanceLog(CoordinatorClientConfig const &config) -> bool;
|
||||
auto AppendUnregisterReplicationInstanceLog(std::string_view instance_name) -> bool;
|
||||
auto AppendSetInstanceAsMainLog(std::string_view instance_name) -> bool;
|
||||
auto AppendSetInstanceAsReplicaLog(std::string_view instance_name) -> bool;
|
||||
auto AppendUpdateUUIDLog(utils::UUID const &uuid) -> bool;
|
||||
|
||||
auto IsLockOpened() const -> bool;
|
||||
auto GetInstances() const -> std::vector<InstanceState>;
|
||||
auto GetUUID() const -> utils::UUID;
|
||||
|
||||
private:
|
||||
// TODO: (andi) I think variables below can be abstracted/clean them.
|
||||
io::network::Endpoint raft_endpoint_;
|
||||
uint32_t coordinator_id_;
|
||||
uint32_t raft_server_id_;
|
||||
uint32_t raft_port_;
|
||||
std::string raft_address_;
|
||||
|
||||
ptr<CoordinatorStateMachine> state_machine_;
|
||||
ptr<CoordinatorStateManager> state_manager_;
|
||||
|
||||
@@ -25,9 +25,7 @@ enum class RegisterInstanceCoordinatorStatus : uint8_t {
|
||||
NOT_LEADER,
|
||||
RPC_FAILED,
|
||||
RAFT_LOG_ERROR,
|
||||
SUCCESS,
|
||||
LOCK_OPENED,
|
||||
OPEN_LOCK
|
||||
SUCCESS
|
||||
};
|
||||
|
||||
enum class UnregisterInstanceCoordinatorStatus : uint8_t {
|
||||
@@ -38,8 +36,6 @@ enum class UnregisterInstanceCoordinatorStatus : uint8_t {
|
||||
NOT_LEADER,
|
||||
RAFT_LOG_ERROR,
|
||||
SUCCESS,
|
||||
LOCK_OPENED,
|
||||
OPEN_LOCK
|
||||
};
|
||||
|
||||
enum class SetInstanceToMainCoordinatorStatus : uint8_t {
|
||||
@@ -51,9 +47,6 @@ enum class SetInstanceToMainCoordinatorStatus : uint8_t {
|
||||
COULD_NOT_PROMOTE_TO_MAIN,
|
||||
SWAP_UUID_FAILED,
|
||||
SUCCESS,
|
||||
LOCK_OPENED,
|
||||
OPEN_LOCK,
|
||||
ENABLE_WRITING_FAILED
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -32,7 +32,7 @@ using HealthCheckInstanceCallback = void (CoordinatorInstance::*)(std::string_vi
|
||||
|
||||
class ReplicationInstance {
|
||||
public:
|
||||
ReplicationInstance(CoordinatorInstance *peer, CoordinatorToReplicaConfig config, HealthCheckClientCallback succ_cb,
|
||||
ReplicationInstance(CoordinatorInstance *peer, CoordinatorClientConfig config, HealthCheckClientCallback succ_cb,
|
||||
HealthCheckClientCallback fail_cb, HealthCheckInstanceCallback succ_instance_cb,
|
||||
HealthCheckInstanceCallback fail_instance_cb);
|
||||
|
||||
@@ -67,7 +67,7 @@ class ReplicationInstance {
|
||||
auto PauseFrequentCheck() -> void;
|
||||
auto ResumeFrequentCheck() -> void;
|
||||
|
||||
auto ReplicationClientInfo() const -> ReplicationClientInfo;
|
||||
auto ReplicationClientInfo() const -> ReplClientInfo;
|
||||
|
||||
auto EnsureReplicaHasCorrectMainUUID(utils::UUID const &curr_main_uuid) -> bool;
|
||||
|
||||
@@ -79,6 +79,10 @@ class ReplicationInstance {
|
||||
|
||||
auto EnableWritingOnMain() -> bool;
|
||||
|
||||
auto SetNewMainUUID(utils::UUID const &main_uuid) -> void;
|
||||
auto ResetMainUUID() -> void;
|
||||
auto GetMainUUID() const -> std::optional<utils::UUID> const &;
|
||||
|
||||
auto GetSuccessCallback() -> HealthCheckInstanceCallback &;
|
||||
auto GetFailCallback() -> HealthCheckInstanceCallback &;
|
||||
|
||||
@@ -88,12 +92,19 @@ class ReplicationInstance {
|
||||
bool is_alive_{false};
|
||||
std::chrono::system_clock::time_point last_check_of_uuid_{};
|
||||
|
||||
// for replica this is main uuid of current main
|
||||
// for "main" main this same as in CoordinatorData
|
||||
// it is set to nullopt when replica is down
|
||||
// TLDR; when replica is down and comes back up we reset uuid of main replica is listening to
|
||||
// so we need to send swap uuid again
|
||||
std::optional<utils::UUID> main_uuid_;
|
||||
|
||||
HealthCheckInstanceCallback succ_cb_;
|
||||
HealthCheckInstanceCallback fail_cb_;
|
||||
|
||||
friend bool operator==(ReplicationInstance const &first, ReplicationInstance const &second) {
|
||||
return first.client_ == second.client_ && first.last_response_time_ == second.last_response_time_ &&
|
||||
first.is_alive_ == second.is_alive_;
|
||||
first.is_alive_ == second.is_alive_ && first.main_uuid_ == second.main_uuid_;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "nuraft/raft_log_action.hpp"
|
||||
#include "replication_coordination_glue/role.hpp"
|
||||
#include "utils/resource_lock.hpp"
|
||||
@@ -21,7 +21,6 @@
|
||||
|
||||
#include <libnuraft/nuraft.hxx>
|
||||
#include <range/v3/view.hpp>
|
||||
#include "json/json.hpp"
|
||||
|
||||
#include <map>
|
||||
#include <numeric>
|
||||
@@ -32,37 +31,12 @@ namespace memgraph::coordination {
|
||||
|
||||
using replication_coordination_glue::ReplicationRole;
|
||||
|
||||
struct ReplicationInstanceState {
|
||||
CoordinatorToReplicaConfig config;
|
||||
ReplicationRole status;
|
||||
|
||||
// for replica this is main uuid of current main
|
||||
// for "main" main this same as current_main_id_
|
||||
// when replica is down and comes back up we reset uuid of main replica is listening to
|
||||
// so we need to send swap uuid again
|
||||
// For MAIN we don't enable writing until cluster is in healthy state
|
||||
utils::UUID instance_uuid;
|
||||
|
||||
friend auto operator==(ReplicationInstanceState const &lhs, ReplicationInstanceState const &rhs) -> bool {
|
||||
return lhs.config == rhs.config && lhs.status == rhs.status && lhs.instance_uuid == rhs.instance_uuid;
|
||||
}
|
||||
struct InstanceState {
|
||||
CoordinatorClientConfig config;
|
||||
ReplicationRole role;
|
||||
};
|
||||
|
||||
// NOTE: Currently instance of coordinator doesn't change from the registration. Hence, just wrap
|
||||
// CoordinatorToCoordinatorConfig.
|
||||
struct CoordinatorInstanceState {
|
||||
CoordinatorToCoordinatorConfig config;
|
||||
|
||||
friend auto operator==(CoordinatorInstanceState const &lhs, CoordinatorInstanceState const &rhs) -> bool {
|
||||
return lhs.config == rhs.config;
|
||||
}
|
||||
};
|
||||
|
||||
void to_json(nlohmann::json &j, ReplicationInstanceState const &instance_state);
|
||||
void from_json(nlohmann::json const &j, ReplicationInstanceState &instance_state);
|
||||
|
||||
using TRaftLog = std::variant<CoordinatorToReplicaConfig, std::string, utils::UUID, CoordinatorToCoordinatorConfig,
|
||||
InstanceUUIDUpdate>;
|
||||
using TRaftLog = std::variant<CoordinatorClientConfig, std::string, utils::UUID>;
|
||||
|
||||
using nuraft::buffer;
|
||||
using nuraft::buffer_serializer;
|
||||
@@ -71,9 +45,6 @@ using nuraft::ptr;
|
||||
class CoordinatorClusterState {
|
||||
public:
|
||||
CoordinatorClusterState() = default;
|
||||
explicit CoordinatorClusterState(std::map<std::string, ReplicationInstanceState, std::less<>> instances,
|
||||
utils::UUID const ¤t_main_uuid, bool is_lock_opened);
|
||||
|
||||
CoordinatorClusterState(CoordinatorClusterState const &);
|
||||
CoordinatorClusterState &operator=(CoordinatorClusterState const &);
|
||||
|
||||
@@ -81,13 +52,15 @@ class CoordinatorClusterState {
|
||||
CoordinatorClusterState &operator=(CoordinatorClusterState &&other) noexcept;
|
||||
~CoordinatorClusterState() = default;
|
||||
|
||||
auto FindCurrentMainInstanceName() const -> std::optional<std::string>;
|
||||
|
||||
auto MainExists() const -> bool;
|
||||
|
||||
auto HasMainState(std::string_view instance_name) const -> bool;
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto HasReplicaState(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto IsCurrentMain(std::string_view instance_name) const -> bool;
|
||||
auto InsertInstance(std::string_view instance_name, ReplicationRole role) -> void;
|
||||
|
||||
auto DoAction(TRaftLog log_entry, RaftLogAction log_action) -> void;
|
||||
|
||||
@@ -95,22 +68,14 @@ class CoordinatorClusterState {
|
||||
|
||||
static auto Deserialize(buffer &data) -> CoordinatorClusterState;
|
||||
|
||||
auto GetReplicationInstances() const -> std::vector<ReplicationInstanceState>;
|
||||
auto GetInstances() const -> std::vector<InstanceState>;
|
||||
|
||||
auto GetCurrentMainUUID() const -> utils::UUID;
|
||||
|
||||
auto GetInstanceUUID(std::string_view) const -> utils::UUID;
|
||||
|
||||
auto IsLockOpened() const -> bool;
|
||||
|
||||
auto GetCoordinatorInstances() const -> std::vector<CoordinatorInstanceState>;
|
||||
auto GetUUID() const -> utils::UUID;
|
||||
|
||||
private:
|
||||
std::vector<CoordinatorInstanceState> coordinators_{};
|
||||
std::map<std::string, ReplicationInstanceState, std::less<>> repl_instances_{};
|
||||
utils::UUID current_main_uuid_{};
|
||||
std::map<std::string, InstanceState, std::less<>> instance_roles_;
|
||||
utils::UUID uuid_{};
|
||||
mutable utils::ResourceLock log_lock_{};
|
||||
bool is_lock_opened_{false};
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "nuraft/coordinator_cluster_state.hpp"
|
||||
#include "nuraft/raft_log_action.hpp"
|
||||
|
||||
@@ -40,21 +40,19 @@ class CoordinatorStateMachine : public state_machine {
|
||||
CoordinatorStateMachine &operator=(CoordinatorStateMachine const &) = delete;
|
||||
CoordinatorStateMachine(CoordinatorStateMachine &&) = delete;
|
||||
CoordinatorStateMachine &operator=(CoordinatorStateMachine &&) = delete;
|
||||
~CoordinatorStateMachine() override = default;
|
||||
~CoordinatorStateMachine() override {}
|
||||
|
||||
auto FindCurrentMainInstanceName() const -> std::optional<std::string>;
|
||||
auto MainExists() const -> bool;
|
||||
auto IsMain(std::string_view instance_name) const -> bool;
|
||||
auto IsReplica(std::string_view instance_name) const -> bool;
|
||||
|
||||
static auto CreateLog(nlohmann::json &&log) -> ptr<buffer>;
|
||||
static auto SerializeOpenLockRegister(CoordinatorToReplicaConfig const &config) -> ptr<buffer>;
|
||||
static auto SerializeOpenLockUnregister(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeOpenLockSetInstanceAsMain(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeOpenLockFailover(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeRegisterInstance(CoordinatorToReplicaConfig const &config) -> ptr<buffer>;
|
||||
static auto SerializeRegisterInstance(CoordinatorClientConfig const &config) -> ptr<buffer>;
|
||||
static auto SerializeUnregisterInstance(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeSetInstanceAsMain(InstanceUUIDUpdate const &instance_uuid_change) -> ptr<buffer>;
|
||||
static auto SerializeSetInstanceAsMain(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeSetInstanceAsReplica(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeUpdateUUIDForNewMain(utils::UUID const &uuid) -> ptr<buffer>;
|
||||
static auto SerializeUpdateUUIDForInstance(InstanceUUIDUpdate const &instance_uuid_change) -> ptr<buffer>;
|
||||
static auto SerializeAddCoordinatorInstance(CoordinatorToCoordinatorConfig const &config) -> ptr<buffer>;
|
||||
static auto SerializeOpenLockSetInstanceAsReplica(std::string_view instance_name) -> ptr<buffer>;
|
||||
static auto SerializeUpdateUUID(utils::UUID const &uuid) -> ptr<buffer>;
|
||||
|
||||
static auto DecodeLog(buffer &data) -> std::pair<TRaftLog, RaftLogAction>;
|
||||
|
||||
@@ -82,19 +80,8 @@ class CoordinatorStateMachine : public state_machine {
|
||||
|
||||
auto create_snapshot(snapshot &s, async_result<bool>::handler_type &when_done) -> void override;
|
||||
|
||||
auto GetReplicationInstances() const -> std::vector<ReplicationInstanceState>;
|
||||
|
||||
auto GetCoordinatorInstances() const -> std::vector<CoordinatorInstanceState>;
|
||||
|
||||
// Getters
|
||||
auto MainExists() const -> bool;
|
||||
auto HasMainState(std::string_view instance_name) const -> bool;
|
||||
auto HasReplicaState(std::string_view instance_name) const -> bool;
|
||||
auto IsCurrentMain(std::string_view instance_name) const -> bool;
|
||||
|
||||
auto GetCurrentMainUUID() const -> utils::UUID;
|
||||
auto GetInstanceUUID(std::string_view instance_name) const -> utils::UUID;
|
||||
auto IsLockOpened() const -> bool;
|
||||
auto GetInstances() const -> std::vector<InstanceState>;
|
||||
auto GetUUID() const -> utils::UUID;
|
||||
|
||||
private:
|
||||
struct SnapshotCtx {
|
||||
@@ -108,8 +95,12 @@ class CoordinatorStateMachine : public state_machine {
|
||||
auto create_snapshot_internal(ptr<snapshot> snapshot) -> void;
|
||||
|
||||
CoordinatorClusterState cluster_state_;
|
||||
|
||||
// mutable utils::RWLock lock{utils::RWLock::Priority::READ};
|
||||
|
||||
std::atomic<uint64_t> last_committed_idx_{0};
|
||||
|
||||
// TODO: (andi) Maybe not needed, remove it
|
||||
std::map<uint64_t, ptr<SnapshotCtx>> snapshots_;
|
||||
std::mutex snapshots_lock_;
|
||||
|
||||
|
||||
@@ -23,34 +23,41 @@
|
||||
namespace memgraph::coordination {
|
||||
|
||||
enum class RaftLogAction : uint8_t {
|
||||
OPEN_LOCK_REGISTER_REPLICATION_INSTANCE,
|
||||
OPEN_LOCK_UNREGISTER_REPLICATION_INSTANCE,
|
||||
OPEN_LOCK_FAILOVER,
|
||||
OPEN_LOCK_SET_INSTANCE_AS_MAIN,
|
||||
OPEN_LOCK_SET_INSTANCE_AS_REPLICA,
|
||||
REGISTER_REPLICATION_INSTANCE,
|
||||
UNREGISTER_REPLICATION_INSTANCE,
|
||||
SET_INSTANCE_AS_MAIN,
|
||||
SET_INSTANCE_AS_REPLICA,
|
||||
UPDATE_UUID_OF_NEW_MAIN,
|
||||
ADD_COORDINATOR_INSTANCE,
|
||||
UPDATE_UUID_FOR_INSTANCE,
|
||||
UPDATE_UUID
|
||||
};
|
||||
|
||||
NLOHMANN_JSON_SERIALIZE_ENUM(RaftLogAction,
|
||||
{{RaftLogAction::REGISTER_REPLICATION_INSTANCE, "register"},
|
||||
{RaftLogAction::UNREGISTER_REPLICATION_INSTANCE, "unregister"},
|
||||
{RaftLogAction::SET_INSTANCE_AS_MAIN, "promote"},
|
||||
{RaftLogAction::SET_INSTANCE_AS_REPLICA, "demote"},
|
||||
{RaftLogAction::UPDATE_UUID_OF_NEW_MAIN, "update_uuid_of_new_main"},
|
||||
{RaftLogAction::ADD_COORDINATOR_INSTANCE, "add_coordinator_instance"},
|
||||
{RaftLogAction::UPDATE_UUID_FOR_INSTANCE, "update_uuid_for_instance"},
|
||||
{RaftLogAction::OPEN_LOCK_REGISTER_REPLICATION_INSTANCE, "open_lock_register_instance"},
|
||||
{RaftLogAction::OPEN_LOCK_UNREGISTER_REPLICATION_INSTANCE,
|
||||
"open_lock_unregister_instance"},
|
||||
{RaftLogAction::OPEN_LOCK_FAILOVER, "open_lock_failover"},
|
||||
{RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_MAIN, "open_lock_set_instance_as_main"},
|
||||
{RaftLogAction::OPEN_LOCK_SET_INSTANCE_AS_REPLICA, "open_lock_set_instance_as_replica"}})
|
||||
NLOHMANN_JSON_SERIALIZE_ENUM(RaftLogAction, {
|
||||
{RaftLogAction::REGISTER_REPLICATION_INSTANCE, "register"},
|
||||
{RaftLogAction::UNREGISTER_REPLICATION_INSTANCE, "unregister"},
|
||||
{RaftLogAction::SET_INSTANCE_AS_MAIN, "promote"},
|
||||
{RaftLogAction::SET_INSTANCE_AS_REPLICA, "demote"},
|
||||
{RaftLogAction::UPDATE_UUID, "update_uuid"},
|
||||
})
|
||||
|
||||
inline auto ParseRaftLogAction(std::string_view action) -> RaftLogAction {
|
||||
if (action == "register") {
|
||||
return RaftLogAction::REGISTER_REPLICATION_INSTANCE;
|
||||
}
|
||||
if (action == "unregister") {
|
||||
return RaftLogAction::UNREGISTER_REPLICATION_INSTANCE;
|
||||
}
|
||||
if (action == "promote") {
|
||||
return RaftLogAction::SET_INSTANCE_AS_MAIN;
|
||||
}
|
||||
if (action == "demote") {
|
||||
return RaftLogAction::SET_INSTANCE_AS_REPLICA;
|
||||
}
|
||||
|
||||
if (action == "update_uuid") {
|
||||
return RaftLogAction::UPDATE_UUID;
|
||||
}
|
||||
|
||||
throw InvalidRaftLogActionException("Invalid Raft log action: {}.", action);
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -10,11 +10,11 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
#include <chrono>
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "coordination/raft_state.hpp"
|
||||
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/coordinator_exceptions.hpp"
|
||||
#include "utils/counter.hpp"
|
||||
|
||||
namespace memgraph::coordination {
|
||||
@@ -30,62 +30,54 @@ using nuraft::raft_server;
|
||||
using nuraft::srv_config;
|
||||
using raft_result = cmd_result<ptr<buffer>>;
|
||||
|
||||
RaftState::RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t coordinator_id,
|
||||
RaftState::RaftState(BecomeLeaderCb become_leader_cb, BecomeFollowerCb become_follower_cb, uint32_t raft_server_id,
|
||||
uint32_t raft_port, std::string raft_address)
|
||||
: raft_endpoint_(raft_address, raft_port),
|
||||
coordinator_id_(coordinator_id),
|
||||
: raft_server_id_(raft_server_id),
|
||||
raft_port_(raft_port),
|
||||
raft_address_(std::move(raft_address)),
|
||||
state_machine_(cs_new<CoordinatorStateMachine>()),
|
||||
state_manager_(cs_new<CoordinatorStateManager>(coordinator_id_, raft_endpoint_.SocketAddress())),
|
||||
state_manager_(
|
||||
cs_new<CoordinatorStateManager>(raft_server_id_, raft_address_ + ":" + std::to_string(raft_port_))),
|
||||
logger_(nullptr),
|
||||
become_leader_cb_(std::move(become_leader_cb)),
|
||||
become_follower_cb_(std::move(become_follower_cb)) {}
|
||||
|
||||
auto RaftState::InitRaftServer() -> void {
|
||||
asio_service::options asio_opts;
|
||||
asio_opts.thread_pool_size_ = 1;
|
||||
asio_opts.thread_pool_size_ = 1; // TODO: (andi) Improve this
|
||||
|
||||
raft_params params;
|
||||
params.heart_beat_interval_ = 100;
|
||||
params.election_timeout_lower_bound_ = 200;
|
||||
params.election_timeout_upper_bound_ = 400;
|
||||
// 5 logs are preserved before the last snapshot
|
||||
params.reserved_log_items_ = 5;
|
||||
// Create snapshot for every 5 log appends
|
||||
params.snapshot_distance_ = 5;
|
||||
params.client_req_timeout_ = 3000;
|
||||
params.return_method_ = raft_params::blocking;
|
||||
|
||||
// If the leader doesn't receive any response from quorum nodes
|
||||
// in 200ms, it will step down.
|
||||
// This allows us to achieve strong consistency even if network partition
|
||||
// happens between the current leader and followers.
|
||||
// The value must be <= election_timeout_lower_bound_ so that cluster can never
|
||||
// have multiple leaders.
|
||||
params.leadership_expiry_ = 200;
|
||||
|
||||
raft_server::init_options init_opts;
|
||||
|
||||
init_opts.raft_callback_ = [this](cb_func::Type event_type, cb_func::Param *param) -> nuraft::CbReturnCode {
|
||||
if (event_type == cb_func::BecomeLeader) {
|
||||
spdlog::info("Node {} became leader", param->leaderId);
|
||||
become_leader_cb_();
|
||||
} else if (event_type == cb_func::BecomeFollower) {
|
||||
// TODO(antoniofilipovic) Check what happens when becoming follower while doing failover
|
||||
// There is no way to stop becoming a follower:
|
||||
// https://github.com/eBay/NuRaft/blob/188947bcc73ce38ab1c3cf9d01015ca8a29decd9/src/raft_server.cxx#L1334-L1335
|
||||
spdlog::trace("Got request to become follower");
|
||||
spdlog::info("Node {} became follower", param->myId);
|
||||
become_follower_cb_();
|
||||
spdlog::trace("Node {} became follower", param->myId);
|
||||
}
|
||||
return CbReturnCode::Ok;
|
||||
};
|
||||
|
||||
raft_launcher launcher;
|
||||
|
||||
raft_server_ =
|
||||
launcher.init(state_machine_, state_manager_, logger_, raft_endpoint_.port, asio_opts, params, init_opts);
|
||||
raft_server_ = launcher.init(state_machine_, state_manager_, logger_, static_cast<int>(raft_port_), asio_opts, params,
|
||||
init_opts);
|
||||
|
||||
if (!raft_server_) {
|
||||
throw RaftServerStartException("Failed to launch raft server on {}", raft_endpoint_.SocketAddress());
|
||||
throw RaftServerStartException("Failed to launch raft server on {}:{}", raft_address_, raft_port_);
|
||||
}
|
||||
|
||||
auto maybe_stop = utils::ResettableCounter<20>();
|
||||
do {
|
||||
if (raft_server_->is_initialized()) {
|
||||
@@ -94,15 +86,15 @@ auto RaftState::InitRaftServer() -> void {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(250));
|
||||
} while (!maybe_stop());
|
||||
|
||||
throw RaftServerStartException("Failed to initialize raft server on {}", raft_endpoint_.SocketAddress());
|
||||
throw RaftServerStartException("Failed to initialize raft server on {}:{}", raft_address_, raft_port_);
|
||||
}
|
||||
|
||||
auto RaftState::MakeRaftState(BecomeLeaderCb &&become_leader_cb, BecomeFollowerCb &&become_follower_cb) -> RaftState {
|
||||
uint32_t coordinator_id = FLAGS_coordinator_id;
|
||||
uint32_t raft_port = FLAGS_coordinator_port;
|
||||
uint32_t raft_server_id = FLAGS_raft_server_id;
|
||||
uint32_t raft_port = FLAGS_raft_server_port;
|
||||
|
||||
auto raft_state =
|
||||
RaftState(std::move(become_leader_cb), std::move(become_follower_cb), coordinator_id, raft_port, "127.0.0.1");
|
||||
RaftState(std::move(become_leader_cb), std::move(become_follower_cb), raft_server_id, raft_port, "127.0.0.1");
|
||||
|
||||
raft_state.InitRaftServer();
|
||||
return raft_state;
|
||||
@@ -110,44 +102,18 @@ auto RaftState::MakeRaftState(BecomeLeaderCb &&become_leader_cb, BecomeFollowerC
|
||||
|
||||
RaftState::~RaftState() { launcher_.shutdown(); }
|
||||
|
||||
auto RaftState::InstanceName() const -> std::string {
|
||||
return fmt::format("coordinator_{}", std::to_string(coordinator_id_));
|
||||
}
|
||||
auto RaftState::InstanceName() const -> std::string { return "coordinator_" + std::to_string(raft_server_id_); }
|
||||
|
||||
auto RaftState::RaftSocketAddress() const -> std::string { return raft_endpoint_.SocketAddress(); }
|
||||
auto RaftState::RaftSocketAddress() const -> std::string { return raft_address_ + ":" + std::to_string(raft_port_); }
|
||||
|
||||
auto RaftState::AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void {
|
||||
auto const endpoint = config.coordinator_server.SocketAddress();
|
||||
srv_config const srv_config_to_add(static_cast<int>(config.coordinator_server_id), endpoint);
|
||||
|
||||
auto cmd_result = raft_server_->add_srv(srv_config_to_add);
|
||||
|
||||
if (cmd_result->get_result_code() == nuraft::cmd_result_code::OK) {
|
||||
spdlog::info("Request to add server {} to the cluster accepted", endpoint);
|
||||
} else {
|
||||
throw RaftAddServerException("Failed to accept request to add server {} to the cluster with error code {}",
|
||||
endpoint, int(cmd_result->get_result_code()));
|
||||
}
|
||||
|
||||
// Waiting for server to join
|
||||
constexpr int max_tries{10};
|
||||
auto maybe_stop = utils::ResettableCounter<max_tries>();
|
||||
constexpr int waiting_period{200};
|
||||
bool added{false};
|
||||
while (!maybe_stop()) {
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(waiting_period));
|
||||
const auto server_config = raft_server_->get_srv_config(static_cast<nuraft::int32>(config.coordinator_server_id));
|
||||
if (server_config) {
|
||||
spdlog::trace("Server with id {} added to cluster", config.coordinator_server_id);
|
||||
added = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!added) {
|
||||
throw RaftAddServerException("Failed to add server {} to the cluster in {}ms", endpoint,
|
||||
max_tries * waiting_period);
|
||||
auto RaftState::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address)
|
||||
-> void {
|
||||
auto const endpoint = fmt::format("{}:{}", raft_address, raft_port);
|
||||
srv_config const srv_config_to_add(static_cast<int>(raft_server_id), endpoint);
|
||||
if (!raft_server_->add_srv(srv_config_to_add)->get_accepted()) {
|
||||
throw RaftAddServerException("Failed to add server {} to the cluster", endpoint);
|
||||
}
|
||||
spdlog::info("Request to add server {} to the cluster accepted", endpoint);
|
||||
}
|
||||
|
||||
auto RaftState::GetAllCoordinators() const -> std::vector<ptr<srv_config>> {
|
||||
@@ -160,79 +126,7 @@ auto RaftState::IsLeader() const -> bool { return raft_server_->is_leader(); }
|
||||
|
||||
auto RaftState::RequestLeadership() -> bool { return raft_server_->is_leader() || raft_server_->request_leadership(); }
|
||||
|
||||
auto RaftState::AppendOpenLockRegister(CoordinatorToReplicaConfig const &config) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeOpenLockRegister(config);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error("Failed to accept request to open lock to register instance {}", config.instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to open lock for registering instance {} with error code {}", config.instance_name,
|
||||
int(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendOpenLockUnregister(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeOpenLockUnregister(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error("Failed to accept request to open lock to unregister instance {}.", instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to open lock for unregistering instance {} with error code {}", instance_name,
|
||||
int(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendOpenLockFailover(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeOpenLockFailover(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error("Failed to accept request to open lock for failover {}", instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to open lock for failover to instance {} with error code {}", instance_name,
|
||||
int(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendOpenLockSetInstanceToMain(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeOpenLockSetInstanceAsMain(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error("Failed to accept request to open lock and set instance {} to MAIN", instance_name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to open lock to set instance {} to MAIN with error code {}", instance_name,
|
||||
int(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendRegisterReplicationInstanceLog(CoordinatorToReplicaConfig const &config) -> bool {
|
||||
auto RaftState::AppendRegisterReplicationInstanceLog(CoordinatorClientConfig const &config) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeRegisterInstance(config);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
|
||||
@@ -248,8 +142,7 @@ auto RaftState::AppendRegisterReplicationInstanceLog(CoordinatorToReplicaConfig
|
||||
spdlog::info("Request for registering instance {} accepted", config.instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to register instance {} with error code {}", config.instance_name,
|
||||
int(res->get_result_code()));
|
||||
spdlog::error("Failed to register instance {} with error code {}", config.instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -270,15 +163,14 @@ auto RaftState::AppendUnregisterReplicationInstanceLog(std::string_view instance
|
||||
spdlog::info("Request for unregistering instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to unregister instance {} with error code {}", instance_name, int(res->get_result_code()));
|
||||
spdlog::error("Failed to unregister instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendSetInstanceAsMainLog(std::string_view instance_name, utils::UUID const &uuid) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeSetInstanceAsMain(
|
||||
InstanceUUIDUpdate{.instance_name = std::string{instance_name}, .uuid = uuid});
|
||||
auto RaftState::AppendSetInstanceAsMainLog(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeSetInstanceAsMain(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
@@ -291,7 +183,7 @@ auto RaftState::AppendSetInstanceAsMainLog(std::string_view instance_name, utils
|
||||
spdlog::info("Request for promoting instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, int(res->get_result_code()));
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@@ -310,35 +202,15 @@ auto RaftState::AppendSetInstanceAsReplicaLog(std::string_view instance_name) ->
|
||||
spdlog::info("Request for demoting instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, int(res->get_result_code()));
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendOpenLockSetInstanceToReplica(std::string_view instance_name) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeOpenLockSetInstanceAsReplica(instance_name);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for demoting instance {}. Most likely the reason is that the instance is not "
|
||||
"the leader.",
|
||||
instance_name);
|
||||
return false;
|
||||
}
|
||||
spdlog::info("Request for demoting instance {} accepted", instance_name);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to promote instance {} with error code {}", instance_name, int(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendUpdateUUIDForNewMainLog(utils::UUID const &uuid) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeUpdateUUIDForNewMain(uuid);
|
||||
auto RaftState::AppendUpdateUUIDLog(utils::UUID const &uuid) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeUpdateUUID(uuid);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
@@ -346,85 +218,31 @@ auto RaftState::AppendUpdateUUIDForNewMainLog(utils::UUID const &uuid) -> bool {
|
||||
"the leader.");
|
||||
return false;
|
||||
}
|
||||
spdlog::trace("Request for updating UUID accepted");
|
||||
spdlog::info("Request for updating UUID accepted");
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to update UUID with error code {}", int(res->get_result_code()));
|
||||
spdlog::error("Failed to update UUID with error code {}", res->get_result_code());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendAddCoordinatorInstanceLog(CoordinatorToCoordinatorConfig const &config) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeAddCoordinatorInstance(config);
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error(
|
||||
"Failed to accept request for adding coordinator instance {}. Most likely the reason is that the instance is "
|
||||
"not the leader.",
|
||||
config.coordinator_server_id);
|
||||
return false;
|
||||
}
|
||||
|
||||
spdlog::info("Request for adding coordinator instance {} accepted", config.coordinator_server_id);
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to add coordinator instance {} with error code {}", config.coordinator_server_id,
|
||||
static_cast<int>(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
auto RaftState::AppendUpdateUUIDForInstanceLog(std::string_view instance_name, const utils::UUID &uuid) -> bool {
|
||||
auto new_log = CoordinatorStateMachine::SerializeUpdateUUIDForInstance(
|
||||
{.instance_name = std::string{instance_name}, .uuid = uuid});
|
||||
auto const res = raft_server_->append_entries({new_log});
|
||||
if (!res->get_accepted()) {
|
||||
spdlog::error("Failed to accept request for updating UUID of instance.");
|
||||
return false;
|
||||
}
|
||||
spdlog::trace("Request for updating UUID of instance accepted");
|
||||
|
||||
if (res->get_result_code() != nuraft::cmd_result_code::OK) {
|
||||
spdlog::error("Failed to update UUID of instance with error code {}", int(res->get_result_code()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
auto RaftState::FindCurrentMainInstanceName() const -> std::optional<std::string> {
|
||||
return state_machine_->FindCurrentMainInstanceName();
|
||||
}
|
||||
|
||||
auto RaftState::MainExists() const -> bool { return state_machine_->MainExists(); }
|
||||
|
||||
auto RaftState::HasMainState(std::string_view instance_name) const -> bool {
|
||||
return state_machine_->HasMainState(instance_name);
|
||||
auto RaftState::IsMain(std::string_view instance_name) const -> bool { return state_machine_->IsMain(instance_name); }
|
||||
|
||||
auto RaftState::IsReplica(std::string_view instance_name) const -> bool {
|
||||
return state_machine_->IsReplica(instance_name);
|
||||
}
|
||||
|
||||
auto RaftState::HasReplicaState(std::string_view instance_name) const -> bool {
|
||||
return state_machine_->HasReplicaState(instance_name);
|
||||
}
|
||||
auto RaftState::GetInstances() const -> std::vector<InstanceState> { return state_machine_->GetInstances(); }
|
||||
|
||||
auto RaftState::GetReplicationInstances() const -> std::vector<ReplicationInstanceState> {
|
||||
return state_machine_->GetReplicationInstances();
|
||||
}
|
||||
|
||||
auto RaftState::GetCurrentMainUUID() const -> utils::UUID { return state_machine_->GetCurrentMainUUID(); }
|
||||
|
||||
auto RaftState::IsCurrentMain(std::string_view instance_name) const -> bool {
|
||||
return state_machine_->IsCurrentMain(instance_name);
|
||||
}
|
||||
|
||||
auto RaftState::IsLockOpened() const -> bool { return state_machine_->IsLockOpened(); }
|
||||
|
||||
auto RaftState::GetInstanceUUID(std::string_view instance_name) const -> utils::UUID {
|
||||
return state_machine_->GetInstanceUUID(instance_name);
|
||||
}
|
||||
|
||||
auto RaftState::GetCoordinatorInstances() const -> std::vector<CoordinatorInstanceState> {
|
||||
return state_machine_->GetCoordinatorInstances();
|
||||
}
|
||||
auto RaftState::GetUUID() const -> utils::UUID { return state_machine_->GetUUID(); }
|
||||
|
||||
} // namespace memgraph::coordination
|
||||
#endif
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
|
||||
namespace memgraph::coordination {
|
||||
|
||||
ReplicationInstance::ReplicationInstance(CoordinatorInstance *peer, CoordinatorToReplicaConfig config,
|
||||
ReplicationInstance::ReplicationInstance(CoordinatorInstance *peer, CoordinatorClientConfig config,
|
||||
HealthCheckClientCallback succ_cb, HealthCheckClientCallback fail_cb,
|
||||
HealthCheckInstanceCallback succ_instance_cb,
|
||||
HealthCheckInstanceCallback fail_instance_cb)
|
||||
@@ -56,6 +56,7 @@ auto ReplicationInstance::PromoteToMain(utils::UUID const &new_uuid, Replication
|
||||
return false;
|
||||
}
|
||||
|
||||
main_uuid_ = new_uuid;
|
||||
succ_cb_ = main_succ_cb;
|
||||
fail_cb_ = main_fail_cb;
|
||||
|
||||
@@ -81,7 +82,7 @@ auto ReplicationInstance::StopFrequentCheck() -> void { client_.StopFrequentChec
|
||||
auto ReplicationInstance::PauseFrequentCheck() -> void { client_.PauseFrequentCheck(); }
|
||||
auto ReplicationInstance::ResumeFrequentCheck() -> void { client_.ResumeFrequentCheck(); }
|
||||
|
||||
auto ReplicationInstance::ReplicationClientInfo() const -> coordination::ReplicationClientInfo {
|
||||
auto ReplicationInstance::ReplicationClientInfo() const -> CoordinatorClientConfig::ReplicationClientInfo {
|
||||
return client_.ReplicationClientInfo();
|
||||
}
|
||||
|
||||
@@ -90,6 +91,9 @@ auto ReplicationInstance::GetFailCallback() -> HealthCheckInstanceCallback & { r
|
||||
|
||||
auto ReplicationInstance::GetClient() -> CoordinatorClient & { return client_; }
|
||||
|
||||
auto ReplicationInstance::SetNewMainUUID(utils::UUID const &main_uuid) -> void { main_uuid_ = main_uuid; }
|
||||
auto ReplicationInstance::GetMainUUID() const -> std::optional<utils::UUID> const & { return main_uuid_; }
|
||||
|
||||
auto ReplicationInstance::EnsureReplicaHasCorrectMainUUID(utils::UUID const &curr_main_uuid) -> bool {
|
||||
if (!IsReadyForUUIDPing()) {
|
||||
return true;
|
||||
@@ -112,6 +116,7 @@ auto ReplicationInstance::SendSwapAndUpdateUUID(utils::UUID const &new_main_uuid
|
||||
if (!replication_coordination_glue::SendSwapMainUUIDRpc(client_.RpcClient(), new_main_uuid)) {
|
||||
return false;
|
||||
}
|
||||
SetNewMainUUID(new_main_uuid);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -119,8 +119,6 @@ class Reader {
|
||||
auto GetHeader() const -> Header const &;
|
||||
auto GetNextRow(utils::MemoryResource *mem) -> std::optional<Row>;
|
||||
|
||||
void Reset();
|
||||
|
||||
private:
|
||||
// Some implementation issues that need clearing up, but this is mainly because
|
||||
// I don't want `boost/iostreams/filtering_stream.hpp` included in this header file
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -34,10 +34,6 @@ struct Reader::impl {
|
||||
|
||||
[[nodiscard]] bool HasHeader() const { return read_config_.with_header; }
|
||||
[[nodiscard]] auto Header() const -> Header const & { return header_; }
|
||||
void Reset() {
|
||||
line_buffer_.clear();
|
||||
line_buffer_.shrink_to_fit();
|
||||
}
|
||||
|
||||
auto GetNextRow(utils::MemoryResource *mem) -> std::optional<Reader::Row>;
|
||||
|
||||
@@ -46,7 +42,7 @@ struct Reader::impl {
|
||||
|
||||
void TryInitializeHeader();
|
||||
|
||||
bool GetNextLine();
|
||||
std::optional<utils::pmr::string> GetNextLine(utils::MemoryResource *mem);
|
||||
|
||||
ParsingResult ParseHeader();
|
||||
|
||||
@@ -59,8 +55,6 @@ struct Reader::impl {
|
||||
Config read_config_;
|
||||
uint64_t line_count_{1};
|
||||
uint16_t number_of_columns_{0};
|
||||
uint64_t estimated_number_of_columns_{0};
|
||||
utils::pmr::string line_buffer_{memory_};
|
||||
Reader::Header header_{memory_};
|
||||
};
|
||||
|
||||
@@ -135,16 +129,17 @@ void Reader::impl::InitializeStream() {
|
||||
MG_ASSERT(csv_stream_.is_complete(), "Should be 'complete' for correct operation");
|
||||
}
|
||||
|
||||
bool Reader::impl::GetNextLine() {
|
||||
if (!std::getline(csv_stream_, line_buffer_)) {
|
||||
std::optional<utils::pmr::string> Reader::impl::GetNextLine(utils::MemoryResource *mem) {
|
||||
utils::pmr::string line(mem);
|
||||
if (!std::getline(csv_stream_, line)) {
|
||||
// reached end of file or an I/0 error occurred
|
||||
if (!csv_stream_.good()) {
|
||||
csv_stream_.reset(); // this will close the file_stream_ and clear the chain
|
||||
}
|
||||
return false;
|
||||
return std::nullopt;
|
||||
}
|
||||
++line_count_;
|
||||
return true;
|
||||
return std::move(line);
|
||||
}
|
||||
|
||||
Reader::ParsingResult Reader::impl::ParseHeader() {
|
||||
@@ -175,8 +170,6 @@ void Reader::impl::TryInitializeHeader() {
|
||||
|
||||
const Reader::Header &Reader::GetHeader() const { return pimpl->Header(); }
|
||||
|
||||
void Reader::Reset() { pimpl->Reset(); }
|
||||
|
||||
namespace {
|
||||
enum class CsvParserState : uint8_t { INITIAL_FIELD, NEXT_FIELD, QUOTING, EXPECT_DELIMITER, DONE };
|
||||
|
||||
@@ -186,8 +179,6 @@ Reader::ParsingResult Reader::impl::ParseRow(utils::MemoryResource *mem) {
|
||||
utils::pmr::vector<utils::pmr::string> row(mem);
|
||||
if (number_of_columns_ != 0) {
|
||||
row.reserve(number_of_columns_);
|
||||
} else if (estimated_number_of_columns_ != 0) {
|
||||
row.reserve(estimated_number_of_columns_);
|
||||
}
|
||||
|
||||
utils::pmr::string column(memory_);
|
||||
@@ -195,12 +186,13 @@ Reader::ParsingResult Reader::impl::ParseRow(utils::MemoryResource *mem) {
|
||||
auto state = CsvParserState::INITIAL_FIELD;
|
||||
|
||||
do {
|
||||
if (!GetNextLine()) {
|
||||
const auto maybe_line = GetNextLine(mem);
|
||||
if (!maybe_line) {
|
||||
// The whole file was processed.
|
||||
break;
|
||||
}
|
||||
|
||||
std::string_view line_string_view = line_buffer_;
|
||||
std::string_view line_string_view = *maybe_line;
|
||||
|
||||
// remove '\r' from the end in case we have dos file format
|
||||
if (line_string_view.back() == '\r') {
|
||||
@@ -320,11 +312,6 @@ Reader::ParsingResult Reader::impl::ParseRow(utils::MemoryResource *mem) {
|
||||
fmt::format("Expected {:d} columns in row {:d}, but got {:d}", number_of_columns_,
|
||||
line_count_ - 1, row.size()));
|
||||
}
|
||||
// To avoid unessisary dynamic growth of the row, remember the number of
|
||||
// columns for future calls
|
||||
if (number_of_columns_ == 0 && estimated_number_of_columns_ == 0) {
|
||||
estimated_number_of_columns_ = row.size();
|
||||
}
|
||||
|
||||
return std::move(row);
|
||||
}
|
||||
@@ -332,7 +319,7 @@ Reader::ParsingResult Reader::impl::ParseRow(utils::MemoryResource *mem) {
|
||||
std::optional<Reader::Row> Reader::impl::GetNextRow(utils::MemoryResource *mem) {
|
||||
auto row = ParseRow(mem);
|
||||
|
||||
if (row.HasError()) [[unlikely]] {
|
||||
if (row.HasError()) {
|
||||
if (!read_config_.ignore_bad) {
|
||||
throw CsvReadException("CSV Reader: Bad row at line {:d}: {}", line_count_ - 1, row.GetError().message);
|
||||
}
|
||||
@@ -346,7 +333,7 @@ std::optional<Reader::Row> Reader::impl::GetNextRow(utils::MemoryResource *mem)
|
||||
} while (row.HasError());
|
||||
}
|
||||
|
||||
if (row->empty()) [[unlikely]] {
|
||||
if (row->empty()) {
|
||||
// reached end of file
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ namespace memgraph::dbms {
|
||||
CoordinatorHandler::CoordinatorHandler(coordination::CoordinatorState &coordinator_state)
|
||||
: coordinator_state_(coordinator_state) {}
|
||||
|
||||
auto CoordinatorHandler::RegisterReplicationInstance(coordination::CoordinatorToReplicaConfig const &config)
|
||||
auto CoordinatorHandler::RegisterReplicationInstance(coordination::CoordinatorClientConfig const &config)
|
||||
-> coordination::RegisterInstanceCoordinatorStatus {
|
||||
return coordinator_state_.RegisterReplicationInstance(config);
|
||||
}
|
||||
@@ -39,8 +39,9 @@ auto CoordinatorHandler::ShowInstances() const -> std::vector<coordination::Inst
|
||||
return coordinator_state_.ShowInstances();
|
||||
}
|
||||
|
||||
auto CoordinatorHandler::AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void {
|
||||
coordinator_state_.AddCoordinatorInstance(config);
|
||||
auto CoordinatorHandler::AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port,
|
||||
std::string_view raft_address) -> void {
|
||||
coordinator_state_.AddCoordinatorInstance(raft_server_id, raft_port, raft_address);
|
||||
}
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
#include "coordination/coordinator_communication_config.hpp"
|
||||
#include "coordination/coordinator_config.hpp"
|
||||
#include "coordination/coordinator_state.hpp"
|
||||
#include "coordination/instance_status.hpp"
|
||||
#include "coordination/register_main_replica_coordinator_status.hpp"
|
||||
@@ -30,7 +30,7 @@ class CoordinatorHandler {
|
||||
|
||||
// TODO: (andi) When moving coordinator state on same instances, rename from RegisterReplicationInstance to
|
||||
// RegisterInstance
|
||||
auto RegisterReplicationInstance(coordination::CoordinatorToReplicaConfig const &config)
|
||||
auto RegisterReplicationInstance(coordination::CoordinatorClientConfig const &config)
|
||||
-> coordination::RegisterInstanceCoordinatorStatus;
|
||||
|
||||
auto UnregisterReplicationInstance(std::string_view instance_name)
|
||||
@@ -40,7 +40,7 @@ class CoordinatorHandler {
|
||||
|
||||
auto ShowInstances() const -> std::vector<coordination::InstanceStatus>;
|
||||
|
||||
auto AddCoordinatorInstance(coordination::CoordinatorToCoordinatorConfig const &config) -> void;
|
||||
auto AddCoordinatorInstance(uint32_t raft_server_id, uint32_t raft_port, std::string_view raft_address) -> void;
|
||||
|
||||
private:
|
||||
coordination::CoordinatorState &coordinator_state_;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -311,7 +311,7 @@ class DbmsHandler {
|
||||
stats.triggers += info.triggers;
|
||||
stats.streams += info.streams;
|
||||
++stats.num_databases;
|
||||
stats.indices += storage_info.label_indices + storage_info.label_property_indices + storage_info.text_indices;
|
||||
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
|
||||
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
|
||||
++stats.storage_modes[(int)storage_info.storage_mode];
|
||||
++stats.isolation_levels[(int)storage_info.isolation_level];
|
||||
|
||||
@@ -589,6 +589,7 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
|
||||
if (timestamp < storage->timestamp_) {
|
||||
continue;
|
||||
}
|
||||
|
||||
SPDLOG_INFO(" Delta {}", applied_deltas);
|
||||
switch (delta.type) {
|
||||
case WalDeltaData::Type::VERTEX_CREATE: {
|
||||
@@ -615,7 +616,6 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
|
||||
auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW);
|
||||
if (!vertex)
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
// NOTE: Text search doesn’t have replication in scope yet (Phases 1 and 2)
|
||||
auto ret = vertex->AddLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label));
|
||||
if (ret.HasError() || !ret.GetValue())
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
@@ -628,21 +628,18 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
|
||||
auto vertex = transaction->FindVertex(delta.vertex_add_remove_label.gid, View::NEW);
|
||||
if (!vertex)
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
// NOTE: Text search doesn’t have replication in scope yet (Phases 1 and 2)
|
||||
auto ret = vertex->RemoveLabel(transaction->NameToLabel(delta.vertex_add_remove_label.label));
|
||||
if (ret.HasError() || !ret.GetValue())
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
|
||||
spdlog::trace(" Vertex {} set property", delta.vertex_edge_set_property.gid.AsUint());
|
||||
// NOLINTNEXTLINE
|
||||
spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
|
||||
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
// NOLINTNEXTLINE
|
||||
auto vertex = transaction->FindVertex(delta.vertex_edge_set_property.gid, View::NEW);
|
||||
if (!vertex)
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
// NOTE: Phase 1 of the text search feature doesn't have replication in scope
|
||||
auto ret = vertex->SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
|
||||
delta.vertex_edge_set_property.value);
|
||||
if (ret.HasError())
|
||||
@@ -687,7 +684,8 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::EDGE_SET_PROPERTY: {
|
||||
spdlog::trace(" Edge {} set property", delta.vertex_edge_set_property.gid.AsUint());
|
||||
spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
|
||||
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
|
||||
if (!storage->config_.salient.items.properties_on_edges)
|
||||
throw utils::BasicException(
|
||||
"Can't set properties on edges because properties on edges "
|
||||
@@ -842,28 +840,6 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
|
||||
transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label));
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::EDGE_INDEX_CREATE: {
|
||||
spdlog::trace(" Create edge index on :{}", delta.operation_edge_type.edge_type);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
if (transaction->CreateIndex(storage->NameToEdgeType(delta.operation_label.label)).HasError())
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::EDGE_INDEX_DROP: {
|
||||
spdlog::trace(" Drop edge index on :{}", delta.operation_edge_type.edge_type);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
if (transaction->DropIndex(storage->NameToEdgeType(delta.operation_label.label)).HasError())
|
||||
throw utils::BasicException("Invalid transaction! Please raise an issue, {}:{}", __FILE__, __LINE__);
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::TEXT_INDEX_CREATE: {
|
||||
// NOTE: Text search doesn’t have replication in scope yet (Phases 1 and 2)
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::TEXT_INDEX_DROP: {
|
||||
// NOTE: Text search doesn’t have replication in scope yet (Phases 1 and 2)
|
||||
break;
|
||||
}
|
||||
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
|
||||
spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
|
||||
delta.operation_label_property.property);
|
||||
@@ -927,4 +903,5 @@ uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage
|
||||
spdlog::debug("Applied {} deltas", applied_deltas);
|
||||
return applied_deltas;
|
||||
}
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -18,15 +18,14 @@
|
||||
|
||||
// Bolt server flags.
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_string(
|
||||
experimental_enabled, "",
|
||||
"Experimental features to be used, comma-separated. Options [system-replication, text-search, high-availability]");
|
||||
DEFINE_string(experimental_enabled, "",
|
||||
"Experimental features to be used, comma seperated. Options [system-replication, high-availability]");
|
||||
|
||||
using namespace std::string_view_literals;
|
||||
|
||||
namespace memgraph::flags {
|
||||
|
||||
auto const mapping = std::map{std::pair{"system-replication"sv, Experiments::SYSTEM_REPLICATION},
|
||||
std::pair{"text-search"sv, Experiments::TEXT_SEARCH},
|
||||
std::pair{"high-availability"sv, Experiments::HIGH_AVAILABILITY}};
|
||||
|
||||
auto ExperimentsInstance() -> Experiments & {
|
||||
@@ -46,7 +45,7 @@ bool AreExperimentsEnabled(Experiments experiments) {
|
||||
void InitializeExperimental() {
|
||||
namespace rv = ranges::views;
|
||||
|
||||
auto const canonicalize_string = [](auto &&rng) {
|
||||
auto const connonicalize_string = [](auto &&rng) {
|
||||
auto const is_space = [](auto c) { return c == ' '; };
|
||||
auto const to_lower = [](unsigned char c) { return std::tolower(c); };
|
||||
|
||||
@@ -57,7 +56,7 @@ void InitializeExperimental() {
|
||||
auto const mapping_end = mapping.cend();
|
||||
using underlying_type = std::underlying_type_t<Experiments>;
|
||||
auto to_set = underlying_type{};
|
||||
for (auto &&experiment : FLAGS_experimental_enabled | rv::split(',') | rv::transform(canonicalize_string)) {
|
||||
for (auto &&experiment : FLAGS_experimental_enabled | rv::split(',') | rv::transform(connonicalize_string)) {
|
||||
if (auto it = mapping.find(experiment); it != mapping_end) {
|
||||
to_set |= static_cast<underlying_type>(it->second);
|
||||
}
|
||||
|
||||
@@ -23,8 +23,7 @@ namespace memgraph::flags {
|
||||
// old experiments can be reused once code cleanup has happened
|
||||
enum class Experiments : uint8_t {
|
||||
SYSTEM_REPLICATION = 1 << 0,
|
||||
TEXT_SEARCH = 1 << 1,
|
||||
HIGH_AVAILABILITY = 1 << 2,
|
||||
HIGH_AVAILABILITY = 1 << 1,
|
||||
};
|
||||
|
||||
bool AreExperimentsEnabled(Experiments experiments);
|
||||
|
||||
@@ -131,10 +131,6 @@ DEFINE_uint64(storage_recovery_thread_count,
|
||||
DEFINE_bool(storage_enable_schema_metadata, false,
|
||||
"Controls whether metadata should be collected about the resident labels and edge types.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(storage_delta_on_identical_property_update, true,
|
||||
"Controls whether updating a property with the same value should create a delta object.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(telemetry_enabled, false,
|
||||
"Set to true to enable telemetry. We collect information about the "
|
||||
|
||||
@@ -84,8 +84,6 @@ DECLARE_bool(storage_parallel_schema_recovery);
|
||||
DECLARE_uint64(storage_recovery_thread_count);
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_bool(storage_enable_schema_metadata);
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_bool(storage_delta_on_identical_property_update);
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_bool(telemetry_enabled);
|
||||
|
||||
@@ -13,11 +13,11 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint32(management_port, 0, "Port on which coordinator servers will be started.");
|
||||
DEFINE_uint32(coordinator_server_port, 0, "Port on which coordinator servers will be started.");
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint32(coordinator_port, 0, "Port on which raft servers will be started.");
|
||||
DEFINE_uint32(raft_server_port, 0, "Port on which raft servers will be started.");
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint32(coordinator_id, 0, "Unique ID of the raft server.");
|
||||
DEFINE_uint32(raft_server_id, 0, "Unique ID of the raft server.");
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint32(instance_down_timeout_sec, 5, "Time duration after which an instance is considered down.");
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
|
||||
@@ -15,11 +15,11 @@
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_uint32(management_port);
|
||||
DECLARE_uint32(coordinator_server_port);
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_uint32(coordinator_port);
|
||||
DECLARE_uint32(raft_server_port);
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_uint32(coordinator_id);
|
||||
DECLARE_uint32(raft_server_id);
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_uint32(instance_down_timeout_sec);
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -73,11 +73,11 @@ constexpr auto kLogToStderrGFlagsKey = "also_log_to_stderr";
|
||||
constexpr auto kCartesianProductEnabledSettingKey = "cartesian-product-enabled";
|
||||
constexpr auto kCartesianProductEnabledGFlagsKey = "cartesian-product-enabled";
|
||||
|
||||
// NOLINTBEGIN(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
// Local cache-like thing
|
||||
std::atomic<double> execution_timeout_sec_;
|
||||
std::atomic<bool> cartesian_product_enabled_{true};
|
||||
// NOLINTEND(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
std::atomic<double> execution_timeout_sec_; // Local cache-like thing
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
std::atomic<bool> cartesian_product_enabled_{true}; // Local cache-like thing
|
||||
|
||||
auto ToLLEnum(std::string_view val) {
|
||||
const auto ll_enum = memgraph::flags::LogLevelToEnum(val);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2024 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user