Compare commits

...

77 Commits

Author SHA1 Message Date
jbajic
3bc1839df9 Fix centos 7 2022-03-18 15:52:49 +01:00
jbajic
e07342e3ff Update change date 2022-03-17 13:49:45 +01:00
jbajic
659c0386e1 Add override version 2022-03-17 09:10:42 +01:00
jbajic
60ad05acff Address review comments 2022-03-14 15:47:41 +01:00
jbajic
4f593c7fca Adapt precommit to run through all files 2022-03-14 15:47:41 +01:00
jbajic
770ea1189a Fix SSL errors 2022-03-14 15:47:41 +01:00
jbajic
695bb343f1 Resolve clang-tidy error 2022-03-14 15:47:41 +01:00
jbajic
12b4ec1589 Add memgraph namespace 2022-03-14 15:47:41 +01:00
Antonio Andelic
b33d2c3940 use boost ssl 2022-03-14 15:47:41 +01:00
Marko Budiselić
477acad1f6 Update CODEOWNERS file (#362) 2022-02-25 13:42:27 +01:00
János Benjamin Antal
ddca2b40f5 Remove unimplemented function declaration (#357) 2022-02-24 11:49:22 +01:00
Antonio Andelic
4817be0add Polishing for release 2022-02-17 14:47:11 +01:00
Jure Bajic
3fb7e5378d Add websocket port argument (#355) 2022-02-17 10:36:15 +01:00
Antonio Andelic
1d88893715 Add Websocket e2e tests (#353) 2022-02-17 10:36:10 +01:00
Jure Bajic
bd2c30fddc Websocket unit tests (#334) 2022-02-17 10:35:56 +01:00
Antonio Andelic
06e6ead4d2 WSS support (#327) 2022-02-17 10:35:56 +01:00
Jure Bajic
728b37080d Fix authorization check (#332) 2022-02-17 10:35:56 +01:00
Antonio Andelic
48a531aac1 Add websocket authentication (#322) 2022-02-17 10:35:48 +01:00
Antonio Andelic
914fc1a656 Send logs over websocket (#316) 2022-02-17 10:31:26 +01:00
Antonio Andelic
1d1c182c2d Add websocket server and log stream (#314) 2022-02-17 10:31:26 +01:00
János Benjamin Antal
c6e19ec09f Fix folly in toolchain-v4 (#346) 2022-02-14 14:31:27 +01:00
János Benjamin Antal
693dab78d2 Remove unused URLs from libs/setup.sh (#352) 2022-02-11 13:56:43 +01:00
Antonio Andelic
69eca9b043 Procedures for handling modules (#330) 2022-02-11 11:29:41 +01:00
Antonio Andelic
5aeaad198b Define SHOW VERSION query (#265) 2022-02-10 10:30:14 +01:00
Antonio Andelic
b23f88c607 Use gzip for packaging docker image (#348) 2022-02-07 15:50:43 +01:00
Kostas Kyrimis
4fd8bdce4c Make temporal types now functions use the query context timestamp (#333) 2022-02-04 12:57:38 +01:00
Antonio Andelic
265b203b00 Build for ARM64 (#340) 2022-02-03 13:03:35 +01:00
János Benjamin Antal
661e5185d8 Fix nested transaction error (#339) 2022-02-03 09:41:23 +01:00
Marko Budiselić
7348ad6800 Add init quicklisp proxy flag (#342) 2022-02-02 09:07:57 +01:00
János Benjamin Antal
6c00d146f2 Add configs for kafka streams (#328) 2022-01-31 17:26:53 +01:00
Antonio Andelic
ced84e17b6 Update nlohmann and remove guards (#336) 2022-01-26 08:12:09 +01:00
Jure Bajic
834ce5f51a Remove support for deb9 (#329) 2022-01-25 12:23:50 +01:00
Antonio Andelic
bb1308acc7 Use libs from toolchain (#326) 2022-01-21 10:22:36 +01:00
János Benjamin Antal
e1f31d3d02 Toolchain v4 with fbthrift (#318) 2022-01-18 12:51:27 +01:00
Josip Matak
339cd9b84e Change Python root directory (#289) 2022-01-18 08:23:17 +01:00
Antonio Andelic
7deac4ac8b Extra procedures transformation info (#310) 2022-01-13 13:46:32 +01:00
Jure Bajic
fec887a67c Adapt header-checker (#323) 2022-01-13 09:18:27 +01:00
Antonio Andelic
f3a3a4ed87 Unsafe git protocol fix (#324) 2022-01-12 11:36:06 +01:00
János Benjamin Antal
ec65187442 Use correct error in log message (#321) 2022-01-10 13:50:46 +01:00
János Benjamin Antal
079c0495c3 Increase reference count when returning Py_None (#320) 2022-01-07 12:56:52 +01:00
Kostas Kyrimis
90cfed2ada Remove unnecessary use of string in server start method (#317) 2021-12-14 16:13:24 +01:00
Antonio Andelic
8716b8e992 Update to newest spdlog and fmt (#309) 2021-12-07 11:05:33 +01:00
Antonio Andelic
a277541354 Update debian base image (#311) 2021-12-07 10:10:04 +01:00
Antonio Andelic
b01564f179 Update license date for 2.1.1 (#312) 2021-12-06 12:35:30 +01:00
Antonio Andelic
d9bb4e2e46 Use default values instantly for batch size/interval (#306) 2021-11-30 16:29:51 +01:00
János Benjamin Antal
05d0aee494 Avoid crash in case of loading old stream (#302) 2021-11-30 15:27:11 +01:00
Antonio Andelic
aabec99a8e Procedure for stream configs (#301) 2021-11-29 08:56:10 +01:00
Antonio Andelic
d277dd49a3 Update stats with query modules (#304) 2021-11-26 15:31:25 +01:00
Antonio Andelic
18f9d19b18 Update pyyaml version (#305) 2021-11-25 10:12:33 +01:00
Antonio Andelic
530eed5c6d Update change date (#303) 2021-11-24 16:26:35 +01:00
Antonio Andelic
12f4e0068a Small polishing and fixes 2021-11-16 17:23:20 +01:00
Antonio Andelic
e8976e0f1c Update mgp API for Pulsar (#298) 2021-11-16 16:00:25 +01:00
Kostas Kyrimis
6eb52581eb Retry transaction on consumer (#294) 2021-11-16 15:58:36 +01:00
Antonio Andelic
8606e69fd6 Add support for type in show streams (#300) 2021-11-16 15:58:36 +01:00
Antonio Andelic
c7b045bffc Pulsar e2e tests (#296) 2021-11-16 15:58:35 +01:00
Antonio Andelic
b66cc66503 Add support for Pulsar queries (#292) 2021-11-16 15:52:58 +01:00
Antonio Andelic
0e4719018a Introduce pulsar dependency (#286) 2021-11-16 15:47:40 +01:00
Antonio Andelic
0ebd52aac3 Support for different consumers (#280) 2021-11-16 15:40:27 +01:00
Antonio Andelic
6c971b856e Fix ORDER BY and temporal types (#299) 2021-11-16 10:59:25 +01:00
Kostas Kyrimis
6cb293688d Fix check stream not reseting back to the correct offset (#295) 2021-11-15 14:44:30 +01:00
Jure Bajic
16709dff6c Add stats and notifications in results metadata (#285) 2021-11-15 13:51:13 +01:00
Kostas Kyrimis
95dd3481c0 Add more information to the MG_ASSERT in release mode (#291) 2021-11-12 09:09:15 +01:00
Kostas Kyrimis
47c0c629c7 Add procedure for setting a kafka stream offset (#282) 2021-11-11 12:07:58 +01:00
Kostas Kyrimis
636c551047 Remove linter concurrency-mt (#287) 2021-11-02 09:19:41 +01:00
Marko Budiselić
72384b2b71 Add internals pages link (#288) 2021-11-01 09:49:22 +01:00
Jure Bajic
a9b1ff9bea Support Bolt v4.3 protocol (#226) 2021-10-28 13:05:09 +02:00
Jure Bajic
e40bc32624 Fix daily benchmark workflow (#284) 2021-10-27 09:05:46 +02:00
Antonio Andelic
bd21bc82b7 Add license to cpp/hpp/py test files (#283) 2021-10-26 08:53:56 +02:00
Jure Bajic
9c1680e82c Change how to get branch name (#281) 2021-10-25 09:01:26 +02:00
Antonio Andelic
1a78c3695d Support multiple types for weighted shortest path (#278) 2021-10-19 14:39:23 +02:00
Kostas Kyrimis
10196f3d7d Add bootstrap servers to create stream (#274) 2021-10-18 10:49:00 +02:00
Jure Bajic
2c86fefbb5 Run diff on master (#275) 2021-10-18 08:40:07 +02:00
Antonio Andelic
df689dccd5 Make Memgraph work with fetch content (#256) 2021-10-14 18:58:42 +02:00
Jure Bajic
6033fbe9bd Clean up topics before starting test (#267) 2021-10-14 16:57:46 +02:00
Jure Bajic
519a204424 Add retry to get_version script (#268) 2021-10-14 16:07:40 +02:00
Antonio Andelic
3fc5e57fef Update dependency source when package all is called (#272) 2021-10-14 15:16:32 +02:00
János Benjamin Antal
906933d2b3 Fix typo in procedure C-API (#273) 2021-10-14 08:50:16 +02:00
686 changed files with 26520 additions and 12921 deletions

View File

@@ -1,7 +1,9 @@
---
Checks: '*,
-abseil-string-find-str-contains,
-altera-id-dependent-backward-branch,
-altera-struct-pack-align,
-altera-unroll-loops,
-android-*,
-cert-err58-cpp,
-cppcoreguidelines-avoid-c-arrays,
@@ -58,7 +60,8 @@ Checks: '*,
-readability-implicit-bool-conversion,
-readability-magic-numbers,
-readability-named-parameter,
-misc-no-recursion'
-misc-no-recursion,
-concurrency-mt-unsafe'
WarningsAsErrors: ''
HeaderFilterRegex: 'src/.*'
AnalyzeTemporaryDtors: false

View File

@@ -16,6 +16,7 @@ tmpdir=$(mktemp -d repo-XXXXXXXX)
trap "rm -rf $tmpdir" EXIT INT
modified_files=$(git diff --cached --name-only --diff-filter=AM $against | sed -nE "/.*\.(cpp|cc|cxx|c|h|hpp)$/p")
FAIL=0
for file in $modified_files; do
echo "Checking $file..."
@@ -25,19 +26,20 @@ for file in $modified_files; do
echo "Running clang-format..."
$project_folder/tools/git-clang-format $tmpdir/$file
code=$?
CODE=$?
if [ $code -ne 0 ]; then
if [ $CODE -ne 0 ]; then
break
fi
# Do not break header checker
echo "Running header checker..."
$project_folder/tools/header-checker.py $tmpdir/$file
code=$?
if [ $code -ne 0 ]; then
break
$project_folder/tools/header-checker.py $tmpdir/$file $file --amend-year
CODE=$?
if [ $CODE -ne 0 ]; then
FAIL=1
fi
done;
return $code
return ${FAIL}

View File

@@ -25,7 +25,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -42,6 +42,16 @@ jobs:
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Get branch name (merge)
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
- name: Get branch name (pull request)
if: github.event_name == 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
@@ -51,7 +61,8 @@ jobs:
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"
- name: Run mgbench
run: |
@@ -67,4 +78,5 @@ jobs:
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -1,13 +1,16 @@
name: Diff
on:
push:
branches:
- master
workflow_dispatch:
pull_request:
paths-ignore:
- 'docs/**'
- '**/*.md'
- '.clang-format'
- 'CODEOWNERS'
- "docs/**"
- "**/*.md"
- ".clang-format"
- "CODEOWNERS"
jobs:
community_build:
@@ -29,7 +32,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -42,7 +45,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -67,7 +70,7 @@ jobs:
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -79,7 +82,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
cd build
@@ -88,7 +91,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Compute code coverage.
cd tools/github
@@ -106,7 +109,7 @@ jobs:
- name: Run clang-tidy
run: |
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
@@ -133,7 +136,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -146,7 +149,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -178,7 +181,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -209,7 +212,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -235,7 +238,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -269,7 +272,7 @@ jobs:
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
cd build
@@ -316,7 +319,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -357,7 +360,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -374,6 +377,16 @@ jobs:
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Get branch name (merge)
if: github.event_name != 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
- name: Get branch name (pull request)
if: github.event_name == 'pull_request'
shell: bash
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
@@ -383,7 +396,8 @@ jobs:
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"
- name: Run mgbench
run: |
@@ -399,4 +413,5 @@ jobs:
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -23,7 +23,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -36,10 +36,10 @@ jobs:
- name: Run clang-tidy
run: |
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v3/bin/clang-tidy "$PWD/src/*" |
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results

View File

@@ -39,23 +39,6 @@ jobs:
name: centos-8
path: build/output/centos-8/memgraph*.rpm
debian-9:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9
path: build/output/debian-9/memgraph*.deb
debian-10:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
@@ -101,7 +84,7 @@ jobs:
- name: "Build package"
run: |
cd release/package
./run.sh package debian-10 --for-docker
./run.sh package debian-11 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
@@ -143,7 +126,7 @@ jobs:
name: ubuntu-2004
path: build/output/ubuntu-20.04/memgraph*.deb
debian-10-platform:
debian-11-platform:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
@@ -153,9 +136,9 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-10 --for-platform
./release/package/run.sh package debian-11 --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10-platform
path: build/output/debian-10/memgraph*.deb
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb

View File

@@ -26,7 +26,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -39,7 +39,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -64,7 +64,7 @@ jobs:
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -77,7 +77,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -86,7 +86,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Compute code coverage.
cd tools/github
@@ -121,7 +121,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -134,7 +134,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -166,7 +166,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -198,7 +198,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -211,7 +211,7 @@ jobs:
- name: Create enterprise RPM package
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
cd build
@@ -233,7 +233,7 @@ jobs:
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run micro benchmark tests.
cd build
@@ -272,7 +272,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build

View File

@@ -26,7 +26,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -39,7 +39,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -64,7 +64,7 @@ jobs:
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -77,7 +77,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -86,7 +86,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Compute code coverage.
cd tools/github
@@ -121,7 +121,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -134,7 +134,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -166,7 +166,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -198,7 +198,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -211,7 +211,7 @@ jobs:
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
cd build
@@ -232,7 +232,7 @@ jobs:
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run micro benchmark tests.
cd build
@@ -271,7 +271,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -333,7 +333,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init

View File

@@ -26,7 +26,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -39,7 +39,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -64,7 +64,7 @@ jobs:
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -77,7 +77,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
@@ -86,7 +86,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Compute code coverage.
cd tools/github
@@ -121,7 +121,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -134,7 +134,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -166,7 +166,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -198,7 +198,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
@@ -211,7 +211,7 @@ jobs:
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
cd build
@@ -232,7 +232,7 @@ jobs:
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run micro benchmark tests.
cd build
@@ -271,7 +271,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v3/activate
source /opt/toolchain-v4/activate
# Run unit tests.
cd build

View File

@@ -54,7 +54,7 @@ option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON)
# Set the current version here to override the automatic version detection. The
# version must be specified as `X.Y.Z`. Primarily used when building new patch
# versions.
set(MEMGRAPH_OVERRIDE_VERSION "")
set(MEMGRAPH_OVERRIDE_VERSION "2.2.1")
# Custom suffix that this version should have. The suffix can be any arbitrary
# string. Primarily used when building a version for a specific customer.
@@ -66,7 +66,7 @@ if (MG_ENTERPRISE)
else()
set(get_version_offering "--open-source")
endif()
set(get_version_script "${CMAKE_SOURCE_DIR}/release/get_version.py")
set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
# Get version that should be used in the binary.
execute_process(
@@ -76,7 +76,7 @@ execute_process(
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph version.")
@@ -93,7 +93,7 @@ execute_process(
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
@@ -110,7 +110,7 @@ execute_process(
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
@@ -204,6 +204,8 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
# release flags
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
#debug flags
set(PREFERRED_DEBUGGER "gdb" CACHE STRING
"Tunes the debug output for your preferred debugger (gdb or lldb).")
@@ -228,6 +230,8 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
# setup external dependencies -------------------------------------------------
# threading
@@ -322,7 +326,8 @@ if (UBSAN)
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
endif()
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact Python version used by the query modules")
set(MG_PYTHON_PATH "" CACHE STRING "Specify the exact Python path used by the query modules")
# Add subprojects
include_directories(src)
@@ -331,8 +336,13 @@ add_subdirectory(src)
# Release configuration
add_subdirectory(release)
enable_testing()
add_subdirectory(tests)
option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
if (MG_ENABLE_TESTING)
enable_testing()
add_subdirectory(tests)
endif()
if(TOOLS)
add_subdirectory(tools)

View File

@@ -1 +1 @@
* @gitbuda @antonio2368 @antaljanosbenjamin @kostasrim @jbajic
* @antaljanosbenjamin @kostasrim

View File

@@ -115,6 +115,14 @@ understand what actions will and will not be tolerated.
Read our [contributing guide](CONTRIBUTING.md) to learn about our development
process and how to propose bug fixes and improvements.
### Internals
Read our
[internal](https://memgraph.notion.site/Memgraph-Internals-12b69132d67a417898972927d6870bd2)
docs to learn more about Memgraph's architecture, how to build the project from
source and how to start contributing. All information related to the database,
can be found in the aforementioned docs.
### :scroll: License
Memgraph Community is available under the [BSL

55
cmake/FindJemalloc.cmake Normal file
View File

@@ -0,0 +1,55 @@
# Try to find jemalloc library
#
# Use this module as:
# find_package(Jemalloc)
#
# or:
# find_package(Jemalloc REQUIRED)
#
# This will define the following variables:
#
# Jemalloc_FOUND True if the system has the jemalloc library.
# Jemalloc_INCLUDE_DIRS Include directories needed to use jemalloc.
# Jemalloc_LIBRARIES Libraries needed to link to jemalloc.
#
# The following cache variables may also be set:
#
# Jemalloc_INCLUDE_DIR The directory containing jemalloc/jemalloc.h.
# Jemalloc_LIBRARY The path to the jemalloc static library.
find_path(Jemalloc_INCLUDE_DIR NAMES jemalloc/jemalloc.h PATH_SUFFIXES include)
find_library(Jemalloc_LIBRARY NAMES libjemalloc.a PATH_SUFFIXES lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Jemalloc
FOUND_VAR Jemalloc_FOUND
REQUIRED_VARS
Jemalloc_LIBRARY
Jemalloc_INCLUDE_DIR
)
if(Jemalloc_FOUND)
set(Jemalloc_LIBRARIES ${Jemalloc_LIBRARY})
set(Jemalloc_INCLUDE_DIRS ${Jemalloc_INCLUDE_DIR})
else()
if(Jemalloc_FIND_REQUIRED)
message(FATAL_ERROR "Cannot find jemalloc!")
else()
message(WARNING "jemalloc is not found!")
endif()
endif()
if(Jemalloc_FOUND AND NOT TARGET Jemalloc::Jemalloc)
add_library(Jemalloc::Jemalloc UNKNOWN IMPORTED)
set_target_properties(Jemalloc::Jemalloc
PROPERTIES
IMPORTED_LOCATION "${Jemalloc_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${Jemalloc_INCLUDE_DIR}"
)
endif()
mark_as_advanced(
Jemalloc_INCLUDE_DIR
Jemalloc_LIBRARY
)

View File

@@ -1,4 +0,0 @@
# Memgraph Build and Run Environments
Please continue in
[Notion](https://www.notion.so/memgraph/Tools-05e0baafb78a49b386e0063b4833d23d).

View File

@@ -11,12 +11,18 @@ TOOLCHAIN_BUILD_DEPS=(
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel libipt-devel libbabeltrace-devel xz-devel python3-devel # gdb
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python3-devel # gdb
texinfo # gdb
libcurl-devel # cmake
curl # snappy
readline-devel # cmake and llvm
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel automake bison # swig
file
openssl-devel
gmp-devel
gperf
patch
)
TOOLCHAIN_RUN_DEPS=(
@@ -26,10 +32,10 @@ TOOLCHAIN_RUN_DEPS=(
expat libipt libbabeltrace xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconfig # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
@@ -49,6 +55,7 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
@@ -94,8 +101,12 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
yum install -y epel-release
yum remove -y ius-release
yum install -y \
https://repo.ius.io/ius-release-el7.rpm
yum update -y
yum install -y wget git python3 python3-pip
yum install -y wget python3 python3-pip
yum install -y git
for pkg in $1; do
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then

View File

@@ -6,16 +6,22 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
patch
)
TOOLCHAIN_RUN_DEPS=(
@@ -25,6 +31,7 @@ TOOLCHAIN_RUN_DEPS=(
expat libipt libbabeltrace xz-libs python36 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
@@ -48,6 +55,7 @@ MEMGRAPH_BUILD_DEPS=(
which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
@@ -109,9 +117,10 @@ install() {
# https://centos.pkgs.org
# Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
# within GDB yet (an assumption).
# http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working
if [ "$pkg" == libbabeltrace-devel ]; then
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
dnf install -y http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
fi
continue
fi

139
environment/os/centos-9.sh Executable file
View File

@@ -0,0 +1,139 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it manually using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "python3-virtualenv" ]; then
continue
fi
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
pushd /tmp/sbcl-1.4.2-arm64-linux
INSTALL_ROOT=/usr/local sh install.sh
popd
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
if [ "$pkg" == python3-virtualenv ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user virtualenv
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
fi
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -15,6 +15,12 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file # for libunwind
libssl-dev # for libevent
libgmp-dev # for gdb
gperf # for proxygen
git # for fbthrift
libedit-dev libpcre3-dev automake bison # for swig
)
@@ -26,6 +32,7 @@ TOOLCHAIN_RUN_DEPS=(
libcurl4 # for cmake
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
@@ -46,6 +53,7 @@ MEMGRAPH_BUILD_DEPS=(
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {

View File

@@ -16,6 +16,12 @@ TOOLCHAIN_BUILD_DEPS=(
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
curl # snappy
file # for libunwind
libssl-dev # for libevent
libgmp-dev
gperf # for proxygen
git # for fbthrift
)
TOOLCHAIN_RUN_DEPS=(
@@ -24,8 +30,10 @@ TOOLCHAIN_RUN_DEPS=(
zlib1g # zlib library used for all builds
libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
libcurl4 # for cmake
file # for CPack
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
@@ -46,6 +54,7 @@ MEMGRAPH_BUILD_DEPS=(
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {

View File

@@ -1,62 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev libipt-dev libbabeltrace-dev libbabeltrace-ctf-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 libipt1 libbabeltrace1 libbabeltrace-ctf1 liblzma5 python3 # for gdb
libcurl3 # for cmake
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
install_all_apt "$1"
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -16,6 +16,12 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # cmake
libreadline-dev # cmake and llvm
libffi-dev libxml2-dev # llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # swig
)
@@ -27,6 +33,7 @@ TOOLCHAIN_RUN_DEPS=(
libcurl4 # for cmake
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
@@ -45,6 +52,7 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {

View File

@@ -15,6 +15,12 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # for swig
)
@@ -26,6 +32,7 @@ TOOLCHAIN_RUN_DEPS=(
libcurl4 # for cmake
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
@@ -46,6 +53,7 @@ MEMGRAPH_BUILD_DEPS=(
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {

View File

@@ -0,0 +1,41 @@
diff -ur a/folly/CMakeLists.txt b/folly/CMakeLists.txt
--- a/folly/CMakeLists.txt 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/CMakeLists.txt 2022-02-03 15:19:41.349693134 +0100
@@ -28,7 +28,6 @@
)
add_subdirectory(experimental/exception_tracer)
-add_subdirectory(logging/example)
if (PYTHON_EXTENSIONS)
# Create tree of symbolic links in structure required for successful
diff -ur a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp
--- a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2022-02-03 15:19:11.003368891 +0100
@@ -96,6 +96,7 @@
#define __builtin_unreachable()
#endif
+#if 0
namespace __cxxabiv1 {
void __cxa_throw(
@@ -154,5 +155,5 @@
}
} // namespace std
-
+#endif
#endif // defined(__GLIBCXX__)
diff -ur a/folly/Portability.h b/folly/Portability.h
--- a/folly/Portability.h 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/Portability.h 2022-02-03 15:19:11.003368891 +0100
@@ -566,7 +566,7 @@
#define FOLLY_HAS_COROUTINES 0
#elif (__cpp_coroutines >= 201703L || __cpp_impl_coroutine >= 201902L) && \
(__has_include(<coroutine>) || __has_include(<experimental/coroutine>))
-#define FOLLY_HAS_COROUTINES 1
+#define FOLLY_HAS_COROUTINES 0
// This is mainly to workaround bugs triggered by LTO, when stack allocated
// variables in await_suspend end up on a coroutine frame.
#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE

View File

@@ -0,0 +1,11 @@
diff -ur a/cmake/proxygen-config.cmake.in b/cmake/proxygen-config.cmake.in
--- a/cmake/proxygen-config.cmake.in 2021-12-13 02:37:05.000000000 +0100
+++ b/cmake/proxygen-config.cmake.in 2022-01-27 17:14:28.284810621 +0100
@@ -21,7 +21,6 @@
find_dependency(folly)
find_dependency(wangle)
find_dependency(Fizz)
-find_dependency(mvfst)
# For now, anything that depends on Proxygen has to copy its FindZstd.cmake
# and issue a `find_package(Zstd)`. Uncommenting this won't work because
# this Zstd module exposes a library called `zstd`. The right fix is

View File

@@ -0,0 +1,29 @@
diff -ur a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2021-05-05 00:53:34.000000000 +0200
+++ b/CMakeLists.txt 2022-01-27 17:18:34.758302398 +0100
@@ -52,9 +52,9 @@
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0)
- # Disable RTTI.
- string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Use -Wall for clang and gcc.
if(NOT CMAKE_CXX_FLAGS MATCHES "-Wall")
@@ -77,9 +77,9 @@
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
- # Disable RTTI.
- string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make

1211
environment/toolchain/v4.sh Executable file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,75 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBEzEOZIBEACxg/IuXERlDB48JBWmF4NxNUuuup1IhJAJyFGFSKh3OGAO2Ard
sNuRLjANsFXA7m7P5eTFcG+BoHHuAVYmKnI3PPZtHVLnUt4pGItPczQZ2BE1WpcI
ayjGTBJeKItX3Npqg9D/odO9WWS1i3FQPVdrLn0YH37/BA66jeMQCRo7g7GLpaNf
IrvYGsqTbxCwsmA37rpE7oyU4Yrf74HT091WBsRIoq/MelhbxTDMR8eu/dUGZQVc
Kj3lN55RepwWwUUKyqarY0zMt4HkFJ7v7yRL+Cvzy92Ouv4Wf2FlhNtEs5LE4Tax
W0PO5AEmUoKjX87SezQK0f652018b4u6Ex52cY7p+n5TII/UyoowH6+tY8UHo9yb
fStrqgNE/mY2bhA6+AwCaOUGsFzVVPTbjtxL3HacUP/jlA1h78V8VTvTs5d55iG7
jSqR9o05wje8rwNiXXK0xtiJahyNzL97Kn/DgPSqPIi45G+8nxWSPFM5eunBKRl9
vAnsvwrdPRsR6YR3uMHTuVhQX9/CY891MHkaZJ6wydWtKt3yQwJLYqwo5d4DwnUX
CduUwSKv+6RmtWI5ZmTQYOcBRcZyGKml9X9Q8iSbm6cnpFXmLrNQwCJN+D3SiYGc
MtbltZo0ysPMa6Xj5xFaYqWk/BI4iLb2Gs+ByGo/+a0Eq4XYBMOpitNniQARAQAB
tCdMYXNzZSBDb2xsaW4gPGxhc3NlLmNvbGxpbkB0dWthYW5pLm9yZz6JAlEEEwEK
ADsCGwMCHgECF4AECwkIBwMVCggFFgIDAQAWIQQ2kMJAzlG0Zw0wrRw47nV9aRhG
IAUCYEt9dQUJFxeR4wAKCRA47nV9aRhGIBNDEACxD6vJ+enZwe3IgkJh5JtLsC9b
MWCQRlPW1EVMsg96Cb5Rtron1eN1pp1TlzENJu1/C7C/VEsr9WwOPg26Men7fNf/
O21QM9IBWd/uB0Pu333WqKh92ESS5x9ST9DrG39nVGSPkQQBMuia72VrA+crPnwT
/h/u1IN6/sff5VDIU24rUiqW2Npy733dANruj7Ny0scRXVPltnVdhqwPHt6qNjC1
t+/cCnwHgW1BR1RYXBPpB42z/m29dL9rPrG0YPGWs2Bc+EATUICfEE6eIvwfciue
IJTjKT9Y9DrogJC2AYFhjC7N04OKdCB2hFs4BjexJwr4X0GJO7LhFl03c951AsIE
GHwrucRPB5bo2vmvQ8IvZn7CmtdUJzXv9JlyU6p+MIK1pz7TK6GgSOSffQIXZn6e
nUPtm9mEwuncOfmW8/ODYPs1gCWYgyiFJx8h7eEu+M4MxHSFBs7MwXf/Ae2fSp+M
P/p198qB8fC5oVBnF95qb0Qi0uc1D+Gb+gpBF+ymMb+s/VBOR3QWiym7AzBrJ62g
UnbC9jMLGnSRI+7p7raUfMTgXr5/oQoBw7ExJVltSSRrim2YH/t4CV47mO6dR9J3
1RtsTFIRNhz+07XPsETcuCV/dgqeC8fOFLt9MY17Sufhb1DcGy4urZBOIhXcpTV7
vHVj5IYH5nYOT49NRYkCOAQTAQIAIgUCTMQ5kgIbAwYLCQgHAwIGFQgCCQoLBBYC
AwECHgECF4AACgkQOO51fWkYRiAg4A/7BXKwoRaXrMbMPOW7vuVF7c2IKB2Yqzn1
vLBCwuEHkqY237lDcXY4/5LR+1gcZ3Duw1n/BRSm0FBdvyX/JTWiWNSDUkKAO/0l
T2Tg44YLrDT3bzwu8dbU9xQt6kH+SCOHvv5Oe4k79l5mro6fF3H1M0bN63x/YoFY
ojy09D7/JptY82oR4f/VdKnfZLJcCViCb0wp8SD2NkDAudKg+K+7PD8HlTWklQQg
TZdRXxVZKIJeU42aJDqnRbAhJd64YHyClhqut9F5LUmiP5qfLfNhkKDhNOwk2Blr
BGBJkSd7wPyzcX4Mun/L6YspHjbeVMt9TD7HQlo+OOd2OjAHCx6pqwkXnzeLPEaE
cPdQ1SHgrBViAxX3DNPubLP0Knw8XwFu96EuhHZgexE1W7bB4LFsJyXAc5k1PqPD
CLsAauxmvI2OfI7opG/8wyxDvNgoPjG8fZNAgY0REqPC0JnTXChH31IxUmhNotH8
tD3DDTZOHw05n5MwwUrEE9xiETVDfFQcMLfxZ9KLz+BC2g1t5LYublRgnCMNJzFg
sNUMM02CphABzl/LCLnumr0eyQQ/weV4twEhLwSDmqLYHL0EdYW0Y3CnnU9vmYxQ
cXKbstS71sEJJYBBmSBbf9GxkOY8BRNtwVwY0kPgxv1WqdVBiAFvfB+pyAsrax9B
3UeB7ZSwRD6JAhwEEAEKAAYFAlS25GwACgkQlbYYGy0z6ew92Q//ZA9/6piQtoW4
PwP/1DtWGyKU8hwR+9FG669iPk/dAG+yoEJtFMOUpg/FUFmCX8Bc4oEHsCVyLxKt
DcCVUIRcYNSFi5hTZaBEbwsOlDT37gtlfIIu34hhHRccKaLnN/N9gNMNw8wGh9xg
Q/KtxZwcbk/bZIlDkKTJkFBRAekdEGAFDWb/AZOy+LQxS8ZAh1eWkfV0i8opmK9k
gPXtLE0WSsqtYyGs58z+BFE9NH3tEUwK6jSvtuLwQl4UrICNbKthcpb8WwH6UXzb
q3QNSYVOpf/cqRdBJA6bvb/ku/xyKVL08lGmxD9v1b137R7mafDAFPTsvH2Mt/0V
YuhtWav3r1Bl9QksDxt2DTS8wiWDUBetGqOVdcw7vBrXPEWDNBmxeJXsiJ7zJlR+
9wrJOm6RV2+l1IPxu96EaPS+kTNBijKrhxb67bww8BTEWTd0wcdJmgWRkM8SIstp
IKqd0L2TFYph2/NtrBhRg+DIEPJPpSTGsUMcCEXCZPQ+cIdlQKsWpk0tZ62DlvEl
r7E+wgUSQolRfx5KrpZifiS2zQlhzdXv28CJhsVbLyw5fUAWUKIH/dCo5NKsNLk2
Lc5DH9VWnFgxAAtW290FqeK/4ulMq7Vs1dQSwyHM2Ni3QqqeaiOrh8gbSY5CMLFN
Y3HYRwuTYPa3AobsozCzBj0Zdf/6AFe5Ag0ETMQ5kgEQAL/FwKdjxgPxtSpgq1SM
zgZtTTyLqhgGD3NZfadHWHYRIL38NDV3JeTA79Y2zj2dj7KQPDT+0aqeizTV2E3j
P3iCQ53VOT4consBaQAgKexpptnS+T1DobtICFJ0GGzf0HRj6KO2zSOuOitWPWlU
wbvX7M0LLI2+hqlx0jTPqbJFZ/Za6KTtbS6xdCPVUpUqYZQpokEZcwQmUp8Q+lGo
JD2sNYCZyap63X/aAOgCGr2RXYddOH5e8vGzGW+mwtCv+WQ9Ay35mGqI5MqkbZd1
Qbuv2b1647E/QEEucfRHVbJVKGGPpFMUJtcItyyIt5jo+r9CCL4Cs47dF/9/RNwu
NvpvHXUyqMBQdWNZRMx4k/NGD/WviPi9m6mIMui6rOQsSOaqYdcUX4Nq2Orr3Oaz
2JPQdUfeI23iot1vK8hxvUCQTV3HfJghizN6spVl0yQOKBiE8miJRgrjHilH3hTb
xoo42xDkNAq+CQo3QAm1ibDxKCDq0RcWPjcCRAN/Q5MmpcodpdKkzV0yGIS4g7s5
frVrgV/kox2r4/Yxsr8K909+4H82AjTKGX/BmsQFCTAqBk6p7I0zxjIqJ/w33TZB
Q0Pn4r3WIlUPafzY6a9/LAvN1fHRxf9SpCByJsszD03Qu5f5TB8gthsdnVmTo7jj
iordEKMtw2aEMLzdWWTQ/TNVABEBAAGJAjwEGAEKACYCGwwWIQQ2kMJAzlG0Zw0w
rRw47nV9aRhGIAUCYEt9YAUJFxeRzgAKCRA47nV9aRhGIMLtD/9HuKM4pngImcuz
YwzQmdv4j26YYyh4jVsKEmVWTiRcehEgUIlrWkCu3qzd5NK+RetS7kJ8MPnzEUfj
YbpdC6yrF6n1mSrZZ4VJMkV2ev37bIgXM+Wp1mCAGbjNxQnjn9RabT/gjIqmGuRn
AP7RsSeOSuO/gO9h2Pteciz23ussTilB+8cTooQEQQZe6Kv/zukvL+ccSehLHsZ7
qVfRUAmtt8nFkXXE+s8jfLfhqstaI2/RJu5witaPcXM8Mnz2E95aASAbZy0eQot9
0Pvf07n9yuC3tueTvzvlXx3h5U3yT44tIOmzANIQjay1TGdm+RBJ2ZYyhyLawlZ2
NVUXXSp4QZZXPA0UWbF+pb7Q9cdKDNFVuvGBljuea0Yd0T2o+ibDq43HziX9ll+l
SXk9mqvW1UcDOaxWrSsm1Gc1O9g3wqH5xHAhtY8GPh/7VgAawskPkmnlkMW6pYPy
zibbeISJL1gd1jIT63y6aoVrtNoo+wYJm280ROflh4+5QOo6QJ+jm70fkXSG/qJ5
a8/qCPTHkJc/rpkL6/TDQAJURi9RhDAC0gb40HtusbN1LZEA+i0cWTmYXap+DB4Y
R4pApilpaG87M+VUokR4xpnx7vTb2MPa7Mdenvi9FEGnKXadmT8038vlfzz5GGUT
MlVin9BQPTpdA+PpRiJvKJgVDeAFOg==
=asTC
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -1405,38 +1405,76 @@ int mgp_must_abort(struct mgp_graph *graph);
/// @}
/// @name Kafka message API
/// Currently the API below is for kafka only but in the future
/// mgp_message and mgp_messages might be generic to support
/// other streaming systems.
/// @name Stream Source message API
/// API for accessing specific data contained in a mgp_message
/// used for defining transformation procedures.
/// Not all methods are available for all stream sources
/// so make sure that your transformation procedure can be used
/// for a specific source, i.e. only valid methods are used.
///@{
/// A single Kafka message
/// A single Stream source message
struct mgp_message;
/// A list of Kafka messages
/// A list of Stream source messages
struct mgp_messages;
/// Stream source type.
enum mgp_source_type {
KAFKA,
PULSAR,
};
/// Get the type of the stream source that produced the message.
enum mgp_error mgp_message_source_type(struct mgp_message *message, enum mgp_source_type *result);
/// Payload is not null terminated and not a string but rather a byte array.
/// You need to call mgp_message_payload_size() first, to read the size of
/// the payload.
/// Supported stream sources:
/// - Kafka
/// - Pulsar
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_payload(struct mgp_message *message, const char **result);
/// Get the payload size
/// Supported stream sources:
/// - Kafka
/// - Pulsar
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_payload_size(struct mgp_message *message, size_t *result);
/// Get the name of topic
/// Supported stream sources:
/// - Kafka
/// - Pulsar
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_topic_name(struct mgp_message *message, const char **result);
/// Get the key of mgp_message as a byte array
/// Supported stream sources:
/// - Kafka
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_key(struct mgp_message *message, const char **result);
/// Get the key size of mgp_message
/// Supported stream sources:
/// - Kafka
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_key_size(struct mgp_message *message, size_t *result);
/// Get the timestamp of mgp_message as a byte array
/// Supported stream sources:
/// - Kafka
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_timestamp(struct mgp_message *message, int64_t *result);
/// Get the message offset from a message.
/// Supported stream sources:
/// - Kafka
/// Return MGP_ERROR_INVALID_ARGUMENT if the message is from an unsupported stream source.
enum mgp_error mgp_message_offset(struct mgp_message *message, int64_t *result);
/// Get the number of messages contained in the mgp_messages list
/// Current implementation always returns without errors.
enum mgp_error mgp_messages_size(struct mgp_messages *message, size_t *result);

View File

@@ -1260,6 +1260,9 @@ class InvalidMessageError(Exception):
pass
SOURCE_TYPE_KAFKA = _mgp.SOURCE_TYPE_KAFKA
SOURCE_TYPE_PULSAR = _mgp.SOURCE_TYPE_PULSAR
class Message:
"""Represents a message from a stream."""
__slots__ = ('_message',)
@@ -1280,26 +1283,73 @@ class Message:
"""Return True if `self` is in valid context and may be used."""
return self._message.is_valid()
def source_type(self) -> str:
"""
Supported in all stream sources
Raise InvalidArgumentError if the message is from an unsupported stream source.
"""
if not self.is_valid():
raise InvalidMessageError()
return self._message.source_type()
def payload(self) -> bytes:
"""
Supported stream sources:
- Kafka
- Pulsar
Raise InvalidArgumentError if the message is from an unsupported stream source.
"""
if not self.is_valid():
raise InvalidMessageError()
return self._message.payload()
def topic_name(self) -> str:
"""
Supported stream sources:
- Kafka
- Pulsar
Raise InvalidArgumentError if the message is from an unsupported stream source.
"""
if not self.is_valid():
raise InvalidMessageError()
return self._message.topic_name()
def key(self) -> bytes:
"""
Supported stream sources:
- Kafka
Raise InvalidArgumentError if the message is from an unsupported stream source.
"""
if not self.is_valid():
raise InvalidMessageError()
return self._message.key()
def timestamp(self) -> int:
"""
Supported stream sources:
- Kafka
Raise InvalidArgumentError if the message is from an unsupported stream source.
"""
if not self.is_valid():
raise InvalidMessageError()
return self._message.timestamp()
def offset(self) -> int:
"""
Supported stream sources:
- Kafka
Raise InvalidArgumentError if the message is from an unsupported stream source.
"""
if not self.is_valid():
raise InvalidMessageError()
return self._message.offset()
class InvalidMessagesError(Exception):
"""Signals using a messages instance outside of the registered transformation."""

59
init
View File

@@ -10,6 +10,8 @@ function print_help () {
echo -e "Check for missing packages and setup the project.\n"
echo "Optional arguments:"
echo -e " -h\tdisplay this help and exit"
echo -e " --without-libs-setup\tskip the step for setting up libs"
echo -e " --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)"
}
function setup_virtualenv () {
@@ -30,21 +32,36 @@ function setup_virtualenv () {
popd > /dev/null
}
if [[ $# -gt 1 ]]; then
wsl_quicklisp_proxy=""
setup_libs=true
if [[ $# -eq 1 && "$1" == "-h" ]]; then
print_help
exit 1
elif [[ $# -eq 1 ]]; then
case "$1" in
-h)
print_help
exit 0
;;
*)
# unknown option
print_help
exit 1
;;
esac
exit 0
else
while(($#)); do
case "$1" in
--wsl-quicklisp-proxy)
shift
if [[ $# -eq 0 ]]; then
echo "Missing proxy URL"
print_help
exit 1
fi
wsl_quicklisp_proxy=":proxy \"http://$1/\""
shift
;;
--without-libs-setup)
shift
setup_libs=false
;;
*)
# unknown option
echo "Invalid argument provided: $1"
print_help
exit 1
;;
esac
done
fi
DISTRO=$(operating_system)
@@ -66,7 +83,7 @@ if [[ ! -f "${quicklisp_install_dir}/setup.lisp" ]]; then
echo \
"
(load \"${DIR}/quicklisp.lisp\")
(quicklisp-quickstart:install :path \"${quicklisp_install_dir}\")
(quicklisp-quickstart:install $wsl_quicklisp_proxy :path \"${quicklisp_install_dir}\")
" | sbcl --script || exit 1
rm -rf quicklisp.lisp || exit 1
fi
@@ -80,11 +97,13 @@ echo \
(ql:quickload '(:lcp :lcp/test) :silent t)
" | sbcl --script
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
if [[ "$setup_libs" == "true" ]]; then
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
fi
# setup gql_behave dependencies
setup_virtualenv tests/gql_behave

1
libs/.gitignore vendored
View File

@@ -5,3 +5,4 @@
!CMakeLists.txt
!__main.cpp
!jemalloc.cmake
!pulsar.patch

View File

@@ -8,6 +8,15 @@ if (NPROC EQUAL 0)
set(NPROC 1)
endif()
find_package(Boost 1.78 REQUIRED)
find_package(BZip2 1.0.6 REQUIRED)
find_package(Threads REQUIRED)
set(GFLAGS_NOTHREADS OFF)
find_package(gflags REQUIRED)
find_package(fmt 8.0.1)
find_package(Jemalloc REQUIRED)
find_package(ZLIB 1.2.11 REQUIRED)
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
# convenience functions
@@ -48,7 +57,7 @@ endfunction(import_library)
# INSTALL_COMMAND arguments.
function(add_external_project name)
set(options NO_C_COMPILER)
set(one_value_kwargs SOURCE_DIR)
set(one_value_kwargs SOURCE_DIR BUILD_IN_SOURCE)
set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
CONFIGURE_COMMAND)
cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
@@ -56,11 +65,16 @@ function(add_external_project name)
if (KW_SOURCE_DIR)
set(source_dir ${KW_SOURCE_DIR})
endif()
set(build_in_source 0)
if (KW_BUILD_IN_SOURCE)
set(build_in_source ${KW_BUILD_IN_SOURCE})
endif()
if (NOT KW_NO_C_COMPILER)
set(KW_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} ${KW_CMAKE_ARGS})
endif()
ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS}
PREFIX ${source_dir} SOURCE_DIR ${source_dir}
BUILD_IN_SOURCE ${build_in_source}
CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND}
CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -97,17 +111,12 @@ import_external_library(antlr4 STATIC
# Setup google benchmark.
import_external_library(benchmark STATIC
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/lib/libbenchmark.a
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/${CMAKE_INSTALL_LIBDIR}/libbenchmark.a
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/include
# Skip testing. The tests don't compile with Clang 8.
CMAKE_ARGS -DBENCHMARK_ENABLE_TESTING=OFF)
# setup fmt format
import_external_library(fmt STATIC
${CMAKE_CURRENT_SOURCE_DIR}/fmt/${CMAKE_INSTALL_LIBDIR}/libfmt.a
${CMAKE_CURRENT_SOURCE_DIR}/fmt/include
# Skip testing.
CMAKE_ARGS -DFMT_TEST=OFF)
include(FetchContent)
# setup rapidcheck (it cannot be external, since it doesn't have install
# target)
@@ -134,55 +143,19 @@ import_library(gtest_main STATIC ${GTEST_MAIN_LIBRARY} ${GTEST_INCLUDE_DIR} gtes
import_library(gmock STATIC ${GMOCK_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj)
import_library(gmock_main STATIC ${GMOCK_MAIN_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj)
# setup google flags
set(GFLAGS_NO_FILENAMES "0")
if ("${CMAKE_BUILD_TYPE}" MATCHES "^(R|r)(E|e)(L|l).+")
set(GFLAGS_NO_FILENAMES "1")
endif()
# setup google flags
import_external_library(gflags STATIC
${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/libgflags.a
${CMAKE_CURRENT_SOURCE_DIR}/gflags/include
# Not needed, since gflags is C++ only.
NO_C_COMPILER
# Don't register installation in ~/.cmake
CMAKE_ARGS -DREGISTER_INSTALL_PREFIX=OFF
-DBUILD_gflags_nothreads_LIB=OFF
-DGFLAGS_NO_FILENAMES=${GFLAGS_NO_FILENAMES})
# Setup cppitertools
import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR})
# Setup json
import_header_library(json ${CMAKE_CURRENT_SOURCE_DIR})
# Setup bzip2
import_external_library(bzip2 STATIC
${CMAKE_CURRENT_SOURCE_DIR}/bzip2/libbz2.a
${CMAKE_CURRENT_SOURCE_DIR}/bzip2
# bzip2's Makefile has -g CFLAG which is redundant
CONFIGURE_COMMAND sed -i "s/-Wall -Winline -O2 -g/-Wall -Winline -O2/g" ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/Makefile
BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/bzip2
CC=${CMAKE_C_COMPILER}
CXX=${CMAKE_CXX_COMPILER}
INSTALL_COMMAND true)
# Setup zlib
import_external_library(zlib STATIC
${CMAKE_CURRENT_SOURCE_DIR}/zlib/lib/libz.a
${CMAKE_CURRENT_SOURCE_DIR}/zlib
CMAKE_ARGS -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
BUILD_COMMAND $(MAKE) zlibstatic)
# Setup RocksDB
import_external_library(rocksdb STATIC
${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/lib/librocksdb.a
${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include
DEPENDS gflags-proj
CMAKE_ARGS -Dgflags_DIR=${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/cmake/gflags
-DUSE_RTTI=ON
CMAKE_ARGS -DUSE_RTTI=ON
-DWITH_TESTS=OFF
-DGFLAGS_NOTHREADS=OFF
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
BUILD_COMMAND $(MAKE) rocksdb)
@@ -217,12 +190,11 @@ add_external_project(mgconsole
add_custom_target(mgconsole DEPENDS mgconsole-proj)
# Setup spdlog
import_external_library(spdlog STATIC
${CMAKE_CURRENT_SOURCE_DIR}/spdlog/${CMAKE_INSTALL_LIBDIR}/libspdlog.a
${CMAKE_CURRENT_SOURCE_DIR}/spdlog/include
BUILD_COMMAND $(MAKE) spdlog)
set(SPDLOG_FMT_EXTERNAL ON)
FetchContent_Declare(spdlog
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/spdlog)
include(jemalloc.cmake)
FetchContent_MakeAvailable(spdlog)
# Setup librdkafka.
import_external_library(librdkafka STATIC
@@ -231,14 +203,55 @@ import_external_library(librdkafka STATIC
CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON
-DRDKAFKA_BUILD_EXAMPLES=OFF
-DRDKAFKA_BUILD_TESTS=OFF
-DWITH_ZSTD=OFF
-DENABLE_LZ4_EXT=OFF
-DCMAKE_INSTALL_LIBDIR=lib
-DWITH_SSL=ON
# If we want SASL, we need to install it on build machines
-DWITH_SASL=OFF)
target_link_libraries(librdkafka INTERFACE ${OPENSSL_LIBRARIES} zlib)
target_link_libraries(librdkafka INTERFACE ${OPENSSL_LIBRARIES} ZLIB::ZLIB)
import_library(librdkafka++ STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka++.a
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/include
)
target_link_libraries(librdkafka++ INTERFACE librdkafka)
set(PROTOBUF_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/protobuf/lib)
import_external_library(protobuf STATIC
${PROTOBUF_ROOT}/lib/libprotobuf.a
${PROTOBUF_ROOT}/include
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND true)
import_external_library(pulsar STATIC
${CMAKE_CURRENT_SOURCE_DIR}/pulsar/pulsar-client-cpp/lib/libpulsarwithdeps.a
${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install/include
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND cmake pulsar-client-cpp
-DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DBUILD_DYNAMIC_LIB=OFF
-DBUILD_STATIC_LIB=ON
-DBUILD_TESTS=OFF
-DLINK_STATIC=ON
-DPROTOC_PATH=${PROTOBUF_ROOT}/bin/protoc
-DBOOST_ROOT=${BOOST_ROOT}
-DCMAKE_PREFIX_PATH=${PROTOBUF_ROOT}
-DProtobuf_INCLUDE_DIRS=${PROTOBUF_ROOT}/include
-DBUILD_PYTHON_WRAPPER=OFF
-DBUILD_PERF_TOOLS=OFF
-DUSE_LOG4CXX=OFF
BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
add_dependencies(pulsar-proj protobuf)
if (${MG_ARCH} STREQUAL "ARM64")
set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON)
endif()
import_external_library(librdtsc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/lib/librdtsc.a
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include
CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS}
BUILD_COMMAND $(MAKE) rdtsc)

43
libs/antlr4.patch Normal file
View File

@@ -0,0 +1,43 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index a8503bb..11362cf 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
ExternalProject_Add(
utfcpp
- GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
- GIT_TAG "v3.1.1"
+ GIT_REPOSITORY "https://github.com/nemtrif/utfcpp"
+ GIT_TAG "v3.2.1"
SOURCE_DIR ${UTFCPP_DIR}
UPDATE_DISCONNECTED 1
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
DESTINATION lib
EXPORT antlr4-targets)
install(TARGETS antlr4_static
diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
index 468db98..65a473b 100644
--- a/runtime/Cpp/runtime/src/support/Any.h
+++ b/runtime/Cpp/runtime/src/support/Any.h
@@ -122,12 +122,12 @@ private:
}
private:
- template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return new Derived<T>(value);
}
- template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return nullptr;
}

29
libs/librdtsc.patch Normal file
View File

@@ -0,0 +1,29 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ee9b58c..31359a9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU "Enables PMU usage on ARM platforms" OFF)
# | Library Build and Install Properties |
# +--------------------------------------------------------+
-add_library(rdtsc SHARED
+add_library(rdtsc
src/cycles.c
src/common_timer.c
src/timer.c
@@ -72,15 +72,6 @@ target_include_directories(rdtsc
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
)
-# Install directory changes depending on build mode
-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug")
- # During debug, the library will be installed into a local directory
- set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_install CACHE PATH "" FORCE)
-else ()
- # This will install in /usr/lib and /usr/include
- set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "" FORCE)
-endif ()
-
# Specifying what to export when installing (GNUInstallDirs required)
install(TARGETS rdtsc
EXPORT librstsc-config

1520
libs/pulsar.patch Normal file

File diff suppressed because it is too large Load Diff

21
libs/rocksdb.patch Normal file
View File

@@ -0,0 +1,21 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6761929..6a369af 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -220,6 +220,7 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
endif()
endif()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-copy -Wno-unused-but-set-variable")
endif()
include(CheckCCompilerFlag)
@@ -997,7 +998,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(ROCKSDB_BUILD_SHARED)
install(
- TARGETS ${ROCKSDB_SHARED_LIB}
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
EXPORT RocksDBTargets
COMPONENT runtime
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"

View File

@@ -14,39 +14,49 @@ clone () {
local git_repo=$1
local dir_name=$2
local checkout_id=$3
shift 3
local shallow=$4
shift 4
# Clone if there's no repo.
if [[ ! -d "$dir_name" ]]; then
echo "Cloning from $git_repo"
# If the clone fails, it doesn't make sense to continue with the function
# execution but the whole script should continue executing because we might
# clone the same repo from a different source.
git clone "$git_repo" "$dir_name" || return 1
if [ "$shallow" = true ]; then
git clone --depth 1 --branch "$checkout_id" "$git_repo" "$dir_name" || return 1
else
git clone "$git_repo" "$dir_name" || return 1
fi
fi
pushd "$dir_name"
# Just fetch new commits from remote repository. Don't merge/pull them in, so
# that we don't clobber local modifications.
git fetch
# Check whether we have any local changes which need to be preserved.
local local_changes=true
if git diff --no-ext-diff --quiet && git diff --no-ext-diff --cached --quiet; then
local_changes=false
fi
# Stash regardless of local_changes, so that a user gets a message on stdout.
git stash
# Checkout the primary commit (there's no need to pull/merge).
# The checkout fail should exit this script immediately because the target
# commit is not there and that will most likely create build-time errors.
git checkout "$checkout_id" || exit 1
# Apply any optional cherry pick fixes.
while [[ $# -ne 0 ]]; do
local cherry_pick_id=$1
shift
# The cherry-pick fail should exit this script immediately because the
# target commit is not there and that will most likely create build-time
# errors.
git cherry-pick -n "$cherry_pick_id" || exit 1
done
if [ "$shallow" = false ]; then
# Stash regardless of local_changes, so that a user gets a message on stdout.
git stash
# Just fetch new commits from remote repository. Don't merge/pull them in, so
# that we don't clobber local modifications.
git fetch
# Checkout the primary commit (there's no need to pull/merge).
# The checkout fail should exit this script immediately because the target
# commit is not there and that will most likely create build-time errors.
git checkout "$checkout_id" || exit 1
# Apply any optional cherry pick fixes.
while [[ $# -ne 0 ]]; do
local cherry_pick_id=$1
shift
# The cherry-pick fail should exit this script immediately because the
# target commit is not there and that will most likely create build-time
# errors.
git cherry-pick -n "$cherry_pick_id" || exit 1
done
fi
# Reapply any local changes.
if [[ $local_changes == true ]]; then
git stash pop
@@ -70,12 +80,13 @@ repo_clone_try_double () {
secondary_url="$2"
folder_name="$3"
ref="$4"
shallow="${5:-false}"
echo "Cloning primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
clone "$primary_url" "$folder_name" "$ref" "$shallow" || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1
echo ""
}
@@ -96,23 +107,21 @@ declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["fmt"]="http://$local_cache_host/git/fmt.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
["gtest"]="http://$local_cache_host/git/googletest.git"
["gflags"]="http://$local_cache_host/git/gflags.git"
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
["bzip2"]="http://$local_cache_host/git/bzip2.git"
["zlib"]="http://$local_cache_host/git/zlib.git"
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
["mgclient"]="http://$local_cache_host/git/mgclient.git"
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
["protobuf"]="http://$local_cache_host/git/protobuf.git"
["pulsar"]="http://$local_cache_host/git/pulsar.git"
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
)
# The goal of secondary urls is to have links to the "source of truth" of
@@ -123,62 +132,47 @@ declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["fmt"]="https://github.com/fmtlib/fmt.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
["gtest"]="https://github.com/google/googletest.git"
["gflags"]="https://github.com/memgraph/gflags.git"
["libbcrypt"]="https://github.com/rg3/libbcrypt"
["bzip2"]="https://github.com/VFR-maniac/bzip2"
["zlib"]="https://github.com/madler/zlib.git"
["rocksdb"]="https://github.com/facebook/rocksdb.git"
["mgclient"]="https://github.com/memgraph/mgclient.git"
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
["spdlog"]="https://github.com/gabime/spdlog"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
["pulsar"]="https://github.com/apache/pulsar.git"
["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
)
# antlr
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="5e5b6d35b4183fd330102c40947b95c4b5c6abb5" # v4.9.2
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
# remove shared library from install dependencies
sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
# fix issue https://github.com/antlr/antlr4/issues/3194 - should update Antlr commit once the PR related to the issue gets merged
sed -i 's/std::is_nothrow_copy_constructible/std::is_copy_constructible/' antlr4/runtime/Cpp/runtime/src/support/Any.h
# replace the utf8cpp version which is used because the older one uses gtest that doesn't
# compile with the newer compilers because of uninitialized variable
# the newer utf8cpp switched to ftest
sed -i 's/v3.1.1/v3.2.1/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
antlr4_tag="4.9.2" # v4.9.2
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
pushd antlr4
git apply ../antlr4.patch
popd
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
# fmt
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
# rapidcheck
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
# google benchmark
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
benchmark_tag="v1.6.0"
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
# google test
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
# google flags
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
googletest_tag="release-1.8.0"
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
# libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
@@ -198,21 +192,14 @@ cd json
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
# remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
# remove shared library from install dependencies
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
rocksdb_tag="v6.14.6" # (2020-10-14)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
pushd rocksdb
git apply ../rocksdb.patch
popd
# mgclient
mgclient_tag="v1.3.0" # (2021-09-23)
mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
@@ -222,30 +209,32 @@ repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient
# mgconsole
mgconsole_tag="v1.1.0" # (2021-10-07)
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag"
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
pushd jemalloc
# ThreadPool select job randomly, and there can be some threads that had been
# performed some memory heavy task before and will be inactive for some time,
# but until it will became active again, the memory will not be freed since by
# default each thread has it's own arena, but there should be not more then
# 4*CPU arenas (see opt.nareans description).
#
# By enabling percpu_arena number of arenas limited to number of CPUs and hence
# this problem should go away.
#
# muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
popd
spdlog_tag="v1.9.2" # (2021-08-12)
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag" true
# librdkafka
librdkafka_tag="v1.7.0" # (2021-05-06)
repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag"
repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag" true
# protobuf
protobuf_tag="v3.12.4"
repo_clone_try_double "${primary_urls[protobuf]}" "${secondary_urls[protobuf]}" "protobuf" "$protobuf_tag" true
pushd protobuf
./autogen.sh && ./configure CC=clang CXX=clang++ --prefix=$(pwd)/lib
popd
#pulsar
pulsar_tag="v2.8.1"
repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pulsar" "$pulsar_tag" true
pushd pulsar
git apply ../pulsar.patch
popd
#librdtsc
librdtsc_tag="v0.3"
repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}" "librdtsc" "$librdtsc_tag" true
pushd librdtsc
git apply ../librdtsc.patch
popd

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2025-10-12
CHANGE DATE: 2026-17-03
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -36,9 +36,18 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
# RPM specific
set(MG_ARCH_EXTENSION "noarch")
if (${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION "x86_64")
elseif (${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION "aarch64")
endif()
set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.x86_64.rpm")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/var /var/lib /var/log /etc/logrotate.d
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)

View File

@@ -1,4 +0,0 @@
# Memgraph Release
Please continue in
[Notion](https://www.notion.so/memgraph/Release-621733518c82456ebc119d2a025b5dd3).

View File

@@ -0,0 +1,19 @@
FROM dokken/centos-stream-9
ARG env_folder
ARG toolchain_version
COPY ${env_folder} /env_folder
RUN yum update && yum install -y curl git
RUN /${env_folder}/os/centos-9.sh install MEMGRAPH_BUILD_DEPS
RUN /${env_folder}/os/centos-9.sh install TOOLCHAIN_RUN_DEPS
RUN rm -rf /env_folder
RUN yum clean all
RUN curl https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/${toolchain_version}/${toolchain_version}-binaries-centos-9-arm64.tar.gz -o /tmp/toolchain.tar.gz \
&& tar xvzf /tmp/toolchain.tar.gz -C /opt \
&& rm /tmp/toolchain.tar.gz

5
release/arm64/build_env.sh Executable file
View File

@@ -0,0 +1,5 @@
#!/bin/bash
cp -r ../../environment env_folder
docker build -f build_env.dockerfile --build-arg env_folder=env_folder --build-arg toolchain_version=toolchain-v4 -t mg_build_env .
rm -rf env_folder

View File

@@ -1 +1 @@
../LICENSE.md
../../LICENSE

View File

@@ -1,19 +1,19 @@
FROM debian:buster
FROM debian:bullseye
# NOTE: If you change the base distro update release/package as well.
ARG deb_release
ARG release
RUN apt-get update && apt-get install -y \
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.7 python3-pip \
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
RUN pip3 install networkx==2.4 numpy==1.19.2 scipy==1.5.2
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${deb_release} /
COPY ${release} /
# Install memgraph package
RUN dpkg -i ${deb_release}
RUN dpkg -i ${release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687

View File

@@ -0,0 +1,31 @@
FROM dokken/centos-stream-9
# NOTE: If you change the base distro update release/package as well.
ARG release
RUN yum update && yum install -y \
openssl libcurl libseccomp python3 python3-pip \
--nobest --allowerasing \
&& rm -rf /tmp/* \
&& yum clean all
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${release} /
# Install memgraph package
RUN rpm -i ${release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687
# Snapshots and logging volumes
VOLUME /var/log/memgraph
VOLUME /var/lib/memgraph
# Configuration volume
VOLUME /etc/memgraph
USER memgraph
WORKDIR /usr/lib/memgraph
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
CMD [""]

View File

@@ -1,46 +0,0 @@
#!/bin/bash -e
# Build and package Docker image of Memgraph.
function print_help () {
echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.deb"
echo "Optional arguments:"
echo -e "\t-h|--help\t\tPrint help."
echo -e "\t--latest\t\tTag image as latest version."
}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
deb_path="$1"
if [[ ! -f "$deb_path" ]]; then
echo "File '$deb_path' does not exist!"
exit 1
fi
# Copy the .deb to working directory.
cp "$deb_path" "${working_dir}/"
cd ${working_dir}
# Extract version and offering from deb name.
deb_name=`echo $(basename "$deb_path") | sed 's/.deb$//'`
version=`echo ${deb_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
dockerfile_path="${working_dir}/memgraph.dockerfile"
image_name="memgraph:${version}"
package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg deb_release=${deb_name}.deb .
docker save ${image_name} ${latest_image} > ${package_name}
rm "${deb_name}.deb"
echo "Built Docker image at '${working_dir}/${package_name}'"

61
release/docker/package_docker Executable file
View File

@@ -0,0 +1,61 @@
#!/bin/bash -e
# Build and package Docker image of Memgraph.
function print_help () {
echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.(deb|rpm)"
echo "Optional arguments:"
echo -e "\t-h|--help\t\tPrint help."
echo -e "\t--latest\t\tTag image as latest version."
}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
package_path="$1"
if [[ ! -f "$package_path" ]]; then
echo "File '$package_path' does not exist!"
exit 1
fi
# Copy the .deb to working directory.
cp "$package_path" "${working_dir}/"
cd ${working_dir}
extension="${package_path##*.}"
if [[ "$extension" == "deb" ]]; then
# Extract version and offering from deb name.
package_name=`echo $(basename "$package_path") | sed 's/.deb$//'`
version=`echo ${package_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
dockerfile_path="${working_dir}/memgraph_deb.dockerfile"
elif [[ "$extension" == "rpm" ]]; then
# Extract version and offering from deb name.
package_name=`echo $(basename "$package_path") | sed 's/.rpm$//'`
version=`echo ${package_name} | cut -d '-' -f 2 | rev | cut -d '-' -f 2- | rev`
version=${version%_1}
dockerfile_path="${working_dir}/memgraph_rpm.dockerfile"
else
echo "Invalid file sent as the package"
print_help
exit 1
fi
image_name="memgraph:${version}"
image_package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
docker save ${image_name} ${latest_image} | gzip > ${image_package_name}
rm "${package_name}.${extension}"
echo "Built Docker image at '${working_dir}/${image_package_name}'"

View File

@@ -1,10 +1,11 @@
#!/usr/bin/env python3
import argparse
import os
import re
import subprocess
import sys
import os
import time
from functools import wraps
# This script is used to determine the current version of Memgraph. The script
# determines the current version using `git` automatically. The user can also
@@ -93,16 +94,29 @@ import os
# https://fedoraproject.org/wiki/Package_Versioning_Examples
def retry(retry_limit, timeout=100):
def inner_func(func):
@wraps(func)
def wrapper(*args, **kwargs):
for _ in range(retry_limit):
try:
return func(*args, **kwargs)
except Exception:
time.sleep(timeout)
return func(*args, **kwargs)
return wrapper
return inner_func
@retry(3)
def get_output(*cmd, multiple=False):
ret = subprocess.run(cmd, stdout=subprocess.PIPE, check=True)
if multiple:
return list(map(lambda x: x.strip(),
ret.stdout.decode("utf-8").strip().split("\n")))
return list(map(lambda x: x.strip(), ret.stdout.decode("utf-8").strip().split("\n")))
return ret.stdout.decode("utf-8").strip()
def format_version(variant, version, offering, distance=None, shorthash=None,
suffix=None):
def format_version(variant, version, offering, distance=None, shorthash=None, suffix=None):
if not distance:
# This is a release version.
if variant == "deb":
@@ -135,8 +149,7 @@ def format_version(variant, version, offering, distance=None, shorthash=None,
return ret
elif variant == "rpm":
# <VERSION>_0.<DISTANCE>.<SHORTHASH>.<OFFERING>[.<SUFFIX>]
ret = "{}_0.{}.{}{}".format(
version, distance, shorthash, "." + offering if offering else "")
ret = "{}_0.{}.{}{}".format(version, distance, shorthash, "." + offering if offering else "")
if suffix:
ret += "." + suffix
return ret
@@ -149,27 +162,23 @@ def format_version(variant, version, offering, distance=None, shorthash=None,
# Parse arguments.
parser = argparse.ArgumentParser(
description="Get the current version of Memgraph.")
parser = argparse.ArgumentParser(description="Get the current version of Memgraph.")
parser.add_argument("--open-source", action="store_true", help="set the current offering to 'open-source'")
parser.add_argument("version", help="manual version override, if supplied the version isn't " "determined using git")
parser.add_argument("suffix", help="custom suffix for the current version being built")
parser.add_argument(
"--open-source", action="store_true",
help="set the current offering to 'open-source'")
"--variant",
choices=("binary", "deb", "rpm"),
default="binary",
help="which variant of the version string should be generated",
)
parser.add_argument(
"version", help="manual version override, if supplied the version isn't "
"determined using git")
parser.add_argument(
"suffix", help="custom suffix for the current version being built")
parser.add_argument(
"--variant", choices=("binary", "deb", "rpm"), default="binary",
help="which variant of the version string should be generated")
parser.add_argument(
"--memgraph-root-dir", help="The root directory of the checked out "
"Memgraph repository.", default=".")
"--memgraph-root-dir", help="The root directory of the checked out " "Memgraph repository.", default="."
)
args = parser.parse_args()
if not os.path.isdir(args.memgraph_root_dir):
raise Exception("The root directory ({}) is not a valid directory".format(
args.memgraph_root_dir))
raise Exception("The root directory ({}) is not a valid directory".format(args.memgraph_root_dir))
os.chdir(args.memgraph_root_dir)
@@ -179,8 +188,7 @@ offering = "open-source" if args.open_source else None
if args.version:
if not re.match(r"^[0-9]+\.[0-9]+\.[0-9]+$", args.version):
raise Exception("Invalid version supplied '{}'!".format(args.version))
print(format_version(args.variant, args.version, offering,
suffix=args.suffix), end="")
print(format_version(args.variant, args.version, offering, suffix=args.suffix), end="")
sys.exit(0)
# Within CI, after the regular checkout, master is sometimes (e.g. in the case
@@ -234,33 +242,28 @@ versions.sort(reverse=True)
current_version = None
for version in versions:
version_tuple, branch, master_branch_merge = version
current_branch_merge = get_output(
"git", "merge-base", current_hash, branch)
master_current_merge = get_output(
"git", "merge-base", current_hash, "master")
current_branch_merge = get_output("git", "merge-base", current_hash, branch)
master_current_merge = get_output("git", "merge-base", current_hash, "master")
# The first check checks whether this commit is a child of `master` and
# the version branch was created before us.
# The second check checks whether this commit is a child of the version
# branch.
if master_branch_merge == current_branch_merge or \
master_branch_merge == master_current_merge:
if master_branch_merge == current_branch_merge or master_branch_merge == master_current_merge:
current_version = version
break
# Determine current version.
if current_version is None:
raise Exception("You are attempting to determine the version for a very "
"old version of Memgraph!")
raise Exception("You are attempting to determine the version for a very " "old version of Memgraph!")
version, branch, master_branch_merge = current_version
distance = int(get_output("git", "rev-list", "--count", "--first-parent",
master_branch_merge + ".." + current_hash))
distance = int(get_output("git", "rev-list", "--count", "--first-parent", master_branch_merge + ".." + current_hash))
version_str = ".".join(map(str, version)) + ".0"
if distance == 0:
print(format_version(args.variant, version_str, offering,
suffix=args.suffix),
end="")
print(format_version(args.variant, version_str, offering, suffix=args.suffix), end="")
else:
print(format_version(args.variant, version_str, offering,
distance=distance, shorthash=current_hash_short,
suffix=args.suffix),
end="")
print(
format_version(
args.variant, version_str, offering, distance=distance, shorthash=current_hash_short, suffix=args.suffix
),
end="",
)

View File

@@ -1,17 +0,0 @@
FROM debian:9
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-9.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-9.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-9.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -8,10 +8,6 @@ services:
build:
context: centos-8
container_name: "mgbuild_centos-8"
mgbuild_debian-9:
build:
context: debian-9
container_name: "mgbuild_debian-9"
mgbuild_debian-10:
build:
context: debian-10

View File

@@ -3,9 +3,9 @@
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v3"
TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
@@ -18,14 +18,21 @@ print_help () {
make_package () {
os="$1"
build_container="mgbuild_$os"
echo "Building Memgraph for $os on $build_container..."
package_command=""
if [[ "$os" =~ ^"centos".* ]]; then
docker exec "$build_container" bash -c "yum -y update"
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
docker exec "$build_container" bash -c "apt update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
docker exec "$build_container" bash -c "apt update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
telemetry_id_override_flag=""
@@ -39,8 +46,6 @@ make_package () {
exit
fi
fi
build_container="mgbuild_$os"
echo "Building Memgraph for $os on $build_container..."
echo "Copying project files..."
# If master is not the current branch, fetch it, because the get_version
@@ -67,7 +72,7 @@ make_package () {
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
docker exec "$build_container" bash -c "cd /memgraph && ./init"
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
# ' is used instead of " because we need to run make within the allowed
@@ -95,13 +100,13 @@ case "$1" in
;;
docker)
# NOTE: Docker is build on top of Debian 10 package.
based_on_os="debian-10"
# NOTE: Docker is build on top of Debian 11 package.
based_on_os="debian-11"
# shellcheck disable=SC2012
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
./package_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
# shellcheck disable=SC2012
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
docker_host_folder="$HOST_OUTPUT_DIR/docker"

View File

@@ -37,7 +37,7 @@ set(mg_single_node_v2_sources
)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth)
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)

View File

@@ -1,5 +1,8 @@
set(audit_src_files log.cpp)
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
add_library(mg-audit STATIC ${audit_src_files})
target_link_libraries(mg-audit json gflags fmt)
target_link_libraries(mg-audit json gflags fmt::fmt)
target_link_libraries(mg-audit mg-utils mg-storage-v2)

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -18,7 +18,7 @@
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace audit {
namespace memgraph::audit {
// Helper function that converts a `storage::PropertyValue` to `nlohmann::json`.
inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) {
@@ -143,4 +143,4 @@ void Log::Flush() {
log_.Sync();
}
} // namespace audit
} // namespace memgraph::audit

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -17,7 +17,7 @@
#include "utils/file.hpp"
#include "utils/scheduler.hpp"
namespace audit {
namespace memgraph::audit {
const uint64_t kBufferSizeDefault = 100000;
const uint64_t kBufferFlushIntervalMillisDefault = 200;
@@ -71,4 +71,4 @@ class Log {
std::mutex lock_;
};
} // namespace audit
} // namespace memgraph::audit

View File

@@ -5,10 +5,13 @@ set(auth_src_files
module.cpp)
find_package(Seccomp REQUIRED)
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
add_library(mg-auth STATIC ${auth_src_files})
target_link_libraries(mg-auth json libbcrypt gflags fmt)
target_link_libraries(mg-auth mg-utils mg-kvstore)
target_link_libraries(mg-auth json libbcrypt gflags fmt::fmt)
target_link_libraries(mg-auth mg-utils mg-kvstore mg-license )
target_link_libraries(mg-auth ${Seccomp_LIBRARIES})
target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS})

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -42,8 +42,7 @@ DEFINE_VALIDATED_int32(auth_module_timeout_ms, 10000,
"response from the auth module.",
FLAG_IN_RANGE(100, 1800000));
namespace auth {
namespace memgraph::auth {
const std::string kUserPrefix = "user:";
const std::string kRolePrefix = "role:";
const std::string kLinkPrefix = "link:";
@@ -316,4 +315,4 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
return ret;
}
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -18,8 +18,7 @@
#include "kvstore/kvstore.hpp"
#include "utils/settings.hpp"
namespace auth {
namespace memgraph::auth {
/**
* This class serves as the main Authentication/Authorization storage.
* It provides functions for managing Users, Roles and Permissions.
@@ -163,4 +162,4 @@ class Auth final {
kvstore::KVStore storage_;
auth::Module module_;
};
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -12,8 +12,7 @@
#include "auth/exceptions.hpp"
namespace auth {
namespace memgraph::auth {
const std::string EncryptPassword(const std::string &password) {
char salt[BCRYPT_HASHSIZE];
char hash[BCRYPT_HASHSIZE];
@@ -40,4 +39,4 @@ bool VerifyPassword(const std::string &password, const std::string &hash) {
return ret == 0;
}
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -10,12 +10,11 @@
#include <string>
namespace auth {
namespace memgraph::auth {
/// @throw AuthException if unable to encrypt the password.
const std::string EncryptPassword(const std::string &password);
/// @throw AuthException if unable to verify the password.
bool VerifyPassword(const std::string &password, const std::string &hash);
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,8 +13,7 @@
#include "utils/exceptions.hpp"
namespace auth {
namespace memgraph::auth {
/**
* This exception class is thrown for all exceptions that can occur when dealing
* with the Auth library.
@@ -23,4 +22,4 @@ class AuthException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
};
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -28,7 +28,17 @@ DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
"The regular expression that should be used to match the entire "
"entered password to ensure its strength.");
namespace auth {
namespace memgraph::auth {
namespace {
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {
Permission::MATCH, Permission::CREATE, Permission::MERGE, Permission::DELETE,
Permission::SET, Permission::REMOVE, Permission::INDEX, Permission::STATS,
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET};
} // namespace
std::string PermissionToString(Permission permission) {
switch (permission) {
@@ -68,6 +78,12 @@ std::string PermissionToString(Permission permission) {
return "AUTH";
case Permission::STREAM:
return "STREAM";
case Permission::MODULE_READ:
return "MODULE_READ";
case Permission::MODULE_WRITE:
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
}
}
@@ -286,4 +302,4 @@ bool operator==(const User &first, const User &second) {
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_;
}
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -13,41 +13,35 @@
#include <json/json.hpp>
namespace auth {
namespace memgraph::auth {
// These permissions must have values that are applicable for usage in a
// bitmask.
// clang-format off
enum class Permission : uint64_t {
MATCH = 1,
CREATE = 1U << 1U,
MERGE = 1U << 2U,
DELETE = 1U << 3U,
SET = 1U << 4U,
REMOVE = 1U << 5U,
INDEX = 1U << 6U,
STATS = 1U << 7U,
CONSTRAINT = 1U << 8U,
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
DURABILITY = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
TRIGGER = 1U << 14U,
CONFIG = 1U << 15U,
AUTH = 1U << 16U,
STREAM = 1U << 17U
MATCH = 1,
CREATE = 1U << 1U,
MERGE = 1U << 2U,
DELETE = 1U << 3U,
SET = 1U << 4U,
REMOVE = 1U << 5U,
INDEX = 1U << 6U,
STATS = 1U << 7U,
CONSTRAINT = 1U << 8U,
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
DURABILITY = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
TRIGGER = 1U << 14U,
CONFIG = 1U << 15U,
AUTH = 1U << 16U,
STREAM = 1U << 17U,
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U
};
// clang-format on
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permission::CREATE, Permission::MERGE,
Permission::DELETE, Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS, Permission::CONSTRAINT,
Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY,
Permission::TRIGGER, Permission::CONFIG, Permission::STREAM};
// Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission);
@@ -159,4 +153,4 @@ class User final {
};
bool operator==(const User &first, const User &second);
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -153,7 +153,7 @@ int Target(void *arg) {
// process and something really bad could happen.
// Get a pointer to the passed arguments.
auto *ta = reinterpret_cast<auth::TargetArguments *>(arg);
auto *ta = reinterpret_cast<memgraph::auth::TargetArguments *>(arg);
// Redirect `stdin` to `/dev/null`.
int fd = open("/dev/null", O_RDONLY | O_CLOEXEC);
@@ -312,8 +312,7 @@ nlohmann::json GetData(int fd, int timeout_millisec) {
} // namespace
namespace auth {
namespace memgraph::auth {
Module::Module(const std::filesystem::path &module_executable_path) {
if (!module_executable_path.empty()) {
module_executable_path_ = std::filesystem::absolute(module_executable_path);
@@ -447,4 +446,4 @@ void Module::Shutdown() {
Module::~Module() { Shutdown(); }
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -16,8 +16,7 @@
#include <json/json.hpp>
namespace auth {
namespace memgraph::auth {
struct TargetArguments {
std::filesystem::path module_executable_path;
int pipe_to_module{-1};
@@ -70,4 +69,4 @@ class Module final {
int pipe_from_module_[2] = {-1, -1};
};
} // namespace auth
} // namespace memgraph::auth

View File

@@ -1,4 +1,11 @@
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
set(communication_src_files
websocket/auth.cpp
websocket/server.cpp
websocket/listener.cpp
websocket/session.cpp
bolt/v1/value.cpp
buffer.cpp
client.cpp
@@ -6,8 +13,10 @@ set(communication_src_files
helpers.cpp
init.cpp)
find_package(Boost REQUIRED)
add_library(mg-communication STATIC ${communication_src_files})
target_link_libraries(mg-communication Threads::Threads mg-utils mg-io fmt gflags)
target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io mg-auth fmt::fmt gflags)
find_package(OpenSSL REQUIRED)
target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,7 +21,7 @@
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/// This exception is thrown whenever an error occurs during query execution
/// that isn't fatal (eg. mistyped query or some transient error occurred).
@@ -315,4 +315,4 @@ class Client final {
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_};
ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>> encoder_{encoder_buffer_};
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,7 +13,7 @@
#include <cstdint>
namespace communication::bolt {
namespace memgraph::communication::bolt {
static constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
static constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
@@ -31,6 +31,7 @@ enum class Signature : uint8_t {
Begin = 0x11,
Commit = 0x12,
Rollback = 0x13,
Route = 0x66,
Record = 0x71,
Success = 0x70,
@@ -99,4 +100,4 @@ static constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList, M
static constexpr Marker Marker8[3] = {Marker::String8, Marker::List8, Marker::Map8};
static constexpr Marker Marker16[3] = {Marker::String16, Marker::List16, Marker::Map16};
static constexpr Marker Marker32[3] = {Marker::String32, Marker::List32, Marker::Map32};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,7 @@
#include <cstddef>
#include <cstdint>
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Sizes related to the chunk defined in Bolt protocol.
@@ -28,8 +28,8 @@ static constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
*/
static constexpr size_t kHandshakeSize = 20;
static constexpr uint16_t kSupportedVersions[3] = {0x0100, 0x0400, 0x0401};
static constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
static constexpr int kPullAll = -1;
static constexpr int kPullLast = -1;
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,7 @@
#include "communication/bolt/v1/constants.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* This class is used as the return value of the GetChunk function of the
@@ -136,4 +136,4 @@ class ChunkedDecoderBuffer {
std::vector<uint8_t> data_;
size_t pos_{0};
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,7 +22,7 @@
#include "utils/logging.hpp"
#include "utils/temporal.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Bolt Decoder.
@@ -591,4 +591,4 @@ class Decoder {
return true;
}
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,7 +22,7 @@ static_assert(std::is_same_v<std::uint8_t, char> || std::is_same_v<std::uint8_t,
"communication::bolt::Encoder requires uint8_t to be "
"implemented as char or unsigned char.");
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Bolt BaseEncoder. Has public interfaces for writing Bolt encoded data.
@@ -273,4 +273,4 @@ class BaseEncoder {
}
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,7 @@
#include "communication/bolt/v1/constants.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* @brief ChunkedEncoderBuffer
@@ -123,4 +123,4 @@ class ChunkedEncoderBuffer {
// Amount of data in chunk array.
size_t have_{0};
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,7 @@
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/encoder/base_encoder.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Bolt Client Encoder.
@@ -169,4 +169,4 @@ class ClientEncoder : private BaseEncoder<Buffer> {
return buffer_.Flush();
}
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,7 @@
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/encoder/base_encoder.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Bolt Encoder.
@@ -158,4 +158,4 @@ class Encoder : private BaseEncoder<Buffer> {
return buffer_.Flush();
}
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -17,7 +17,7 @@
#include "utils/exceptions.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Used to indicate something is wrong with the client but the transaction is
@@ -83,4 +83,4 @@ class VerboseError : public utils::BasicException {
std::string code_;
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -27,7 +27,7 @@
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Bolt Session Exception
@@ -195,4 +195,4 @@ class Session {
}
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,7 +13,7 @@
#include <cstdint>
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* This class represents states in execution of the Bolt protocol.
@@ -55,4 +55,4 @@ enum class State : uint8_t {
*/
Close
};
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,7 @@
#include "utils/likely.hpp"
#include "utils/logging.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/**
* Error state run function
@@ -95,4 +95,4 @@ State StateErrorRun(TSession &session, State state) {
return state;
}
}
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,6 +12,7 @@
#pragma once
#include <map>
#include <memory>
#include <new>
#include <string>
@@ -19,339 +20,69 @@
#include "communication/bolt/v1/constants.hpp"
#include "communication/bolt/v1/exceptions.hpp"
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/states/handlers.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
#include "utils/likely.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace communication::bolt {
// TODO (mferencevic): revise these error messages
inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::exception &e) {
if (auto *verbose = dynamic_cast<const VerboseError *>(&e)) {
return {verbose->code(), verbose->what()};
}
if (dynamic_cast<const ClientError *>(&e)) {
// Clients expect 4 strings separated by dots. First being database name
// (for example: Neo, Memgraph...), second being either ClientError,
// TransientError or DatabaseError (or ClientNotification for warnings).
// ClientError means wrong query, do not retry. DatabaseError means
// something wrong in database, do not retry. TransientError means query
// failed, but if retried it may succeed, retry it.
//
// Third and fourth strings being namespace and specific error name.
// It is not really important what we put there since we don't expect
// any special handling of specific exceptions on client side, but we
// need to make sure that we don't accidentally return some exception
// name which clients handle in a special way. For example, if client
// receives *.TransientError.Transaction.Terminate it will not rerun
// query even though TransientError was returned, because of Neo's
// semantics of that error.
return {"Memgraph.ClientError.MemgraphError.MemgraphError", e.what()};
}
if (dynamic_cast<const utils::BasicException *>(&e)) {
// Exception not derived from QueryException was thrown which means that
// database probably aborted transaction because of some timeout,
// deadlock, serialization error or something similar. We return
// TransientError since retry of same transaction could succeed.
return {"Memgraph.TransientError.MemgraphError.MemgraphError", e.what()};
}
if (dynamic_cast<const std::bad_alloc *>(&e)) {
// std::bad_alloc was thrown, God knows in which state is database ->
// terminate.
LOG_FATAL("Memgraph is out of memory");
}
// All exceptions used in memgraph are derived from BasicException. Since
// we caught some other exception we don't know what is going on. Return
// DatabaseError, log real message and return generic string.
spdlog::error(
utils::MessageWithLink("Unknown exception occurred during query execution {}.", e.what(), "https://memgr.ph/unknown"));
return {"Memgraph.DatabaseError.MemgraphError.MemgraphError",
"An unknown exception occurred, this is unexpected. Real message "
"should be in database logs."};
}
namespace memgraph::communication::bolt {
template <typename TSession>
inline State HandleFailure(TSession &session, const std::exception &e) {
spdlog::trace("Error message: {}", e.what());
if (const auto *p = dynamic_cast<const utils::StacktraceException *>(&e)) {
spdlog::trace("Error trace: {}", p->trace());
}
session.encoder_buffer_.Clear();
auto code_message = ExceptionToErrorMessage(e);
bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
if (!fail_sent) {
spdlog::trace("Couldn't send failure message!");
return State::Close;
}
return State::Error;
}
template <typename TSession>
State HandleRun(TSession &session, State state, Marker marker) {
const std::map<std::string, Value> kEmptyFields = {{"fields", std::vector<Value>{}}};
const auto expected_marker = session.version_.major == 1 ? Marker::TinyStruct2 : Marker::TinyStruct3;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!",
session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query, params, extra;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
}
if (!session.decoder_.ReadValue(&params, Value::Type::Map)) {
spdlog::trace("Couldn't read parameters!");
return State::Close;
}
if (session.version_.major == 4) {
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra field!");
}
}
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
State RunHandlerV1(Signature signature, TSession &session, State state, Marker marker) {
switch (signature) {
case Signature::Run:
return HandleRunV1<TSession>(session, state, marker);
case Signature::Pull:
return HandlePullV1<TSession>(session, state, marker);
case Signature::Discard:
return HandleDiscardV1<TSession>(session, state, marker);
case Signature::Reset:
return HandleReset<TSession>(session, marker);
default:
spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
namespace detail {
template <bool is_pull, typename TSession>
State HandlePullDiscard(TSession &session, State state, Marker marker) {
const auto expected_marker = session.version_.major == 1 ? Marker::TinyStruct : Marker::TinyStruct1;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!",
session.version_.major == 1 ? "TinyStruct" : "TinyStruct1", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Result) {
if constexpr (is_pull) {
spdlog::trace("Unexpected PULL!");
} else {
spdlog::trace("Unexpected DISCARD!");
}
// Same as `unexpected RUN` case.
return State::Close;
}
try {
std::optional<int> n;
std::optional<int> qid;
if (session.version_.major == 4) {
Value extra;
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra field!");
}
const auto &extra_map = extra.ValueMap();
if (extra_map.count("n")) {
if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
n = n_value;
}
}
if (extra_map.count("qid")) {
if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
qid = qid_value;
}
template <typename TSession, int bolt_minor = 0>
State RunHandlerV4(Signature signature, TSession &session, State state, Marker marker) {
switch (signature) {
case Signature::Run:
return HandleRunV4<TSession>(session, state, marker);
case Signature::Pull:
return HandlePullV4<TSession>(session, state, marker);
case Signature::Discard:
return HandleDiscardV4<TSession>(session, state, marker);
case Signature::Reset:
return HandleReset<TSession>(session, marker);
case Signature::Begin:
return HandleBegin<TSession>(session, state, marker);
case Signature::Commit:
return HandleCommit<TSession>(session, state, marker);
case Signature::Goodbye:
return HandleGoodbye<TSession>();
case Signature::Rollback:
return HandleRollback<TSession>(session, state, marker);
case Signature::Noop: {
if constexpr (bolt_minor >= 1) {
return HandleNoop<TSession>(state);
} else {
spdlog::trace("Supported only in bolt v4.1");
return State::Close;
}
}
std::map<std::string, Value> summary;
if constexpr (is_pull) {
// Pull can throw.
summary = session.Pull(&session.encoder_, n, qid);
} else {
summary = session.Discard(n, qid);
case Signature::Route: {
if constexpr (bolt_minor >= 3) {
if (signature == Signature::Route) return HandleRoute<TSession>(session);
} else {
spdlog::trace("Supported only in bolt v4.3");
return State::Close;
}
}
if (!session.encoder_.MessageSuccess(summary)) {
spdlog::trace("Couldn't send query summary!");
default:
spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
return State::Close;
}
if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
return State::Result;
}
return State::Idle;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
} // namespace detail
template <typename Session>
State HandlePull(Session &session, State state, Marker marker) {
return detail::HandlePullDiscard<true>(session, state, marker);
}
template <typename Session>
State HandleDiscard(Session &session, State state, Marker marker) {
return detail::HandlePullDiscard<false>(session, state, marker);
}
template <typename Session>
State HandleReset(Session &session, State, Marker marker) {
// IMPORTANT: This implementation of the Bolt RESET command isn't fully
// compliant to the protocol definition. In the protocol it is defined
// that this command should immediately stop any running commands and
// reset the session to a clean state. That means that we should always
// make a look-ahead for the RESET command before processing anything.
// Our implementation, for now, does everything in a blocking fashion
// so we cannot simply "kill" a transaction while it is running. So
// now this command only resets the session to a clean state. It
// does not IGNORE running and pending commands as it should.
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return State::Close;
}
// Clear all pending data and send a success message.
session.encoder_buffer_.Clear();
if (!session.encoder_.MessageSuccess()) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
session.Abort();
return State::Idle;
}
template <typename Session>
State HandleBegin(Session &session, State state, Marker marker) {
if (session.version_.major == 1) {
spdlog::trace("BEGIN messsage not supported in Bolt v1!");
return State::Close;
}
if (marker != Marker::TinyStruct1) {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
Value extra;
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra fields!");
return State::Close;
}
if (state != State::Idle) {
spdlog::trace("Unexpected BEGIN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
try {
session.BeginTransaction();
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
return State::Idle;
}
template <typename Session>
State HandleCommit(Session &session, State state, Marker marker) {
if (session.version_.major == 1) {
spdlog::trace("COMMIT messsage not supported in Bolt v1!");
return State::Close;
}
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Idle) {
spdlog::trace("Unexpected COMMIT command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
try {
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
session.CommitTransaction();
return State::Idle;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <typename Session>
State HandleRollback(Session &session, State state, Marker marker) {
if (session.version_.major == 1) {
spdlog::trace("ROLLBACK messsage not supported in Bolt v1!");
return State::Close;
}
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Idle) {
spdlog::trace("Unexpected ROLLBACK command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
try {
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
session.RollbackTransaction();
return State::Idle;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
@@ -361,8 +92,8 @@ State HandleRollback(Session &session, State state, Marker marker) {
* It executes: RUN, PULL_ALL, DISCARD_ALL & RESET.
* @param session the session that should be used for the run
*/
template <typename Session>
State StateExecutingRun(Session &session, State state) {
template <typename TSession>
State StateExecutingRun(TSession &session, State state) {
Marker marker;
Signature signature;
if (!session.decoder_.ReadMessageHeader(&signature, &marker)) {
@@ -370,30 +101,21 @@ State StateExecutingRun(Session &session, State state) {
return State::Close;
}
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
spdlog::trace("Received NOOP message");
return state;
}
if (signature == Signature::Run) {
return HandleRun(session, state, marker);
} else if (signature == Signature::Pull) {
return HandlePull(session, state, marker);
} else if (signature == Signature::Discard) {
return HandleDiscard(session, state, marker);
} else if (signature == Signature::Begin) {
return HandleBegin(session, state, marker);
} else if (signature == Signature::Commit) {
return HandleCommit(session, state, marker);
} else if (signature == Signature::Rollback) {
return HandleRollback(session, state, marker);
} else if (signature == Signature::Reset) {
return HandleReset(session, state, marker);
} else if (signature == Signature::Goodbye && session.version_.major != 1) {
throw SessionClosedException("Closing connection.");
} else {
spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
return State::Close;
switch (session.version_.major) {
case 1:
return RunHandlerV1(signature, session, state, marker);
case 4: {
if (session.version_.minor >= 3) {
return RunHandlerV4<TSession, 3>(signature, session, state, marker);
}
if (session.version_.minor >= 1) {
return RunHandlerV4<TSession, 1>(signature, session, state, marker);
}
return RunHandlerV4<TSession>(signature, session, state, marker);
}
default:
spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
return State::Close;
}
}
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -0,0 +1,418 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <map>
#include <string>
#include <string_view>
#include <vector>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace memgraph::communication::bolt {
// TODO: Revise these error messages
inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::exception &e) {
if (const auto *verbose = dynamic_cast<const VerboseError *>(&e)) {
return {verbose->code(), verbose->what()};
}
if (dynamic_cast<const ClientError *>(&e)) {
// Clients expect 4 strings separated by dots. First being database name
// (for example: Neo, Memgraph...), second being either ClientError,
// TransientError or DatabaseError (or ClientNotification for warnings).
// ClientError means wrong query, do not retry. DatabaseError means
// something wrong in database, do not retry. TransientError means query
// failed, but if retried it may succeed, retry it.
//
// Third and fourth strings being namespace and specific error name.
// It is not really important what we put there since we don't expect
// any special handling of specific exceptions on client side, but we
// need to make sure that we don't accidentally return some exception
// name which clients handle in a special way. For example, if client
// receives *.TransientError.Transaction.Terminate it will not rerun
// query even though TransientError was returned, because of Neo's
// semantics of that error.
return {"Memgraph.ClientError.MemgraphError.MemgraphError", e.what()};
}
if (dynamic_cast<const utils::BasicException *>(&e)) {
// Exception not derived from QueryException was thrown which means that
// database probably aborted transaction because of some timeout,
// deadlock, serialization error or something similar. We return
// TransientError since retry of same transaction could succeed.
return {"Memgraph.TransientError.MemgraphError.MemgraphError", e.what()};
}
if (dynamic_cast<const std::bad_alloc *>(&e)) {
// std::bad_alloc was thrown, God knows in which state is database ->
// terminate.
LOG_FATAL("Memgraph is out of memory");
}
// All exceptions used in memgraph are derived from BasicException. Since
// we caught some other exception we don't know what is going on. Return
// DatabaseError, log real message and return generic string.
spdlog::error(utils::MessageWithLink("Unknown exception occurred during query execution {}.", e.what(),
"https://memgr.ph/unknown"));
return {"Memgraph.DatabaseError.MemgraphError.MemgraphError",
"An unknown exception occurred, this is unexpected. Real message "
"should be in database logs."};
}
namespace details {
template <typename TSession>
State HandleRun(TSession &session, const State state, const Value &query, const Value &params) {
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <bool is_pull, typename TSession>
State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<int> qid) {
try {
std::map<std::string, Value> summary;
if constexpr (is_pull) {
// Pull can throw.
summary = session.Pull(&session.encoder_, n, qid);
} else {
summary = session.Discard(n, qid);
}
if (!session.encoder_.MessageSuccess(summary)) {
spdlog::trace("Couldn't send query summary!");
return State::Close;
}
if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
return State::Result;
}
return State::Idle;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <bool is_pull, typename TSession>
State HandlePullDiscardV1(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Result) {
if constexpr (is_pull) {
spdlog::trace("Unexpected PULL!");
} else {
spdlog::trace("Unexpected DISCARD!");
}
// Same as `unexpected RUN` case.
return State::Close;
}
return HandlePullDiscard<is_pull, TSession>(session, std::nullopt, std::nullopt);
}
template <bool is_pull, typename TSession>
State HandlePullDiscardV4(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct1;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct1", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Result) {
if constexpr (is_pull) {
spdlog::trace("Unexpected PULL!");
} else {
spdlog::trace("Unexpected DISCARD!");
}
// Same as `unexpected RUN` case.
return State::Close;
}
std::optional<int> n;
std::optional<int> qid;
Value extra;
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra field!");
}
const auto &extra_map = extra.ValueMap();
if (extra_map.count("n")) {
if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
n = n_value;
}
}
if (extra_map.count("qid")) {
if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
qid = qid_value;
}
}
return HandlePullDiscard<is_pull, TSession>(session, n, qid);
}
} // namespace details
template <typename TSession>
inline State HandleFailure(TSession &session, const std::exception &e) {
spdlog::trace("Error message: {}", e.what());
if (const auto *p = dynamic_cast<const utils::StacktraceException *>(&e)) {
spdlog::trace("Error trace: {}", p->trace());
}
session.encoder_buffer_.Clear();
auto code_message = ExceptionToErrorMessage(e);
bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
if (!fail_sent) {
spdlog::trace("Couldn't send failure message!");
return State::Close;
}
return State::Error;
}
template <typename TSession>
State HandleRunV1(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct2;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!",
session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query, params;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
}
if (!session.decoder_.ReadValue(&params, Value::Type::Map)) {
spdlog::trace("Couldn't read parameters!");
return State::Close;
}
return details::HandleRun(session, state, query, params);
}
template <typename TSession>
State HandleRunV4(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct3;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query, params, extra;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
}
if (!session.decoder_.ReadValue(&params, Value::Type::Map)) {
spdlog::trace("Couldn't read parameters!");
return State::Close;
}
// Even though this part seems unnecessary it is needed to move the buffer
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra field!");
}
return details::HandleRun(session, state, query, params);
}
template <typename TSession>
State HandlePullV1(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV1<true>(session, state, marker);
}
template <typename TSession>
State HandlePullV4(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV4<true>(session, state, marker);
}
template <typename TSession>
State HandleDiscardV1(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV1<false>(session, state, marker);
}
template <typename TSession>
State HandleDiscardV4(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV4<false>(session, state, marker);
}
template <typename TSession>
State HandleReset(TSession &session, const Marker marker) {
// IMPORTANT: This implementation of the Bolt RESET command isn't fully
// compliant to the protocol definition. In the protocol it is defined
// that this command should immediately stop any running commands and
// reset the session to a clean state. That means that we should always
// make a look-ahead for the RESET command before processing anything.
// Our implementation, for now, does everything in a blocking fashion
// so we cannot simply "kill" a transaction while it is running. So
// now this command only resets the session to a clean state. It
// does not IGNORE running and pending commands as it should.
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return State::Close;
}
// Clear all pending data and send a success message.
session.encoder_buffer_.Clear();
if (!session.encoder_.MessageSuccess()) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
session.Abort();
return State::Idle;
}
template <typename TSession>
State HandleBegin(TSession &session, const State state, const Marker marker) {
if (marker != Marker::TinyStruct1) {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
Value extra;
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra fields!");
return State::Close;
}
if (state != State::Idle) {
spdlog::trace("Unexpected BEGIN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
try {
session.BeginTransaction();
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
return State::Idle;
}
template <typename TSession>
State HandleCommit(TSession &session, const State state, const Marker marker) {
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Idle) {
spdlog::trace("Unexpected COMMIT command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
try {
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
session.CommitTransaction();
return State::Idle;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <typename TSession>
State HandleRollback(TSession &session, const State state, const Marker marker) {
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
if (state != State::Idle) {
spdlog::trace("Unexpected ROLLBACK command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
try {
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
session.RollbackTransaction();
return State::Idle;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <typename TSession>
State HandleNoop(const State state) {
spdlog::trace("Received NOOP message");
return state;
}
template <typename TSession>
State HandleGoodbye() {
throw SessionClosedException("Closing connection.");
}
template <typename TSession>
State HandleRoute(TSession &session) {
// Route message is not implemented since it is neo4j specific, therefore we
// will receive it an inform user that there is no implementation.
session.encoder_buffer_.Clear();
bool fail_sent =
session.encoder_.MessageFailure({{"code", 66}, {"message", "Route message not supported in Memgraph!"}});
if (!fail_sent) {
spdlog::trace("Couldn't send failure message!");
return State::Close;
}
return State::Error;
}
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,6 +12,9 @@
#pragma once
#include <fmt/format.h>
#include <algorithm>
#include <cstdint>
#include <iterator>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/constants.hpp"
@@ -19,7 +22,36 @@
#include "utils/likely.hpp"
#include "utils/logging.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
inline bool CopyProtocolInformationIfSupported(uint16_t version, uint8_t *protocol) {
const auto *supported_version = std::find(std::begin(kSupportedVersions), std::end(kSupportedVersions), version);
if (supported_version != std::end(kSupportedVersions)) {
std::memcpy(protocol, &version, sizeof(version));
return true;
}
return false;
}
inline bool CopyProtocolInformationIfSupportedWithOffset(auto data_position, uint8_t *protocol) {
struct bolt_range_version {
uint8_t offset;
uint8_t minor;
uint8_t major;
} bolt_range_version;
std::memcpy(&bolt_range_version, data_position, sizeof(bolt_range_version));
if (bolt_range_version.major == 0 || bolt_range_version.minor == 0) return false;
bolt_range_version.offset = std::min(bolt_range_version.offset, bolt_range_version.minor);
for (uint8_t i{0U}; i <= bolt_range_version.offset; i++) {
uint8_t current_minor = bolt_range_version.minor - i;
if (CopyProtocolInformationIfSupported(static_cast<uint16_t>((bolt_range_version.major << 8U) + current_minor),
protocol)) {
return true;
}
}
return false;
}
/**
* Handshake state run function
@@ -29,7 +61,7 @@ namespace communication::bolt {
template <typename TSession>
State StateHandshakeRun(TSession &session) {
auto precmp = std::memcmp(session.input_stream_.data(), kPreamble, sizeof(kPreamble));
if (UNLIKELY(precmp != 0)) {
if (precmp != 0) [[unlikely]] {
spdlog::trace("Received a wrong preamble!");
return State::Close;
}
@@ -37,22 +69,22 @@ State StateHandshakeRun(TSession &session) {
DMG_ASSERT(session.input_stream_.size() >= kHandshakeSize, "Wrong size of the handshake data!");
auto dataPosition = session.input_stream_.data() + sizeof(kPreamble);
uint8_t protocol[4] = {0x00};
for (int i = 0; i < 4 && !protocol[3]; ++i) {
dataPosition += 2; // version is defined only by the last 2 bytes
uint16_t version = 0;
for (int i = 0; i < 4 && !protocol[3]; ++i) {
// If there is an offset defined (e.g. 0x00 0x03 0x03 0x04) the second byte
// That would enable the client to pick between 4.0 and 4.3 versions
// as per changes in handshake bolt protocol in v4.3
if (CopyProtocolInformationIfSupportedWithOffset(dataPosition + 1, protocol + 2)) break;
dataPosition += 2; // version is defined only by the last 2 bytes
uint16_t version{0};
std::memcpy(&version, dataPosition, sizeof(version));
if (!version) {
break;
}
for (const auto supportedVersion : kSupportedVersions) {
if (supportedVersion == version) {
std::memcpy(protocol + 2, &version, sizeof(version));
break;
}
if (CopyProtocolInformationIfSupported(version, protocol + 2)) {
break;
}
dataPosition += 2;
@@ -78,4 +110,4 @@ State StateHandshakeRun(TSession &session) {
return State::Init;
}
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,6 +12,7 @@
#pragma once
#include <fmt/format.h>
#include <optional>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/state.hpp"
@@ -20,12 +21,47 @@
#include "utils/likely.hpp"
#include "utils/logging.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
namespace detail {
namespace details {
template <typename TSession>
std::optional<Value> StateInitRunV1(TSession &session, const Marker marker) {
if (UNLIKELY(marker != Marker::TinyStruct2)) {
std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
// Get authentication data.
auto &data = metadata.ValueMap();
if (!data.count("scheme")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
}
std::string username;
std::string password;
if (data["scheme"].ValueString() == "basic") {
if (!data.count("principal") || !data.count("credentials")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
}
username = data["principal"].ValueString();
password = data["credentials"].ValueString();
} else if (data["scheme"].ValueString() != "none") {
spdlog::warn("Unsupported authentication scheme: {}", data["scheme"].ValueString());
return State::Close;
}
// Authenticate the user.
if (!session.Authenticate(username, password)) {
if (!session.encoder_.MessageFailure(
{{"code", "Memgraph.ClientError.Security.Unauthenticated"}, {"message", "Authentication failure"}})) {
spdlog::trace("Couldn't send failure message to the client!");
}
// Throw an exception to indicate to the network stack that the session
// should be closed and cleaned up.
throw SessionClosedException("The client is not authenticated!");
}
return std::nullopt;
}
template <typename TSession>
std::optional<Value> GetMetadataV1(TSession &session, const Marker marker) {
if (marker != Marker::TinyStruct2) [[unlikely]] {
spdlog::trace("Expected TinyStruct2 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace(
"The client sent malformed data, but we are continuing "
@@ -53,8 +89,8 @@ std::optional<Value> StateInitRunV1(TSession &session, const Marker marker) {
}
template <typename TSession>
std::optional<Value> StateInitRunV4(TSession &session, const Marker marker) {
if (UNLIKELY(marker != Marker::TinyStruct1)) {
std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace(
"The client sent malformed data, but we are continuing "
@@ -80,15 +116,79 @@ std::optional<Value> StateInitRunV4(TSession &session, const Marker marker) {
return metadata;
}
} // namespace detail
template <typename TSession>
State SendSuccessMessage(TSession &session) {
// Neo4j's Java driver 4.1.1+ requires connection_id.
// The only usage in the mentioned version is for logging purposes.
// Because it's not critical for the regular usage of the driver
// we send a hardcoded value for now.
std::map<std::string, Value> metadata{{"connection_id", "bolt-1"}};
if (auto server_name = session.GetServerNameForInit(); server_name) {
metadata.insert({"server", *server_name});
}
bool success_sent = session.encoder_.MessageSuccess(metadata);
if (!success_sent) {
spdlog::trace("Couldn't send success message to the client!");
return State::Close;
}
return State::Idle;
}
template <typename TSession>
State StateInitRunV1(TSession &session, const Marker marker, const Signature signature) {
if (signature != Signature::Init) [[unlikely]] {
spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
return State::Close;
}
auto maybeMetadata = GetMetadataV1(session, marker);
if (!maybeMetadata) {
return State::Close;
}
if (auto result = AuthenticateUser(session, *maybeMetadata)) {
return result.value();
}
return SendSuccessMessage(session);
}
template <typename TSession, int bolt_minor = 0>
State StateInitRunV4(TSession &session, Marker marker, Signature signature) {
if constexpr (bolt_minor > 0) {
if (signature == Signature::Noop) [[unlikely]] {
SPDLOG_DEBUG("Received NOOP message");
return State::Init;
}
}
if (signature != Signature::Init) [[unlikely]] {
spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
return State::Close;
}
auto maybeMetadata = GetMetadataV4(session, marker);
if (!maybeMetadata) {
return State::Close;
}
if (auto result = AuthenticateUser(session, *maybeMetadata)) {
return result.value();
}
return SendSuccessMessage(session);
}
} // namespace details
/**
* Init state run function.
* This function runs everything to initialize a Bolt session with the client.
* @param session the session that should be used for the run.
*/
template <typename Session>
State StateInitRun(Session &session) {
template <typename TSession>
State StateInitRun(TSession &session) {
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
Marker marker;
@@ -98,72 +198,18 @@ State StateInitRun(Session &session) {
return State::Close;
}
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
SPDLOG_DEBUG("Received NOOP message");
return State::Init;
}
if (UNLIKELY(signature != Signature::Init)) {
spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
return State::Close;
}
auto maybeMetadata =
session.version_.major == 1 ? detail::StateInitRunV1(session, marker) : detail::StateInitRunV4(session, marker);
if (!maybeMetadata) {
return State::Close;
}
// Get authentication data.
std::string username;
std::string password;
auto &data = maybeMetadata->ValueMap();
if (!data.count("scheme")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
}
if (data["scheme"].ValueString() == "basic") {
if (!data.count("principal") || !data.count("credentials")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
switch (session.version_.major) {
case 1: {
return details::StateInitRunV1<TSession>(session, marker, signature);
}
username = data["principal"].ValueString();
password = data["credentials"].ValueString();
} else if (data["scheme"].ValueString() != "none") {
spdlog::warn("Unsupported authentication scheme: {}", data["scheme"].ValueString());
return State::Close;
}
// Authenticate the user.
if (!session.Authenticate(username, password)) {
if (!session.encoder_.MessageFailure(
{{"code", "Memgraph.ClientError.Security.Unauthenticated"}, {"message", "Authentication failure"}})) {
spdlog::trace("Couldn't send failure message to the client!");
}
// Throw an exception to indicate to the network stack that the session
// should be closed and cleaned up.
throw SessionClosedException("The client is not authenticated!");
}
// Return success.
{
bool success_sent = false;
// Neo4j's Java driver 4.1.1+ requires connection_id.
// The only usage in the mentioned version is for logging purposes.
// Because it's not critical for the regular usage of the driver
// we send a hardcoded value for now.
std::map<std::string, Value> metadata{{"connection_id", "bolt-1"}};
if (auto server_name = session.GetServerNameForInit(); server_name) {
metadata.insert({"server", *server_name});
}
success_sent = session.encoder_.MessageSuccess(metadata);
if (!success_sent) {
spdlog::trace("Couldn't send success message to the client!");
return State::Close;
case 4: {
if (session.version_.minor > 0) {
return details::StateInitRunV4<TSession, 1>(session, marker, signature);
}
return details::StateInitRunV4<TSession>(session, marker, signature);
}
}
return State::Idle;
spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
return State::Close;
}
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,7 @@
#include "utils/algorithm.hpp"
#include "utils/string.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
#define DEF_GETTER_BY_VAL(type, value_type, field) \
value_type &Value::Value##type() { \
@@ -461,4 +461,4 @@ std::ostream &operator<<(std::ostream &os, const Value::Type type) {
return os << "duration";
}
}
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,7 @@
#include "utils/exceptions.hpp"
#include "utils/temporal.hpp"
namespace communication::bolt {
namespace memgraph::communication::bolt {
/** Forward declaration of Value class. */
class Value;
@@ -282,4 +282,4 @@ std::ostream &operator<<(std::ostream &os, const UnboundedEdge &edge);
std::ostream &operator<<(std::ostream &os, const Path &path);
std::ostream &operator<<(std::ostream &os, const Value &value);
std::ostream &operator<<(std::ostream &os, const Value::Type type);
} // namespace communication::bolt
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,7 +13,7 @@
#include "utils/logging.hpp"
namespace communication {
namespace memgraph::communication {
Buffer::Buffer() : data_(kBufferInitialSize, 0), read_end_(this), write_end_(this) {}
@@ -77,4 +77,4 @@ void Buffer::Resize(size_t len) {
void Buffer::Clear() { have_ = 0; }
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,7 +15,7 @@
#include "io/network/stream_buffer.hpp"
namespace communication {
namespace memgraph::communication {
/**
* @brief Buffer
@@ -171,4 +171,4 @@ class Buffer final {
ReadEnd read_end_;
WriteEnd write_end_;
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,7 +14,7 @@
#include "communication/helpers.hpp"
#include "utils/logging.hpp"
namespace communication {
namespace memgraph::communication {
Client::Client(ClientContext *context) : context_(context) {}
@@ -239,4 +239,4 @@ bool ClientOutputStream::Write(const uint8_t *data, size_t len, bool have_more)
}
bool ClientOutputStream::Write(const std::string &str, bool have_more) { return client_.Write(str, have_more); }
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,14 +21,14 @@
#include "io/network/endpoint.hpp"
#include "io/network/socket.hpp"
namespace communication {
namespace memgraph::communication {
/**
* This class implements a generic network Client.
* It uses blocking sockets and provides an API that can be used to receive/send
* data over the network connection.
*
* NOTE: If you use this client you **must** create `communication::SSLInit`
* NOTE: If you use this client you **must** create `memgraph::communication::SSLInit`
* from the `main` function before using the client!
*/
class Client final {
@@ -167,4 +167,4 @@ class ClientOutputStream final {
Client &client_;
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,10 +10,13 @@
// licenses/APL.txt.
#include "communication/context.hpp"
#include <boost/asio/ssl/context.hpp>
#include <boost/asio/ssl/verify_mode.hpp>
#include <boost/system/detail/error_code.hpp>
#include "utils/logging.hpp"
namespace communication {
namespace memgraph::communication {
ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) {
if (use_ssl_) {
@@ -73,80 +76,62 @@ SSL_CTX *ClientContext::context() { return ctx_; }
bool ClientContext::use_ssl() { return use_ssl_; }
ServerContext::ServerContext() : use_ssl_(false), ctx_(nullptr) {}
ServerContext::ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file,
bool verify_peer)
: use_ssl_(true),
#if OPENSSL_VERSION_NUMBER < 0x10100000L
ctx_(SSL_CTX_new(SSLv23_server_method()))
#else
ctx_(SSL_CTX_new(TLS_server_method()))
#endif
{
// TODO (mferencevic): add support for encrypted private keys
// TODO (mferencevic): add certificate revocation list (CRL)
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load server certificate from file: {}", cert_file);
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load server private key from file: {}", key_file);
bool verify_peer) {
ctx_.emplace(boost::asio::ssl::context::tls_server);
ctx_->set_default_verify_paths();
// TODO: add support for encrypted private keys
// TODO: add certificate revocation list (CRL)
boost::system::error_code ec;
ctx_->use_certificate_chain_file(cert_file, ec);
MG_ASSERT(!ec, "Couldn't load server certificate from file: {}", cert_file);
ctx_->use_private_key_file(key_file, boost::asio::ssl::context::pem, ec);
MG_ASSERT(!ec, "Couldn't load server private key from file: {}", key_file);
// Disable legacy SSL support. Other options can be seen here:
// https://www.openssl.org/docs/man1.0.2/ssl/SSL_CTX_set_options.html
SSL_CTX_set_options(ctx_, SSL_OP_NO_SSLv3);
ctx_->set_options(SSL_OP_NO_SSLv3, ec);
MG_ASSERT(!ec, "Setting options to SSL context failed!");
if (ca_file != "") {
if (!ca_file.empty()) {
// Load the certificate authority file.
MG_ASSERT(SSL_CTX_load_verify_locations(ctx_, ca_file.c_str(), nullptr) == 1,
"Couldn't load certificate authority from file: {}", ca_file);
boost::system::error_code ec;
ctx_->load_verify_file(ca_file, ec);
MG_ASSERT(!ec, "Couldn't load certificate authority from file: {}", ca_file);
if (verify_peer) {
// Add the CA to list of accepted CAs that is sent to the client.
STACK_OF(X509_NAME) *ca_names = SSL_load_client_CA_file(ca_file.c_str());
MG_ASSERT(ca_names != nullptr, "Couldn't load certificate authority from file: {}", ca_file);
// `ca_names` doesn' need to be free'd because we pass it to
// `SSL_CTX_set_client_CA_list`:
// https://mta.openssl.org/pipermail/openssl-users/2015-May/001363.html
SSL_CTX_set_client_CA_list(ctx_, ca_names);
// Enable verification of the client certificate.
SSL_CTX_set_verify(ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
// NOLINTNEXTLINE(hicpp-signed-bitwise)
ctx_->set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert, ec);
MG_ASSERT(!ec, "Setting SSL verification mode failed!");
}
}
}
ServerContext::ServerContext(ServerContext &&other) noexcept : use_ssl_(other.use_ssl_), ctx_(other.ctx_) {
other.use_ssl_ = false;
other.ctx_ = nullptr;
}
ServerContext::ServerContext(ServerContext &&other) noexcept { std::swap(ctx_, other.ctx_); }
ServerContext &ServerContext::operator=(ServerContext &&other) noexcept {
if (this == &other) return *this;
// destroy my objects
if (use_ssl_) {
SSL_CTX_free(ctx_);
}
// move other objects to self
use_ssl_ = other.use_ssl_;
ctx_ = other.ctx_;
ctx_ = std::move(other.ctx_);
// reset other objects
other.use_ssl_ = false;
other.ctx_ = nullptr;
other.ctx_.reset();
return *this;
}
ServerContext::~ServerContext() {
if (use_ssl_) {
SSL_CTX_free(ctx_);
}
ServerContext::~ServerContext() {}
SSL_CTX *ServerContext::context() {
MG_ASSERT(ctx_);
return ctx_->native_handle();
}
SSL_CTX *ServerContext::context() { return ctx_; }
boost::asio::ssl::context &ServerContext::context_clone() {
MG_ASSERT(ctx_);
return *ctx_;
}
bool ServerContext::use_ssl() { return use_ssl_; }
bool ServerContext::use_ssl() const { return ctx_.has_value(); }
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,11 +11,13 @@
#pragma once
#include <optional>
#include <string>
#include <openssl/ssl.h>
#include <boost/asio/ssl/context.hpp>
namespace communication {
namespace memgraph::communication {
/**
* This class represents a context that should be used with network clients. One
@@ -69,11 +71,7 @@ class ClientContext final {
*/
class ServerContext final {
public:
/**
* This constructor constructs a ServerContext that doesn't use SSL.
*/
ServerContext();
ServerContext() = default;
/**
* This constructor constructs a ServerContext that uses SSL. The parameters
* `key_file` and `cert_file` can't be "" because when setting up a server it
@@ -95,16 +93,15 @@ class ServerContext final {
ServerContext(ServerContext &&other) noexcept;
ServerContext &operator=(ServerContext &&other) noexcept;
// Destructor that handles ownership of the SSL object.
~ServerContext();
SSL_CTX *context();
boost::asio::ssl::context &context_clone();
bool use_ssl();
bool use_ssl() const;
private:
bool use_ssl_;
SSL_CTX *ctx_;
std::optional<boost::asio::ssl::context> ctx_;
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,7 +13,7 @@
#include "utils/exceptions.hpp"
namespace communication {
namespace memgraph::communication {
/**
* This exception is thrown to indicate to the communication stack that the
@@ -22,4 +22,4 @@ namespace communication {
class SessionClosedException : public utils::BasicException {
using utils::BasicException::BasicException;
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,7 +13,7 @@
#include "communication/helpers.hpp"
namespace communication {
namespace memgraph::communication {
const std::string SslGetLastError() {
char buff[2048];
@@ -21,4 +21,4 @@ const std::string SslGetLastError() {
ERR_error_string_n(err, buff, sizeof(buff));
return std::string(buff);
}
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,11 +13,11 @@
#include <string>
namespace communication {
namespace memgraph::communication {
/**
* This function reads and returns a string describing the last OpenSSL error.
*/
const std::string SslGetLastError();
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,7 @@
#include "utils/signals.hpp"
#include "utils/spin_lock.hpp"
namespace communication {
namespace memgraph::communication {
namespace {
// OpenSSL before 1.1 did not have a out-of-the-box multithreading support
@@ -72,4 +72,4 @@ SSLInit::SSLInit() {
}
SSLInit::~SSLInit() { Cleanup(); }
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,7 +11,7 @@
#pragma once
namespace communication {
namespace memgraph::communication {
/**
* Create this object in each `main` file that uses the Communication stack. It
@@ -36,4 +36,4 @@ struct SSLInit {
~SSLInit();
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -28,7 +28,7 @@
#include "utils/spin_lock.hpp"
#include "utils/thread.hpp"
namespace communication {
namespace memgraph::communication {
/**
* This class listens to events on an epoll object and processes them.
@@ -273,4 +273,4 @@ class Listener final {
const std::string service_name_;
const size_t workers_count_;
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -29,7 +29,7 @@
*/
class ResultStreamFaker {
public:
explicit ResultStreamFaker(storage::Storage *store) : store_(store) {}
explicit ResultStreamFaker(memgraph::storage::Storage *store) : store_(store) {}
ResultStreamFaker(const ResultStreamFaker &) = delete;
ResultStreamFaker &operator=(const ResultStreamFaker &) = delete;
@@ -38,25 +38,25 @@ class ResultStreamFaker {
void Header(const std::vector<std::string> &fields) { header_ = fields; }
void Result(const std::vector<communication::bolt::Value> &values) { results_.push_back(values); }
void Result(const std::vector<memgraph::communication::bolt::Value> &values) { results_.push_back(values); }
void Result(const std::vector<query::TypedValue> &values) {
std::vector<communication::bolt::Value> bvalues;
void Result(const std::vector<memgraph::query::TypedValue> &values) {
std::vector<memgraph::communication::bolt::Value> bvalues;
bvalues.reserve(values.size());
for (const auto &value : values) {
auto maybe_value = glue::ToBoltValue(value, *store_, storage::View::NEW);
auto maybe_value = memgraph::glue::ToBoltValue(value, *store_, memgraph::storage::View::NEW);
MG_ASSERT(maybe_value.HasValue());
bvalues.push_back(std::move(*maybe_value));
}
results_.push_back(std::move(bvalues));
}
void Summary(const std::map<std::string, communication::bolt::Value> &summary) { summary_ = summary; }
void Summary(const std::map<std::string, memgraph::communication::bolt::Value> &summary) { summary_ = summary; }
void Summary(const std::map<std::string, query::TypedValue> &summary) {
std::map<std::string, communication::bolt::Value> bsummary;
void Summary(const std::map<std::string, memgraph::query::TypedValue> &summary) {
std::map<std::string, memgraph::communication::bolt::Value> bsummary;
for (const auto &item : summary) {
auto maybe_value = glue::ToBoltValue(item.second, *store_, storage::View::NEW);
auto maybe_value = memgraph::glue::ToBoltValue(item.second, *store_, memgraph::storage::View::NEW);
MG_ASSERT(maybe_value.HasValue());
bsummary.insert({item.first, std::move(*maybe_value)});
}
@@ -119,17 +119,17 @@ class ResultStreamFaker {
// output the summary
os << "Query summary: {";
utils::PrintIterable(os, results.GetSummary(), ", ",
[&](auto &stream, const auto &kv) { stream << kv.first << ": " << kv.second; });
memgraph::utils::PrintIterable(os, results.GetSummary(), ", ",
[&](auto &stream, const auto &kv) { stream << kv.first << ": " << kv.second; });
os << "}" << std::endl;
return os;
}
private:
storage::Storage *store_;
memgraph::storage::Storage *store_;
// the data that the record stream can accept
std::vector<std::string> header_;
std::vector<std::vector<communication::bolt::Value>> results_;
std::map<std::string, communication::bolt::Value> summary_;
std::vector<std::vector<memgraph::communication::bolt::Value>> results_;
std::map<std::string, memgraph::communication::bolt::Value> summary_;
};

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -27,7 +27,7 @@
#include "utils/message.hpp"
#include "utils/thread.hpp"
namespace communication {
namespace memgraph::communication {
/**
* Communication server.
@@ -37,10 +37,10 @@ namespace communication {
* that has `num_workers` threads. It is started automatically on constructor,
* and stopped at destructor.
*
* Current Server achitecture:
* Current Server architecture:
* incoming connection -> server -> listener -> session
*
* NOTE: If you use this server you **must** create `communication::SSLInit`
* NOTE: If you use this server you **must** create communication::SSLInit`
* from the `main` function before using the server!
*
* @tparam TSession the server can handle different Sessions, each session
@@ -89,7 +89,8 @@ class Server final {
alive_.store(true);
if (!socket_.Bind(endpoint_)) {
spdlog::error(utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint_, "https://memgr.ph/socket"));
spdlog::error(
utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint_, "https://memgr.ph/socket"));
alive_.store(false);
return false;
}
@@ -102,9 +103,8 @@ class Server final {
listener_.Start();
std::string service_name(service_name_);
thread_ = std::thread([this, service_name]() {
utils::ThreadSetName(fmt::format("{} server", service_name));
thread_ = std::thread([this]() {
utils::ThreadSetName(fmt::format("{} server", service_name_));
spdlog::info("{} server is fully armed and operational", service_name_);
spdlog::info("{} listening on {}", service_name_, socket_.endpoint());
@@ -161,4 +161,4 @@ class Server final {
const std::string service_name_;
};
} // namespace communication
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,7 +33,7 @@
#include "utils/on_scope_exit.hpp"
#include "utils/spin_lock.hpp"
namespace communication {
namespace memgraph::communication {
/**
* This is used to provide input to user sessions. All sessions used with the
@@ -327,4 +327,4 @@ class Session final {
SSL *ssl_{nullptr};
BIO *bio_{nullptr};
}; // namespace communication
} // namespace communication
} // namespace memgraph::communication

Some files were not shown because too many files have changed in this diff Show More