Compare commits
284 Commits
v2.2.1
...
MG-determi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18edf17496 | ||
|
|
eeb7ab6272 | ||
|
|
5c0e41ed44 | ||
|
|
6df2db0d19 | ||
|
|
968584a8fc | ||
|
|
33add3ecd0 | ||
|
|
91b5092c71 | ||
|
|
c0f576c187 | ||
|
|
51fb4a6e7a | ||
|
|
cba183898a | ||
|
|
5f5fe7eb29 | ||
|
|
efa4378fb4 | ||
|
|
3c07a5dc04 | ||
|
|
e442963466 | ||
|
|
32fe4d94d9 | ||
|
|
94b66a4e81 | ||
|
|
61b9457718 | ||
|
|
159b30ba5f | ||
|
|
b67e5b9a6c | ||
|
|
79756ae6fb | ||
|
|
a54bcb9819 | ||
|
|
c16f948de9 | ||
|
|
2a7ed1ad82 | ||
|
|
5201db46d2 | ||
|
|
91550128a5 | ||
|
|
d5966101ad | ||
|
|
88487e2513 | ||
|
|
b10b1eb239 | ||
|
|
dca94f42bb | ||
|
|
39b40ecf00 | ||
|
|
baacc52a65 | ||
|
|
5273d319e2 | ||
|
|
23838b50ea | ||
|
|
d85fb94bc7 | ||
|
|
4d2036249e | ||
|
|
d23643d4ff | ||
|
|
e52ce1e363 | ||
|
|
1abfe28806 | ||
|
|
d17970f6d9 | ||
|
|
9e72c7cb54 | ||
|
|
528e30a9be | ||
|
|
bb7c7f7627 | ||
|
|
bab5e1386a | ||
|
|
3d954e7abc | ||
|
|
fa5c9a2568 | ||
|
|
c745f8c877 | ||
|
|
486231b1b9 | ||
|
|
43ad5855c4 | ||
|
|
24864ff7d2 | ||
|
|
8598f6edf4 | ||
|
|
e41073bc2c | ||
|
|
14e3e72565 | ||
|
|
1cee7ecb8a | ||
|
|
690a389563 | ||
|
|
9203616283 | ||
|
|
d198819bf1 | ||
|
|
3a8f01af79 | ||
|
|
7e6ec8bb26 | ||
|
|
6138d76690 | ||
|
|
25fdb1a1f0 | ||
|
|
2de1d6c359 | ||
|
|
0364311dd0 | ||
|
|
9235515dab | ||
|
|
b83fb287ad | ||
|
|
70dc19dfdb | ||
|
|
b685a21171 | ||
|
|
4db83b8159 | ||
|
|
dd8dd4f6c4 | ||
|
|
fa1ddfea12 | ||
|
|
a815ec9617 | ||
|
|
78528bd609 | ||
|
|
6239f4fc3e | ||
|
|
c5eb3ff2c0 | ||
|
|
9f71ce0f78 | ||
|
|
589dd36bf2 | ||
|
|
57e7169203 | ||
|
|
bb3b053375 | ||
|
|
a96f489756 | ||
|
|
e909e7d2d8 | ||
|
|
2d91a7fd1e | ||
|
|
1d18f1197f | ||
|
|
edeebf46ec | ||
|
|
1148fe9aad | ||
|
|
c92c795b1a | ||
|
|
0fee412f92 | ||
|
|
7596e85358 | ||
|
|
014836ca3c | ||
|
|
68654b5a19 | ||
|
|
4d96bf5006 | ||
|
|
9bb70eb0b9 | ||
|
|
7076788dd5 | ||
|
|
ee4be9aa5b | ||
|
|
96a2ad4b63 | ||
|
|
84509fa477 | ||
|
|
19b5fe3caf | ||
|
|
74db669e23 | ||
|
|
599802033f | ||
|
|
6dd57426f8 | ||
|
|
3c9f0c48e9 | ||
|
|
9a1258a708 | ||
|
|
c55ca836a4 | ||
|
|
5674ef4016 | ||
|
|
77c2afc9e7 | ||
|
|
27292dd921 | ||
|
|
e4f1fd8647 | ||
|
|
4608af9d00 | ||
|
|
50e72a7c28 | ||
|
|
17a104677d | ||
|
|
a6add80fc9 | ||
|
|
5d3eaf6a55 | ||
|
|
a13f260236 | ||
|
|
1f778ba5f3 | ||
|
|
787987168c | ||
|
|
7b4b1ba8ed | ||
|
|
bb1e8aa164 | ||
|
|
27a1311966 | ||
|
|
30ff6487f2 | ||
|
|
119da2d7a7 | ||
|
|
fa972813d2 | ||
|
|
951b058116 | ||
|
|
d0cad6e6ba | ||
|
|
6138277972 | ||
|
|
2f77eb96ff | ||
|
|
d7bc93c55f | ||
|
|
039d5f51d3 | ||
|
|
cebe6f62fa | ||
|
|
6834ce01fe | ||
|
|
4c5cd1f847 | ||
|
|
f9e2a66961 | ||
|
|
d5700ab5ff | ||
|
|
b0aaba6765 | ||
|
|
e1f18f3733 | ||
|
|
c90b38faf0 | ||
|
|
acbf3c764c | ||
|
|
37df41796f | ||
|
|
f04e1cda4b | ||
|
|
f28ba89584 | ||
|
|
60485311c8 | ||
|
|
36a1c43851 | ||
|
|
bae8c084b1 | ||
|
|
2898120eeb | ||
|
|
d62a45752a | ||
|
|
c5ee6ffbc2 | ||
|
|
5ef08f841a | ||
|
|
59c7d81ae8 | ||
|
|
03c095e780 | ||
|
|
b542e49b3e | ||
|
|
ddb30f49ea | ||
|
|
d920d7c293 | ||
|
|
6b0168cb3d | ||
|
|
03cf264b76 | ||
|
|
9448e23dc9 | ||
|
|
eafccaea84 | ||
|
|
0c4f591b3c | ||
|
|
534e365271 | ||
|
|
281ae158ec | ||
|
|
39c9c215b1 | ||
|
|
f6e78ce6da | ||
|
|
3cf79f5bbf | ||
|
|
6a31c49432 | ||
|
|
fea92f4829 | ||
|
|
1703cd039d | ||
|
|
ca2351124b | ||
|
|
5784c0d473 | ||
|
|
8ebc704819 | ||
|
|
5939fb2b0c | ||
|
|
ee64684b0b | ||
|
|
1e4c02f8a5 | ||
|
|
8353228ba7 | ||
|
|
332afadf21 | ||
|
|
acc655f4fd | ||
|
|
12e7a261aa | ||
|
|
1c3bb969e9 | ||
|
|
e5437080c5 | ||
|
|
17090dd8ac | ||
|
|
281cebd386 | ||
|
|
f89a2bbf42 | ||
|
|
85b8ce9101 | ||
|
|
6bb40a7f49 | ||
|
|
1707ee648e | ||
|
|
07f34838bd | ||
|
|
b6f8b5e130 | ||
|
|
58243f4a26 | ||
|
|
d06132cb33 | ||
|
|
f063c1b1ad | ||
|
|
5347c06d76 | ||
|
|
4634ac484a | ||
|
|
a5dc818e19 | ||
|
|
4cb3b064c4 | ||
|
|
23171e76b6 | ||
|
|
6fd64d31f2 | ||
|
|
87111b2f89 | ||
|
|
b5c7078c7d | ||
|
|
925835b080 | ||
|
|
ce788f5f65 | ||
|
|
b8186bea2e | ||
|
|
3c4856dcb7 | ||
|
|
b4d6dc0930 | ||
|
|
817161a915 | ||
|
|
58eb2caf0f | ||
|
|
ecda71168c | ||
|
|
8e1f83acc9 | ||
|
|
f910cb770c | ||
|
|
38d0b89b04 | ||
|
|
a2a6a3855b | ||
|
|
e9f0360fb3 | ||
|
|
fe4955447e | ||
|
|
1631c20df2 | ||
|
|
947baedbe6 | ||
|
|
c6447eb48b | ||
|
|
7e84744d07 | ||
|
|
c0d03888f4 | ||
|
|
efb3c8d03d | ||
|
|
95dbc022c0 | ||
|
|
a40403e3ce | ||
|
|
14c9e68456 | ||
|
|
68b26275a3 | ||
|
|
5012824e05 | ||
|
|
a12a1ea358 | ||
|
|
2891041468 | ||
|
|
cc5ee6a496 | ||
|
|
f57f30c8cf | ||
|
|
462daf3a2b | ||
|
|
2009fefc8a | ||
|
|
eb3f96d1f6 | ||
|
|
80e0e439b7 | ||
|
|
351258ace8 | ||
|
|
74d3663821 | ||
|
|
264b233053 | ||
|
|
2ceaf59767 | ||
|
|
eb0b3141d5 | ||
|
|
c0bee760bf | ||
|
|
ff2f8031a9 | ||
|
|
094d4f282d | ||
|
|
3dd2657320 | ||
|
|
6fe474282a | ||
|
|
7fc0fb6520 | ||
|
|
3f4f66b57f | ||
|
|
063e297e1e | ||
|
|
2998f92595 | ||
|
|
86b1688192 | ||
|
|
f629de7e60 | ||
|
|
b737e53456 | ||
|
|
1bdc32ba5d | ||
|
|
10ca68bb2a | ||
|
|
bfbd8538d4 | ||
|
|
3e0e17d469 | ||
|
|
b57f91fcfc | ||
|
|
066a96c0ae | ||
|
|
1ae6b71c5f | ||
|
|
21870a0e7e | ||
|
|
cbe15e7f44 | ||
|
|
65a7ba01da | ||
|
|
7a2bbd4bb3 | ||
|
|
589e0e098b | ||
|
|
41d4185156 | ||
|
|
599c0a641f | ||
|
|
1fb49c4865 | ||
|
|
df1485aeec | ||
|
|
b2e1056389 | ||
|
|
e4c9411e63 | ||
|
|
a0bc1371dd | ||
|
|
21ad5d4328 | ||
|
|
8e3ab1ad0f | ||
|
|
cccf32e79d | ||
|
|
22bd60c613 | ||
|
|
8059a3e653 | ||
|
|
a7f4c98bea | ||
|
|
483f4d04bd | ||
|
|
3e7aef432f | ||
|
|
10ea9c773e | ||
|
|
b782271be8 | ||
|
|
a8ffcfa046 | ||
|
|
7b78665cd8 | ||
|
|
4abaf27765 | ||
|
|
ea2806bd57 | ||
|
|
c8dbaf5979 | ||
|
|
17049ada09 | ||
|
|
1b619f51b2 | ||
|
|
537855a0b2 | ||
|
|
5822b44b15 | ||
|
|
bf01c58ed9 | ||
|
|
89a5566f3f | ||
|
|
29452f8774 |
@@ -6,6 +6,7 @@ Checks: '*,
|
||||
-altera-unroll-loops,
|
||||
-android-*,
|
||||
-cert-err58-cpp,
|
||||
-cert-str34-c,
|
||||
-cppcoreguidelines-avoid-c-arrays,
|
||||
-cppcoreguidelines-avoid-goto,
|
||||
-cppcoreguidelines-avoid-magic-numbers,
|
||||
@@ -49,6 +50,7 @@ Checks: '*,
|
||||
-misc-non-private-member-variables-in-classes,
|
||||
-modernize-avoid-c-arrays,
|
||||
-modernize-concat-nested-namespaces,
|
||||
-modernize-loop-convert,
|
||||
-modernize-pass-by-value,
|
||||
-modernize-use-equals-default,
|
||||
-modernize-use-nodiscard,
|
||||
@@ -61,7 +63,9 @@ Checks: '*,
|
||||
-readability-magic-numbers,
|
||||
-readability-named-parameter,
|
||||
-misc-no-recursion,
|
||||
-concurrency-mt-unsafe'
|
||||
-concurrency-mt-unsafe,
|
||||
-bugprone-easily-swappable-parameters'
|
||||
|
||||
WarningsAsErrors: ''
|
||||
HeaderFilterRegex: 'src/.*'
|
||||
AnalyzeTemporaryDtors: false
|
||||
@@ -88,4 +92,3 @@ CheckOptions:
|
||||
- key: modernize-use-nullptr.NullMacros
|
||||
value: 'NULL'
|
||||
...
|
||||
|
||||
|
||||
@@ -24,14 +24,6 @@ for file in $modified_files; do
|
||||
|
||||
git checkout-index --prefix="$tmpdir/" -- $file
|
||||
|
||||
echo "Running clang-format..."
|
||||
$project_folder/tools/git-clang-format $tmpdir/$file
|
||||
CODE=$?
|
||||
|
||||
if [ $CODE -ne 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Do not break header checker
|
||||
echo "Running header checker..."
|
||||
$project_folder/tools/header-checker.py $tmpdir/$file $file --amend-year
|
||||
@@ -39,7 +31,6 @@ for file in $modified_files; do
|
||||
if [ $CODE -ne 0 ]; then
|
||||
FAIL=1
|
||||
fi
|
||||
|
||||
done;
|
||||
|
||||
return ${FAIL}
|
||||
|
||||
2
.github/pull_request_template.md
vendored
2
.github/pull_request_template.md
vendored
@@ -3,7 +3,9 @@
|
||||
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
|
||||
- [ ] Write E2E tests
|
||||
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
|
||||
- [ ] Provide the full content or a guide for the final git message
|
||||
|
||||
[master < Task] PR
|
||||
- [ ] Check, and update documentation if necessary
|
||||
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
|
||||
- [ ] Provide the full content or a guide for the final git message
|
||||
|
||||
2
.github/workflows/daily_benchmark.yaml
vendored
2
.github/workflows/daily_benchmark.yaml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
300
.github/workflows/diff.yaml
vendored
300
.github/workflows/diff.yaml
vendored
@@ -1,4 +1,7 @@
|
||||
name: Diff
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -23,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -61,20 +64,50 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
# This is also needed if we want do to comparison against other branches
|
||||
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
|
||||
- name: Fetch all history for all tags and branches
|
||||
run: git fetch
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Set base branch
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set base branch # if we manually dispatch or push to master
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
|
||||
|
||||
- name: Python code analysis
|
||||
run: |
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
|
||||
for file in ${CHANGED_FILES}; do
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
python3 -m black --check --diff ${file}
|
||||
python3 -m isort --check-only --diff ${file}
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
@@ -102,17 +135,27 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
- name: Set base branch
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set base branch # if we manually dispatch or push to master
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
|
||||
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -extra-arg="-DMG_CLANG_TIDY_CHECK" -path build | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
@@ -127,7 +170,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -146,52 +189,23 @@ jobs:
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run leftover CTest tests
|
||||
- name: Run simulation tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run leftover CTest tests (all except unit and benchmark tests).
|
||||
# Run simulation tests.
|
||||
cd build
|
||||
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
|
||||
ctest -R memgraph__simulation --output-on-failure -j$THREADS
|
||||
|
||||
- name: Run drivers tests
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
cd tests/integration
|
||||
for name in *; do
|
||||
if [ ! -d $name ]; then continue; fi
|
||||
pushd $name >/dev/null
|
||||
echo "Running: $name"
|
||||
if [ -x prepare.sh ]; then
|
||||
./prepare.sh
|
||||
fi
|
||||
if [ -x runner.py ]; then
|
||||
./runner.py
|
||||
elif [ -x runner.sh ]; then
|
||||
./runner.sh
|
||||
fi
|
||||
echo
|
||||
popd >/dev/null
|
||||
done
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
- name: Run single benchmark test
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run cppcheck and clang-format.
|
||||
cd tools/github
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
# Run simulation tests.
|
||||
cd tests/mgbench
|
||||
./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
@@ -203,7 +217,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -222,19 +236,6 @@ jobs:
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests/gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -244,6 +245,24 @@ jobs:
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
|
||||
- name: Run simulation tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run simulation tests.
|
||||
cd build
|
||||
ctest -R memgraph__simulation --output-on-failure -j$THREADS
|
||||
|
||||
- name: Run single benchmark test
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run simulation tests.
|
||||
cd tests/mgbench
|
||||
./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
# TODO(gitbuda): Setup mgclient and pymgclient properly.
|
||||
@@ -251,167 +270,4 @@ jobs:
|
||||
./setup.sh
|
||||
source ve3/bin/activate
|
||||
cd e2e
|
||||
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
|
||||
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration
|
||||
|
||||
- name: Run stress test (SSL)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run durability test
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 5
|
||||
|
||||
- name: Create enterprise DEB package
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
cd build
|
||||
|
||||
# create mgconsole
|
||||
# we use the -B to force the build
|
||||
make -j$THREADS -B mgconsole
|
||||
|
||||
# Create enterprise DEB package.
|
||||
mkdir output && cd output
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
#continue-on-error: true
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Get branch name (merge)
|
||||
if: github.event_name != 'pull_request'
|
||||
shell: bash
|
||||
run: echo "BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> $GITHUB_ENV
|
||||
|
||||
- name: Get branch name (pull request)
|
||||
if: github.event_name == 'pull_request'
|
||||
shell: bash
|
||||
run: echo "BRANCH_NAME=$(echo ${GITHUB_HEAD_REF} | tr / -)" >> $GITHUB_ENV
|
||||
|
||||
- name: Upload macro benchmark results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory ./distributed_queries
|
||||
|
||||
4
.github/workflows/full_clang_tidy.yaml
vendored
4
.github/workflows/full_clang_tidy.yaml
vendored
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# The results are also written to standard output in order to retain them in the logs
|
||||
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
|
||||
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -extra-arg="-DMG_CLANG_TIDY_CHECK" -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
|
||||
tee ./build/full_clang_tidy_output.txt
|
||||
|
||||
- name: Summarize clang-tidy results
|
||||
|
||||
92
.github/workflows/package_all.yaml
vendored
92
.github/workflows/package_all.yaml
vendored
@@ -6,79 +6,79 @@ on: workflow_dispatch
|
||||
|
||||
jobs:
|
||||
centos-7:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-7
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: centos-7
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
centos-8:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
centos-9:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-8
|
||||
./release/package/run.sh package centos-9
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: centos-8
|
||||
path: build/output/centos-8/memgraph*.rpm
|
||||
name: centos-9
|
||||
path: build/output/centos-9/memgraph*.rpm
|
||||
|
||||
debian-10:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-10
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-10
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
debian-11:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
docker:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -87,58 +87,92 @@ jobs:
|
||||
./run.sh package debian-11 --for-docker
|
||||
./run.sh docker
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: docker
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
ubuntu-1804:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-18.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-1804
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-20.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-2004
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
ubuntu-2204:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-2204
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
debian-11-arm:
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-arm
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
16
.github/workflows/release_centos8.yaml
vendored
16
.github/workflows/release_centos8.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -225,7 +225,7 @@ jobs:
|
||||
rpmlint memgraph*.rpm
|
||||
|
||||
- name: Save enterprise RPM package
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Enterprise RPM package"
|
||||
path: build/output/memgraph*.rpm
|
||||
@@ -262,7 +262,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
|
||||
20
.github/workflows/release_debian10.yaml
vendored
20
.github/workflows/release_debian10.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
@@ -261,7 +261,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
@@ -324,7 +324,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -349,7 +349,7 @@ jobs:
|
||||
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
|
||||
69
.github/workflows/release_docker.yaml
vendored
Normal file
69
.github/workflows/release_docker.yaml
vendored
Normal file
@@ -0,0 +1,69 @@
|
||||
name: Publish Docker images
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Memgraph binary version to publish on DockerHub."
|
||||
required: true
|
||||
force_release:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
docker_publish:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_ORGANIZATION_NAME: memgraph
|
||||
DOCKER_REPOSITORY_NAME: memgraph
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Download memgraph binary
|
||||
run: |
|
||||
cd release/docker
|
||||
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
|
||||
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
|
||||
|
||||
- name: Check if specified version is already pushed
|
||||
run: |
|
||||
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
|
||||
echo $EXISTS
|
||||
if [[ ${EXISTS} -eq 0 ]]; then
|
||||
echo 'The specified version has been already released to DockerHub.'
|
||||
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
|
||||
echo 'Forcing the release!'
|
||||
else
|
||||
echo 'Stopping the release!'
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo 'All good the specified version has not been release to DockerHub.'
|
||||
fi
|
||||
|
||||
- name: Build & push docker images
|
||||
run: |
|
||||
cd release/docker
|
||||
docker buildx build \
|
||||
--build-arg BINARY_NAME="memgraph-" \
|
||||
--build-arg EXTENSION="deb" \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
|
||||
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
|
||||
--file memgraph_deb.dockerfile \
|
||||
--push .
|
||||
16
.github/workflows/release_ubuntu2004.yaml
vendored
16
.github/workflows/release_ubuntu2004.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
@@ -261,7 +261,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v2
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
|
||||
12
.gitignore
vendored
12
.gitignore
vendored
@@ -23,6 +23,8 @@ cmake-build-*
|
||||
cmake/DownloadProject/
|
||||
dist/
|
||||
src/query/frontend/opencypher/generated/
|
||||
src/query/v2/frontend/opencypher/generated/
|
||||
src/parser/opencypher/generated
|
||||
tags
|
||||
ve/
|
||||
ve3/
|
||||
@@ -50,15 +52,25 @@ src/distributed/pull_produce_rpc_messages.hpp
|
||||
src/distributed/storage_gc_rpc_messages.hpp
|
||||
src/distributed/token_sharing_rpc_messages.hpp
|
||||
src/distributed/updates_rpc_messages.hpp
|
||||
src/query/v2/frontend/ast/ast.hpp
|
||||
src/query/frontend/ast/ast.hpp
|
||||
src/storage/v3/bindings/ast/ast.hpp
|
||||
src/query/distributed/frontend/ast/ast_serialization.hpp
|
||||
src/query/v2/distributed/frontend/ast/ast_serialization.hpp
|
||||
src/durability/distributed/state_delta.hpp
|
||||
src/durability/single_node/state_delta.hpp
|
||||
src/durability/single_node_ha/state_delta.hpp
|
||||
src/query/frontend/semantic/symbol.hpp
|
||||
src/query/v2/frontend/semantic/symbol.hpp
|
||||
src/expr/semantic/symbol.hpp
|
||||
src/query/distributed/frontend/semantic/symbol_serialization.hpp
|
||||
src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
|
||||
src/query/distributed/plan/ops.hpp
|
||||
src/query/v2/distributed/plan/ops.hpp
|
||||
src/query/plan/operator.hpp
|
||||
src/query/v2/plan/operator.hpp
|
||||
src/parser/opencypher/generated
|
||||
src/expr/semantic/symbol.hpp
|
||||
src/raft/log_entry.hpp
|
||||
src/raft/raft_rpc_messages.hpp
|
||||
src/raft/snapshot_metadata.hpp
|
||||
|
||||
20
.pre-commit-config.yaml
Normal file
20
.pre-commit-config.yaml
Normal file
@@ -0,0 +1,20 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v2.3.0
|
||||
hooks:
|
||||
- id: check-yaml
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.10.0
|
||||
hooks:
|
||||
- id: black
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.10.1
|
||||
hooks:
|
||||
- id: isort
|
||||
name: isort (python)
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v13.0.0
|
||||
hooks:
|
||||
- id: clang-format
|
||||
181
CMakeLists.txt
181
CMakeLists.txt
@@ -4,7 +4,6 @@ cmake_minimum_required(VERSION 3.8)
|
||||
|
||||
# !! IMPORTANT !! run ./project_root/init.sh before cmake command
|
||||
# to download dependencies
|
||||
|
||||
if(NOT UNIX)
|
||||
message(FATAL_ERROR "Unsupported operating system.")
|
||||
endif()
|
||||
@@ -19,6 +18,7 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM TRUE)
|
||||
find_program(CCACHE_FOUND ccache)
|
||||
option(USE_CCACHE "ccache:" ON)
|
||||
message(STATUS "CCache: ${USE_CCACHE}")
|
||||
|
||||
if(CCACHE_FOUND AND USE_CCACHE)
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
|
||||
@@ -28,7 +28,8 @@ endif(CCACHE_FOUND AND USE_CCACHE)
|
||||
# NOTE: must be choosen before use of project() or enable_language()
|
||||
find_program(CLANG_FOUND clang)
|
||||
find_program(CLANGXX_FOUND clang++)
|
||||
if (CLANG_FOUND AND CLANGXX_FOUND)
|
||||
|
||||
if(CLANG_FOUND AND CLANGXX_FOUND)
|
||||
set(CMAKE_C_COMPILER ${CLANG_FOUND})
|
||||
set(CMAKE_CXX_COMPILER ${CLANGXX_FOUND})
|
||||
else()
|
||||
@@ -36,12 +37,11 @@ else()
|
||||
endif()
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
project(memgraph)
|
||||
|
||||
# Install licenses.
|
||||
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
|
||||
DESTINATION share/doc/memgraph)
|
||||
DESTINATION share/doc/memgraph)
|
||||
|
||||
# For more information about how to release a new version of Memgraph, see
|
||||
# `release/README.md`.
|
||||
@@ -61,61 +61,65 @@ set(MEMGRAPH_OVERRIDE_VERSION "")
|
||||
set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "")
|
||||
|
||||
# Variables used to generate the versions.
|
||||
if (MG_ENTERPRISE)
|
||||
if(MG_ENTERPRISE)
|
||||
set(get_version_offering "")
|
||||
else()
|
||||
set(get_version_offering "--open-source")
|
||||
endif()
|
||||
|
||||
set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
|
||||
|
||||
# Get version that should be used in the binary.
|
||||
execute_process(
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
|
||||
message(FATAL_ERROR "Unable to get Memgraph version.")
|
||||
message(FATAL_ERROR "Unable to get Memgraph version.")
|
||||
else()
|
||||
MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
|
||||
MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
|
||||
endif()
|
||||
|
||||
# Get version that should be used in the DEB package.
|
||||
execute_process(
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant deb
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant deb
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
|
||||
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
|
||||
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
|
||||
else()
|
||||
MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
|
||||
MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
|
||||
endif()
|
||||
|
||||
# Get version that should be used in the RPM package.
|
||||
execute_process(
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant rpm
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant rpm
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
|
||||
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
|
||||
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
|
||||
else()
|
||||
MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
|
||||
MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
|
||||
endif()
|
||||
|
||||
# We want the above variables to be updated each time something is committed to
|
||||
@@ -135,22 +139,24 @@ endif()
|
||||
# unnecessary recalculations of the release version. The release version only
|
||||
# changes on every `git commit` or `git checkout`. That is why we watch the
|
||||
# following files for changes:
|
||||
# - `.git/HEAD` -> changes each time a `git checkout` is issued
|
||||
# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
|
||||
# you are on a branch) and you have to monitor the file of the specific
|
||||
# branch to detect when a `git commit` was issued
|
||||
# - `.git/HEAD` -> changes each time a `git checkout` is issued
|
||||
# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
|
||||
# you are on a branch) and you have to monitor the file of the specific
|
||||
# branch to detect when a `git commit` was issued
|
||||
# More details about the contents of the `.git` directory and the specific
|
||||
# files used can be seen here:
|
||||
# https://git-scm.com/book/en/v2/Git-Internals-Git-References
|
||||
set(git_directory "${CMAKE_SOURCE_DIR}/.git")
|
||||
if (EXISTS "${git_directory}")
|
||||
|
||||
if(EXISTS "${git_directory}")
|
||||
set_property(DIRECTORY APPEND PROPERTY
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
|
||||
file(STRINGS "${git_directory}/HEAD" git_head_data)
|
||||
if (git_head_data MATCHES "^ref: ")
|
||||
|
||||
if(git_head_data MATCHES "^ref: ")
|
||||
string(SUBSTRING "${git_head_data}" 5 -1 git_head_ref)
|
||||
set_property(DIRECTORY APPEND PROPERTY
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -159,16 +165,19 @@ endif()
|
||||
# setup CMake module path, defines path for include() and find_package()
|
||||
# https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake)
|
||||
|
||||
# custom function definitions
|
||||
include(functions)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# We want out of source builds, so that cmake generated files don't get mixed
|
||||
# with source files. This allows for easier clean up.
|
||||
disallow_in_source_build()
|
||||
add_custom_target(clean_all
|
||||
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
|
||||
COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
|
||||
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
|
||||
COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# build flags -----------------------------------------------------------------
|
||||
@@ -179,16 +188,19 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# c99-designator is disabled because of required mixture of designated and
|
||||
# non-designated initializers in Python Query Module code (`py_module.cpp`).
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
|
||||
-Werror=switch -Werror=switch-bool -Werror=return-type \
|
||||
-Werror=return-stack-address \
|
||||
-Wno-c99-designator")
|
||||
-Wno-c99-designator \
|
||||
-DBOOST_STACKTRACE_USE_ADDR2LINE \
|
||||
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
|
||||
|
||||
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
|
||||
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
|
||||
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
|
||||
# Statically link libgcc and libstdc++, the GCC allows this according to:
|
||||
# https://gcc.gnu.org/onlinedocs/gcc-10.2.0/libstdc++/manual/manual/license.html
|
||||
@@ -206,42 +218,47 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
|
||||
|
||||
SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
|
||||
|
||||
#debug flags
|
||||
# debug flags
|
||||
set(PREFERRED_DEBUGGER "gdb" CACHE STRING
|
||||
"Tunes the debug output for your preferred debugger (gdb or lldb).")
|
||||
if ("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
|
||||
elseif ("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
|
||||
"Tunes the debug output for your preferred debugger (gdb or lldb).")
|
||||
|
||||
if("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
|
||||
elseif("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
|
||||
else()
|
||||
message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
|
||||
"'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g")
|
||||
message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
|
||||
"'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g")
|
||||
endif()
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# default build type is debug
|
||||
if (NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE "Debug")
|
||||
if(NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE "Debug")
|
||||
endif()
|
||||
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
|
||||
|
||||
# setup external dependencies -------------------------------------------------
|
||||
|
||||
# threading
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# optional readline
|
||||
option(USE_READLINE "Use GNU Readline library if available (default ON). \
|
||||
Set this to OFF to prevent linking with Readline even if it is available." ON)
|
||||
if (USE_READLINE)
|
||||
|
||||
if(USE_READLINE)
|
||||
find_package(Readline)
|
||||
if (READLINE_FOUND)
|
||||
|
||||
if(READLINE_FOUND)
|
||||
add_definitions(-DHAS_READLINE)
|
||||
endif()
|
||||
endif()
|
||||
@@ -257,24 +274,27 @@ option(ASAN "Build with Address Sanitizer. To get a reasonable performance optio
|
||||
option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
|
||||
option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
|
||||
|
||||
if (TEST_COVERAGE)
|
||||
if(TEST_COVERAGE)
|
||||
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
if (NOT lower_build_type STREQUAL "debug")
|
||||
|
||||
if(NOT lower_build_type STREQUAL "debug")
|
||||
message(FATAL_ERROR "Generating test coverage unsupported in non Debug builds. Current build type is '${CMAKE_BUILD_TYPE}'")
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
|
||||
endif()
|
||||
|
||||
if (MG_ENTERPRISE)
|
||||
if(MG_ENTERPRISE)
|
||||
add_definitions(-DMG_ENTERPRISE)
|
||||
endif()
|
||||
|
||||
set(ENABLE_JEMALLOC ON)
|
||||
|
||||
if (ASAN)
|
||||
if(ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
set(ENABLE_JEMALLOC OFF)
|
||||
|
||||
# Enable Addres sanitizer and get nicer stack traces in error messages.
|
||||
# NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
|
||||
# distribution to symbolize the stack traces (note that ideally the
|
||||
@@ -283,19 +303,20 @@ if (ASAN)
|
||||
# provide it in separate ASAN_SYMBOLIZER_PATH environment variable.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
|
||||
|
||||
# To detect Stack-use-after-return bugs set run-time flag:
|
||||
# ASAN_OPTIONS=detect_stack_use_after_return=1
|
||||
# ASAN_OPTIONS=detect_stack_use_after_return=1
|
||||
# To check initialization order bugs set run-time flag:
|
||||
# ASAN_OPTIONS=check_initialization_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# dynamically initialized global from another translation unit, which is
|
||||
# not yet initialized
|
||||
# ASAN_OPTIONS=strict_init_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# any dynamically initialized global from another translation unit.
|
||||
# ASAN_OPTIONS=check_initialization_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# dynamically initialized global from another translation unit, which is
|
||||
# not yet initialized
|
||||
# ASAN_OPTIONS=strict_init_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# any dynamically initialized global from another translation unit.
|
||||
endif()
|
||||
|
||||
if (TSAN)
|
||||
if(TSAN)
|
||||
# ThreadSanitizer generally requires all code to be compiled with -fsanitize=thread.
|
||||
# If some code (e.g. dynamic libraries) is not compiled with the flag, it can
|
||||
# lead to false positive race reports, false negative race reports and/or
|
||||
@@ -308,19 +329,21 @@ if (TSAN)
|
||||
# (unwinding stack on each memory access is too expensive).
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
|
||||
|
||||
# By default ThreadSanitizer uses addr2line utility to symbolize reports.
|
||||
# llvm-symbolizer is faster, consumes less memory and produces much better
|
||||
# reports. To use it set runtime flag:
|
||||
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
|
||||
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
|
||||
# For more runtime flags see: https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
|
||||
endif()
|
||||
|
||||
if (UBSAN)
|
||||
if(UBSAN)
|
||||
# Compile with UBSAN but disable vptr check. This is disabled because it
|
||||
# requires linking with clang++ to make sure C++ specific parts of the
|
||||
# runtime library and c++ standard libraries are present.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
|
||||
|
||||
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
|
||||
# Make sure llvm-symbolizer binary is in path.
|
||||
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
|
||||
@@ -339,7 +362,7 @@ add_subdirectory(release)
|
||||
option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
|
||||
message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
|
||||
|
||||
if (MG_ENABLE_TESTING)
|
||||
if(MG_ENABLE_TESTING)
|
||||
enable_testing()
|
||||
add_subdirectory(tests)
|
||||
endif()
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
* @antaljanosbenjamin @kostasrim
|
||||
12
README.md
12
README.md
@@ -54,6 +54,18 @@ Memgraph is implemented in C/C++ and leverages an in-memory first architecture
|
||||
to ensure that you’re getting the best possible performance consistently and
|
||||
without surprises. It’s also ACID-compliant and highly available.
|
||||
|
||||
## :video_game: Memgraph Playground
|
||||
|
||||
You don't need to install anything to try out Memgraph. Check out
|
||||
our **[Memgraph Playground](https://playground.memgraph.com/)** sandboxes in
|
||||
your browser.
|
||||
|
||||
<p align="left">
|
||||
<a href="https://playground.memgraph.com/">
|
||||
<img width="450px" alt="Memgraph Playground" src="https://download.memgraph.com/asset/github/memgraph/memgraph-playground.png">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
## :floppy_disk: Download & Install
|
||||
|
||||
### Windows
|
||||
|
||||
@@ -47,6 +47,14 @@ modifications:
|
||||
value: ""
|
||||
override: false
|
||||
|
||||
- name: "bolt_cert_file"
|
||||
value: "/etc/memgraph/ssl/cert.pem"
|
||||
override: false
|
||||
|
||||
- name: "bolt_key_file"
|
||||
value: "/etc/memgraph/ssl/key.pem"
|
||||
override: false
|
||||
|
||||
- name: "storage_properties_on_edges"
|
||||
value: "true"
|
||||
override: true
|
||||
@@ -75,13 +83,9 @@ modifications:
|
||||
value: "true"
|
||||
override: true
|
||||
|
||||
- name: "query_modules_directory"
|
||||
value: "/usr/lib/memgraph/query_modules"
|
||||
override: true
|
||||
|
||||
- name: "auth_module_executable"
|
||||
value: "/usr/lib/memgraph/auth_module/example.py"
|
||||
override: false
|
||||
# - name: "query_modules_directory"
|
||||
# value: "/usr/lib/memgraph/query_modules"
|
||||
# override: true
|
||||
|
||||
- name: "memory_limit"
|
||||
value: "0"
|
||||
|
||||
@@ -5,6 +5,9 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "centos-7"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
@@ -106,7 +109,7 @@ install() {
|
||||
https://repo.ius.io/ius-release-el7.rpm
|
||||
yum update -y
|
||||
yum install -y wget python3 python3-pip
|
||||
yum install -y git224
|
||||
yum install -y git
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == libipt ]; then
|
||||
if ! yum list installed libipt >/dev/null 2>/dev/null; then
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
|
||||
libcurl-devel # for cmake
|
||||
curl # snappy
|
||||
readline-devel # for cmake and llvm
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel automake bison # for swig
|
||||
file
|
||||
openssl-devel
|
||||
gmp-devel
|
||||
gperf
|
||||
patch
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz # used for archive unpacking
|
||||
zlib # zlib library used for all builds
|
||||
expat libipt libbabeltrace xz-libs python36 # for gdb
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
openssl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconf-pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python36-devel # for query modules
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python36 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
#
|
||||
# IMPORTANT: python3-yaml does NOT exist on CentOS
|
||||
# Install it manually using `pip3 install PyYAML`
|
||||
#
|
||||
PyYAML # Package name here does not correspond to the yum package!
|
||||
libcurl-devel # mg-requests
|
||||
rpm-build rpmlint # for RPM package building
|
||||
doxygen graphviz # source documentation generators
|
||||
which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == "PyYAML" ]; then
|
||||
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
done
|
||||
if [ "$missing" != "" ]; then
|
||||
echo "MISSING PACKAGES: $missing"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
dnf install -y epel-release
|
||||
dnf install -y 'dnf-command(config-manager)'
|
||||
dnf config-manager --set-enabled powertools # Required to install texinfo.
|
||||
dnf update -y
|
||||
dnf install -y wget git python36 python3-pip
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == libipt ]; then
|
||||
if ! dnf list installed libipt >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libipt-devel ]; then
|
||||
if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
# Install GDB dependencies not present in the standard repos.
|
||||
# https://bugs.centos.org/view.php?id=17068
|
||||
# https://centos.pkgs.org
|
||||
# Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
|
||||
# within GDB yet (an assumption).
|
||||
# http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working
|
||||
if [ "$pkg" == libbabeltrace-devel ]; then
|
||||
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-3.1 ]; then
|
||||
if ! dnf list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
|
||||
rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm
|
||||
dnf update -y
|
||||
dnf install -y dotnet-sdk-3.1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == PyYAML ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user PyYAML
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
dnf install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -5,15 +5,16 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "centos-9"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc gcc-c++ make # generic build tools
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
expat-devel xz-devel python3-devel texinfo # for gdb
|
||||
libcurl-devel # for cmake
|
||||
curl # snappy
|
||||
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
|
||||
readline-devel # for cmake and llvm
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel automake bison # for swig
|
||||
@@ -21,6 +22,9 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
openssl-devel
|
||||
gmp-devel
|
||||
gperf
|
||||
diffutils
|
||||
libipt libipt-devel # intel
|
||||
patch
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
@@ -31,18 +35,19 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
openssl-devel
|
||||
perl # for openssl
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconf-pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
wget # for downloading libs
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
#
|
||||
# IMPORTANT: python3-yaml does NOT exist on CentOS
|
||||
# Install it manually using `pip3 install PyYAML`
|
||||
@@ -73,12 +78,6 @@ check() {
|
||||
if [ "$pkg" == "python3-virtualenv" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! sbcl --version &> /dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
@@ -105,13 +104,37 @@ install() {
|
||||
yum update -y
|
||||
yum install -y wget git python3 python3-pip
|
||||
for pkg in $1; do
|
||||
# Since there is no support for libipt-devel for CentOS 9 we install
|
||||
# Fedoras version of same libs, they are the same version but released
|
||||
# for different OS
|
||||
# TODO Update when libipt-devel releases for CentOS 9
|
||||
if [ "$pkg" == libipt ]; then
|
||||
if ! dnf list installed libipt >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libipt-devel ]; then
|
||||
if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libbabeltrace-devel ]; then
|
||||
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://mirror.stream.centos.org/9-stream/CRB/x86_64/os/Packages/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! sbcl --version &> /dev/null; then
|
||||
curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
|
||||
tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
|
||||
pushd /tmp/sbcl-1.4.2-arm64-linux
|
||||
INSTALL_ROOT=/usr/local sh install.sh
|
||||
popd
|
||||
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
@@ -125,9 +148,11 @@ install() {
|
||||
fi
|
||||
if [ "$pkg" == python3-virtualenv ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user virtualenv
|
||||
pip3 install virtualenv
|
||||
pip3 install virtualenvwrapper
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -5,6 +5,9 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "debian-10"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
|
||||
98
environment/os/debian-11-arm.sh
Executable file
98
environment/os/debian-11-arm.sh
Executable file
@@ -0,0 +1,98 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "debian-11"
|
||||
check_architecture "arm64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg # used for archive signature verification
|
||||
tar gzip bzip2 xz-utils unzip # used for archive unpacking
|
||||
zlib1g-dev # zlib library used for all builds
|
||||
libexpat1-dev liblzma-dev python3-dev texinfo # for gdb
|
||||
libcurl4-openssl-dev # for cmake
|
||||
libreadline-dev # for cmake and llvm
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
curl # snappy
|
||||
file # for libunwind
|
||||
libssl-dev # for libevent
|
||||
libgmp-dev
|
||||
gperf # for proxygen
|
||||
git # for fbthrift
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
zlib1g # zlib library used for all builds
|
||||
libexpat1 liblzma5 python3 # for gdb
|
||||
libcurl4 # for cmake
|
||||
file # for CPack
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi7 libxml2 # for llvm
|
||||
libssl-dev # for libevent
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
uuid-dev default-jre-headless # required by antlr
|
||||
libreadline-dev # for memgraph console
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat # tests are using nc to wait for memgraph
|
||||
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
|
||||
golang nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
cat >/etc/apt/sources.list <<EOF
|
||||
deb http://deb.debian.org/debian bullseye main
|
||||
deb-src http://deb.debian.org/debian bullseye main
|
||||
|
||||
deb http://deb.debian.org/debian-security/ bullseye-security main
|
||||
deb-src http://deb.debian.org/debian-security/ bullseye-security main
|
||||
|
||||
deb http://deb.debian.org/debian bullseye-updates main
|
||||
deb-src http://deb.debian.org/debian bullseye-updates main
|
||||
EOF
|
||||
cd "$DIR"
|
||||
apt update
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
apt install -y wget
|
||||
for pkg in $1; do
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -5,6 +5,9 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "debian-11"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
@@ -87,7 +90,7 @@ EOF
|
||||
fi
|
||||
apt install -y wget
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == dotnet-sdk-3.1 ]; then
|
||||
if [ "$pkg" == dotnet-sdk-3.1 ]; then
|
||||
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
|
||||
@@ -5,6 +5,8 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "todo-os-name"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
@@ -5,6 +5,9 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "ubuntu-18.04"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # archive download
|
||||
|
||||
@@ -5,6 +5,9 @@ set -Eeuo pipefail
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "ubuntu-20.04"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
|
||||
96
environment/os/ubuntu-22.04.sh
Executable file
96
environment/os/ubuntu-22.04.sh
Executable file
@@ -0,0 +1,96 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
check_operating_system "ubuntu-22.04"
|
||||
check_architecture "x86_64"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg # used for archive signature verification
|
||||
tar gzip bzip2 xz-utils unzip # used for archive unpacking
|
||||
zlib1g-dev # zlib library used for all builds
|
||||
libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
|
||||
libcurl4-openssl-dev # for cmake
|
||||
libreadline-dev # for cmake and llvm
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
curl # snappy
|
||||
file
|
||||
git # for thrift
|
||||
libgmp-dev # for gdb
|
||||
gperf # for proxygen
|
||||
libssl-dev
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
zlib1g # zlib library used for all builds
|
||||
libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
|
||||
libcurl4 # for cmake
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi7 libxml2 # for llvm
|
||||
libssl-dev # for libevent
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
uuid-dev default-jre-headless # required by antlr
|
||||
libreadline-dev # for memgraph console
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat # tests are using nc to wait for memgraph
|
||||
python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
|
||||
dotnet-sdk-6.0 golang nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
apt update
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
apt install -y wget
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == dotnet-sdk-6.0 ]; then
|
||||
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https dotnet-sdk-6.0
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -675,7 +675,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
|
||||
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
|
||||
SNAPPY_VERSION=1.1.9
|
||||
XZ_VERSION=5.2.5 # for LZMA
|
||||
ZLIB_VERSION=1.2.11
|
||||
ZLIB_VERSION=1.2.12
|
||||
ZSTD_VERSION=1.5.0
|
||||
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
|
||||
|
||||
@@ -1178,12 +1178,21 @@ popd
|
||||
|
||||
# create toolchain archive
|
||||
if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
|
||||
DISTRO_FULL_NAME=$DISTRO
|
||||
if [ "$for_arm" = true ]; then
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
|
||||
DISTRO_FULL_NAME=${DISTRO}
|
||||
if [[ "${DISTRO}" == centos* ]]; then
|
||||
if [[ "$for_arm" = "true" ]]; then
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
|
||||
else
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
|
||||
fi
|
||||
else
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
|
||||
if [[ "$for_arm" = "true" ]]; then
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-arm64"
|
||||
else
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
|
||||
fi
|
||||
fi
|
||||
|
||||
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
|
||||
fi
|
||||
|
||||
|
||||
@@ -5,6 +5,28 @@ operating_system() {
|
||||
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
|
||||
}
|
||||
|
||||
check_operating_system() {
|
||||
if [ "$(operating_system)" != "$1" ]; then
|
||||
echo "Not the right operating system!"
|
||||
exit 1
|
||||
else
|
||||
echo "The right operating system."
|
||||
fi
|
||||
}
|
||||
|
||||
architecture() {
|
||||
uname -m
|
||||
}
|
||||
|
||||
check_architecture() {
|
||||
if [ "$(architecture)" != "$1" ]; then
|
||||
echo "Not the right architecture!"
|
||||
exit 1
|
||||
else
|
||||
echo "The right architecture."
|
||||
fi
|
||||
}
|
||||
|
||||
check_all_yum() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
998
include/mgp.py
998
include/mgp.py
File diff suppressed because it is too large
Load Diff
26
init
26
init
@@ -24,7 +24,7 @@ function setup_virtualenv () {
|
||||
fi
|
||||
|
||||
# create new virtualenv
|
||||
virtualenv -p python3 ve3 || exit 1
|
||||
python3 -m virtualenv -p python3 ve3 || exit 1
|
||||
source ve3/bin/activate
|
||||
pip --timeout 1000 install -r requirements.txt || exit 1
|
||||
deactivate
|
||||
@@ -65,8 +65,14 @@ else
|
||||
fi
|
||||
|
||||
DISTRO=$(operating_system)
|
||||
echo "ALL BUILD PACKAGES: $($DIR/environment/os/$DISTRO.sh list MEMGRAPH_BUILD_DEPS)"
|
||||
$DIR/environment/os/$DISTRO.sh check MEMGRAPH_BUILD_DEPS
|
||||
ARCHITECTURE=$(architecture)
|
||||
if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
|
||||
OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
|
||||
else
|
||||
OS_SCRIPT=$DIR/environment/os/$DISTRO.sh
|
||||
fi
|
||||
echo "ALL BUILD PACKAGES: $($OS_SCRIPT list MEMGRAPH_BUILD_DEPS)"
|
||||
$OS_SCRIPT check MEMGRAPH_BUILD_DEPS
|
||||
echo "All packages are in-place..."
|
||||
|
||||
# create a default build directory
|
||||
@@ -129,3 +135,17 @@ for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
|
||||
ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook"
|
||||
echo "Added $hook hook"
|
||||
done;
|
||||
|
||||
# Install precommit hook
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pre_commit install
|
||||
|
||||
# Install py format tools
|
||||
echo "Install black formatter"
|
||||
python3 -m pip install black==22.10.*
|
||||
echo "Install isort"
|
||||
python3 -m pip install isort==5.10.*
|
||||
|
||||
|
||||
# Link `include/mgp.py` with `release/mgp/mgp.py`
|
||||
ln -v -f include/mgp.py release/mgp/mgp.py
|
||||
|
||||
2
libs/.gitignore
vendored
2
libs/.gitignore
vendored
@@ -4,5 +4,5 @@
|
||||
!cleanup.sh
|
||||
!CMakeLists.txt
|
||||
!__main.cpp
|
||||
!jemalloc.cmake
|
||||
!pulsar.patch
|
||||
!antlr4.10.1.patch
|
||||
|
||||
@@ -106,6 +106,7 @@ import_external_library(antlr4 STATIC
|
||||
-DWITH_LIBCXX=OFF # because of debian bug
|
||||
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
|
||||
-DCMAKE_CXX_STANDARD=20
|
||||
-DANTLR_BUILD_CPP_TESTS=OFF
|
||||
BUILD_COMMAND $(MAKE) antlr4_static
|
||||
INSTALL_COMMAND $(MAKE) install)
|
||||
|
||||
|
||||
13
libs/antlr4.10.1.patch
Normal file
13
libs/antlr4.10.1.patch
Normal file
@@ -0,0 +1,13 @@
|
||||
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
|
||||
index baf46cac9..2e7756de8 100644
|
||||
--- a/runtime/Cpp/runtime/CMakeLists.txt
|
||||
+++ b/runtime/Cpp/runtime/CMakeLists.txt
|
||||
@@ -134,7 +134,7 @@ set_target_properties(antlr4_static
|
||||
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
|
||||
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
|
||||
|
||||
-install(TARGETS antlr4_shared
|
||||
+install(TARGETS antlr4_shared OPTIONAL
|
||||
EXPORT antlr4-targets
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
@@ -1,43 +0,0 @@
|
||||
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
|
||||
index a8503bb..11362cf 100644
|
||||
--- a/runtime/Cpp/runtime/CMakeLists.txt
|
||||
+++ b/runtime/Cpp/runtime/CMakeLists.txt
|
||||
@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
|
||||
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
|
||||
ExternalProject_Add(
|
||||
utfcpp
|
||||
- GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
|
||||
- GIT_TAG "v3.1.1"
|
||||
+ GIT_REPOSITORY "https://github.com/nemtrif/utfcpp"
|
||||
+ GIT_TAG "v3.2.1"
|
||||
SOURCE_DIR ${UTFCPP_DIR}
|
||||
UPDATE_DISCONNECTED 1
|
||||
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
|
||||
@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
|
||||
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
|
||||
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
|
||||
|
||||
-install(TARGETS antlr4_shared
|
||||
+install(TARGETS antlr4_shared OPTIONAL
|
||||
DESTINATION lib
|
||||
EXPORT antlr4-targets)
|
||||
install(TARGETS antlr4_static
|
||||
diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
|
||||
index 468db98..65a473b 100644
|
||||
--- a/runtime/Cpp/runtime/src/support/Any.h
|
||||
+++ b/runtime/Cpp/runtime/src/support/Any.h
|
||||
@@ -122,12 +122,12 @@ private:
|
||||
}
|
||||
|
||||
private:
|
||||
- template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
|
||||
+ template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
|
||||
Base* clone() const {
|
||||
return new Derived<T>(value);
|
||||
}
|
||||
|
||||
- template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
|
||||
+ template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
|
||||
Base* clone() const {
|
||||
return nullptr;
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
|
||||
|
||||
set(JEMALLOC_SRCS
|
||||
${JEMALLOC_DIR}/src/arena.c
|
||||
${JEMALLOC_DIR}/src/background_thread.c
|
||||
${JEMALLOC_DIR}/src/base.c
|
||||
${JEMALLOC_DIR}/src/bin.c
|
||||
${JEMALLOC_DIR}/src/bitmap.c
|
||||
${JEMALLOC_DIR}/src/ckh.c
|
||||
${JEMALLOC_DIR}/src/ctl.c
|
||||
${JEMALLOC_DIR}/src/div.c
|
||||
${JEMALLOC_DIR}/src/extent.c
|
||||
${JEMALLOC_DIR}/src/extent_dss.c
|
||||
${JEMALLOC_DIR}/src/extent_mmap.c
|
||||
${JEMALLOC_DIR}/src/hash.c
|
||||
${JEMALLOC_DIR}/src/hook.c
|
||||
${JEMALLOC_DIR}/src/jemalloc.c
|
||||
${JEMALLOC_DIR}/src/large.c
|
||||
${JEMALLOC_DIR}/src/log.c
|
||||
${JEMALLOC_DIR}/src/malloc_io.c
|
||||
${JEMALLOC_DIR}/src/mutex.c
|
||||
${JEMALLOC_DIR}/src/mutex_pool.c
|
||||
${JEMALLOC_DIR}/src/nstime.c
|
||||
${JEMALLOC_DIR}/src/pages.c
|
||||
${JEMALLOC_DIR}/src/prng.c
|
||||
${JEMALLOC_DIR}/src/prof.c
|
||||
${JEMALLOC_DIR}/src/rtree.c
|
||||
${JEMALLOC_DIR}/src/sc.c
|
||||
${JEMALLOC_DIR}/src/stats.c
|
||||
${JEMALLOC_DIR}/src/sz.c
|
||||
${JEMALLOC_DIR}/src/tcache.c
|
||||
${JEMALLOC_DIR}/src/test_hooks.c
|
||||
${JEMALLOC_DIR}/src/ticker.c
|
||||
${JEMALLOC_DIR}/src/tsd.c
|
||||
${JEMALLOC_DIR}/src/witness.c
|
||||
${JEMALLOC_DIR}/src/safety_check.c
|
||||
)
|
||||
|
||||
add_library(jemalloc ${JEMALLOC_SRCS})
|
||||
target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(jemalloc PUBLIC Threads::Threads)
|
||||
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
|
||||
endif()
|
||||
|
||||
target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
|
||||
# for RTLD_NEXT
|
||||
target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
|
||||
|
||||
set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)
|
||||
@@ -105,7 +105,7 @@ repo_clone_try_double () {
|
||||
# Download from primary_urls might fail because the cache is not installed.
|
||||
declare -A primary_urls=(
|
||||
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
|
||||
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
|
||||
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.10.1-complete.jar"
|
||||
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
|
||||
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
|
||||
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
|
||||
@@ -116,7 +116,7 @@ declare -A primary_urls=(
|
||||
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
|
||||
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
|
||||
["spdlog"]="http://$local_cache_host/git/spdlog"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
|
||||
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
|
||||
["protobuf"]="http://$local_cache_host/git/protobuf.git"
|
||||
@@ -130,7 +130,7 @@ declare -A primary_urls=(
|
||||
# should fail.
|
||||
declare -A secondary_urls=(
|
||||
["antlr4-code"]="https://github.com/antlr/antlr4.git"
|
||||
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
|
||||
["antlr4-generator"]="https://www.antlr.org/download/antlr-4.10.1-complete.jar"
|
||||
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
|
||||
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
|
||||
["gbenchmark"]="https://github.com/google/benchmark.git"
|
||||
@@ -141,7 +141,7 @@ declare -A secondary_urls=(
|
||||
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
|
||||
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
|
||||
["spdlog"]="https://github.com/gabime/spdlog"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
|
||||
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
|
||||
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
|
||||
@@ -152,10 +152,10 @@ declare -A secondary_urls=(
|
||||
# antlr
|
||||
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
|
||||
|
||||
antlr4_tag="4.9.2" # v4.9.2
|
||||
antlr4_tag="4.10.1" # v4.10.1
|
||||
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
|
||||
pushd antlr4
|
||||
git apply ../antlr4.patch
|
||||
git apply ../antlr4.10.1.patch
|
||||
popd
|
||||
|
||||
# cppitertools v2.0 2019-12-23
|
||||
@@ -199,7 +199,7 @@ git apply ../rocksdb.patch
|
||||
popd
|
||||
|
||||
# mgclient
|
||||
mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08)
|
||||
mgclient_tag="v1.4.0" # (2022-06-14)
|
||||
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
|
||||
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
|
||||
3. using the Licensed Work to create a work or solution
|
||||
which competes (or might reasonably be expected to
|
||||
compete) with the Licensed Work.
|
||||
CHANGE DATE: 2026-18-02
|
||||
CHANGE DATE: 2026-27-04
|
||||
CHANGE LICENSE: Apache License, Version 2.0
|
||||
|
||||
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.
|
||||
|
||||
12
pyproject.toml
Normal file
12
pyproject.toml
Normal file
@@ -0,0 +1,12 @@
|
||||
[tool.black]
|
||||
line-length = 120
|
||||
include = '\.pyi?$'
|
||||
extend-exclude = '''
|
||||
/(
|
||||
| .git
|
||||
| .__pycache__
|
||||
| build
|
||||
| libs
|
||||
| .cache
|
||||
)/
|
||||
'''
|
||||
@@ -10,6 +10,14 @@ set(CPACK_PACKAGE_VENDOR "Memgraph Ltd.")
|
||||
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
|
||||
"High performance, in-memory, transactional graph database")
|
||||
|
||||
# Setting arhitecture extension for deb packages
|
||||
set(MG_ARCH_EXTENSION_DEB "all")
|
||||
if (${MG_ARCH} STREQUAL "x86_64")
|
||||
set(MG_ARCH_EXTENSION_DEB "amd64")
|
||||
elseif (${MG_ARCH} STREQUAL "ARM64")
|
||||
set(MG_ARCH_EXTENSION_DEB "arm64")
|
||||
endif()
|
||||
|
||||
# DEB specific
|
||||
# Instead of using "name <email>" format, we use "email (name)" to prevent
|
||||
# errors due to full stop, '.' at the end of "Ltd". (See: RFC 822)
|
||||
@@ -17,7 +25,7 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "tech@memgraph.com (Memgraph Ltd.)")
|
||||
set(CPACK_DEBIAN_PACKAGE_SECTION non-free/database)
|
||||
set(CPACK_DEBIAN_PACKAGE_HOMEPAGE https://memgraph.com)
|
||||
set(CPACK_DEBIAN_PACKAGE_VERSION "${MEMGRAPH_VERSION_DEB}")
|
||||
set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_amd64.deb")
|
||||
set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_${MG_ARCH_EXTENSION_DEB}.deb")
|
||||
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/debian/conffiles;"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/debian/copyright;"
|
||||
@@ -33,21 +41,20 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
|
||||
applications driver by real-time connected data.")
|
||||
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
|
||||
# We also depend on `python3` because we embed it in Memgraph.
|
||||
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
|
||||
|
||||
# RPM specific
|
||||
|
||||
set(MG_ARCH_EXTENSION "noarch")
|
||||
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6")
|
||||
|
||||
# Setting arhitecture extension for rpm packages
|
||||
set(MG_ARCH_EXTENSION_RPM "noarch")
|
||||
if (${MG_ARCH} STREQUAL "x86_64")
|
||||
set(MG_ARCH_EXTENSION "x86_64")
|
||||
set(MG_ARCH_EXTENSION_RPM "x86_64")
|
||||
elseif (${MG_ARCH} STREQUAL "ARM64")
|
||||
set(MG_ARCH_EXTENSION "aarch64")
|
||||
set(MG_ARCH_EXTENSION_RPM "aarch64")
|
||||
endif()
|
||||
|
||||
# RPM specific
|
||||
set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
|
||||
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
|
||||
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
|
||||
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION_RPM}.rpm")
|
||||
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
|
||||
/var /var/lib /var/log /etc/logrotate.d
|
||||
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
|
||||
@@ -60,7 +67,7 @@ It aims to deliver developers the speed, simplicity and scale required to build
|
||||
the next generation of applications driver by real-time connected data.")
|
||||
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
|
||||
# We also depend on `python3` because we embed it in Memgraph.
|
||||
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0")
|
||||
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate")
|
||||
|
||||
# All variables must be set before including.
|
||||
include(CPack)
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
FROM debian:bullseye
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG release
|
||||
ARG BINARY_NAME
|
||||
ARG EXTENSION
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
|
||||
--no-install-recommends \
|
||||
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
|
||||
--no-install-recommends \
|
||||
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
|
||||
|
||||
COPY ${release} /
|
||||
COPY "${BINARY_NAME}${TARGETARCH}.${EXTENSION}" /
|
||||
|
||||
# Install memgraph package
|
||||
RUN dpkg -i ${release}
|
||||
RUN dpkg -i "${BINARY_NAME}${TARGETARCH}.deb"
|
||||
|
||||
# Memgraph listens for Bolt Protocol on this port by default.
|
||||
EXPOSE 7687
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
FROM dokken/centos-stream-9
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG release
|
||||
|
||||
RUN yum update && yum install -y \
|
||||
openssl libcurl libseccomp python3 python3-pip \
|
||||
--nobest --allowerasing \
|
||||
&& rm -rf /tmp/* \
|
||||
&& yum clean all
|
||||
|
||||
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
|
||||
|
||||
COPY ${release} /
|
||||
|
||||
# Install memgraph package
|
||||
RUN rpm -i ${release}
|
||||
|
||||
# Memgraph listens for Bolt Protocol on this port by default.
|
||||
EXPOSE 7687
|
||||
# Snapshots and logging volumes
|
||||
VOLUME /var/log/memgraph
|
||||
VOLUME /var/lib/memgraph
|
||||
# Configuration volume
|
||||
VOLUME /etc/memgraph
|
||||
|
||||
USER memgraph
|
||||
WORKDIR /usr/lib/memgraph
|
||||
|
||||
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
|
||||
CMD [""]
|
||||
@@ -55,7 +55,10 @@ image_name="memgraph:${version}"
|
||||
image_package_name="memgraph-${version}-docker.tar.gz"
|
||||
|
||||
# Build docker image.
|
||||
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
|
||||
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
|
||||
--build-arg BINARY_NAME=${package_name} \
|
||||
--build-arg EXTENSION=${extension} \
|
||||
--build-arg TARGETARCH="" .
|
||||
docker save ${image_name} ${latest_image} | gzip > ${image_package_name}
|
||||
rm "${package_name}.${extension}"
|
||||
echo "Built Docker image at '${working_dir}/${image_package_name}'"
|
||||
|
||||
3
release/mgp/.gitignore
vendored
Normal file
3
release/mgp/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
.venv
|
||||
dist
|
||||
mgp.py
|
||||
201
release/mgp/LICENSE
Normal file
201
release/mgp/LICENSE
Normal file
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
4
release/mgp/README.md
Normal file
4
release/mgp/README.md
Normal file
@@ -0,0 +1,4 @@
|
||||
# mgp
|
||||
|
||||
PyPi package used for type hinting when creating MAGE modules. The get started
|
||||
using MAGE repository checkout the repository here: https://github.com/memgraph/mage.
|
||||
255
release/mgp/_mgp.py
Normal file
255
release/mgp/_mgp.py
Normal file
@@ -0,0 +1,255 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
class MgpIterable:
|
||||
def get() -> Any:
|
||||
pass
|
||||
|
||||
def next() -> Any:
|
||||
pass
|
||||
|
||||
|
||||
class Vertex:
|
||||
def is_valid() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
def underlying_graph_is_mutable() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
def iter_properties() -> MgpIterable: # type: ignore
|
||||
pass
|
||||
|
||||
def get_property(self, property_name: str) -> "Property": # type: ignore
|
||||
pass
|
||||
|
||||
def set_property(self, property_name: str, value: Any) -> "Property": # type: ignore
|
||||
pass
|
||||
|
||||
def get_id() -> "VertexId": # type: ignore
|
||||
pass
|
||||
|
||||
def label_at(self, index: int) -> "Label": # type: ignore
|
||||
pass
|
||||
|
||||
def labels_count() -> int: # type: ignore
|
||||
pass
|
||||
|
||||
def add_label(self, label: Any):
|
||||
pass
|
||||
|
||||
def remove_label(self, label: Any):
|
||||
pass
|
||||
|
||||
def iter_in_edges() -> MgpIterable: # type: ignore
|
||||
pass
|
||||
|
||||
def iter_out_edges() -> MgpIterable: # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
class Edge:
|
||||
def is_valid() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
def underlying_graph_is_mutable() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
def iter_properties() -> MgpIterable: # type: ignore
|
||||
pass
|
||||
|
||||
def get_property(self, property_name: str) -> "Property": # type: ignore
|
||||
pass
|
||||
|
||||
def set_property(self, property_name: str, valuse: Any) -> "Property": # type: ignore
|
||||
pass
|
||||
|
||||
def get_type_name() -> str: # type: ignore
|
||||
pass
|
||||
|
||||
def get_id() -> "EdgeId": # type: ignore
|
||||
pass
|
||||
|
||||
def from_vertex() -> Vertex: # type: ignore
|
||||
pass
|
||||
|
||||
def to_vertex() -> Vertex: # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
class Path:
|
||||
def is_valid() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def make_with_start(vertex: Vertex) -> "Path": # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
class Graph:
|
||||
def is_valid() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
class CypherType:
|
||||
pass
|
||||
|
||||
|
||||
class Message:
|
||||
def is_valid() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
def source_type() -> str: # type: ignore
|
||||
pass
|
||||
|
||||
def topic_name() -> str: # type: ignore
|
||||
pass
|
||||
|
||||
def key() -> bytes: # type: ignore
|
||||
pass
|
||||
|
||||
def timestamp() -> int: # type: ignore
|
||||
pass
|
||||
|
||||
def offset() -> int: # type: ignore
|
||||
pass
|
||||
|
||||
def payload() -> bytes: # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
class Messages:
|
||||
def is_valid() -> bool: # type: ignore
|
||||
pass
|
||||
|
||||
def message_at(self, id: int) -> Message: # type: ignore
|
||||
pass
|
||||
|
||||
def total_messages() -> int: # type: ignore
|
||||
pass
|
||||
|
||||
|
||||
class UnknownError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnableToAllocateError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InsufficientBufferError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class OutOfRangeError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class LogicErrorError(Exception):
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class DeletedObjectError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidArgumentError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class KeyAlreadyExistsError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ImmutableObjectError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ValueConversionError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class SerializationError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def type_nullable(elem: Any):
|
||||
pass
|
||||
|
||||
|
||||
def type_list(elem: Any):
|
||||
pass
|
||||
|
||||
|
||||
def type_bool():
|
||||
pass
|
||||
|
||||
|
||||
def type_string():
|
||||
pass
|
||||
|
||||
|
||||
def type_int():
|
||||
pass
|
||||
|
||||
|
||||
def type_float():
|
||||
pass
|
||||
|
||||
|
||||
def type_number():
|
||||
pass
|
||||
|
||||
|
||||
def type_map():
|
||||
pass
|
||||
|
||||
|
||||
def type_node():
|
||||
pass
|
||||
|
||||
|
||||
def type_relationship():
|
||||
pass
|
||||
|
||||
|
||||
def type_path():
|
||||
pass
|
||||
|
||||
|
||||
def type_date():
|
||||
pass
|
||||
|
||||
|
||||
def type_local_time():
|
||||
pass
|
||||
|
||||
|
||||
def type_local_date_time():
|
||||
pass
|
||||
|
||||
|
||||
def type_duration():
|
||||
pass
|
||||
|
||||
|
||||
def type_any():
|
||||
pass
|
||||
|
||||
|
||||
class _MODULE:
|
||||
@staticmethod
|
||||
def add_read_procedure(wrapper):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def add_write_procedure(wrapper):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def add_transformation(wrapper):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def add_function(wrapper):
|
||||
pass
|
||||
22
release/mgp/instructions.md
Normal file
22
release/mgp/instructions.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# How to publish new versions
|
||||
## Prerequisites
|
||||
1. Installed poetry
|
||||
```
|
||||
pip install poetry
|
||||
```
|
||||
2. Set up [API tokens](https://pypi.org/help/#apitoken)
|
||||
3. Be a collaborator on [pypi](https://pypi.org/project/mgp/)
|
||||
|
||||
## Making changes
|
||||
1. Make changes to the package
|
||||
2. Bump version in `pyproject.tml`
|
||||
3. `poetry build`
|
||||
4. `poetry publish`
|
||||
|
||||
## Why is this not automatized?
|
||||
|
||||
Because someone always has to manually bump up the version in `pyproject.toml`
|
||||
|
||||
## Why does `_mgp.py` exists?
|
||||
Because we are mocking here all the types that are created by Memgraph
|
||||
in order to fix typing errors in `mgp.py`.
|
||||
23
release/mgp/pyproject.toml
Normal file
23
release/mgp/pyproject.toml
Normal file
@@ -0,0 +1,23 @@
|
||||
[tool.poetry]
|
||||
name = "mgp"
|
||||
version = "1.0.0"
|
||||
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
|
||||
authors = [
|
||||
"MasterMedo <mislav.vuletic@gmail.com>",
|
||||
"jbajic <jure.bajic@memgraph.io>",
|
||||
"katarinasupe <katarina.supe@memgraph.io>",
|
||||
"antejavor <ante.javor@memgraph.io>",
|
||||
"antaljanosbenjamin <benjamin.antal@memgraph.io>",
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
readme = "README.md"
|
||||
include = ["mgp.py", "_mgp.py"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.7"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
@@ -7,8 +7,8 @@ RUN yum -y update \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
FROM centos:8
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
RUN dnf -y update \
|
||||
&& dnf install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
14
release/package/centos-9/Dockerfile
Normal file
14
release/package/centos-9/Dockerfile
Normal file
@@ -0,0 +1,14 @@
|
||||
FROM quay.io/centos/centos:stream9
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
RUN yum -y update \
|
||||
&& yum install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
17
release/package/debian-11-arm/Dockerfile
Normal file
17
release/package/debian-11-arm/Dockerfile
Normal file
@@ -0,0 +1,17 @@
|
||||
FROM debian:11
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -4,10 +4,10 @@ services:
|
||||
build:
|
||||
context: centos-7
|
||||
container_name: "mgbuild_centos-7"
|
||||
mgbuild_centos-8:
|
||||
mgbuild_centos-9:
|
||||
build:
|
||||
context: centos-8
|
||||
container_name: "mgbuild_centos-8"
|
||||
context: centos-9
|
||||
container_name: "mgbuild_centos-9"
|
||||
mgbuild_debian-10:
|
||||
build:
|
||||
context: debian-10
|
||||
@@ -24,3 +24,7 @@ services:
|
||||
build:
|
||||
context: ubuntu-20.04
|
||||
container_name: "mgbuild_ubuntu-20.04"
|
||||
mgbuild_ubuntu-22.04:
|
||||
build:
|
||||
context: ubuntu-22.04
|
||||
container_name: "mgbuild_ubuntu-22.04"
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
|
||||
SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm)
|
||||
PROJECT_ROOT="$SCRIPT_DIR/../.."
|
||||
TOOLCHAIN_VERSION="toolchain-v4"
|
||||
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
|
||||
@@ -67,14 +67,18 @@ make_package () {
|
||||
# environment/os/{os}.sh does not come within the toolchain package. When
|
||||
# migrating to the next version of toolchain do that, and remove the
|
||||
# TOOLCHAIN_RUN_DEPS installation from here.
|
||||
echo "Installing dependencies..."
|
||||
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
|
||||
|
||||
echo "Building targeted package..."
|
||||
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
|
||||
if [[ "$os" == "debian-11-arm" ]]; then
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
|
||||
else
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
|
||||
fi
|
||||
# ' is used instead of " because we need to run make within the allowed
|
||||
# container resources.
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
17
release/package/ubuntu-22.04/Dockerfile
Normal file
17
release/package/ubuntu-22.04/Dockerfile
Normal file
@@ -0,0 +1,17 @@
|
||||
FROM ubuntu:22.04
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -5,16 +5,22 @@ add_subdirectory(lisp)
|
||||
add_subdirectory(utils)
|
||||
add_subdirectory(requests)
|
||||
add_subdirectory(io)
|
||||
add_subdirectory(io/simulator)
|
||||
add_subdirectory(kvstore)
|
||||
add_subdirectory(telemetry)
|
||||
add_subdirectory(communication)
|
||||
add_subdirectory(memory)
|
||||
add_subdirectory(storage/v2)
|
||||
add_subdirectory(storage/v3)
|
||||
add_subdirectory(integrations)
|
||||
add_subdirectory(query)
|
||||
add_subdirectory(query/v2)
|
||||
add_subdirectory(slk)
|
||||
add_subdirectory(rpc)
|
||||
add_subdirectory(auth)
|
||||
add_subdirectory(parser)
|
||||
add_subdirectory(expr)
|
||||
add_subdirectory(coordinator)
|
||||
|
||||
if (MG_ENTERPRISE)
|
||||
add_subdirectory(audit)
|
||||
@@ -31,13 +37,13 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
|
||||
# Memgraph Single Node v2 Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
set(mg_single_node_v2_sources
|
||||
glue/communication.cpp
|
||||
memgraph.cpp
|
||||
glue/auth.cpp
|
||||
glue/v2/communication.cpp
|
||||
memgraph.cpp
|
||||
glue/v2/auth.cpp
|
||||
)
|
||||
|
||||
set(mg_single_node_v2_libs stdc++fs Threads::Threads
|
||||
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
|
||||
telemetry_lib mg-query-v2 mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-io mg-coordinator)
|
||||
if (MG_ENTERPRISE)
|
||||
# These are enterprise subsystems
|
||||
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
|
||||
@@ -120,19 +126,3 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
|
||||
# ----------------------------------------------------------------------------
|
||||
# Memgraph CSV Import Tool Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
add_executable(mg_import_csv mg_import_csv.cpp)
|
||||
target_link_libraries(mg_import_csv mg-storage-v2)
|
||||
|
||||
# Strip the executable in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET mg_import_csv POST_BUILD
|
||||
COMMAND strip -s mg_import_csv
|
||||
COMMENT "Stripping symbols and sections from mg_import_csv")
|
||||
endif()
|
||||
|
||||
install(TARGETS mg_import_csv RUNTIME DESTINATION bin)
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Memgraph CSV Import Tool Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(auth_password_permit_null, true, "Set to false to disable null passwords.");
|
||||
|
||||
constexpr std::string_view default_password_regex = ".+";
|
||||
inline constexpr std::string_view default_password_regex = ".+";
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
|
||||
"The regular expression that should be used to match the entire "
|
||||
@@ -37,7 +37,7 @@ const std::vector<Permission> kPermissionsAll = {
|
||||
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
|
||||
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
|
||||
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
|
||||
Permission::WEBSOCKET};
|
||||
Permission::WEBSOCKET, Permission::SCHEMA};
|
||||
} // namespace
|
||||
|
||||
std::string PermissionToString(Permission permission) {
|
||||
@@ -84,6 +84,8 @@ std::string PermissionToString(Permission permission) {
|
||||
return "MODULE_WRITE";
|
||||
case Permission::WEBSOCKET:
|
||||
return "WEBSOCKET";
|
||||
case Permission::SCHEMA:
|
||||
return "SCHEMA";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,8 @@ enum class Permission : uint64_t {
|
||||
STREAM = 1U << 17U,
|
||||
MODULE_READ = 1U << 18U,
|
||||
MODULE_WRITE = 1U << 19U,
|
||||
WEBSOCKET = 1U << 20U
|
||||
WEBSOCKET = 1U << 20U,
|
||||
SCHEMA = 1U << 21U
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
|
||||
19
src/common/types.hpp
Normal file
19
src/common/types.hpp
Normal file
@@ -0,0 +1,19 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace memgraph::common {
|
||||
enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
|
||||
|
||||
} // namespace memgraph::common
|
||||
@@ -15,8 +15,8 @@
|
||||
|
||||
namespace memgraph::communication::bolt {
|
||||
|
||||
static constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
|
||||
static constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
|
||||
inline constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
|
||||
inline constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
|
||||
|
||||
enum class Signature : uint8_t {
|
||||
Noop = 0x00,
|
||||
@@ -95,9 +95,9 @@ enum class Marker : uint8_t {
|
||||
Struct16 = 0xDD,
|
||||
};
|
||||
|
||||
static constexpr uint8_t MarkerString = 0, MarkerList = 1, MarkerMap = 2;
|
||||
static constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList, Marker::TinyMap};
|
||||
static constexpr Marker Marker8[3] = {Marker::String8, Marker::List8, Marker::Map8};
|
||||
static constexpr Marker Marker16[3] = {Marker::String16, Marker::List16, Marker::Map16};
|
||||
static constexpr Marker Marker32[3] = {Marker::String32, Marker::List32, Marker::Map32};
|
||||
inline constexpr uint8_t MarkerString = 0, MarkerList = 1, MarkerMap = 2;
|
||||
inline constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList, Marker::TinyMap};
|
||||
inline constexpr Marker Marker8[3] = {Marker::String8, Marker::List8, Marker::Map8};
|
||||
inline constexpr Marker Marker16[3] = {Marker::String16, Marker::List16, Marker::Map16};
|
||||
inline constexpr Marker Marker32[3] = {Marker::String32, Marker::List32, Marker::Map32};
|
||||
} // namespace memgraph::communication::bolt
|
||||
|
||||
@@ -19,17 +19,17 @@ namespace memgraph::communication::bolt {
|
||||
/**
|
||||
* Sizes related to the chunk defined in Bolt protocol.
|
||||
*/
|
||||
static constexpr size_t kChunkHeaderSize = 2;
|
||||
static constexpr size_t kChunkMaxDataSize = 65535;
|
||||
static constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
|
||||
inline constexpr size_t kChunkHeaderSize = 2;
|
||||
inline constexpr size_t kChunkMaxDataSize = 65535;
|
||||
inline constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
|
||||
|
||||
/**
|
||||
* Handshake size defined in the Bolt protocol.
|
||||
*/
|
||||
static constexpr size_t kHandshakeSize = 20;
|
||||
inline constexpr size_t kHandshakeSize = 20;
|
||||
|
||||
static constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
|
||||
inline constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
|
||||
|
||||
static constexpr int kPullAll = -1;
|
||||
static constexpr int kPullLast = -1;
|
||||
inline constexpr int kPullAll = -1;
|
||||
inline constexpr int kPullLast = -1;
|
||||
} // namespace memgraph::communication::bolt
|
||||
|
||||
@@ -78,14 +78,18 @@ bool ClientContext::use_ssl() { return use_ssl_; }
|
||||
|
||||
ServerContext::ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file,
|
||||
bool verify_peer) {
|
||||
ctx_.emplace(boost::asio::ssl::context::tls_server);
|
||||
namespace ssl = boost::asio::ssl;
|
||||
ctx_.emplace(ssl::context::tls_server);
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
ctx_->set_options(ssl::context::default_workarounds | ssl::context::no_sslv2 | ssl::context::no_sslv3 |
|
||||
ssl::context::single_dh_use);
|
||||
ctx_->set_default_verify_paths();
|
||||
// TODO: add support for encrypted private keys
|
||||
// TODO: add certificate revocation list (CRL)
|
||||
boost::system::error_code ec;
|
||||
ctx_->use_certificate_chain_file(cert_file, ec);
|
||||
MG_ASSERT(!ec, "Couldn't load server certificate from file: {}", cert_file);
|
||||
ctx_->use_private_key_file(key_file, boost::asio::ssl::context::pem, ec);
|
||||
ctx_->use_private_key_file(key_file, ssl::context::pem, ec);
|
||||
MG_ASSERT(!ec, "Couldn't load server private key from file: {}", key_file);
|
||||
|
||||
ctx_->set_options(SSL_OP_NO_SSLv3, ec);
|
||||
@@ -100,7 +104,7 @@ ServerContext::ServerContext(const std::string &key_file, const std::string &cer
|
||||
if (verify_peer) {
|
||||
// Enable verification of the client certificate.
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
ctx_->set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert, ec);
|
||||
ctx_->set_verify_mode(ssl::verify_peer | ssl::verify_fail_if_no_peer_cert, ec);
|
||||
MG_ASSERT(!ec, "Setting SSL verification mode failed!");
|
||||
}
|
||||
}
|
||||
|
||||
136
src/communication/v2/listener.hpp
Normal file
136
src/communication/v2/listener.hpp
Normal file
@@ -0,0 +1,136 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/strand.hpp>
|
||||
#include <boost/beast/core.hpp>
|
||||
#include <boost/system/detail/error_code.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/v2/pool.hpp"
|
||||
#include "communication/v2/session.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
template <class TSession, class TSessionData>
|
||||
class Listener final : public std::enable_shared_from_this<Listener<TSession, TSessionData>> {
|
||||
using tcp = boost::asio::ip::tcp;
|
||||
using SessionHandler = Session<TSession, TSessionData>;
|
||||
using std::enable_shared_from_this<Listener<TSession, TSessionData>>::shared_from_this;
|
||||
|
||||
public:
|
||||
Listener(const Listener &) = delete;
|
||||
Listener(Listener &&) = delete;
|
||||
Listener &operator=(const Listener &) = delete;
|
||||
Listener &operator=(Listener &&) = delete;
|
||||
~Listener() {}
|
||||
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Listener> Create(Args &&...args) {
|
||||
return std::shared_ptr<Listener>{new Listener(std::forward<Args>(args)...)};
|
||||
}
|
||||
|
||||
void Start() { DoAccept(); }
|
||||
|
||||
bool IsRunning() const noexcept { return alive_.load(std::memory_order_relaxed); }
|
||||
|
||||
private:
|
||||
Listener(boost::asio::io_context &io_context, TSessionData &data, ServerContext *server_context,
|
||||
tcp::endpoint &endpoint, const std::string_view service_name, const uint64_t inactivity_timeout_sec)
|
||||
: io_context_(io_context),
|
||||
data_(&data),
|
||||
server_context_(server_context),
|
||||
acceptor_(io_context_),
|
||||
endpoint_{endpoint},
|
||||
service_name_{service_name},
|
||||
inactivity_timeout_{inactivity_timeout_sec} {
|
||||
boost::system::error_code ec;
|
||||
// Open the acceptor
|
||||
acceptor_.open(endpoint.protocol(), ec);
|
||||
if (ec) {
|
||||
OnError(ec, "open");
|
||||
return;
|
||||
}
|
||||
|
||||
// Allow address reuse
|
||||
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
|
||||
if (ec) {
|
||||
OnError(ec, "set_option");
|
||||
return;
|
||||
}
|
||||
|
||||
// Bind to the server address
|
||||
acceptor_.bind(endpoint, ec);
|
||||
if (ec) {
|
||||
spdlog::error(
|
||||
utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint, "https://memgr.ph/socket"));
|
||||
OnError(ec, "bind");
|
||||
return;
|
||||
}
|
||||
|
||||
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
|
||||
if (ec) {
|
||||
OnError(ec, "listen");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void DoAccept() {
|
||||
acceptor_.async_accept(io_context_,
|
||||
[shared_this = shared_from_this()](auto ec, boost::asio::ip::tcp::socket &&socket) {
|
||||
shared_this->OnAccept(ec, std::move(socket));
|
||||
});
|
||||
}
|
||||
|
||||
void OnAccept(boost::system::error_code ec, tcp::socket socket) {
|
||||
if (ec) {
|
||||
return OnError(ec, "accept");
|
||||
}
|
||||
|
||||
auto session = SessionHandler::Create(std::move(socket), *data_, *server_context_, endpoint_, inactivity_timeout_,
|
||||
service_name_);
|
||||
session->Start();
|
||||
DoAccept();
|
||||
}
|
||||
|
||||
void OnError(const boost::system::error_code &ec, const std::string_view what) {
|
||||
spdlog::error("Listener failed on {}: {}", what, ec.message());
|
||||
alive_.store(false, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
boost::asio::io_context &io_context_;
|
||||
TSessionData *data_;
|
||||
ServerContext *server_context_;
|
||||
tcp::acceptor acceptor_;
|
||||
|
||||
tcp::endpoint endpoint_;
|
||||
std::string_view service_name_;
|
||||
std::chrono::seconds inactivity_timeout_;
|
||||
|
||||
std::atomic<bool> alive_;
|
||||
};
|
||||
} // namespace memgraph::communication::v2
|
||||
68
src/communication/v2/pool.hpp
Normal file
68
src/communication/v2/pool.hpp
Normal file
@@ -0,0 +1,68 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/asio/executor_work_guard.hpp>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
class IOContextThreadPool final {
|
||||
private:
|
||||
using IOContext = boost::asio::io_context;
|
||||
using IOContextGuard = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
|
||||
|
||||
public:
|
||||
explicit IOContextThreadPool(size_t pool_size) : guard_{io_context_.get_executor()}, pool_size_{pool_size} {
|
||||
MG_ASSERT(pool_size != 0, "Pool size must be greater then 0!");
|
||||
}
|
||||
|
||||
IOContextThreadPool(const IOContextThreadPool &) = delete;
|
||||
IOContextThreadPool &operator=(const IOContextThreadPool &) = delete;
|
||||
IOContextThreadPool(IOContextThreadPool &&) = delete;
|
||||
IOContextThreadPool &operator=(IOContextThreadPool &&) = delete;
|
||||
~IOContextThreadPool() = default;
|
||||
|
||||
void Run() {
|
||||
background_threads_.reserve(pool_size_);
|
||||
for (size_t i = 0; i < pool_size_; ++i) {
|
||||
background_threads_.emplace_back([this]() { io_context_.run(); });
|
||||
}
|
||||
running_ = true;
|
||||
}
|
||||
|
||||
void Shutdown() {
|
||||
io_context_.stop();
|
||||
running_ = false;
|
||||
}
|
||||
|
||||
void AwaitShutdown() { background_threads_.clear(); }
|
||||
|
||||
bool IsRunning() const noexcept { return running_; }
|
||||
|
||||
IOContext &GetIOContext() noexcept { return io_context_; }
|
||||
|
||||
private:
|
||||
/// The pool of io_context.
|
||||
IOContext io_context_;
|
||||
IOContextGuard guard_;
|
||||
size_t pool_size_;
|
||||
std::vector<std::jthread> background_threads_;
|
||||
bool running_{false};
|
||||
};
|
||||
} // namespace memgraph::communication::v2
|
||||
128
src/communication/v2/server.hpp
Normal file
128
src/communication/v2/server.hpp
Normal file
@@ -0,0 +1,128 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/ip/address.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/init.hpp"
|
||||
#include "communication/v2/listener.hpp"
|
||||
#include "communication/v2/pool.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/thread.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
using Socket = boost::asio::ip::tcp::socket;
|
||||
using ServerEndpoint = boost::asio::ip::tcp::endpoint;
|
||||
/**
|
||||
* Communication server.
|
||||
*
|
||||
* Listens for incoming connections on the server port and assigns them to the
|
||||
* connection listener. The listener and session are implemented using asio
|
||||
* async model. Currently the implemented model is thread per core model
|
||||
* opposed to io_context per core. The reasoning for opting for the former model
|
||||
* is the robustness to the multiple resource demanding queries that can be split
|
||||
* across multiple threads, and then a single thread would not block io_context,
|
||||
* unlike in the latter model where it is possible that thread that accepts
|
||||
* request is being blocked by demanding query.
|
||||
* All logic is contained within handlers that are being dispatched
|
||||
* on a single strand per session. The only exception is write which is
|
||||
* synchronous since the nature of the clients conenction is synchronous as
|
||||
* well.
|
||||
*
|
||||
* Current Server architecture:
|
||||
* incoming connection -> server -> listener -> session
|
||||
|
||||
*
|
||||
* @tparam TSession the server can handle different Sessions, each session
|
||||
* represents a different protocol so the same network infrastructure
|
||||
* can be used for handling different protocols
|
||||
* @tparam TSessionData the class with objects that will be forwarded to the
|
||||
* session
|
||||
*/
|
||||
template <typename TSession, typename TSessionData>
|
||||
class Server final {
|
||||
using ServerHandler = Server<TSession, TSessionData>;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Constructs and binds server to endpoint, operates on session data and
|
||||
* invokes workers_count workers
|
||||
*/
|
||||
Server(ServerEndpoint &endpoint, TSessionData &session_data, ServerContext *server_context,
|
||||
const int inactivity_timeout_sec, const std::string_view service_name,
|
||||
size_t workers_count = std::thread::hardware_concurrency())
|
||||
: endpoint_{endpoint},
|
||||
service_name_{service_name},
|
||||
context_thread_pool_{workers_count},
|
||||
listener_{Listener<TSession, TSessionData>::Create(context_thread_pool_.GetIOContext(), session_data,
|
||||
server_context, endpoint_, service_name_,
|
||||
inactivity_timeout_sec)} {}
|
||||
|
||||
~Server() { MG_ASSERT(!IsRunning(), "Server wasn't shutdown properly"); }
|
||||
|
||||
Server(const Server &) = delete;
|
||||
Server(Server &&) = delete;
|
||||
Server &operator=(const Server &) = delete;
|
||||
Server &operator=(Server &&) = delete;
|
||||
|
||||
const auto &Endpoint() const {
|
||||
MG_ASSERT(IsRunning(), "You can't get the server endpoint when it's not running!");
|
||||
return endpoint_;
|
||||
}
|
||||
|
||||
bool Start() {
|
||||
if (IsRunning()) {
|
||||
spdlog::error("The server is already running");
|
||||
return false;
|
||||
}
|
||||
listener_->Start();
|
||||
|
||||
spdlog::info("{} server is fully armed and operational", service_name_);
|
||||
spdlog::info("{} listening on {}", service_name_, endpoint_.address());
|
||||
context_thread_pool_.Run();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Shutdown() {
|
||||
context_thread_pool_.Shutdown();
|
||||
spdlog::info("{} shutting down...", service_name_);
|
||||
}
|
||||
|
||||
void AwaitShutdown() { context_thread_pool_.AwaitShutdown(); }
|
||||
|
||||
bool IsRunning() const noexcept { return context_thread_pool_.IsRunning() && listener_->IsRunning(); }
|
||||
|
||||
private:
|
||||
ServerEndpoint endpoint_;
|
||||
std::string service_name_;
|
||||
|
||||
IOContextThreadPool context_thread_pool_;
|
||||
std::shared_ptr<Listener<TSession, TSessionData>> listener_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::communication::v2
|
||||
517
src/communication/v2/session.hpp
Normal file
517
src/communication/v2/session.hpp
Normal file
@@ -0,0 +1,517 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <boost/asio/bind_executor.hpp>
|
||||
#include <boost/asio/buffer.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/read.hpp>
|
||||
#include <boost/asio/socket_base.hpp>
|
||||
#include <boost/asio/ssl/stream.hpp>
|
||||
#include <boost/asio/ssl/stream_base.hpp>
|
||||
#include <boost/asio/steady_timer.hpp>
|
||||
#include <boost/asio/strand.hpp>
|
||||
#include <boost/asio/system_context.hpp>
|
||||
#include <boost/asio/write.hpp>
|
||||
#include <boost/beast/core/tcp_stream.hpp>
|
||||
#include <boost/beast/http.hpp>
|
||||
#include <boost/beast/websocket.hpp>
|
||||
#include <boost/beast/websocket/rfc6455.hpp>
|
||||
#include <boost/system/detail/error_code.hpp>
|
||||
|
||||
#include "communication/buffer.hpp"
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
/**
|
||||
* This is used to provide input to user Sessions. All Sessions used with the
|
||||
* network stack should use this class as their input stream.
|
||||
*/
|
||||
using InputStream = communication::Buffer::ReadEnd;
|
||||
using tcp = boost::asio::ip::tcp;
|
||||
|
||||
/**
|
||||
* This is used to provide output from user Sessions. All Sessions used with the
|
||||
* network stack should use this class for their output stream.
|
||||
*/
|
||||
class OutputStream final {
|
||||
public:
|
||||
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
|
||||
: write_function_(write_function) {}
|
||||
|
||||
OutputStream(const OutputStream &) = delete;
|
||||
OutputStream(OutputStream &&) = delete;
|
||||
OutputStream &operator=(const OutputStream &) = delete;
|
||||
OutputStream &operator=(OutputStream &&) = delete;
|
||||
~OutputStream() = default;
|
||||
|
||||
bool Write(const uint8_t *data, size_t len, bool have_more = false) { return write_function_(data, len, have_more); }
|
||||
|
||||
bool Write(const std::string &str, bool have_more = false) {
|
||||
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(), have_more);
|
||||
}
|
||||
|
||||
private:
|
||||
std::function<bool(const uint8_t *, size_t, bool)> write_function_;
|
||||
};
|
||||
|
||||
/**
|
||||
* This class is used internally in the communication stack to handle all user
|
||||
* Websocket Sessions. It handles socket ownership, inactivity timeout and protocol
|
||||
* wrapping.
|
||||
*/
|
||||
template <typename TSession, typename TSessionData>
|
||||
class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TSession, TSessionData>> {
|
||||
using WebSocket = boost::beast::websocket::stream<boost::beast::tcp_stream>;
|
||||
using std::enable_shared_from_this<WebsocketSession<TSession, TSessionData>>::shared_from_this;
|
||||
|
||||
public:
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<WebsocketSession> Create(Args &&...args) {
|
||||
return std::shared_ptr<WebsocketSession>(new WebsocketSession(std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
// Start the asynchronous accept operation
|
||||
template <class Body, class Allocator>
|
||||
void DoAccept(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> req) {
|
||||
execution_active_ = true;
|
||||
// Set suggested timeout settings for the websocket
|
||||
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
|
||||
boost::asio::socket_base::keep_alive option(true);
|
||||
|
||||
// Set a decorator to change the Server of the handshake
|
||||
ws_.set_option(boost::beast::websocket::stream_base::decorator([&req](boost::beast::websocket::response_type &res) {
|
||||
res.set(boost::beast::http::field::server, std::string("Memgraph Bolt WS"));
|
||||
|
||||
// We need to do this to support WASM clients, which explicitly send this flag
|
||||
// in their upgrade request
|
||||
// Neo4j client breaks when this flag is sent
|
||||
if (const auto secondary_protocol = req.base().find(boost::beast::http::field::sec_websocket_protocol);
|
||||
secondary_protocol != res.base().end() && secondary_protocol->value() == "binary") {
|
||||
res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
|
||||
}
|
||||
}));
|
||||
ws_.binary(true);
|
||||
|
||||
// Accept the websocket handshake
|
||||
ws_.async_accept(
|
||||
req, boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnAccept, shared_from_this())));
|
||||
}
|
||||
|
||||
bool Write(const uint8_t *data, size_t len) {
|
||||
if (!IsConnected()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boost::system::error_code ec;
|
||||
ws_.write(boost::asio::buffer(data, len), ec);
|
||||
if (ec) {
|
||||
OnError(ec, "write");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
// Take ownership of the socket
|
||||
explicit WebsocketSession(tcp::socket &&socket, TSessionData &data, tcp::endpoint endpoint,
|
||||
std::string_view service_name)
|
||||
: ws_(std::move(socket)),
|
||||
strand_{boost::asio::make_strand(ws_.get_executor())},
|
||||
output_stream_([this](const uint8_t *data, size_t len, bool /*have_more*/) { return Write(data, len); }),
|
||||
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
|
||||
endpoint_{endpoint},
|
||||
remote_endpoint_{ws_.next_layer().socket().remote_endpoint()},
|
||||
service_name_{service_name} {}
|
||||
|
||||
void OnAccept(boost::beast::error_code ec) {
|
||||
if (ec) {
|
||||
return OnError(ec, "accept");
|
||||
}
|
||||
|
||||
// Read a message
|
||||
DoRead();
|
||||
}
|
||||
|
||||
void DoRead() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
// Read a message into our buffer
|
||||
auto buffer = input_buffer_.write_end()->Allocate();
|
||||
ws_.async_read_some(
|
||||
boost::asio::buffer(buffer.data, buffer.len),
|
||||
boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnRead, shared_from_this())));
|
||||
}
|
||||
|
||||
void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
|
||||
// This indicates that the WebsocketSession was closed
|
||||
if (ec == boost::beast::websocket::error::closed) {
|
||||
return;
|
||||
}
|
||||
if (ec) {
|
||||
OnError(ec, "read");
|
||||
}
|
||||
input_buffer_.write_end()->Written(bytes_transferred);
|
||||
|
||||
try {
|
||||
session_.Execute();
|
||||
DoRead();
|
||||
} catch (const SessionClosedException &e) {
|
||||
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
DoClose();
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::error(
|
||||
"Exception was thrown while processing event in {} session "
|
||||
"associated with {}:{}",
|
||||
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
|
||||
spdlog::debug("Exception message: {}", e.what());
|
||||
DoClose();
|
||||
}
|
||||
}
|
||||
|
||||
void OnError(const boost::system::error_code &ec, const std::string_view action) {
|
||||
spdlog::error("Websocket Bolt session error: {} on {}", ec.message(), action);
|
||||
|
||||
DoClose();
|
||||
}
|
||||
|
||||
void DoClose() {
|
||||
ws_.async_close(
|
||||
boost::beast::websocket::close_code::normal,
|
||||
boost::asio::bind_executor(
|
||||
strand_, [shared_this = shared_from_this()](boost::beast::error_code ec) { shared_this->OnClose(ec); }));
|
||||
}
|
||||
|
||||
void OnClose(const boost::system::error_code &ec) {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
if (ec) {
|
||||
return OnError(ec, "close");
|
||||
}
|
||||
}
|
||||
|
||||
bool IsConnected() const { return ws_.is_open() && execution_active_; }
|
||||
|
||||
WebSocket ws_;
|
||||
boost::asio::strand<WebSocket::executor_type> strand_;
|
||||
|
||||
communication::Buffer input_buffer_;
|
||||
OutputStream output_stream_;
|
||||
TSession session_;
|
||||
tcp::endpoint endpoint_;
|
||||
tcp::endpoint remote_endpoint_;
|
||||
std::string_view service_name_;
|
||||
bool execution_active_{false};
|
||||
};
|
||||
|
||||
/**
|
||||
* This class is used internally in the communication stack to handle all user
|
||||
* Sessions. It handles socket ownership, inactivity timeout and protocol
|
||||
* wrapping.
|
||||
*/
|
||||
template <typename TSession, typename TSessionData>
|
||||
class Session final : public std::enable_shared_from_this<Session<TSession, TSessionData>> {
|
||||
using TCPSocket = tcp::socket;
|
||||
using SSLSocket = boost::asio::ssl::stream<TCPSocket>;
|
||||
using std::enable_shared_from_this<Session<TSession, TSessionData>>::shared_from_this;
|
||||
|
||||
public:
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Session> Create(Args &&...args) {
|
||||
return std::shared_ptr<Session>(new Session(std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
Session(const Session &) = delete;
|
||||
Session(Session &&) = delete;
|
||||
Session &operator=(const Session &) = delete;
|
||||
Session &operator=(Session &&) = delete;
|
||||
~Session() = default;
|
||||
|
||||
bool Start() {
|
||||
if (execution_active_) {
|
||||
return false;
|
||||
}
|
||||
execution_active_ = true;
|
||||
timeout_timer_.async_wait(boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
|
||||
|
||||
if (std::holds_alternative<SSLSocket>(socket_)) {
|
||||
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoHandshake(); });
|
||||
} else {
|
||||
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Write(const uint8_t *data, size_t len, bool have_more = false) {
|
||||
if (!IsConnected()) {
|
||||
return false;
|
||||
}
|
||||
return std::visit(
|
||||
utils::Overloaded{[shared_this = shared_from_this(), data, len, have_more](TCPSocket &socket) mutable {
|
||||
boost::system::error_code ec;
|
||||
while (len > 0) {
|
||||
const auto sent = socket.send(boost::asio::buffer(data, len),
|
||||
MSG_NOSIGNAL | (have_more ? MSG_MORE : 0), ec);
|
||||
if (ec) {
|
||||
shared_this->OnError(ec);
|
||||
return false;
|
||||
}
|
||||
data += sent;
|
||||
len -= sent;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
[shared_this = shared_from_this(), data, len](SSLSocket &socket) mutable {
|
||||
boost::system::error_code ec;
|
||||
while (len > 0) {
|
||||
const auto sent = socket.write_some(boost::asio::buffer(data, len), ec);
|
||||
if (ec) {
|
||||
shared_this->OnError(ec);
|
||||
return false;
|
||||
}
|
||||
data += sent;
|
||||
len -= sent;
|
||||
}
|
||||
return true;
|
||||
}},
|
||||
socket_);
|
||||
}
|
||||
|
||||
bool IsConnected() const {
|
||||
return std::visit([this](const auto &socket) { return execution_active_ && socket.lowest_layer().is_open(); },
|
||||
socket_);
|
||||
}
|
||||
|
||||
private:
|
||||
explicit Session(tcp::socket &&socket, TSessionData &data, ServerContext &server_context, tcp::endpoint endpoint,
|
||||
const std::chrono::seconds inactivity_timeout_sec, std::string_view service_name)
|
||||
: socket_(CreateSocket(std::move(socket), server_context)),
|
||||
strand_{boost::asio::make_strand(GetExecutor())},
|
||||
output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
|
||||
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
|
||||
data_{&data},
|
||||
endpoint_{endpoint},
|
||||
remote_endpoint_{GetRemoteEndpoint()},
|
||||
service_name_{service_name},
|
||||
timeout_seconds_(inactivity_timeout_sec),
|
||||
timeout_timer_(GetExecutor()) {
|
||||
ExecuteForSocket([](auto &&socket) {
|
||||
socket.lowest_layer().set_option(tcp::no_delay(true)); // enable PSH
|
||||
socket.lowest_layer().set_option(boost::asio::socket_base::keep_alive(true)); // enable SO_KEEPALIVE
|
||||
socket.lowest_layer().non_blocking(false);
|
||||
});
|
||||
timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
|
||||
spdlog::info("Accepted a connection from {}: {}:{}", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
}
|
||||
|
||||
void DoRead() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
timeout_timer_.expires_after(timeout_seconds_);
|
||||
ExecuteForSocket([this](auto &&socket) {
|
||||
auto buffer = input_buffer_.write_end()->Allocate();
|
||||
socket.async_read_some(
|
||||
boost::asio::buffer(buffer.data, buffer.len),
|
||||
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnRead, shared_from_this())));
|
||||
});
|
||||
}
|
||||
|
||||
bool IsWebsocketUpgrade(boost::beast::http::request_parser<boost::beast::http::string_body> &parser) {
|
||||
boost::system::error_code error_code_parsing;
|
||||
parser.put(boost::asio::buffer(input_buffer_.read_end()->data(), input_buffer_.read_end()->size()),
|
||||
error_code_parsing);
|
||||
if (error_code_parsing) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return boost::beast::websocket::is_upgrade(parser.get());
|
||||
}
|
||||
|
||||
void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
|
||||
if (ec) {
|
||||
return OnError(ec);
|
||||
}
|
||||
input_buffer_.write_end()->Written(bytes_transferred);
|
||||
|
||||
// Can be a websocket connection only on the first read, since it is not
|
||||
// expected from clients to upgrade from tcp to websocket
|
||||
if (!has_received_msg_) {
|
||||
has_received_msg_ = true;
|
||||
boost::beast::http::request_parser<boost::beast::http::string_body> parser;
|
||||
|
||||
if (IsWebsocketUpgrade(parser)) {
|
||||
spdlog::info("Switching {} to websocket connection", remote_endpoint_);
|
||||
if (std::holds_alternative<TCPSocket>(socket_)) {
|
||||
auto sock = std::get<TCPSocket>(std::move(socket_));
|
||||
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), *data_, endpoint_, service_name_)
|
||||
->DoAccept(parser.release());
|
||||
execution_active_ = false;
|
||||
return;
|
||||
}
|
||||
spdlog::error("Error while upgrading connection to websocket");
|
||||
DoShutdown();
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
session_.Execute();
|
||||
DoRead();
|
||||
} catch (const SessionClosedException &e) {
|
||||
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
DoShutdown();
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::error(
|
||||
"Exception was thrown while processing event in {} session "
|
||||
"associated with {}:{}",
|
||||
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
|
||||
spdlog::debug("Exception message: {}", e.what());
|
||||
DoShutdown();
|
||||
}
|
||||
}
|
||||
|
||||
void OnError(const boost::system::error_code &ec) {
|
||||
if (ec == boost::asio::error::operation_aborted) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (ec == boost::asio::error::eof) {
|
||||
spdlog::info("Session closed by peer");
|
||||
} else {
|
||||
spdlog::error("Session error: {}", ec.message());
|
||||
}
|
||||
|
||||
DoShutdown();
|
||||
}
|
||||
|
||||
void DoShutdown() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
execution_active_ = false;
|
||||
timeout_timer_.cancel();
|
||||
ExecuteForSocket([](auto &socket) {
|
||||
boost::system::error_code ec;
|
||||
auto &lowest_layer = socket.lowest_layer();
|
||||
lowest_layer.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
|
||||
if (ec) {
|
||||
spdlog::error("Session shutdown failed: {}", ec.what());
|
||||
}
|
||||
lowest_layer.close();
|
||||
});
|
||||
}
|
||||
|
||||
void DoHandshake() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
if (auto *socket = std::get_if<SSLSocket>(&socket_); socket) {
|
||||
socket->async_handshake(
|
||||
boost::asio::ssl::stream_base::server,
|
||||
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnHandshake, shared_from_this())));
|
||||
}
|
||||
}
|
||||
|
||||
void OnHandshake(const boost::system::error_code &ec) {
|
||||
if (ec) {
|
||||
return OnError(ec);
|
||||
}
|
||||
DoRead();
|
||||
}
|
||||
|
||||
void OnClose(const boost::system::error_code &ec) {
|
||||
if (ec) {
|
||||
return OnError(ec);
|
||||
}
|
||||
}
|
||||
|
||||
void OnTimeout() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
// Check whether the deadline has passed. We compare the deadline against
|
||||
// the current time since a new asynchronous operation may have moved the
|
||||
// deadline before this actor had a chance to run.
|
||||
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
|
||||
// The deadline has passed. Stop the session. The other actors will
|
||||
// terminate as soon as possible.
|
||||
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_.count());
|
||||
DoShutdown();
|
||||
} else {
|
||||
// Put the actor back to sleep.
|
||||
timeout_timer_.async_wait(
|
||||
boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
|
||||
}
|
||||
}
|
||||
|
||||
std::variant<TCPSocket, SSLSocket> CreateSocket(tcp::socket &&socket, ServerContext &context) {
|
||||
if (context.use_ssl()) {
|
||||
ssl_context_.emplace(context.context_clone());
|
||||
return SSLSocket{std::move(socket), *ssl_context_};
|
||||
}
|
||||
|
||||
return TCPSocket{std::move(socket)};
|
||||
}
|
||||
|
||||
auto GetExecutor() {
|
||||
return std::visit(utils::Overloaded{[](auto &&socket) { return socket.get_executor(); }}, socket_);
|
||||
}
|
||||
|
||||
auto GetRemoteEndpoint() const {
|
||||
return std::visit(utils::Overloaded{[](const auto &socket) { return socket.lowest_layer().remote_endpoint(); }},
|
||||
socket_);
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
decltype(auto) ExecuteForSocket(F &&fun) {
|
||||
return std::visit(utils::Overloaded{std::forward<F>(fun)}, socket_);
|
||||
}
|
||||
|
||||
std::variant<TCPSocket, SSLSocket> socket_;
|
||||
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ssl_context_;
|
||||
boost::asio::strand<tcp::socket::executor_type> strand_;
|
||||
|
||||
communication::Buffer input_buffer_;
|
||||
OutputStream output_stream_;
|
||||
TSession session_;
|
||||
TSessionData *data_;
|
||||
tcp::endpoint endpoint_;
|
||||
tcp::endpoint remote_endpoint_;
|
||||
std::string_view service_name_;
|
||||
std::chrono::seconds timeout_seconds_;
|
||||
boost::asio::steady_timer timeout_timer_;
|
||||
bool execution_active_{false};
|
||||
bool has_received_msg_{false};
|
||||
};
|
||||
} // namespace memgraph::communication::v2
|
||||
@@ -11,8 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
|
||||
|
||||
#include <list>
|
||||
#include <memory>
|
||||
|
||||
|
||||
@@ -44,10 +44,10 @@ class QuoteEscapeFormatter : public spdlog::custom_flag_formatter {
|
||||
void format(const spdlog::details::log_msg &msg, const std::tm & /*time*/, spdlog::memory_buf_t &dest) override {
|
||||
for (const auto c : msg.payload) {
|
||||
if (c == '"') {
|
||||
constexpr std::string_view escaped_quote = "\\\"";
|
||||
static constexpr std::string_view escaped_quote = "\\\"";
|
||||
dest.append(escaped_quote.data(), escaped_quote.data() + escaped_quote.size());
|
||||
} else if (c == '\n') {
|
||||
constexpr std::string_view escaped_newline = "\\n";
|
||||
static constexpr std::string_view escaped_newline = "\\n";
|
||||
dest.append(escaped_newline.data(), escaped_newline.data() + escaped_newline.size());
|
||||
} else {
|
||||
dest.push_back(c);
|
||||
|
||||
@@ -11,8 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include <spdlog/sinks/base_sink.h>
|
||||
|
||||
@@ -11,8 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
9
src/coordinator/CMakeLists.txt
Normal file
9
src/coordinator/CMakeLists.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
set(coordinator_src_files
|
||||
coordinator.cpp
|
||||
shard_map.cpp)
|
||||
|
||||
find_package(fmt REQUIRED)
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
add_library(mg-coordinator STATIC ${coordinator_src_files})
|
||||
target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils mg-storage-v3)
|
||||
129
src/coordinator/coordinator.cpp
Normal file
129
src/coordinator/coordinator.cpp
Normal file
@@ -0,0 +1,129 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <coordinator/coordinator.hpp>
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_request) {
|
||||
spdlog::info("Coordinator handling HeartbeatRequest");
|
||||
|
||||
// add this storage engine to any under-replicated shards that it is not already a part of
|
||||
|
||||
auto initializing_rsms_for_shard_manager =
|
||||
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
|
||||
|
||||
return HeartbeatResponse{
|
||||
.shards_to_initialize = initializing_rsms_for_shard_manager,
|
||||
};
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
|
||||
HlcResponse res{};
|
||||
|
||||
auto hlc_shard_map = shard_map_.GetHlc();
|
||||
|
||||
MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
|
||||
|
||||
res.new_hlc = Hlc{
|
||||
.logical_id = ++highest_allocated_timestamp_,
|
||||
// TODO(tyler) probably pass some more context to the Coordinator here
|
||||
// so that we can use our wall clock and enforce monotonicity.
|
||||
// .coordinator_wall_clock = io_.Now(),
|
||||
};
|
||||
|
||||
// Allways return fresher shard_map for now.
|
||||
res.fresher_shard_map = std::make_optional(shard_map_);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
|
||||
AllocateEdgeIdBatchResponse res{};
|
||||
|
||||
uint64_t low = highest_allocated_edge_id_;
|
||||
|
||||
highest_allocated_edge_id_ += ahr.batch_size;
|
||||
|
||||
uint64_t high = highest_allocated_edge_id_;
|
||||
|
||||
res.low = low;
|
||||
res.high = high;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/// This splits the shard immediately beneath the provided
|
||||
/// split key, keeping the assigned peers identical for now,
|
||||
/// but letting them be gradually migrated over time.
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
|
||||
SplitShardResponse res{};
|
||||
|
||||
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
|
||||
res.success = false;
|
||||
} else {
|
||||
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
|
||||
split_shard_request.split_key);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/// This adds the provided storage engine to the standby storage engine pool,
|
||||
/// which can be used to rebalance storage over time.
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(
|
||||
RegisterStorageEngineRequest && /* register_storage_engine_request */) {
|
||||
RegisterStorageEngineResponse res{};
|
||||
// TODO
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/// This begins the process of draining the provided storage engine from all raft
|
||||
/// clusters that it might be participating in.
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(
|
||||
DeregisterStorageEngineRequest && /* register_storage_engine_request */) {
|
||||
DeregisterStorageEngineResponse res{};
|
||||
// TODO
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
|
||||
InitializeLabelResponse res{};
|
||||
|
||||
std::optional<LabelId> new_label_id = shard_map_.InitializeNewLabel(
|
||||
initialize_label_request.label_name, initialize_label_request.schema, initialize_label_request.replication_factor,
|
||||
initialize_label_request.last_shard_map_version);
|
||||
|
||||
if (new_label_id) {
|
||||
res.new_label_id = new_label_id.value();
|
||||
res.fresher_shard_map = std::nullopt;
|
||||
res.success = true;
|
||||
} else {
|
||||
res.fresher_shard_map = shard_map_;
|
||||
res.success = false;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request) {
|
||||
AllocatePropertyIdsResponse res{};
|
||||
|
||||
auto property_ids = shard_map_.AllocatePropertyIds(allocate_property_ids_request.property_names);
|
||||
|
||||
res.property_ids = property_ids;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
196
src/coordinator/coordinator.hpp
Normal file
196
src/coordinator/coordinator.hpp
Normal file
@@ -0,0 +1,196 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
|
||||
#include <coordinator/hybrid_logical_clock.hpp>
|
||||
#include <coordinator/shard_map.hpp>
|
||||
#include <io/simulator/simulator.hpp>
|
||||
#include <io/time.hpp>
|
||||
#include <io/transport.hpp>
|
||||
#include <storage/v3/id_types.hpp>
|
||||
#include <storage/v3/schemas.hpp>
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
using memgraph::io::Address;
|
||||
using memgraph::storage::v3::LabelId;
|
||||
using memgraph::storage::v3::PropertyId;
|
||||
using memgraph::storage::v3::SchemaProperty;
|
||||
using SimT = memgraph::io::simulator::SimulatorTransport;
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
|
||||
struct HlcRequest {
|
||||
Hlc last_shard_map_version;
|
||||
};
|
||||
|
||||
struct HlcResponse {
|
||||
Hlc new_hlc;
|
||||
std::optional<ShardMap> fresher_shard_map;
|
||||
};
|
||||
|
||||
struct GetShardMapRequest {
|
||||
// No state
|
||||
};
|
||||
|
||||
struct GetShardMapResponse {
|
||||
ShardMap shard_map;
|
||||
};
|
||||
|
||||
struct AllocateHlcBatchRequest {
|
||||
Hlc low;
|
||||
Hlc high;
|
||||
};
|
||||
|
||||
struct AllocateHlcBatchResponse {
|
||||
bool success;
|
||||
Hlc low;
|
||||
Hlc high;
|
||||
};
|
||||
|
||||
struct AllocateEdgeIdBatchRequest {
|
||||
size_t batch_size;
|
||||
};
|
||||
|
||||
struct AllocateEdgeIdBatchResponse {
|
||||
uint64_t low;
|
||||
uint64_t high;
|
||||
};
|
||||
|
||||
struct AllocatePropertyIdsRequest {
|
||||
std::vector<std::string> property_names;
|
||||
};
|
||||
|
||||
struct AllocatePropertyIdsResponse {
|
||||
std::map<std::string, PropertyId> property_ids;
|
||||
};
|
||||
|
||||
struct SplitShardRequest {
|
||||
Hlc previous_shard_map_version;
|
||||
LabelId label_id;
|
||||
PrimaryKey split_key;
|
||||
};
|
||||
|
||||
struct SplitShardResponse {
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct RegisterStorageEngineRequest {
|
||||
Address address;
|
||||
};
|
||||
|
||||
struct RegisterStorageEngineResponse {
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct DeregisterStorageEngineRequest {
|
||||
Address address;
|
||||
};
|
||||
|
||||
struct DeregisterStorageEngineResponse {
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct InitializeLabelRequest {
|
||||
std::string label_name;
|
||||
std::vector<SchemaProperty> schema;
|
||||
size_t replication_factor;
|
||||
Hlc last_shard_map_version;
|
||||
};
|
||||
|
||||
struct InitializeLabelResponse {
|
||||
bool success;
|
||||
LabelId new_label_id;
|
||||
std::optional<ShardMap> fresher_shard_map;
|
||||
};
|
||||
|
||||
struct HeartbeatRequest {
|
||||
Address from_storage_manager;
|
||||
std::set<boost::uuids::uuid> initialized_rsms;
|
||||
};
|
||||
|
||||
struct HeartbeatResponse {
|
||||
std::vector<ShardToInitialize> shards_to_initialize;
|
||||
};
|
||||
|
||||
using CoordinatorWriteRequests =
|
||||
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
|
||||
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
|
||||
using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
|
||||
RegisterStorageEngineResponse, DeregisterStorageEngineResponse,
|
||||
InitializeLabelResponse, AllocatePropertyIdsResponse, HeartbeatResponse>;
|
||||
|
||||
using CoordinatorReadRequests = std::variant<GetShardMapRequest>;
|
||||
using CoordinatorReadResponses = std::variant<GetShardMapResponse>;
|
||||
|
||||
class Coordinator {
|
||||
public:
|
||||
explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {}
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
|
||||
CoordinatorReadResponses Read(CoordinatorReadRequests requests) {
|
||||
return std::visit([&](auto &&request) { return HandleRead(std::forward<decltype(request)>(request)); },
|
||||
std::move(requests)); // NOLINT(hicpp-move-const-arg,performance-move-const-arg)
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
|
||||
CoordinatorWriteResponses Apply(CoordinatorWriteRequests requests) {
|
||||
return std::visit([&](auto &&request) mutable { return ApplyWrite(std::forward<decltype(request)>(request)); },
|
||||
std::move(requests));
|
||||
}
|
||||
|
||||
private:
|
||||
ShardMap shard_map_;
|
||||
uint64_t highest_allocated_timestamp_{0};
|
||||
|
||||
/// Query engines need to periodically request batches of unique edge IDs.
|
||||
uint64_t highest_allocated_edge_id_{0};
|
||||
|
||||
CoordinatorReadResponses HandleRead(GetShardMapRequest && /* get_shard_map_request */) {
|
||||
GetShardMapResponse res;
|
||||
res.shard_map = shard_map_;
|
||||
return res;
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses ApplyWrite(HeartbeatRequest &&heartbeat_request);
|
||||
|
||||
CoordinatorWriteResponses ApplyWrite(HlcRequest &&hlc_request);
|
||||
|
||||
CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr);
|
||||
|
||||
/// This splits the shard immediately beneath the provided
|
||||
/// split key, keeping the assigned peers identical for now,
|
||||
/// but letting them be gradually migrated over time.
|
||||
CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request);
|
||||
|
||||
/// This adds the provided storage engine to the standby storage engine pool,
|
||||
/// which can be used to rebalance storage over time.
|
||||
static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */);
|
||||
|
||||
/// This begins the process of draining the provided storage engine from all raft
|
||||
/// clusters that it might be participating in.
|
||||
static CoordinatorWriteResponses ApplyWrite(DeregisterStorageEngineRequest && /* register_storage_engine_request */);
|
||||
|
||||
CoordinatorWriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request);
|
||||
|
||||
CoordinatorWriteResponses ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request);
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
24
src/coordinator/coordinator_client.hpp
Normal file
24
src/coordinator/coordinator_client.hpp
Normal file
@@ -0,0 +1,24 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "io/rsm/rsm_client.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
using memgraph::io::rsm::RsmClient;
|
||||
|
||||
template <typename IoImpl>
|
||||
using CoordinatorClient = RsmClient<IoImpl, CoordinatorWriteRequests, CoordinatorWriteResponses,
|
||||
CoordinatorReadRequests, CoordinatorReadResponses>;
|
||||
} // namespace memgraph::coordinator
|
||||
23
src/coordinator/coordinator_rsm.hpp
Normal file
23
src/coordinator/coordinator_rsm.hpp
Normal file
@@ -0,0 +1,23 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
template <typename IoImpl>
|
||||
using CoordinatorRsm = memgraph::io::rsm::Raft<IoImpl, Coordinator, CoordinatorWriteRequests, CoordinatorWriteResponses,
|
||||
CoordinatorReadRequests, CoordinatorReadResponses>;
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
156
src/coordinator/coordinator_worker.hpp
Normal file
156
src/coordinator/coordinator_worker.hpp
Normal file
@@ -0,0 +1,156 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
#include <variant>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/coordinator_rsm.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "io/future.hpp"
|
||||
#include "io/messages.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
|
||||
namespace memgraph::coordinator::coordinator_worker {
|
||||
|
||||
/// Obligations:
|
||||
/// * ShutDown
|
||||
/// * Cron
|
||||
/// * RouteMessage
|
||||
|
||||
using coordinator::Coordinator;
|
||||
using coordinator::CoordinatorRsm;
|
||||
using io::Address;
|
||||
using io::RequestId;
|
||||
using io::Time;
|
||||
using io::messages::CoordinatorMessages;
|
||||
using msgs::ReadRequests;
|
||||
using msgs::ReadResponses;
|
||||
using msgs::WriteRequests;
|
||||
using msgs::WriteResponses;
|
||||
|
||||
struct ShutDown {};
|
||||
|
||||
struct Cron {};
|
||||
|
||||
struct RouteMessage {
|
||||
CoordinatorMessages message;
|
||||
RequestId request_id;
|
||||
Address to;
|
||||
Address from;
|
||||
};
|
||||
|
||||
using Message = std::variant<RouteMessage, Cron, ShutDown>;
|
||||
|
||||
struct QueueInner {
|
||||
std::mutex mu{};
|
||||
std::condition_variable cv;
|
||||
// TODO(tyler) handle simulator communication std::shared_ptr<std::atomic<int>> blocked;
|
||||
|
||||
// TODO(tyler) investigate using a priority queue that prioritizes messages in a way that
|
||||
// improves overall QoS. For example, maybe we want to schedule raft Append messages
|
||||
// ahead of Read messages or generally writes before reads for lowering the load on the
|
||||
// overall system faster etc... When we do this, we need to make sure to avoid
|
||||
// starvation by sometimes randomizing priorities, rather than following a strict
|
||||
// prioritization.
|
||||
std::deque<Message> queue;
|
||||
};
|
||||
|
||||
/// There are two reasons to implement our own Queue instead of using
|
||||
/// one off-the-shelf:
|
||||
/// 1. we will need to know in the simulator when all threads are waiting
|
||||
/// 2. we will want to implement our own priority queue within this for QoS
|
||||
class Queue {
|
||||
std::shared_ptr<QueueInner> inner_ = std::make_shared<QueueInner>();
|
||||
|
||||
public:
|
||||
void Push(Message &&message) {
|
||||
{
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->queue.emplace_back(std::move(message));
|
||||
} // lock dropped before notifying condition variable
|
||||
|
||||
inner_->cv.notify_all();
|
||||
}
|
||||
|
||||
Message Pop() {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
while (inner_->queue.empty()) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
|
||||
Message message = std::move(inner_->queue.front());
|
||||
inner_->queue.pop_front();
|
||||
|
||||
return message;
|
||||
}
|
||||
};
|
||||
|
||||
/// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
|
||||
template <typename IoImpl>
|
||||
class CoordinatorWorker {
|
||||
io::Io<IoImpl> io_;
|
||||
Queue queue_;
|
||||
CoordinatorRsm<IoImpl> coordinator_;
|
||||
|
||||
bool Process(ShutDown && /*shut_down*/) { return false; }
|
||||
|
||||
bool Process(Cron && /* cron */) {
|
||||
coordinator_.Cron();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Process(RouteMessage &&route_message) {
|
||||
coordinator_.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public:
|
||||
CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
|
||||
: io_(std::move(io)),
|
||||
queue_(std::move(queue)),
|
||||
coordinator_{std::move(io_.ForkLocal()), {}, std::move(coordinator)} {}
|
||||
|
||||
CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
|
||||
CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
|
||||
CoordinatorWorker(const CoordinatorWorker &) = delete;
|
||||
CoordinatorWorker &operator=(const CoordinatorWorker &) = delete;
|
||||
~CoordinatorWorker() = default;
|
||||
|
||||
void Run() {
|
||||
while (true) {
|
||||
Message message = queue_.Pop();
|
||||
|
||||
const bool should_continue = std::visit(
|
||||
[this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); }, std::move(message));
|
||||
|
||||
if (!should_continue) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator::coordinator_worker
|
||||
48
src/coordinator/hybrid_logical_clock.hpp
Normal file
48
src/coordinator/hybrid_logical_clock.hpp
Normal file
@@ -0,0 +1,48 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <compare>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
|
||||
#include "io/time.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
using Time = memgraph::io::Time;
|
||||
|
||||
/// Hybrid-logical clock
|
||||
struct Hlc {
|
||||
uint64_t logical_id = 0;
|
||||
Time coordinator_wall_clock = Time::min();
|
||||
|
||||
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
|
||||
|
||||
bool operator==(const Hlc &other) const = default;
|
||||
bool operator<(const Hlc &other) const = default;
|
||||
bool operator==(const uint64_t other) const { return logical_id == other; }
|
||||
bool operator<(const uint64_t other) const { return logical_id < other; }
|
||||
bool operator>=(const uint64_t other) const { return logical_id >= other; }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
|
||||
auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);
|
||||
in << "Hlc { logical_id: " << hlc.logical_id;
|
||||
in << ", coordinator_wall_clock: " << std::put_time(std::localtime(&wall_clock), "%F %T");
|
||||
in << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
552
src/coordinator/shard_map.cpp
Normal file
552
src/coordinator/shard_map.cpp
Normal file
@@ -0,0 +1,552 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "common/types.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/temporal.hpp"
|
||||
#include "utils/cast.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
using memgraph::common::SchemaType;
|
||||
using memgraph::storage::v3::TemporalData;
|
||||
using memgraph::storage::v3::TemporalType;
|
||||
|
||||
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
|
||||
PrimaryKey ret{};
|
||||
|
||||
const int64_t min_int = std::numeric_limits<int64_t>::min();
|
||||
|
||||
const TemporalData date{TemporalType::Date, min_int};
|
||||
const TemporalData local_time{TemporalType::LocalTime, min_int};
|
||||
const TemporalData local_date_time{TemporalType::LocalDateTime, min_int};
|
||||
const TemporalData duration{TemporalType::Duration, min_int};
|
||||
|
||||
for (const auto &schema_property : schema) {
|
||||
switch (schema_property.type) {
|
||||
case SchemaType::BOOL:
|
||||
ret.emplace_back(PropertyValue(false));
|
||||
break;
|
||||
case SchemaType::INT:
|
||||
ret.emplace_back(PropertyValue(min_int));
|
||||
break;
|
||||
case SchemaType::STRING:
|
||||
ret.emplace_back(PropertyValue(""));
|
||||
break;
|
||||
case SchemaType::DATE:
|
||||
ret.emplace_back(PropertyValue(date));
|
||||
break;
|
||||
case SchemaType::LOCALTIME:
|
||||
ret.emplace_back(PropertyValue(local_time));
|
||||
break;
|
||||
case SchemaType::LOCALDATETIME:
|
||||
ret.emplace_back(PropertyValue(local_date_time));
|
||||
break;
|
||||
case SchemaType::DURATION:
|
||||
ret.emplace_back(PropertyValue(duration));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ShardMap ShardMap::Parse(std::istream &input_stream) {
|
||||
ShardMap shard_map;
|
||||
const auto read_size = [&input_stream] {
|
||||
size_t size{0};
|
||||
input_stream >> size;
|
||||
return size;
|
||||
};
|
||||
|
||||
// Reads a string until the next whitespace
|
||||
const auto read_word = [&input_stream] {
|
||||
std::string word;
|
||||
input_stream >> word;
|
||||
return word;
|
||||
};
|
||||
|
||||
const auto read_names = [&read_size, &read_word] {
|
||||
const auto number_of_names = read_size();
|
||||
spdlog::trace("Reading {} names", number_of_names);
|
||||
std::vector<std::string> names;
|
||||
names.reserve(number_of_names);
|
||||
|
||||
for (auto name_index = 0; name_index < number_of_names; ++name_index) {
|
||||
names.push_back(read_word());
|
||||
spdlog::trace("Read '{}'", names.back());
|
||||
}
|
||||
return names;
|
||||
};
|
||||
|
||||
const auto read_line = [&input_stream] {
|
||||
std::string line;
|
||||
std::getline(input_stream, line);
|
||||
return line;
|
||||
};
|
||||
|
||||
const auto parse_type = [](const std::string &type) {
|
||||
static const auto type_map = std::unordered_map<std::string, common::SchemaType>{
|
||||
{"string", common::SchemaType::STRING}, {"int", common::SchemaType::INT}, {"bool", common::SchemaType::BOOL}};
|
||||
const auto lower_case_type = utils::ToLowerCase(type);
|
||||
auto it = type_map.find(lower_case_type);
|
||||
MG_ASSERT(it != type_map.end(), "Invalid type in split files: {}", type);
|
||||
return it->second;
|
||||
};
|
||||
|
||||
const auto parse_property_value = [](std::string text, const common::SchemaType type) {
|
||||
if (type == common::SchemaType::STRING) {
|
||||
return storage::v3::PropertyValue{std::move(text)};
|
||||
}
|
||||
if (type == common::SchemaType::INT) {
|
||||
size_t processed{0};
|
||||
int64_t value = std::stoll(text, &processed);
|
||||
MG_ASSERT(processed == text.size() || text[processed] == ' ', "Invalid integer format: '{}'", text);
|
||||
return storage::v3::PropertyValue{value};
|
||||
}
|
||||
LOG_FATAL("Not supported type: {}", utils::UnderlyingCast(type));
|
||||
};
|
||||
|
||||
spdlog::debug("Reading properties");
|
||||
const auto properties = read_names();
|
||||
MG_ASSERT(shard_map.AllocatePropertyIds(properties).size() == properties.size(),
|
||||
"Unexpected number of properties created!");
|
||||
|
||||
spdlog::debug("Reading edge types");
|
||||
const auto edge_types = read_names();
|
||||
MG_ASSERT(shard_map.AllocateEdgeTypeIds(edge_types).size() == edge_types.size(),
|
||||
"Unexpected number of properties created!");
|
||||
|
||||
spdlog::debug("Reading primary labels");
|
||||
const auto number_of_primary_labels = read_size();
|
||||
spdlog::debug("Reading {} primary labels", number_of_primary_labels);
|
||||
|
||||
for (auto label_index = 0; label_index < number_of_primary_labels; ++label_index) {
|
||||
const auto primary_label = read_word();
|
||||
spdlog::debug("Reading primary label named '{}'", primary_label);
|
||||
const auto number_of_primary_properties = read_size();
|
||||
spdlog::debug("Reading {} primary properties", number_of_primary_properties);
|
||||
std::vector<std::string> pp_names;
|
||||
std::vector<common::SchemaType> pp_types;
|
||||
pp_names.reserve(number_of_primary_properties);
|
||||
pp_types.reserve(number_of_primary_properties);
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
pp_names.push_back(read_word());
|
||||
spdlog::debug("Reading primary property named '{}'", pp_names.back());
|
||||
pp_types.push_back(parse_type(read_word()));
|
||||
}
|
||||
auto pp_mapping = shard_map.AllocatePropertyIds(pp_names);
|
||||
std::vector<SchemaProperty> schema;
|
||||
schema.reserve(number_of_primary_properties);
|
||||
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
schema.push_back(storage::v3::SchemaProperty{pp_mapping.at(pp_names[property_index]), pp_types[property_index]});
|
||||
}
|
||||
const auto hlc = shard_map.GetHlc();
|
||||
MG_ASSERT(shard_map.InitializeNewLabel(primary_label, schema, 1, hlc).has_value(),
|
||||
"Cannot initialize new label: {}", primary_label);
|
||||
|
||||
const auto number_of_split_points = read_size();
|
||||
spdlog::debug("Reading {} split points", number_of_split_points);
|
||||
|
||||
[[maybe_unused]] const auto remainder_from_last_line = read_line();
|
||||
for (auto split_point_index = 0; split_point_index < number_of_split_points; ++split_point_index) {
|
||||
const auto line = read_line();
|
||||
spdlog::debug("Read split point '{}'", line);
|
||||
MG_ASSERT(line.front() == '[', "Invalid split file format!");
|
||||
MG_ASSERT(line.back() == ']', "Invalid split file format!");
|
||||
std::string_view line_view{line};
|
||||
line_view.remove_prefix(1);
|
||||
line_view.remove_suffix(1);
|
||||
static constexpr std::string_view kDelimiter{","};
|
||||
auto pk_values_as_text = utils::Split(line_view, kDelimiter);
|
||||
std::vector<PropertyValue> pk;
|
||||
pk.reserve(number_of_primary_properties);
|
||||
MG_ASSERT(pk_values_as_text.size() == number_of_primary_properties,
|
||||
"Split point contains invalid number of values '{}'", line);
|
||||
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
pk.push_back(parse_property_value(std::move(pk_values_as_text[property_index]), schema[property_index].type));
|
||||
}
|
||||
shard_map.SplitShard(shard_map.GetHlc(), shard_map.labels.at(primary_label), pk);
|
||||
}
|
||||
}
|
||||
|
||||
return shard_map;
|
||||
}
|
||||
|
||||
std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map) {
|
||||
using utils::print_helpers::operator<<;
|
||||
|
||||
in << "ShardMap { shard_map_version: " << shard_map.shard_map_version;
|
||||
in << ", max_property_id: " << shard_map.max_property_id;
|
||||
in << ", max_edge_type_id: " << shard_map.max_edge_type_id;
|
||||
in << ", properties: " << shard_map.properties;
|
||||
in << ", edge_types: " << shard_map.edge_types;
|
||||
in << ", max_label_id: " << shard_map.max_label_id;
|
||||
in << ", labels: " << shard_map.labels;
|
||||
in << ", label_spaces: " << shard_map.label_spaces;
|
||||
in << ", schemas: " << shard_map.schemas;
|
||||
in << "}";
|
||||
return in;
|
||||
}
|
||||
|
||||
Shards ShardMap::GetShardsForLabel(const LabelName &label) const {
|
||||
const auto id = labels.at(label);
|
||||
const auto &shards = label_spaces.at(id).shards;
|
||||
return shards;
|
||||
}
|
||||
|
||||
std::vector<Shards> ShardMap::GetAllShards() const {
|
||||
std::vector<Shards> all_shards;
|
||||
all_shards.reserve(label_spaces.size());
|
||||
std::transform(label_spaces.begin(), label_spaces.end(), std::back_inserter(all_shards),
|
||||
[](const auto &label_space) { return label_space.second.shards; });
|
||||
return all_shards;
|
||||
}
|
||||
|
||||
// TODO(gabor) later we will want to update the wallclock time with
|
||||
// the given Io<impl>'s time as well
|
||||
Hlc ShardMap::IncrementShardMapVersion() noexcept {
|
||||
++shard_map_version.logical_id;
|
||||
return shard_map_version;
|
||||
}
|
||||
|
||||
// TODO(antaljanosbenjamin) use a single map for all name id
|
||||
// mapping and a single counter to maintain the next id
|
||||
std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
|
||||
const auto map_type_ids = [&id_to_names](const auto &name_to_id_type) {
|
||||
for (const auto &[name, id] : name_to_id_type) {
|
||||
id_to_names.emplace(id.AsUint(), name);
|
||||
}
|
||||
};
|
||||
|
||||
map_type_ids(edge_types);
|
||||
map_type_ids(labels);
|
||||
map_type_ids(properties);
|
||||
|
||||
return id_to_names;
|
||||
}
|
||||
|
||||
Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
|
||||
|
||||
std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
|
||||
std::set<boost::uuids::uuid> initialized) {
|
||||
std::vector<ShardToInitialize> ret{};
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (auto &[label_id, label_space] : label_spaces) {
|
||||
for (auto it = label_space.shards.begin(); it != label_space.shards.end(); it++) {
|
||||
auto &[low_key, shard] = *it;
|
||||
std::optional<PrimaryKey> high_key;
|
||||
if (const auto next_it = std::next(it); next_it != label_space.shards.end()) {
|
||||
high_key = next_it->first;
|
||||
}
|
||||
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
|
||||
bool machine_contains_shard = false;
|
||||
|
||||
for (auto &aas : shard) {
|
||||
if (initialized.contains(aas.address.unique_id)) {
|
||||
machine_contains_shard = true;
|
||||
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
|
||||
aas.status = Status::CONSENSUS_PARTICIPANT;
|
||||
}
|
||||
} else {
|
||||
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
|
||||
aas.address.last_known_port == storage_manager.last_known_port;
|
||||
if (same_machine) {
|
||||
machine_contains_shard = true;
|
||||
spdlog::info("reminding shard manager that they should begin participating in shard");
|
||||
|
||||
ret.push_back(ShardToInitialize{
|
||||
.uuid = aas.address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
|
||||
Address address = storage_manager;
|
||||
|
||||
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
|
||||
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
|
||||
spdlog::info("assigning shard manager to shard");
|
||||
|
||||
ret.push_back(ShardToInitialize{
|
||||
.uuid = address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
|
||||
AddressAndStatus aas = {
|
||||
.address = address,
|
||||
.status = Status::INITIALIZING,
|
||||
};
|
||||
|
||||
shard.emplace_back(aas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
|
||||
if (previous_shard_map_version != shard_map_version) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto &label_space = label_spaces.at(label_id);
|
||||
auto &shards_in_map = label_space.shards;
|
||||
|
||||
MG_ASSERT(!shards_in_map.empty());
|
||||
MG_ASSERT(!shards_in_map.contains(key));
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
// Finding the Shard that the new PrimaryKey should map to.
|
||||
auto prev = std::prev(shards_in_map.upper_bound(key));
|
||||
Shard duplicated_shard = prev->second;
|
||||
|
||||
// Apply the split
|
||||
shards_in_map[key] = duplicated_shard;
|
||||
|
||||
IncrementShardMapVersion();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, Hlc last_shard_map_version) {
|
||||
if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const LabelId label_id = LabelId::FromUint(++max_label_id);
|
||||
|
||||
labels.emplace(std::move(label_name), label_id);
|
||||
|
||||
PrimaryKey initial_key = SchemaToMinKey(schema);
|
||||
Shard empty_shard = {};
|
||||
|
||||
Shards shards = {
|
||||
{initial_key, empty_shard},
|
||||
};
|
||||
|
||||
LabelSpace label_space{
|
||||
.schema = schema,
|
||||
.shards = shards,
|
||||
.replication_factor = replication_factor,
|
||||
};
|
||||
schemas[label_id] = std::move(schema);
|
||||
|
||||
label_spaces.emplace(label_id, label_space);
|
||||
|
||||
IncrementShardMapVersion();
|
||||
|
||||
return label_id;
|
||||
}
|
||||
|
||||
void ShardMap::AddServer(Address server_address) {
|
||||
// Find a random place for the server to plug in
|
||||
}
|
||||
std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
|
||||
if (const auto it = labels.find(label); it != labels.end()) {
|
||||
return it->second;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string &ShardMap::GetLabelName(const LabelId label) const {
|
||||
if (const auto it =
|
||||
std::ranges::find_if(labels, [label](const auto &name_id_pair) { return name_id_pair.second == label; });
|
||||
it != labels.end()) {
|
||||
return it->first;
|
||||
}
|
||||
throw utils::BasicException("GetLabelName fails on the given label id!");
|
||||
}
|
||||
|
||||
std::optional<PropertyId> ShardMap::GetPropertyId(const std::string &property_name) const {
|
||||
if (const auto it = properties.find(property_name); it != properties.end()) {
|
||||
return it->second;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string &ShardMap::GetPropertyName(const PropertyId property) const {
|
||||
if (const auto it = std::ranges::find_if(
|
||||
properties, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
|
||||
it != properties.end()) {
|
||||
return it->first;
|
||||
}
|
||||
throw utils::BasicException("PropertyId not found!");
|
||||
}
|
||||
|
||||
std::optional<EdgeTypeId> ShardMap::GetEdgeTypeId(const std::string &edge_type) const {
|
||||
if (const auto it = edge_types.find(edge_type); it != edge_types.end()) {
|
||||
return it->second;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string &ShardMap::GetEdgeTypeName(const EdgeTypeId property) const {
|
||||
if (const auto it = std::ranges::find_if(
|
||||
edge_types, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
|
||||
it != edge_types.end()) {
|
||||
return it->first;
|
||||
}
|
||||
throw utils::BasicException("EdgeTypeId not found!");
|
||||
}
|
||||
|
||||
Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key,
|
||||
const PrimaryKey &end_key) const {
|
||||
MG_ASSERT(start_key <= end_key);
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
LabelId label_id = labels.at(label_name);
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
const auto &shards_for_label = label_space.shards;
|
||||
|
||||
MG_ASSERT(shards_for_label.begin()->first <= start_key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
auto it = std::prev(shards_for_label.upper_bound(start_key));
|
||||
const auto end_it = shards_for_label.upper_bound(end_key);
|
||||
|
||||
Shards shards{};
|
||||
|
||||
std::copy(it, end_it, std::inserter(shards, shards.end()));
|
||||
|
||||
return shards;
|
||||
}
|
||||
|
||||
Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
LabelId label_id = labels.at(label_name);
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
MG_ASSERT(label_space.shards.begin()->first <= key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
Shard ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
MG_ASSERT(label_space.shards.begin()->first <= key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
PropertyMap ShardMap::AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
|
||||
PropertyMap ret{};
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (const auto &property_name : new_properties) {
|
||||
if (properties.contains(property_name)) {
|
||||
auto property_id = properties.at(property_name);
|
||||
ret.emplace(property_name, property_id);
|
||||
} else {
|
||||
mutated = true;
|
||||
|
||||
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
|
||||
ret.emplace(property_name, property_id);
|
||||
properties.emplace(property_name, property_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types) {
|
||||
EdgeTypeIdMap ret;
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (const auto &edge_type_name : new_edge_types) {
|
||||
if (edge_types.contains(edge_type_name)) {
|
||||
auto edge_type_id = edge_types.at(edge_type_name);
|
||||
ret.emplace(edge_type_name, edge_type_id);
|
||||
} else {
|
||||
mutated = true;
|
||||
|
||||
const EdgeTypeId edge_type_id = EdgeTypeId::FromUint(++max_edge_type_id);
|
||||
ret.emplace(edge_type_name, edge_type_id);
|
||||
edge_types.emplace(edge_type_name, edge_type_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ShardMap::ClusterInitialized() const {
|
||||
for (const auto &[label_id, label_space] : label_spaces) {
|
||||
for (const auto &[low_key, shard] : label_space.shards) {
|
||||
if (shard.size() < label_space.replication_factor) {
|
||||
spdlog::info("label_space below desired replication factor");
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto &aas : shard) {
|
||||
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
177
src/coordinator/shard_map.hpp
Normal file
177
src/coordinator/shard_map.hpp
Normal file
@@ -0,0 +1,177 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
#include <boost/uuid/uuid_generators.hpp>
|
||||
|
||||
#include "common/types.hpp"
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/temporal.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/print_helpers.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
constexpr int64_t kNotExistingId{0};
|
||||
|
||||
using memgraph::io::Address;
|
||||
using memgraph::storage::v3::Config;
|
||||
using memgraph::storage::v3::EdgeTypeId;
|
||||
using memgraph::storage::v3::LabelId;
|
||||
using memgraph::storage::v3::PropertyId;
|
||||
using memgraph::storage::v3::PropertyValue;
|
||||
using memgraph::storage::v3::SchemaProperty;
|
||||
|
||||
enum class Status : uint8_t {
|
||||
CONSENSUS_PARTICIPANT,
|
||||
INITIALIZING,
|
||||
// TODO(tyler) this will possibly have more states,
|
||||
// depending on the reconfiguration protocol that we
|
||||
// implement.
|
||||
};
|
||||
|
||||
struct AddressAndStatus {
|
||||
memgraph::io::Address address;
|
||||
Status status;
|
||||
|
||||
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const AddressAndStatus &address_and_status) {
|
||||
in << "AddressAndStatus { address: ";
|
||||
in << address_and_status.address;
|
||||
if (address_and_status.status == Status::CONSENSUS_PARTICIPANT) {
|
||||
in << ", status: CONSENSUS_PARTICIPANT }";
|
||||
} else {
|
||||
in << ", status: INITIALIZING }";
|
||||
}
|
||||
|
||||
return in;
|
||||
}
|
||||
|
||||
friend bool operator==(const AddressAndStatus &lhs, const AddressAndStatus &rhs) {
|
||||
return lhs.address == rhs.address;
|
||||
}
|
||||
};
|
||||
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
using Shard = std::vector<AddressAndStatus>;
|
||||
using Shards = std::map<PrimaryKey, Shard>;
|
||||
using LabelName = std::string;
|
||||
using PropertyName = std::string;
|
||||
using EdgeTypeName = std::string;
|
||||
using PropertyMap = std::map<PropertyName, PropertyId>;
|
||||
using EdgeTypeIdMap = std::map<EdgeTypeName, EdgeTypeId>;
|
||||
|
||||
struct ShardToInitialize {
|
||||
boost::uuids::uuid uuid;
|
||||
LabelId label_id;
|
||||
PrimaryKey min_key;
|
||||
std::optional<PrimaryKey> max_key;
|
||||
std::vector<SchemaProperty> schema;
|
||||
Config config;
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
};
|
||||
|
||||
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
|
||||
|
||||
struct LabelSpace {
|
||||
std::vector<SchemaProperty> schema;
|
||||
// Maps between the smallest primary key stored in the shard and the shard
|
||||
std::map<PrimaryKey, Shard> shards;
|
||||
size_t replication_factor;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
|
||||
using utils::print_helpers::operator<<;
|
||||
|
||||
in << "LabelSpace { schema: ";
|
||||
in << label_space.schema;
|
||||
in << ", shards: ";
|
||||
in << label_space.shards;
|
||||
in << ", replication_factor: " << label_space.replication_factor << "}";
|
||||
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
struct ShardMap {
|
||||
Hlc shard_map_version;
|
||||
uint64_t max_property_id{kNotExistingId};
|
||||
uint64_t max_edge_type_id{kNotExistingId};
|
||||
std::map<PropertyName, PropertyId> properties;
|
||||
std::map<EdgeTypeName, EdgeTypeId> edge_types;
|
||||
uint64_t max_label_id{kNotExistingId};
|
||||
std::map<LabelId, LabelSpace> label_spaces;
|
||||
std::map<LabelId, std::vector<SchemaProperty>> schemas;
|
||||
std::map<LabelName, LabelId> labels;
|
||||
|
||||
[[nodiscard]] static ShardMap Parse(std::istream &input_stream);
|
||||
friend std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map);
|
||||
|
||||
Shards GetShardsForLabel(const LabelName &label) const;
|
||||
|
||||
std::vector<Shards> GetAllShards() const;
|
||||
|
||||
// TODO(gabor) later we will want to update the wallclock time with
|
||||
// the given Io<impl>'s time as well
|
||||
Hlc IncrementShardMapVersion() noexcept;
|
||||
Hlc GetHlc() const noexcept;
|
||||
|
||||
std::unordered_map<uint64_t, std::string> IdToNames();
|
||||
|
||||
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
|
||||
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
|
||||
|
||||
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
|
||||
|
||||
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, Hlc last_shard_map_version);
|
||||
|
||||
void AddServer(Address server_address);
|
||||
|
||||
std::optional<LabelId> GetLabelId(const std::string &label) const;
|
||||
// TODO(antaljanosbenjamin): Remove this and instead use NameIdMapper
|
||||
const std::string &GetLabelName(LabelId label) const;
|
||||
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const;
|
||||
const std::string &GetPropertyName(PropertyId property) const;
|
||||
std::optional<EdgeTypeId> GetEdgeTypeId(const std::string &edge_type) const;
|
||||
const std::string &GetEdgeTypeName(EdgeTypeId property) const;
|
||||
|
||||
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
|
||||
|
||||
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
|
||||
|
||||
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
|
||||
|
||||
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
|
||||
|
||||
EdgeTypeIdMap AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types);
|
||||
|
||||
/// Returns true if all shards have the desired number of replicas and they are in
|
||||
/// the CONSENSUS_PARTICIPANT state. Note that this does not necessarily mean that
|
||||
/// there is also an active leader for each shard.
|
||||
bool ClusterInitialized() const;
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
20
src/expr/CMakeLists.txt
Normal file
20
src/expr/CMakeLists.txt
Normal file
@@ -0,0 +1,20 @@
|
||||
define_add_lcp(add_lcp_expr lcp_expr_cpp_files generated_lcp_expr_files)
|
||||
|
||||
add_lcp_expr(semantic/symbol.lcp)
|
||||
|
||||
add_custom_target(generate_lcp_expr DEPENDS ${generated_lcp_expr_files})
|
||||
|
||||
set(mg_expr_sources
|
||||
${lcp_expr_cpp_files}
|
||||
parsing.cpp)
|
||||
|
||||
find_package(Boost REQUIRED)
|
||||
|
||||
add_library(mg-expr STATIC ${mg_expr_sources})
|
||||
add_dependencies(mg-expr generate_lcp_expr)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
|
||||
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)
|
||||
33
src/expr/ast.hpp
Normal file
33
src/expr/ast.hpp
Normal file
@@ -0,0 +1,33 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
#pragma once
|
||||
|
||||
#ifndef MG_AST_INCLUDE_PATH
|
||||
#ifdef MG_CLANG_TIDY_CHECK
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#include "query/v2/bindings/bindings.hpp"
|
||||
#else
|
||||
#error Missing AST include path
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifndef MG_INJECTED_NAMESPACE_NAME
|
||||
#ifdef MG_CLANG_TIDY_CHECK
|
||||
#else
|
||||
#error Missing AST namespace
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include MG_AST_INCLUDE_PATH
|
||||
|
||||
namespace memgraph::expr {
|
||||
using namespace MG_INJECTED_NAMESPACE_NAME; // NOLINT(google-build-using-namespace)
|
||||
} // namespace memgraph::expr
|
||||
135
src/expr/ast/ast_visitor.hpp
Normal file
135
src/expr/ast/ast_visitor.hpp
Normal file
@@ -0,0 +1,135 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/visitor.hpp"
|
||||
|
||||
namespace MG_INJECTED_NAMESPACE_NAME {
|
||||
|
||||
// Forward declares for Tree visitors.
|
||||
class CypherQuery;
|
||||
class SingleQuery;
|
||||
class CypherUnion;
|
||||
class NamedExpression;
|
||||
class Identifier;
|
||||
class PropertyLookup;
|
||||
class LabelsTest;
|
||||
class Aggregation;
|
||||
class Function;
|
||||
class Reduce;
|
||||
class Coalesce;
|
||||
class Extract;
|
||||
class All;
|
||||
class Single;
|
||||
class Any;
|
||||
class None;
|
||||
class ParameterLookup;
|
||||
class CallProcedure;
|
||||
class Create;
|
||||
class Match;
|
||||
class Return;
|
||||
class With;
|
||||
class Pattern;
|
||||
class NodeAtom;
|
||||
class EdgeAtom;
|
||||
class PrimitiveLiteral;
|
||||
class ListLiteral;
|
||||
class MapLiteral;
|
||||
class OrOperator;
|
||||
class XorOperator;
|
||||
class AndOperator;
|
||||
class NotOperator;
|
||||
class AdditionOperator;
|
||||
class SubtractionOperator;
|
||||
class MultiplicationOperator;
|
||||
class DivisionOperator;
|
||||
class ModOperator;
|
||||
class UnaryPlusOperator;
|
||||
class UnaryMinusOperator;
|
||||
class IsNullOperator;
|
||||
class NotEqualOperator;
|
||||
class EqualOperator;
|
||||
class LessOperator;
|
||||
class GreaterOperator;
|
||||
class LessEqualOperator;
|
||||
class GreaterEqualOperator;
|
||||
class InListOperator;
|
||||
class SubscriptOperator;
|
||||
class ListSlicingOperator;
|
||||
class IfOperator;
|
||||
class Delete;
|
||||
class Where;
|
||||
class SetProperty;
|
||||
class SetProperties;
|
||||
class SetLabels;
|
||||
class RemoveProperty;
|
||||
class RemoveLabels;
|
||||
class Merge;
|
||||
class Unwind;
|
||||
class AuthQuery;
|
||||
class ExplainQuery;
|
||||
class ProfileQuery;
|
||||
class IndexQuery;
|
||||
class InfoQuery;
|
||||
class ConstraintQuery;
|
||||
class RegexMatch;
|
||||
class DumpQuery;
|
||||
class ReplicationQuery;
|
||||
class LockPathQuery;
|
||||
class LoadCsv;
|
||||
class FreeMemoryQuery;
|
||||
class TriggerQuery;
|
||||
class IsolationLevelQuery;
|
||||
class CreateSnapshotQuery;
|
||||
class StreamQuery;
|
||||
class SettingQuery;
|
||||
class VersionQuery;
|
||||
class Foreach;
|
||||
class SchemaQuery;
|
||||
|
||||
using TreeCompositeVisitor = memgraph::utils::CompositeVisitor<
|
||||
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
|
||||
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
|
||||
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
|
||||
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
|
||||
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
|
||||
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
|
||||
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach>;
|
||||
|
||||
using TreeLeafVisitor = memgraph::utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
|
||||
|
||||
class HierarchicalTreeVisitor : public TreeCompositeVisitor, public TreeLeafVisitor {
|
||||
public:
|
||||
using TreeCompositeVisitor::PostVisit;
|
||||
using TreeCompositeVisitor::PreVisit;
|
||||
using TreeLeafVisitor::Visit;
|
||||
using typename TreeLeafVisitor::ReturnType;
|
||||
};
|
||||
|
||||
template <class TResult>
|
||||
class ExpressionVisitor
|
||||
: public memgraph::utils::Visitor<
|
||||
TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
|
||||
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
|
||||
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
|
||||
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral,
|
||||
MapLiteral, PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any,
|
||||
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch> {};
|
||||
|
||||
template <class TResult>
|
||||
class QueryVisitor
|
||||
: public memgraph::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
|
||||
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
|
||||
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery,
|
||||
StreamQuery, SettingQuery, VersionQuery, SchemaQuery> {};
|
||||
|
||||
} // namespace MG_INJECTED_NAMESPACE_NAME
|
||||
3035
src/expr/ast/cypher_main_visitor.hpp
Normal file
3035
src/expr/ast/cypher_main_visitor.hpp
Normal file
File diff suppressed because it is too large
Load Diff
271
src/expr/ast/pretty_print.hpp
Normal file
271
src/expr/ast/pretty_print.hpp
Normal file
@@ -0,0 +1,271 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <type_traits>
|
||||
|
||||
#include "expr/ast.hpp"
|
||||
#include "expr/typed_value.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const T &arg) {
|
||||
static_assert(!std::is_convertible<T, Expression *>::value,
|
||||
"This overload shouldn't be called with pointers convertible "
|
||||
"to Expression *. This means your other PrintObject overloads aren't "
|
||||
"being called for certain AST nodes when they should (or perhaps such "
|
||||
"overloads don't exist yet).");
|
||||
*out << arg;
|
||||
}
|
||||
|
||||
inline void PrintObject(std::ostream *out, const std::string &str) { *out << utils::Escape(str); }
|
||||
|
||||
inline void PrintObject(std::ostream *out, Aggregation::Op op) { *out << Aggregation::OpToString(op); }
|
||||
|
||||
inline void PrintObject(std::ostream *out, Expression *expr);
|
||||
|
||||
inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const std::vector<T> &vec) {
|
||||
*out << "[";
|
||||
utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); });
|
||||
*out << "]";
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const std::vector<T, utils::Allocator<T>> &vec) {
|
||||
*out << "[";
|
||||
utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); });
|
||||
*out << "]";
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void PrintObject(std::ostream *out, const std::map<K, V> &map) {
|
||||
*out << "{";
|
||||
utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) {
|
||||
PrintObject(&stream, item.first);
|
||||
stream << ": ";
|
||||
PrintObject(&stream, item.second);
|
||||
});
|
||||
*out << "}";
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const utils::pmr::map<utils::pmr::string, T> &map) {
|
||||
*out << "{";
|
||||
utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) {
|
||||
PrintObject(&stream, item.first);
|
||||
stream << ": ";
|
||||
PrintObject(&stream, item.second);
|
||||
});
|
||||
*out << "}";
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename T3>
|
||||
inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
|
||||
using TypedValue = TypedValueT<T1, T2, T3>;
|
||||
switch (value.type()) {
|
||||
case TypedValue::Type::Null:
|
||||
*out << "null";
|
||||
break;
|
||||
case TypedValue::Type::String:
|
||||
PrintObject(out, value.ValueString());
|
||||
break;
|
||||
case TypedValue::Type::Bool:
|
||||
*out << (value.ValueBool() ? "true" : "false");
|
||||
break;
|
||||
case TypedValue::Type::Int:
|
||||
PrintObject(out, value.ValueInt());
|
||||
break;
|
||||
case TypedValue::Type::Double:
|
||||
PrintObject(out, value.ValueDouble());
|
||||
break;
|
||||
case TypedValue::Type::List:
|
||||
PrintObject(out, value.ValueList());
|
||||
break;
|
||||
case TypedValue::Type::Map:
|
||||
PrintObject(out, value.ValueMap());
|
||||
break;
|
||||
case TypedValue::Type::Date:
|
||||
PrintObject(out, value.ValueDate());
|
||||
break;
|
||||
case TypedValue::Type::Duration:
|
||||
PrintObject(out, value.ValueDuration());
|
||||
break;
|
||||
case TypedValue::Type::LocalTime:
|
||||
PrintObject(out, value.ValueLocalTime());
|
||||
break;
|
||||
case TypedValue::Type::LocalDateTime:
|
||||
PrintObject(out, value.ValueLocalDateTime());
|
||||
break;
|
||||
default:
|
||||
MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintOperatorArgs(std::ostream *out, const T &arg) {
|
||||
*out << " ";
|
||||
PrintObject(out, arg);
|
||||
*out << ")";
|
||||
}
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
|
||||
*out << " ";
|
||||
PrintObject(out, arg);
|
||||
PrintOperatorArgs(out, args...);
|
||||
}
|
||||
|
||||
template <typename... Ts>
|
||||
void PrintOperator(std::ostream *out, const std::string &name, const Ts &...args) {
|
||||
*out << "(" << name;
|
||||
PrintOperatorArgs(out, args...);
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
|
||||
public:
|
||||
explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
|
||||
|
||||
// Unary operators
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE &op) override { detail::PrintOperator(out_, OP_STR, op.expression_); }
|
||||
|
||||
UNARY_OPERATOR_VISIT(NotOperator, "Not");
|
||||
UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
|
||||
UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
|
||||
UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
|
||||
|
||||
#undef UNARY_OPERATOR_VISIT
|
||||
|
||||
// Binary operators
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE &op) override { detail::PrintOperator(out_, OP_STR, op.expression1_, op.expression2_); }
|
||||
|
||||
BINARY_OPERATOR_VISIT(OrOperator, "Or");
|
||||
BINARY_OPERATOR_VISIT(XorOperator, "Xor");
|
||||
BINARY_OPERATOR_VISIT(AndOperator, "And");
|
||||
BINARY_OPERATOR_VISIT(AdditionOperator, "+");
|
||||
BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
|
||||
BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
|
||||
BINARY_OPERATOR_VISIT(DivisionOperator, "/");
|
||||
BINARY_OPERATOR_VISIT(ModOperator, "%");
|
||||
BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
|
||||
BINARY_OPERATOR_VISIT(EqualOperator, "==");
|
||||
BINARY_OPERATOR_VISIT(LessOperator, "<");
|
||||
BINARY_OPERATOR_VISIT(GreaterOperator, ">");
|
||||
BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
|
||||
BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
|
||||
BINARY_OPERATOR_VISIT(InListOperator, "In");
|
||||
BINARY_OPERATOR_VISIT(SubscriptOperator, "Subscript");
|
||||
|
||||
#undef BINARY_OPERATOR_VISIT
|
||||
|
||||
// Other
|
||||
void Visit(ListSlicingOperator &op) override {
|
||||
detail::PrintOperator(out_, "ListSlicing", op.list_, op.lower_bound_, op.upper_bound_);
|
||||
}
|
||||
|
||||
void Visit(IfOperator &op) override {
|
||||
detail::PrintOperator(out_, "If", op.condition_, op.then_expression_, op.else_expression_);
|
||||
}
|
||||
|
||||
void Visit(ListLiteral &op) override { detail::PrintOperator(out_, "ListLiteral", op.elements_); }
|
||||
|
||||
void Visit(MapLiteral &op) override {
|
||||
std::map<std::string, Expression *> map;
|
||||
for (const auto &kv : op.elements_) {
|
||||
map[kv.first.name] = kv.second;
|
||||
}
|
||||
detail::PrintObject(out_, map);
|
||||
}
|
||||
|
||||
void Visit(LabelsTest &op) override { detail::PrintOperator(out_, "LabelsTest", op.expression_); }
|
||||
|
||||
void Visit(Aggregation &op) override { detail::PrintOperator(out_, "Aggregation", op.op_); }
|
||||
|
||||
void Visit(Function &op) override { detail::PrintOperator(out_, "Function", op.function_name_, op.arguments_); }
|
||||
|
||||
void Visit(Reduce &op) override {
|
||||
detail::PrintOperator(out_, "Reduce", op.accumulator_, op.initializer_, op.identifier_, op.list_, op.expression_);
|
||||
}
|
||||
|
||||
void Visit(Coalesce &op) override { detail::PrintOperator(out_, "Coalesce", op.expressions_); }
|
||||
|
||||
void Visit(Extract &op) override { detail::PrintOperator(out_, "Extract", op.identifier_, op.list_, op.expression_); }
|
||||
|
||||
void Visit(All &op) override {
|
||||
detail::PrintOperator(out_, "All", op.identifier_, op.list_expression_, op.where_->expression_);
|
||||
}
|
||||
|
||||
void Visit(Single &op) override {
|
||||
detail::PrintOperator(out_, "Single", op.identifier_, op.list_expression_, op.where_->expression_);
|
||||
}
|
||||
|
||||
void Visit(Any &op) override {
|
||||
detail::PrintOperator(out_, "Any", op.identifier_, op.list_expression_, op.where_->expression_);
|
||||
}
|
||||
|
||||
void Visit(None &op) override {
|
||||
detail::PrintOperator(out_, "None", op.identifier_, op.list_expression_, op.where_->expression_);
|
||||
}
|
||||
|
||||
void Visit(Identifier &op) override { detail::PrintOperator(out_, "Identifier", op.name_); }
|
||||
|
||||
void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
|
||||
|
||||
void Visit(PropertyLookup &op) override {
|
||||
detail::PrintOperator(out_, "PropertyLookup", op.expression_, op.property_.name);
|
||||
}
|
||||
|
||||
void Visit(ParameterLookup &op) override { detail::PrintOperator(out_, "ParameterLookup", op.token_position_); }
|
||||
|
||||
void Visit(NamedExpression &op) override { detail::PrintOperator(out_, "NamedExpression", op.name_, op.expression_); }
|
||||
|
||||
void Visit(RegexMatch &op) override { detail::PrintOperator(out_, "=~", op.string_expr_, op.regex_); }
|
||||
|
||||
private:
|
||||
std::ostream *out_;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
inline void PrintObject(std::ostream *out, Expression *expr) {
|
||||
if (expr) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
} else {
|
||||
*out << "<null>";
|
||||
}
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline void PrintExpression(Expression *expr, std::ostream *out) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
}
|
||||
|
||||
inline void PrintExpression(NamedExpression *expr, std::ostream *out) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
}
|
||||
} // namespace memgraph::expr
|
||||
279
src/expr/ast/pretty_print_ast_to_original_expression.hpp
Normal file
279
src/expr/ast/pretty_print_ast_to_original_expression.hpp
Normal file
@@ -0,0 +1,279 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <type_traits>
|
||||
|
||||
#include "expr/ast.hpp"
|
||||
#include "expr/typed_value.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
inline constexpr const char *identifier_node_symbol = "MG_SYMBOL_NODE";
|
||||
inline constexpr const char *identifier_edge_symbol = "MG_SYMBOL_EDGE";
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const T &arg) {
|
||||
static_assert(!std::is_convertible<T, Expression *>::value,
|
||||
"This overload shouldn't be called with pointers convertible "
|
||||
"to Expression *. This means your other PrintObject overloads aren't "
|
||||
"being called for certain AST nodes when they should (or perhaps such "
|
||||
"overloads don't exist yet).");
|
||||
*out << arg;
|
||||
}
|
||||
|
||||
inline void PrintObject(std::ostream *out, const std::string &str) { *out << str; }
|
||||
|
||||
inline void PrintObject(std::ostream * /*out*/, Aggregation::Op /*op*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: Aggregation::Op");
|
||||
}
|
||||
|
||||
inline void PrintObject(std::ostream * /*out*/, Expression * /*expr*/);
|
||||
|
||||
inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream * /*out*/, const std::vector<T> & /*vec*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: vector<T>");
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream * /*out*/, const std::vector<T, utils::Allocator<T>> & /*vec*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: vector<T, utils::Allocator<T>>");
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void PrintObject(std::ostream * /*out*/, const std::map<K, V> & /*map*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: map<K, V>");
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream * /*out*/, const utils::pmr::map<utils::pmr::string, T> & /*map*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: map<utils::pmr::string, V>");
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename T3>
|
||||
inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
|
||||
using TypedValue = TypedValueT<T1, T2, T3>;
|
||||
switch (value.type()) {
|
||||
case TypedValue::Type::Null:
|
||||
*out << "null";
|
||||
break;
|
||||
case TypedValue::Type::String:
|
||||
PrintObject(out, value.ValueString());
|
||||
break;
|
||||
case TypedValue::Type::Bool:
|
||||
*out << (value.ValueBool() ? "true" : "false");
|
||||
break;
|
||||
case TypedValue::Type::Int:
|
||||
PrintObject(out, value.ValueInt());
|
||||
break;
|
||||
case TypedValue::Type::Double:
|
||||
PrintObject(out, value.ValueDouble());
|
||||
break;
|
||||
case TypedValue::Type::List:
|
||||
PrintObject(out, value.ValueList());
|
||||
break;
|
||||
case TypedValue::Type::Map:
|
||||
PrintObject(out, value.ValueMap());
|
||||
break;
|
||||
case TypedValue::Type::Date:
|
||||
PrintObject(out, value.ValueDate());
|
||||
break;
|
||||
case TypedValue::Type::Duration:
|
||||
PrintObject(out, value.ValueDuration());
|
||||
break;
|
||||
case TypedValue::Type::LocalTime:
|
||||
PrintObject(out, value.ValueLocalTime());
|
||||
break;
|
||||
case TypedValue::Type::LocalDateTime:
|
||||
PrintObject(out, value.ValueLocalDateTime());
|
||||
break;
|
||||
default:
|
||||
MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintOperatorArgs(const std::string & /*name*/, std::ostream *out, bool with_parenthesis, const T &arg) {
|
||||
PrintObject(out, arg);
|
||||
if (with_parenthesis) {
|
||||
*out << ")";
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
void PrintOperatorArgs(const std::string &name, std::ostream *out, bool with_parenthesis, const T &arg,
|
||||
const Ts &...args) {
|
||||
PrintObject(out, arg);
|
||||
*out << " " << name << " ";
|
||||
PrintOperatorArgs(name, out, with_parenthesis, args...);
|
||||
}
|
||||
|
||||
template <typename... Ts>
|
||||
void PrintOperator(const std::string &name, std::ostream *out, bool with_parenthesis, const Ts &...args) {
|
||||
if (with_parenthesis) {
|
||||
*out << "(";
|
||||
}
|
||||
PrintOperatorArgs(name, out, with_parenthesis, args...);
|
||||
}
|
||||
|
||||
// new
|
||||
template <typename T>
|
||||
void PrintOperatorArgs(std::ostream *out, const T &arg) {
|
||||
PrintObject(out, arg);
|
||||
}
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
|
||||
PrintObject(out, arg);
|
||||
PrintOperatorArgs(out, args...);
|
||||
}
|
||||
|
||||
template <typename... Ts>
|
||||
void PrintOperator(std::ostream *out, const Ts &...args) {
|
||||
PrintOperatorArgs(out, args...);
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
|
||||
public:
|
||||
explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
|
||||
|
||||
// Unary operators
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE &op) override { detail::PrintOperator(OP_STR, out_, false /*with_parenthesis*/, op.expression_); }
|
||||
|
||||
UNARY_OPERATOR_VISIT(NotOperator, "Not");
|
||||
UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
|
||||
UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
|
||||
UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
|
||||
|
||||
#undef UNARY_OPERATOR_VISIT
|
||||
|
||||
// Binary operators
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE &op) override { \
|
||||
detail::PrintOperator(OP_STR, out_, true /*with_parenthesis*/, op.expression1_, op.expression2_); \
|
||||
}
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define BINARY_OPERATOR_VISIT_NOT_IMPL(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE & /*op*/) override { throw utils::NotYetImplemented("OP_NODE"); }
|
||||
|
||||
BINARY_OPERATOR_VISIT(OrOperator, "Or");
|
||||
BINARY_OPERATOR_VISIT(XorOperator, "Xor");
|
||||
BINARY_OPERATOR_VISIT(AndOperator, "And");
|
||||
BINARY_OPERATOR_VISIT(AdditionOperator, "+");
|
||||
BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
|
||||
BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
|
||||
BINARY_OPERATOR_VISIT(DivisionOperator, "/");
|
||||
BINARY_OPERATOR_VISIT(ModOperator, "%");
|
||||
BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
|
||||
BINARY_OPERATOR_VISIT(EqualOperator, "=");
|
||||
BINARY_OPERATOR_VISIT(LessOperator, "<");
|
||||
BINARY_OPERATOR_VISIT(GreaterOperator, ">");
|
||||
BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
|
||||
BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
|
||||
BINARY_OPERATOR_VISIT_NOT_IMPL(InListOperator, "In");
|
||||
BINARY_OPERATOR_VISIT_NOT_IMPL(SubscriptOperator, "Subscript");
|
||||
|
||||
#undef BINARY_OPERATOR_VISIT
|
||||
#undef BINARY_OPERATOR_VISIT_NOT_IMPL
|
||||
|
||||
// Other
|
||||
void Visit(ListSlicingOperator & /*op*/) override { throw utils::NotYetImplemented("ListSlicingOperator"); }
|
||||
|
||||
void Visit(IfOperator & /*op*/) override { throw utils::NotYetImplemented("IfOperator"); }
|
||||
|
||||
void Visit(ListLiteral & /*op*/) override { throw utils::NotYetImplemented("ListLiteral"); }
|
||||
|
||||
void Visit(MapLiteral & /*op*/) override { throw utils::NotYetImplemented("MapLiteral"); }
|
||||
|
||||
void Visit(LabelsTest & /*op*/) override { throw utils::NotYetImplemented("LabelsTest"); }
|
||||
|
||||
void Visit(Aggregation & /*op*/) override { throw utils::NotYetImplemented("Aggregation"); }
|
||||
|
||||
void Visit(Function & /*op*/) override { throw utils::NotYetImplemented("Function"); }
|
||||
|
||||
void Visit(Reduce & /*op*/) override { throw utils::NotYetImplemented("Reduce"); }
|
||||
|
||||
void Visit(Coalesce & /*op*/) override { throw utils::NotYetImplemented("Coalesce"); }
|
||||
|
||||
void Visit(Extract & /*op*/) override { throw utils::NotYetImplemented("Extract"); }
|
||||
|
||||
void Visit(All & /*op*/) override { throw utils::NotYetImplemented("All"); }
|
||||
|
||||
void Visit(Single & /*op*/) override { throw utils::NotYetImplemented("Single"); }
|
||||
|
||||
void Visit(Any & /*op*/) override { throw utils::NotYetImplemented("Any"); }
|
||||
|
||||
void Visit(None & /*op*/) override { throw utils::NotYetImplemented("None"); }
|
||||
|
||||
void Visit(Identifier &op) override {
|
||||
auto is_node = true;
|
||||
auto is_edge = false;
|
||||
auto is_other = false;
|
||||
if (is_node) {
|
||||
detail::PrintOperator(out_, identifier_node_symbol);
|
||||
} else if (is_edge) {
|
||||
detail::PrintOperator(out_, identifier_edge_symbol);
|
||||
} else {
|
||||
MG_ASSERT(is_other);
|
||||
detail::PrintOperator(out_, op.name_);
|
||||
}
|
||||
}
|
||||
|
||||
void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
|
||||
|
||||
void Visit(PropertyLookup &op) override { detail::PrintOperator(out_, op.expression_, ".", op.property_.name); }
|
||||
|
||||
void Visit(ParameterLookup & /*op*/) override { throw utils::NotYetImplemented("ParameterLookup"); }
|
||||
|
||||
void Visit(NamedExpression & /*op*/) override { throw utils::NotYetImplemented("NamedExpression"); }
|
||||
|
||||
void Visit(RegexMatch & /*op*/) override { throw utils::NotYetImplemented("RegexMatch"); }
|
||||
|
||||
private:
|
||||
std::ostream *out_;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
inline void PrintObject(std::ostream *out, Expression *expr) {
|
||||
if (expr) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
} else {
|
||||
*out << "<null>";
|
||||
}
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline void PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(Expression *expr, std::ostream *out) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
}
|
||||
|
||||
inline std::string ExpressiontoStringWhileReplacingNodeAndEdgeSymbols(Expression *expr) {
|
||||
std::ostringstream ss;
|
||||
expr::PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(expr, &ss);
|
||||
return ss.str();
|
||||
}
|
||||
} // namespace memgraph::expr
|
||||
52
src/expr/exceptions.hpp
Normal file
52
src/expr/exceptions.hpp
Normal file
@@ -0,0 +1,52 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
class SyntaxException : public utils::BasicException {
|
||||
public:
|
||||
using utils::BasicException::BasicException;
|
||||
SyntaxException() : SyntaxException("") {}
|
||||
};
|
||||
|
||||
class SemanticException : public utils::BasicException {
|
||||
public:
|
||||
using utils::BasicException::BasicException;
|
||||
SemanticException() : BasicException("") {}
|
||||
};
|
||||
|
||||
class ExpressionRuntimeException : public utils::BasicException {
|
||||
public:
|
||||
using utils::BasicException::BasicException;
|
||||
};
|
||||
|
||||
class RedeclareVariableError : public SemanticException {
|
||||
public:
|
||||
explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {}
|
||||
};
|
||||
|
||||
class UnboundVariableError : public SemanticException {
|
||||
public:
|
||||
explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {}
|
||||
};
|
||||
|
||||
class TypeMismatchError : public SemanticException {
|
||||
public:
|
||||
TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected)
|
||||
: SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) {
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::expr
|
||||
829
src/expr/interpret/eval.hpp
Normal file
829
src/expr/interpret/eval.hpp
Normal file
@@ -0,0 +1,829 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
/// @file
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <regex>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
|
||||
#include "expr/ast.hpp"
|
||||
#include "expr/exceptions.hpp"
|
||||
#include "expr/interpret/frame.hpp"
|
||||
#include "expr/semantic/symbol_table.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
struct StorageTag {};
|
||||
struct QueryEngineTag {};
|
||||
|
||||
template <typename TypedValue, typename EvaluationContext, typename DbAccessor, typename StorageView, typename LabelId,
|
||||
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
|
||||
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
public:
|
||||
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
|
||||
DbAccessor *dba, StorageView view)
|
||||
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
|
||||
|
||||
using ExpressionVisitor<TypedValue>::Visit;
|
||||
|
||||
utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; }
|
||||
|
||||
TypedValue Visit(NamedExpression &named_expression) override {
|
||||
const auto &symbol = symbol_table_->at(named_expression);
|
||||
auto value = named_expression.expression_->Accept(*this);
|
||||
frame_->at(symbol) = value;
|
||||
return value;
|
||||
}
|
||||
|
||||
TypedValue Visit(Identifier &ident) override {
|
||||
return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define BINARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
TypedValue Visit(OP_NODE &op) override { \
|
||||
auto val1 = op.expression1_->Accept(*this); \
|
||||
auto val2 = op.expression2_->Accept(*this); \
|
||||
try { \
|
||||
return val1 CPP_OP val2; \
|
||||
} catch (const TypedValueException &) { \
|
||||
throw ExpressionRuntimeException("Invalid types: {} and {} for '{}'.", val1.type(), val2.type(), #CYPHER_OP); \
|
||||
} \
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define UNARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
TypedValue Visit(OP_NODE &op) override { \
|
||||
auto val = op.expression_->Accept(*this); \
|
||||
try { \
|
||||
return CPP_OP val; \
|
||||
} catch (const TypedValueException &) { \
|
||||
throw ExpressionRuntimeException("Invalid type {} for '{}'.", val.type(), #CYPHER_OP); \
|
||||
} \
|
||||
}
|
||||
|
||||
BINARY_OPERATOR_VISITOR(OrOperator, ||, OR);
|
||||
BINARY_OPERATOR_VISITOR(XorOperator, ^, XOR);
|
||||
BINARY_OPERATOR_VISITOR(AdditionOperator, +, +);
|
||||
BINARY_OPERATOR_VISITOR(SubtractionOperator, -, -);
|
||||
BINARY_OPERATOR_VISITOR(MultiplicationOperator, *, *);
|
||||
BINARY_OPERATOR_VISITOR(DivisionOperator, /, /);
|
||||
BINARY_OPERATOR_VISITOR(ModOperator, %, %);
|
||||
BINARY_OPERATOR_VISITOR(NotEqualOperator, !=, <>);
|
||||
BINARY_OPERATOR_VISITOR(EqualOperator, ==, =);
|
||||
BINARY_OPERATOR_VISITOR(LessOperator, <, <);
|
||||
BINARY_OPERATOR_VISITOR(GreaterOperator, >, >);
|
||||
BINARY_OPERATOR_VISITOR(LessEqualOperator, <=, <=);
|
||||
BINARY_OPERATOR_VISITOR(GreaterEqualOperator, >=, >=);
|
||||
|
||||
UNARY_OPERATOR_VISITOR(NotOperator, !, NOT);
|
||||
UNARY_OPERATOR_VISITOR(UnaryPlusOperator, +, +);
|
||||
UNARY_OPERATOR_VISITOR(UnaryMinusOperator, -, -);
|
||||
|
||||
#undef BINARY_OPERATOR_VISITOR
|
||||
#undef UNARY_OPERATOR_VISITOR
|
||||
|
||||
TypedValue Visit(AndOperator &op) override {
|
||||
auto value1 = op.expression1_->Accept(*this);
|
||||
if (value1.IsBool() && !value1.ValueBool()) {
|
||||
// If first expression is false, don't evaluate the second one.
|
||||
return value1;
|
||||
}
|
||||
auto value2 = op.expression2_->Accept(*this);
|
||||
try {
|
||||
return value1 && value2;
|
||||
} catch (const TypedValueException &) {
|
||||
throw ExpressionRuntimeException("Invalid types: {} and {} for AND.", value1.type(), value2.type());
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(IfOperator &if_operator) override {
|
||||
auto condition = if_operator.condition_->Accept(*this);
|
||||
if (condition.IsNull()) {
|
||||
return if_operator.else_expression_->Accept(*this);
|
||||
}
|
||||
if (condition.type() != TypedValue::Type::Bool) {
|
||||
// At the moment IfOperator is used only in CASE construct.
|
||||
throw ExpressionRuntimeException("CASE expected boolean expression, got {}.", condition.type());
|
||||
}
|
||||
if (condition.ValueBool()) {
|
||||
return if_operator.then_expression_->Accept(*this);
|
||||
}
|
||||
return if_operator.else_expression_->Accept(*this);
|
||||
}
|
||||
|
||||
TypedValue Visit(InListOperator &in_list) override {
|
||||
auto literal = in_list.expression1_->Accept(*this);
|
||||
auto _list = in_list.expression2_->Accept(*this);
|
||||
if (_list.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
// Exceptions have higher priority than returning nulls when list expression
|
||||
// is not null.
|
||||
if (_list.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("IN expected a list, got {}.", _list.type());
|
||||
}
|
||||
const auto &list = _list.ValueList();
|
||||
|
||||
// If literal is NULL there is no need to try to compare it with every
|
||||
// element in the list since result of every comparison will be NULL. There
|
||||
// is one special case that we must test explicitly: if list is empty then
|
||||
// result is false since no comparison will be performed.
|
||||
if (list.empty()) return TypedValue(false, ctx_->memory);
|
||||
if (literal.IsNull()) return TypedValue(ctx_->memory);
|
||||
|
||||
auto has_null = false;
|
||||
for (const auto &element : list) {
|
||||
auto result = literal == element;
|
||||
if (result.IsNull()) {
|
||||
has_null = true;
|
||||
} else if (result.ValueBool()) {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
}
|
||||
if (has_null) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
return TypedValue(false, ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(SubscriptOperator &list_indexing) override {
|
||||
auto lhs = list_indexing.expression1_->Accept(*this);
|
||||
auto index = list_indexing.expression2_->Accept(*this);
|
||||
if (!lhs.IsList() && !lhs.IsMap() && !lhs.IsVertex() && !lhs.IsEdge() && !lhs.IsNull())
|
||||
throw ExpressionRuntimeException(
|
||||
"Expected a list, a map, a node or an edge to index with '[]', got "
|
||||
"{}.",
|
||||
lhs.type());
|
||||
if (lhs.IsNull() || index.IsNull()) return TypedValue(ctx_->memory);
|
||||
if (lhs.IsList()) {
|
||||
if (!index.IsInt())
|
||||
throw ExpressionRuntimeException("Expected an integer as a list index, got {}.", index.type());
|
||||
auto index_int = index.ValueInt();
|
||||
// NOTE: Take non-const reference to list, so that we can move out the
|
||||
// indexed element as the result.
|
||||
auto &list = lhs.ValueList();
|
||||
if (index_int < 0) {
|
||||
index_int += static_cast<int64_t>(list.size());
|
||||
}
|
||||
if (index_int >= static_cast<int64_t>(list.size()) || index_int < 0) return TypedValue(ctx_->memory);
|
||||
// NOTE: Explicit move is needed, so that we return the move constructed
|
||||
// value and preserve the correct MemoryResource.
|
||||
return std::move(list[index_int]);
|
||||
}
|
||||
|
||||
if (lhs.IsMap()) {
|
||||
if (!index.IsString())
|
||||
throw ExpressionRuntimeException("Expected a string as a map index, got {}.", index.type());
|
||||
// NOTE: Take non-const reference to map, so that we can move out the
|
||||
// looked-up element as the result.
|
||||
auto &map = lhs.ValueMap();
|
||||
auto found = map.find(index.ValueString());
|
||||
if (found == map.end()) return TypedValue(ctx_->memory);
|
||||
// NOTE: Explicit move is needed, so that we return the move constructed
|
||||
// value and preserve the correct MemoryResource.
|
||||
return std::move(found->second);
|
||||
}
|
||||
|
||||
if (lhs.IsVertex()) {
|
||||
if (!index.IsString())
|
||||
throw ExpressionRuntimeException("Expected a string as a property name, got {}.", index.type());
|
||||
return TypedValue(GetProperty(lhs.ValueVertex(), index.ValueString()), ctx_->memory);
|
||||
}
|
||||
|
||||
if (lhs.IsEdge()) {
|
||||
if (!index.IsString())
|
||||
throw ExpressionRuntimeException("Expected a string as a property name, got {}.", index.type());
|
||||
return TypedValue(GetProperty(lhs.ValueEdge(), index.ValueString()), ctx_->memory);
|
||||
}
|
||||
|
||||
// lhs is Null
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(ListSlicingOperator &op) override {
|
||||
// If some type is null we can't return null, because throwing exception
|
||||
// on illegal type has higher priority.
|
||||
auto is_null = false;
|
||||
auto get_bound = [&](Expression *bound_expr, int64_t default_value) {
|
||||
if (bound_expr) {
|
||||
auto bound = bound_expr->Accept(*this);
|
||||
if (bound.type() == TypedValue::Type::Null) {
|
||||
is_null = true;
|
||||
} else if (bound.type() != TypedValue::Type::Int) {
|
||||
throw ExpressionRuntimeException("Expected an integer for a bound in list slicing, got {}.", bound.type());
|
||||
}
|
||||
return bound;
|
||||
}
|
||||
return TypedValue(default_value, ctx_->memory);
|
||||
};
|
||||
auto _upper_bound = get_bound(op.upper_bound_, std::numeric_limits<int64_t>::max());
|
||||
auto _lower_bound = get_bound(op.lower_bound_, 0);
|
||||
|
||||
auto _list = op.list_->Accept(*this);
|
||||
if (_list.type() == TypedValue::Type::Null) {
|
||||
is_null = true;
|
||||
} else if (_list.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("Expected a list to slice, got {}.", _list.type());
|
||||
}
|
||||
|
||||
if (is_null) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
const auto &list = _list.ValueList();
|
||||
auto normalise_bound = [&](int64_t bound) {
|
||||
if (bound < 0) {
|
||||
bound = static_cast<int64_t>(list.size()) + bound;
|
||||
}
|
||||
return std::max(static_cast<int64_t>(0), std::min(bound, static_cast<int64_t>(list.size())));
|
||||
};
|
||||
auto lower_bound = normalise_bound(_lower_bound.ValueInt());
|
||||
auto upper_bound = normalise_bound(_upper_bound.ValueInt());
|
||||
if (upper_bound <= lower_bound) {
|
||||
return TypedValue(typename TypedValue::TVector(ctx_->memory), ctx_->memory);
|
||||
}
|
||||
return TypedValue(
|
||||
typename TypedValue::TVector(list.begin() + lower_bound, list.begin() + upper_bound, ctx_->memory));
|
||||
}
|
||||
|
||||
TypedValue Visit(IsNullOperator &is_null) override {
|
||||
auto value = is_null.expression_->Accept(*this);
|
||||
return TypedValue(value.IsNull(), ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(PropertyLookup &property_lookup) override {
|
||||
auto expression_result = property_lookup.expression_->Accept(*this);
|
||||
auto maybe_date = [this](const auto &date, const auto &prop_name) -> std::optional<TypedValue> {
|
||||
if (prop_name == "year") {
|
||||
return TypedValue(date.year, ctx_->memory);
|
||||
}
|
||||
if (prop_name == "month") {
|
||||
return TypedValue(date.month, ctx_->memory);
|
||||
}
|
||||
if (prop_name == "day") {
|
||||
return TypedValue(date.day, ctx_->memory);
|
||||
}
|
||||
return std::nullopt;
|
||||
};
|
||||
auto maybe_local_time = [this](const auto <, const auto &prop_name) -> std::optional<TypedValue> {
|
||||
if (prop_name == "hour") {
|
||||
return TypedValue(lt.hour, ctx_->memory);
|
||||
}
|
||||
if (prop_name == "minute") {
|
||||
return TypedValue(lt.minute, ctx_->memory);
|
||||
}
|
||||
if (prop_name == "second") {
|
||||
return TypedValue(lt.second, ctx_->memory);
|
||||
}
|
||||
if (prop_name == "millisecond") {
|
||||
return TypedValue(lt.millisecond, ctx_->memory);
|
||||
}
|
||||
if (prop_name == "microsecond") {
|
||||
return TypedValue(lt.microsecond, ctx_->memory);
|
||||
}
|
||||
return std::nullopt;
|
||||
};
|
||||
auto maybe_duration = [this](const auto &dur, const auto &prop_name) -> std::optional<TypedValue> {
|
||||
if (prop_name == "day") {
|
||||
return TypedValue(dur.Days(), ctx_->memory);
|
||||
}
|
||||
if (prop_name == "hour") {
|
||||
return TypedValue(dur.SubDaysAsHours(), ctx_->memory);
|
||||
}
|
||||
if (prop_name == "minute") {
|
||||
return TypedValue(dur.SubDaysAsMinutes(), ctx_->memory);
|
||||
}
|
||||
if (prop_name == "second") {
|
||||
return TypedValue(dur.SubDaysAsSeconds(), ctx_->memory);
|
||||
}
|
||||
if (prop_name == "millisecond") {
|
||||
return TypedValue(dur.SubDaysAsMilliseconds(), ctx_->memory);
|
||||
}
|
||||
if (prop_name == "microsecond") {
|
||||
return TypedValue(dur.SubDaysAsMicroseconds(), ctx_->memory);
|
||||
}
|
||||
if (prop_name == "nanosecond") {
|
||||
return TypedValue(dur.SubDaysAsNanoseconds(), ctx_->memory);
|
||||
}
|
||||
return std::nullopt;
|
||||
};
|
||||
switch (expression_result.type()) {
|
||||
case TypedValue::Type::Null:
|
||||
return TypedValue(ctx_->memory);
|
||||
case TypedValue::Type::Vertex:
|
||||
return GetProperty(expression_result.ValueVertex(), property_lookup.property_);
|
||||
case TypedValue::Type::Edge:
|
||||
return GetProperty(expression_result.ValueEdge(), property_lookup.property_);
|
||||
case TypedValue::Type::Map: {
|
||||
// NOTE: Take non-const reference to map, so that we can move out the
|
||||
// looked-up element as the result.
|
||||
auto &map = expression_result.ValueMap();
|
||||
auto found = map.find(property_lookup.property_.name.c_str());
|
||||
if (found == map.end()) return TypedValue(ctx_->memory);
|
||||
// NOTE: Explicit move is needed, so that we return the move constructed
|
||||
// value and preserve the correct MemoryResource.
|
||||
return std::move(found->second);
|
||||
}
|
||||
case TypedValue::Type::Duration: {
|
||||
const auto &prop_name = property_lookup.property_.name;
|
||||
const auto &dur = expression_result.ValueDuration();
|
||||
if (auto dur_field = maybe_duration(dur, prop_name); dur_field) {
|
||||
return std::move(*dur_field);
|
||||
}
|
||||
throw ExpressionRuntimeException("Invalid property name {} for Duration", prop_name);
|
||||
}
|
||||
case TypedValue::Type::Date: {
|
||||
const auto &prop_name = property_lookup.property_.name;
|
||||
const auto &date = expression_result.ValueDate();
|
||||
if (auto date_field = maybe_date(date, prop_name); date_field) {
|
||||
return std::move(*date_field);
|
||||
}
|
||||
throw ExpressionRuntimeException("Invalid property name {} for Date", prop_name);
|
||||
}
|
||||
case TypedValue::Type::LocalTime: {
|
||||
const auto &prop_name = property_lookup.property_.name;
|
||||
const auto < = expression_result.ValueLocalTime();
|
||||
if (auto lt_field = maybe_local_time(lt, prop_name); lt_field) {
|
||||
return std::move(*lt_field);
|
||||
}
|
||||
throw ExpressionRuntimeException("Invalid property name {} for LocalTime", prop_name);
|
||||
}
|
||||
case TypedValue::Type::LocalDateTime: {
|
||||
const auto &prop_name = property_lookup.property_.name;
|
||||
const auto &ldt = expression_result.ValueLocalDateTime();
|
||||
if (auto date_field = maybe_date(ldt.date, prop_name); date_field) {
|
||||
return std::move(*date_field);
|
||||
}
|
||||
if (auto lt_field = maybe_local_time(ldt.local_time, prop_name); lt_field) {
|
||||
return std::move(*lt_field);
|
||||
}
|
||||
throw ExpressionRuntimeException("Invalid property name {} for LocalDateTime", prop_name);
|
||||
}
|
||||
default:
|
||||
throw ExpressionRuntimeException("Only nodes, edges, maps and temporal types have properties to be looked-up.");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename VertexAccessor, typename TTag = Tag,
|
||||
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, bool>>
|
||||
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label, StorageTag /*tag*/) {
|
||||
auto has_label = vertex.HasLabel(view_, GetLabel(label));
|
||||
if (has_label.HasError() && has_label.GetError() == Error::NONEXISTENT_OBJECT) {
|
||||
// This is a very nasty and temporary hack in order to make MERGE
|
||||
// work. The old storage had the following logic when returning an
|
||||
// `OLD` view: `return old ? old : new`. That means that if the
|
||||
// `OLD` view didn't exist, it returned the NEW view. With this hack
|
||||
// we simulate that behavior.
|
||||
// TODO (mferencevic, teon.banek): Remove once MERGE is
|
||||
// reimplemented.
|
||||
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
|
||||
}
|
||||
if (has_label.HasError()) {
|
||||
switch (has_label.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
|
||||
}
|
||||
}
|
||||
return *has_label;
|
||||
}
|
||||
|
||||
template <typename VertexAccessor, typename TTag = Tag,
|
||||
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
|
||||
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
|
||||
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
|
||||
auto has_label = vertex.HasLabel(label);
|
||||
return !has_label;
|
||||
}
|
||||
|
||||
TypedValue Visit(LabelsTest &labels_test) override {
|
||||
auto expression_result = labels_test.expression_->Accept(*this);
|
||||
switch (expression_result.type()) {
|
||||
case TypedValue::Type::Null:
|
||||
return TypedValue(ctx_->memory);
|
||||
case TypedValue::Type::Vertex: {
|
||||
const auto &vertex = expression_result.ValueVertex();
|
||||
if (std::ranges::all_of(labels_test.labels_, [&vertex, this](const auto label_test) {
|
||||
return this->HasLabelImpl(vertex, label_test, Tag{});
|
||||
})) {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
return TypedValue(false, ctx_->memory);
|
||||
}
|
||||
default:
|
||||
throw ExpressionRuntimeException("Only nodes have labels.");
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(PrimitiveLiteral &literal) override {
|
||||
// TODO: no need to evaluate constants, we can write it to frame in one
|
||||
// of the previous phases.
|
||||
return TypedValue(literal.value_, ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(ListLiteral &literal) override {
|
||||
typename TypedValue::TVector result(ctx_->memory);
|
||||
result.reserve(literal.elements_.size());
|
||||
for (const auto &expression : literal.elements_) result.emplace_back(expression->Accept(*this));
|
||||
return TypedValue(result, ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(MapLiteral &literal) override {
|
||||
typename TypedValue::TMap result(ctx_->memory);
|
||||
for (const auto &pair : literal.elements_) result.emplace(pair.first.name, pair.second->Accept(*this));
|
||||
return TypedValue(result, ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(Aggregation &aggregation) override {
|
||||
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(Coalesce &coalesce) override {
|
||||
auto &exprs = coalesce.expressions_;
|
||||
|
||||
if (exprs.size() == 0) {
|
||||
throw ExpressionRuntimeException("'coalesce' requires at least one argument.");
|
||||
}
|
||||
|
||||
for (int64_t i = 0; i < exprs.size(); ++i) {
|
||||
TypedValue val(exprs[i]->Accept(*this), ctx_->memory);
|
||||
if (!val.IsNull()) {
|
||||
return val;
|
||||
}
|
||||
}
|
||||
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(Function &function) override {
|
||||
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
|
||||
// Stack allocate evaluated arguments when there's a small number of them.
|
||||
if (function.arguments_.size() <= 8) {
|
||||
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
|
||||
TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
|
||||
TypedValue(ctx_->memory), TypedValue(ctx_->memory)};
|
||||
for (size_t i = 0; i < function.arguments_.size(); ++i) {
|
||||
arguments[i] = function.arguments_[i]->Accept(*this);
|
||||
}
|
||||
auto res = function.function_(arguments, function.arguments_.size(), function_ctx);
|
||||
MG_ASSERT(res.GetMemoryResource() == ctx_->memory);
|
||||
return res;
|
||||
} else {
|
||||
typename TypedValue::TVector arguments(ctx_->memory);
|
||||
arguments.reserve(function.arguments_.size());
|
||||
for (const auto &argument : function.arguments_) {
|
||||
arguments.emplace_back(argument->Accept(*this));
|
||||
}
|
||||
auto res = function.function_(arguments.data(), arguments.size(), function_ctx);
|
||||
MG_ASSERT(res.GetMemoryResource() == ctx_->memory);
|
||||
return res;
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(Reduce &reduce) override {
|
||||
auto list_value = reduce.list_->Accept(*this);
|
||||
if (list_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (list_value.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("REDUCE expected a list, got {}.", list_value.type());
|
||||
}
|
||||
const auto &list = list_value.ValueList();
|
||||
const auto &element_symbol = symbol_table_->at(*reduce.identifier_);
|
||||
const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
|
||||
auto accumulator = reduce.initializer_->Accept(*this);
|
||||
for (const auto &element : list) {
|
||||
frame_->at(accumulator_symbol) = accumulator;
|
||||
frame_->at(element_symbol) = element;
|
||||
accumulator = reduce.expression_->Accept(*this);
|
||||
}
|
||||
return accumulator;
|
||||
}
|
||||
|
||||
TypedValue Visit(Extract &extract) override {
|
||||
auto list_value = extract.list_->Accept(*this);
|
||||
if (list_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (list_value.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("EXTRACT expected a list, got {}.", list_value.type());
|
||||
}
|
||||
const auto &list = list_value.ValueList();
|
||||
const auto &element_symbol = symbol_table_->at(*extract.identifier_);
|
||||
typename TypedValue::TVector result(ctx_->memory);
|
||||
result.reserve(list.size());
|
||||
for (const auto &element : list) {
|
||||
if (element.IsNull()) {
|
||||
result.emplace_back();
|
||||
} else {
|
||||
frame_->at(element_symbol) = element;
|
||||
result.emplace_back(extract.expression_->Accept(*this));
|
||||
}
|
||||
}
|
||||
return TypedValue(result, ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(All &all) override {
|
||||
auto list_value = all.list_expression_->Accept(*this);
|
||||
if (list_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (list_value.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("ALL expected a list, got {}.", list_value.type());
|
||||
}
|
||||
const auto &list = list_value.ValueList();
|
||||
const auto &symbol = symbol_table_->at(*all.identifier_);
|
||||
bool has_null_elements = false;
|
||||
bool has_value = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->at(symbol) = element;
|
||||
auto result = all.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
|
||||
}
|
||||
if (!result.IsNull()) {
|
||||
has_value = true;
|
||||
if (!result.ValueBool()) {
|
||||
return TypedValue(false, ctx_->memory);
|
||||
}
|
||||
} else {
|
||||
has_null_elements = true;
|
||||
}
|
||||
}
|
||||
if (!has_value) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (has_null_elements) {
|
||||
return TypedValue(false, ctx_->memory);
|
||||
} else {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(Single &single) override {
|
||||
auto list_value = single.list_expression_->Accept(*this);
|
||||
if (list_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (list_value.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("SINGLE expected a list, got {}.", list_value.type());
|
||||
}
|
||||
const auto &list = list_value.ValueList();
|
||||
const auto &symbol = symbol_table_->at(*single.identifier_);
|
||||
bool has_value = false;
|
||||
bool predicate_satisfied = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->at(symbol) = element;
|
||||
auto result = single.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
|
||||
}
|
||||
if (result.type() == TypedValue::Type::Bool) {
|
||||
has_value = true;
|
||||
}
|
||||
if (result.IsNull() || !result.ValueBool()) {
|
||||
continue;
|
||||
}
|
||||
// Return false if more than one element satisfies the predicate.
|
||||
if (predicate_satisfied) {
|
||||
return TypedValue(false, ctx_->memory);
|
||||
} else {
|
||||
predicate_satisfied = true;
|
||||
}
|
||||
}
|
||||
if (!has_value) {
|
||||
return TypedValue(ctx_->memory);
|
||||
} else {
|
||||
return TypedValue(predicate_satisfied, ctx_->memory);
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(Any &any) override {
|
||||
auto list_value = any.list_expression_->Accept(*this);
|
||||
if (list_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (list_value.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("ANY expected a list, got {}.", list_value.type());
|
||||
}
|
||||
const auto &list = list_value.ValueList();
|
||||
const auto &symbol = symbol_table_->at(*any.identifier_);
|
||||
bool has_value = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->at(symbol) = element;
|
||||
auto result = any.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
|
||||
}
|
||||
if (!result.IsNull()) {
|
||||
has_value = true;
|
||||
if (result.ValueBool()) {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Return Null if all elements are Null
|
||||
if (!has_value) {
|
||||
return TypedValue(ctx_->memory);
|
||||
} else {
|
||||
return TypedValue(false, ctx_->memory);
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(None &none) override {
|
||||
auto list_value = none.list_expression_->Accept(*this);
|
||||
if (list_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (list_value.type() != TypedValue::Type::List) {
|
||||
throw ExpressionRuntimeException("NONE expected a list, got {}.", list_value.type());
|
||||
}
|
||||
const auto &list = list_value.ValueList();
|
||||
const auto &symbol = symbol_table_->at(*none.identifier_);
|
||||
bool has_value = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->at(symbol) = element;
|
||||
auto result = none.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
|
||||
}
|
||||
if (!result.IsNull()) {
|
||||
has_value = true;
|
||||
if (result.ValueBool()) {
|
||||
return TypedValue(false, ctx_->memory);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Return Null if all elements are Null
|
||||
if (!has_value) {
|
||||
return TypedValue(ctx_->memory);
|
||||
} else {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(ParameterLookup ¶m_lookup) override {
|
||||
return TypedValue(conv_(ctx_->parameters.AtTokenPosition(param_lookup.token_position_)), ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(RegexMatch ®ex_match) override {
|
||||
auto target_string_value = regex_match.string_expr_->Accept(*this);
|
||||
auto regex_value = regex_match.regex_->Accept(*this);
|
||||
if (target_string_value.IsNull() || regex_value.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
if (regex_value.type() != TypedValue::Type::String) {
|
||||
throw ExpressionRuntimeException("Regular expression must evaluate to a string, got {}.", regex_value.type());
|
||||
}
|
||||
if (target_string_value.type() != TypedValue::Type::String) {
|
||||
// Instead of error, we return Null which makes it compatible in case we
|
||||
// use indexed lookup which filters out any non-string properties.
|
||||
// Assuming a property lookup is the target_string_value.
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
const auto &target_string = target_string_value.ValueString();
|
||||
try {
|
||||
std::regex regex(regex_value.ValueString());
|
||||
return TypedValue(std::regex_match(target_string, regex), ctx_->memory);
|
||||
} catch (const std::regex_error &e) {
|
||||
throw ExpressionRuntimeException("Regex error in '{}': {}", regex_value.ValueString(), e.what());
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <class TRecordAccessor, class TTag = Tag,
|
||||
class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
|
||||
TReturnType GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
|
||||
auto maybe_prop = record_accessor.GetProperty(prop.name);
|
||||
// Handler non existent property
|
||||
return conv_(maybe_prop, dba_);
|
||||
}
|
||||
|
||||
template <class TRecordAccessor, class TTag = Tag,
|
||||
class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
|
||||
TReturnType GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
|
||||
auto maybe_prop = record_accessor.GetProperty(std::string(name));
|
||||
// Handler non existent property
|
||||
return conv_(maybe_prop, dba_);
|
||||
}
|
||||
|
||||
template <class TRecordAccessor, class TTag = Tag,
|
||||
class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
|
||||
TypedValue GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
|
||||
auto maybe_prop = record_accessor.GetProperty(view_, ctx_->properties[prop.ix]);
|
||||
if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
|
||||
// This is a very nasty and temporary hack in order to make MERGE work.
|
||||
// The old storage had the following logic when returning an `OLD` view:
|
||||
// `return old ? old : new`. That means that if the `OLD` view didn't
|
||||
// exist, it returned the NEW view. With this hack we simulate that
|
||||
// behavior.
|
||||
// TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented.
|
||||
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
|
||||
}
|
||||
if (maybe_prop.HasError()) {
|
||||
switch (maybe_prop.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
throw ExpressionRuntimeException("Unexpected error when getting a property.");
|
||||
}
|
||||
}
|
||||
return conv_(*maybe_prop, ctx_->memory);
|
||||
}
|
||||
|
||||
template <class TRecordAccessor, class TTag = Tag,
|
||||
class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
|
||||
TypedValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
|
||||
auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
|
||||
if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
|
||||
// This is a very nasty and temporary hack in order to make MERGE work.
|
||||
// The old storage had the following logic when returning an `OLD` view:
|
||||
// `return old ? old : new`. That means that if the `OLD` view didn't
|
||||
// exist, it returned the NEW view. With this hack we simulate that
|
||||
// behavior.
|
||||
// TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented.
|
||||
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
|
||||
}
|
||||
if (maybe_prop.HasError()) {
|
||||
switch (maybe_prop.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
throw ExpressionRuntimeException("Unexpected error when getting a property.");
|
||||
}
|
||||
}
|
||||
return conv_(*maybe_prop, ctx_->memory);
|
||||
}
|
||||
|
||||
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
|
||||
|
||||
Frame<TypedValue> *frame_;
|
||||
const SymbolTable *symbol_table_;
|
||||
const EvaluationContext *ctx_;
|
||||
DbAccessor *dba_;
|
||||
// which switching approach should be used when evaluating
|
||||
StorageView view_;
|
||||
ConvFunctor conv_;
|
||||
};
|
||||
|
||||
/// A helper function for evaluating an expression that's an int.
|
||||
///
|
||||
/// @param what - Name of what's getting evaluated. Used for user feedback (via
|
||||
/// exception) when the evaluated value is not an int.
|
||||
/// @throw ExpressionRuntimeException if expression doesn't evaluate to an int.
|
||||
template <typename ExpressionEvaluator>
|
||||
int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what) {
|
||||
TypedValue value = expr->Accept(*evaluator);
|
||||
try {
|
||||
return value.ValueInt();
|
||||
} catch (TypedValueException &e) {
|
||||
throw ExpressionRuntimeException(what + " must be an int");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ExpressionEvaluator>
|
||||
std::optional<size_t> EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale) {
|
||||
if (!memory_limit) return std::nullopt;
|
||||
auto limit_value = memory_limit->Accept(*eval);
|
||||
if (!limit_value.IsInt() || limit_value.ValueInt() <= 0)
|
||||
throw ExpressionRuntimeException("Memory limit must be a non-negative integer.");
|
||||
size_t limit = limit_value.ValueInt();
|
||||
if (std::numeric_limits<size_t>::max() / memory_scale < limit)
|
||||
throw ExpressionRuntimeException("Memory limit overflow.");
|
||||
return limit * memory_scale;
|
||||
}
|
||||
|
||||
} // namespace memgraph::expr
|
||||
45
src/expr/interpret/frame.hpp
Normal file
45
src/expr/interpret/frame.hpp
Normal file
@@ -0,0 +1,45 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "expr/semantic/symbol_table.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/pmr/vector.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
template <typename TypedValue>
|
||||
class Frame {
|
||||
public:
|
||||
/// Create a Frame of given size backed by a utils::NewDeleteResource()
|
||||
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
|
||||
|
||||
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
|
||||
|
||||
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
|
||||
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
|
||||
|
||||
TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
|
||||
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
|
||||
|
||||
auto &elems() { return elems_; }
|
||||
|
||||
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
|
||||
|
||||
private:
|
||||
utils::pmr::vector<TypedValue> elems_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::expr
|
||||
184
src/expr/parsing.cpp
Normal file
184
src/expr/parsing.cpp
Normal file
@@ -0,0 +1,184 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "expr/parsing.hpp"
|
||||
|
||||
#include <cctype>
|
||||
#include <codecvt>
|
||||
#include <locale>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "expr/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
int64_t ParseIntegerLiteral(const std::string &s) {
|
||||
try {
|
||||
// Not really correct since long long can have a bigger range than int64_t.
|
||||
return static_cast<int64_t>(std::stoll(s, 0, 0));
|
||||
} catch (const std::out_of_range &) {
|
||||
throw SemanticException("Integer literal exceeds 64 bits.");
|
||||
}
|
||||
}
|
||||
|
||||
std::string ParseStringLiteral(const std::string &s) {
|
||||
// These functions is declared as lambda since its semantics is highly
|
||||
// specific for this conxtext and shouldn't be used elsewhere.
|
||||
auto EncodeEscapedUnicodeCodepointUtf32 = [](const std::string &s, int &i) {
|
||||
const int kLongUnicodeLength = 8;
|
||||
int j = i + 1;
|
||||
while (j < static_cast<int>(s.size()) - 1 && j < i + kLongUnicodeLength + 1 && isxdigit(s[j])) {
|
||||
++j;
|
||||
}
|
||||
if (j - i == kLongUnicodeLength + 1) {
|
||||
char32_t t = stoi(s.substr(i + 1, kLongUnicodeLength), 0, 16);
|
||||
i += kLongUnicodeLength;
|
||||
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
|
||||
return converter.to_bytes(t);
|
||||
}
|
||||
throw SyntaxException(
|
||||
"Expected 8 hex digits as unicode codepoint started with \\U. "
|
||||
"Use \\u for 4 hex digits format.");
|
||||
};
|
||||
auto EncodeEscapedUnicodeCodepointUtf16 = [](const std::string &s, int &i) {
|
||||
const int kShortUnicodeLength = 4;
|
||||
int j = i + 1;
|
||||
while (j < static_cast<int>(s.size()) - 1 && j < i + kShortUnicodeLength + 1 && isxdigit(s[j])) {
|
||||
++j;
|
||||
}
|
||||
if (j - i >= kShortUnicodeLength + 1) {
|
||||
char16_t t = stoi(s.substr(i + 1, kShortUnicodeLength), 0, 16);
|
||||
if (t >= 0xD800 && t <= 0xDBFF) {
|
||||
// t is high surrogate pair. Expect one more utf16 codepoint.
|
||||
j = i + kShortUnicodeLength + 1;
|
||||
if (j >= static_cast<int>(s.size()) - 1 || s[j] != '\\') {
|
||||
throw SemanticException("Invalid UTF codepoint.");
|
||||
}
|
||||
++j;
|
||||
if (j >= static_cast<int>(s.size()) - 1 || (s[j] != 'u' && s[j] != 'U')) {
|
||||
throw SemanticException("Invalid UTF codepoint.");
|
||||
}
|
||||
++j;
|
||||
int k = j;
|
||||
while (k < static_cast<int>(s.size()) - 1 && k < j + kShortUnicodeLength && isxdigit(s[k])) {
|
||||
++k;
|
||||
}
|
||||
if (k != j + kShortUnicodeLength) {
|
||||
throw SemanticException("Invalid UTF codepoint.");
|
||||
}
|
||||
char16_t surrogates[3] = {t, static_cast<char16_t>(stoi(s.substr(j, kShortUnicodeLength), 0, 16)), 0};
|
||||
i += kShortUnicodeLength + 2 + kShortUnicodeLength;
|
||||
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
|
||||
return converter.to_bytes(surrogates);
|
||||
} else {
|
||||
i += kShortUnicodeLength;
|
||||
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
|
||||
return converter.to_bytes(t);
|
||||
}
|
||||
}
|
||||
throw SyntaxException(
|
||||
"Expected 4 hex digits as unicode codepoint started with \\u. "
|
||||
"Use \\U for 8 hex digits format.");
|
||||
};
|
||||
|
||||
std::string unescaped;
|
||||
bool escape = false;
|
||||
|
||||
// First and last char is quote, we don't need to look at them.
|
||||
for (int i = 1; i < static_cast<int>(s.size()) - 1; ++i) {
|
||||
if (escape) {
|
||||
switch (s[i]) {
|
||||
case '\\':
|
||||
unescaped += '\\';
|
||||
break;
|
||||
case '\'':
|
||||
unescaped += '\'';
|
||||
break;
|
||||
case '"':
|
||||
unescaped += '"';
|
||||
break;
|
||||
case 'B':
|
||||
case 'b':
|
||||
unescaped += '\b';
|
||||
break;
|
||||
case 'F':
|
||||
case 'f':
|
||||
unescaped += '\f';
|
||||
break;
|
||||
case 'N':
|
||||
case 'n':
|
||||
unescaped += '\n';
|
||||
break;
|
||||
case 'R':
|
||||
case 'r':
|
||||
unescaped += '\r';
|
||||
break;
|
||||
case 'T':
|
||||
case 't':
|
||||
unescaped += '\t';
|
||||
break;
|
||||
case 'U':
|
||||
try {
|
||||
unescaped += EncodeEscapedUnicodeCodepointUtf32(s, i);
|
||||
} catch (const std::range_error &) {
|
||||
throw SemanticException("Invalid UTF codepoint.");
|
||||
}
|
||||
break;
|
||||
case 'u':
|
||||
try {
|
||||
unescaped += EncodeEscapedUnicodeCodepointUtf16(s, i);
|
||||
} catch (const std::range_error &) {
|
||||
throw SemanticException("Invalid UTF codepoint.");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
// This should never happen, except grammar changes and we don't
|
||||
// notice change in this production.
|
||||
DLOG_FATAL("can't happen");
|
||||
throw std::exception();
|
||||
}
|
||||
escape = false;
|
||||
} else if (s[i] == '\\') {
|
||||
escape = true;
|
||||
} else {
|
||||
unescaped += s[i];
|
||||
}
|
||||
}
|
||||
return unescaped;
|
||||
}
|
||||
|
||||
double ParseDoubleLiteral(const std::string &s) {
|
||||
try {
|
||||
return utils::ParseDouble(s);
|
||||
} catch (const utils::BasicException &) {
|
||||
throw SemanticException("Couldn't parse string to double.");
|
||||
}
|
||||
}
|
||||
|
||||
std::string ParseParameter(const std::string &s) {
|
||||
DMG_ASSERT(s[0] == '$', "Invalid string passed as parameter name");
|
||||
if (s[1] != '`') return s.substr(1);
|
||||
// If parameter name is escaped symbolic name then symbolic name should be
|
||||
// unescaped and leading and trailing backquote should be removed.
|
||||
DMG_ASSERT(s.size() > 3U && s.back() == '`', "Invalid string passed as parameter name");
|
||||
std::string out;
|
||||
for (int i = 2; i < static_cast<int>(s.size()) - 1; ++i) {
|
||||
if (s[i] == '`') {
|
||||
++i;
|
||||
}
|
||||
out.push_back(s[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace memgraph::expr
|
||||
27
src/expr/parsing.hpp
Normal file
27
src/expr/parsing.hpp
Normal file
@@ -0,0 +1,27 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
/// @file
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
// These are the functions for parsing literals and parameter names from
|
||||
// opencypher query.
|
||||
int64_t ParseIntegerLiteral(const std::string &s);
|
||||
std::string ParseStringLiteral(const std::string &s);
|
||||
double ParseDoubleLiteral(const std::string &s);
|
||||
std::string ParseParameter(const std::string &s);
|
||||
|
||||
} // namespace memgraph::expr
|
||||
87
src/expr/semantic/symbol.lcp
Normal file
87
src/expr/semantic/symbol.lcp
Normal file
@@ -0,0 +1,87 @@
|
||||
;; Copyright 2022 Memgraph Ltd.
|
||||
;;
|
||||
;; Use of this software is governed by the Business Source License
|
||||
;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
;; License, and you may not use this file except in compliance with the Business Source License.
|
||||
;;
|
||||
;; As of the Change Date specified in that file, in accordance with
|
||||
;; the Business Source License, use of this software will be governed
|
||||
;; by the Apache License, Version 2.0, included in the file
|
||||
;; licenses/APL.txt.
|
||||
|
||||
#>cpp
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "utils/typeinfo.hpp"
|
||||
cpp<#
|
||||
|
||||
(lcp:namespace memgraph)
|
||||
(lcp:namespace expr)
|
||||
|
||||
(lcp:define-class symbol ()
|
||||
((name "std::string" :scope :public)
|
||||
(position :int64_t :scope :public)
|
||||
(user-declared :bool :initval "true" :scope :public)
|
||||
(type "Type" :initval "Type::ANY" :scope :public)
|
||||
(token-position :int64_t :initval "-1" :scope :public))
|
||||
(:public
|
||||
;; This is similar to TypedValue::Type, but this has `Any` type.
|
||||
;; TODO: Make a better Type structure which can store a generic List.
|
||||
(lcp:define-enum type (any vertex edge path number edge-list)
|
||||
(:serialize))
|
||||
#>cpp
|
||||
// TODO: Generate enum to string conversion from LCP. Note, that this is
|
||||
// displayed to the end user, so we may want to have a pretty name of each
|
||||
// value.
|
||||
static std::string TypeToString(Type type) {
|
||||
const char *enum_string[] = {"Any", "Vertex", "Edge",
|
||||
"Path", "Number", "EdgeList"};
|
||||
return enum_string[static_cast<int>(type)];
|
||||
}
|
||||
|
||||
Symbol() {}
|
||||
Symbol(const std::string &name, int position, bool user_declared,
|
||||
Type type = Type::ANY, int token_position = -1)
|
||||
: name_(name),
|
||||
position_(position),
|
||||
user_declared_(user_declared),
|
||||
type_(type),
|
||||
token_position_(token_position) {}
|
||||
|
||||
bool operator==(const Symbol &other) const {
|
||||
return position_ == other.position_ && name_ == other.name_ &&
|
||||
type_ == other.type_;
|
||||
}
|
||||
bool operator!=(const Symbol &other) const { return !operator==(other); }
|
||||
|
||||
// TODO: Remove these since members are public
|
||||
const auto &name() const { return name_; }
|
||||
int position() const { return position_; }
|
||||
Type type() const { return type_; }
|
||||
bool user_declared() const { return user_declared_; }
|
||||
int token_position() const { return token_position_; }
|
||||
cpp<#)
|
||||
(:serialize (:slk)))
|
||||
|
||||
(lcp:pop-namespace) ;; expr
|
||||
(lcp:pop-namespace) ;; memgraph
|
||||
|
||||
#>cpp
|
||||
namespace std {
|
||||
|
||||
template <>
|
||||
struct hash<memgraph::expr::Symbol> {
|
||||
size_t operator()(const memgraph::expr::Symbol &symbol) const {
|
||||
size_t prime = 265443599u;
|
||||
size_t hash = std::hash<int>{}(symbol.position());
|
||||
hash ^= prime * std::hash<std::string>{}(symbol.name());
|
||||
hash ^= prime * std::hash<int>{}(static_cast<int>(symbol.type()));
|
||||
return hash;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace std
|
||||
|
||||
cpp<#
|
||||
712
src/expr/semantic/symbol_generator.hpp
Normal file
712
src/expr/semantic/symbol_generator.hpp
Normal file
@@ -0,0 +1,712 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
// Copyright 2017 Memgraph
|
||||
//
|
||||
// Created by Teon Banek on 11-03-2017
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
#include <ranges>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include "expr/ast.hpp"
|
||||
#include "expr/ast/ast_visitor.hpp"
|
||||
#include "expr/exceptions.hpp"
|
||||
#include "expr/semantic/symbol_table.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
namespace detail {
|
||||
inline std::unordered_map<std::string, Identifier *> GeneratePredefinedIdentifierMap(
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
std::unordered_map<std::string, Identifier *> identifier_map;
|
||||
for (const auto &identifier : predefined_identifiers) {
|
||||
identifier_map.emplace(identifier->name_, identifier);
|
||||
}
|
||||
|
||||
return identifier_map;
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
/// Visits the AST and generates symbols for variables.
|
||||
///
|
||||
/// During the process of symbol generation, simple semantic checks are
|
||||
/// performed. Such as, redeclaring a variable or conflicting expectations of
|
||||
/// variable types.
|
||||
class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
public:
|
||||
explicit SymbolGenerator(SymbolTable *symbol_table, const std::vector<Identifier *> &predefined_identifiers)
|
||||
: symbol_table_(symbol_table),
|
||||
predefined_identifiers_{detail::GeneratePredefinedIdentifierMap(predefined_identifiers)},
|
||||
scopes_(1, Scope()) {}
|
||||
|
||||
using HierarchicalTreeVisitor::PostVisit;
|
||||
using HierarchicalTreeVisitor::PreVisit;
|
||||
using HierarchicalTreeVisitor::Visit;
|
||||
using typename HierarchicalTreeVisitor::ReturnType;
|
||||
|
||||
// Query
|
||||
bool PreVisit(SingleQuery & /*unused*/) override {
|
||||
prev_return_names_ = curr_return_names_;
|
||||
curr_return_names_.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Union
|
||||
bool PreVisit(CypherUnion & /*unused*/) override {
|
||||
scopes_.back() = Scope();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(CypherUnion &cypher_union) override {
|
||||
if (prev_return_names_ != curr_return_names_) {
|
||||
throw SemanticException("All subqueries in an UNION must have the same column names.");
|
||||
}
|
||||
|
||||
// create new symbols for the result of the union
|
||||
for (const auto &name : curr_return_names_) {
|
||||
auto symbol = CreateSymbol(name, false);
|
||||
cypher_union.union_symbols_.push_back(symbol);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
// Clauses
|
||||
bool PreVisit(Create & /*unused*/) override {
|
||||
scopes_.back().in_create = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Create & /*unused*/) override {
|
||||
scopes_.back().in_create = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(CallProcedure &call_proc) override {
|
||||
for (auto *expr : call_proc.arguments_) {
|
||||
expr->Accept(*this);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PostVisit(CallProcedure &call_proc) override {
|
||||
for (auto *ident : call_proc.result_identifiers_) {
|
||||
if (HasSymbolLocalScope(ident->name_)) {
|
||||
throw RedeclareVariableError(ident->name_);
|
||||
}
|
||||
ident->MapTo(CreateSymbol(ident->name_, true));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(LoadCsv & /*unused*/) override { return false; }
|
||||
|
||||
bool PostVisit(LoadCsv &load_csv) override {
|
||||
if (HasSymbolLocalScope(load_csv.row_var_->name_)) {
|
||||
throw RedeclareVariableError(load_csv.row_var_->name_);
|
||||
}
|
||||
load_csv.row_var_->MapTo(CreateSymbol(load_csv.row_var_->name_, true));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(Return &ret) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_return = true;
|
||||
VisitReturnBody(ret.body_);
|
||||
scope.in_return = false;
|
||||
return false; // We handled the traversal ourselves.
|
||||
}
|
||||
|
||||
bool PostVisit(Return & /*unused*/) override {
|
||||
for (const auto &name_symbol : scopes_.back().symbols) curr_return_names_.insert(name_symbol.first);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(With &with) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_with = true;
|
||||
VisitReturnBody(with.body_, with.where_);
|
||||
scope.in_with = false;
|
||||
return false; // We handled the traversal ourselves.
|
||||
}
|
||||
|
||||
bool PreVisit(Where & /*unused*/) override {
|
||||
scopes_.back().in_where = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Where & /*unused*/) override {
|
||||
scopes_.back().in_where = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(Merge & /*unused*/) override {
|
||||
scopes_.back().in_merge = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Merge & /*unused*/) override {
|
||||
scopes_.back().in_merge = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Unwind &unwind) override {
|
||||
const auto &name = unwind.named_expression_->name_;
|
||||
if (HasSymbolLocalScope(name)) {
|
||||
throw RedeclareVariableError(name);
|
||||
}
|
||||
unwind.named_expression_->MapTo(CreateSymbol(name, true));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(Match & /*unused*/) override {
|
||||
scopes_.back().in_match = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Match & /*unused*/) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_match = false;
|
||||
// Check variables in property maps after visiting Match, so that they can
|
||||
// reference symbols out of bind order.
|
||||
for (auto &ident : scope.identifiers_in_match) {
|
||||
if (!HasSymbolLocalScope(ident->name_) && !ConsumePredefinedIdentifier(ident->name_))
|
||||
throw UnboundVariableError(ident->name_);
|
||||
ident->MapTo(scope.symbols[ident->name_]);
|
||||
}
|
||||
scope.identifiers_in_match.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(Foreach &for_each) override {
|
||||
const auto &name = for_each.named_expression_->name_;
|
||||
scopes_.emplace_back(Scope());
|
||||
scopes_.back().in_foreach = true;
|
||||
for_each.named_expression_->MapTo(
|
||||
CreateSymbol(name, true, Symbol::Type::ANY, for_each.named_expression_->token_position_));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Foreach & /*unused*/) override {
|
||||
scopes_.pop_back();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Expressions
|
||||
ReturnType Visit(Identifier &ident) override {
|
||||
auto &scope = scopes_.back();
|
||||
if (scope.in_skip || scope.in_limit) {
|
||||
throw SemanticException("Variables are not allowed in {}.", scope.in_skip ? "SKIP" : "LIMIT");
|
||||
}
|
||||
Symbol symbol;
|
||||
if (scope.in_pattern && !(scope.in_node_atom || scope.visiting_edge)) {
|
||||
// If we are in the pattern, and outside of a node or an edge, the
|
||||
// identifier is the pattern name.
|
||||
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, Symbol::Type::PATH);
|
||||
} else if (scope.in_pattern && scope.in_pattern_atom_identifier) {
|
||||
// Patterns used to create nodes and edges cannot redeclare already
|
||||
// established bindings. Declaration only happens in single node
|
||||
// patterns and in edge patterns. OpenCypher example,
|
||||
// `MATCH (n) CREATE (n)` should throw an error that `n` is already
|
||||
// declared. While `MATCH (n) CREATE (n) -[:R]-> (n)` is allowed,
|
||||
// since `n` now references the bound node instead of declaring it.
|
||||
if ((scope.in_create_node || scope.in_create_edge) && HasSymbolLocalScope(ident.name_)) {
|
||||
throw RedeclareVariableError(ident.name_);
|
||||
}
|
||||
auto type = Symbol::Type::VERTEX;
|
||||
if (scope.visiting_edge) {
|
||||
// Edge referencing is not allowed (like in Neo4j):
|
||||
// `MATCH (n) - [r] -> (n) - [r] -> (n) RETURN r` is not allowed.
|
||||
if (HasSymbolLocalScope(ident.name_)) {
|
||||
throw RedeclareVariableError(ident.name_);
|
||||
}
|
||||
type = scope.visiting_edge->IsVariable() ? Symbol::Type::EDGE_LIST : Symbol::Type::EDGE;
|
||||
}
|
||||
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, type);
|
||||
} else if (scope.in_pattern && !scope.in_pattern_atom_identifier && scope.in_match) {
|
||||
if (scope.in_edge_range && scope.visiting_edge->identifier_->name_ == ident.name_) {
|
||||
// Prevent variable path bounds to reference the identifier which is bound
|
||||
// by the variable path itself.
|
||||
throw UnboundVariableError(ident.name_);
|
||||
}
|
||||
// Variables in property maps or bounds of variable length path during MATCH
|
||||
// can reference symbols bound later in the same MATCH. We collect them
|
||||
// here, so that they can be checked after visiting Match.
|
||||
scope.identifiers_in_match.emplace_back(&ident);
|
||||
} else {
|
||||
// Everything else references a bound symbol.
|
||||
if (!HasSymbol(ident.name_) && !ConsumePredefinedIdentifier(ident.name_)) throw UnboundVariableError(ident.name_);
|
||||
symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, Symbol::Type::ANY);
|
||||
}
|
||||
ident.MapTo(symbol);
|
||||
return true;
|
||||
}
|
||||
|
||||
ReturnType Visit(PrimitiveLiteral & /*unused*/) override { return true; }
|
||||
|
||||
ReturnType Visit(ParameterLookup & /*unused*/) override { return true; }
|
||||
|
||||
bool PreVisit(Aggregation &aggr) override {
|
||||
auto &scope = scopes_.back();
|
||||
// Check if the aggregation can be used in this context. This check should
|
||||
// probably move to a separate phase, which checks if the query is well
|
||||
// formed.
|
||||
if ((!scope.in_return && !scope.in_with) || scope.in_order_by || scope.in_skip || scope.in_limit ||
|
||||
scope.in_where) {
|
||||
throw SemanticException("Aggregation functions are only allowed in WITH and RETURN.");
|
||||
}
|
||||
if (scope.in_aggregation) {
|
||||
throw SemanticException(
|
||||
"Using aggregation functions inside aggregation functions is not "
|
||||
"allowed.");
|
||||
}
|
||||
if (scope.num_if_operators) {
|
||||
// Neo allows aggregations here and produces very interesting behaviors.
|
||||
// To simplify implementation at this moment we decided to completely
|
||||
// disallow aggregations inside of the CASE.
|
||||
// However, in some cases aggregation makes perfect sense, for example:
|
||||
// CASE count(n) WHEN 10 THEN "YES" ELSE "NO" END.
|
||||
// TODO: Rethink of allowing aggregations in some parts of the CASE
|
||||
// construct.
|
||||
throw SemanticException("Using aggregation functions inside of CASE is not allowed.");
|
||||
}
|
||||
// Create a virtual symbol for aggregation result.
|
||||
// Currently, we only have aggregation operators which return numbers.
|
||||
auto aggr_name = Aggregation::OpToString(aggr.op_) + std::to_string(aggr.symbol_pos_);
|
||||
aggr.MapTo(CreateSymbol(aggr_name, false, Symbol::Type::NUMBER));
|
||||
scope.in_aggregation = true;
|
||||
scope.has_aggregation = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Aggregation & /*unused*/) override {
|
||||
scopes_.back().in_aggregation = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(IfOperator & /*unused*/) override {
|
||||
++scopes_.back().num_if_operators;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(IfOperator & /*unused*/) override {
|
||||
--scopes_.back().num_if_operators;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(All &all) override {
|
||||
all.list_expression_->Accept(*this);
|
||||
VisitWithIdentifiers(all.where_->expression_, {all.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PreVisit(Single &single) override {
|
||||
single.list_expression_->Accept(*this);
|
||||
VisitWithIdentifiers(single.where_->expression_, {single.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PreVisit(Any &any) override {
|
||||
any.list_expression_->Accept(*this);
|
||||
VisitWithIdentifiers(any.where_->expression_, {any.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PreVisit(None &none) override {
|
||||
none.list_expression_->Accept(*this);
|
||||
VisitWithIdentifiers(none.where_->expression_, {none.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PreVisit(Reduce &reduce) override {
|
||||
reduce.initializer_->Accept(*this);
|
||||
reduce.list_->Accept(*this);
|
||||
VisitWithIdentifiers(reduce.expression_, {reduce.accumulator_, reduce.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PreVisit(Extract &extract) override {
|
||||
extract.list_->Accept(*this);
|
||||
VisitWithIdentifiers(extract.expression_, {extract.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
// Pattern and its subparts.
|
||||
bool PreVisit(Pattern &pattern) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_pattern = true;
|
||||
if ((scope.in_create || scope.in_merge) && pattern.atoms_.size() == 1U) {
|
||||
MG_ASSERT(utils::IsSubtype(*pattern.atoms_[0], NodeAtom::kType), "Expected a single NodeAtom in Pattern");
|
||||
scope.in_create_node = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Pattern & /*unused*/) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_pattern = false;
|
||||
scope.in_create_node = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(NodeAtom &node_atom) override {
|
||||
auto &scope = scopes_.back();
|
||||
auto check_node_semantic = [&node_atom, &scope, this](const bool props_or_labels) {
|
||||
const auto &node_name = node_atom.identifier_->name_;
|
||||
if ((scope.in_create || scope.in_merge) && props_or_labels && HasSymbolLocalScope(node_name)) {
|
||||
throw SemanticException("Cannot create node '" + node_name +
|
||||
"' with labels or properties, because it is already declared.");
|
||||
}
|
||||
scope.in_pattern_atom_identifier = true;
|
||||
node_atom.identifier_->Accept(*this);
|
||||
scope.in_pattern_atom_identifier = false;
|
||||
};
|
||||
|
||||
scope.in_node_atom = true;
|
||||
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&node_atom.properties_)) {
|
||||
bool props_or_labels = !properties->empty() || !node_atom.labels_.empty();
|
||||
|
||||
check_node_semantic(props_or_labels);
|
||||
for (auto kv : *properties) {
|
||||
kv.second->Accept(*this);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
auto &properties_parameter = std::get<ParameterLookup *>(node_atom.properties_);
|
||||
bool props_or_labels = !properties_parameter || !node_atom.labels_.empty();
|
||||
|
||||
check_node_semantic(props_or_labels);
|
||||
properties_parameter->Accept(*this);
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PostVisit(NodeAtom & /*unused*/) override {
|
||||
scopes_.back().in_node_atom = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(EdgeAtom &edge_atom) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.visiting_edge = &edge_atom;
|
||||
if (scope.in_create || scope.in_merge) {
|
||||
scope.in_create_edge = true;
|
||||
if (edge_atom.edge_types_.size() != 1U) {
|
||||
throw SemanticException(
|
||||
"A single relationship type must be specified "
|
||||
"when creating an edge.");
|
||||
}
|
||||
if (scope.in_create && // Merge allows bidirectionality
|
||||
edge_atom.direction_ == EdgeAtom::Direction::BOTH) {
|
||||
throw SemanticException(
|
||||
"Bidirectional relationship are not supported "
|
||||
"when creating an edge");
|
||||
}
|
||||
if (edge_atom.IsVariable()) {
|
||||
throw SemanticException(
|
||||
"Variable length relationships are not supported when creating an "
|
||||
"edge.");
|
||||
}
|
||||
}
|
||||
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&edge_atom.properties_)) {
|
||||
for (auto kv : *properties) {
|
||||
kv.second->Accept(*this);
|
||||
}
|
||||
} else {
|
||||
std::get<ParameterLookup *>(edge_atom.properties_)->Accept(*this);
|
||||
}
|
||||
if (edge_atom.IsVariable()) {
|
||||
scope.in_edge_range = true;
|
||||
if (edge_atom.lower_bound_) {
|
||||
edge_atom.lower_bound_->Accept(*this);
|
||||
}
|
||||
if (edge_atom.upper_bound_) {
|
||||
edge_atom.upper_bound_->Accept(*this);
|
||||
}
|
||||
scope.in_edge_range = false;
|
||||
scope.in_pattern = false;
|
||||
if (edge_atom.filter_lambda_.expression) {
|
||||
VisitWithIdentifiers(edge_atom.filter_lambda_.expression,
|
||||
{edge_atom.filter_lambda_.inner_edge, edge_atom.filter_lambda_.inner_node});
|
||||
} else {
|
||||
// Create inner symbols, but don't bind them in scope, since they are to
|
||||
// be used in the missing filter expression.
|
||||
auto *inner_edge = edge_atom.filter_lambda_.inner_edge;
|
||||
inner_edge->MapTo(
|
||||
symbol_table_->CreateSymbol(inner_edge->name_, inner_edge->user_declared_, Symbol::Type::EDGE));
|
||||
auto *inner_node = edge_atom.filter_lambda_.inner_node;
|
||||
inner_node->MapTo(
|
||||
symbol_table_->CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX));
|
||||
}
|
||||
if (edge_atom.weight_lambda_.expression) {
|
||||
VisitWithIdentifiers(edge_atom.weight_lambda_.expression,
|
||||
{edge_atom.weight_lambda_.inner_edge, edge_atom.weight_lambda_.inner_node});
|
||||
}
|
||||
scope.in_pattern = true;
|
||||
}
|
||||
scope.in_pattern_atom_identifier = true;
|
||||
edge_atom.identifier_->Accept(*this);
|
||||
scope.in_pattern_atom_identifier = false;
|
||||
if (edge_atom.total_weight_) {
|
||||
if (HasSymbolLocalScope(edge_atom.total_weight_->name_)) {
|
||||
throw RedeclareVariableError(edge_atom.total_weight_->name_);
|
||||
}
|
||||
edge_atom.total_weight_->MapTo(GetOrCreateSymbolLocalScope(
|
||||
edge_atom.total_weight_->name_, edge_atom.total_weight_->user_declared_, Symbol::Type::NUMBER));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PostVisit(EdgeAtom & /*unused*/) override {
|
||||
auto &scope = scopes_.back();
|
||||
scope.visiting_edge = nullptr;
|
||||
scope.in_create_edge = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
// Scope stores the state of where we are when visiting the AST and a map of
|
||||
// names to symbols.
|
||||
struct Scope {
|
||||
bool in_pattern{false};
|
||||
bool in_merge{false};
|
||||
bool in_create{false};
|
||||
// in_create_node is true if we are creating or merging *only* a node.
|
||||
// Therefore, it is *not* equivalent to (in_create || in_merge) &&
|
||||
// in_node_atom.
|
||||
bool in_create_node{false};
|
||||
// True if creating an edge;
|
||||
// shortcut for (in_create || in_merge) && visiting_edge.
|
||||
bool in_create_edge{false};
|
||||
bool in_node_atom{false};
|
||||
EdgeAtom *visiting_edge{nullptr};
|
||||
bool in_aggregation{false};
|
||||
bool in_return{false};
|
||||
bool in_with{false};
|
||||
bool in_skip{false};
|
||||
bool in_limit{false};
|
||||
bool in_order_by{false};
|
||||
bool in_where{false};
|
||||
bool in_match{false};
|
||||
bool in_foreach{false};
|
||||
// True when visiting a pattern atom (node or edge) identifier, which can be
|
||||
// reused or created in the pattern itself.
|
||||
bool in_pattern_atom_identifier{false};
|
||||
// True when visiting range bounds of a variable path.
|
||||
bool in_edge_range{false};
|
||||
// True if the return/with contains an aggregation in any named expression.
|
||||
bool has_aggregation{false};
|
||||
// Map from variable names to symbols.
|
||||
std::map<std::string, Symbol> symbols;
|
||||
// Identifiers found in property maps of patterns or as variable length path
|
||||
// bounds in a single Match clause. They need to be checked after visiting
|
||||
// Match. Identifiers created by naming vertices, edges and paths are *not*
|
||||
// stored in here.
|
||||
std::vector<Identifier *> identifiers_in_match;
|
||||
// Number of nested IfOperators.
|
||||
int num_if_operators{0};
|
||||
};
|
||||
|
||||
inline static std::optional<Symbol> FindSymbolInScope(const std::string &name, const Scope &scope,
|
||||
Symbol::Type type) {
|
||||
if (auto it = scope.symbols.find(name); it != scope.symbols.end()) {
|
||||
const auto &symbol = it->second;
|
||||
// Unless we have `ANY` type, check that types match.
|
||||
if (type != Symbol::Type::ANY && symbol.type() != Symbol::Type::ANY && type != symbol.type()) {
|
||||
throw TypeMismatchError(name, Symbol::TypeToString(symbol.type()), Symbol::TypeToString(type));
|
||||
}
|
||||
return symbol;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool HasSymbol(const std::string &name) const {
|
||||
return std::ranges::any_of(scopes_, [&name](const auto &scope) { return scope.symbols.contains(name); });
|
||||
}
|
||||
|
||||
bool HasSymbolLocalScope(const std::string &name) const { return scopes_.back().symbols.contains(name); }
|
||||
|
||||
// @return true if it added a predefined identifier with that name
|
||||
bool ConsumePredefinedIdentifier(const std::string &name) {
|
||||
auto it = predefined_identifiers_.find(name);
|
||||
|
||||
if (it == predefined_identifiers_.end()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// we can only use the predefined identifier in a single scope so we remove it after creating
|
||||
// a symbol for it
|
||||
auto &identifier = it->second;
|
||||
MG_ASSERT(!identifier->user_declared_, "Predefined symbols cannot be user declared!");
|
||||
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
|
||||
predefined_identifiers_.erase(it);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Returns a freshly generated symbol. Previous mapping of the same name to a
|
||||
// different symbol is replaced with the new one.
|
||||
Symbol CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
|
||||
int token_position = -1) {
|
||||
auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position);
|
||||
scopes_.back().symbols[name] = symbol;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
Symbol GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY) {
|
||||
// NOLINTNEXTLINE
|
||||
for (auto scope = scopes_.rbegin(); scope != scopes_.rend(); ++scope) {
|
||||
if (auto maybe_symbol = FindSymbolInScope(name, *scope, type); maybe_symbol) {
|
||||
return *maybe_symbol;
|
||||
}
|
||||
}
|
||||
return CreateSymbol(name, user_declared, type);
|
||||
}
|
||||
|
||||
// Returns the symbol by name. If the mapping already exists, checks if the
|
||||
// types match. Otherwise, returns a new symbol.
|
||||
Symbol GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared,
|
||||
Symbol::Type type = Symbol::Type::ANY) {
|
||||
auto &scope = scopes_.back();
|
||||
if (auto maybe_symbol = FindSymbolInScope(name, scope, type); maybe_symbol) {
|
||||
return *maybe_symbol;
|
||||
}
|
||||
return CreateSymbol(name, user_declared, type);
|
||||
}
|
||||
|
||||
void VisitReturnBody(ReturnBody &body, Where *where = nullptr) {
|
||||
auto &scope = scopes_.back();
|
||||
for (auto &expr : body.named_expressions) {
|
||||
expr->Accept(*this);
|
||||
}
|
||||
std::vector<Symbol> user_symbols;
|
||||
if (body.all_identifiers) {
|
||||
// Carry over user symbols because '*' appeared.
|
||||
for (const auto &sym_pair : scope.symbols) {
|
||||
if (!sym_pair.second.user_declared()) {
|
||||
continue;
|
||||
}
|
||||
user_symbols.emplace_back(sym_pair.second);
|
||||
}
|
||||
if (user_symbols.empty()) {
|
||||
throw SemanticException("There are no variables in scope to use for '*'.");
|
||||
}
|
||||
}
|
||||
// WITH/RETURN clause removes declarations of all the previous variables and
|
||||
// declares only those established through named expressions. New declarations
|
||||
// must not be visible inside named expressions themselves.
|
||||
bool removed_old_names = false;
|
||||
if ((!where && body.order_by.empty()) || scope.has_aggregation) {
|
||||
// WHERE and ORDER BY need to see both the old and new symbols, unless we
|
||||
// have an aggregation. Therefore, we can clear the symbols immediately if
|
||||
// there is neither ORDER BY nor WHERE, or we have an aggregation.
|
||||
scope.symbols.clear();
|
||||
removed_old_names = true;
|
||||
}
|
||||
// Create symbols for named expressions.
|
||||
std::unordered_set<std::string> new_names;
|
||||
for (const auto &user_sym : user_symbols) {
|
||||
new_names.insert(user_sym.name());
|
||||
scope.symbols[user_sym.name()] = user_sym;
|
||||
}
|
||||
for (auto &named_expr : body.named_expressions) {
|
||||
const auto &name = named_expr->name_;
|
||||
if (!new_names.insert(name).second) {
|
||||
throw SemanticException("Multiple results with the same name '{}' are not allowed.", name);
|
||||
}
|
||||
// An improvement would be to infer the type of the expression, so that the
|
||||
// new symbol would have a more specific type.
|
||||
named_expr->MapTo(CreateSymbol(name, true, Symbol::Type::ANY, named_expr->token_position_));
|
||||
}
|
||||
scope.in_order_by = true;
|
||||
for (const auto &order_pair : body.order_by) {
|
||||
order_pair.expression->Accept(*this);
|
||||
}
|
||||
scope.in_order_by = false;
|
||||
if (body.skip) {
|
||||
scope.in_skip = true;
|
||||
body.skip->Accept(*this);
|
||||
scope.in_skip = false;
|
||||
}
|
||||
if (body.limit) {
|
||||
scope.in_limit = true;
|
||||
body.limit->Accept(*this);
|
||||
scope.in_limit = false;
|
||||
}
|
||||
if (where) where->Accept(*this);
|
||||
if (!removed_old_names) {
|
||||
// We have an ORDER BY or WHERE, but no aggregation, which means we didn't
|
||||
// clear the old symbols, so do it now. We cannot just call clear, because
|
||||
// we've added new symbols.
|
||||
for (auto sym_it = scope.symbols.begin(); sym_it != scope.symbols.end();) {
|
||||
if (new_names.find(sym_it->first) == new_names.end()) {
|
||||
sym_it = scope.symbols.erase(sym_it);
|
||||
} else {
|
||||
sym_it++;
|
||||
}
|
||||
}
|
||||
}
|
||||
scopes_.back().has_aggregation = false;
|
||||
}
|
||||
|
||||
void VisitWithIdentifiers(Expression *expr, const std::vector<Identifier *> &identifiers) {
|
||||
auto &scope = scopes_.back();
|
||||
std::vector<std::pair<std::optional<Symbol>, Identifier *>> prev_symbols;
|
||||
// Collect previous symbols if they exist.
|
||||
for (const auto &identifier : identifiers) {
|
||||
std::optional<Symbol> prev_symbol;
|
||||
auto prev_symbol_it = scope.symbols.find(identifier->name_);
|
||||
if (prev_symbol_it != scope.symbols.end()) {
|
||||
prev_symbol = prev_symbol_it->second;
|
||||
}
|
||||
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
|
||||
prev_symbols.emplace_back(prev_symbol, identifier);
|
||||
}
|
||||
// Visit the expression with the new symbols bound.
|
||||
expr->Accept(*this);
|
||||
// Restore back to previous symbols.
|
||||
for (const auto &prev : prev_symbols) {
|
||||
const auto &prev_symbol = prev.first;
|
||||
const auto &identifier = prev.second;
|
||||
if (prev_symbol) {
|
||||
scope.symbols[identifier->name_] = *prev_symbol;
|
||||
} else {
|
||||
scope.symbols.erase(identifier->name_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
SymbolTable *symbol_table_;
|
||||
|
||||
// Identifiers which are injected from outside the query. Each identifier
|
||||
// is mapped by its name.
|
||||
std::unordered_map<std::string, Identifier *> predefined_identifiers_;
|
||||
std::vector<Scope> scopes_;
|
||||
std::unordered_set<std::string> prev_return_names_;
|
||||
std::unordered_set<std::string> curr_return_names_;
|
||||
};
|
||||
|
||||
inline SymbolTable MakeSymbolTable(CypherQuery *query, const std::vector<Identifier *> &predefined_identifiers = {}) {
|
||||
SymbolTable symbol_table;
|
||||
SymbolGenerator symbol_generator(&symbol_table, predefined_identifiers);
|
||||
query->single_query_->Accept(symbol_generator);
|
||||
for (auto *cypher_union : query->cypher_unions_) {
|
||||
cypher_union->Accept(symbol_generator);
|
||||
}
|
||||
return symbol_table;
|
||||
}
|
||||
|
||||
} // namespace memgraph::expr
|
||||
64
src/expr/semantic/symbol_table.hpp
Normal file
64
src/expr/semantic/symbol_table.hpp
Normal file
@@ -0,0 +1,64 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "expr/ast.hpp"
|
||||
#include "expr/semantic/symbol.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
class SymbolTable final {
|
||||
public:
|
||||
SymbolTable() {}
|
||||
const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
|
||||
int32_t token_position = -1) {
|
||||
MG_ASSERT(table_.size() <= std::numeric_limits<int32_t>::max(),
|
||||
"SymbolTable size doesn't fit into 32-bit integer!");
|
||||
auto got = table_.emplace(position_, Symbol(name, position_, user_declared, type, token_position));
|
||||
MG_ASSERT(got.second, "Duplicate symbol ID!");
|
||||
position_++;
|
||||
return got.first->second;
|
||||
}
|
||||
|
||||
// TODO(buda): This is the same logic as in the cypher_main_visitor. During
|
||||
// parsing phase symbol table doesn't exist. Figure out a better solution.
|
||||
const Symbol &CreateAnonymousSymbol(Symbol::Type type = Symbol::Type::ANY) {
|
||||
int id = 1;
|
||||
while (true) {
|
||||
static const std::string &kAnonPrefix = "anon";
|
||||
std::string name_candidate = kAnonPrefix + std::to_string(id++);
|
||||
if (std::find_if(std::begin(table_), std::end(table_), [&name_candidate](const auto &item) -> bool {
|
||||
return item.second.name_ == name_candidate;
|
||||
}) == std::end(table_)) {
|
||||
return CreateSymbol(name_candidate, false, type);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const Symbol &at(const Identifier &ident) const { return table_.at(ident.symbol_pos_); }
|
||||
const Symbol &at(const NamedExpression &nexpr) const { return table_.at(nexpr.symbol_pos_); }
|
||||
const Symbol &at(const Aggregation &aggr) const { return table_.at(aggr.symbol_pos_); }
|
||||
|
||||
// TODO: Remove these since members are public
|
||||
int32_t max_position() const { return static_cast<int32_t>(table_.size()); }
|
||||
|
||||
const auto &table() const { return table_; }
|
||||
|
||||
int32_t position_{0};
|
||||
std::map<int32_t, Symbol> table_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::expr
|
||||
1505
src/expr/typed_value.hpp
Normal file
1505
src/expr/typed_value.hpp
Normal file
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user