Compare commits

..

43 Commits

Author SHA1 Message Date
gvolfing
2faa58fe21 restructure test 2022-09-29 08:05:23 +02:00
János Benjamin Antal
40b2e12eaf Use include from bindings instead of directly from expr 2022-09-27 11:30:40 +02:00
gvolfing
0e613a1ff9 Fix expression test 2022-09-23 19:56:00 +02:00
jbajic
4b5cbb99b6 Test out expr in storage 2022-09-23 14:14:03 +02:00
Kostas Kyrimis
925835b080 Implement query engine client (#531)
- Add shard request manager
2022-09-22 15:05:43 +02:00
Tyler Neely
ce788f5f65 Machine manager and shard stitch (#569) 2022-09-22 13:55:16 +02:00
János Benjamin Antal
b8186bea2e Lexicographically ordered storage (#454) 2022-09-21 18:58:31 +02:00
János Benjamin Antal
3c4856dcb7 Merge remote-tracking branch 'origin/project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-21 18:50:00 +02:00
Marko Budiselić
b4d6dc0930 Add proper transaction handling (#550) 2022-09-21 18:25:51 +02:00
Jure Bajic
817161a915 Implement expression evaluator in storage v3 (#534)
- Enable `mg-expr` in mg-storage-v3
-  Adapt id_mapper
- Change conversion function from PropertyValue to TypedValue
- Remove memgraph functions
- Enable expression tests on for storage
2022-09-21 17:26:55 +02:00
Tyler Neely
58eb2caf0f Add machine manager prototype (#533) 2022-09-21 16:57:25 +02:00
gvolfing
ecda71168c Implement message based actions (#538)
Create shard-side handlers for basic messages

Implement the handlers for CreateVertices, CreateEdges and ScanAll. Use
or modify the defined messages to interact with individual Shards and
test their behavior. Shard is currently being owned by ShardRsm
instances. The two top level dispatching functions Read() and Apply()
are responsible for read- and write operations respectively. Currently
there are a handful of messages that are defined but not utilized, these
will be used in the near future, as well as a couple of handler
functions with empty implementations.
2022-09-20 11:15:19 +02:00
János Benjamin Antal
8e1f83acc9 Reference vertices by id in edges (#532) 2022-09-14 13:11:14 +02:00
jbajic
f910cb770c Merge branch 'project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-09 12:39:28 +02:00
Kostas Kyrimis
38d0b89b04 Implement expression evaluator library (#486) 2022-09-07 18:15:32 +03:00
Jure Bajic
a2a6a3855b Remove primary label from vertex (#529)
- Remove the primary label from vertex
- Pass vertex_validator instead of schema validator
- Fix vertex ctors
2022-09-07 13:36:10 +02:00
Jure Bajic
e9f0360fb3 Enable indices for storage v3 (#528)
- Enable indices in storage v3
- Add new test cases
- Change `CreateVertexAndValidate` to call `SetProperty` and `AddLabel`
2022-09-06 22:48:25 +02:00
János Benjamin Antal
fe4955447e Merge branch 'project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-05 14:59:01 +02:00
Tyler Neely
1631c20df2 In memory shard map (#505) 2022-09-05 15:50:54 +03:00
János Benjamin Antal
947baedbe6 Merge remote-tracking branch 'origin/project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-09-02 11:41:03 +02:00
Kostas Kyrimis
c6447eb48b Add shard requests responses (#526) 2022-09-01 18:54:47 +03:00
Jure Bajic
7e84744d07 Split storage and shards (#519)
- Rename storage to shard
- Add primary label and range for shard
- Remove id_mapper functionality from shard
- Adapt tests
2022-09-01 09:10:40 +02:00
Tyler Neely
c0d03888f4 Implement basic raft version (#498) 2022-08-30 15:07:34 +02:00
János Benjamin Antal
efb3c8d03d Remove multi-threaded related logic and variables (#460)
* Remove logic that was necessary for optimal multi-threaded performance, such
  as accumulating deleted object in local containers and appending them to a
  global one, handling overlapping locking.
* Remove background GC thread.
* Remove mutexes, locks and atomics throughout storage.
2022-08-30 13:41:53 +02:00
Jure Bajic
95dbc022c0 Integrate schema and keystore (#497)
- Integrate schema and keystore on vertex creation
- Add GC test for storage v3
- Add tests for accessors
- Fix all tests related to this except for query v2
- Fix labels not returning primary label
2022-08-29 14:38:25 +02:00
Tyler Neely
a40403e3ce Add local transport (#512)
* Create LocalTransport Io provider for sending messages to components on the same machine
* Move src/io/simulation/message_conversion.hpp to src/io/message_conversion.hpp for use in other Io providers
2022-08-29 13:49:51 +02:00
Tyler Neely
14c9e68456 Transport prototype (#466) 2022-08-12 08:24:32 +02:00
jbajic
68b26275a3 Merge branch 'project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-08-08 11:34:46 +02:00
jbajic
5012824e05 Address review comments 2022-08-04 11:45:16 +02:00
Jure Bajic
a12a1ea358 Move schema to storage v3 and query v2
* Move schema to storage v3

* Remove schema from v2

* Move schema to query v2

* Remove schema from query v1

* Make glue v2

* Move schema related tests to newer versions of query and storage

* Fix typo in CMake

* Fix interpreter test

* Fix clang tidy errors

* Change temp dir name
2022-08-04 09:50:02 +02:00
János Benjamin Antal
2891041468 Make storage use KeyStore (#455) 2022-08-03 18:10:58 +02:00
János Benjamin Antal
cc5ee6a496 Merge remote-tracking branch 'origin/project-pineapples' into E118-MG-lexicographically-ordered-storage 2022-08-02 08:19:43 +02:00
jbajic
f57f30c8cf Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-08-01 10:46:11 +02:00
Jure Bajic
462daf3a2b Enforce schema on vertex creation
- Separating schema definition from schema validation
- Updating vertex_accessor and db_accessors with necessary methods
- Adding a primary label to Vertex
- Adding schema tests
- Updating existing tests for storage v3, and deprecating old:
  - interpreter => interpreter_v2
  - query_plan_accumulate_aggregate => storage_v3_query_plan_accumulate_aggregate
  - query_plan_create_set_remove_delete => storage_v3_query_plan_create_set_remove_delete
  - query_plan_bag_semantics => storage_v3_query_plan_bag_semantics
  - query_plan_edge_cases => storage_v3_query_plan_edge_cases
  - query_plan_v2_create_set_remove_delete => storage_v3_query_plan_v2_create_set_remove_delete
  - query_plan_match_filter_return => storage_v3_query_plan_match_filter_return
2022-07-29 13:38:17 +02:00
Marko Budiselic
2009fefc8a Merge branch 'master' into project-pineapples 2022-07-28 15:36:51 +02:00
Marko Budiselić
eb3f96d1f6 Bring changes from master to project-pineapples (#477)
* Fix aggregation functions on `null` and group-by inputs (#448)
* Upgrade Antrl to 4.10.1 and remove antlr_lock (#441)
* Update clang-tidy job (#476)
* Add parser stress test (#463)

NOTE: Doing this to have buildable comments on the project-pineapples branch

Co-authored-by: gvolfing <107616712+gvolfing@users.noreply.github.com>
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 15:36:17 +02:00
jbajic
264b233053 Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-07-22 11:48:45 +02:00
Jure Bajic
2ceaf59767 Create query engine v2 (#444)
Create version v2 of the query engine.
Adjust CMake and lisp files
Connect query engine v2 with storage engine v3
2022-07-19 12:28:19 +02:00
János Benjamin Antal
c0bee760bf git 2022-07-18 08:21:04 +02:00
Jure Bajic
3f4f66b57f Create schema DDL expressions
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Add schema operations in storage

* Add create and show schema queries

* Add privileges for schema

* Add missing keywords into lexer

* Add drop schema query

* Add schema visitors

* Update metadata

* Add PrepareSchemaQuery function

* Implement show schemas

* Add show schema query

* Fix schema visitor

* Add common schema type

* Fix grammar

* Temporary create ddl logic

* Fix naming for schemaproperty type to schema type

* Rename schemaproperty to schemapropertytype

* Enable Create schema ddl

* Override visitPropertyType

* Add initial schema implementation

* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

* Add verification on creation and deletion

* Rename DeleteSchema to DropSchema

* Remove list and map from lexer

* Fix grammar with schemaTypeMap

* Add privilege and cypher visitor tests

* Catch repeating type name in schema definition

* Fix conflicting keywords

* Add notifications

* Drop float support

* Finish interpreter tests

* Fix tests

* Fix clang tidy errors

* Fix GetSchema

* Replace for with transfrom

* Add cloning og schema_property_map

* Address review comments

* Rename SchemaPropertyType to SchemaType

* Remove inline

* Assert of schema properties

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-11 09:20:15 +02:00
Jure Bajic
2998f92595 Add initial schema implementation
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-08 10:33:43 +02:00
János Benjamin Antal
1bdc32ba5d Copy storage v2 to create storage v3 (#416)
* Copy storage v2 to v3

* Integrate v3 to cmake

* Fix clang-tidy warnings

* Add dummy unit test for storage-v3 to trigger build for code analysis builds
2022-07-05 08:20:59 +02:00
János Benjamin Antal
21870a0e7e Merge branch 'master' into project-pineapples 2022-06-23 14:49:14 +02:00
527 changed files with 89998 additions and 26267 deletions

View File

@@ -6,6 +6,7 @@ Checks: '*,
-altera-unroll-loops,
-android-*,
-cert-err58-cpp,
-cert-str34-c,
-cppcoreguidelines-avoid-c-arrays,
-cppcoreguidelines-avoid-goto,
-cppcoreguidelines-avoid-magic-numbers,
@@ -49,6 +50,7 @@ Checks: '*,
-misc-non-private-member-variables-in-classes,
-modernize-avoid-c-arrays,
-modernize-concat-nested-namespaces,
-modernize-loop-convert,
-modernize-pass-by-value,
-modernize-use-equals-default,
-modernize-use-nodiscard,

View File

@@ -16,7 +16,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -26,7 +26,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -64,7 +64,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -75,7 +75,7 @@ jobs:
- name: Fetch all history for all tags and branches
run: git fetch
- name: Initialize deps
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
@@ -83,32 +83,6 @@ jobs:
# Initialize dependencies.
./init
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Python code analysis
run: |
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
for file in ${CHANGED_FILES}; do
echo ${file}
if [[ ${file} == *.py ]]; then
python3 -m black --check --diff ${file}
python3 -m isort --check-only --diff ${file}
fi
done
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
@@ -136,17 +110,27 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -extra-arg="-DMG_CLANG_TIDY_CHECK" -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
@@ -161,7 +145,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -187,7 +171,7 @@ jobs:
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
- name: Run drivers tests
run: |
@@ -222,7 +206,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -237,7 +221,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -262,7 +246,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
@@ -278,6 +262,15 @@ jobs:
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run simulation tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__simulation --output-on-failure -j$THREADS
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
@@ -319,13 +312,13 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
@@ -344,7 +337,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -369,7 +362,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: "Jepsen Report"
@@ -385,7 +378,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -39,7 +39,7 @@ jobs:
source /opt/toolchain-v4/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -extra-arg="-DMG_CLANG_TIDY_CHECK" -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results

View File

@@ -17,7 +17,7 @@ jobs:
run: |
./release/package/run.sh package centos-7
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
@@ -34,7 +34,7 @@ jobs:
run: |
./release/package/run.sh package centos-9
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
@@ -51,7 +51,7 @@ jobs:
run: |
./release/package/run.sh package debian-10
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
@@ -68,7 +68,7 @@ jobs:
run: |
./release/package/run.sh package debian-11
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
@@ -87,7 +87,7 @@ jobs:
./run.sh package debian-11 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: docker
path: build/output/docker/memgraph*.tar.gz
@@ -104,7 +104,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804
path: build/output/ubuntu-18.04/memgraph*.deb
@@ -121,7 +121,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004
path: build/output/ubuntu-20.04/memgraph*.deb
@@ -138,7 +138,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-22.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-2204
path: build/output/ubuntu-22.04/memgraph*.deb
@@ -155,7 +155,7 @@ jobs:
run: |
./release/package/run.sh package debian-11 --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
@@ -172,7 +172,7 @@ jobs:
run: |
./release/package/run.sh package debian-11-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11-arm
path: build/output/debian-11-arm/memgraph*.deb

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -225,7 +225,7 @@ jobs:
rpmlint memgraph*.rpm
- name: Save enterprise RPM package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise RPM package"
path: build/output/memgraph*.rpm
@@ -262,7 +262,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
@@ -324,7 +324,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -349,7 +349,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: "Jepsen Report"

View File

@@ -19,17 +19,17 @@ jobs:
DOCKER_REPOSITORY_NAME: memgraph
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
- name: Log in to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |

12
.gitignore vendored
View File

@@ -23,6 +23,8 @@ cmake-build-*
cmake/DownloadProject/
dist/
src/query/frontend/opencypher/generated/
src/query/v2/frontend/opencypher/generated/
src/parser/opencypher/generated
tags
ve/
ve3/
@@ -50,15 +52,25 @@ src/distributed/pull_produce_rpc_messages.hpp
src/distributed/storage_gc_rpc_messages.hpp
src/distributed/token_sharing_rpc_messages.hpp
src/distributed/updates_rpc_messages.hpp
src/query/v2/frontend/ast/ast.hpp
src/query/frontend/ast/ast.hpp
src/storage/v3/bindings/ast/ast.hpp
src/query/distributed/frontend/ast/ast_serialization.hpp
src/query/v2/distributed/frontend/ast/ast_serialization.hpp
src/durability/distributed/state_delta.hpp
src/durability/single_node/state_delta.hpp
src/durability/single_node_ha/state_delta.hpp
src/query/frontend/semantic/symbol.hpp
src/query/v2/frontend/semantic/symbol.hpp
src/expr/semantic/symbol.hpp
src/query/distributed/frontend/semantic/symbol_serialization.hpp
src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
src/query/distributed/plan/ops.hpp
src/query/v2/distributed/plan/ops.hpp
src/query/plan/operator.hpp
src/query/v2/plan/operator.hpp
src/parser/opencypher/generated
src/expr/semantic/symbol.hpp
src/raft/log_entry.hpp
src/raft/raft_rpc_messages.hpp
src/raft/snapshot_metadata.hpp

View File

@@ -6,14 +6,18 @@ repos:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.10.0
rev: 22.3.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.10.1
hooks:
- id: isort
name: isort (python)
args: # arguments to configure black
- --line-length=120
- --include='\.pyi?$'
# these folders wont be formatted by black
- --exclude="""\.git |
\.__pycache__|
build|
libs|
.cache"""
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: v13.0.0
hooks:

View File

@@ -185,8 +185,6 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-Werror=switch -Werror=switch-bool -Werror=return-type \
-Werror=return-stack-address \
-Wno-c99-designator \
-mcpu=native \
-mtune=native \
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.

View File

@@ -22,7 +22,7 @@ Build modern, graph-based applications on top of your streaming data in minutes.
<p align="center">
<a href="https://github.com/memgraph/memgraph">
<img src="https://img.shields.io/github/actions/workflow/status/memgraph/memgraph/release_debian10.yaml?branch=master&label=build%20and%20test&logo=github"/>
<img src="https://img.shields.io/github/workflow/status/memgraph/memgraph/Release%20Ubuntu%2020.04/master" alt="build" title="build"/>
</a>
<a href="https://memgraph.com/docs/" alt="Documentation">
<img src="https://img.shields.io/badge/documentation-Memgraph-orange" />
@@ -37,10 +37,9 @@ Build modern, graph-based applications on top of your streaming data in minutes.
## :clipboard: Description
Memgraph is an open source graph database built for real-time streaming and
compatible with Neo4j. Whether you're a developer or a data scientist with
interconnected data, Memgraph will get you the immediate actionable insights
fast.
Memgraph is a streaming graph application platform that helps you wrangle your
streaming data, build sophisticated models that you can query in real-time, and
develop graph applications.
Memgraph directly connects to your streaming infrastructure. You can ingest data
from sources like Kafka, SQL, or plain CSV files. Memgraph provides a standard
@@ -52,9 +51,8 @@ natural and effective way to model many real-world problems without relying on
complex SQL schemas.
Memgraph is implemented in C/C++ and leverages an in-memory first architecture
to ensure that youre getting the [best possible
performance](http://memgraph.com/benchgraph) consistently and without surprises.
Its also ACID-compliant and highly available.
to ensure that youre getting the best possible performance consistently and
without surprises. Its also ACID-compliant and highly available.
## :video_game: Memgraph Playground
@@ -143,15 +141,6 @@ Memgraph Community is available under the [BSL
license](./licenses/BSL.txt).</br> Memgraph Enterprise is available under the
[MEL license](./licenses/MEL.txt).
## 🙋 Community
- :purple_heart: [**Discord**](https://discord.gg/memgraph)
- :busts_in_silhouette: [**Discourse forum**](https://discourse.memgraph.com/)
- :open_file_folder: [**Memgraph GitHub**](https://github.com/memgraph)
- :bird: [**Twitter**](https://twitter.com/memgraphdb)
- :movie_camera:
[**YouTube**](https://www.youtube.com/channel/UCZ3HOJvHGxtQ_JHxOselBYg)
<p align="center">
<a href="#">
<img src="https://img.shields.io/badge/⬆back_to_top_⬆-white" alt="Back to top" title="Back to top"/>

View File

@@ -5,10 +5,12 @@ import os
import subprocess
import sys
import textwrap
import xml.etree.ElementTree as ET
import yaml
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(SCRIPT_DIR, "flags.yaml")
WIDTH = 80
@@ -16,13 +18,14 @@ WIDTH = 80
def wrap_text(s, initial_indent="# "):
return "\n#\n".join(
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "), s.split("\n"))
)
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent,
subsequent_indent="# "), s.split("\n")))
def extract_flags(binary_path):
ret = {}
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
data = subprocess.run([binary_path, "--help-xml"],
stdout=subprocess.PIPE).stdout.decode("utf-8")
root = ET.fromstring(data)
for child in root:
if child.tag == "usage" and child.text.lower().count("warning"):
@@ -43,7 +46,8 @@ def apply_config_to_flags(config, flags):
for modification in config["modifications"]:
name = modification["name"]
if name not in flags:
print("WARNING: Flag '" + name + "' missing from binary!", file=sys.stderr)
print("WARNING: Flag '" + name + "' missing from binary!",
file=sys.stderr)
continue
flags[name]["default"] = modification["value"]
flags[name]["override"] = modification["override"]
@@ -71,9 +75,8 @@ def extract_sections(flags):
else:
sections.append((current_section, current_flags))
sections.append(("other", other))
assert set(sum(map(lambda x: x[1], sections), [])) == set(
flags.keys()
), "The section extraction algorithm lost some flags!"
assert set(sum(map(lambda x: x[1], sections), [])) == set(flags.keys()), \
"The section extraction algorithm lost some flags!"
return sections
@@ -86,7 +89,8 @@ def generate_config_file(sections, flags):
helpstr = flag["meaning"] + " [" + flag["type"] + "]"
ret += wrap_text(helpstr) + "\n"
prefix = "# " if not flag["override"] else ""
ret += prefix + "--" + flag["name"].replace("_", "-") + "=" + flag["default"] + "\n\n"
ret += prefix + "--" + flag["name"].replace("_", "-") + \
"=" + flag["default"] + "\n\n"
ret += "\n"
ret += wrap_text(config["footer"])
return ret.strip() + "\n"
@@ -94,9 +98,13 @@ def generate_config_file(sections, flags):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("memgraph_binary", help="path to Memgraph binary")
parser.add_argument("output_file", help="path where to store the generated Memgraph " "configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE, help="path to generator configuration file")
parser.add_argument("memgraph_binary",
help="path to Memgraph binary")
parser.add_argument("output_file",
help="path where to store the generated Memgraph "
"configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE,
help="path to generator configuration file")
args = parser.parse_args()
flags = extract_flags(args.memgraph_binary)

View File

@@ -1,204 +0,0 @@
# CSV Import Tool Documentation
CSV is a universal and very versatile data format used to store large quantities
of data. Each Memgraph database instance has a CSV import tool installed called
`mg_import_csv`. The CSV import tool should be used for initial bulk ingestion
of data into the database. Upon ingestion, the CSV importer creates a snapshot
that will be used by the database to recover its state on its next startup.
If you are already familiar with the Neo4j bulk import tool, then using the
`mg_import_csv` tool should be easy. The CSV import tool is fully compatible
with the [Neo4j CSV
format](https://neo4j.com/docs/operations-manual/current/tools/import/). If you
already have a pipeline set-up for Neo4j, you should only replace `neo4j-admin
import` with `mg_import_csv`.
## CSV File Format
Each row of a CSV file represents a single entry that should be imported into
the database. Both nodes and relationships can be imported into the database
using CSV files.
Each set of CSV files must have a header that describes the data that is stored
in the CSV files. Each field in the CSV header is in the format
`<name>[:<type>]` which identifies the name that should be used for that column
and the type that should be used for that column. The type is optional and
defaults to `string` (see the following chapter).
Each CSV field must be divided using the delimiter and each CSV field can either
be quoted or unquoted. When the field is quoted, the first and last character in
the field *must* be the quote character. If the field isn't quoted, and a quote
character appears in it, it is treated as a regular character. If a quote
character appears inside a quoted string then the quote character must be
doubled in order to escape it. Line feeds and carriage returns are ignored in
the CSV file, also, the file can't contain a NULL character.
## Properties
Both nodes and relationships can have properties added to them. When importing
properties, the CSV importer uses the name specified in the header of the
corresponding CSV column for the name of the property. A property is designated
by specifying one of the following types in the header:
- `integer`, `int`, `long`, `byte`, `short`: creates an integer property
- `float`, `double`: creates a float property
- `boolean`, `bool`: creates a boolean property
- `string`, `char`: creates a string property
When importing a boolean value, the CSV field should contain exactly the text
`true` to import a `True` boolean value. All other text values are treated as a
boolean value `False`.
If you want to import an array of values, you can do so by appending `[]` to any
of the above types. The values of the array are then determined by splitting
the raw CSV value using the array delimiter character.
Assuming that the array delimiter is `;`, the following example:
```plaintext
first_name,last_name:string,number:integer,aliases:string[]
John,Doe,1,Johnny;Jo;J-man
Melissa,Doe,2,Mel
```
Will yield these results:
```plaintext
CREATE ({first_name: "John", last_name: "Doe", number: 1, aliases: ["Johnny", "Jo", "J-man"]});
CREATE ({first_name: "Melissa", last_name: "Doe", number: 2, aliases: ["Mel"]});
```
### Nodes
When importing nodes, several more types can be specified in the header of the
CSV file (along with all property types):
- `ID`: id of the node that should be used as the node ID when importing
relationships
- `LABEL`: designates that the field contains additional labels for the node
- `IGNORE`: designates that the field should be ignored
The `ID` field type sets the internal ID that will be used for the node when
creating relationships. It is optional and nodes that don't have an ID value
specified will be imported, but can't be connected to any relationships. If you
want to save the ID value as a property in the database, just specify a name for
the ID (`user_id:ID`). If you just want to use the ID during the import, leave
out the name of the field (`:ID`). The `ID` field also supports creating
separate ID spaces. The ID space is specified with the ID space name appended
to the `ID` type in parentheses (`ID(user)`). That allows you to have the same
IDs (by value) for multiple different node files (for example, numbers from 1 to
N). The IDs in each ID space will be treated as an independent set of IDs that
don't interfere with IDs in another ID space.
The `LABEL` field type adds additional labels to the node. The value is treated
as an array type so that multiple additional labels can be specified for each
node. The value is split using the array delimiter (`--array-delimiter` flag).
### Relationships
In order to be able to import relationships, you must import the nodes in the
same invocation of `mg_import_csv` that is used to import the relationships.
When importing relationships, several more types can be specified in the header
of the CSV file (along with all property types):
- `START_ID`: id of the start node that should be connected with the
relationship
- `END_ID`: id of the end node that should be connected with the relationship
- `TYPE`: designates the type of the relationship
- `IGNORE`: designates that the field should be ignored
The `START_ID` field type sets the start node that should be connected with the
relationship to the end node. The field *must* be specified and the node ID
must be one of the node IDs that were specified in the node CSV files. The name
of this field is ignored. If the node ID is in an ID space, you can specify the
ID space for the in the same way as for the node ID (`START_ID(user)`).
The `END_ID` field type sets the end node that should be connected with the
relationship to the start node. The field *must* be specified and the node ID
must be one of the node IDs that were specified in the node CSV files. The name
of this field is ignored. If the node ID is in an ID space, you can specify the
ID space for the in the same way as for the node ID (`END_ID(user)`).
The `TYPE` field type sets the type of the relationship. Each relationship
*must* have a relationship type, but it doesn't necessarily need to be specified
in the CSV file, it can also be set externally for the whole CSV file. The name
of this field is ignored.
## CSV Importer Flags
The importer has many command line options that allow you to customize the way
the importer loads your data.
The two main flags that are used to specify the input CSV files are `--nodes`
and `--relationships`. Basic description of these flags is provided in the table
and more detailed explainion can be found further down bellow.
| Flag | Description |
|-----------------------| -------------- |
|`--nodes` | Used to specify CSV files that contain the nodes to the importer. |
|`--relationships` | Used to specify CSV files that contain the relationships to the importer.|
|`--delimiter` | Sets the delimiter that should be used when splitting the CSV fields (default `,`)|
|`--quote` | Sets the quote character that should be used to quote a CSV field (default `"`)|
|`--array-delimiter` | Sets the delimiter that should be used when splitting array values (default `;`)|
|`--id-type` | Specifies which data type should be used to store the supplied <br /> node IDs when storing them as properties (if the field name is supplied). <br /> The supported values are either `STRING` or `INTEGER`. (default `STRING`)|
|`--ignore-empty-strings` | Instructs the importer to treat all empty strings as `Null` values <br /> instead of an empty string value (default `false`)|
|`--ignore-extra-columns` | Instructs the importer to ignore all columns (instead of raising an error) <br /> that aren't specified after the last specified column in the CSV header. (default `false`) |
| `--skip-bad-relationships`| Instructs the importer to ignore all relationships (instead of raising an error) <br /> that refer to nodes that don't exist in the node files. (default `false`) |
|`--skip-duplicate-nodes` | Instructs the importer to ignore all duplicate nodes (instead of raising an error). <br /> Duplicate nodes are nodes that have an ID that is the same as another node that was already imported. (default `false`) |
| `--trim-strings`| Instructs the importer to trim all of the loaded CSV field values before processing them further. <br /> Trimming the fields removes all leading and trailing whitespace from them. (default `false`) |
The `--nodes` and `--relationships` flags are used to specify CSV files that
contain the nodes and relationships to the importer. Multiple files can be
specified in each supplied `--nodes` or `--relationships` flag. Files that are
supplied in one `--nodes` or `--relationships` flag are treated by the CSV
parser as one big CSV file. Only the first line of the first file is parsed for
the CSV header, all other files (and rows) are treated as data. This is useful
when you have a very large CSV file and don't want to edit its first line just
to add a CSV header. Instead, you can specify the header in a separate file
(e.g. `users_header.csv` or `friendships_header.csv`) and have the data intact
in the large file (e.g. `users.csv` or `friendships.csv`). Also, you can supply
additional labels for each set of node files.
The format of `--nodes` flag is:
`[<label>[:<label>]...=]<file>[,<file>][,<file>]...`. Take note that only the
first `<file>` part is mandatory, all other parts of the flag value are
optional. Multiple `--nodes` flags can be supplied to describe multiple sets of
different node files. For the importer to work, at least one `--nodes` flag
*must* be supplied.
The format of `--relationships` flag is: `[<type>=]<file>[,<file>][,<file>]...`.
Take note that only the first `<file>` part is mandatory, all other parts of the
flag value are optional. Multiple `--relationships` flags can be supplied to
describe multiple sets of different relationship files. The `--relationships`
flag isn't mandatory.
## CSV Parser Logic
The CSV parser uses the same logic as the standard Python CSV parser. The data
is parsed in the same way as the following snippet:
```python
import csv
for row in csv.reader(stream, strict=True):
# process 'row'
```
Python uses 'excel' as the default dialect when parsing CSV files and the
default settings for the CSV parser are:
- delimiter: `','`
- doublequote: `True`
- escapechar: `None`
- lineterminator: `'\r\n'`
- quotechar: `'"'`
- skipinitialspace: `False`
The above snippet can be expanded to:
```python
import csv
for row in csv.reader(stream, delimiter=',', doublequote=True,
escapechar=None, lineterminator='\r\n',
quotechar='"', skipinitialspace=False,
strict=True):
# process 'row'
```
For more information about the meaning of the above values, see:
https://docs.python.org/3/library/csv.html#csv.Dialect

View File

@@ -6,7 +6,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "arm64" "aarch64"
check_architecture "arm64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools

View File

@@ -1,98 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "fedora-36"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
perl # for openssl
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1 +0,0 @@
*.tar.gz

View File

@@ -10,14 +10,6 @@ cd "$DIR"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
function log_tool_name () {
echo ""
echo ""
echo "#### $1 ####"
echo ""
echo ""
}
for_arm=false
if [[ "$#" -eq 1 ]]; then
if [[ "$1" == "--for-arm" ]]; then
@@ -28,11 +20,9 @@ if [[ "$#" -eq 1 ]]; then
fi
fi
TOOLCHAIN_STDCXX="${TOOLCHAIN_STDCXX:-libstdc++}"
if [[ "$TOOLCHAIN_STDCXX" != "libstdc++" && "$TOOLCHAIN_STDCXX" != "libc++" ]]; then
echo "Only GCC (libstdc++) or LLVM (libc++) C++ standard library implementations are supported."
exit 1
fi
os="$1"
# toolchain version
TOOLCHAIN_VERSION=4
# package versions used
@@ -109,8 +99,6 @@ if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxx-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxxabi-$LLVM_VERSION.src.tar.xz
fi
if [ ! -f pahole-gdb-master.zip ]; then
wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
@@ -168,8 +156,6 @@ if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxx-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxxabi-$LLVM_VERSION.src.tar.xz.sig
fi
# list of valid llvm gnupg keys: https://releases.llvm.org/download.html
$GPG --keyserver $KEYSERVER --recv-keys 0x474E22316ABF4785A88C6E8EA2C794A986419D8A
@@ -179,8 +165,6 @@ $GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
$GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
$GPG --verify libunwind-$LLVM_VERSION.src.tar.xz.sig libunwind-$LLVM_VERSION.src.tar.xz
$GPG --verify libcxx-$LLVM_VERSION.src.tar.xz.sig libcxx-$LLVM_VERSION.src.tar.xz
$GPG --verify libcxxabi-$LLVM_VERSION.src.tar.xz.sig libcxxabi-$LLVM_VERSION.src.tar.xz
popd
@@ -188,7 +172,7 @@ popd
mkdir -p build
pushd build
log_tool_name "GCC $GCC_VERSION"
# compile gcc
if [ ! -f $PREFIX/bin/gcc ]; then
if [ -d gcc-$GCC_VERSION ]; then
rm -rf gcc-$GCC_VERSION
@@ -279,7 +263,7 @@ fi
export PATH=$PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$PREFIX/lib64
log_tool_name "binutils $BINUTILS_VERSION"
# compile binutils
if [ ! -f $PREFIX/bin/ld.gold ]; then
if [ -d binutils-$BINUTILS_VERSION ]; then
rm -rf binutils-$BINUTILS_VERSION
@@ -343,7 +327,7 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
popd && popd
fi
log_tool_name "GDB $GDB_VERSION"
# compile gdb
if [ ! -f $PREFIX/bin/gdb ]; then
if [ -d gdb-$GDB_VERSION ]; then
rm -rf gdb-$GDB_VERSION
@@ -414,13 +398,13 @@ if [ ! -f $PREFIX/bin/gdb ]; then
popd && popd
fi
log_tool_name "install pahole"
# install pahole
if [ ! -d $PREFIX/share/pahole-gdb ]; then
unzip ../archives/pahole-gdb-master.zip
mv pahole-gdb-master $PREFIX/share/pahole-gdb
fi
log_tool_name "setup system gdbinit"
# setup system gdbinit
if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
mkdir -p $PREFIX/etc/gdb
cat >$PREFIX/etc/gdb/gdbinit <<EOF
@@ -446,7 +430,7 @@ end
EOF
fi
log_tool_name "cmake $CMAKE_VERSION"
# compile cmake
if [ ! -f $PREFIX/bin/cmake ]; then
if [ -d cmake-$CMAKE_VERSION ]; then
rm -rf cmake-$CMAKE_VERSION
@@ -472,7 +456,7 @@ if [ ! -f $PREFIX/bin/cmake ]; then
popd && popd
fi
log_tool_name "cppcheck $CPPCHECK_VERSION"
# compile cppcheck
if [ ! -f $PREFIX/bin/cppcheck ]; then
if [ -d cppcheck-$CPPCHECK_VERSION ]; then
rm -rf cppcheck-$CPPCHECK_VERSION
@@ -496,7 +480,7 @@ if [ ! -f $PREFIX/bin/cppcheck ]; then
popd
fi
log_tool_name "swig $SWIG_VERSION"
# compile swig
if [ ! -d swig-$SWIG_VERSION/install ]; then
if [ -d swig-$SWIG_VERSION ]; then
rm -rf swig-$SWIG_VERSION
@@ -512,7 +496,7 @@ if [ ! -d swig-$SWIG_VERSION/install ]; then
popd && popd
fi
log_tool_name "LLVM $LLVM_VERSION"
# compile llvm
if [ ! -f $PREFIX/bin/clang ]; then
if [ -d llvm-$LLVM_VERSION ]; then
rm -rf llvm-$LLVM_VERSION
@@ -529,19 +513,8 @@ if [ ! -f $PREFIX/bin/clang ]; then
mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz
mv libunwind-$LLVM_VERSION.src/include/mach-o llvm-$LLVM_VERSION/tools/lld/include
# The following is required because of libc++
tar -xvf ../archives/libcxx-$LLVM_VERSION.src.tar.xz
mv libcxx-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libcxx
tar -xvf ../archives/libcxxabi-$LLVM_VERSION.src.tar.xz
mv libcxxabi-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libcxxabi
# NOTE: We moved part of the libunwind in one of the previous step.
rm -r libunwind-$LLVM_VERSION.src
tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz
mv libunwind-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libunwind
pushd llvm-$LLVM_VERSION
mkdir -p build && pushd build
mkdir build && pushd build
# activate swig
export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
@@ -847,11 +820,7 @@ source $PREFIX/activate
export CC=$PREFIX/bin/clang
export CXX=$PREFIX/bin/clang++
export CFLAGS="$CFLAGS -fPIC"
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
export CXXFLAGS="$CXXFLAGS -fPIC"
else
export CXXFLAGS="$CXXFLAGS -fPIC -stdlib=libc++"
fi
export CXXFLAGS="$CXXFLAGS -fPIC"
COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
-DCMAKE_PREFIX_PATH=$PREFIX
-DCMAKE_BUILD_TYPE=Release
@@ -865,7 +834,7 @@ COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
COMMON_CONFIGURE_FLAGS="--enable-shared=no --prefix=$PREFIX"
COMMON_MAKE_INSTALL_FLAGS="-j$CPUS BUILD_SHARED=no PREFIX=$PREFIX install"
log_tool_name "bzip2 $BZIP2_VERSION"
# install bzip2
if [ ! -f $PREFIX/include/bzlib.h ]; then
if [ -d bzip2-$BZIP2_VERSION ]; then
rm -rf bzip2-$BZIP2_VERSION
@@ -876,7 +845,7 @@ if [ ! -f $PREFIX/include/bzlib.h ]; then
popd
fi
log_tool_name "fmt $FMT_VERSION"
# install fmt
if [ ! -d $PREFIX/include/fmt ]; then
if [ -d fmt-$FMT_VERSION ]; then
rm -rf fmt-$FMT_VERSION
@@ -889,7 +858,7 @@ if [ ! -d $PREFIX/include/fmt ]; then
popd && popd
fi
log_tool_name "lz4 $LZ4_VERSION"
# install lz4
if [ ! -f $PREFIX/include/lz4.h ]; then
if [ -d lz4-$LZ4_VERSION ]; then
rm -rf lz4-$LZ4_VERSION
@@ -900,7 +869,7 @@ if [ ! -f $PREFIX/include/lz4.h ]; then
popd
fi
log_tool_name "xz $XZ_VERSION"
# install xz
if [ ! -f $PREFIX/include/lzma.h ]; then
if [ -d xz-$XZ_VERSION ]; then
rm -rf xz-$XZ_VERSION
@@ -912,7 +881,7 @@ if [ ! -f $PREFIX/include/lzma.h ]; then
popd
fi
log_tool_name "zlib $ZLIB_VERSION"
# install zlib
if [ ! -f $PREFIX/include/zlib.h ]; then
if [ -d zlib-$ZLIB_VERSION ]; then
rm -rf zlib-$ZLIB_VERSION
@@ -926,7 +895,7 @@ if [ ! -f $PREFIX/include/zlib.h ]; then
popd && popd
fi
log_tool_name "zstd $ZSTD_VERSION"
# install zstd
if [ ! -f $PREFIX/include/zstd.h ]; then
if [ -d zstd-$ZSTD_VERSION ]; then
rm -rf zstd-$ZSTD_VERSION
@@ -941,8 +910,7 @@ if [ ! -f $PREFIX/include/zstd.h ]; then
popd && popd
fi
# TODO(gitbuda): Freeze jmalloc version.
log_tool_name "jmalloc"
#install jemalloc
if [ ! -d $PREFIX/include/jemalloc ]; then
if [ -d jemalloc ]; then
rm -rf jemalloc
@@ -959,7 +927,7 @@ if [ ! -d $PREFIX/include/jemalloc ]; then
popd
fi
log_tool_name "BOOST $BOOST_VERSION"
# install boost
if [ ! -d $PREFIX/include/boost ]; then
if [ -d boost_$BOOST_VERSION_UNDERSCORES ]; then
rm -rf boost_$BOOST_VERSION_UNDERSCORES
@@ -967,24 +935,15 @@ if [ ! -d $PREFIX/include/boost ]; then
tar -xzf ../archives/boost_$BOOST_VERSION_UNDERSCORES.tar.gz
pushd boost_$BOOST_VERSION_UNDERSCORES
./bootstrap.sh --prefix=$PREFIX --with-toolset=clang --with-python=python3 --without-icu
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
else
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
fi
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
popd
fi
log_tool_name "double-conversion $DOUBLE_CONVERSION_VERSION"
# install double-conversion
if [ ! -d $PREFIX/include/double-conversion ]; then
if [ -d double-conversion-$DOUBLE_CONVERSION_VERSION ]; then
rm -rf double-conversion-$DOUBLE_CONVERSION_VERSION
@@ -999,8 +958,7 @@ if [ ! -d $PREFIX/include/double-conversion ]; then
popd && popd
fi
# TODO(gitbuda): Freeze gflags version.
log_tool_name "gflags"
# install gflags
if [ ! -d $PREFIX/include/gflags ]; then
if [ -d gflags ]; then
rm -rf gflags
@@ -1019,7 +977,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
popd && popd
fi
log_tool_name "libunwind $LIBUNWIND_VERSION"
# install libunwind
if [ ! -f $PREFIX/include/libunwind.h ]; then
if [ -d libunwind-$LIBUNWIND_VERSION ]; then
rm -rf libunwind-$LIBUNWIND_VERSION
@@ -1032,7 +990,7 @@ if [ ! -f $PREFIX/include/libunwind.h ]; then
popd
fi
log_tool_name "glog $GLOG_VERSION"
# install glog
if [ ! -d $PREFIX/include/glog ]; then
if [ -d glog-$GLOG_VERSION ]; then
rm -rf glog-$GLOG_VERSION
@@ -1046,7 +1004,7 @@ if [ ! -d $PREFIX/include/glog ]; then
popd && popd
fi
log_tool_name "libevent $LIBEVENT_VERSION"
# install libevent
if [ ! -d $PREFIX/include/event2 ]; then
if [ -d libevent-$LIBEVENT_VERSION ]; then
rm -rf libevent-$LIBEVENT_VERSION
@@ -1065,7 +1023,7 @@ if [ ! -d $PREFIX/include/event2 ]; then
popd && popd
fi
log_tool_name "snappy $SNAPPY_VERSION"
# install snappy
if [ ! -f $PREFIX/include/snappy.h ]; then
if [ -d snappy-$SNAPPY_VERSION ]; then
rm -rf snappy-$SNAPPY_VERSION
@@ -1083,7 +1041,7 @@ if [ ! -f $PREFIX/include/snappy.h ]; then
popd && popd
fi
log_tool_name "libsodium $LIBSODIUM_VERSION"
# install libsodium
if [ ! -f $PREFIX/include/sodium.h ]; then
if [ -d libsodium-$LIBSODIUM_VERSION ]; then
rm -rf libsodium-$LIBSODIUM_VERSION
@@ -1095,7 +1053,7 @@ if [ ! -f $PREFIX/include/sodium.h ]; then
popd
fi
log_tool_name "libaio $LIBAIO_VERSION"
# install libaio
if [ ! -f $PREFIX/include/libaio.h ]; then
if [ -d libaio-$LIBAIO_VERSION ]; then
rm -rf libaio-$LIBAIO_VERSION
@@ -1106,7 +1064,7 @@ if [ ! -f $PREFIX/include/libaio.h ]; then
popd
fi
log_tool_name "folly $FBLIBS_VERSION"
# install folly
if [ ! -d $PREFIX/include/folly ]; then
if [ -d folly-$FBLIBS_VERSION ]; then
rm -rf folly-$FBLIBS_VERSION
@@ -1127,7 +1085,7 @@ if [ ! -d $PREFIX/include/folly ]; then
popd && popd
fi
log_tool_name "fizz $FBLIBS_VERSION"
# install fizz
if [ ! -d $PREFIX/include/fizz ]; then
if [ -d fizz-$FBLIBS_VERSION ]; then
rm -rf fizz-$FBLIBS_VERSION
@@ -1146,7 +1104,7 @@ if [ ! -d $PREFIX/include/fizz ]; then
popd && popd
fi
log_tool_name "wangle FBLIBS_VERSION"
# install wangle
if [ ! -d $PREFIX/include/wangle ]; then
if [ -d wangle-$FBLIBS_VERSION ]; then
rm -rf wangle-$FBLIBS_VERSION
@@ -1165,7 +1123,7 @@ if [ ! -d $PREFIX/include/wangle ]; then
popd && popd
fi
log_tool_name "proxygen $FBLIBS_VERSION"
# install proxygen
if [ ! -d $PREFIX/include/proxygen ]; then
if [ -d proxygen-$FBLIBS_VERSION ]; then
rm -rf proxygen-$FBLIBS_VERSION
@@ -1186,7 +1144,7 @@ if [ ! -d $PREFIX/include/proxygen ]; then
popd && popd
fi
log_tool_name "flex $FBLIBS_VERSION"
# install flex
if [ ! -f $PREFIX/include/FlexLexer.h ]; then
if [ -d flex-$FLEX_VERSION ]; then
rm -rf flex-$FLEX_VERSION
@@ -1198,7 +1156,7 @@ if [ ! -f $PREFIX/include/FlexLexer.h ]; then
popd
fi
log_tool_name "fbthrift $FBLIBS_VERSION"
# install fbthrift
if [ ! -d $PREFIX/include/thrift ]; then
if [ -d fbthrift-$FBLIBS_VERSION ]; then
rm -rf fbthrift-$FBLIBS_VERSION
@@ -1208,15 +1166,10 @@ if [ ! -d $PREFIX/include/thrift ]; then
# build is used by facebook builder
mkdir _build
pushd _build
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
CMAKE_CXX_FLAGS="-fsized-deallocation"
else
CMAKE_CXX_FLAGS="-fsized-deallocation -stdlib=libc++"
fi
cmake .. $COMMON_CMAKE_FLAGS \
-Denable_tests=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"
-DCMAKE_CXX_FLAGS=-fsized-deallocation
make -j$CPUS install
popd
fi
@@ -1239,12 +1192,7 @@ if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
fi
fi
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
# Pass because infra scripts assume there is not C++ standard lib in the name.
echo "NOTE: Not adding anything to the archive name that GCC C++ standard lib is used."
else
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-libc++"
fi
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
fi

View File

@@ -1,11 +1,11 @@
#!/bin/bash
function operating_system() {
operating_system() {
grep -E '^(VERSION_)?ID=' /etc/os-release | \
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
}
function check_operating_system() {
check_operating_system() {
if [ "$(operating_system)" != "$1" ]; then
echo "Not the right operating system!"
exit 1
@@ -14,22 +14,20 @@ function check_operating_system() {
fi
}
function architecture() {
architecture() {
uname -m
}
check_architecture() {
for arch in "$@"; do
if [ "$(architecture)" = "$arch" ]; then
echo "The right architecture!"
return 0
fi
done
echo "Not the right architecture!"
exit 1
if [ "$(architecture)" != "$1" ]; then
echo "Not the right architecture!"
exit 1
else
echo "The right architecture."
fi
}
function check_all_yum() {
check_all_yum() {
local missing=""
for pkg in $1; do
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
@@ -42,7 +40,7 @@ function check_all_yum() {
fi
}
function check_all_dpkg() {
check_all_dpkg() {
local missing=""
for pkg in $1; do
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
@@ -55,7 +53,7 @@ function check_all_dpkg() {
fi
}
function check_all_dnf() {
check_all_dnf() {
local missing=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
@@ -67,8 +65,7 @@ function check_all_dnf() {
exit 1
fi
}
function install_all_apt() {
install_all_apt() {
for pkg in $1; do
apt install -y "$pkg"
done

View File

@@ -1,726 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file _mgp.hpp
///
/// The file contains methods that connect mg procedures and the outside code
/// Methods like mapping a graph into memory or assigning new mg results or
/// their properties are implemented.
#pragma once
#include "mg_exceptions.hpp"
#include "mg_procedure.h"
namespace mgp {
namespace {
inline void MgExceptionHandle(mgp_error result_code) {
switch (result_code) {
case mgp_error::MGP_ERROR_UNKNOWN_ERROR:
throw mg_exception::UnknownException();
case mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE:
throw mg_exception::AllocationException();
case mgp_error::MGP_ERROR_INSUFFICIENT_BUFFER:
throw mg_exception::InsufficientBufferException();
case mgp_error::MGP_ERROR_OUT_OF_RANGE:
throw mg_exception::OutOfRangeException();
case mgp_error::MGP_ERROR_LOGIC_ERROR:
throw mg_exception::LogicException();
case mgp_error::MGP_ERROR_DELETED_OBJECT:
throw mg_exception::DeletedObjectException();
case mgp_error::MGP_ERROR_INVALID_ARGUMENT:
throw mg_exception::InvalidArgumentException();
case mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS:
throw mg_exception::KeyAlreadyExistsException();
case mgp_error::MGP_ERROR_IMMUTABLE_OBJECT:
throw mg_exception::ImmutableObjectException();
case mgp_error::MGP_ERROR_VALUE_CONVERSION:
throw mg_exception::ValueConversionException();
case mgp_error::MGP_ERROR_SERIALIZATION_ERROR:
throw mg_exception::SerializationException();
default:
return;
}
}
template <typename TResult, typename TFunc, typename... TArgs>
TResult MgInvoke(TFunc func, TArgs... args) {
TResult result{};
auto result_code = func(args..., &result);
MgExceptionHandle(result_code);
return result;
}
template <typename TFunc, typename... TArgs>
inline void MgInvokeVoid(TFunc func, TArgs... args) {
auto result_code = func(args...);
MgExceptionHandle(result_code);
}
} // namespace
// mgp_value
// Make value
inline mgp_value *value_make_null(mgp_memory *memory) { return MgInvoke<mgp_value *>(mgp_value_make_null, memory); }
inline mgp_value *value_make_bool(int val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_bool, val, memory);
}
inline mgp_value *value_make_int(int64_t val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_int, val, memory);
}
inline mgp_value *value_make_double(double val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_double, val, memory);
}
inline mgp_value *value_make_string(const char *val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_string, val, memory);
}
inline mgp_value *value_make_list(mgp_list *val) { return MgInvoke<mgp_value *>(mgp_value_make_list, val); }
inline mgp_value *value_make_map(mgp_map *val) { return MgInvoke<mgp_value *>(mgp_value_make_map, val); }
inline mgp_value *value_make_vertex(mgp_vertex *val) { return MgInvoke<mgp_value *>(mgp_value_make_vertex, val); }
inline mgp_value *value_make_edge(mgp_edge *val) { return MgInvoke<mgp_value *>(mgp_value_make_edge, val); }
inline mgp_value *value_make_path(mgp_path *val) { return MgInvoke<mgp_value *>(mgp_value_make_path, val); }
inline mgp_value *value_make_date(mgp_date *val) { return MgInvoke<mgp_value *>(mgp_value_make_date, val); }
inline mgp_value *value_make_local_time(mgp_local_time *val) {
return MgInvoke<mgp_value *>(mgp_value_make_local_time, val);
}
inline mgp_value *value_make_local_date_time(mgp_local_date_time *val) {
return MgInvoke<mgp_value *>(mgp_value_make_local_date_time, val);
}
inline mgp_value *value_make_duration(mgp_duration *val) { return MgInvoke<mgp_value *>(mgp_value_make_duration, val); }
// Copy value
// TODO: implement within MGP API
// with primitive types ({bool, int, double, string}), create a new identical value
// otherwise call mgp_##TYPE_copy and convert tpye
inline mgp_value *value_copy(mgp_value *val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_copy, val, memory);
}
// Destroy value
inline void value_destroy(mgp_value *val) { mgp_value_destroy(val); }
// Get value of type
inline mgp_value_type value_get_type(mgp_value *val) { return MgInvoke<mgp_value_type>(mgp_value_get_type, val); }
inline bool value_get_bool(mgp_value *val) { return MgInvoke<int>(mgp_value_get_bool, val); }
inline int64_t value_get_int(mgp_value *val) { return MgInvoke<int64_t>(mgp_value_get_int, val); }
inline double value_get_double(mgp_value *val) { return MgInvoke<double>(mgp_value_get_double, val); }
inline const char *value_get_string(mgp_value *val) { return MgInvoke<const char *>(mgp_value_get_string, val); }
inline mgp_list *value_get_list(mgp_value *val) { return MgInvoke<mgp_list *>(mgp_value_get_list, val); }
inline mgp_map *value_get_map(mgp_value *val) { return MgInvoke<mgp_map *>(mgp_value_get_map, val); }
inline mgp_vertex *value_get_vertex(mgp_value *val) { return MgInvoke<mgp_vertex *>(mgp_value_get_vertex, val); }
inline mgp_edge *value_get_edge(mgp_value *val) { return MgInvoke<mgp_edge *>(mgp_value_get_edge, val); }
inline mgp_path *value_get_path(mgp_value *val) { return MgInvoke<mgp_path *>(mgp_value_get_path, val); }
inline mgp_date *value_get_date(mgp_value *val) { return MgInvoke<mgp_date *>(mgp_value_get_date, val); }
inline mgp_local_time *value_get_local_time(mgp_value *val) {
return MgInvoke<mgp_local_time *>(mgp_value_get_local_time, val);
}
inline mgp_local_date_time *value_get_local_date_time(mgp_value *val) {
return MgInvoke<mgp_local_date_time *>(mgp_value_get_local_date_time, val);
}
inline mgp_duration *value_get_duration(mgp_value *val) {
return MgInvoke<mgp_duration *>(mgp_value_get_duration, val);
}
// Check type of value
inline bool value_is_null(mgp_value *val) { return MgInvoke<int>(mgp_value_is_null, val); }
inline bool value_is_bool(mgp_value *val) { return MgInvoke<int>(mgp_value_is_bool, val); }
inline bool value_is_int(mgp_value *val) { return MgInvoke<int>(mgp_value_is_int, val); }
inline bool value_is_double(mgp_value *val) { return MgInvoke<int>(mgp_value_is_double, val); }
inline bool value_is_string(mgp_value *val) { return MgInvoke<int>(mgp_value_is_string, val); }
inline bool value_is_list(mgp_value *val) { return MgInvoke<int>(mgp_value_is_list, val); }
inline bool value_is_map(mgp_value *val) { return MgInvoke<int>(mgp_value_is_map, val); }
inline bool value_is_vertex(mgp_value *val) { return MgInvoke<int>(mgp_value_is_vertex, val); }
inline bool value_is_edge(mgp_value *val) { return MgInvoke<int>(mgp_value_is_edge, val); }
inline bool value_is_path(mgp_value *val) { return MgInvoke<int>(mgp_value_is_path, val); }
inline bool value_is_date(mgp_value *val) { return MgInvoke<int>(mgp_value_is_date, val); }
inline bool value_is_local_time(mgp_value *val) { return MgInvoke<int>(mgp_value_is_local_time, val); }
inline bool value_is_local_date_time(mgp_value *val) { return MgInvoke<int>(mgp_value_is_local_date_time, val); }
inline bool value_is_duration(mgp_value *val) { return MgInvoke<int>(mgp_value_is_duration, val); }
// Get type
inline mgp_type *type_any() { return MgInvoke<mgp_type *>(mgp_type_any); }
inline mgp_type *type_bool() { return MgInvoke<mgp_type *>(mgp_type_bool); }
inline mgp_type *type_string() { return MgInvoke<mgp_type *>(mgp_type_string); }
inline mgp_type *type_int() { return MgInvoke<mgp_type *>(mgp_type_int); }
inline mgp_type *type_float() { return MgInvoke<mgp_type *>(mgp_type_float); }
inline mgp_type *type_number() { return MgInvoke<mgp_type *>(mgp_type_number); }
inline mgp_type *type_list(mgp_type *element_type) { return MgInvoke<mgp_type *>(mgp_type_list, element_type); }
inline mgp_type *type_map() { return MgInvoke<mgp_type *>(mgp_type_map); }
inline mgp_type *type_node() { return MgInvoke<mgp_type *>(mgp_type_node); }
inline mgp_type *type_relationship() { return MgInvoke<mgp_type *>(mgp_type_relationship); }
inline mgp_type *type_path() { return MgInvoke<mgp_type *>(mgp_type_path); }
inline mgp_type *type_date() { return MgInvoke<mgp_type *>(mgp_type_date); }
inline mgp_type *type_local_time() { return MgInvoke<mgp_type *>(mgp_type_local_time); }
inline mgp_type *type_local_date_time() { return MgInvoke<mgp_type *>(mgp_type_local_date_time); }
inline mgp_type *type_duration() { return MgInvoke<mgp_type *>(mgp_type_duration); }
inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke<mgp_type *>(mgp_type_nullable, type); }
// mgp_graph
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_graph_create_vertex, graph, memory);
}
inline void graph_delete_vertex(mgp_graph *graph, mgp_vertex *vertex) {
MgInvokeVoid(mgp_graph_delete_vertex, graph, vertex);
}
inline void graph_detach_delete_vertex(mgp_graph *graph, mgp_vertex *vertex) {
MgInvokeVoid(mgp_graph_detach_delete_vertex, graph, vertex);
}
inline mgp_edge *graph_create_edge(mgp_graph *graph, mgp_vertex *from, mgp_vertex *to, mgp_edge_type type,
mgp_memory *memory) {
return MgInvoke<mgp_edge *>(mgp_graph_create_edge, graph, from, to, type, memory);
}
inline void graph_delete_edge(mgp_graph *graph, mgp_edge *edge) { MgInvokeVoid(mgp_graph_delete_edge, graph, edge); }
inline mgp_vertex *graph_get_vertex_by_id(mgp_graph *g, mgp_vertex_id id, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_graph_get_vertex_by_id, g, id, memory);
}
inline mgp_vertices_iterator *graph_iter_vertices(mgp_graph *g, mgp_memory *memory) {
return MgInvoke<mgp_vertices_iterator *>(mgp_graph_iter_vertices, g, memory);
}
// mgp_vertices_iterator
inline void vertices_iterator_destroy(mgp_vertices_iterator *it) { mgp_vertices_iterator_destroy(it); }
inline mgp_vertex *vertices_iterator_get(mgp_vertices_iterator *it) {
return MgInvoke<mgp_vertex *>(mgp_vertices_iterator_get, it);
}
inline mgp_vertex *vertices_iterator_next(mgp_vertices_iterator *it) {
return MgInvoke<mgp_vertex *>(mgp_vertices_iterator_next, it);
}
// mgp_edges_iterator
inline void edges_iterator_destroy(mgp_edges_iterator *it) { mgp_edges_iterator_destroy(it); }
inline mgp_edge *edges_iterator_get(mgp_edges_iterator *it) { return MgInvoke<mgp_edge *>(mgp_edges_iterator_get, it); }
inline mgp_edge *edges_iterator_next(mgp_edges_iterator *it) {
return MgInvoke<mgp_edge *>(mgp_edges_iterator_next, it);
}
// mgp_properties_iterator
inline void properties_iterator_destroy(mgp_properties_iterator *it) { mgp_properties_iterator_destroy(it); }
inline mgp_property *properties_iterator_get(mgp_properties_iterator *it) {
return MgInvoke<mgp_property *>(mgp_properties_iterator_get, it);
}
inline mgp_property *properties_iterator_next(mgp_properties_iterator *it) {
return MgInvoke<mgp_property *>(mgp_properties_iterator_next, it);
}
// Container {mgp_list, mgp_map} methods
// mgp_list
inline mgp_list *list_make_empty(size_t capacity, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_make_empty, capacity, memory);
}
inline mgp_list *list_copy(mgp_list *list, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_copy, list, memory);
}
inline void list_destroy(mgp_list *list) { mgp_list_destroy(list); }
inline void list_append(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append, list, val); }
inline void list_append_extend(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append_extend, list, val); }
inline size_t list_size(mgp_list *list) { return MgInvoke<size_t>(mgp_list_size, list); }
inline size_t list_capacity(mgp_list *list) { return MgInvoke<size_t>(mgp_list_capacity, list); }
inline mgp_value *list_at(mgp_list *list, size_t index) { return MgInvoke<mgp_value *>(mgp_list_at, list, index); }
// mgp_map
inline mgp_map *map_make_empty(mgp_memory *memory) { return MgInvoke<mgp_map *>(mgp_map_make_empty, memory); }
inline mgp_map *map_copy(mgp_map *map, mgp_memory *memory) { return MgInvoke<mgp_map *>(mgp_map_copy, map, memory); }
inline void map_destroy(mgp_map *map) { mgp_map_destroy(map); }
inline void map_insert(mgp_map *map, const char *key, mgp_value *value) {
MgInvokeVoid(mgp_map_insert, map, key, value);
}
inline size_t map_size(mgp_map *map) { return MgInvoke<size_t>(mgp_map_size, map); }
inline mgp_value *map_at(mgp_map *map, const char *key) { return MgInvoke<mgp_value *>(mgp_map_at, map, key); }
inline const char *map_item_key(mgp_map_item *item) { return MgInvoke<const char *>(mgp_map_item_key, item); }
inline mgp_value *map_item_value(mgp_map_item *item) { return MgInvoke<mgp_value *>(mgp_map_item_value, item); }
inline mgp_map_items_iterator *map_iter_items(mgp_map *map, mgp_memory *memory) {
return MgInvoke<mgp_map_items_iterator *>(mgp_map_iter_items, map, memory);
}
inline void map_items_iterator_destroy(mgp_map_items_iterator *it) { mgp_map_items_iterator_destroy(it); }
inline mgp_map_item *map_items_iterator_get(mgp_map_items_iterator *it) {
return MgInvoke<mgp_map_item *>(mgp_map_items_iterator_get, it);
}
inline mgp_map_item *map_items_iterator_next(mgp_map_items_iterator *it) {
return MgInvoke<mgp_map_item *>(mgp_map_items_iterator_next, it);
}
// mgp_vertex
inline mgp_vertex_id vertex_get_id(mgp_vertex *v) { return MgInvoke<mgp_vertex_id>(mgp_vertex_get_id, v); }
inline mgp_vertex *vertex_copy(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_vertex_copy, v, memory);
}
inline void vertex_destroy(mgp_vertex *v) { mgp_vertex_destroy(v); }
inline bool vertex_equal(mgp_vertex *v1, mgp_vertex *v2) { return MgInvoke<int>(mgp_vertex_equal, v1, v2); }
inline size_t vertex_labels_count(mgp_vertex *v) { return MgInvoke<size_t>(mgp_vertex_labels_count, v); }
inline mgp_label vertex_label_at(mgp_vertex *v, size_t index) {
return MgInvoke<mgp_label>(mgp_vertex_label_at, v, index);
}
inline bool vertex_has_label(mgp_vertex *v, mgp_label label) { return MgInvoke<int>(mgp_vertex_has_label, v, label); }
inline bool vertex_has_label_named(mgp_vertex *v, const char *label_name) {
return MgInvoke<int>(mgp_vertex_has_label_named, v, label_name);
}
inline void vertex_add_label(mgp_vertex *vertex, mgp_label label) { MgInvokeVoid(mgp_vertex_add_label, vertex, label); }
inline mgp_value *vertex_get_property(mgp_vertex *v, const char *property_name, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_vertex_get_property, v, property_name, memory);
}
inline mgp_properties_iterator *vertex_iter_properties(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_properties_iterator *>(mgp_vertex_iter_properties, v, memory);
}
inline mgp_edges_iterator *vertex_iter_in_edges(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_edges_iterator *>(mgp_vertex_iter_in_edges, v, memory);
}
inline mgp_edges_iterator *vertex_iter_out_edges(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_edges_iterator *>(mgp_vertex_iter_out_edges, v, memory);
}
// mgp_edge
inline mgp_edge_id edge_get_id(mgp_edge *e) { return MgInvoke<mgp_edge_id>(mgp_edge_get_id, e); }
inline mgp_edge *edge_copy(mgp_edge *e, mgp_memory *memory) { return MgInvoke<mgp_edge *>(mgp_edge_copy, e, memory); }
inline void edge_destroy(mgp_edge *e) { mgp_edge_destroy(e); }
inline bool edge_equal(mgp_edge *e1, mgp_edge *e2) { return MgInvoke<int>(mgp_edge_equal, e1, e2); }
inline mgp_edge_type edge_get_type(mgp_edge *e) { return MgInvoke<mgp_edge_type>(mgp_edge_get_type, e); }
inline mgp_vertex *edge_get_from(mgp_edge *e) { return MgInvoke<mgp_vertex *>(mgp_edge_get_from, e); }
inline mgp_vertex *edge_get_to(mgp_edge *e) { return MgInvoke<mgp_vertex *>(mgp_edge_get_to, e); }
inline mgp_value *edge_get_property(mgp_edge *e, const char *property_name, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_edge_get_property, e, property_name, memory);
}
inline mgp_properties_iterator *edge_iter_properties(mgp_edge *e, mgp_memory *memory) {
return MgInvoke<mgp_properties_iterator *>(mgp_edge_iter_properties, e, memory);
}
// mgp_path
inline mgp_path *path_make_with_start(mgp_vertex *vertex, mgp_memory *memory) {
return MgInvoke<mgp_path *>(mgp_path_make_with_start, vertex, memory);
}
inline mgp_path *path_copy(mgp_path *path, mgp_memory *memory) {
return MgInvoke<mgp_path *>(mgp_path_copy, path, memory);
}
inline void path_destroy(mgp_path *path) { mgp_path_destroy(path); }
inline void path_expand(mgp_path *path, mgp_edge *edge) { MgInvokeVoid(mgp_path_expand, path, edge); }
inline size_t path_size(mgp_path *path) { return MgInvoke<size_t>(mgp_path_size, path); }
inline mgp_vertex *path_vertex_at(mgp_path *path, size_t index) {
return MgInvoke<mgp_vertex *>(mgp_path_vertex_at, path, index);
}
inline mgp_edge *path_edge_at(mgp_path *path, size_t index) {
return MgInvoke<mgp_edge *>(mgp_path_edge_at, path, index);
}
inline bool path_equal(mgp_path *p1, mgp_path *p2) { return MgInvoke<int>(mgp_path_equal, p1, p2); }
// Temporal type {mgp_date, mgp_local_time, mgp_local_date_time, mgp_duration} methods
// mgp_date
inline mgp_date *date_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_from_string, string, memory);
}
inline mgp_date *date_from_parameters(mgp_date_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_from_parameters, parameters, memory);
}
inline mgp_date *date_copy(mgp_date *date, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_copy, date, memory);
}
inline void date_destroy(mgp_date *date) { mgp_date_destroy(date); }
inline bool date_equal(mgp_date *first, mgp_date *second) { return MgInvoke<int>(mgp_date_equal, first, second); }
inline int date_get_year(mgp_date *date) { return MgInvoke<int>(mgp_date_get_year, date); }
inline int date_get_month(mgp_date *date) { return MgInvoke<int>(mgp_date_get_month, date); }
inline int date_get_day(mgp_date *date) { return MgInvoke<int>(mgp_date_get_day, date); }
inline int64_t date_timestamp(mgp_date *date) { return MgInvoke<int64_t>(mgp_date_timestamp, date); }
inline mgp_date *date_now(mgp_memory *memory) { return MgInvoke<mgp_date *>(mgp_date_now, memory); }
inline mgp_date *date_add_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_add_duration, date, dur, memory);
}
inline mgp_date *date_sub_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_sub_duration, date, dur, memory);
}
inline mgp_duration *date_diff(mgp_date *first, mgp_date *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_date_diff, first, second, memory);
}
// mgp_local_time
inline mgp_local_time *local_time_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_from_string, string, memory);
}
inline mgp_local_time *local_time_from_parameters(mgp_local_time_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_from_parameters, parameters, memory);
}
inline mgp_local_time *local_time_copy(mgp_local_time *local_time, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_copy, local_time, memory);
}
inline void local_time_destroy(mgp_local_time *local_time) { mgp_local_time_destroy(local_time); }
inline bool local_time_equal(mgp_local_time *first, mgp_local_time *second) {
return MgInvoke<int>(mgp_local_time_equal, first, second);
}
inline int local_time_get_hour(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_hour, local_time);
}
inline int local_time_get_minute(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_minute, local_time);
}
inline int local_time_get_second(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_second, local_time);
}
inline int local_time_get_millisecond(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_millisecond, local_time);
}
inline int local_time_get_microsecond(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_microsecond, local_time);
}
inline int64_t local_time_timestamp(mgp_local_time *local_time) {
return MgInvoke<int64_t>(mgp_local_time_timestamp, local_time);
}
inline mgp_local_time *local_time_now(mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_now, memory);
}
inline mgp_local_time *local_time_add_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_add_duration, local_time, dur, memory);
}
inline mgp_local_time *local_time_sub_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_sub_duration, local_time, dur, memory);
}
inline mgp_duration *local_time_diff(mgp_local_time *first, mgp_local_time *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_local_time_diff, first, second, memory);
}
// mgp_local_date_time
inline mgp_local_date_time *local_date_time_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_from_string, string, memory);
}
inline mgp_local_date_time *local_date_time_from_parameters(mgp_local_date_time_parameters *parameters,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_from_parameters, parameters, memory);
}
inline mgp_local_date_time *local_date_time_copy(mgp_local_date_time *local_date_time, mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_copy, local_date_time, memory);
}
inline void local_date_time_destroy(mgp_local_date_time *local_date_time) {
mgp_local_date_time_destroy(local_date_time);
}
inline bool local_date_time_equal(mgp_local_date_time *first, mgp_local_date_time *second) {
return MgInvoke<int>(mgp_local_date_time_equal, first, second);
}
inline int local_date_time_get_year(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_year, local_date_time);
}
inline int local_date_time_get_month(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_month, local_date_time);
}
inline int local_date_time_get_day(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_day, local_date_time);
}
inline int local_date_time_get_hour(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_hour, local_date_time);
}
inline int local_date_time_get_minute(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_minute, local_date_time);
}
inline int local_date_time_get_second(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_second, local_date_time);
}
inline int local_date_time_get_millisecond(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_millisecond, local_date_time);
}
inline int local_date_time_get_microsecond(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_microsecond, local_date_time);
}
inline int64_t local_date_time_timestamp(mgp_local_date_time *local_date_time) {
return MgInvoke<int64_t>(mgp_local_date_time_timestamp, local_date_time);
}
inline mgp_local_date_time *local_date_time_now(mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_now, memory);
}
inline mgp_local_date_time *local_date_time_add_duration(mgp_local_date_time *local_date_time, mgp_duration *dur,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_add_duration, local_date_time, dur, memory);
}
inline mgp_local_date_time *local_date_time_sub_duration(mgp_local_date_time *local_date_time, mgp_duration *dur,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_sub_duration, local_date_time, dur, memory);
}
inline mgp_duration *local_date_time_diff(mgp_local_date_time *first, mgp_local_date_time *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_local_date_time_diff, first, second, memory);
}
// mgp_duration
inline mgp_duration *duration_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_string, string, memory);
}
inline mgp_duration *duration_from_parameters(mgp_duration_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_parameters, parameters, memory);
}
inline mgp_duration *duration_from_microseconds(int64_t microseconds, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_microseconds, microseconds, memory);
}
inline mgp_duration *duration_copy(mgp_duration *duration, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_copy, duration, memory);
}
inline void duration_destroy(mgp_duration *duration) { mgp_duration_destroy(duration); }
inline int64_t duration_get_microseconds(mgp_duration *duration) {
return MgInvoke<int64_t>(mgp_duration_get_microseconds, duration);
}
inline bool duration_equal(mgp_duration *first, mgp_duration *second) {
return MgInvoke<int>(mgp_duration_equal, first, second);
}
inline mgp_duration *duration_neg(mgp_duration *duration, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_neg, duration, memory);
}
inline mgp_duration *duration_add(mgp_duration *first, mgp_duration *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_add, first, second, memory);
}
inline mgp_duration *duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_sub, first, second, memory);
}
// Procedure
inline mgp_proc *module_add_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb) {
return MgInvoke<mgp_proc *>(mgp_module_add_read_procedure, module, name, cb);
}
inline mgp_proc *module_add_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb) {
return MgInvoke<mgp_proc *>(mgp_module_add_write_procedure, module, name, cb);
}
inline void proc_add_arg(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_arg, proc, name, type);
}
inline void proc_add_opt_arg(mgp_proc *proc, const char *name, mgp_type *type, mgp_value *default_value) {
MgInvokeVoid(mgp_proc_add_opt_arg, proc, name, type, default_value);
}
inline void proc_add_result(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_result, proc, name, type);
}
inline void proc_add_deprecated_result(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_deprecated_result, proc, name, type);
}
inline bool must_abort(mgp_graph *graph) { return mgp_must_abort(graph); }
// mgp_result
inline void result_set_error_msg(mgp_result *res, const char *error_msg) {
MgInvokeVoid(mgp_result_set_error_msg, res, error_msg);
}
inline mgp_result_record *result_new_record(mgp_result *res) {
return MgInvoke<mgp_result_record *>(mgp_result_new_record, res);
}
inline void result_record_insert(mgp_result_record *record, const char *field_name, mgp_value *val) {
MgInvokeVoid(mgp_result_record_insert, record, field_name, val);
}
// Function
inline mgp_func *module_add_function(mgp_module *module, const char *name, mgp_func_cb cb) {
return MgInvoke<mgp_func *>(mgp_module_add_function, module, name, cb);
}
inline void func_add_arg(mgp_func *func, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_func_add_arg, func, name, type);
}
inline void func_add_opt_arg(mgp_func *func, const char *name, mgp_type *type, mgp_value *default_value) {
MgInvokeVoid(mgp_func_add_opt_arg, func, name, type, default_value);
}
inline void func_result_set_error_msg(mgp_func_result *res, const char *msg, mgp_memory *memory) {
MgInvokeVoid(mgp_func_result_set_error_msg, res, msg, memory);
}
inline void func_result_set_value(mgp_func_result *res, mgp_value *value, mgp_memory *memory) {
MgInvokeVoid(mgp_func_result_set_value, res, value, memory);
}
} // namespace mgp

View File

@@ -1,57 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <exception>
#include <iostream>
namespace mg_exception {
struct NotEnoughMemoryException : public std::exception {
const char *what() const throw() { return "Not enough memory!"; }
};
struct UnknownException : public std::exception {
const char *what() const throw() { return "Unknown exception!"; }
};
struct AllocationException : public std::exception {
const char *what() const throw() { return "Could not allocate memory!"; }
};
struct InsufficientBufferException : public std::exception {
const char *what() const throw() { return "Buffer is not sufficient to process procedure!"; }
};
struct OutOfRangeException : public std::exception {
const char *what() const throw() { return "Index out of range!"; }
};
struct LogicException : public std::exception {
const char *what() const throw() { return "Logic exception, check the procedure signature!"; }
};
struct DeletedObjectException : public std::exception {
const char *what() const throw() { return "Object is deleted!"; }
};
struct InvalidArgumentException : public std::exception {
const char *what() const throw() { return "Invalid argument!"; }
};
struct InvalidIDException : public std::exception {
const char *what() const throw() { return "Invalid ID!"; }
};
struct KeyAlreadyExistsException : public std::exception {
const char *what() const throw() { return "Key you are trying to set already exists!"; }
};
struct ImmutableObjectException : public std::exception {
const char *what() const throw() { return "Object you are trying to change is immutable!"; }
};
struct ValueConversionException : public std::exception {
const char *what() const throw() { return "Error in value conversion!"; }
};
struct SerializationException : public std::exception {
const char *what() const throw() { return "Error in serialization!"; }
};
} // namespace mg_exception

View File

@@ -37,19 +37,12 @@ extern "C" {
/// All functions return an error code that can be used to figure out whether the API call was successful or not. In
/// case of failure, the specific error code can be used to identify the reason of the failure.
MGP_ENUM_CLASS MGP_NODISCARD mgp_error{
MGP_ERROR_NO_ERROR,
MGP_ERROR_UNKNOWN_ERROR,
MGP_ERROR_UNABLE_TO_ALLOCATE,
MGP_ERROR_INSUFFICIENT_BUFFER,
MGP_ERROR_OUT_OF_RANGE,
MGP_ERROR_LOGIC_ERROR,
MGP_ERROR_DELETED_OBJECT,
MGP_ERROR_INVALID_ARGUMENT,
MGP_ERROR_KEY_ALREADY_EXISTS,
MGP_ERROR_IMMUTABLE_OBJECT,
MGP_ERROR_VALUE_CONVERSION,
MGP_ERROR_SERIALIZATION_ERROR,
MGP_ERROR_AUTHORIZATION_ERROR,
MGP_ERROR_NO_ERROR, MGP_ERROR_UNKNOWN_ERROR,
MGP_ERROR_UNABLE_TO_ALLOCATE, MGP_ERROR_INSUFFICIENT_BUFFER,
MGP_ERROR_OUT_OF_RANGE, MGP_ERROR_LOGIC_ERROR,
MGP_ERROR_DELETED_OBJECT, MGP_ERROR_INVALID_ARGUMENT,
MGP_ERROR_KEY_ALREADY_EXISTS, MGP_ERROR_IMMUTABLE_OBJECT,
MGP_ERROR_VALUE_CONVERSION, MGP_ERROR_SERIALIZATION_ERROR,
};
///@}
@@ -171,8 +164,6 @@ enum mgp_value_type {
MGP_VALUE_TYPE_DURATION,
};
enum mgp_error mgp_value_copy(struct mgp_value *val, struct mgp_memory *memory, struct mgp_value **result);
/// Free the memory used by the given mgp_value instance.
void mgp_value_destroy(struct mgp_value *val);
@@ -408,8 +399,6 @@ enum mgp_error mgp_value_get_duration(struct mgp_value *val, struct mgp_duration
/// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_list.
enum mgp_error mgp_list_make_empty(size_t capacity, struct mgp_memory *memory, struct mgp_list **result);
enum mgp_error mgp_list_copy(struct mgp_list *list, struct mgp_memory *memory, struct mgp_list **result);
/// Free the memory used by the given mgp_list and contained elements.
void mgp_list_destroy(struct mgp_list *list);
@@ -448,8 +437,6 @@ enum mgp_error mgp_list_at(struct mgp_list *list, size_t index, struct mgp_value
/// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_map.
enum mgp_error mgp_map_make_empty(struct mgp_memory *memory, struct mgp_map **result);
enum mgp_error mgp_map_copy(struct mgp_map *map, struct mgp_memory *memory, struct mgp_map **result);
/// Free the memory used by the given mgp_map and contained items.
void mgp_map_destroy(struct mgp_map *map);
@@ -1305,12 +1292,6 @@ struct mgp_proc;
/// Describes a Memgraph magic function.
struct mgp_func;
/// All available log levels that can be used in mgp_log function
MGP_ENUM_CLASS mgp_log_level{
MGP_LOG_LEVEL_TRACE, MGP_LOG_LEVEL_DEBUG, MGP_LOG_LEVEL_INFO,
MGP_LOG_LEVEL_WARN, MGP_LOG_LEVEL_ERROR, MGP_LOG_LEVEL_CRITICAL,
};
/// Entry-point for a query module read procedure, invoked through openCypher.
///
/// Passed in arguments will not live longer than the callback's execution.
@@ -1405,9 +1386,6 @@ enum mgp_error mgp_proc_add_result(struct mgp_proc *proc, const char *name, stru
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid result name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a result field with the same name was already added.
enum mgp_error mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char *name, struct mgp_type *type);
/// Log a message on a certain level.
enum mgp_error mgp_log(enum mgp_log_level log_level, const char *output);
///@}
/// @name Execution
@@ -1534,10 +1512,6 @@ enum mgp_error mgp_module_add_transformation(struct mgp_module *module, const ch
///
///@{
/// State of the database that is exposed to magic functions. Currently it is unused, but it enables extending the
/// functionalities of magic functions in future without breaking the API.
struct mgp_func_context;
/// Add a required argument to a function.
///
/// The order of the added arguments corresponds to the signature of the openCypher function.

File diff suppressed because it is too large Load Diff

View File

@@ -134,15 +134,6 @@ class SerializationError(_mgp.SerializationError):
pass
class AuthorizationError(_mgp.AuthorizationError):
"""
Signals that the user doesn't have sufficient permissions to perform
procedure call.
"""
pass
class Label:
"""Label of a `Vertex`."""
@@ -155,7 +146,7 @@ class Label:
def name(self) -> str:
"""
Get the name of the label.
Returns:
A string that represents the name of the label.
@@ -204,20 +195,20 @@ class Properties:
def get(self, property_name: str, default=None) -> object:
"""
Get the value of a property with the given name or return default value.
Args:
Args:
property_name: String that represents property name.
default: Default value return if there is no property.
Returns:
Any object value that property under `property_name` has or default value otherwise.
Raises:
InvalidContextError: If `edge` or `vertex` is out of context.
Returns:
Any object value that property under `property_name` has or default value otherwise.
Raises:
InvalidContextError: If `edge` or `vertex` is out of context.
UnableToAllocateError: If unable to allocate a `mgp.Value`.
DeletedObjectError: If the `object` has been deleted.
Examples:
Examples:
```
vertex.properties.get(property_name)
edge.properties.get(property_name)
@@ -236,23 +227,23 @@ class Properties:
Set the value of the property. When the value is `None`, then the
property is removed.
Args:
property_name: String that represents property name.
Args:
property_name: String that represents property name.
value: Object that represents value to be set.
Raises:
Raises:
UnableToAllocateError: If unable to allocate memory for storing the property.
ImmutableObjectError: If the object is immutable.
DeletedObjectError: If the object has been deleted.
SerializationError: If the object has been modified by another transaction.
ValueConversionError: If `value` is vertex, edge or path.
Examples:
```
vertex.properties.set(property_name, value)
edge.properties.set(property_name, value)
```
"""
self[property_name] = value
@@ -261,15 +252,15 @@ class Properties:
Iterate over the properties. Doesnt return a dynamic view of the properties but copies the
current properties.
Returns:
Returns:
Iterable `Property` of names and values.
Raises:
InvalidContextError: If edge or vertex is out of context.
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
items = vertex.properties.items()
for it in items:
@@ -299,15 +290,15 @@ class Properties:
Iterate over property names. Doesnt return a dynamic view of the property names but copies the
name of the current properties.
Returns:
Returns:
Iterable list of strings that represent names/keys of properties.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
graph.vertex.properties.keys()
graph.edge.properties.keys()
@@ -323,20 +314,20 @@ class Properties:
Iterate over property values. Doesnt return a dynamic view of the property values but copies the
value of the current properties.
Returns:
Returns:
Iterable list of property values.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
vertex.properties.values()
edge.properties.values()
```
"""
if not self._vertex_or_edge.is_valid():
raise InvalidContextError()
@@ -347,15 +338,15 @@ class Properties:
"""
Get the number of properties.
Returns:
Returns:
A number of properties on vertex or edge.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
len(vertex.properties)
len(edge.properties)
@@ -372,15 +363,15 @@ class Properties:
"""
Iterate over property names.
Returns:
Returns:
Iterable list of strings that represent names of properties.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
iter(vertex.properties)
iter(edge.properties)
@@ -395,24 +386,24 @@ class Properties:
def __getitem__(self, property_name: str) -> object:
"""
Get the value of a property with the given name or raise KeyError.
Args:
Args:
property_name: String that represents property name.
Returns:
Returns:
Any value that property under property_name have.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate a mgp.Value.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
vertex.properties[property_name]
edge.properties[property_name]
```
"""
if not self._vertex_or_edge.is_valid():
raise InvalidContextError()
@@ -426,18 +417,18 @@ class Properties:
Set the value of the property. When the value is `None`, then the
property is removed.
Args:
Args:
property_name: String that represents property name.
value: Object that represents value to be set.
Raises:
UnableToAllocateError: If unable to allocate memory for storing the property.
ImmutableObjectError: If the object is immutable.
DeletedObjectError: If the object has been deleted.
SerializationError: If the object has been modified by another transaction.
ValueConversionError: If `value` is vertex, edge or path.
Examples:
Examples:
```
vertex.properties[property_name] = value
edge.properties[property_name] = value
@@ -452,18 +443,18 @@ class Properties:
"""
Check if there is a property with the given name.
Args:
Args:
property_name: String that represents property name
Returns:
Bool value that depends if there is with a given name.
Raises:
Bool value that depends if there is with a given name.
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate a mgp.Value.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
if property_name in vertex.properties:
```
@@ -492,7 +483,7 @@ class EdgeType:
def name(self) -> str:
"""
Get the name of EdgeType.
Returns:
A string that represents the name of EdgeType.
@@ -521,7 +512,7 @@ class Edge:
Access to an Edge is only valid during a single execution of a procedure in
a query. You should not globally store an instance of an Edge. Using an
invalid Edge instance will raise InvalidContextError.
"""
__slots__ = ("_edge",)
@@ -541,10 +532,10 @@ class Edge:
def is_valid(self) -> bool:
"""
Check if `edge` is in a valid context and may be used.
Returns:
A `bool` value depends on if the `edge` is in a valid context.
A `bool` value depends on if the `edge` is in a valid context.
Examples:
```edge.is_valid()```
@@ -552,15 +543,15 @@ class Edge:
return self._edge.is_valid()
def underlying_graph_is_mutable(self) -> bool:
"""
Check if the `graph` can be modified.
"""
Check if the `graph` can be modified.
Returns:
Returns:
A `bool` value depends on if the `graph` is mutable.
Examples:
Examples:
```edge.underlying_graph_is_mutable()```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -573,10 +564,10 @@ class Edge:
Returns:
`EdgeId` represents ID of the edge.
Raises:
Raises:
InvalidContextError: If edge is out of context.
Examples:
```edge.id```
"""
@@ -590,12 +581,12 @@ class Edge:
Get the type of edge.
Returns:
`EdgeType` describing the type of edge.
`EdgeType` describing the type of edge.
Raises:
InvalidContextError: If edge is out of context.
Examples:
Examples:
```edge.type```
"""
if not self.is_valid():
@@ -607,10 +598,10 @@ class Edge:
"""
Get the source vertex.
Returns:
Returns:
`Vertex` from where the edge is directed.
Raises:
Raises:
InvalidContextError: If edge is out of context.
Examples:
@@ -624,14 +615,14 @@ class Edge:
def to_vertex(self) -> "Vertex":
"""
Get the destination vertex.
Returns:
Returns:
`Vertex` to where the edge is directed.
Raises:
Raises:
InvalidContextError: If edge is out of context.
Examples:
Examples:
```edge.to_vertex```
"""
if not self.is_valid():
@@ -644,7 +635,7 @@ class Edge:
Get the properties of the edge.
Returns:
All `Properties` of edge.
All `Properties` of edge.
Raises:
InvalidContextError: If edge is out of context.
@@ -701,9 +692,9 @@ class Vertex:
Checks if `Vertex` is in valid context and may be used.
Returns:
A `bool` value depends on if the `Vertex` is in a valid context.
Examples:
A `bool` value depends on if the `Vertex` is in a valid context.
Examples:
```vertex.is_valid()```
"""
@@ -711,14 +702,14 @@ class Vertex:
def underlying_graph_is_mutable(self) -> bool:
"""
Check if the `graph` is mutable.
Check if the `graph` is mutable.
Returns:
Returns:
A `bool` value depends on if the `graph` is mutable.
Examples:
Examples:
```vertex.underlying_graph_is_mutable()```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -731,10 +722,10 @@ class Vertex:
Returns:
`VertexId` represents ID of the vertex.
Raises:
Raises:
InvalidContextError: If vertex is out of context.
Examples:
```vertex.id```
"""
@@ -747,15 +738,15 @@ class Vertex:
"""
Get the labels of the vertex.
Returns:
Returns:
A tuple of `Label` representing vertex Labels
Raises:
Raises:
InvalidContextError: If vertex is out of context.
OutOfRangeError: If some of the labels are removed while collecting the labels.
DeletedObjectError: If `Vertex` has been deleted.
Examples:
Examples:
```vertex.labels```
"""
if not self.is_valid():
@@ -766,17 +757,17 @@ class Vertex:
"""
Add the label to the vertex.
Args:
label: String label to be added.
Raises:
Args:
label: String label to be added.
Raises:
InvalidContextError: If `Vertex` is out of context.
UnableToAllocateError: If unable to allocate memory for storing the label.
ImmutableObjectError: If `Vertex` is immutable.
DeletedObjectError: If `Vertex` has been deleted.
SerializationError: If `Vertex` has been modified by another transaction.
Examples:
Examples:
```vertex.add_label(label)```
"""
if not self.is_valid():
@@ -787,15 +778,15 @@ class Vertex:
"""
Remove the label from the vertex.
Args:
label: String label to be deleted
Raises:
Args:
label: String label to be deleted
Raises:
InvalidContextError: If `Vertex` is out of context.
ImmutableObjectError: If `Vertex` is immutable.
DeletedObjectError: If `Vertex` has been deleted.
SerializationError: If `Vertex` has been modified by another transaction.
Examples:
Examples:
```vertex.remove_label(label)```
"""
if not self.is_valid():
@@ -807,13 +798,13 @@ class Vertex:
"""
Get the properties of the vertex.
Returns:
Returns:
`Properties` on a current vertex.
Raises:
Raises:
InvalidContextError: If `Vertex` is out of context.
Examples:
Examples:
```vertex.properties```
"""
if not self.is_valid():
@@ -829,13 +820,13 @@ class Vertex:
Returns:
Iterable list of `Edge` objects that are directed in towards the current vertex.
Raises:
Raises:
InvalidContextError: If `Vertex` is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If `Vertex` has been deleted.
Examples:
Examples:
```for edge in vertex.in_edges:```
"""
if not self.is_valid():
@@ -859,12 +850,12 @@ class Vertex:
Returns:
Iterable list of `Edge` objects that are directed out of the current vertex.
Raises:
Raises:
InvalidContextError: If `Vertex` is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If `Vertex` has been deleted.
Examples:
Examples:
```for edge in vertex.out_edges:```
"""
if not self.is_valid():
@@ -897,7 +888,7 @@ class Path:
def __init__(self, starting_vertex_or_path: typing.Union[_mgp.Path, Vertex]):
"""Initialize with a starting Vertex.
Raises:
Raises:
InvalidContextError: If passed in Vertex is invalid.
UnableToAllocateError: If cannot allocate a path.
"""
@@ -941,10 +932,10 @@ class Path:
def is_valid(self) -> bool:
"""
Check if `Path` is in valid context and may be used.
Returns:
A `bool` value depends on if the `Path` is in a valid context.
A `bool` value depends on if the `Path` is in a valid context.
Examples:
```path.is_valid()```
"""
@@ -957,15 +948,15 @@ class Path:
The last vertex on the path will become the other endpoint of the given
edge, as continued from the current last vertex.
Args:
Args:
edge: `Edge` that is added to the path
Raises:
Raises:
InvalidContextError: If using an invalid `Path` instance or if passed in `Edge` is invalid.
LogicErrorError: If the current last vertex in the path is not part of the given edge.
UnableToAllocateError: If unable to allocate memory for path extension.
Examples:
Examples:
```path.expand(edge)```
"""
if not isinstance(edge, Edge):
@@ -982,14 +973,14 @@ class Path:
"""
Vertices are ordered from the start to the end of the path.
Returns:
A tuple of `Vertex` objects order from start to end of the path.
Returns:
A tuple of `Vertex` objects order from start to end of the path.
Raises:
InvalidContextError: If using an invalid Path instance.
Examples:
```path.vertices```
```path.vertices```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -1003,11 +994,11 @@ class Path:
"""
Edges are ordered from the start to the end of the path.
Returns:
Returns:
A tuple of `Edge` objects order from start to end of the path
Raises:
Raises:
InvalidContextError: If using an invalid `Path` instance.
Examples:
Examples:
```path.edges```
"""
if not self.is_valid():
@@ -1048,10 +1039,10 @@ class Vertices:
def is_valid(self) -> bool:
"""
Check if `Vertices` is in valid context and may be used.
Returns:
A `bool` value depends on if the `Vertices` is in valid context.
A `bool` value depends on if the `Vertices` is in valid context.
Examples:
```vertices.is_valid()```
"""
@@ -1061,14 +1052,14 @@ class Vertices:
"""
Iterate over vertices.
Returns:
Iterable list of `Vertex` objects.
Returns:
Iterable list of `Vertex` objects.
Raises:
Raises:
InvalidContextError: If context is invalid.
UnableToAllocateError: If unable to allocate an iterator or a vertex.
Examples:
Examples:
```
for vertex in graph.vertices:
```
@@ -1089,18 +1080,18 @@ class Vertices:
def __contains__(self, vertex):
"""
Check if Vertices contain the given vertex.
Check if Vertices contain the given vertex.
Args:
Args:
vertex: `Vertex` to be checked if it is a part of graph `Vertices`.
Returns:
Bool value depends if there is `Vertex` in graph `Vertices`.
Bool value depends if there is `Vertex` in graph `Vertices`.
Raises:
UnableToAllocateError: If unable to allocate the vertex.
Examples:
Examples:
```if vertex in graph.vertices:```
"""
try:
@@ -1113,14 +1104,14 @@ class Vertices:
"""
Get the number of vertices.
Returns:
Returns:
A number of vertices in the graph.
Raises:
Raises:
InvalidContextError: If context is invalid.
UnableToAllocateError: If unable to allocate an iterator or a vertex.
Examples:
Examples:
```len(graph.vertices)```
"""
if not self._len:
@@ -1149,9 +1140,9 @@ class Graph:
Check if `graph` is in a valid context and may be used.
Returns:
A `bool` value depends on if the `graph` is in a valid context.
Examples:
A `bool` value depends on if the `graph` is in a valid context.
Examples:
```graph.is_valid()```
"""
@@ -1178,7 +1169,7 @@ class Graph:
Examples:
```graph.get_vertex_by_id(vertex_id)```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -1216,11 +1207,11 @@ class Graph:
def is_mutable(self) -> bool:
"""
Check if the graph is mutable. Thus it can be used to modify vertices and edges.
Returns:
A `bool` value that depends if the graph is mutable or not.
Returns:
A `bool` value that depends if the graph is mutable or not.
Examples:
Examples:
```graph.is_mutable()```
"""
if not self.is_valid():
@@ -1231,14 +1222,14 @@ class Graph:
"""
Create an empty vertex.
Returns:
Created `Vertex`.
Returns:
Created `Vertex`.
Raises:
Raises:
ImmutableObjectError: If `graph` is immutable.
UnableToAllocateError: If unable to allocate a vertex.
Examples:
Examples:
Creating an empty vertex.
```vertex = graph.create_vertex()```
@@ -1258,7 +1249,7 @@ class Graph:
LogicErrorError: If `vertex` has edges.
SerializationError: If `vertex` has been modified by
another transaction.
Examples:
Examples:
```graph.delete_vertex(vertex)```
"""
@@ -1269,39 +1260,36 @@ class Graph:
def detach_delete_vertex(self, vertex: Vertex) -> None:
"""
Delete a vertex and all of its edges.
Args:
Args:
vertex: `Vertex` to be deleted with all of its edges
Raises:
Raises:
ImmutableObjectError: If `graph` is immutable.
SerializationError: If `vertex` has been modified by another transaction.
Examples:
Examples:
```graph.detach_delete_vertex(vertex)```
"""
if not self.is_valid():
raise InvalidContextError()
self._graph.detach_delete_vertex(vertex._vertex)
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> Edge:
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> None:
"""
Create an edge.
Args:
from_vertex: `Vertex` from where edge is directed.
to_vertex: `Vertex' to where edge is directed.
edge_type: `EdgeType` defines the type of edge.
Returns:
Created `Edge`.
Args:
from_vertex: `Vertex` from where edge is directed.
to_vertex: `Vertex' to where edge is directed.
edge_type: `EdgeType` defines the type of edge.
Raises:
ImmutableObjectError: If `graph` is immutable.
UnableToAllocateError: If unable to allocate an edge.
DeletedObjectError: If `from_vertex` or `to_vertex` has been deleted.
SerializationError: If `from_vertex` or `to_vertex` has been modified by another transaction.
Examples:
```edge = graph.create_edge(from_vertex, vertex, edge_type)```
Examples:
```graph.create_edge(from_vertex, vertex, edge_type)```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -1313,8 +1301,8 @@ class Graph:
Args:
edge: `Edge` to be deleted
Raises:
Raises:
ImmutableObjectError if `graph` is immutable.
Raise SerializationError if `edge`, its source or destination vertex has been modified by another transaction.
"""
@@ -1349,15 +1337,15 @@ class ProcCtx:
@property
def graph(self) -> Graph:
"""
Access to `Graph` object.
Access to `Graph` object.
Returns:
Graph object.
Returns:
Graph object.
Raises:
Raises:
InvalidContextError: If context is invalid.
Examples:
Examples:
```context.graph```
"""
if not self.is_valid():
@@ -1981,8 +1969,6 @@ def _wrap_exceptions():
raise ValueConversionError(e)
except _mgp.SerializationError as e:
raise SerializationError(e)
except _mgp.AuthorizationError as e:
raise AuthorizationError(e)
return wrapped_func
@@ -2018,82 +2004,4 @@ def _wrap_exceptions():
setattr(module, name, wrap_function(obj))
class Logger:
"""Represents a Logger through which it is possible
to send logs via API to the graph database.
The best way to use this Logger is to have one per query module."""
__slots__ = ("_logger",)
def __init__(self):
self._logger = _mgp._LOGGER
def info(self, out: str) -> None:
"""
Log message on INFO level..
Args:
out: String message to be logged.
Examples:
```logger.info("Hello from query module.")```
"""
self._logger.info(out)
def warning(self, out: str) -> None:
"""
Log message on WARNING level..
Args:
out: String message to be logged.
Examples:
```logger.warning("Hello from query module.")```
"""
self._logger.warning(out)
def critical(self, out: str) -> None:
"""
Log message on CRITICAL level..
Args:
out: String message to be logged.
Examples:
```logger.critical("Hello from query module.")```
"""
self._logger.critical(out)
def error(self, out: str) -> None:
"""
Log message on ERROR level..
Args:
out: String message to be logged.
Examples:
```logger.error("Hello from query module.")```
"""
self._logger.error(out)
def trace(self, out: str) -> None:
"""
Log message on TRACE level..
Args:
out: String message to be logged.
Examples:
```logger.trace("Hello from query module.")```
"""
self._logger.trace(out)
def debug(self, out: str) -> None:
"""
Log message on DEBUG level..
Args:
out: String message to be logged.
Examples:
```logger.debug("Hello from query module.")```
"""
self._logger.debug(out)
_wrap_exceptions()

17
init
View File

@@ -5,9 +5,6 @@ cd "$DIR"
source "$DIR/environment/util.sh"
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
function print_help () {
echo "Usage: $0 [OPTION]"
echo -e "Check for missing packages and setup the project.\n"
@@ -67,6 +64,8 @@ else
done
fi
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
else
@@ -112,12 +111,6 @@ if [[ "$setup_libs" == "true" ]]; then
cd ..
fi
# Fix for centos 7 during release
if [ "${ARCHITECTURE}" = "centos-7" ]; then
python3 -m pip uninstall virtualenv
python3 -m pip install virtualenv
fi
# setup gql_behave dependencies
setup_virtualenv tests/gql_behave
@@ -147,11 +140,5 @@ done;
python3 -m pip install pre-commit
python3 -m pre_commit install
# Install py format tools
echo "Install black formatter"
python3 -m pip install black==22.10.*
echo "Install isort"
python3 -m pip install isort==5.10.*
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

View File

@@ -208,7 +208,7 @@ pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
# mgconsole
mgconsole_tag="v1.3.0" # (2022-11-20)
mgconsole_tag="v1.1.0" # (2021-10-07)
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
spdlog_tag="v1.9.2" # (2021-08-12)

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2026-13-12
CHANGE DATE: 2026-27-04
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -2,8 +2,8 @@ MEMGRAPH
ENTERPRISE LICENCE AGREEMENT
Memgraph Limited is registered in England under registration 10195084 and has its registered office at 90a High Street,
Hertfordshire, Berkhamsted, HP4 2BL United Kingdom ("Memgraph").
Memgraph Limited is registered in England under registration 10195084 and has its registered office at Suite 4,
Ironstone House, Ironstone Way, Brixworth, Northampton, NN6 9UD (Memgraph).
Memgraph agrees to license and/or grant you (the “Customer”) access to the Software ( as defined below) and provide

View File

@@ -1,12 +0,0 @@
[tool.black]
line-length = 120
include = '\.pyi?$'
extend-exclude = '''
/(
| .git
| .__pycache__
| build
| libs
| .cache
)/
'''

View File

@@ -8,40 +8,29 @@ disallow_in_source_build()
# Everything that is installed here, should be under the "query_modules" component.
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules")
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
add_library(example_c SHARED example.c)
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_c PRIVATE -Wall)
# Strip C example in release build.
add_library(example SHARED example.c)
target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example PRIVATE -Wall)
# Strip the library in release build.
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_c POST_BUILD
COMMAND strip -s $<TARGET_FILE:example_c>
COMMENT "Stripping symbols and sections from the C example module")
add_custom_command(TARGET example POST_BUILD
COMMAND strip -s $<TARGET_FILE:example>
COMMENT "Stripping symbols and sections from example module")
endif()
install(PROGRAMS $<TARGET_FILE:example_c>
install(PROGRAMS $<TARGET_FILE:example>
DESTINATION lib/memgraph/query_modules
RENAME example_c.so)
RENAME example.so)
# Also install the source of the example, so user can read it.
install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
add_library(example_cpp SHARED example.cpp)
target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_cpp PRIVATE -Wall)
# Strip C++ example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_cpp POST_BUILD
COMMAND strip -s $<TARGET_FILE:example_cpp>
COMMENT "Stripping symbols and sections from the C++ example module")
endif()
install(PROGRAMS $<TARGET_FILE:example_cpp>
DESTINATION lib/memgraph/query_modules
RENAME example_cpp.so)
# Also install the source of the example, so user can read it.
install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
# Install the Python example and modules
# Install the Python example
install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
# Install the Python modules
install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)

View File

@@ -1,103 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <mgp.hpp>
void ProcImpl(std::vector<mgp::Value> arguments, mgp::Graph graph, mgp::RecordFactory record_factory) {
auto record = record_factory.NewRecord();
record.Insert("out", true);
}
void SampleReadProc(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
try {
mgp::memory = memory;
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
ProcImpl(arguments, mgp::Graph(memgraph_graph), mgp::RecordFactory(result));
} catch (const std::exception &e) {
mgp::result_set_error_msg(result, e.what());
return;
}
}
void AddXNodes(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::memory = memory;
auto graph = mgp::Graph(memgraph_graph);
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
for (int i = 0; i < arguments[0].ValueInt(); i++) {
graph.CreateNode();
}
}
void Multiply(mgp_list *args, mgp_func_context *ctx, mgp_func_result *res, mgp_memory *memory) {
mgp::memory = memory;
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
auto result = mgp::Result(res);
auto first = arguments[0].ValueInt();
auto second = arguments[1].ValueInt();
result.SetValue(first * second);
}
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
try {
mgp::memory = memory;
AddProcedure(SampleReadProc, "return_true", mgp::ProcedureType::Read,
{mgp::Parameter("param_1", mgp::Type::Int), mgp::Parameter("param_2", mgp::Type::Double, 2.3)},
{mgp::Return("out", mgp::Type::Bool)}, module, memory);
} catch (const std::exception &e) {
return 1;
}
try {
mgp::memory = memory;
mgp::AddProcedure(AddXNodes, "add_x_nodes", mgp::ProcedureType::Write, {mgp::Parameter("param_1", mgp::Type::Int)},
{}, module, memory);
} catch (const std::exception &e) {
return 1;
}
try {
mgp::memory = memory;
mgp::AddFunction(Multiply, "multiply",
{mgp::Parameter("int", mgp::Type::Int), mgp::Parameter("int", mgp::Type::Int, (int64_t)3)}, module,
memory);
} catch (const std::exception &e) {
return 1;
}
return 0;
}
extern "C" int mgp_shutdown_module() { return 0; }

View File

@@ -1,4 +1,3 @@
.venv
dist
mgp.py
poetry.lock

View File

@@ -1,11 +1,4 @@
# mgp
PyPi package used for type hinting when creating query modules. Repository of already available query modules is called [MAGE](https://github.com/memgraph/mage).
## 🎬 Get started
To learn more, head over to the [docs for the query modules Python API](https://memgraph.com/docs/memgraph/reference-guide/query-modules/api/python-api). To get started with query modules, check out the [how-to guide](https://memgraph.com/docs/memgraph/how-to-guides/query-modules) on Memgraph docs.
## 🔢 Versioning
- mgp v1.1 is compatible with Memgraph >= 2.4.0
PyPi package used for type hinting when creating MAGE modules. The get started
using MAGE repository checkout the repository here: https://github.com/memgraph/mage.

View File

@@ -173,10 +173,6 @@ class SerializationError(Exception):
pass
class AuthorizationError(Exception):
pass
def type_nullable(elem: Any):
pass
@@ -257,11 +253,3 @@ class _MODULE:
@staticmethod
def add_function(wrapper):
pass
class SOURCE_TYPE_KAFKA:
pass
class SOURCE_TYPE_PULSAR:
pass

View File

@@ -1,13 +1,13 @@
[tool.poetry]
name = "mgp"
version = "1.1.1"
version = "1.0.0"
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
authors = [
"katarinasupe <katarina.supe@memgraph.io>",
"MasterMedo <mislav.vuletic@gmail.com>",
"jbajic <jure.bajic@memgraph.io>",
"katarinasupe <katarina.supe@memgraph.io>",
"antejavor <ante.javor@memgraph.io>",
"antaljanosbenjamin <benjamin.antal@memgraph.io>",
"MasterMedo <mislav.vuletic@gmail.com>",
]
license = "Apache-2.0"
readme = "README.md"

View File

@@ -72,8 +72,6 @@ make_package () {
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
# Fix issue with git marking directory as not safe
docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'"
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
if [[ "$os" == "debian-11-arm" ]]; then

View File

@@ -5,21 +5,25 @@ add_subdirectory(lisp)
add_subdirectory(utils)
add_subdirectory(requests)
add_subdirectory(io)
add_subdirectory(io/simulator)
add_subdirectory(kvstore)
add_subdirectory(telemetry)
add_subdirectory(communication)
add_subdirectory(memory)
add_subdirectory(storage/v2)
add_subdirectory(storage/v3)
add_subdirectory(integrations)
add_subdirectory(query)
add_subdirectory(glue)
add_subdirectory(query/v2)
add_subdirectory(slk)
add_subdirectory(rpc)
add_subdirectory(license)
add_subdirectory(auth)
add_subdirectory(parser)
add_subdirectory(expr)
add_subdirectory(coordinator)
if(MG_ENTERPRISE)
add_subdirectory(audit)
if (MG_ENTERPRISE)
add_subdirectory(audit)
endif()
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -33,94 +37,80 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
# Memgraph Single Node v2 Executable
# ----------------------------------------------------------------------------
set(mg_single_node_v2_sources
memgraph.cpp
glue/communication.cpp
memgraph.cpp
glue/auth.cpp
)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
mg-telemetry mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-glue)
if(MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
endif()
# memgraph main executable
add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph ${mg_single_node_v2_libs})
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the
# symbols in custom procedure module libraries.
target_link_libraries(memgraph "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
set_target_properties(memgraph PROPERTIES
# Set the executable output name to include version information.
OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
# Output the executable in main binary dir.
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Set the executable output name to include version information.
OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
# Output the executable in main binary dir.
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Create symlink to the built executable.
add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph
BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph
COMMENT "Creating symlink to memgraph executable")
COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph
BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph
COMMENT "Creating symlink to memgraph executable")
# Emulate the installed python_support, by creating a symlink
add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support
BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support
COMMENT "Creating symlink for python_support")
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support
BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support
COMMENT "Creating symlink for python_support")
# Strip the executable in release build.
if(lower_build_type STREQUAL "release")
add_custom_command(TARGET memgraph POST_BUILD
COMMAND strip -s $<TARGET_FILE:memgraph>
COMMENT "Stripping symbols and sections from memgraph")
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET memgraph POST_BUILD
COMMAND strip -s $<TARGET_FILE:memgraph>
COMMENT "Stripping symbols and sections from memgraph")
endif()
# Generate the configuration file.
add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_BINARY_DIR}/memgraph
${CMAKE_BINARY_DIR}/config/memgraph.conf
DEPENDS ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_SOURCE_DIR}/config/flags.yaml
BYPRODUCTS ${CMAKE_BINARY_DIR}/config/memgraph.conf
COMMENT "Generating memgraph configuration file")
COMMAND ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_BINARY_DIR}/memgraph
${CMAKE_BINARY_DIR}/config/memgraph.conf
DEPENDS ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_SOURCE_DIR}/config/flags.yaml
BYPRODUCTS ${CMAKE_BINARY_DIR}/config/memgraph.conf
COMMENT "Generating memgraph configuration file")
# Everything here is under "memgraph" install component.
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph")
# TODO: Default directory permissions to 755
# NOTE: This is added in CMake 3.11, so enable it then
# set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
# OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ)
#set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
# OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ)
# Install and rename executable to just 'memgraph' Since we have to rename,
# we cannot use the recommended `install(TARGETS ...)`.
install(PROGRAMS $<TARGET_FILE:memgraph>
DESTINATION lib/memgraph RENAME memgraph)
# Install Python source for supporting our embedded Python.
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.py
DESTINATION lib/memgraph/python_support)
# Install the includes file for writing custom procedures in C and C++>
# Install the include file for writing custom procedures.
install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/_mgp.hpp
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/mg_exceptions.hpp
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.hpp
DESTINATION include/memgraph)
# Install the config file (must use absolute path).
install(FILES ${CMAKE_BINARY_DIR}/config/memgraph.conf
DESTINATION /etc/memgraph RENAME memgraph.conf)
# Install logrotate configuration (must use absolute path).
install(FILES ${CMAKE_SOURCE_DIR}/release/logrotate.conf
DESTINATION /etc/logrotate.d RENAME memgraph)
@@ -136,14 +126,15 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
# ----------------------------------------------------------------------------
# Memgraph CSV Import Tool Executable
# ----------------------------------------------------------------------------
add_executable(mg_import_csv mg_import_csv.cpp)
target_link_libraries(mg_import_csv mg-storage-v2)
# Strip the executable in release build.
if(lower_build_type STREQUAL "release")
add_custom_command(TARGET mg_import_csv POST_BUILD
COMMAND strip -s mg_import_csv
COMMENT "Stripping symbols and sections from mg_import_csv")
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET mg_import_csv POST_BUILD
COMMAND strip -s mg_import_csv
COMMENT "Stripping symbols and sections from mg_import_csv")
endif()
install(TARGETS mg_import_csv RUNTIME DESTINATION bin)

View File

@@ -16,8 +16,8 @@
#include <fmt/format.h>
#include "auth/exceptions.hpp"
#include "license/license.hpp"
#include "utils/flag_validation.hpp"
#include "utils/license.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/settings.hpp"
@@ -68,9 +68,10 @@ Auth::Auth(const std::string &storage_directory) : storage_(storage_directory),
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
if (module_.IsUsed()) {
const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings);
if (license_check_result.HasError()) {
spdlog::warn(license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
spdlog::warn(
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
return std::nullopt;
}
@@ -225,7 +226,7 @@ std::vector<auth::User> Auth::AllUsers() const {
if (username != utils::ToLowerCase(username)) continue;
auto user = GetUser(username);
if (user) {
ret.push_back(std::move(*user));
ret.push_back(*user);
}
}
return ret;
@@ -305,7 +306,7 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
if (it->second == rolename) {
auto user = GetUser(username);
if (user) {
ret.push_back(std::move(*user));
ret.push_back(*user);
} else {
throw AuthException("Couldn't load user '{}'!", username);
}

View File

@@ -21,7 +21,7 @@
namespace memgraph::auth {
/**
* This class serves as the main Authentication/Authorization storage.
* It provides functions for managing Users, Roles, Permissions and FineGrainedAccessPermissions.
* It provides functions for managing Users, Roles and Permissions.
* NOTE: The non-const functions in this class aren't thread safe.
* TODO (mferencevic): Disable user/role modification functions when they are
* being managed by the auth module.

View File

@@ -8,16 +8,14 @@
#include "auth/models.hpp"
#include <cstdint>
#include <regex>
#include <gflags/gflags.h>
#include "auth/crypto.hpp"
#include "auth/exceptions.hpp"
#include "license/license.hpp"
#include "utils/cast.hpp"
#include "utils/logging.hpp"
#include "utils/license.hpp"
#include "utils/settings.hpp"
#include "utils/string.hpp"
@@ -32,7 +30,6 @@ DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
namespace memgraph::auth {
namespace {
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {
Permission::MATCH, Permission::CREATE, Permission::MERGE, Permission::DELETE,
@@ -40,7 +37,7 @@ const std::vector<Permission> kPermissionsAll = {
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET};
Permission::WEBSOCKET, Permission::SCHEMA};
} // namespace
std::string PermissionToString(Permission permission) {
@@ -87,6 +84,8 @@ std::string PermissionToString(Permission permission) {
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
case Permission::SCHEMA:
return "SCHEMA";
}
}
@@ -101,55 +100,6 @@ std::string PermissionLevelToString(PermissionLevel level) {
}
}
#ifdef MG_ENTERPRISE
FineGrainedPermission PermissionToFineGrainedPermission(const uint64_t permission) {
if (permission & FineGrainedPermission::CREATE_DELETE) {
return FineGrainedPermission::CREATE_DELETE;
}
if (permission & FineGrainedPermission::UPDATE) {
return FineGrainedPermission::UPDATE;
}
if (permission & FineGrainedPermission::READ) {
return FineGrainedPermission::READ;
}
return FineGrainedPermission::NOTHING;
}
std::string FineGrainedPermissionToString(const FineGrainedPermission level) {
switch (level) {
case FineGrainedPermission::CREATE_DELETE:
return "CREATE_DELETE";
case FineGrainedPermission::UPDATE:
return "UPDATE";
case FineGrainedPermission::READ:
return "READ";
case FineGrainedPermission::NOTHING:
return "NOTHING";
}
}
FineGrainedAccessPermissions Merge(const FineGrainedAccessPermissions &first,
const FineGrainedAccessPermissions &second) {
std::unordered_map<std::string, uint64_t> permissions{first.GetPermissions()};
std::optional<uint64_t> global_permission;
if (second.GetGlobalPermission().has_value()) {
global_permission = *second.GetGlobalPermission();
} else if (first.GetGlobalPermission().has_value()) {
global_permission = *first.GetGlobalPermission();
}
for (const auto &[label_name, permission] : second.GetPermissions()) {
permissions[label_name] = permission;
}
return FineGrainedAccessPermissions(permissions, global_permission);
}
#endif
Permissions::Permissions(uint64_t grants, uint64_t denies) {
// The deny bitmask has higher priority than the grant bitmask.
denies_ = denies;
@@ -223,7 +173,7 @@ Permissions Permissions::Deserialize(const nlohmann::json &data) {
if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) {
throw AuthException("Couldn't load permissions data!");
}
return Permissions{data["grants"], data["denies"]};
return {data["grants"], data["denies"]};
}
uint64_t Permissions::grants() const { return grants_; }
@@ -235,191 +185,19 @@ bool operator==(const Permissions &first, const Permissions &second) {
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions::FineGrainedAccessPermissions(const std::unordered_map<std::string, uint64_t> &permissions,
const std::optional<uint64_t> &global_permission)
: permissions_(permissions), global_permission_(global_permission) {}
PermissionLevel FineGrainedAccessPermissions::Has(const std::string &permission,
const FineGrainedPermission fine_grained_permission) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return PermissionLevel::GRANT;
}
const auto concrete_permission = std::invoke([&]() -> uint64_t {
if (permissions_.contains(permission)) {
return permissions_.at(permission);
}
if (global_permission_.has_value()) {
return global_permission_.value();
}
return 0;
});
const auto temp_permission = concrete_permission & fine_grained_permission;
return temp_permission > 0 ? PermissionLevel::GRANT : PermissionLevel::DENY;
}
void FineGrainedAccessPermissions::Grant(const std::string &permission,
const FineGrainedPermission fine_grained_permission) {
if (permission == kAsterisk) {
global_permission_ = CalculateGrant(fine_grained_permission);
} else {
permissions_[permission] = CalculateGrant(fine_grained_permission);
}
}
void FineGrainedAccessPermissions::Revoke(const std::string &permission) {
if (permission == kAsterisk) {
permissions_.clear();
global_permission_ = std::nullopt;
} else {
permissions_.erase(permission);
}
}
nlohmann::json FineGrainedAccessPermissions::Serialize() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
nlohmann::json data = nlohmann::json::object();
data["permissions"] = permissions_;
data["global_permission"] = global_permission_.has_value() ? global_permission_.value() : -1;
return data;
}
FineGrainedAccessPermissions FineGrainedAccessPermissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!");
}
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
std::optional<uint64_t> global_permission;
if (data["global_permission"].empty() || data["global_permission"] == -1) {
global_permission = std::nullopt;
} else {
global_permission = data["global_permission"];
}
return FineGrainedAccessPermissions(data["permissions"], global_permission);
}
const std::unordered_map<std::string, uint64_t> &FineGrainedAccessPermissions::GetPermissions() const {
return permissions_;
}
const std::optional<uint64_t> &FineGrainedAccessPermissions::GetGlobalPermission() const { return global_permission_; };
uint64_t FineGrainedAccessPermissions::CalculateGrant(FineGrainedPermission fine_grained_permission) {
uint64_t shift{1};
uint64_t result{0};
auto uint_fine_grained_permission = static_cast<uint64_t>(fine_grained_permission);
while (uint_fine_grained_permission > 0) {
result |= uint_fine_grained_permission;
uint_fine_grained_permission >>= shift;
}
return result;
}
bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return first.GetPermissions() == second.GetPermissions() &&
first.GetGlobalPermission() == second.GetGlobalPermission();
}
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return !(first == second);
}
FineGrainedAccessHandler::FineGrainedAccessHandler(FineGrainedAccessPermissions labelPermissions,
FineGrainedAccessPermissions edgeTypePermissions)
: label_permissions_(std::move(labelPermissions)), edge_type_permissions_(std::move(edgeTypePermissions)) {}
const FineGrainedAccessPermissions &FineGrainedAccessHandler::label_permissions() const { return label_permissions_; }
FineGrainedAccessPermissions &FineGrainedAccessHandler::label_permissions() { return label_permissions_; }
const FineGrainedAccessPermissions &FineGrainedAccessHandler::edge_type_permissions() const {
return edge_type_permissions_;
}
FineGrainedAccessPermissions &FineGrainedAccessHandler::edge_type_permissions() { return edge_type_permissions_; }
nlohmann::json FineGrainedAccessHandler::Serialize() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
nlohmann::json data = nlohmann::json::object();
data["label_permissions"] = label_permissions_.Serialize();
data["edge_type_permissions"] = edge_type_permissions_.Serialize();
return data;
}
FineGrainedAccessHandler FineGrainedAccessHandler::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load role data!");
}
if (!data["label_permissions"].is_object() || !data["edge_type_permissions"].is_object()) {
throw AuthException("Couldn't load label_permissions or edge_type_permissions data!");
}
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessHandler{};
}
auto label_permissions = FineGrainedAccessPermissions::Deserialize(data["label_permissions"]);
auto edge_type_permissions = FineGrainedAccessPermissions::Deserialize(data["edge_type_permissions"]);
return FineGrainedAccessHandler(std::move(label_permissions), std::move(edge_type_permissions));
}
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second) {
return first.label_permissions_ == second.label_permissions_ &&
first.edge_type_permissions_ == second.edge_type_permissions_;
}
bool operator!=(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second) {
return !(first == second);
}
#endif
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
Role::Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler)
: rolename_(utils::ToLowerCase(rolename)),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
#endif
const std::string &Role::rolename() const { return rolename_; }
const Permissions &Role::permissions() const { return permissions_; }
Permissions &Role::permissions() { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &Role::fine_grained_access_handler() const { return fine_grained_access_handler_; }
FineGrainedAccessHandler &Role::fine_grained_access_handler() { return fine_grained_access_handler_; }
const FineGrainedAccessPermissions &Role::GetFineGrainedAccessLabelPermissions() const {
return fine_grained_access_handler_.label_permissions();
}
const FineGrainedAccessPermissions &Role::GetFineGrainedAccessEdgeTypePermissions() const {
return fine_grained_access_handler_.edge_type_permissions();
}
#endif
nlohmann::json Role::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["rolename"] = rolename_;
data["permissions"] = permissions_.Serialize();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
} else {
data["fine_grained_access_handler"] = {};
}
#endif
return data;
}
@@ -431,43 +209,18 @@ Role Role::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load role data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
if (!data["fine_grained_access_handler"].is_object()) {
throw AuthException("Couldn't load user data!");
}
auto fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
return {data["rolename"], permissions, std::move(fine_grained_access_handler)};
}
#endif
return {data["rolename"], permissions};
}
bool operator==(const Role &first, const Role &second) {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_ &&
first.fine_grained_access_handler_ == second.fine_grained_access_handler_;
}
#endif
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_;
}
User::User() {}
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler)
: username_(utils::ToLowerCase(username)),
password_hash_(password_hash),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
#endif
bool User::CheckPassword(const std::string &password) {
if (password_hash_.empty()) return true;
return VerifyPassword(password, password_hash_);
@@ -483,13 +236,13 @@ void User::UpdatePassword(const std::optional<std::string> &password) {
}
if (FLAGS_auth_password_strength_regex != default_password_regex) {
if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
if (const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings);
license_check_result.HasError()) {
throw AuthException(
"Custom password regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
default_password_regex,
license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
}
}
std::regex re(FLAGS_auth_password_strength_regex);
@@ -509,47 +262,17 @@ void User::ClearRole() { role_ = std::nullopt; }
Permissions User::GetPermissions() const {
if (role_) {
return Permissions{permissions_.grants() | role_->permissions().grants(),
permissions_.denies() | role_->permissions().denies()};
return Permissions(permissions_.grants() | role_->permissions().grants(),
permissions_.denies() | role_->permissions().denies());
}
return permissions_;
}
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions User::GetFineGrainedAccessLabelPermissions() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
if (role_) {
return Merge(role()->fine_grained_access_handler().label_permissions(),
fine_grained_access_handler_.label_permissions());
}
return fine_grained_access_handler_.label_permissions();
}
FineGrainedAccessPermissions User::GetFineGrainedAccessEdgeTypePermissions() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
if (role_) {
return Merge(role()->fine_grained_access_handler().edge_type_permissions(),
fine_grained_access_handler_.edge_type_permissions());
}
return fine_grained_access_handler_.edge_type_permissions();
}
#endif
const std::string &User::username() const { return username_; }
const Permissions &User::permissions() const { return permissions_; }
Permissions &User::permissions() { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &User::fine_grained_access_handler() const { return fine_grained_access_handler_; }
FineGrainedAccessHandler &User::fine_grained_access_handler() { return fine_grained_access_handler_; }
#endif
const Role *User::role() const {
if (role_.has_value()) {
return &role_.value();
@@ -562,13 +285,6 @@ nlohmann::json User::Serialize() const {
data["username"] = username_;
data["password_hash"] = password_hash_;
data["permissions"] = permissions_.Serialize();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
} else {
data["fine_grained_access_handler"] = {};
}
#endif
// The role shouldn't be serialized here, it is stored as a foreign key.
return data;
}
@@ -581,28 +297,11 @@ User User::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load user data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
if (!data["fine_grained_access_handler"].is_object()) {
throw AuthException("Couldn't load user data!");
}
auto fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
return {data["username"], data["password_hash"], permissions, fine_grained_access_handler};
}
#endif
return {data["username"], data["password_hash"], permissions};
}
bool operator==(const User &first, const User &second) {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_ &&
first.fine_grained_access_handler_ == second.fine_grained_access_handler_;
}
#endif
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_;
}
} // namespace memgraph::auth

View File

@@ -10,12 +10,10 @@
#include <optional>
#include <string>
#include <unordered_map>
#include <json/json.hpp>
namespace memgraph::auth {
const std::string kAsterisk = "*";
// These permissions must have values that are applicable for usage in a
// bitmask.
// clang-format off
@@ -40,65 +38,27 @@ enum class Permission : uint64_t {
STREAM = 1U << 17U,
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U
WEBSOCKET = 1U << 20U,
SCHEMA = 1U << 21U
};
// clang-format on
#ifdef MG_ENTERPRISE
// clang-format off
enum class FineGrainedPermission : uint64_t {
NOTHING = 0,
READ = 1,
UPDATE = 1U << 1U,
CREATE_DELETE = 1U << 2U
};
// clang-format on
constexpr inline uint64_t operator|(FineGrainedPermission lhs, FineGrainedPermission rhs) {
return static_cast<uint64_t>(lhs) | static_cast<uint64_t>(rhs);
}
constexpr inline uint64_t operator|(uint64_t lhs, FineGrainedPermission rhs) {
return lhs | static_cast<uint64_t>(rhs);
}
constexpr inline uint64_t operator&(uint64_t lhs, FineGrainedPermission rhs) {
return (lhs & static_cast<uint64_t>(rhs)) != 0;
}
constexpr uint64_t kLabelPermissionAll = memgraph::auth::FineGrainedPermission::CREATE_DELETE |
memgraph::auth::FineGrainedPermission::UPDATE |
memgraph::auth::FineGrainedPermission::READ;
constexpr uint64_t kLabelPermissionMax = static_cast<uint64_t>(memgraph::auth::FineGrainedPermission::CREATE_DELETE);
constexpr uint64_t kLabelPermissionMin = static_cast<uint64_t>(memgraph::auth::FineGrainedPermission::READ);
#endif
// Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission);
// Class that indicates what permission level the user/role has.
enum class PermissionLevel : uint8_t { GRANT, NEUTRAL, DENY };
enum class PermissionLevel {
GRANT,
NEUTRAL,
DENY,
};
// Function that converts a permission level to its string representation.
std::string PermissionLevelToString(PermissionLevel level);
#ifdef MG_ENTERPRISE
// Function that converts a label permission level to its string representation.
std::string FineGrainedPermissionToString(FineGrainedPermission level);
// Constructs a label permission from a permission
FineGrainedPermission PermissionToFineGrainedPermission(uint64_t permission);
#endif
class Permissions final {
public:
explicit Permissions(uint64_t grants = 0, uint64_t denies = 0);
Permissions(const Permissions &) = default;
Permissions &operator=(const Permissions &) = default;
Permissions(Permissions &&) noexcept = default;
Permissions &operator=(Permissions &&) noexcept = default;
~Permissions() = default;
Permissions(uint64_t grants = 0, uint64_t denies = 0);
PermissionLevel Has(Permission permission) const;
@@ -129,96 +89,16 @@ bool operator==(const Permissions &first, const Permissions &second);
bool operator!=(const Permissions &first, const Permissions &second);
#ifdef MG_ENTERPRISE
class FineGrainedAccessPermissions final {
public:
explicit FineGrainedAccessPermissions(const std::unordered_map<std::string, uint64_t> &permissions = {},
const std::optional<uint64_t> &global_permission = std::nullopt);
FineGrainedAccessPermissions(const FineGrainedAccessPermissions &) = default;
FineGrainedAccessPermissions &operator=(const FineGrainedAccessPermissions &) = default;
FineGrainedAccessPermissions(FineGrainedAccessPermissions &&) = default;
FineGrainedAccessPermissions &operator=(FineGrainedAccessPermissions &&) = default;
~FineGrainedAccessPermissions() = default;
PermissionLevel Has(const std::string &permission, FineGrainedPermission fine_grained_permission) const;
void Grant(const std::string &permission, FineGrainedPermission fine_grained_permission);
void Revoke(const std::string &permission);
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static FineGrainedAccessPermissions Deserialize(const nlohmann::json &data);
const std::unordered_map<std::string, uint64_t> &GetPermissions() const;
const std::optional<uint64_t> &GetGlobalPermission() const;
private:
std::unordered_map<std::string, uint64_t> permissions_{};
std::optional<uint64_t> global_permission_;
static uint64_t CalculateGrant(FineGrainedPermission fine_grained_permission);
};
bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
class FineGrainedAccessHandler final {
public:
explicit FineGrainedAccessHandler(FineGrainedAccessPermissions labelPermissions = FineGrainedAccessPermissions(),
FineGrainedAccessPermissions edgeTypePermissions = FineGrainedAccessPermissions());
FineGrainedAccessHandler(const FineGrainedAccessHandler &) = default;
FineGrainedAccessHandler &operator=(const FineGrainedAccessHandler &) = default;
FineGrainedAccessHandler(FineGrainedAccessHandler &&) noexcept = default;
FineGrainedAccessHandler &operator=(FineGrainedAccessHandler &&) noexcept = default;
~FineGrainedAccessHandler() = default;
const FineGrainedAccessPermissions &label_permissions() const;
FineGrainedAccessPermissions &label_permissions();
const FineGrainedAccessPermissions &edge_type_permissions() const;
FineGrainedAccessPermissions &edge_type_permissions();
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static FineGrainedAccessHandler Deserialize(const nlohmann::json &data);
friend bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
private:
FineGrainedAccessPermissions label_permissions_;
FineGrainedAccessPermissions edge_type_permissions_;
};
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
#endif
class Role final {
public:
explicit Role(const std::string &rolename);
Role(const std::string &rolename);
Role(const std::string &rolename, const Permissions &permissions);
#ifdef MG_ENTERPRISE
Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler);
#endif
Role(const Role &) = default;
Role &operator=(const Role &) = default;
Role(Role &&) noexcept = default;
Role &operator=(Role &&) noexcept = default;
~Role() = default;
const std::string &rolename() const;
const Permissions &permissions() const;
Permissions &permissions();
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
const FineGrainedAccessPermissions &GetFineGrainedAccessLabelPermissions() const;
const FineGrainedAccessPermissions &GetFineGrainedAccessEdgeTypePermissions() const;
#endif
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
@@ -229,9 +109,6 @@ class Role final {
private:
std::string rolename_;
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
#endif
};
bool operator==(const Role &first, const Role &second);
@@ -239,19 +116,9 @@ bool operator==(const Role &first, const Role &second);
// TODO (mferencevic): Implement password expiry.
class User final {
public:
User();
User(const std::string &username);
explicit User(const std::string &username);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
#ifdef MG_ENTERPRISE
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler);
#endif
User(const User &) = default;
User &operator=(const User &) = default;
User(User &&) noexcept = default;
User &operator=(User &&) noexcept = default;
~User() = default;
/// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password);
@@ -265,12 +132,6 @@ class User final {
Permissions GetPermissions() const;
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions GetFineGrainedAccessLabelPermissions() const;
FineGrainedAccessPermissions GetFineGrainedAccessEdgeTypePermissions() const;
const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
#endif
const std::string &username() const;
const Permissions &permissions() const;
@@ -289,16 +150,8 @@ class User final {
std::string username_;
std::string password_hash_;
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
#endif
std::optional<Role> role_;
};
bool operator==(const User &first, const User &second);
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions Merge(const FineGrainedAccessPermissions &first,
const FineGrainedAccessPermissions &second);
#endif
} // namespace memgraph::auth

19
src/common/types.hpp Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
namespace memgraph::common {
enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
} // namespace memgraph::common

View File

@@ -7,7 +7,6 @@ set(communication_src_files
websocket/listener.cpp
websocket/session.cpp
bolt/v1/value.cpp
bolt/client.cpp
buffer.cpp
client.cpp
context.cpp

View File

@@ -1,262 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "communication/bolt/client.hpp"
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/value.hpp"
#include "utils/logging.hpp"
namespace {
constexpr uint8_t kBoltV43Version[4] = {0x00, 0x00, 0x03, 0x04};
constexpr uint8_t kEmptyBoltVersion[4] = {0x00, 0x00, 0x00, 0x00};
} // namespace
namespace memgraph::communication::bolt {
Client::Client(communication::ClientContext &context) : client_{&context} {}
void Client::Connect(const io::network::Endpoint &endpoint, const std::string &username, const std::string &password,
const std::string &client_name) {
if (!client_.Connect(endpoint)) {
throw ClientFatalException("Couldn't connect to {}!", endpoint);
}
if (!client_.Write(kPreamble, sizeof(kPreamble), true)) {
spdlog::error("Couldn't send preamble!");
throw ServerCommunicationException();
}
if (!client_.Write(kBoltV43Version, sizeof(kBoltV43Version), true)) {
spdlog::error("Couldn't send protocol version!");
throw ServerCommunicationException();
}
for (int i = 0; i < 3; ++i) {
if (!client_.Write(kEmptyBoltVersion, sizeof(kEmptyBoltVersion), i != 2)) {
spdlog::error("Couldn't send protocol version!");
throw ServerCommunicationException();
}
}
if (!client_.Read(sizeof(kBoltV43Version))) {
spdlog::error("Couldn't get negotiated protocol version!");
throw ServerCommunicationException();
}
if (memcmp(kBoltV43Version, client_.GetData(), sizeof(kBoltV43Version)) != 0) {
spdlog::error("Server negotiated unsupported protocol version!");
throw ClientFatalException("The server negotiated an usupported protocol version!");
}
client_.ShiftData(sizeof(kBoltV43Version));
if (!encoder_.MessageInit({{"user_agent", client_name},
{"scheme", "basic"},
{"principal", username},
{"credentials", password},
{"routing", {}}})) {
spdlog::error("Couldn't send init message!");
throw ServerCommunicationException();
}
Signature signature{};
Value metadata;
if (!ReadMessage(signature, metadata)) {
spdlog::error("Couldn't read init message response!");
throw ServerCommunicationException();
}
if (signature != Signature::Success) {
spdlog::error("Handshake failed!");
throw ClientFatalException("Handshake with the server failed!");
}
spdlog::debug("Metadata of init message response: {}", metadata);
}
QueryData Client::Execute(const std::string &query, const std::map<std::string, Value> &parameters) {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
spdlog::debug("Sending run message with statement: '{}'; parameters: {}", query, parameters);
// It is super critical from performance point of view to send the pull message right after the run message. Otherwise
// the performance will degrade multiple magnitudes.
encoder_.MessageRun(query, parameters, {});
encoder_.MessagePull({{"n", Value(-1)}});
spdlog::debug("Reading run message response");
Signature signature{};
Value fields;
if (!ReadMessage(signature, fields)) {
throw ServerCommunicationException();
}
if (fields.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
if (signature == Signature::Failure) {
HandleFailure<ClientQueryException>(fields.ValueMap());
}
if (signature != Signature::Success) {
throw ServerMalformedDataException();
}
spdlog::debug("Reading pull_all message response");
Marker marker{};
Value metadata;
std::vector<std::vector<Value>> records;
while (true) {
if (!GetMessage()) {
throw ServerCommunicationException();
}
if (!decoder_.ReadMessageHeader(&signature, &marker)) {
throw ServerCommunicationException();
}
if (signature == Signature::Record) {
Value record;
if (!decoder_.ReadValue(&record, Value::Type::List)) {
throw ServerCommunicationException();
}
records.emplace_back(std::move(record.ValueList()));
} else if (signature == Signature::Success) {
if (!decoder_.ReadValue(&metadata)) {
throw ServerCommunicationException();
}
break;
} else if (signature == Signature::Failure) {
Value data;
if (!decoder_.ReadValue(&data)) {
throw ServerCommunicationException();
}
HandleFailure<ClientQueryException>(data.ValueMap());
} else {
throw ServerMalformedDataException();
}
}
if (metadata.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
QueryData ret{{}, std::move(records), std::move(metadata.ValueMap())};
auto &header = fields.ValueMap();
if (header.find("fields") == header.end()) {
throw ServerMalformedDataException();
}
if (header["fields"].type() != Value::Type::List) {
throw ServerMalformedDataException();
}
auto &field_vector = header["fields"].ValueList();
for (auto &field_item : field_vector) {
if (field_item.type() != Value::Type::String) {
throw ServerMalformedDataException();
}
ret.fields.emplace_back(std::move(field_item.ValueString()));
}
return ret;
}
void Client::Reset() {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
spdlog::debug("Sending reset message");
encoder_.MessageReset();
Signature signature{};
Value fields;
// In Execute the pull message is sent right after the run message without reading the answer for the run message.
// That means some of the messages sent might get ignored.
while (true) {
if (!ReadMessage(signature, fields)) {
throw ServerCommunicationException();
}
if (signature == Signature::Success) {
break;
}
if (signature != Signature::Ignored) {
throw ServerMalformedDataException();
}
}
}
std::optional<std::map<std::string, Value>> Client::Route(const std::map<std::string, Value> &routing,
const std::vector<Value> &bookmarks,
const std::optional<std::string> &db) {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
spdlog::debug("Sending route message with routing: {}; bookmarks: {}; db: {}", routing, bookmarks,
db.has_value() ? *db : Value());
encoder_.MessageRoute(routing, bookmarks, db);
spdlog::debug("Reading route message response");
Signature signature{};
Value fields;
if (!ReadMessage(signature, fields)) {
throw ServerCommunicationException();
}
if (signature == Signature::Ignored) {
return std::nullopt;
}
if (signature == Signature::Failure) {
HandleFailure(fields.ValueMap());
}
if (signature != Signature::Success) {
throw ServerMalformedDataException{};
}
return fields.ValueMap();
}
void Client::Close() { client_.Close(); };
bool Client::GetMessage() {
client_.ClearData();
while (true) {
if (!client_.Read(kChunkHeaderSize)) return false;
size_t chunk_size = client_.GetData()[0];
chunk_size <<= 8U;
chunk_size += client_.GetData()[1];
if (chunk_size == 0) return true;
if (!client_.Read(chunk_size)) return false;
if (decoder_buffer_.GetChunk() != ChunkState::Whole) return false;
client_.ClearData();
}
return true;
}
bool Client::ReadMessage(Signature &signature, Value &ret) {
Marker marker{};
if (!GetMessage()) return false;
if (!decoder_.ReadMessageHeader(&signature, &marker)) return false;
return ReadMessageData(marker, ret);
}
bool Client::ReadMessageData(Marker marker, Value &ret) {
if (marker == Marker::TinyStruct) {
ret = Value();
return true;
}
if (marker == Marker::TinyStruct1) {
return decoder_.ReadValue(&ret);
}
return false;
}
} // namespace memgraph::communication::bolt

View File

@@ -11,12 +11,6 @@
#pragma once
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/decoder/chunked_decoder_buffer.hpp"
#include "communication/bolt/v1/decoder/decoder.hpp"
#include "communication/bolt/v1/encoder/chunked_encoder_buffer.hpp"
@@ -25,17 +19,22 @@
#include "communication/context.hpp"
#include "io/network/endpoint.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
namespace memgraph::communication::bolt {
class FailureResponseException : public utils::BasicException {
/// This exception is thrown whenever an error occurs during query execution
/// that isn't fatal (eg. mistyped query or some transient error occurred).
/// It should be handled by everyone who uses the client.
class ClientQueryException : public utils::BasicException {
public:
FailureResponseException() : utils::BasicException{"Couldn't execute query!"} {}
using utils::BasicException::BasicException;
explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {}
ClientQueryException() : utils::BasicException("Couldn't execute query!") {}
FailureResponseException(const std::string &code, const std::string &message)
: utils::BasicException{message}, code_{code} {}
template <class... Args>
ClientQueryException(const std::string &code, Args &&...args)
: utils::BasicException(std::forward<Args>(args)...), code_(code) {}
const std::string &code() const { return code_; }
@@ -43,14 +42,6 @@ class FailureResponseException : public utils::BasicException {
std::string code_;
};
/// This exception is thrown whenever an error occurs during query execution
/// that isn't fatal (eg. mistyped query or some transient error occurred).
/// It should be handled by everyone who uses the client.
class ClientQueryException : public FailureResponseException {
public:
using FailureResponseException::FailureResponseException;
};
/// This exception is thrown whenever a fatal error occurs during query
/// execution and/or connecting to the server.
/// It should be handled by everyone who uses the client.
@@ -85,13 +76,12 @@ struct QueryData {
/// server. It supports both SSL and plaintext connections.
class Client final {
public:
explicit Client(communication::ClientContext &context);
explicit Client(communication::ClientContext *context) : client_(context) {}
Client(const Client &) = delete;
Client(Client &&) = delete;
Client &operator=(const Client &) = delete;
Client &operator=(Client &&) = delete;
~Client() = default;
/// Method used to connect to the server. Before executing queries this method
/// should be called to set-up the connection to the server. After the
@@ -99,7 +89,50 @@ class Client final {
/// established connection.
/// @throws ClientFatalException when we couldn't connect to the server
void Connect(const io::network::Endpoint &endpoint, const std::string &username, const std::string &password,
const std::string &client_name = "memgraph-bolt");
const std::string &client_name = "memgraph-bolt") {
if (!client_.Connect(endpoint)) {
throw ClientFatalException("Couldn't connect to {}!", endpoint);
}
if (!client_.Write(kPreamble, sizeof(kPreamble), true)) {
SPDLOG_ERROR("Couldn't send preamble!");
throw ServerCommunicationException();
}
for (int i = 0; i < 4; ++i) {
if (!client_.Write(kProtocol, sizeof(kProtocol), i != 3)) {
SPDLOG_ERROR("Couldn't send protocol version!");
throw ServerCommunicationException();
}
}
if (!client_.Read(sizeof(kProtocol))) {
SPDLOG_ERROR("Couldn't get negotiated protocol version!");
throw ServerCommunicationException();
}
if (memcmp(kProtocol, client_.GetData(), sizeof(kProtocol)) != 0) {
SPDLOG_ERROR("Server negotiated unsupported protocol version!");
throw ClientFatalException("The server negotiated an usupported protocol version!");
}
client_.ShiftData(sizeof(kProtocol));
if (!encoder_.MessageInit(client_name, {{"scheme", "basic"}, {"principal", username}, {"credentials", password}})) {
SPDLOG_ERROR("Couldn't send init message!");
throw ServerCommunicationException();
}
Signature signature;
Value metadata;
if (!ReadMessage(&signature, &metadata)) {
SPDLOG_ERROR("Couldn't read init message response!");
throw ServerCommunicationException();
}
if (signature != Signature::Success) {
SPDLOG_ERROR("Handshake failed!");
throw ClientFatalException("Handshake with the server failed!");
}
SPDLOG_INFO("Metadata of init message response: {}", metadata);
}
/// Function used to execute queries against the server. Before you can
/// execute queries you must connect the client to the server.
@@ -107,40 +140,167 @@ class Client final {
/// executing the query (eg. mistyped query,
/// etc.)
/// @throws ClientFatalException when we couldn't communicate with the server
QueryData Execute(const std::string &query, const std::map<std::string, Value> &parameters);
/// Close the active client connection.
void Close();
/// Can be used to reset the active client connection. Reset is automatically sent after receiving a failure message
/// from the server, which result in throwing an FailureResponseException or any exception derived from it.
void Reset();
/// Can be used to send a route message.
std::optional<std::map<std::string, Value>> Route(const std::map<std::string, Value> &routing,
const std::vector<Value> &bookmarks,
const std::optional<std::string> &db);
private:
using ClientEncoder = ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>>;
template <typename TException = FailureResponseException>
[[noreturn]] void HandleFailure(const std::map<std::string, Value> &response_map) {
Reset();
auto it = response_map.find("message");
if (it != response_map.end()) {
auto it_code = response_map.find("code");
if (it_code != response_map.end()) {
throw TException(it_code->second.ValueString(), it->second.ValueString());
}
throw TException("", it->second.ValueString());
QueryData Execute(const std::string &query, const std::map<std::string, Value> &parameters) {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
throw TException();
SPDLOG_INFO("Sending run message with statement: '{}'; parameters: {}", query, parameters);
encoder_.MessageRun(query, parameters);
encoder_.MessagePullAll();
SPDLOG_INFO("Reading run message response");
Signature signature;
Value fields;
if (!ReadMessage(&signature, &fields)) {
throw ServerCommunicationException();
}
if (fields.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
if (signature == Signature::Failure) {
HandleFailure();
auto &tmp = fields.ValueMap();
auto it = tmp.find("message");
if (it != tmp.end()) {
auto it_code = tmp.find("code");
if (it_code != tmp.end()) {
throw ClientQueryException(it_code->second.ValueString(), it->second.ValueString());
} else {
throw ClientQueryException("", it->second.ValueString());
}
}
throw ClientQueryException();
} else if (signature != Signature::Success) {
throw ServerMalformedDataException();
}
SPDLOG_INFO("Reading pull_all message response");
Marker marker;
Value metadata;
std::vector<std::vector<Value>> records;
while (true) {
if (!GetMessage()) {
throw ServerCommunicationException();
}
if (!decoder_.ReadMessageHeader(&signature, &marker)) {
throw ServerCommunicationException();
}
if (signature == Signature::Record) {
Value record;
if (!decoder_.ReadValue(&record, Value::Type::List)) {
throw ServerCommunicationException();
}
records.emplace_back(std::move(record.ValueList()));
} else if (signature == Signature::Success) {
if (!decoder_.ReadValue(&metadata)) {
throw ServerCommunicationException();
}
break;
} else if (signature == Signature::Failure) {
Value data;
if (!decoder_.ReadValue(&data)) {
throw ServerCommunicationException();
}
HandleFailure();
auto &tmp = data.ValueMap();
auto it = tmp.find("message");
if (it != tmp.end()) {
auto it_code = tmp.find("code");
if (it_code != tmp.end()) {
throw ClientQueryException(it_code->second.ValueString(), it->second.ValueString());
} else {
throw ClientQueryException("", it->second.ValueString());
}
}
throw ClientQueryException();
} else {
throw ServerMalformedDataException();
}
}
if (metadata.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
QueryData ret{{}, std::move(records), std::move(metadata.ValueMap())};
auto &header = fields.ValueMap();
if (header.find("fields") == header.end()) {
throw ServerMalformedDataException();
}
if (header["fields"].type() != Value::Type::List) {
throw ServerMalformedDataException();
}
auto &field_vector = header["fields"].ValueList();
for (auto &field_item : field_vector) {
if (field_item.type() != Value::Type::String) {
throw ServerMalformedDataException();
}
ret.fields.emplace_back(std::move(field_item.ValueString()));
}
return ret;
}
bool GetMessage();
bool ReadMessage(Signature &signature, Value &ret);
bool ReadMessageData(Marker marker, Value &ret);
/// Close the active client connection.
void Close() { client_.Close(); };
private:
bool GetMessage() {
client_.ClearData();
while (true) {
if (!client_.Read(kChunkHeaderSize)) return false;
size_t chunk_size = client_.GetData()[0];
chunk_size <<= 8;
chunk_size += client_.GetData()[1];
if (chunk_size == 0) return true;
if (!client_.Read(chunk_size)) return false;
if (decoder_buffer_.GetChunk() != ChunkState::Whole) return false;
client_.ClearData();
}
return true;
}
bool ReadMessage(Signature *signature, Value *ret) {
Marker marker;
if (!GetMessage()) return false;
if (!decoder_.ReadMessageHeader(signature, &marker)) return false;
return ReadMessageData(marker, ret);
}
bool ReadMessageData(Marker marker, Value *ret) {
if (marker == Marker::TinyStruct) {
*ret = Value();
return true;
} else if (marker == Marker::TinyStruct1) {
return decoder_.ReadValue(ret);
}
return false;
}
void HandleFailure() {
if (!encoder_.MessageAckFailure()) {
throw ServerCommunicationException();
}
while (true) {
Signature signature;
Value data;
if (!ReadMessage(&signature, &data)) {
throw ServerCommunicationException();
}
if (signature == Signature::Success) {
break;
} else if (signature != Signature::Ignored) {
throw ServerMalformedDataException();
}
}
}
// client
communication::Client client_;
@@ -153,6 +313,6 @@ class Client final {
// encoder objects
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_};
ClientEncoder encoder_{encoder_buffer_};
ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>> encoder_{encoder_buffer_};
};
} // namespace memgraph::communication::bolt

View File

@@ -16,6 +16,7 @@
namespace memgraph::communication::bolt {
inline constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
inline constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
enum class Signature : uint8_t {
Noop = 0x00,

View File

@@ -11,11 +11,6 @@
#pragma once
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/encoder/base_encoder.hpp"
@@ -35,7 +30,6 @@ class ClientEncoder : private BaseEncoder<Buffer> {
using BaseEncoder<Buffer>::WriteList;
using BaseEncoder<Buffer>::WriteMap;
using BaseEncoder<Buffer>::WriteString;
using BaseEncoder<Buffer>::WriteNull;
using BaseEncoder<Buffer>::buffer_;
public:
@@ -44,9 +38,10 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a Init message.
*
* From the Bolt v4.3 documentation:
* HelloMess (signature=0x01) {
* Map<String,Value> extra
* From the Bolt v1 documentation:
* InitMessage (signature=0x01) {
* String clientName
* Map<String,Value> authToken
* }
*
* @param client_name the name of the connected client
@@ -54,10 +49,11 @@ class ClientEncoder : private BaseEncoder<Buffer> {
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageInit(const std::map<std::string, Value> &extra) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
bool MessageInit(const std::string client_name, const std::map<std::string, Value> &auth_token) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
WriteRAW(utils::UnderlyingCast(Signature::Init));
WriteMap(extra);
WriteString(client_name);
WriteMap(auth_token);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -68,11 +64,10 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a Run message.
*
* From the Bolt v4.3 documentation:
* From the Bolt v1 documentation:
* RunMessage (signature=0x10) {
* String statement
* Map<String,Value> parameters
* Map<String,Value> extra
* String statement
* Map<String,Value> parameters
* }
*
* @param statement the statement that should be executed
@@ -80,13 +75,11 @@ class ClientEncoder : private BaseEncoder<Buffer> {
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageRun(const std::string &statement, const std::map<std::string, Value> &parameters,
const std::map<std::string, Value> &extra, bool have_more = true) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct3));
bool MessageRun(const std::string &statement, const std::map<std::string, Value> &parameters, bool have_more = true) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
WriteRAW(utils::UnderlyingCast(Signature::Run));
WriteString(statement);
WriteMap(parameters);
WriteMap(extra);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -97,20 +90,18 @@ class ClientEncoder : private BaseEncoder<Buffer> {
}
/**
* Writes a Discard message.
* Writes a DiscardAll message.
*
* From the Bolt v4.3 documentation:
* From the Bolt v1 documentation:
* DiscardMessage (signature=0x2F) {
* Map<String,Value> extra
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageDiscard(const std::map<std::string, Value> &extra) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
bool MessageDiscardAll() {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
WriteRAW(utils::UnderlyingCast(Signature::Discard));
WriteMap(extra);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -121,18 +112,36 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a PullAll message.
*
* From the Bolt v4.3 documentation:
* PullMessage (signature=0x3F) {
* Map<String,Value> extra
* From the Bolt v1 documentation:
* PullAllMessage (signature=0x3F) {
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessagePull(const std::map<std::string, Value> &extra) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
bool MessagePullAll() {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
WriteRAW(utils::UnderlyingCast(Signature::Pull));
WriteMap(extra);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
/**
* Writes a AckFailure message.
*
* From the Bolt v1 documentation:
* AckFailureMessage (signature=0x0E) {
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageAckFailure() {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
WriteRAW(utils::UnderlyingCast(Signature::AckFailure));
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -143,7 +152,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a Reset message.
*
* From the Bolt v4.3 documentation:
* From the Bolt v1 documentation:
* ResetMessage (signature=0x0F) {
* }
*
@@ -159,36 +168,5 @@ class ClientEncoder : private BaseEncoder<Buffer> {
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
/**
* Writes a Route message.
*
* From the Bolt v4.3 documentation:
* RouteMessage (signature=0x0F) {
* Map<String,Value> routing
* List<String> bookmarks
* String db
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageRoute(const std::map<std::string, Value> &routing, const std::vector<Value> &bookmarks,
const std::optional<std::string> &db) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct3));
WriteRAW(utils::UnderlyingCast(Signature::Route));
WriteMap(routing);
WriteList(bookmarks);
if (db.has_value()) {
WriteString(*db);
} else {
WriteNull();
}
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
};
} // namespace memgraph::communication::bolt

View File

@@ -117,6 +117,29 @@ class Encoder : private BaseEncoder<Buffer> {
return buffer_.Flush();
}
/**
* Sends an Ignored message.
*
* From the bolt v1 documentation:
* IgnoredMessage (signature=0x7E) {
* Map<String,Value> metadata
* }
*
* @param metadata the metadata map object that should be sent
* @returns true if the data was successfully sent to the client,
* false otherwise
*/
bool MessageIgnored(const std::map<std::string, Value> &metadata) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
WriteRAW(utils::UnderlyingCast(Signature::Ignored));
WriteMap(metadata);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
/**
* Sends an Ignored message.
*

View File

@@ -15,7 +15,6 @@
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/states/handlers.hpp"
#include "communication/bolt/v1/value.hpp"
#include "utils/cast.hpp"
#include "utils/likely.hpp"
@@ -31,8 +30,8 @@ namespace memgraph::communication::bolt {
*/
template <typename TSession>
State StateErrorRun(TSession &session, State state) {
Marker marker{};
Signature signature{};
Marker marker;
Signature signature;
if (!session.decoder_.ReadMessageHeader(&signature, &marker)) {
spdlog::trace("Missing header data!");
return State::Close;
@@ -46,49 +45,54 @@ State StateErrorRun(TSession &session, State state) {
// Clear the data buffer if it has any leftover data.
session.encoder_buffer_.Clear();
if (session.version_.major == 1 && signature == Signature::AckFailure) {
spdlog::trace("AckFailure received");
if ((session.version_.major == 1 && signature == Signature::AckFailure) || signature == Signature::Reset) {
if (signature == Signature::AckFailure) {
spdlog::trace("AckFailure received");
} else {
spdlog::trace("Reset received");
}
if (!session.encoder_.MessageSuccess()) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
if (signature == Signature::Reset) {
session.Abort();
return State::Idle;
}
// We got AckFailure get back to right state.
MG_ASSERT(state == State::Error, "Shouldn't happen");
return State::Idle;
}
if (signature == Signature::Reset) {
spdlog::trace("Reset received");
return HandleReset(session, marker);
}
} else {
uint8_t value = utils::UnderlyingCast(marker);
uint8_t value = utils::UnderlyingCast(marker);
// All bolt client messages have less than 15 parameters so if we receive
// anything than a TinyStruct it's an error.
if ((value & 0xF0U) != utils::UnderlyingCast(Marker::TinyStruct)) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", value);
return State::Close;
}
// We need to clean up all parameters from this command.
value &= 0x0FU; // The length is stored in the lower nibble.
Value dv;
for (int i = 0; i < value; ++i) {
if (!session.decoder_.ReadValue(&dv)) {
spdlog::trace("Couldn't clean up parameter {} / {}!", i, value);
// All bolt client messages have less than 15 parameters so if we receive
// anything than a TinyStruct it's an error.
if ((value & 0xF0) != utils::UnderlyingCast(Marker::TinyStruct)) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", value);
return State::Close;
}
}
// Ignore this message.
if (!session.encoder_.MessageIgnored()) {
spdlog::trace("Couldn't send ignored message!");
return State::Close;
}
// We need to clean up all parameters from this command.
value &= 0x0F; // The length is stored in the lower nibble.
Value dv;
for (int i = 0; i < value; ++i) {
if (!session.decoder_.ReadValue(&dv)) {
spdlog::trace("Couldn't clean up parameter {} / {}!", i, value);
return State::Close;
}
}
// Cleanup done, command ignored, stay in error state.
return state;
// Ignore this message.
if (!session.encoder_.MessageIgnored()) {
spdlog::trace("Couldn't send ignored message!");
return State::Close;
}
// Cleanup done, command ignored, stay in error state.
return state;
}
}
} // namespace memgraph::communication::bolt

View File

@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
}
case Signature::Route: {
if constexpr (bolt_minor >= 3) {
if (signature == Signature::Route) return HandleRoute<TSession>(session, marker);
if (signature == Signature::Route) return HandleRoute<TSession>(session);
} else {
spdlog::trace("Supported only in bolt v4.3");
return State::Close;

View File

@@ -18,7 +18,6 @@
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/bolt/v1/exceptions.hpp"
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
@@ -137,7 +136,7 @@ template <bool is_pull, typename TSession>
State HandlePullDiscardV1(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct;
if (marker != expected_marker) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -158,7 +157,7 @@ template <bool is_pull, typename TSession>
State HandlePullDiscardV4(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct1;
if (marker != expected_marker) {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct1", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -217,8 +216,7 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query;
Value params;
Value query, params;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
@@ -236,12 +234,10 @@ template <typename TSession>
State HandleRunV4(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct3;
if (marker != expected_marker) {
spdlog::trace("Expected TinyStruct3 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query;
Value params;
Value extra;
Value query, params, extra;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
@@ -296,6 +292,9 @@ State HandleReset(TSession &session, const Marker marker) {
return State::Close;
}
// Clear all pending data and send a success message.
session.encoder_buffer_.Clear();
if (!session.encoder_.MessageSuccess()) {
spdlog::trace("Couldn't send success message!");
return State::Close;
@@ -404,33 +403,12 @@ State HandleGoodbye() {
}
template <typename TSession>
State HandleRoute(TSession &session, const Marker marker) {
// Route message is not implemented since it is Neo4j specific, therefore we will receive it and inform user that
// there is no implementation. Before that, we have to read out the fields from the buffer to leave it in a clean
// state.
if (marker != Marker::TinyStruct3) {
spdlog::trace("Expected TinyStruct3 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
Value routing;
if (!session.decoder_.ReadValue(&routing, Value::Type::Map)) {
spdlog::trace("Couldn't read routing field!");
return State::Close;
}
Value bookmarks;
if (!session.decoder_.ReadValue(&bookmarks, Value::Type::List)) {
spdlog::trace("Couldn't read bookmarks field!");
return State::Close;
}
Value db;
if (!session.decoder_.ReadValue(&db)) {
spdlog::trace("Couldn't read db field!");
return State::Close;
}
State HandleRoute(TSession &session) {
// Route message is not implemented since it is neo4j specific, therefore we
// will receive it an inform user that there is no implementation.
session.encoder_buffer_.Clear();
bool fail_sent =
session.encoder_.MessageFailure({{"code", "66"}, {"message", "Route message is not supported in Memgraph!"}});
session.encoder_.MessageFailure({{"code", 66}, {"message", "Route message not supported in Memgraph!"}});
if (!fail_sent) {
spdlog::trace("Couldn't send failure message!");
return State::Close;

View File

@@ -0,0 +1,9 @@
set(coordinator_src_files
coordinator.cpp
shard_map.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-coordinator STATIC ${coordinator_src_files})
target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils mg-storage-v3)

View File

@@ -0,0 +1,129 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <coordinator/coordinator.hpp>
namespace memgraph::coordinator {
CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_request) {
spdlog::info("Coordinator handling HeartbeatRequest");
// add this storage engine to any under-replicated shards that it is not already a part of
auto initializing_rsms_for_shard_manager =
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
return HeartbeatResponse{
.shards_to_initialize = initializing_rsms_for_shard_manager,
};
}
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
HlcResponse res{};
auto hlc_shard_map = shard_map_.GetHlc();
MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
res.new_hlc = Hlc{
.logical_id = ++highest_allocated_timestamp_,
// TODO(tyler) probably pass some more context to the Coordinator here
// so that we can use our wall clock and enforce monotonicity.
// .coordinator_wall_clock = io_.Now(),
};
// Allways return fresher shard_map for now.
res.fresher_shard_map = std::make_optional(shard_map_);
return res;
}
CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
AllocateEdgeIdBatchResponse res{};
uint64_t low = highest_allocated_edge_id_;
highest_allocated_edge_id_ += ahr.batch_size;
uint64_t high = highest_allocated_edge_id_;
res.low = low;
res.high = high;
return res;
}
/// This splits the shard immediately beneath the provided
/// split key, keeping the assigned peers identical for now,
/// but letting them be gradually migrated over time.
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
SplitShardResponse res{};
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
res.success = false;
} else {
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
split_shard_request.split_key);
}
return res;
}
/// This adds the provided storage engine to the standby storage engine pool,
/// which can be used to rebalance storage over time.
CoordinatorWriteResponses Coordinator::ApplyWrite(
RegisterStorageEngineRequest && /* register_storage_engine_request */) {
RegisterStorageEngineResponse res{};
// TODO
return res;
}
/// This begins the process of draining the provided storage engine from all raft
/// clusters that it might be participating in.
CoordinatorWriteResponses Coordinator::ApplyWrite(
DeregisterStorageEngineRequest && /* register_storage_engine_request */) {
DeregisterStorageEngineResponse res{};
// TODO
return res;
}
CoordinatorWriteResponses Coordinator::ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
InitializeLabelResponse res{};
std::optional<LabelId> new_label_id = shard_map_.InitializeNewLabel(
initialize_label_request.label_name, initialize_label_request.schema, initialize_label_request.replication_factor,
initialize_label_request.last_shard_map_version);
if (new_label_id) {
res.new_label_id = new_label_id.value();
res.fresher_shard_map = std::nullopt;
res.success = true;
} else {
res.fresher_shard_map = shard_map_;
res.success = false;
}
return res;
}
CoordinatorWriteResponses Coordinator::ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request) {
AllocatePropertyIdsResponse res{};
auto property_ids = shard_map_.AllocatePropertyIds(allocate_property_ids_request.property_names);
res.property_ids = property_ids;
return res;
}
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,196 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <optional>
#include <set>
#include <string>
#include <unordered_set>
#include <variant>
#include <vector>
#include <boost/uuid/uuid.hpp>
#include <coordinator/hybrid_logical_clock.hpp>
#include <coordinator/shard_map.hpp>
#include <io/simulator/simulator.hpp>
#include <io/time.hpp>
#include <io/transport.hpp>
#include <storage/v3/id_types.hpp>
#include <storage/v3/schemas.hpp>
namespace memgraph::coordinator {
using memgraph::io::Address;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyId;
using memgraph::storage::v3::SchemaProperty;
using SimT = memgraph::io::simulator::SimulatorTransport;
using PrimaryKey = std::vector<PropertyValue>;
struct HlcRequest {
Hlc last_shard_map_version;
};
struct HlcResponse {
Hlc new_hlc;
std::optional<ShardMap> fresher_shard_map;
};
struct GetShardMapRequest {
// No state
};
struct GetShardMapResponse {
ShardMap shard_map;
};
struct AllocateHlcBatchRequest {
Hlc low;
Hlc high;
};
struct AllocateHlcBatchResponse {
bool success;
Hlc low;
Hlc high;
};
struct AllocateEdgeIdBatchRequest {
size_t batch_size;
};
struct AllocateEdgeIdBatchResponse {
uint64_t low;
uint64_t high;
};
struct AllocatePropertyIdsRequest {
std::vector<std::string> property_names;
};
struct AllocatePropertyIdsResponse {
std::map<std::string, PropertyId> property_ids;
};
struct SplitShardRequest {
Hlc previous_shard_map_version;
LabelId label_id;
PrimaryKey split_key;
};
struct SplitShardResponse {
bool success;
};
struct RegisterStorageEngineRequest {
Address address;
};
struct RegisterStorageEngineResponse {
bool success;
};
struct DeregisterStorageEngineRequest {
Address address;
};
struct DeregisterStorageEngineResponse {
bool success;
};
struct InitializeLabelRequest {
std::string label_name;
std::vector<SchemaProperty> schema;
size_t replication_factor;
Hlc last_shard_map_version;
};
struct InitializeLabelResponse {
bool success;
LabelId new_label_id;
std::optional<ShardMap> fresher_shard_map;
};
struct HeartbeatRequest {
Address from_storage_manager;
std::set<boost::uuids::uuid> initialized_rsms;
};
struct HeartbeatResponse {
std::vector<ShardToInitialize> shards_to_initialize;
};
using CoordinatorWriteRequests =
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
using CoordinatorWriteResponses = std::variant<HlcResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
RegisterStorageEngineResponse, DeregisterStorageEngineResponse,
InitializeLabelResponse, AllocatePropertyIdsResponse, HeartbeatResponse>;
using CoordinatorReadRequests = std::variant<GetShardMapRequest>;
using CoordinatorReadResponses = std::variant<GetShardMapResponse>;
class Coordinator {
public:
explicit Coordinator(ShardMap sm) : shard_map_{std::move(sm)} {}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
CoordinatorReadResponses Read(CoordinatorReadRequests requests) {
return std::visit([&](auto &&request) { return HandleRead(std::forward<decltype(request)>(request)); },
std::move(requests)); // NOLINT(hicpp-move-const-arg,performance-move-const-arg)
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static
CoordinatorWriteResponses Apply(CoordinatorWriteRequests requests) {
return std::visit([&](auto &&request) mutable { return ApplyWrite(std::forward<decltype(request)>(request)); },
std::move(requests));
}
private:
ShardMap shard_map_;
uint64_t highest_allocated_timestamp_;
/// Query engines need to periodically request batches of unique edge IDs.
uint64_t highest_allocated_edge_id_;
CoordinatorReadResponses HandleRead(GetShardMapRequest && /* get_shard_map_request */) {
GetShardMapResponse res;
res.shard_map = shard_map_;
return res;
}
CoordinatorWriteResponses ApplyWrite(HeartbeatRequest &&heartbeat_request);
CoordinatorWriteResponses ApplyWrite(HlcRequest &&hlc_request);
CoordinatorWriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr);
/// This splits the shard immediately beneath the provided
/// split key, keeping the assigned peers identical for now,
/// but letting them be gradually migrated over time.
CoordinatorWriteResponses ApplyWrite(SplitShardRequest &&split_shard_request);
/// This adds the provided storage engine to the standby storage engine pool,
/// which can be used to rebalance storage over time.
static CoordinatorWriteResponses ApplyWrite(RegisterStorageEngineRequest && /* register_storage_engine_request */);
/// This begins the process of draining the provided storage engine from all raft
/// clusters that it might be participating in.
static CoordinatorWriteResponses ApplyWrite(DeregisterStorageEngineRequest && /* register_storage_engine_request */);
CoordinatorWriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request);
CoordinatorWriteResponses ApplyWrite(AllocatePropertyIdsRequest &&allocate_property_ids_request);
};
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,24 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "coordinator/coordinator.hpp"
#include "io/rsm/rsm_client.hpp"
namespace memgraph::coordinator {
using memgraph::io::rsm::RsmClient;
template <typename IoImpl>
using CoordinatorClient = RsmClient<IoImpl, CoordinatorWriteRequests, CoordinatorWriteResponses,
CoordinatorReadRequests, CoordinatorReadResponses>;
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "coordinator/coordinator.hpp"
#include "io/rsm/raft.hpp"
namespace memgraph::coordinator {
template <typename IoImpl>
using CoordinatorRsm = memgraph::io::rsm::Raft<IoImpl, Coordinator, CoordinatorWriteRequests, CoordinatorWriteResponses,
CoordinatorReadRequests, CoordinatorReadResponses>;
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,36 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <compare>
#include "io/time.hpp"
namespace memgraph::coordinator {
using Time = memgraph::io::Time;
/// Hybrid-logical clock
struct Hlc {
uint64_t logical_id;
Time coordinator_wall_clock;
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
bool operator==(const Hlc &other) const = default;
bool operator<(const Hlc &other) const = default;
bool operator==(const uint64_t other) const { return logical_id == other; }
bool operator<(const uint64_t other) const { return logical_id < other; }
bool operator>=(const uint64_t other) const { return logical_id >= other; }
};
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,90 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "coordinator/shard_map.hpp"
#include "storage/v3/temporal.hpp"
namespace memgraph::coordinator {
using memgraph::common::SchemaType;
using memgraph::storage::v3::TemporalData;
using memgraph::storage::v3::TemporalType;
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
PrimaryKey ret{};
const int64_t min_int = std::numeric_limits<int64_t>::min();
const TemporalData date{TemporalType::Date, min_int};
const TemporalData local_time{TemporalType::LocalTime, min_int};
const TemporalData local_date_time{TemporalType::LocalDateTime, min_int};
const TemporalData duration{TemporalType::Duration, min_int};
for (const auto &schema_property : schema) {
switch (schema_property.type) {
case SchemaType::BOOL:
ret.emplace_back(PropertyValue(false));
break;
case SchemaType::INT:
ret.emplace_back(PropertyValue(min_int));
break;
case SchemaType::STRING:
ret.emplace_back(PropertyValue(""));
break;
case SchemaType::DATE:
ret.emplace_back(PropertyValue(date));
break;
case SchemaType::LOCALTIME:
ret.emplace_back(PropertyValue(local_time));
break;
case SchemaType::LOCALDATETIME:
ret.emplace_back(PropertyValue(local_date_time));
break;
case SchemaType::DURATION:
ret.emplace_back(PropertyValue(duration));
break;
}
}
return ret;
}
std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
size_t replication_factor, Hlc last_shard_map_version) {
if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
return std::nullopt;
}
const LabelId label_id = LabelId::FromUint(++max_label_id);
labels.emplace(std::move(label_name), label_id);
PrimaryKey initial_key = SchemaToMinKey(schema);
Shard empty_shard = {};
Shards shards = {
{initial_key, empty_shard},
};
LabelSpace label_space{
.schema = std::move(schema),
.shards = shards,
.replication_factor = replication_factor,
};
label_spaces.emplace(label_id, label_space);
IncrementShardMapVersion();
return label_id;
}
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,276 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <limits>
#include <map>
#include <set>
#include <vector>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include "common/types.hpp"
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/temporal.hpp"
namespace memgraph::coordinator {
using memgraph::io::Address;
using memgraph::storage::v3::Config;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyId;
using memgraph::storage::v3::PropertyValue;
using memgraph::storage::v3::SchemaProperty;
enum class Status : uint8_t {
CONSENSUS_PARTICIPANT,
INITIALIZING,
// TODO(tyler) this will possibly have more states,
// depending on the reconfiguration protocol that we
// implement.
};
struct AddressAndStatus {
memgraph::io::Address address;
Status status;
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
};
using PrimaryKey = std::vector<PropertyValue>;
using Shard = std::vector<AddressAndStatus>;
using Shards = std::map<PrimaryKey, Shard>;
using LabelName = std::string;
using PropertyName = std::string;
using PropertyMap = std::map<PropertyName, PropertyId>;
struct ShardToInitialize {
boost::uuids::uuid uuid;
LabelId label_id;
PrimaryKey min_key;
std::optional<PrimaryKey> max_key;
Config config;
};
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
struct LabelSpace {
std::vector<SchemaProperty> schema;
std::map<PrimaryKey, Shard> shards;
size_t replication_factor;
};
struct ShardMap {
Hlc shard_map_version;
uint64_t max_property_id;
std::map<PropertyName, PropertyId> properties;
uint64_t max_label_id;
std::map<LabelName, LabelId> labels;
std::map<LabelId, LabelSpace> label_spaces;
std::map<LabelId, std::vector<SchemaProperty>> schemas;
Shards GetShards(const LabelName &label) {
const auto id = labels.at(label);
auto &shards = label_spaces.at(id).shards;
return shards;
}
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well
Hlc IncrementShardMapVersion() noexcept {
++shard_map_version.logical_id;
return shard_map_version;
}
Hlc GetHlc() const noexcept { return shard_map_version; }
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
std::vector<ShardToInitialize> ret{};
bool mutated = false;
for (auto &[label_id, label_space] : label_spaces) {
for (auto &[low_key, shard] : label_space.shards) {
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
bool machine_contains_shard = false;
for (auto &aas : shard) {
if (initialized.contains(aas.address.unique_id)) {
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
aas.status = Status::CONSENSUS_PARTICIPANT;
machine_contains_shard = true;
} else {
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
aas.address.last_known_port == storage_manager.last_known_port;
if (same_machine) {
machine_contains_shard = true;
ret.push_back(ShardToInitialize{
.uuid = aas.address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = std::nullopt,
.config = Config{},
});
}
}
}
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
Address address = storage_manager;
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
ret.push_back(ShardToInitialize{
.uuid = address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = std::nullopt,
.config = Config{},
});
AddressAndStatus aas = {
.address = address,
.status = Status::INITIALIZING,
};
shard.emplace_back(aas);
}
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
if (previous_shard_map_version != shard_map_version) {
return false;
}
auto &label_space = label_spaces.at(label_id);
auto &shards_in_map = label_space.shards;
MG_ASSERT(!shards_in_map.empty());
MG_ASSERT(!shards_in_map.contains(key));
MG_ASSERT(label_spaces.contains(label_id));
// Finding the Shard that the new PrimaryKey should map to.
auto prev = std::prev(shards_in_map.upper_bound(key));
Shard duplicated_shard = prev->second;
// Apply the split
shards_in_map[key] = duplicated_shard;
return true;
}
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
size_t replication_factor, Hlc last_shard_map_version);
void AddServer(Address server_address) {
// Find a random place for the server to plug in
}
LabelId GetLabelId(const std::string &label) const { return labels.at(label); }
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const {
MG_ASSERT(start_key <= end_key);
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
const auto &shards_for_label = label_space.shards;
MG_ASSERT(shards_for_label.begin()->first <= start_key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
auto it = std::prev(shards_for_label.upper_bound(start_key));
const auto end_it = shards_for_label.upper_bound(end_key);
Shards shards{};
std::copy(it, end_it, std::inserter(shards, shards.end()));
return shards;
}
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
MG_ASSERT(label_spaces.contains(label_id));
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
PropertyMap ret{};
bool mutated = false;
for (const auto &property_name : new_properties) {
if (properties.contains(property_name)) {
auto property_id = properties.at(property_name);
ret.emplace(property_name, property_id);
} else {
mutated = true;
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
ret.emplace(property_name, property_id);
properties.emplace(property_name, property_id);
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const {
if (properties.contains(property_name)) {
return properties.at(property_name);
}
return std::nullopt;
}
};
} // namespace memgraph::coordinator

20
src/expr/CMakeLists.txt Normal file
View File

@@ -0,0 +1,20 @@
define_add_lcp(add_lcp_expr lcp_expr_cpp_files generated_lcp_expr_files)
add_lcp_expr(semantic/symbol.lcp)
add_custom_target(generate_lcp_expr DEPENDS ${generated_lcp_expr_files})
set(mg_expr_sources
${lcp_expr_cpp_files}
parsing.cpp)
find_package(Boost REQUIRED)
add_library(mg-expr STATIC ${mg_expr_sources})
add_dependencies(mg-expr generate_lcp_expr)
target_include_directories(mg-expr PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)

35
src/expr/ast.hpp Normal file
View File

@@ -0,0 +1,35 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#ifndef MG_AST_INCLUDE_PATH
#ifdef MG_CLANG_TIDY_CHECK
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp"
#else
#error Missing AST include path
#endif
#endif
#ifndef MG_INJECTED_NAMESPACE_NAME
#ifdef MG_CLANG_TIDY_CHECK
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2
#else
#error Missing AST namespace
#endif
#endif
#include MG_AST_INCLUDE_PATH
namespace memgraph::expr {
using namespace MG_INJECTED_NAMESPACE_NAME; // NOLINT(google-build-using-namespace)
} // namespace memgraph::expr

View File

@@ -0,0 +1,135 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "utils/visitor.hpp"
namespace MG_INJECTED_NAMESPACE_NAME {
// Forward declares for Tree visitors.
class CypherQuery;
class SingleQuery;
class CypherUnion;
class NamedExpression;
class Identifier;
class PropertyLookup;
class LabelsTest;
class Aggregation;
class Function;
class Reduce;
class Coalesce;
class Extract;
class All;
class Single;
class Any;
class None;
class ParameterLookup;
class CallProcedure;
class Create;
class Match;
class Return;
class With;
class Pattern;
class NodeAtom;
class EdgeAtom;
class PrimitiveLiteral;
class ListLiteral;
class MapLiteral;
class OrOperator;
class XorOperator;
class AndOperator;
class NotOperator;
class AdditionOperator;
class SubtractionOperator;
class MultiplicationOperator;
class DivisionOperator;
class ModOperator;
class UnaryPlusOperator;
class UnaryMinusOperator;
class IsNullOperator;
class NotEqualOperator;
class EqualOperator;
class LessOperator;
class GreaterOperator;
class LessEqualOperator;
class GreaterEqualOperator;
class InListOperator;
class SubscriptOperator;
class ListSlicingOperator;
class IfOperator;
class Delete;
class Where;
class SetProperty;
class SetProperties;
class SetLabels;
class RemoveProperty;
class RemoveLabels;
class Merge;
class Unwind;
class AuthQuery;
class ExplainQuery;
class ProfileQuery;
class IndexQuery;
class InfoQuery;
class ConstraintQuery;
class RegexMatch;
class DumpQuery;
class ReplicationQuery;
class LockPathQuery;
class LoadCsv;
class FreeMemoryQuery;
class TriggerQuery;
class IsolationLevelQuery;
class CreateSnapshotQuery;
class StreamQuery;
class SettingQuery;
class VersionQuery;
class Foreach;
class SchemaQuery;
using TreeCompositeVisitor = memgraph::utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach>;
using TreeLeafVisitor = memgraph::utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
class HierarchicalTreeVisitor : public TreeCompositeVisitor, public TreeLeafVisitor {
public:
using TreeCompositeVisitor::PostVisit;
using TreeCompositeVisitor::PreVisit;
using TreeLeafVisitor::Visit;
using typename TreeLeafVisitor::ReturnType;
};
template <class TResult>
class ExpressionVisitor
: public memgraph::utils::Visitor<
TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral,
MapLiteral, PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any,
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch> {};
template <class TResult>
class QueryVisitor
: public memgraph::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery,
StreamQuery, SettingQuery, VersionQuery, SchemaQuery> {};
} // namespace MG_INJECTED_NAMESPACE_NAME

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,271 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iostream>
#include <type_traits>
#include "expr/ast.hpp"
#include "expr/typed_value.hpp"
#include "utils/algorithm.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace memgraph::expr {
namespace detail {
template <typename T>
void PrintObject(std::ostream *out, const T &arg) {
static_assert(!std::is_convertible<T, Expression *>::value,
"This overload shouldn't be called with pointers convertible "
"to Expression *. This means your other PrintObject overloads aren't "
"being called for certain AST nodes when they should (or perhaps such "
"overloads don't exist yet).");
*out << arg;
}
inline void PrintObject(std::ostream *out, const std::string &str) { *out << utils::Escape(str); }
inline void PrintObject(std::ostream *out, Aggregation::Op op) { *out << Aggregation::OpToString(op); }
inline void PrintObject(std::ostream *out, Expression *expr);
inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
template <typename T>
void PrintObject(std::ostream *out, const std::vector<T> &vec) {
*out << "[";
utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); });
*out << "]";
}
template <typename T>
void PrintObject(std::ostream *out, const std::vector<T, utils::Allocator<T>> &vec) {
*out << "[";
utils::PrintIterable(*out, vec, ", ", [](auto &stream, const auto &item) { PrintObject(&stream, item); });
*out << "]";
}
template <typename K, typename V>
void PrintObject(std::ostream *out, const std::map<K, V> &map) {
*out << "{";
utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) {
PrintObject(&stream, item.first);
stream << ": ";
PrintObject(&stream, item.second);
});
*out << "}";
}
template <typename T>
void PrintObject(std::ostream *out, const utils::pmr::map<utils::pmr::string, T> &map) {
*out << "{";
utils::PrintIterable(*out, map, ", ", [](auto &stream, const auto &item) {
PrintObject(&stream, item.first);
stream << ": ";
PrintObject(&stream, item.second);
});
*out << "}";
}
template <typename T1, typename T2, typename T3>
inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
using TypedValue = TypedValueT<T1, T2, T3>;
switch (value.type()) {
case TypedValue::Type::Null:
*out << "null";
break;
case TypedValue::Type::String:
PrintObject(out, value.ValueString());
break;
case TypedValue::Type::Bool:
*out << (value.ValueBool() ? "true" : "false");
break;
case TypedValue::Type::Int:
PrintObject(out, value.ValueInt());
break;
case TypedValue::Type::Double:
PrintObject(out, value.ValueDouble());
break;
case TypedValue::Type::List:
PrintObject(out, value.ValueList());
break;
case TypedValue::Type::Map:
PrintObject(out, value.ValueMap());
break;
case TypedValue::Type::Date:
PrintObject(out, value.ValueDate());
break;
case TypedValue::Type::Duration:
PrintObject(out, value.ValueDuration());
break;
case TypedValue::Type::LocalTime:
PrintObject(out, value.ValueLocalTime());
break;
case TypedValue::Type::LocalDateTime:
PrintObject(out, value.ValueLocalDateTime());
break;
default:
MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
}
}
template <typename T>
void PrintOperatorArgs(std::ostream *out, const T &arg) {
*out << " ";
PrintObject(out, arg);
*out << ")";
}
template <typename T, typename... Ts>
void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
*out << " ";
PrintObject(out, arg);
PrintOperatorArgs(out, args...);
}
template <typename... Ts>
void PrintOperator(std::ostream *out, const std::string &name, const Ts &...args) {
*out << "(" << name;
PrintOperatorArgs(out, args...);
}
} // namespace detail
class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
public:
explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
// Unary operators
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE &op) override { detail::PrintOperator(out_, OP_STR, op.expression_); }
UNARY_OPERATOR_VISIT(NotOperator, "Not");
UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
#undef UNARY_OPERATOR_VISIT
// Binary operators
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE &op) override { detail::PrintOperator(out_, OP_STR, op.expression1_, op.expression2_); }
BINARY_OPERATOR_VISIT(OrOperator, "Or");
BINARY_OPERATOR_VISIT(XorOperator, "Xor");
BINARY_OPERATOR_VISIT(AndOperator, "And");
BINARY_OPERATOR_VISIT(AdditionOperator, "+");
BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
BINARY_OPERATOR_VISIT(DivisionOperator, "/");
BINARY_OPERATOR_VISIT(ModOperator, "%");
BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
BINARY_OPERATOR_VISIT(EqualOperator, "==");
BINARY_OPERATOR_VISIT(LessOperator, "<");
BINARY_OPERATOR_VISIT(GreaterOperator, ">");
BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
BINARY_OPERATOR_VISIT(InListOperator, "In");
BINARY_OPERATOR_VISIT(SubscriptOperator, "Subscript");
#undef BINARY_OPERATOR_VISIT
// Other
void Visit(ListSlicingOperator &op) override {
detail::PrintOperator(out_, "ListSlicing", op.list_, op.lower_bound_, op.upper_bound_);
}
void Visit(IfOperator &op) override {
detail::PrintOperator(out_, "If", op.condition_, op.then_expression_, op.else_expression_);
}
void Visit(ListLiteral &op) override { detail::PrintOperator(out_, "ListLiteral", op.elements_); }
void Visit(MapLiteral &op) override {
std::map<std::string, Expression *> map;
for (const auto &kv : op.elements_) {
map[kv.first.name] = kv.second;
}
detail::PrintObject(out_, map);
}
void Visit(LabelsTest &op) override { detail::PrintOperator(out_, "LabelsTest", op.expression_); }
void Visit(Aggregation &op) override { detail::PrintOperator(out_, "Aggregation", op.op_); }
void Visit(Function &op) override { detail::PrintOperator(out_, "Function", op.function_name_, op.arguments_); }
void Visit(Reduce &op) override {
detail::PrintOperator(out_, "Reduce", op.accumulator_, op.initializer_, op.identifier_, op.list_, op.expression_);
}
void Visit(Coalesce &op) override { detail::PrintOperator(out_, "Coalesce", op.expressions_); }
void Visit(Extract &op) override { detail::PrintOperator(out_, "Extract", op.identifier_, op.list_, op.expression_); }
void Visit(All &op) override {
detail::PrintOperator(out_, "All", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(Single &op) override {
detail::PrintOperator(out_, "Single", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(Any &op) override {
detail::PrintOperator(out_, "Any", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(None &op) override {
detail::PrintOperator(out_, "None", op.identifier_, op.list_expression_, op.where_->expression_);
}
void Visit(Identifier &op) override { detail::PrintOperator(out_, "Identifier", op.name_); }
void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
void Visit(PropertyLookup &op) override {
detail::PrintOperator(out_, "PropertyLookup", op.expression_, op.property_.name);
}
void Visit(ParameterLookup &op) override { detail::PrintOperator(out_, "ParameterLookup", op.token_position_); }
void Visit(NamedExpression &op) override { detail::PrintOperator(out_, "NamedExpression", op.name_, op.expression_); }
void Visit(RegexMatch &op) override { detail::PrintOperator(out_, "=~", op.string_expr_, op.regex_); }
private:
std::ostream *out_;
};
namespace detail {
inline void PrintObject(std::ostream *out, Expression *expr) {
if (expr) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
} else {
*out << "<null>";
}
}
} // namespace detail
inline void PrintExpression(Expression *expr, std::ostream *out) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
}
inline void PrintExpression(NamedExpression *expr, std::ostream *out) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
}
} // namespace memgraph::expr

52
src/expr/exceptions.hpp Normal file
View File

@@ -0,0 +1,52 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "utils/exceptions.hpp"
namespace memgraph::expr {
class SyntaxException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SyntaxException() : SyntaxException("") {}
};
class SemanticException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SemanticException() : BasicException("") {}
};
class ExpressionRuntimeException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
};
class RedeclareVariableError : public SemanticException {
public:
explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {}
};
class UnboundVariableError : public SemanticException {
public:
explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {}
};
class TypeMismatchError : public SemanticException {
public:
TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected)
: SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) {
}
};
} // namespace memgraph::expr

826
src/expr/interpret/eval.hpp Normal file
View File

@@ -0,0 +1,826 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file
#pragma once
#include <algorithm>
#include <limits>
#include <map>
#include <optional>
#include <regex>
#include <type_traits>
#include <vector>
#include "expr/ast.hpp"
#include "expr/exceptions.hpp"
#include "expr/interpret/frame.hpp"
#include "expr/semantic/symbol_table.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::expr {
struct StorageTag {};
struct QueryEngineTag {};
template <typename TypedValue, typename EvaluationContext, typename DbAccessor, typename StorageView, typename LabelId,
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
public:
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
DbAccessor *dba, StorageView view)
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
using ExpressionVisitor<TypedValue>::Visit;
utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; }
TypedValue Visit(NamedExpression &named_expression) override {
const auto &symbol = symbol_table_->at(named_expression);
auto value = named_expression.expression_->Accept(*this);
frame_->at(symbol) = value;
return value;
}
TypedValue Visit(Identifier &ident) override {
return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define BINARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
TypedValue Visit(OP_NODE &op) override { \
auto val1 = op.expression1_->Accept(*this); \
auto val2 = op.expression2_->Accept(*this); \
try { \
return val1 CPP_OP val2; \
} catch (const TypedValueException &) { \
throw ExpressionRuntimeException("Invalid types: {} and {} for '{}'.", val1.type(), val2.type(), #CYPHER_OP); \
} \
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define UNARY_OPERATOR_VISITOR(OP_NODE, CPP_OP, CYPHER_OP) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
TypedValue Visit(OP_NODE &op) override { \
auto val = op.expression_->Accept(*this); \
try { \
return CPP_OP val; \
} catch (const TypedValueException &) { \
throw ExpressionRuntimeException("Invalid type {} for '{}'.", val.type(), #CYPHER_OP); \
} \
}
BINARY_OPERATOR_VISITOR(OrOperator, ||, OR);
BINARY_OPERATOR_VISITOR(XorOperator, ^, XOR);
BINARY_OPERATOR_VISITOR(AdditionOperator, +, +);
BINARY_OPERATOR_VISITOR(SubtractionOperator, -, -);
BINARY_OPERATOR_VISITOR(MultiplicationOperator, *, *);
BINARY_OPERATOR_VISITOR(DivisionOperator, /, /);
BINARY_OPERATOR_VISITOR(ModOperator, %, %);
BINARY_OPERATOR_VISITOR(NotEqualOperator, !=, <>);
BINARY_OPERATOR_VISITOR(EqualOperator, ==, =);
BINARY_OPERATOR_VISITOR(LessOperator, <, <);
BINARY_OPERATOR_VISITOR(GreaterOperator, >, >);
BINARY_OPERATOR_VISITOR(LessEqualOperator, <=, <=);
BINARY_OPERATOR_VISITOR(GreaterEqualOperator, >=, >=);
UNARY_OPERATOR_VISITOR(NotOperator, !, NOT);
UNARY_OPERATOR_VISITOR(UnaryPlusOperator, +, +);
UNARY_OPERATOR_VISITOR(UnaryMinusOperator, -, -);
#undef BINARY_OPERATOR_VISITOR
#undef UNARY_OPERATOR_VISITOR
TypedValue Visit(AndOperator &op) override {
auto value1 = op.expression1_->Accept(*this);
if (value1.IsBool() && !value1.ValueBool()) {
// If first expression is false, don't evaluate the second one.
return value1;
}
auto value2 = op.expression2_->Accept(*this);
try {
return value1 && value2;
} catch (const TypedValueException &) {
throw ExpressionRuntimeException("Invalid types: {} and {} for AND.", value1.type(), value2.type());
}
}
TypedValue Visit(IfOperator &if_operator) override {
auto condition = if_operator.condition_->Accept(*this);
if (condition.IsNull()) {
return if_operator.else_expression_->Accept(*this);
}
if (condition.type() != TypedValue::Type::Bool) {
// At the moment IfOperator is used only in CASE construct.
throw ExpressionRuntimeException("CASE expected boolean expression, got {}.", condition.type());
}
if (condition.ValueBool()) {
return if_operator.then_expression_->Accept(*this);
}
return if_operator.else_expression_->Accept(*this);
}
TypedValue Visit(InListOperator &in_list) override {
auto literal = in_list.expression1_->Accept(*this);
auto _list = in_list.expression2_->Accept(*this);
if (_list.IsNull()) {
return TypedValue(ctx_->memory);
}
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("IN expected a list, got {}.", _list.type());
}
const auto &list = _list.ValueList();
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
auto has_null = false;
for (const auto &element : list) {
auto result = literal == element;
if (result.IsNull()) {
has_null = true;
} else if (result.ValueBool()) {
return TypedValue(true, ctx_->memory);
}
}
if (has_null) {
return TypedValue(ctx_->memory);
}
return TypedValue(false, ctx_->memory);
}
TypedValue Visit(SubscriptOperator &list_indexing) override {
auto lhs = list_indexing.expression1_->Accept(*this);
auto index = list_indexing.expression2_->Accept(*this);
if (!lhs.IsList() && !lhs.IsMap() && !lhs.IsVertex() && !lhs.IsEdge() && !lhs.IsNull())
throw ExpressionRuntimeException(
"Expected a list, a map, a node or an edge to index with '[]', got "
"{}.",
lhs.type());
if (lhs.IsNull() || index.IsNull()) return TypedValue(ctx_->memory);
if (lhs.IsList()) {
if (!index.IsInt())
throw ExpressionRuntimeException("Expected an integer as a list index, got {}.", index.type());
auto index_int = index.ValueInt();
// NOTE: Take non-const reference to list, so that we can move out the
// indexed element as the result.
auto &list = lhs.ValueList();
if (index_int < 0) {
index_int += static_cast<int64_t>(list.size());
}
if (index_int >= static_cast<int64_t>(list.size()) || index_int < 0) return TypedValue(ctx_->memory);
// NOTE: Explicit move is needed, so that we return the move constructed
// value and preserve the correct MemoryResource.
return std::move(list[index_int]);
}
if (lhs.IsMap()) {
if (!index.IsString())
throw ExpressionRuntimeException("Expected a string as a map index, got {}.", index.type());
// NOTE: Take non-const reference to map, so that we can move out the
// looked-up element as the result.
auto &map = lhs.ValueMap();
auto found = map.find(index.ValueString());
if (found == map.end()) return TypedValue(ctx_->memory);
// NOTE: Explicit move is needed, so that we return the move constructed
// value and preserve the correct MemoryResource.
return std::move(found->second);
}
if (lhs.IsVertex()) {
if (!index.IsString())
throw ExpressionRuntimeException("Expected a string as a property name, got {}.", index.type());
return TypedValue(GetProperty(lhs.ValueVertex(), index.ValueString()), ctx_->memory);
}
if (lhs.IsEdge()) {
if (!index.IsString())
throw ExpressionRuntimeException("Expected a string as a property name, got {}.", index.type());
return TypedValue(GetProperty(lhs.ValueEdge(), index.ValueString()), ctx_->memory);
}
// lhs is Null
return TypedValue(ctx_->memory);
}
TypedValue Visit(ListSlicingOperator &op) override {
// If some type is null we can't return null, because throwing exception
// on illegal type has higher priority.
auto is_null = false;
auto get_bound = [&](Expression *bound_expr, int64_t default_value) {
if (bound_expr) {
auto bound = bound_expr->Accept(*this);
if (bound.type() == TypedValue::Type::Null) {
is_null = true;
} else if (bound.type() != TypedValue::Type::Int) {
throw ExpressionRuntimeException("Expected an integer for a bound in list slicing, got {}.", bound.type());
}
return bound;
}
return TypedValue(default_value, ctx_->memory);
};
auto _upper_bound = get_bound(op.upper_bound_, std::numeric_limits<int64_t>::max());
auto _lower_bound = get_bound(op.lower_bound_, 0);
auto _list = op.list_->Accept(*this);
if (_list.type() == TypedValue::Type::Null) {
is_null = true;
} else if (_list.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("Expected a list to slice, got {}.", _list.type());
}
if (is_null) {
return TypedValue(ctx_->memory);
}
const auto &list = _list.ValueList();
auto normalise_bound = [&](int64_t bound) {
if (bound < 0) {
bound = static_cast<int64_t>(list.size()) + bound;
}
return std::max(static_cast<int64_t>(0), std::min(bound, static_cast<int64_t>(list.size())));
};
auto lower_bound = normalise_bound(_lower_bound.ValueInt());
auto upper_bound = normalise_bound(_upper_bound.ValueInt());
if (upper_bound <= lower_bound) {
return TypedValue(typename TypedValue::TVector(ctx_->memory), ctx_->memory);
}
return TypedValue(
typename TypedValue::TVector(list.begin() + lower_bound, list.begin() + upper_bound, ctx_->memory));
}
TypedValue Visit(IsNullOperator &is_null) override {
auto value = is_null.expression_->Accept(*this);
return TypedValue(value.IsNull(), ctx_->memory);
}
TypedValue Visit(PropertyLookup &property_lookup) override {
auto expression_result = property_lookup.expression_->Accept(*this);
auto maybe_date = [this](const auto &date, const auto &prop_name) -> std::optional<TypedValue> {
if (prop_name == "year") {
return TypedValue(date.year, ctx_->memory);
}
if (prop_name == "month") {
return TypedValue(date.month, ctx_->memory);
}
if (prop_name == "day") {
return TypedValue(date.day, ctx_->memory);
}
return std::nullopt;
};
auto maybe_local_time = [this](const auto &lt, const auto &prop_name) -> std::optional<TypedValue> {
if (prop_name == "hour") {
return TypedValue(lt.hour, ctx_->memory);
}
if (prop_name == "minute") {
return TypedValue(lt.minute, ctx_->memory);
}
if (prop_name == "second") {
return TypedValue(lt.second, ctx_->memory);
}
if (prop_name == "millisecond") {
return TypedValue(lt.millisecond, ctx_->memory);
}
if (prop_name == "microsecond") {
return TypedValue(lt.microsecond, ctx_->memory);
}
return std::nullopt;
};
auto maybe_duration = [this](const auto &dur, const auto &prop_name) -> std::optional<TypedValue> {
if (prop_name == "day") {
return TypedValue(dur.Days(), ctx_->memory);
}
if (prop_name == "hour") {
return TypedValue(dur.SubDaysAsHours(), ctx_->memory);
}
if (prop_name == "minute") {
return TypedValue(dur.SubDaysAsMinutes(), ctx_->memory);
}
if (prop_name == "second") {
return TypedValue(dur.SubDaysAsSeconds(), ctx_->memory);
}
if (prop_name == "millisecond") {
return TypedValue(dur.SubDaysAsMilliseconds(), ctx_->memory);
}
if (prop_name == "microsecond") {
return TypedValue(dur.SubDaysAsMicroseconds(), ctx_->memory);
}
if (prop_name == "nanosecond") {
return TypedValue(dur.SubDaysAsNanoseconds(), ctx_->memory);
}
return std::nullopt;
};
switch (expression_result.type()) {
case TypedValue::Type::Null:
return TypedValue(ctx_->memory);
case TypedValue::Type::Vertex:
return GetProperty(expression_result.ValueVertex(), property_lookup.property_);
case TypedValue::Type::Edge:
return GetProperty(expression_result.ValueEdge(), property_lookup.property_);
case TypedValue::Type::Map: {
// NOTE: Take non-const reference to map, so that we can move out the
// looked-up element as the result.
auto &map = expression_result.ValueMap();
auto found = map.find(property_lookup.property_.name.c_str());
if (found == map.end()) return TypedValue(ctx_->memory);
// NOTE: Explicit move is needed, so that we return the move constructed
// value and preserve the correct MemoryResource.
return std::move(found->second);
}
case TypedValue::Type::Duration: {
const auto &prop_name = property_lookup.property_.name;
const auto &dur = expression_result.ValueDuration();
if (auto dur_field = maybe_duration(dur, prop_name); dur_field) {
return std::move(*dur_field);
}
throw ExpressionRuntimeException("Invalid property name {} for Duration", prop_name);
}
case TypedValue::Type::Date: {
const auto &prop_name = property_lookup.property_.name;
const auto &date = expression_result.ValueDate();
if (auto date_field = maybe_date(date, prop_name); date_field) {
return std::move(*date_field);
}
throw ExpressionRuntimeException("Invalid property name {} for Date", prop_name);
}
case TypedValue::Type::LocalTime: {
const auto &prop_name = property_lookup.property_.name;
const auto &lt = expression_result.ValueLocalTime();
if (auto lt_field = maybe_local_time(lt, prop_name); lt_field) {
return std::move(*lt_field);
}
throw ExpressionRuntimeException("Invalid property name {} for LocalTime", prop_name);
}
case TypedValue::Type::LocalDateTime: {
const auto &prop_name = property_lookup.property_.name;
const auto &ldt = expression_result.ValueLocalDateTime();
if (auto date_field = maybe_date(ldt.date, prop_name); date_field) {
return std::move(*date_field);
}
if (auto lt_field = maybe_local_time(ldt.local_time, prop_name); lt_field) {
return std::move(*lt_field);
}
throw ExpressionRuntimeException("Invalid property name {} for LocalDateTime", prop_name);
}
default:
throw ExpressionRuntimeException("Only nodes, edges, maps and temporal types have properties to be looked-up.");
}
}
template <typename VertexAccessor, typename TTag = Tag,
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label, StorageTag /*tag*/) {
auto has_label = vertex.HasLabel(view_, GetLabel(label));
if (has_label.HasError() && has_label.GetError() == Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE
// work. The old storage had the following logic when returning an
// `OLD` view: `return old ? old : new`. That means that if the
// `OLD` view didn't exist, it returned the NEW view. With this hack
// we simulate that behavior.
// TODO (mferencevic, teon.banek): Remove once MERGE is
// reimplemented.
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
}
if (has_label.HasError()) {
switch (has_label.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
}
}
return *has_label;
}
template <typename VertexAccessor, typename TTag = Tag,
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
auto has_label = vertex.HasLabel(label);
return !has_label;
}
TypedValue Visit(LabelsTest &labels_test) override {
auto expression_result = labels_test.expression_->Accept(*this);
switch (expression_result.type()) {
case TypedValue::Type::Null:
return TypedValue(ctx_->memory);
case TypedValue::Type::Vertex: {
const auto &vertex = expression_result.ValueVertex();
if (std::ranges::all_of(labels_test.labels_, [&vertex, this](const auto label_test) {
return this->HasLabelImpl(vertex, label_test, Tag{});
})) {
return TypedValue(true, ctx_->memory);
}
return TypedValue(false, ctx_->memory);
}
default:
throw ExpressionRuntimeException("Only nodes have labels.");
}
}
TypedValue Visit(PrimitiveLiteral &literal) override {
// TODO: no need to evaluate constants, we can write it to frame in one
// of the previous phases.
return TypedValue(literal.value_, ctx_->memory);
}
TypedValue Visit(ListLiteral &literal) override {
typename TypedValue::TVector result(ctx_->memory);
result.reserve(literal.elements_.size());
for (const auto &expression : literal.elements_) result.emplace_back(expression->Accept(*this));
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(MapLiteral &literal) override {
typename TypedValue::TMap result(ctx_->memory);
for (const auto &pair : literal.elements_) result.emplace(pair.first.name, pair.second->Accept(*this));
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(Aggregation &aggregation) override {
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
}
TypedValue Visit(Coalesce &coalesce) override {
auto &exprs = coalesce.expressions_;
if (exprs.size() == 0) {
throw ExpressionRuntimeException("'coalesce' requires at least one argument.");
}
for (int64_t i = 0; i < exprs.size(); ++i) {
TypedValue val(exprs[i]->Accept(*this), ctx_->memory);
if (!val.IsNull()) {
return val;
}
}
return TypedValue(ctx_->memory);
}
TypedValue Visit(Function &function) override {
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
// Stack allocate evaluated arguments when there's a small number of them.
if (function.arguments_.size() <= 8) {
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
TypedValue(ctx_->memory), TypedValue(ctx_->memory)};
for (size_t i = 0; i < function.arguments_.size(); ++i) {
arguments[i] = function.arguments_[i]->Accept(*this);
}
auto res = function.function_(arguments, function.arguments_.size(), function_ctx);
MG_ASSERT(res.GetMemoryResource() == ctx_->memory);
return res;
} else {
typename TypedValue::TVector arguments(ctx_->memory);
arguments.reserve(function.arguments_.size());
for (const auto &argument : function.arguments_) {
arguments.emplace_back(argument->Accept(*this));
}
auto res = function.function_(arguments.data(), arguments.size(), function_ctx);
MG_ASSERT(res.GetMemoryResource() == ctx_->memory);
return res;
}
}
TypedValue Visit(Reduce &reduce) override {
auto list_value = reduce.list_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("REDUCE expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &element_symbol = symbol_table_->at(*reduce.identifier_);
const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
auto accumulator = reduce.initializer_->Accept(*this);
for (const auto &element : list) {
frame_->at(accumulator_symbol) = accumulator;
frame_->at(element_symbol) = element;
accumulator = reduce.expression_->Accept(*this);
}
return accumulator;
}
TypedValue Visit(Extract &extract) override {
auto list_value = extract.list_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("EXTRACT expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &element_symbol = symbol_table_->at(*extract.identifier_);
typename TypedValue::TVector result(ctx_->memory);
result.reserve(list.size());
for (const auto &element : list) {
if (element.IsNull()) {
result.emplace_back();
} else {
frame_->at(element_symbol) = element;
result.emplace_back(extract.expression_->Accept(*this));
}
}
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(All &all) override {
auto list_value = all.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("ALL expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*all.identifier_);
bool has_null_elements = false;
bool has_value = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = all.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
}
if (!result.IsNull()) {
has_value = true;
if (!result.ValueBool()) {
return TypedValue(false, ctx_->memory);
}
} else {
has_null_elements = true;
}
}
if (!has_value) {
return TypedValue(ctx_->memory);
}
if (has_null_elements) {
return TypedValue(false, ctx_->memory);
} else {
return TypedValue(true, ctx_->memory);
}
}
TypedValue Visit(Single &single) override {
auto list_value = single.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("SINGLE expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*single.identifier_);
bool has_value = false;
bool predicate_satisfied = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = single.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
}
if (result.type() == TypedValue::Type::Bool) {
has_value = true;
}
if (result.IsNull() || !result.ValueBool()) {
continue;
}
// Return false if more than one element satisfies the predicate.
if (predicate_satisfied) {
return TypedValue(false, ctx_->memory);
} else {
predicate_satisfied = true;
}
}
if (!has_value) {
return TypedValue(ctx_->memory);
} else {
return TypedValue(predicate_satisfied, ctx_->memory);
}
}
TypedValue Visit(Any &any) override {
auto list_value = any.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("ANY expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*any.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = any.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
}
if (!result.IsNull()) {
has_value = true;
if (result.ValueBool()) {
return TypedValue(true, ctx_->memory);
}
}
}
// Return Null if all elements are Null
if (!has_value) {
return TypedValue(ctx_->memory);
} else {
return TypedValue(false, ctx_->memory);
}
}
TypedValue Visit(None &none) override {
auto list_value = none.list_expression_->Accept(*this);
if (list_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (list_value.type() != TypedValue::Type::List) {
throw ExpressionRuntimeException("NONE expected a list, got {}.", list_value.type());
}
const auto &list = list_value.ValueList();
const auto &symbol = symbol_table_->at(*none.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->at(symbol) = element;
auto result = none.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
}
if (!result.IsNull()) {
has_value = true;
if (result.ValueBool()) {
return TypedValue(false, ctx_->memory);
}
}
}
// Return Null if all elements are Null
if (!has_value) {
return TypedValue(ctx_->memory);
} else {
return TypedValue(true, ctx_->memory);
}
}
TypedValue Visit(ParameterLookup &param_lookup) override {
return TypedValue(conv_(ctx_->parameters.AtTokenPosition(param_lookup.token_position_)), ctx_->memory);
}
TypedValue Visit(RegexMatch &regex_match) override {
auto target_string_value = regex_match.string_expr_->Accept(*this);
auto regex_value = regex_match.regex_->Accept(*this);
if (target_string_value.IsNull() || regex_value.IsNull()) {
return TypedValue(ctx_->memory);
}
if (regex_value.type() != TypedValue::Type::String) {
throw ExpressionRuntimeException("Regular expression must evaluate to a string, got {}.", regex_value.type());
}
if (target_string_value.type() != TypedValue::Type::String) {
// Instead of error, we return Null which makes it compatible in case we
// use indexed lookup which filters out any non-string properties.
// Assuming a property lookup is the target_string_value.
return TypedValue(ctx_->memory);
}
const auto &target_string = target_string_value.ValueString();
try {
std::regex regex(regex_value.ValueString());
return TypedValue(std::regex_match(target_string, regex), ctx_->memory);
} catch (const std::regex_error &e) {
throw ExpressionRuntimeException("Regex error in '{}': {}", regex_value.ValueString(), e.what());
}
}
private:
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
TReturnType GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
auto maybe_prop = record_accessor.GetProperty(prop.name);
// Handler non existent property
return conv_(maybe_prop);
}
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, TypedValue>>
TReturnType GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
auto maybe_prop = record_accessor.GetProperty(std::string(name));
// Handler non existent property
return conv_(maybe_prop);
}
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
TypedValue GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
auto maybe_prop = record_accessor.GetProperty(view_, ctx_->properties[prop.ix]);
if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE work.
// The old storage had the following logic when returning an `OLD` view:
// `return old ? old : new`. That means that if the `OLD` view didn't
// exist, it returned the NEW view. With this hack we simulate that
// behavior.
// TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented.
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
}
if (maybe_prop.HasError()) {
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
template <class TRecordAccessor, class TTag = Tag,
class TReturnType = std::enable_if_t<std::is_same_v<TTag, StorageTag>, TypedValue>>
TypedValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
if (maybe_prop.HasError() && maybe_prop.GetError() == Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE work.
// The old storage had the following logic when returning an `OLD` view:
// `return old ? old : new`. That means that if the `OLD` view didn't
// exist, it returned the NEW view. With this hack we simulate that
// behavior.
// TODO (mferencevic, teon.banek): Remove once MERGE is reimplemented.
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
}
if (maybe_prop.HasError()) {
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
Frame<TypedValue> *frame_;
const SymbolTable *symbol_table_;
const EvaluationContext *ctx_;
DbAccessor *dba_;
// which switching approach should be used when evaluating
StorageView view_;
ConvFunctor conv_;
};
/// A helper function for evaluating an expression that's an int.
///
/// @param what - Name of what's getting evaluated. Used for user feedback (via
/// exception) when the evaluated value is not an int.
/// @throw ExpressionRuntimeException if expression doesn't evaluate to an int.
template <typename ExpressionEvaluator>
int64_t EvaluateInt(ExpressionEvaluator *evaluator, Expression *expr, const std::string &what) {
TypedValue value = expr->Accept(*evaluator);
try {
return value.ValueInt();
} catch (TypedValueException &e) {
throw ExpressionRuntimeException(what + " must be an int");
}
}
template <typename ExpressionEvaluator>
std::optional<size_t> EvaluateMemoryLimit(ExpressionEvaluator *eval, Expression *memory_limit, size_t memory_scale) {
if (!memory_limit) return std::nullopt;
auto limit_value = memory_limit->Accept(*eval);
if (!limit_value.IsInt() || limit_value.ValueInt() <= 0)
throw ExpressionRuntimeException("Memory limit must be a non-negative integer.");
size_t limit = limit_value.ValueInt();
if (std::numeric_limits<size_t>::max() / memory_scale < limit)
throw ExpressionRuntimeException("Memory limit overflow.");
return limit * memory_scale;
}
} // namespace memgraph::expr

View File

@@ -0,0 +1,45 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include "expr/semantic/symbol_table.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/pmr/vector.hpp"
namespace memgraph::expr {
template <typename TypedValue>
class Frame {
public:
/// Create a Frame of given size backed by a utils::NewDeleteResource()
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
auto &elems() { return elems_; }
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
private:
utils::pmr::vector<TypedValue> elems_;
};
} // namespace memgraph::expr

184
src/expr/parsing.cpp Normal file
View File

@@ -0,0 +1,184 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "expr/parsing.hpp"
#include <cctype>
#include <codecvt>
#include <locale>
#include <stdexcept>
#include "expr/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace memgraph::expr {
int64_t ParseIntegerLiteral(const std::string &s) {
try {
// Not really correct since long long can have a bigger range than int64_t.
return static_cast<int64_t>(std::stoll(s, 0, 0));
} catch (const std::out_of_range &) {
throw SemanticException("Integer literal exceeds 64 bits.");
}
}
std::string ParseStringLiteral(const std::string &s) {
// These functions is declared as lambda since its semantics is highly
// specific for this conxtext and shouldn't be used elsewhere.
auto EncodeEscapedUnicodeCodepointUtf32 = [](const std::string &s, int &i) {
const int kLongUnicodeLength = 8;
int j = i + 1;
while (j < static_cast<int>(s.size()) - 1 && j < i + kLongUnicodeLength + 1 && isxdigit(s[j])) {
++j;
}
if (j - i == kLongUnicodeLength + 1) {
char32_t t = stoi(s.substr(i + 1, kLongUnicodeLength), 0, 16);
i += kLongUnicodeLength;
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> converter;
return converter.to_bytes(t);
}
throw SyntaxException(
"Expected 8 hex digits as unicode codepoint started with \\U. "
"Use \\u for 4 hex digits format.");
};
auto EncodeEscapedUnicodeCodepointUtf16 = [](const std::string &s, int &i) {
const int kShortUnicodeLength = 4;
int j = i + 1;
while (j < static_cast<int>(s.size()) - 1 && j < i + kShortUnicodeLength + 1 && isxdigit(s[j])) {
++j;
}
if (j - i >= kShortUnicodeLength + 1) {
char16_t t = stoi(s.substr(i + 1, kShortUnicodeLength), 0, 16);
if (t >= 0xD800 && t <= 0xDBFF) {
// t is high surrogate pair. Expect one more utf16 codepoint.
j = i + kShortUnicodeLength + 1;
if (j >= static_cast<int>(s.size()) - 1 || s[j] != '\\') {
throw SemanticException("Invalid UTF codepoint.");
}
++j;
if (j >= static_cast<int>(s.size()) - 1 || (s[j] != 'u' && s[j] != 'U')) {
throw SemanticException("Invalid UTF codepoint.");
}
++j;
int k = j;
while (k < static_cast<int>(s.size()) - 1 && k < j + kShortUnicodeLength && isxdigit(s[k])) {
++k;
}
if (k != j + kShortUnicodeLength) {
throw SemanticException("Invalid UTF codepoint.");
}
char16_t surrogates[3] = {t, static_cast<char16_t>(stoi(s.substr(j, kShortUnicodeLength), 0, 16)), 0};
i += kShortUnicodeLength + 2 + kShortUnicodeLength;
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
return converter.to_bytes(surrogates);
} else {
i += kShortUnicodeLength;
std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t> converter;
return converter.to_bytes(t);
}
}
throw SyntaxException(
"Expected 4 hex digits as unicode codepoint started with \\u. "
"Use \\U for 8 hex digits format.");
};
std::string unescaped;
bool escape = false;
// First and last char is quote, we don't need to look at them.
for (int i = 1; i < static_cast<int>(s.size()) - 1; ++i) {
if (escape) {
switch (s[i]) {
case '\\':
unescaped += '\\';
break;
case '\'':
unescaped += '\'';
break;
case '"':
unescaped += '"';
break;
case 'B':
case 'b':
unescaped += '\b';
break;
case 'F':
case 'f':
unescaped += '\f';
break;
case 'N':
case 'n':
unescaped += '\n';
break;
case 'R':
case 'r':
unescaped += '\r';
break;
case 'T':
case 't':
unescaped += '\t';
break;
case 'U':
try {
unescaped += EncodeEscapedUnicodeCodepointUtf32(s, i);
} catch (const std::range_error &) {
throw SemanticException("Invalid UTF codepoint.");
}
break;
case 'u':
try {
unescaped += EncodeEscapedUnicodeCodepointUtf16(s, i);
} catch (const std::range_error &) {
throw SemanticException("Invalid UTF codepoint.");
}
break;
default:
// This should never happen, except grammar changes and we don't
// notice change in this production.
DLOG_FATAL("can't happen");
throw std::exception();
}
escape = false;
} else if (s[i] == '\\') {
escape = true;
} else {
unescaped += s[i];
}
}
return unescaped;
}
double ParseDoubleLiteral(const std::string &s) {
try {
return utils::ParseDouble(s);
} catch (const utils::BasicException &) {
throw SemanticException("Couldn't parse string to double.");
}
}
std::string ParseParameter(const std::string &s) {
DMG_ASSERT(s[0] == '$', "Invalid string passed as parameter name");
if (s[1] != '`') return s.substr(1);
// If parameter name is escaped symbolic name then symbolic name should be
// unescaped and leading and trailing backquote should be removed.
DMG_ASSERT(s.size() > 3U && s.back() == '`', "Invalid string passed as parameter name");
std::string out;
for (int i = 2; i < static_cast<int>(s.size()) - 1; ++i) {
if (s[i] == '`') {
++i;
}
out.push_back(s[i]);
}
return out;
}
} // namespace memgraph::expr

27
src/expr/parsing.hpp Normal file
View File

@@ -0,0 +1,27 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file
#pragma once
#include <cstdint>
#include <string>
namespace memgraph::expr {
// These are the functions for parsing literals and parameter names from
// opencypher query.
int64_t ParseIntegerLiteral(const std::string &s);
std::string ParseStringLiteral(const std::string &s);
double ParseDoubleLiteral(const std::string &s);
std::string ParseParameter(const std::string &s);
} // namespace memgraph::expr

View File

@@ -0,0 +1,87 @@
;; Copyright 2022 Memgraph Ltd.
;;
;; Use of this software is governed by the Business Source License
;; included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
;; License, and you may not use this file except in compliance with the Business Source License.
;;
;; As of the Change Date specified in that file, in accordance with
;; the Business Source License, use of this software will be governed
;; by the Apache License, Version 2.0, included in the file
;; licenses/APL.txt.
#>cpp
#pragma once
#include <string>
#include "utils/typeinfo.hpp"
cpp<#
(lcp:namespace memgraph)
(lcp:namespace expr)
(lcp:define-class symbol ()
((name "std::string" :scope :public)
(position :int64_t :scope :public)
(user-declared :bool :initval "true" :scope :public)
(type "Type" :initval "Type::ANY" :scope :public)
(token-position :int64_t :initval "-1" :scope :public))
(:public
;; This is similar to TypedValue::Type, but this has `Any` type.
;; TODO: Make a better Type structure which can store a generic List.
(lcp:define-enum type (any vertex edge path number edge-list)
(:serialize))
#>cpp
// TODO: Generate enum to string conversion from LCP. Note, that this is
// displayed to the end user, so we may want to have a pretty name of each
// value.
static std::string TypeToString(Type type) {
const char *enum_string[] = {"Any", "Vertex", "Edge",
"Path", "Number", "EdgeList"};
return enum_string[static_cast<int>(type)];
}
Symbol() {}
Symbol(const std::string &name, int position, bool user_declared,
Type type = Type::ANY, int token_position = -1)
: name_(name),
position_(position),
user_declared_(user_declared),
type_(type),
token_position_(token_position) {}
bool operator==(const Symbol &other) const {
return position_ == other.position_ && name_ == other.name_ &&
type_ == other.type_;
}
bool operator!=(const Symbol &other) const { return !operator==(other); }
// TODO: Remove these since members are public
const auto &name() const { return name_; }
int position() const { return position_; }
Type type() const { return type_; }
bool user_declared() const { return user_declared_; }
int token_position() const { return token_position_; }
cpp<#)
(:serialize (:slk)))
(lcp:pop-namespace) ;; expr
(lcp:pop-namespace) ;; memgraph
#>cpp
namespace std {
template <>
struct hash<memgraph::expr::Symbol> {
size_t operator()(const memgraph::expr::Symbol &symbol) const {
size_t prime = 265443599u;
size_t hash = std::hash<int>{}(symbol.position());
hash ^= prime * std::hash<std::string>{}(symbol.name());
hash ^= prime * std::hash<int>{}(static_cast<int>(symbol.type()));
return hash;
}
};
} // namespace std
cpp<#

View File

@@ -0,0 +1,712 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// Copyright 2017 Memgraph
//
// Created by Teon Banek on 11-03-2017
#pragma once
#include <algorithm>
#include <optional>
#include <ranges>
#include <unordered_map>
#include <unordered_set>
#include <variant>
#include <vector>
#include "expr/ast.hpp"
#include "expr/ast/ast_visitor.hpp"
#include "expr/exceptions.hpp"
#include "expr/semantic/symbol_table.hpp"
namespace memgraph::expr {
namespace detail {
inline std::unordered_map<std::string, Identifier *> GeneratePredefinedIdentifierMap(
const std::vector<Identifier *> &predefined_identifiers) {
std::unordered_map<std::string, Identifier *> identifier_map;
for (const auto &identifier : predefined_identifiers) {
identifier_map.emplace(identifier->name_, identifier);
}
return identifier_map;
}
} // namespace detail
/// Visits the AST and generates symbols for variables.
///
/// During the process of symbol generation, simple semantic checks are
/// performed. Such as, redeclaring a variable or conflicting expectations of
/// variable types.
class SymbolGenerator : public HierarchicalTreeVisitor {
public:
explicit SymbolGenerator(SymbolTable *symbol_table, const std::vector<Identifier *> &predefined_identifiers)
: symbol_table_(symbol_table),
predefined_identifiers_{detail::GeneratePredefinedIdentifierMap(predefined_identifiers)},
scopes_(1, Scope()) {}
using HierarchicalTreeVisitor::PostVisit;
using HierarchicalTreeVisitor::PreVisit;
using HierarchicalTreeVisitor::Visit;
using typename HierarchicalTreeVisitor::ReturnType;
// Query
bool PreVisit(SingleQuery & /*unused*/) override {
prev_return_names_ = curr_return_names_;
curr_return_names_.clear();
return true;
}
// Union
bool PreVisit(CypherUnion & /*unused*/) override {
scopes_.back() = Scope();
return true;
}
bool PostVisit(CypherUnion &cypher_union) override {
if (prev_return_names_ != curr_return_names_) {
throw SemanticException("All subqueries in an UNION must have the same column names.");
}
// create new symbols for the result of the union
for (const auto &name : curr_return_names_) {
auto symbol = CreateSymbol(name, false);
cypher_union.union_symbols_.push_back(symbol);
}
return true;
}
// Clauses
bool PreVisit(Create & /*unused*/) override {
scopes_.back().in_create = true;
return true;
}
bool PostVisit(Create & /*unused*/) override {
scopes_.back().in_create = false;
return true;
}
bool PreVisit(CallProcedure &call_proc) override {
for (auto *expr : call_proc.arguments_) {
expr->Accept(*this);
}
return false;
}
bool PostVisit(CallProcedure &call_proc) override {
for (auto *ident : call_proc.result_identifiers_) {
if (HasSymbolLocalScope(ident->name_)) {
throw RedeclareVariableError(ident->name_);
}
ident->MapTo(CreateSymbol(ident->name_, true));
}
return true;
}
bool PreVisit(LoadCsv & /*unused*/) override { return false; }
bool PostVisit(LoadCsv &load_csv) override {
if (HasSymbolLocalScope(load_csv.row_var_->name_)) {
throw RedeclareVariableError(load_csv.row_var_->name_);
}
load_csv.row_var_->MapTo(CreateSymbol(load_csv.row_var_->name_, true));
return true;
}
bool PreVisit(Return &ret) override {
auto &scope = scopes_.back();
scope.in_return = true;
VisitReturnBody(ret.body_);
scope.in_return = false;
return false; // We handled the traversal ourselves.
}
bool PostVisit(Return & /*unused*/) override {
for (const auto &name_symbol : scopes_.back().symbols) curr_return_names_.insert(name_symbol.first);
return true;
}
bool PreVisit(With &with) override {
auto &scope = scopes_.back();
scope.in_with = true;
VisitReturnBody(with.body_, with.where_);
scope.in_with = false;
return false; // We handled the traversal ourselves.
}
bool PreVisit(Where & /*unused*/) override {
scopes_.back().in_where = true;
return true;
}
bool PostVisit(Where & /*unused*/) override {
scopes_.back().in_where = false;
return true;
}
bool PreVisit(Merge & /*unused*/) override {
scopes_.back().in_merge = true;
return true;
}
bool PostVisit(Merge & /*unused*/) override {
scopes_.back().in_merge = false;
return true;
}
bool PostVisit(Unwind &unwind) override {
const auto &name = unwind.named_expression_->name_;
if (HasSymbolLocalScope(name)) {
throw RedeclareVariableError(name);
}
unwind.named_expression_->MapTo(CreateSymbol(name, true));
return true;
}
bool PreVisit(Match & /*unused*/) override {
scopes_.back().in_match = true;
return true;
}
bool PostVisit(Match & /*unused*/) override {
auto &scope = scopes_.back();
scope.in_match = false;
// Check variables in property maps after visiting Match, so that they can
// reference symbols out of bind order.
for (auto &ident : scope.identifiers_in_match) {
if (!HasSymbolLocalScope(ident->name_) && !ConsumePredefinedIdentifier(ident->name_))
throw UnboundVariableError(ident->name_);
ident->MapTo(scope.symbols[ident->name_]);
}
scope.identifiers_in_match.clear();
return true;
}
bool PreVisit(Foreach &for_each) override {
const auto &name = for_each.named_expression_->name_;
scopes_.emplace_back(Scope());
scopes_.back().in_foreach = true;
for_each.named_expression_->MapTo(
CreateSymbol(name, true, Symbol::Type::ANY, for_each.named_expression_->token_position_));
return true;
}
bool PostVisit(Foreach & /*unused*/) override {
scopes_.pop_back();
return true;
}
// Expressions
ReturnType Visit(Identifier &ident) override {
auto &scope = scopes_.back();
if (scope.in_skip || scope.in_limit) {
throw SemanticException("Variables are not allowed in {}.", scope.in_skip ? "SKIP" : "LIMIT");
}
Symbol symbol;
if (scope.in_pattern && !(scope.in_node_atom || scope.visiting_edge)) {
// If we are in the pattern, and outside of a node or an edge, the
// identifier is the pattern name.
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, Symbol::Type::PATH);
} else if (scope.in_pattern && scope.in_pattern_atom_identifier) {
// Patterns used to create nodes and edges cannot redeclare already
// established bindings. Declaration only happens in single node
// patterns and in edge patterns. OpenCypher example,
// `MATCH (n) CREATE (n)` should throw an error that `n` is already
// declared. While `MATCH (n) CREATE (n) -[:R]-> (n)` is allowed,
// since `n` now references the bound node instead of declaring it.
if ((scope.in_create_node || scope.in_create_edge) && HasSymbolLocalScope(ident.name_)) {
throw RedeclareVariableError(ident.name_);
}
auto type = Symbol::Type::VERTEX;
if (scope.visiting_edge) {
// Edge referencing is not allowed (like in Neo4j):
// `MATCH (n) - [r] -> (n) - [r] -> (n) RETURN r` is not allowed.
if (HasSymbolLocalScope(ident.name_)) {
throw RedeclareVariableError(ident.name_);
}
type = scope.visiting_edge->IsVariable() ? Symbol::Type::EDGE_LIST : Symbol::Type::EDGE;
}
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, type);
} else if (scope.in_pattern && !scope.in_pattern_atom_identifier && scope.in_match) {
if (scope.in_edge_range && scope.visiting_edge->identifier_->name_ == ident.name_) {
// Prevent variable path bounds to reference the identifier which is bound
// by the variable path itself.
throw UnboundVariableError(ident.name_);
}
// Variables in property maps or bounds of variable length path during MATCH
// can reference symbols bound later in the same MATCH. We collect them
// here, so that they can be checked after visiting Match.
scope.identifiers_in_match.emplace_back(&ident);
} else {
// Everything else references a bound symbol.
if (!HasSymbol(ident.name_) && !ConsumePredefinedIdentifier(ident.name_)) throw UnboundVariableError(ident.name_);
symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, Symbol::Type::ANY);
}
ident.MapTo(symbol);
return true;
}
ReturnType Visit(PrimitiveLiteral & /*unused*/) override { return true; }
ReturnType Visit(ParameterLookup & /*unused*/) override { return true; }
bool PreVisit(Aggregation &aggr) override {
auto &scope = scopes_.back();
// Check if the aggregation can be used in this context. This check should
// probably move to a separate phase, which checks if the query is well
// formed.
if ((!scope.in_return && !scope.in_with) || scope.in_order_by || scope.in_skip || scope.in_limit ||
scope.in_where) {
throw SemanticException("Aggregation functions are only allowed in WITH and RETURN.");
}
if (scope.in_aggregation) {
throw SemanticException(
"Using aggregation functions inside aggregation functions is not "
"allowed.");
}
if (scope.num_if_operators) {
// Neo allows aggregations here and produces very interesting behaviors.
// To simplify implementation at this moment we decided to completely
// disallow aggregations inside of the CASE.
// However, in some cases aggregation makes perfect sense, for example:
// CASE count(n) WHEN 10 THEN "YES" ELSE "NO" END.
// TODO: Rethink of allowing aggregations in some parts of the CASE
// construct.
throw SemanticException("Using aggregation functions inside of CASE is not allowed.");
}
// Create a virtual symbol for aggregation result.
// Currently, we only have aggregation operators which return numbers.
auto aggr_name = Aggregation::OpToString(aggr.op_) + std::to_string(aggr.symbol_pos_);
aggr.MapTo(CreateSymbol(aggr_name, false, Symbol::Type::NUMBER));
scope.in_aggregation = true;
scope.has_aggregation = true;
return true;
}
bool PostVisit(Aggregation & /*unused*/) override {
scopes_.back().in_aggregation = false;
return true;
}
bool PreVisit(IfOperator & /*unused*/) override {
++scopes_.back().num_if_operators;
return true;
}
bool PostVisit(IfOperator & /*unused*/) override {
--scopes_.back().num_if_operators;
return true;
}
bool PreVisit(All &all) override {
all.list_expression_->Accept(*this);
VisitWithIdentifiers(all.where_->expression_, {all.identifier_});
return false;
}
bool PreVisit(Single &single) override {
single.list_expression_->Accept(*this);
VisitWithIdentifiers(single.where_->expression_, {single.identifier_});
return false;
}
bool PreVisit(Any &any) override {
any.list_expression_->Accept(*this);
VisitWithIdentifiers(any.where_->expression_, {any.identifier_});
return false;
}
bool PreVisit(None &none) override {
none.list_expression_->Accept(*this);
VisitWithIdentifiers(none.where_->expression_, {none.identifier_});
return false;
}
bool PreVisit(Reduce &reduce) override {
reduce.initializer_->Accept(*this);
reduce.list_->Accept(*this);
VisitWithIdentifiers(reduce.expression_, {reduce.accumulator_, reduce.identifier_});
return false;
}
bool PreVisit(Extract &extract) override {
extract.list_->Accept(*this);
VisitWithIdentifiers(extract.expression_, {extract.identifier_});
return false;
}
// Pattern and its subparts.
bool PreVisit(Pattern &pattern) override {
auto &scope = scopes_.back();
scope.in_pattern = true;
if ((scope.in_create || scope.in_merge) && pattern.atoms_.size() == 1U) {
MG_ASSERT(utils::IsSubtype(*pattern.atoms_[0], NodeAtom::kType), "Expected a single NodeAtom in Pattern");
scope.in_create_node = true;
}
return true;
}
bool PostVisit(Pattern & /*unused*/) override {
auto &scope = scopes_.back();
scope.in_pattern = false;
scope.in_create_node = false;
return true;
}
bool PreVisit(NodeAtom &node_atom) override {
auto &scope = scopes_.back();
auto check_node_semantic = [&node_atom, &scope, this](const bool props_or_labels) {
const auto &node_name = node_atom.identifier_->name_;
if ((scope.in_create || scope.in_merge) && props_or_labels && HasSymbolLocalScope(node_name)) {
throw SemanticException("Cannot create node '" + node_name +
"' with labels or properties, because it is already declared.");
}
scope.in_pattern_atom_identifier = true;
node_atom.identifier_->Accept(*this);
scope.in_pattern_atom_identifier = false;
};
scope.in_node_atom = true;
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&node_atom.properties_)) {
bool props_or_labels = !properties->empty() || !node_atom.labels_.empty();
check_node_semantic(props_or_labels);
for (auto kv : *properties) {
kv.second->Accept(*this);
}
return false;
}
auto &properties_parameter = std::get<ParameterLookup *>(node_atom.properties_);
bool props_or_labels = !properties_parameter || !node_atom.labels_.empty();
check_node_semantic(props_or_labels);
properties_parameter->Accept(*this);
return false;
}
bool PostVisit(NodeAtom & /*unused*/) override {
scopes_.back().in_node_atom = false;
return true;
}
bool PreVisit(EdgeAtom &edge_atom) override {
auto &scope = scopes_.back();
scope.visiting_edge = &edge_atom;
if (scope.in_create || scope.in_merge) {
scope.in_create_edge = true;
if (edge_atom.edge_types_.size() != 1U) {
throw SemanticException(
"A single relationship type must be specified "
"when creating an edge.");
}
if (scope.in_create && // Merge allows bidirectionality
edge_atom.direction_ == EdgeAtom::Direction::BOTH) {
throw SemanticException(
"Bidirectional relationship are not supported "
"when creating an edge");
}
if (edge_atom.IsVariable()) {
throw SemanticException(
"Variable length relationships are not supported when creating an "
"edge.");
}
}
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&edge_atom.properties_)) {
for (auto kv : *properties) {
kv.second->Accept(*this);
}
} else {
std::get<ParameterLookup *>(edge_atom.properties_)->Accept(*this);
}
if (edge_atom.IsVariable()) {
scope.in_edge_range = true;
if (edge_atom.lower_bound_) {
edge_atom.lower_bound_->Accept(*this);
}
if (edge_atom.upper_bound_) {
edge_atom.upper_bound_->Accept(*this);
}
scope.in_edge_range = false;
scope.in_pattern = false;
if (edge_atom.filter_lambda_.expression) {
VisitWithIdentifiers(edge_atom.filter_lambda_.expression,
{edge_atom.filter_lambda_.inner_edge, edge_atom.filter_lambda_.inner_node});
} else {
// Create inner symbols, but don't bind them in scope, since they are to
// be used in the missing filter expression.
auto *inner_edge = edge_atom.filter_lambda_.inner_edge;
inner_edge->MapTo(
symbol_table_->CreateSymbol(inner_edge->name_, inner_edge->user_declared_, Symbol::Type::EDGE));
auto *inner_node = edge_atom.filter_lambda_.inner_node;
inner_node->MapTo(
symbol_table_->CreateSymbol(inner_node->name_, inner_node->user_declared_, Symbol::Type::VERTEX));
}
if (edge_atom.weight_lambda_.expression) {
VisitWithIdentifiers(edge_atom.weight_lambda_.expression,
{edge_atom.weight_lambda_.inner_edge, edge_atom.weight_lambda_.inner_node});
}
scope.in_pattern = true;
}
scope.in_pattern_atom_identifier = true;
edge_atom.identifier_->Accept(*this);
scope.in_pattern_atom_identifier = false;
if (edge_atom.total_weight_) {
if (HasSymbolLocalScope(edge_atom.total_weight_->name_)) {
throw RedeclareVariableError(edge_atom.total_weight_->name_);
}
edge_atom.total_weight_->MapTo(GetOrCreateSymbolLocalScope(
edge_atom.total_weight_->name_, edge_atom.total_weight_->user_declared_, Symbol::Type::NUMBER));
}
return false;
}
bool PostVisit(EdgeAtom & /*unused*/) override {
auto &scope = scopes_.back();
scope.visiting_edge = nullptr;
scope.in_create_edge = false;
return true;
}
private:
// Scope stores the state of where we are when visiting the AST and a map of
// names to symbols.
struct Scope {
bool in_pattern{false};
bool in_merge{false};
bool in_create{false};
// in_create_node is true if we are creating or merging *only* a node.
// Therefore, it is *not* equivalent to (in_create || in_merge) &&
// in_node_atom.
bool in_create_node{false};
// True if creating an edge;
// shortcut for (in_create || in_merge) && visiting_edge.
bool in_create_edge{false};
bool in_node_atom{false};
EdgeAtom *visiting_edge{nullptr};
bool in_aggregation{false};
bool in_return{false};
bool in_with{false};
bool in_skip{false};
bool in_limit{false};
bool in_order_by{false};
bool in_where{false};
bool in_match{false};
bool in_foreach{false};
// True when visiting a pattern atom (node or edge) identifier, which can be
// reused or created in the pattern itself.
bool in_pattern_atom_identifier{false};
// True when visiting range bounds of a variable path.
bool in_edge_range{false};
// True if the return/with contains an aggregation in any named expression.
bool has_aggregation{false};
// Map from variable names to symbols.
std::map<std::string, Symbol> symbols;
// Identifiers found in property maps of patterns or as variable length path
// bounds in a single Match clause. They need to be checked after visiting
// Match. Identifiers created by naming vertices, edges and paths are *not*
// stored in here.
std::vector<Identifier *> identifiers_in_match;
// Number of nested IfOperators.
int num_if_operators{0};
};
inline static std::optional<Symbol> FindSymbolInScope(const std::string &name, const Scope &scope,
Symbol::Type type) {
if (auto it = scope.symbols.find(name); it != scope.symbols.end()) {
const auto &symbol = it->second;
// Unless we have `ANY` type, check that types match.
if (type != Symbol::Type::ANY && symbol.type() != Symbol::Type::ANY && type != symbol.type()) {
throw TypeMismatchError(name, Symbol::TypeToString(symbol.type()), Symbol::TypeToString(type));
}
return symbol;
}
return std::nullopt;
}
bool HasSymbol(const std::string &name) const {
return std::ranges::any_of(scopes_, [&name](const auto &scope) { return scope.symbols.contains(name); });
}
bool HasSymbolLocalScope(const std::string &name) const { return scopes_.back().symbols.contains(name); }
// @return true if it added a predefined identifier with that name
bool ConsumePredefinedIdentifier(const std::string &name) {
auto it = predefined_identifiers_.find(name);
if (it == predefined_identifiers_.end()) {
return false;
}
// we can only use the predefined identifier in a single scope so we remove it after creating
// a symbol for it
auto &identifier = it->second;
MG_ASSERT(!identifier->user_declared_, "Predefined symbols cannot be user declared!");
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
predefined_identifiers_.erase(it);
return true;
}
// Returns a freshly generated symbol. Previous mapping of the same name to a
// different symbol is replaced with the new one.
Symbol CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
int token_position = -1) {
auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position);
scopes_.back().symbols[name] = symbol;
return symbol;
}
Symbol GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY) {
// NOLINTNEXTLINE
for (auto scope = scopes_.rbegin(); scope != scopes_.rend(); ++scope) {
if (auto maybe_symbol = FindSymbolInScope(name, *scope, type); maybe_symbol) {
return *maybe_symbol;
}
}
return CreateSymbol(name, user_declared, type);
}
// Returns the symbol by name. If the mapping already exists, checks if the
// types match. Otherwise, returns a new symbol.
Symbol GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared,
Symbol::Type type = Symbol::Type::ANY) {
auto &scope = scopes_.back();
if (auto maybe_symbol = FindSymbolInScope(name, scope, type); maybe_symbol) {
return *maybe_symbol;
}
return CreateSymbol(name, user_declared, type);
}
void VisitReturnBody(ReturnBody &body, Where *where = nullptr) {
auto &scope = scopes_.back();
for (auto &expr : body.named_expressions) {
expr->Accept(*this);
}
std::vector<Symbol> user_symbols;
if (body.all_identifiers) {
// Carry over user symbols because '*' appeared.
for (const auto &sym_pair : scope.symbols) {
if (!sym_pair.second.user_declared()) {
continue;
}
user_symbols.emplace_back(sym_pair.second);
}
if (user_symbols.empty()) {
throw SemanticException("There are no variables in scope to use for '*'.");
}
}
// WITH/RETURN clause removes declarations of all the previous variables and
// declares only those established through named expressions. New declarations
// must not be visible inside named expressions themselves.
bool removed_old_names = false;
if ((!where && body.order_by.empty()) || scope.has_aggregation) {
// WHERE and ORDER BY need to see both the old and new symbols, unless we
// have an aggregation. Therefore, we can clear the symbols immediately if
// there is neither ORDER BY nor WHERE, or we have an aggregation.
scope.symbols.clear();
removed_old_names = true;
}
// Create symbols for named expressions.
std::unordered_set<std::string> new_names;
for (const auto &user_sym : user_symbols) {
new_names.insert(user_sym.name());
scope.symbols[user_sym.name()] = user_sym;
}
for (auto &named_expr : body.named_expressions) {
const auto &name = named_expr->name_;
if (!new_names.insert(name).second) {
throw SemanticException("Multiple results with the same name '{}' are not allowed.", name);
}
// An improvement would be to infer the type of the expression, so that the
// new symbol would have a more specific type.
named_expr->MapTo(CreateSymbol(name, true, Symbol::Type::ANY, named_expr->token_position_));
}
scope.in_order_by = true;
for (const auto &order_pair : body.order_by) {
order_pair.expression->Accept(*this);
}
scope.in_order_by = false;
if (body.skip) {
scope.in_skip = true;
body.skip->Accept(*this);
scope.in_skip = false;
}
if (body.limit) {
scope.in_limit = true;
body.limit->Accept(*this);
scope.in_limit = false;
}
if (where) where->Accept(*this);
if (!removed_old_names) {
// We have an ORDER BY or WHERE, but no aggregation, which means we didn't
// clear the old symbols, so do it now. We cannot just call clear, because
// we've added new symbols.
for (auto sym_it = scope.symbols.begin(); sym_it != scope.symbols.end();) {
if (new_names.find(sym_it->first) == new_names.end()) {
sym_it = scope.symbols.erase(sym_it);
} else {
sym_it++;
}
}
}
scopes_.back().has_aggregation = false;
}
void VisitWithIdentifiers(Expression *expr, const std::vector<Identifier *> &identifiers) {
auto &scope = scopes_.back();
std::vector<std::pair<std::optional<Symbol>, Identifier *>> prev_symbols;
// Collect previous symbols if they exist.
for (const auto &identifier : identifiers) {
std::optional<Symbol> prev_symbol;
auto prev_symbol_it = scope.symbols.find(identifier->name_);
if (prev_symbol_it != scope.symbols.end()) {
prev_symbol = prev_symbol_it->second;
}
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
prev_symbols.emplace_back(prev_symbol, identifier);
}
// Visit the expression with the new symbols bound.
expr->Accept(*this);
// Restore back to previous symbols.
for (const auto &prev : prev_symbols) {
const auto &prev_symbol = prev.first;
const auto &identifier = prev.second;
if (prev_symbol) {
scope.symbols[identifier->name_] = *prev_symbol;
} else {
scope.symbols.erase(identifier->name_);
}
}
}
SymbolTable *symbol_table_;
// Identifiers which are injected from outside the query. Each identifier
// is mapped by its name.
std::unordered_map<std::string, Identifier *> predefined_identifiers_;
std::vector<Scope> scopes_;
std::unordered_set<std::string> prev_return_names_;
std::unordered_set<std::string> curr_return_names_;
};
inline SymbolTable MakeSymbolTable(CypherQuery *query, const std::vector<Identifier *> &predefined_identifiers = {}) {
SymbolTable symbol_table;
SymbolGenerator symbol_generator(&symbol_table, predefined_identifiers);
query->single_query_->Accept(symbol_generator);
for (auto *cypher_union : query->cypher_unions_) {
cypher_union->Accept(symbol_generator);
}
return symbol_table;
}
} // namespace memgraph::expr

View File

@@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <map>
#include <string>
#include "expr/ast.hpp"
#include "expr/semantic/symbol.hpp"
#include "utils/logging.hpp"
namespace memgraph::expr {
class SymbolTable final {
public:
SymbolTable() {}
const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
int32_t token_position = -1) {
MG_ASSERT(table_.size() <= std::numeric_limits<int32_t>::max(),
"SymbolTable size doesn't fit into 32-bit integer!");
auto got = table_.emplace(position_, Symbol(name, position_, user_declared, type, token_position));
MG_ASSERT(got.second, "Duplicate symbol ID!");
position_++;
return got.first->second;
}
// TODO(buda): This is the same logic as in the cypher_main_visitor. During
// parsing phase symbol table doesn't exist. Figure out a better solution.
const Symbol &CreateAnonymousSymbol(Symbol::Type type = Symbol::Type::ANY) {
int id = 1;
while (true) {
static const std::string &kAnonPrefix = "anon";
std::string name_candidate = kAnonPrefix + std::to_string(id++);
if (std::find_if(std::begin(table_), std::end(table_), [&name_candidate](const auto &item) -> bool {
return item.second.name_ == name_candidate;
}) == std::end(table_)) {
return CreateSymbol(name_candidate, false, type);
}
}
}
const Symbol &at(const Identifier &ident) const { return table_.at(ident.symbol_pos_); }
const Symbol &at(const NamedExpression &nexpr) const { return table_.at(nexpr.symbol_pos_); }
const Symbol &at(const Aggregation &aggr) const { return table_.at(aggr.symbol_pos_); }
// TODO: Remove these since members are public
int32_t max_position() const { return static_cast<int32_t>(table_.size()); }
const auto &table() const { return table_; }
int32_t position_{0};
std::map<int32_t, Symbol> table_;
};
} // namespace memgraph::expr

1514
src/expr/typed_value.hpp Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -1,4 +0,0 @@
set(mg_glue_sources auth.cpp auth_checker.cpp auth_handler.cpp communication.cpp)
add_library(mg-glue STATIC ${mg_glue_sources})
target_link_libraries(mg-glue mg-query mg-auth)

View File

@@ -10,7 +10,6 @@
// licenses/APL.txt.
#include "glue/auth.hpp"
#include "auth/models.hpp"
namespace memgraph::glue {
@@ -60,20 +59,4 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::WEBSOCKET;
}
}
#ifdef MG_ENTERPRISE
auth::FineGrainedPermission FineGrainedPrivilegeToFineGrainedPermission(
const query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
switch (fine_grained_privilege) {
case query::AuthQuery::FineGrainedPrivilege::NOTHING:
return auth::FineGrainedPermission::NOTHING;
case query::AuthQuery::FineGrainedPrivilege::READ:
return auth::FineGrainedPermission::READ;
case query::AuthQuery::FineGrainedPrivilege::UPDATE:
return auth::FineGrainedPermission::UPDATE;
case query::AuthQuery::FineGrainedPrivilege::CREATE_DELETE:
return auth::FineGrainedPermission::CREATE_DELETE;
}
}
#endif
} // namespace memgraph::glue

View File

@@ -20,12 +20,4 @@ namespace memgraph::glue {
*/
auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege);
#ifdef MG_ENTERPRISE
/**
* Converts query::AuthQuery::FineGrainedPrivilege to its corresponding
* auth::EntityPermission.
*/
auth::FineGrainedPermission FineGrainedPrivilegeToFineGrainedPermission(
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege);
#endif
} // namespace memgraph::glue

View File

@@ -1,171 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/auth_checker.hpp"
#include "auth/auth.hpp"
#include "auth/models.hpp"
#include "glue/auth.hpp"
#include "license/license.hpp"
#include "query/frontend/ast/ast.hpp"
#include "utils/synchronized.hpp"
#ifdef MG_ENTERPRISE
namespace {
bool IsUserAuthorizedLabels(const memgraph::auth::User &user, const memgraph::query::DbAccessor *dba,
const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return std::all_of(labels.begin(), labels.end(), [dba, &user, fine_grained_privilege](const auto &label) {
return user.GetFineGrainedAccessLabelPermissions().Has(
dba->LabelToName(label), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) == memgraph::auth::PermissionLevel::GRANT;
});
}
bool IsUserAuthorizedGloballyLabels(const memgraph::auth::User &user,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return user.GetFineGrainedAccessLabelPermissions().Has(memgraph::auth::kAsterisk, fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}
bool IsUserAuthorizedGloballyEdges(const memgraph::auth::User &user,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return user.GetFineGrainedAccessEdgeTypePermissions().Has(memgraph::auth::kAsterisk, fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}
bool IsUserAuthorizedEdgeType(const memgraph::auth::User &user, const memgraph::query::DbAccessor *dba,
const memgraph::storage::EdgeTypeId &edgeType,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return user.GetFineGrainedAccessEdgeTypePermissions().Has(
dba->EdgeTypeToName(edgeType), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) == memgraph::auth::PermissionLevel::GRANT;
}
} // namespace
#endif
namespace memgraph::glue {
AuthChecker::AuthChecker(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth)
: auth_(auth) {}
bool AuthChecker::IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) const {
std::optional<memgraph::auth::User> maybe_user;
{
auto locked_auth = auth_->ReadLock();
if (!locked_auth->HasUsers()) {
return true;
}
if (username.has_value()) {
maybe_user = locked_auth->GetUser(*username);
}
}
return maybe_user.has_value() && IsUserAuthorized(*maybe_user, privileges);
}
#ifdef MG_ENTERPRISE
std::unique_ptr<memgraph::query::FineGrainedAuthChecker> AuthChecker::GetFineGrainedAuthChecker(
const std::string &username, const memgraph::query::DbAccessor *dba) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
return std::make_unique<memgraph::glue::FineGrainedAuthChecker>(std::move(*user), dba);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
#endif
bool AuthChecker::IsUserAuthorized(const memgraph::auth::User &user,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) {
const auto user_permissions = user.GetPermissions();
return std::all_of(privileges.begin(), privileges.end(), [&user_permissions](const auto privilege) {
return user_permissions.Has(memgraph::glue::PrivilegeToPermission(privilege)) ==
memgraph::auth::PermissionLevel::GRANT;
});
}
#ifdef MG_ENTERPRISE
FineGrainedAuthChecker::FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba)
: user_{std::move(user)}, dba_(dba){};
bool FineGrainedAuthChecker::Has(const memgraph::query::VertexAccessor &vertex, const memgraph::storage::View view,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) {
switch (maybe_labels.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:
throw memgraph::query::QueryRuntimeException("Trying to get labels from a deleted node.");
case memgraph::storage::Error::NONEXISTENT_OBJECT:
throw memgraph::query::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
case memgraph::storage::Error::SERIALIZATION_ERROR:
case memgraph::storage::Error::VERTEX_HAS_EDGES:
case memgraph::storage::Error::PROPERTIES_DISABLED:
throw memgraph::query::QueryRuntimeException("Unexpected error when getting labels.");
}
}
return IsUserAuthorizedLabels(user_, dba_, *maybe_labels, fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const memgraph::query::EdgeAccessor &edge,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsUserAuthorizedEdgeType(user_, dba_, edge.EdgeType(), fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsUserAuthorizedLabels(user_, dba_, labels, fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const memgraph::storage::EdgeTypeId &edge_type,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsUserAuthorizedEdgeType(user_, dba_, edge_type, fine_grained_privilege);
}
bool FineGrainedAuthChecker::HasGlobalPrivilegeOnVertices(
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return IsUserAuthorizedGloballyLabels(user_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
}
bool FineGrainedAuthChecker::HasGlobalPrivilegeOnEdges(
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return IsUserAuthorizedGloballyEdges(user_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
};
#endif
} // namespace memgraph::glue

View File

@@ -1,67 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "auth/auth.hpp"
#include "glue/auth.hpp"
#include "query/auth_checker.hpp"
#include "query/db_accessor.hpp"
#include "query/frontend/ast/ast.hpp"
namespace memgraph::glue {
class AuthChecker : public query::AuthChecker {
public:
explicit AuthChecker(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth);
bool IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<query::AuthQuery::Privilege> &privileges) const override;
#ifdef MG_ENTERPRISE
std::unique_ptr<memgraph::query::FineGrainedAuthChecker> GetFineGrainedAuthChecker(
const std::string &username, const memgraph::query::DbAccessor *dba) const override;
#endif
[[nodiscard]] static bool IsUserAuthorized(const memgraph::auth::User &user,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges);
private:
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
};
#ifdef MG_ENTERPRISE
class FineGrainedAuthChecker : public query::FineGrainedAuthChecker {
public:
explicit FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba);
bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const query::EdgeAccessor &edge,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const std::vector<memgraph::storage::LabelId> &labels,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const memgraph::storage::EdgeTypeId &edge_type,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnVertices(
memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnEdges(
memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
private:
auth::User user_;
const memgraph::query::DbAccessor *dba_;
};
#endif
} // namespace memgraph::glue

View File

@@ -1,664 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/auth_handler.hpp"
#include <sstream>
#include <fmt/format.h>
#include "auth/models.hpp"
#include "glue/auth.hpp"
#include "license/license.hpp"
namespace {
struct PermissionForPrivilegeResult {
std::string permission;
memgraph::auth::PermissionLevel permission_level;
std::string description;
};
struct FineGrainedPermissionForPrivilegeResult {
std::string permission;
#ifdef MG_ENTERPRISE
memgraph::auth::FineGrainedPermission permission_level;
#endif
std::string description;
};
PermissionForPrivilegeResult GetPermissionForPrivilegeForUserOrRole(
const memgraph::auth::Permissions &permissions, const memgraph::query::AuthQuery::Privilege &privilege,
const std::string &user_or_role) {
PermissionForPrivilegeResult container;
const auto permission = memgraph::glue::PrivilegeToPermission(privilege);
container.permission = memgraph::auth::PermissionToString(permission);
container.permission_level = permissions.Has(permission);
switch (container.permission_level) {
case memgraph::auth::PermissionLevel::GRANT:
container.description = "GRANTED TO " + user_or_role;
break;
case memgraph::auth::PermissionLevel::DENY:
container.description = "DENIED TO " + user_or_role;
break;
case memgraph::auth::PermissionLevel::NEUTRAL:
break;
}
return container;
}
std::vector<std::vector<memgraph::query::TypedValue>> ConstructPrivilegesResult(
const std::vector<PermissionForPrivilegeResult> &privileges) {
std::vector<std::vector<memgraph::query::TypedValue>> grants;
grants.reserve(privileges.size());
for (const auto &permission : privileges) {
grants.push_back({memgraph::query::TypedValue(permission.permission),
memgraph::query::TypedValue(memgraph::auth::PermissionLevelToString(permission.permission_level)),
memgraph::query::TypedValue(permission.description)});
}
return grants;
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowUserPrivileges(
const std::optional<memgraph::auth::User> &user) {
std::vector<PermissionForPrivilegeResult> privilege_results;
const auto &permissions = user->GetPermissions();
const auto &user_level_permissions = user->permissions();
for (const auto &privilege : memgraph::query::kPrivilegesAll) {
auto user_permission_result = GetPermissionForPrivilegeForUserOrRole(permissions, privilege, "USER");
auto user_only_permissions_result =
GetPermissionForPrivilegeForUserOrRole(user_level_permissions, privilege, "USER");
if (user_permission_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
std::vector<std::string> full_description;
if (user_only_permissions_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
full_description.emplace_back(user_only_permissions_result.description);
}
if (const auto *role = user->role(); role != nullptr) {
auto role_permission_result = GetPermissionForPrivilegeForUserOrRole(role->permissions(), privilege, "ROLE");
if (role_permission_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
full_description.emplace_back(role_permission_result.description);
}
}
privilege_results.push_back(PermissionForPrivilegeResult{user_permission_result.permission,
user_permission_result.permission_level,
memgraph::utils::Join(full_description, ", ")});
}
}
return ConstructPrivilegesResult(privilege_results);
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowRolePrivileges(
const std::optional<memgraph::auth::Role> &role) {
std::vector<PermissionForPrivilegeResult> privilege_results;
const auto &permissions = role->permissions();
for (const auto &privilege : memgraph::query::kPrivilegesAll) {
auto role_permission_result = GetPermissionForPrivilegeForUserOrRole(permissions, privilege, "ROLE");
if (role_permission_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
privilege_results.push_back(role_permission_result);
}
}
return ConstructPrivilegesResult(privilege_results);
}
#ifdef MG_ENTERPRISE
std::vector<FineGrainedPermissionForPrivilegeResult> GetFineGrainedPermissionForPrivilegeForUserOrRole(
const memgraph::auth::FineGrainedAccessPermissions &permissions, const std::string &permission_type,
const std::string &user_or_role) {
std::vector<FineGrainedPermissionForPrivilegeResult> fine_grained_permissions;
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return fine_grained_permissions;
}
const auto global_permission = permissions.GetGlobalPermission();
if (global_permission.has_value()) {
const auto &permission_level = memgraph::auth::PermissionToFineGrainedPermission(global_permission.value());
std::stringstream permission_representation;
permission_representation << "ALL " << permission_type << "S";
const auto &permission_level_representation =
permission_level == memgraph::auth::FineGrainedPermission::NOTHING ? "DENIED" : "GRANTED";
const auto permission_description =
fmt::format("GLOBAL {0} PERMISSION {1} TO {2}", permission_type, permission_level_representation, user_or_role);
fine_grained_permissions.push_back(FineGrainedPermissionForPrivilegeResult{
permission_representation.str(), permission_level, permission_description});
}
for (const auto &[label, permission] : permissions.GetPermissions()) {
auto permission_level = memgraph::auth::PermissionToFineGrainedPermission(permission);
std::stringstream permission_representation;
permission_representation << permission_type << " :" << label;
const auto &permission_level_representation =
permission_level == memgraph::auth::FineGrainedPermission::NOTHING ? "DENIED" : "GRANTED";
const auto permission_description =
fmt::format("{0} PERMISSION {1} TO {2}", permission_type, permission_level_representation, user_or_role);
fine_grained_permissions.push_back(FineGrainedPermissionForPrivilegeResult{
permission_representation.str(), permission_level, permission_description});
}
return fine_grained_permissions;
}
std::vector<std::vector<memgraph::query::TypedValue>> ConstructFineGrainedPrivilegesResult(
const std::vector<FineGrainedPermissionForPrivilegeResult> &privileges) {
std::vector<std::vector<memgraph::query::TypedValue>> grants;
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
grants.reserve(privileges.size());
for (const auto &permission : privileges) {
grants.push_back(
{memgraph::query::TypedValue(permission.permission),
memgraph::query::TypedValue(memgraph::auth::FineGrainedPermissionToString(permission.permission_level)),
memgraph::query::TypedValue(permission.description)});
}
return grants;
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowFineGrainedUserPrivileges(
const std::optional<memgraph::auth::User> &user) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
const auto &label_permissions = user->GetFineGrainedAccessLabelPermissions();
const auto &edge_type_permissions = user->GetFineGrainedAccessEdgeTypePermissions();
auto all_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(label_permissions, "LABEL", "USER");
auto edge_type_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(edge_type_permissions, "EDGE_TYPE", "USER");
all_fine_grained_permissions.insert(all_fine_grained_permissions.end(), edge_type_fine_grained_permissions.begin(),
edge_type_fine_grained_permissions.end());
return ConstructFineGrainedPrivilegesResult(all_fine_grained_permissions);
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowFineGrainedRolePrivileges(
const std::optional<memgraph::auth::Role> &role) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
const auto &label_permissions = role->GetFineGrainedAccessLabelPermissions();
const auto &edge_type_permissions = role->GetFineGrainedAccessEdgeTypePermissions();
auto all_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(label_permissions, "LABEL", "USER");
auto edge_type_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(edge_type_permissions, "EDGE_TYPE", "USER");
all_fine_grained_permissions.insert(all_fine_grained_permissions.end(), edge_type_fine_grained_permissions.begin(),
edge_type_fine_grained_permissions.end());
return ConstructFineGrainedPrivilegesResult(all_fine_grained_permissions);
}
#endif
} // namespace
namespace memgraph::glue {
AuthQueryHandler::AuthQueryHandler(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::string name_regex_string)
: auth_(auth), name_regex_string_(std::move(name_regex_string)), name_regex_(name_regex_string_) {}
bool AuthQueryHandler::CreateUser(const std::string &username, const std::optional<std::string> &password) {
if (name_regex_string_ != kDefaultUserRoleRegex) {
if (const auto license_check_result =
memgraph::license::global_license_checker.IsEnterpriseValid(memgraph::utils::global_settings);
license_check_result.HasError()) {
throw memgraph::auth::AuthException(
"Custom user/role regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-user-or-role-name-regex\") to its default value (\"{}\") or remove the flag.\n{}",
kDefaultUserRoleRegex,
memgraph::license::LicenseCheckErrorToString(license_check_result.GetError(), "user/role regex"));
}
}
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
try {
const auto [first_user, user_added] = std::invoke([&, this] {
auto locked_auth = auth_->Lock();
const auto first_user = !locked_auth->HasUsers();
const auto user_added = locked_auth->AddUser(username, password).has_value();
return std::make_pair(first_user, user_added);
});
if (first_user) {
spdlog::info("{} is first created user. Granting all privileges.", username);
GrantPrivilege(username, memgraph::query::kPrivilegesAll
#ifdef MG_ENTERPRISE
,
{{{memgraph::query::AuthQuery::FineGrainedPrivilege::CREATE_DELETE, {memgraph::auth::kAsterisk}}}},
{
{
{
memgraph::query::AuthQuery::FineGrainedPrivilege::CREATE_DELETE, {
memgraph::auth::kAsterisk
}
}
}
}
#endif
);
}
return user_added;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
bool AuthQueryHandler::DropUser(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) return false;
return locked_auth->RemoveUser(username);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::SetPassword(const std::string &username, const std::optional<std::string> &password) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist.", username);
}
user->UpdatePassword(password);
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
bool AuthQueryHandler::CreateRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
return locked_auth->AddRole(rolename).has_value();
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
bool AuthQueryHandler::DropRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
auto role = locked_auth->GetRole(rolename);
if (!role) {
return false;
};
return locked_auth->RemoveRole(rolename);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetUsernames() {
try {
auto locked_auth = auth_->ReadLock();
std::vector<memgraph::query::TypedValue> usernames;
const auto &users = locked_auth->AllUsers();
usernames.reserve(users.size());
for (const auto &user : users) {
usernames.emplace_back(user.username());
}
return usernames;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetRolenames() {
try {
auto locked_auth = auth_->ReadLock();
std::vector<memgraph::query::TypedValue> rolenames;
const auto &roles = locked_auth->AllRoles();
rolenames.reserve(roles.size());
for (const auto &role : roles) {
rolenames.emplace_back(role.rolename());
}
return rolenames;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::optional<std::string> AuthQueryHandler::GetRolenameForUser(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
if (const auto *role = user->role(); role != nullptr) {
return role->rolename();
}
return std::nullopt;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetUsernamesForRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto role = locked_auth->GetRole(rolename);
if (!role) {
throw memgraph::query::QueryRuntimeException("Role '{}' doesn't exist.", rolename);
}
std::vector<memgraph::query::TypedValue> usernames;
const auto &users = locked_auth->AllUsersForRole(rolename);
usernames.reserve(users.size());
for (const auto &user : users) {
usernames.emplace_back(user.username());
}
return usernames;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::SetRole(const std::string &username, const std::string &rolename) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
auto role = locked_auth->GetRole(rolename);
if (!role) {
throw memgraph::query::QueryRuntimeException("Role '{}' doesn't exist .", rolename);
}
if (const auto *current_role = user->role(); current_role != nullptr) {
throw memgraph::query::QueryRuntimeException("User '{}' is already a member of role '{}'.", username,
current_role->rolename());
}
user->SetRole(*role);
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::ClearRole(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
user->ClearRole();
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<std::vector<memgraph::query::TypedValue>> AuthQueryHandler::GetPrivileges(const std::string &user_or_role) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
auto locked_auth = auth_->ReadLock();
std::vector<std::vector<memgraph::query::TypedValue>> grants;
#ifdef MG_ENTERPRISE
std::vector<std::vector<memgraph::query::TypedValue>> fine_grained_grants;
#endif
auto user = locked_auth->GetUser(user_or_role);
auto role = locked_auth->GetRole(user_or_role);
if (!user && !role) {
throw memgraph::query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role);
}
if (user) {
grants = ShowUserPrivileges(user);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
fine_grained_grants = ShowFineGrainedUserPrivileges(user);
}
#endif
} else {
grants = ShowRolePrivileges(role);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
fine_grained_grants = ShowFineGrainedRolePrivileges(role);
}
#endif
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
grants.insert(grants.end(), fine_grained_grants.begin(), fine_grained_grants.end());
}
#endif
return grants;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::GrantPrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) {
EditPermissions(
user_or_role, privileges,
#ifdef MG_ENTERPRISE
label_privileges, edge_type_privileges,
#endif
[](auto &permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions.Grant(permission);
}
#ifdef MG_ENTERPRISE
,
[](auto &fine_grained_permissions, const auto &privilege_collection) {
for (const auto &[privilege, entities] : privilege_collection) {
const auto &permission = memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(privilege);
for (const auto &entity : entities) {
fine_grained_permissions.Grant(entity, permission);
}
}
}
#endif
);
} // namespace memgraph::glue
void AuthQueryHandler::DenyPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) {
EditPermissions(
user_or_role, privileges,
#ifdef MG_ENTERPRISE
{}, {},
#endif
[](auto &permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions.Deny(permission);
}
#ifdef MG_ENTERPRISE
,
[](auto &fine_grained_permissions, const auto &privilege_collection) {}
#endif
);
}
void AuthQueryHandler::RevokePrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) {
EditPermissions(
user_or_role, privileges,
#ifdef MG_ENTERPRISE
label_privileges, edge_type_privileges,
#endif
[](auto &permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions.Revoke(permission);
}
#ifdef MG_ENTERPRISE
,
[](auto &fine_grained_permissions, const auto &privilege_collection) {
for ([[maybe_unused]] const auto &[privilege, entities] : privilege_collection) {
for (const auto &entity : entities) {
fine_grained_permissions.Revoke(entity);
}
}
}
#endif
);
} // namespace memgraph::glue
template <class TEditPermissionsFun
#ifdef MG_ENTERPRISE
,
class TEditFineGrainedPermissionsFun
#endif
>
void AuthQueryHandler::EditPermissions(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
,
const TEditPermissionsFun &edit_permissions_fun
#ifdef MG_ENTERPRISE
,
const TEditFineGrainedPermissionsFun &edit_fine_grained_permissions_fun
#endif
) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
std::vector<memgraph::auth::Permission> permissions;
permissions.reserve(privileges.size());
for (const auto &privilege : privileges) {
permissions.push_back(memgraph::glue::PrivilegeToPermission(privilege));
}
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(user_or_role);
auto role = locked_auth->GetRole(user_or_role);
if (!user && !role) {
throw memgraph::query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role);
}
if (user) {
for (const auto &permission : permissions) {
edit_permissions_fun(user->permissions(), permission);
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
for (const auto &label_privilege_collection : label_privileges) {
edit_fine_grained_permissions_fun(user->fine_grained_access_handler().label_permissions(),
label_privilege_collection);
}
for (const auto &edge_type_privilege_collection : edge_type_privileges) {
edit_fine_grained_permissions_fun(user->fine_grained_access_handler().edge_type_permissions(),
edge_type_privilege_collection);
}
}
#endif
locked_auth->SaveUser(*user);
} else {
for (const auto &permission : permissions) {
edit_permissions_fun(role->permissions(), permission);
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
for (const auto &label_privilege : label_privileges) {
edit_fine_grained_permissions_fun(role->fine_grained_access_handler().label_permissions(), label_privilege);
}
for (const auto &edge_type_privilege : edge_type_privileges) {
edit_fine_grained_permissions_fun(role->fine_grained_access_handler().edge_type_permissions(),
edge_type_privilege);
}
}
#endif
locked_auth->SaveRole(*role);
}
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
} // namespace memgraph::glue

View File

@@ -1,109 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <regex>
#include "auth/auth.hpp"
#include "glue/auth.hpp"
#include "license/license.hpp"
#include "query/interpreter.hpp"
#include "utils/string.hpp"
namespace memgraph::glue {
inline constexpr std::string_view kDefaultUserRoleRegex = "[a-zA-Z0-9_.+-@]+";
class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
std::string name_regex_string_;
std::regex name_regex_;
public:
AuthQueryHandler(memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::string name_regex_string);
bool CreateUser(const std::string &username, const std::optional<std::string> &password) override;
bool DropUser(const std::string &username) override;
void SetPassword(const std::string &username, const std::optional<std::string> &password) override;
bool CreateRole(const std::string &rolename) override;
bool DropRole(const std::string &rolename) override;
std::vector<memgraph::query::TypedValue> GetUsernames() override;
std::vector<memgraph::query::TypedValue> GetRolenames() override;
std::optional<std::string> GetRolenameForUser(const std::string &username) override;
std::vector<memgraph::query::TypedValue> GetUsernamesForRole(const std::string &rolename) override;
void SetRole(const std::string &username, const std::string &rolename) override;
void ClearRole(const std::string &username) override;
std::vector<std::vector<memgraph::query::TypedValue>> GetPrivileges(const std::string &user_or_role) override;
void GrantPrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) override;
void DenyPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override;
void RevokePrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) override;
private:
template <class TEditPermissionsFun
#ifdef MG_ENTERPRISE
,
class TEditFineGrainedPermissionsFun
#endif
>
void EditPermissions(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
,
const TEditPermissionsFun &edit_permissions_fun
#ifdef MG_ENTERPRISE
,
const TEditFineGrainedPermissionsFun &edit_fine_grained_permissions_fun
#endif
);
};
} // namespace memgraph::glue

View File

@@ -127,10 +127,6 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage
return Value(value.ValueLocalDateTime());
case query::TypedValue::Type::Duration:
return Value(value.ValueDuration());
case query::TypedValue::Type::Graph:
auto maybe_graph = ToBoltGraph(value.ValueGraph(), db, view);
if (maybe_graph.HasError()) return maybe_graph.GetError();
return Value(std::move(*maybe_graph));
}
}
@@ -187,30 +183,6 @@ storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, c
return communication::bolt::Path(vertices, edges);
}
storage::Result<std::map<std::string, Value>> ToBoltGraph(const query::Graph &graph, const storage::Storage &db,
storage::View view) {
std::map<std::string, Value> map;
std::vector<Value> vertices;
vertices.reserve(graph.vertices().size());
for (const auto &v : graph.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(Value(std::move(*maybe_vertex)));
}
map.emplace("nodes", Value(vertices));
std::vector<Value> edges;
edges.reserve(graph.edges().size());
for (const auto &e : graph.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(Value(std::move(*maybe_edge)));
}
map.emplace("edges", Value(edges));
return std::move(map);
}
storage::PropertyValue ToPropertyValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:

View File

@@ -51,15 +51,6 @@ storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccesso
storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, const storage::Storage &db,
storage::View view);
/// @param query::Graph for converting to communication::bolt::Map.
/// @param storage::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::Result<std::map<std::string, communication::bolt::Value>> ToBoltGraph(const query::Graph &graph,
const storage::Storage &db,
storage::View view);
/// @param query::TypedValue for converting to communication::bolt::Value.
/// @param storage::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::View for ToBoltVertex and ToBoltEdge.

64
src/glue/v2/auth.cpp Normal file
View File

@@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/auth.hpp"
namespace memgraph::glue::v2 {
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege) {
switch (privilege) {
case query::v2::AuthQuery::Privilege::MATCH:
return auth::Permission::MATCH;
case query::v2::AuthQuery::Privilege::CREATE:
return auth::Permission::CREATE;
case query::v2::AuthQuery::Privilege::MERGE:
return auth::Permission::MERGE;
case query::v2::AuthQuery::Privilege::DELETE:
return auth::Permission::DELETE;
case query::v2::AuthQuery::Privilege::SET:
return auth::Permission::SET;
case query::v2::AuthQuery::Privilege::REMOVE:
return auth::Permission::REMOVE;
case query::v2::AuthQuery::Privilege::INDEX:
return auth::Permission::INDEX;
case query::v2::AuthQuery::Privilege::STATS:
return auth::Permission::STATS;
case query::v2::AuthQuery::Privilege::CONSTRAINT:
return auth::Permission::CONSTRAINT;
case query::v2::AuthQuery::Privilege::DUMP:
return auth::Permission::DUMP;
case query::v2::AuthQuery::Privilege::REPLICATION:
return auth::Permission::REPLICATION;
case query::v2::AuthQuery::Privilege::DURABILITY:
return auth::Permission::DURABILITY;
case query::v2::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::v2::AuthQuery::Privilege::FREE_MEMORY:
return auth::Permission::FREE_MEMORY;
case query::v2::AuthQuery::Privilege::TRIGGER:
return auth::Permission::TRIGGER;
case query::v2::AuthQuery::Privilege::CONFIG:
return auth::Permission::CONFIG;
case query::v2::AuthQuery::Privilege::AUTH:
return auth::Permission::AUTH;
case query::v2::AuthQuery::Privilege::STREAM:
return auth::Permission::STREAM;
case query::v2::AuthQuery::Privilege::MODULE_READ:
return auth::Permission::MODULE_READ;
case query::v2::AuthQuery::Privilege::MODULE_WRITE:
return auth::Permission::MODULE_WRITE;
case query::v2::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::v2::AuthQuery::Privilege::SCHEMA:
return auth::Permission::SCHEMA;
}
}
} // namespace memgraph::glue::v2

23
src/glue/v2/auth.hpp Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "auth/models.hpp"
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::glue::v2 {
/**
* This function converts query::AuthQuery::Privilege to its corresponding
* auth::Permission.
*/
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege);
} // namespace memgraph::glue::v2

View File

@@ -0,0 +1,277 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/communication.hpp"
#include <map>
#include <string>
#include <vector>
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/temporal.hpp"
using memgraph::communication::bolt::Value;
namespace memgraph::glue::v2 {
query::v2::TypedValue ToTypedValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return query::v2::TypedValue(value.ValueBool());
case Value::Type::Int:
return query::v2::TypedValue(value.ValueInt());
case Value::Type::Double:
return query::v2::TypedValue(value.ValueDouble());
case Value::Type::String:
return query::v2::TypedValue(value.ValueString());
case Value::Type::List: {
std::vector<query::v2::TypedValue> list;
list.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) list.push_back(ToTypedValue(v));
return query::v2::TypedValue(std::move(list));
}
case Value::Type::Map: {
std::map<std::string, query::v2::TypedValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToTypedValue(kv.second));
return query::v2::TypedValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to TypedValue");
case Value::Type::Date:
return query::v2::TypedValue(value.ValueDate());
case Value::Type::LocalTime:
return query::v2::TypedValue(value.ValueLocalTime());
case Value::Type::LocalDateTime:
return query::v2::TypedValue(value.ValueLocalDateTime());
case Value::Type::Duration:
return query::v2::TypedValue(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
return Value();
case query::v2::TypedValue::Type::Bool:
return Value(value.ValueBool());
case query::v2::TypedValue::Type::Int:
return Value(value.ValueInt());
case query::v2::TypedValue::Type::Double:
return Value(value.ValueDouble());
case query::v2::TypedValue::Type::String:
return Value(std::string(value.ValueString()));
case query::v2::TypedValue::Type::List: {
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
auto maybe_value = ToBoltValue(v, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
values.emplace_back(std::move(*maybe_value));
}
return Value(std::move(values));
}
case query::v2::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return Value(std::move(map));
}
case query::v2::TypedValue::Type::Vertex: {
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::v2::TypedValue::Type::Edge: {
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::v2::TypedValue::Type::Path: {
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::v2::TypedValue::Type::Date:
return Value(value.ValueDate());
case query::v2::TypedValue::Type::LocalTime:
return Value(value.ValueLocalTime());
case query::v2::TypedValue::Type::LocalDateTime:
return Value(value.ValueLocalDateTime());
case query::v2::TypedValue::Type::Duration:
return Value(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
std::vector<std::string> labels;
labels.reserve(maybe_labels->size());
for (const auto &label : *maybe_labels) {
labels.push_back(db.LabelToName(label));
}
auto maybe_properties = vertex.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Vertex{id, labels, properties};
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto from = communication::bolt::Id::FromUint(0);
auto to = communication::bolt::Id::FromUint(0);
const auto &type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());
for (const auto &v : path.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(std::move(*maybe_vertex));
}
std::vector<communication::bolt::Edge> edges;
edges.reserve(path.edges().size());
for (const auto &e : path.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(std::move(*maybe_edge));
}
return communication::bolt::Path(vertices, edges);
}
storage::v3::PropertyValue ToPropertyValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return storage::v3::PropertyValue(value.ValueBool());
case Value::Type::Int:
return storage::v3::PropertyValue(value.ValueInt());
case Value::Type::Double:
return storage::v3::PropertyValue(value.ValueDouble());
case Value::Type::String:
return storage::v3::PropertyValue(value.ValueString());
case Value::Type::List: {
std::vector<storage::v3::PropertyValue> vec;
vec.reserve(value.ValueList().size());
for (const auto &value : value.ValueList()) vec.emplace_back(ToPropertyValue(value));
return storage::v3::PropertyValue(std::move(vec));
}
case Value::Type::Map: {
std::map<std::string, storage::v3::PropertyValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToPropertyValue(kv.second));
return storage::v3::PropertyValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to PropertyValue");
case Value::Type::Date:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Date, value.ValueDate().MicrosecondsSinceEpoch()));
case Value::Type::LocalTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalTime,
value.ValueLocalTime().MicrosecondsSinceEpoch()));
case Value::Type::LocalDateTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalDateTime,
value.ValueLocalDateTime().MicrosecondsSinceEpoch()));
case Value::Type::Duration:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Duration, value.ValueDuration().microseconds));
}
}
Value ToBoltValue(const storage::v3::PropertyValue &value) {
switch (value.type()) {
case storage::v3::PropertyValue::Type::Null:
return {};
case storage::v3::PropertyValue::Type::Bool:
return {value.ValueBool()};
case storage::v3::PropertyValue::Type::Int:
return {value.ValueInt()};
break;
case storage::v3::PropertyValue::Type::Double:
return {value.ValueDouble()};
case storage::v3::PropertyValue::Type::String:
return {value.ValueString()};
case storage::v3::PropertyValue::Type::List: {
const auto &values = value.ValueList();
std::vector<Value> vec;
vec.reserve(values.size());
for (const auto &v : values) {
vec.push_back(ToBoltValue(v));
}
return {std::move(vec)};
}
case storage::v3::PropertyValue::Type::Map: {
const auto &map = value.ValueMap();
std::map<std::string, Value> dv_map;
for (const auto &kv : map) {
dv_map.emplace(kv.first, ToBoltValue(kv.second));
}
return {std::move(dv_map)};
}
case storage::v3::PropertyValue::Type::TemporalData:
const auto &type = value.ValueTemporalData();
switch (type.type) {
case storage::v3::TemporalType::Date:
return {utils::Date(type.microseconds)};
case storage::v3::TemporalType::LocalTime:
return {utils::LocalTime(type.microseconds)};
case storage::v3::TemporalType::LocalDateTime:
return {utils::LocalDateTime(type.microseconds)};
case storage::v3::TemporalType::Duration:
return {utils::Duration(type.microseconds)};
}
}
}
} // namespace memgraph::glue::v2

View File

@@ -0,0 +1,68 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file Conversion functions between Value and other memgraph types.
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
class Storage;
class VertexAccessor;
} // namespace memgraph::storage::v3
namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param storage::v3::Shard for getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param storage::v3::Shard for getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
const storage::v3::Shard &db, storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
} // namespace memgraph::glue::v2

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,7 +12,6 @@
#pragma once
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
@@ -62,42 +61,3 @@ inline void LoadConfig(const std::string &product_name) {
for (int i = 0; i < custom_argc; ++i) free(custom_argv[i]);
delete[] custom_argv;
}
std::pair<std::string, std::string> LoadUsernameAndPassword(const std::string &pass_file) {
std::ifstream file(pass_file);
if (file.fail()) {
spdlog::warn("Problem with opening MG_PASSFILE, memgraph server will start without user");
return {};
}
std::vector<std::string> result;
std::string line;
std::getline(file, line);
size_t pos = 0;
std::string token;
static constexpr std::string_view delimiter{":"};
while ((pos = line.find(delimiter)) != std::string::npos) {
if (line[pos - 1] == '\\') {
line.erase(pos - 1, 1);
token += line.substr(0, pos);
line.erase(0, pos);
} else {
token += line.substr(0, pos);
result.push_back(token);
line.erase(0, pos + delimiter.length());
token = "";
}
}
result.push_back(line);
file.close();
if (result.size() != 2) {
spdlog::warn(
"Wrong data format. Data should be store in format: username:password, memgraph server will start without "
"user");
return {};
}
return {result[0], result[1]};
}

77
src/io/address.hpp Normal file
View File

@@ -0,0 +1,77 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <compare>
#include <fmt/format.h>
#include <boost/asio/ip/tcp.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
namespace memgraph::io {
struct Address {
// It's important for all participants to have a
// unique identifier - IP and port alone are not
// enough, and may change over the lifecycle of
// the nodes. Particularly storage nodes may change
// their IP addresses over time, and the system
// should gracefully update its information
// about them.
boost::uuids::uuid unique_id;
boost::asio::ip::address last_known_ip;
uint16_t last_known_port;
static Address TestAddress(uint16_t port) {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
.last_known_port = port,
};
}
static Address UniqueLocalAddress() {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
};
}
/// Returns a new ID with the same IP and port but a unique UUID.
Address ForkUniqueAddress() {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
.last_known_ip = last_known_ip,
.last_known_port = last_known_port,
};
}
friend bool operator==(const Address &lhs, const Address &rhs) = default;
/// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
friend bool operator<(const Address &lhs, const Address &rhs) {
if (lhs.unique_id != rhs.unique_id) {
return lhs.unique_id < rhs.unique_id;
}
if (lhs.last_known_ip != rhs.last_known_ip) {
return lhs.last_known_ip < rhs.last_known_ip;
}
return lhs.last_known_port < rhs.last_known_port;
}
std::string ToString() const {
return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
}
};
}; // namespace memgraph::io

26
src/io/errors.hpp Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::io {
// Signifies that a retriable operation was unable to
// complete after a configured number of retries.
struct RetriesExhausted {};
// Signifies that a request was unable to receive a response
// within some configured timeout duration. It is important
// to remember that in distributed systems, a timeout does
// not signify that a request was not received or processed.
// It may be the case that the request was fully processed
// but that the response was not received.
struct TimedOut {};
}; // namespace memgraph::io

256
src/io/future.hpp Normal file
View File

@@ -0,0 +1,256 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <condition_variable>
#include <memory>
#include <mutex>
#include <optional>
#include <thread>
#include <utility>
#include "io/errors.hpp"
#include "utils/logging.hpp"
namespace memgraph::io {
// Shared is in an anonymous namespace, and the only way to
// construct a Promise or Future is to pass a Shared in. This
// ensures that Promises and Futures can only be constructed
// in this translation unit.
namespace details {
template <typename T>
class Shared {
mutable std::condition_variable cv_;
mutable std::mutex mu_;
std::optional<T> item_;
bool consumed_ = false;
bool waiting_ = false;
std::function<bool()> simulator_notifier_ = nullptr;
public:
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
Shared() = default;
Shared(Shared &&) = delete;
Shared &operator=(Shared &&) = delete;
Shared(const Shared &) = delete;
Shared &operator=(const Shared &) = delete;
~Shared() = default;
/// Takes the item out of our optional item_ and returns it.
T Take() {
MG_ASSERT(item_, "Take called without item_ being present");
MG_ASSERT(!consumed_, "Take called on already-consumed Future");
T ret = std::move(item_).value();
item_.reset();
consumed_ = true;
return ret;
}
T Wait() {
std::unique_lock<std::mutex> lock(mu_);
waiting_ = true;
while (!item_) {
bool simulator_progressed = false;
if (simulator_notifier_) [[unlikely]] {
// We can't hold our own lock while notifying
// the simulator because notifying the simulator
// involves acquiring the simulator's mutex
// to guarantee that our notification linearizes
// with the simulator's condition variable.
// However, the simulator may acquire our
// mutex to check if we are being awaited,
// while determining system quiescence,
// so we have to get out of its way to avoid
// a cyclical deadlock.
lock.unlock();
simulator_progressed = std::invoke(simulator_notifier_);
lock.lock();
if (item_) {
// item may have been filled while we
// had dropped our mutex while notifying
// the simulator of our waiting_ status.
break;
}
}
if (!simulator_progressed) [[likely]] {
cv_.wait(lock);
}
MG_ASSERT(!consumed_, "Future consumed twice!");
}
waiting_ = false;
return Take();
}
bool IsReady() const {
std::unique_lock<std::mutex> lock(mu_);
return item_.has_value();
}
std::optional<T> TryGet() {
std::unique_lock<std::mutex> lock(mu_);
if (item_) {
return Take();
}
return std::nullopt;
}
void Fill(T item) {
{
std::unique_lock<std::mutex> lock(mu_);
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!item_, "Promise filled twice!");
item_ = item;
} // lock released before condition variable notification
cv_.notify_all();
}
bool IsAwaited() const {
std::unique_lock<std::mutex> lock(mu_);
return waiting_;
}
};
} // namespace details
template <typename T>
class Future {
bool consumed_or_moved_ = false;
std::shared_ptr<details::Shared<T>> shared_;
public:
explicit Future(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Future() = delete;
Future(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
consumed_or_moved_ = old.consumed_or_moved_;
old.consumed_or_moved_ = true;
}
Future(const Future &) = delete;
Future &operator=(const Future &) = delete;
~Future() = default;
/// Returns true if the Future is ready to
/// be consumed using TryGet or Wait (prefer Wait
/// if you know it's ready, because it doesn't
/// return an optional.
bool IsReady() {
MG_ASSERT(!consumed_or_moved_, "Called IsReady after Future already consumed!");
return shared_->IsReady();
}
/// Non-blocking method that returns the inner
/// item if it's already ready, or std::nullopt
/// if it is not ready yet.
std::optional<T> TryGet() {
MG_ASSERT(!consumed_or_moved_, "Called TryGet after Future already consumed!");
std::optional<T> ret = shared_->TryGet();
if (ret) {
consumed_or_moved_ = true;
}
return ret;
}
/// Block on the corresponding promise to be filled,
/// returning the inner item when ready.
T Wait() && {
MG_ASSERT(!consumed_or_moved_, "Future should only be consumed with Wait once!");
T ret = shared_->Wait();
consumed_or_moved_ = true;
return ret;
}
/// Marks this Future as canceled.
void Cancel() {
MG_ASSERT(!consumed_or_moved_, "Future::Cancel called on a future that was already moved or consumed!");
consumed_or_moved_ = true;
}
};
template <typename T>
class Promise {
std::shared_ptr<details::Shared<T>> shared_;
bool filled_or_moved_ = false;
public:
explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Promise() = delete;
Promise(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise &operator=(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise(const Promise &) = delete;
Promise &operator=(const Promise &) = delete;
~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
// Fill the expected item into the Future.
void Fill(T item) {
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
shared_->Fill(item);
filled_or_moved_ = true;
}
bool IsAwaited() { return shared_->IsAwaited(); }
/// Moves this Promise into a unique_ptr.
std::unique_ptr<Promise<T>> ToUnique() && {
std::unique_ptr<Promise<T>> up = std::make_unique<Promise<T>>(std::move(shared_));
filled_or_moved_ = true;
return up;
}
};
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePair() {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>();
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
}; // namespace memgraph::io

View File

@@ -0,0 +1,35 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <random>
#include "io/address.hpp"
#include "io/local_transport/local_transport.hpp"
#include "io/local_transport/local_transport_handle.hpp"
#include "io/transport.hpp"
namespace memgraph::io::local_transport {
class LocalSystem {
std::shared_ptr<LocalTransportHandle> local_transport_handle_ = std::make_shared<LocalTransportHandle>();
public:
Io<LocalTransport> Register(Address address) {
LocalTransport local_transport(local_transport_handle_);
return Io{local_transport, address};
}
void ShutDown() { local_transport_handle_->ShutDown(); }
};
} // namespace memgraph::io::local_transport

View File

@@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <memory>
#include <random>
#include <utility>
#include "io/address.hpp"
#include "io/local_transport/local_transport_handle.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::local_transport {
class LocalTransport {
std::shared_ptr<LocalTransportHandle> local_transport_handle_;
public:
explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
: local_transport_handle_(std::move(local_transport_handle)) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
Duration timeout) {
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
local_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
return local_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
}
Time Now() const { return local_transport_handle_->Now(); }
bool ShouldShutDown() const { return local_transport_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::random_device rng;
return distrib(rng);
}
};
}; // namespace memgraph::io::local_transport

View File

@@ -0,0 +1,151 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <map>
#include <mutex>
#include "io/errors.hpp"
#include "io/message_conversion.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::local_transport {
class LocalTransportHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
bool should_shut_down_ = false;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::vector<OpaqueMessage> can_receive_;
public:
~LocalTransportHandle() {
for (auto &&[pk, promise] : promises_) {
std::move(promise.promise).TimeOut();
}
promises_.clear();
}
void ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
static Time Now() {
auto nano_time = std::chrono::system_clock::now();
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
std::unique_lock lock(mu_);
Time before = Now();
spdlog::info("can_receive_ size: {}", can_receive_.size());
while (can_receive_.empty()) {
Time now = Now();
// protection against non-monotonic timesources
auto maxed_now = std::max(now, before);
auto elapsed = maxed_now - before;
if (timeout < elapsed) {
return TimedOut{};
}
Duration relative_timeout = timeout - elapsed;
std::cv_status cv_status_value = cv_.wait_for(lock, relative_timeout);
if (cv_status_value == std::cv_status::timeout) {
return TimedOut{};
}
}
auto current_message = std::move(can_receive_.back());
can_receive_.pop_back();
auto m_opt = std::move(current_message).Take<Ms...>();
return std::move(m_opt).value();
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
std::any message_any(std::forward<M>(message));
OpaqueMessage opaque_message{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any)};
PromiseKey promise_key{
.requester_address = to_address, .request_id = opaque_message.request_id, .replier_address = from_address};
{
std::unique_lock<std::mutex> lock(mu_);
if (promises_.contains(promise_key)) {
spdlog::info("using message to fill promise");
// complete waiting promise if it's there
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
dop.promise.Fill(std::move(opaque_message));
} else {
spdlog::info("placing message in can_receive_");
can_receive_.emplace_back(std::move(opaque_message));
}
} // lock dropped
cv_.notify_all();
}
template <Message RequestT, Message ResponseT>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
Duration timeout, ResponsePromise<ResponseT> promise) {
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
MG_ASSERT(port_matches && ip_matches);
const Time deadline = Now() + timeout;
{
std::unique_lock<std::mutex> lock(mu_);
PromiseKey promise_key{
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
} // lock dropped
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
}
};
} // namespace memgraph::io::local_transport

View File

@@ -0,0 +1,232 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/transport.hpp"
namespace memgraph::io {
using memgraph::io::Duration;
using memgraph::io::Message;
using memgraph::io::Time;
struct PromiseKey {
Address requester_address;
uint64_t request_id;
// TODO(tyler) possibly remove replier_address from promise key
// once we want to support DSR.
Address replier_address;
public:
friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
if (lhs.requester_address != rhs.requester_address) {
return lhs.requester_address < rhs.requester_address;
}
if (lhs.request_id != rhs.request_id) {
return lhs.request_id < rhs.request_id;
}
return lhs.replier_address < rhs.replier_address;
}
};
struct OpaqueMessage {
Address to_address;
Address from_address;
uint64_t request_id;
std::any message;
/// Recursively tries to match a specific type from the outer
/// variant's parameter pack against the type of the std::any,
/// and if it matches, make it concrete and return it. Otherwise,
/// move on and compare the any with the next type from the
/// parameter pack.
///
/// Return is the full std::variant<Ts...> type that holds the
/// full parameter pack without interfering with recursive
/// narrowing expansion.
template <typename Return, Message Head, Message... Rest>
std::optional<Return> Unpack(std::any &&a) {
if (typeid(Head) == a.type()) {
Head concrete = std::any_cast<Head>(std::move(a));
return concrete;
}
if constexpr (sizeof...(Rest) > 0) {
return Unpack<Return, Rest...>(std::move(a));
} else {
return std::nullopt;
}
}
/// High level "user-facing" conversion function that lets
/// people interested in conversion only supply a single
/// parameter pack for the types that they want to compare
/// with the any and potentially include in the returned
/// variant.
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
if (m_opt) {
return RequestEnvelope<Ms...>{
.message = std::move(*m_opt),
.request_id = request_id,
.to_address = to_address,
.from_address = from_address,
};
}
return std::nullopt;
}
};
class OpaquePromiseTraitBase {
public:
virtual const std::type_info *TypeInfo() const = 0;
virtual bool IsAwaited(void *ptr) const = 0;
virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
virtual void TimeOut(void *ptr) const = 0;
virtual ~OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase &operator=(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase(OpaquePromiseTraitBase &&old) = delete;
OpaquePromiseTraitBase &operator=(OpaquePromiseTraitBase &&) = delete;
};
template <typename T>
class OpaquePromiseTrait : public OpaquePromiseTraitBase {
public:
const std::type_info *TypeInfo() const override { return &typeid(T); };
bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
T message = std::any_cast<T>(std::move(opaque_message.message));
auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
.request_id = opaque_message.request_id,
.to_address = opaque_message.to_address,
.from_address = opaque_message.from_address};
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
unique_promise->Fill(std::move(response_envelope));
};
void TimeOut(void *ptr) const override {
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
ResponseResult<T> result = TimedOut{};
unique_promise->Fill(std::move(result));
}
};
class OpaquePromise {
void *ptr_;
std::unique_ptr<OpaquePromiseTraitBase> trait_;
public:
OpaquePromise(OpaquePromise &&old) noexcept : ptr_(old.ptr_), trait_(std::move(old.trait_)) {
MG_ASSERT(old.ptr_ != nullptr);
old.ptr_ = nullptr;
}
OpaquePromise &operator=(OpaquePromise &&old) noexcept {
MG_ASSERT(ptr_ == nullptr);
MG_ASSERT(old.ptr_ != nullptr);
MG_ASSERT(this != &old);
ptr_ = old.ptr_;
trait_ = std::move(old.trait_);
old.ptr_ = nullptr;
return *this;
}
OpaquePromise(const OpaquePromise &) = delete;
OpaquePromise &operator=(const OpaquePromise &) = delete;
template <typename T>
std::unique_ptr<ResponsePromise<T>> Take() && {
MG_ASSERT(typeid(T) == *trait_->TypeInfo());
MG_ASSERT(ptr_ != nullptr);
auto ptr = static_cast<ResponsePromise<T> *>(ptr_);
ptr_ = nullptr;
return std::unique_ptr<T>(ptr);
}
template <typename T>
explicit OpaquePromise(std::unique_ptr<ResponsePromise<T>> promise)
: ptr_(static_cast<void *>(promise.release())), trait_(std::make_unique<OpaquePromiseTrait<T>>()) {}
bool IsAwaited() {
MG_ASSERT(ptr_ != nullptr);
return trait_->IsAwaited(ptr_);
}
void TimeOut() {
MG_ASSERT(ptr_ != nullptr);
trait_->TimeOut(ptr_);
ptr_ = nullptr;
}
void Fill(OpaqueMessage &&opaque_message) {
MG_ASSERT(ptr_ != nullptr);
trait_->Fill(ptr_, std::move(opaque_message));
ptr_ = nullptr;
}
~OpaquePromise() {
MG_ASSERT(ptr_ == nullptr, "OpaquePromise destroyed without being explicitly timed out or filled");
}
};
struct DeadlineAndOpaquePromise {
Time deadline;
OpaquePromise promise;
};
template <class From>
std::type_info const &type_info_for_variant(From const &from) {
return std::visit([](auto &&x) -> decltype(auto) { return typeid(x); }, from);
}
template <typename From, typename Return, typename Head, typename... Rest>
std::optional<Return> ConvertVariantInner(From &&a) {
if (typeid(Head) == type_info_for_variant(a)) {
Head concrete = std::get<Head>(std::forward<From>(a));
return concrete;
}
if constexpr (sizeof...(Rest) > 0) {
return ConvertVariantInner<From, Return, Rest...>(std::forward<From>(a));
} else {
return std::nullopt;
}
}
/// This function converts a variant to another variant holding a subset OR superset of
/// possible types.
template <class From, class... Ms>
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> ConvertVariant(From &&from) {
return ConvertVariantInner<From, std::variant<Ms...>, Ms...>(std::forward<From>(from));
}
} // namespace memgraph::io

46
src/io/messages.hpp Normal file
View File

@@ -0,0 +1,46 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <variant>
#include <coordinator/coordinator.hpp>
#include <io/rsm/raft.hpp>
#include "query/v2/requests.hpp"
#include "utils/concepts.hpp"
namespace memgraph::io::messages {
using memgraph::coordinator::CoordinatorReadRequests;
using memgraph::coordinator::CoordinatorWriteRequests;
using memgraph::coordinator::CoordinatorWriteResponses;
using StorageReadRequest = msgs::ReadRequests;
using StorageWriteRequest = msgs::WriteRequests;
using memgraph::io::rsm::AppendRequest;
using memgraph::io::rsm::AppendResponse;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::VoteRequest;
using memgraph::io::rsm::VoteResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using CoordinatorMessages =
std::variant<ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>, AppendResponse,
WriteRequest<CoordinatorWriteRequests>, VoteRequest, VoteResponse>;
using ShardMessages = std::variant<ReadRequest<StorageReadRequest>, AppendRequest<StorageWriteRequest>, AppendResponse,
WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>;
using ShardManagerMessages = std::variant<WriteResponse<CoordinatorWriteResponses>>;
} // namespace memgraph::io::messages

926
src/io/rsm/raft.hpp Normal file
View File

@@ -0,0 +1,926 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// TODO(tyler) buffer out-of-order Append buffers on the Followers to reassemble more quickly
// TODO(tyler) handle granular batch sizes based on simple flow control
#pragma once
#include <deque>
#include <iostream>
#include <map>
#include <set>
#include <thread>
#include <unordered_map>
#include <vector>
#include "io/simulator/simulator.hpp"
#include "io/transport.hpp"
#include "utils/concepts.hpp"
namespace memgraph::io::rsm {
/// Timeout and replication tunables
using namespace std::chrono_literals;
static constexpr auto kMinimumElectionTimeout = 100ms;
static constexpr auto kMaximumElectionTimeout = 200ms;
static constexpr auto kMinimumBroadcastTimeout = 40ms;
static constexpr auto kMaximumBroadcastTimeout = 60ms;
static constexpr auto kMinimumCronInterval = 1ms;
static constexpr auto kMaximumCronInterval = 2ms;
static constexpr auto kMinimumReceiveTimeout = 40ms;
static constexpr auto kMaximumReceiveTimeout = 60ms;
static_assert(kMinimumElectionTimeout > kMaximumBroadcastTimeout,
"The broadcast timeout has to be smaller than the election timeout!");
static_assert(kMinimumElectionTimeout < kMaximumElectionTimeout,
"The minimum election timeout has to be smaller than the maximum election timeout!");
static_assert(kMinimumBroadcastTimeout < kMaximumBroadcastTimeout,
"The minimum broadcast timeout has to be smaller than the maximum broadcast timeout!");
static_assert(kMinimumCronInterval < kMaximumCronInterval,
"The minimum cron interval has to be smaller than the maximum cron interval!");
static_assert(kMinimumReceiveTimeout < kMaximumReceiveTimeout,
"The minimum receive timeout has to be smaller than the maximum receive timeout!");
static constexpr size_t kMaximumAppendBatchSize = 1024;
using Term = uint64_t;
using LogIndex = uint64_t;
using LogSize = uint64_t;
template <typename WriteOperation>
struct WriteRequest {
WriteOperation operation;
};
/// WriteResponse is returned to a client after
/// their WriteRequest was entered in to the raft
/// log and it reached consensus.
///
/// WriteReturn is the result of applying the WriteRequest to
/// ReplicatedState, and if the ReplicatedState::write
/// method is deterministic, all replicas will
/// have the same ReplicatedState after applying
/// the submitted WriteRequest.
template <typename WriteReturn>
struct WriteResponse {
bool success;
WriteReturn write_return;
std::optional<Address> retry_leader;
LogIndex raft_index;
};
template <typename ReadOperation>
struct ReadRequest {
ReadOperation operation;
};
template <typename ReadReturn>
struct ReadResponse {
bool success;
ReadReturn read_return;
std::optional<Address> retry_leader;
};
/// AppendRequest is a raft-level message that the Leader
/// periodically broadcasts to all Follower peers. This
/// serves three main roles:
/// 1. acts as a heartbeat from the Leader to the Follower
/// 2. replicates new data that the Leader has received to the Follower
/// 3. informs Follower peers when the commit index has increased,
/// signalling that it is now safe to apply log items to the
/// replicated state machine
template <typename WriteRequest>
struct AppendRequest {
Term term = 0;
LogIndex batch_start_log_index;
Term last_log_term;
std::vector<std::pair<Term, WriteRequest>> entries;
LogSize leader_commit;
};
struct AppendResponse {
bool success;
Term term;
Term last_log_term;
// a small optimization over the raft paper, tells
// the leader the offset that we are interested in
// to send log offsets from for us. This will only
// be useful at the beginning of a leader's term.
LogSize log_size;
};
struct VoteRequest {
Term term = 0;
LogSize log_size;
Term last_log_term;
};
struct VoteResponse {
Term term = 0;
LogSize committed_log_size;
bool vote_granted = false;
};
template <typename WriteRequest>
struct CommonState {
Term term = 0;
std::vector<std::pair<Term, WriteRequest>> log;
LogSize committed_log_size = 0;
LogSize applied_size = 0;
};
struct FollowerTracker {
LogIndex next_index = 0;
LogSize confirmed_log_size = 0;
};
struct PendingClientRequest {
RequestId request_id;
Address address;
Time received_at;
};
struct Leader {
std::map<Address, FollowerTracker> followers;
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
Time last_broadcast = Time::min();
std::string static ToString() { return "\tLeader \t"; }
};
struct Candidate {
std::map<Address, LogSize> successful_votes;
Time election_began = Time::min();
std::set<Address> outstanding_votes;
std::string static ToString() { return "\tCandidate\t"; }
};
struct Follower {
Time last_received_append_entries_timestamp;
Address leader_address;
std::string static ToString() { return "\tFollower \t"; }
};
using Role = std::variant<Candidate, Leader, Follower>;
template <typename Role>
concept AllRoles = memgraph::utils::SameAsAnyOf<Role, Leader, Follower, Candidate>;
template <typename Role>
concept LeaderOrFollower = memgraph::utils::SameAsAnyOf<Role, Leader, Follower>;
template <typename Role>
concept FollowerOrCandidate = memgraph::utils::SameAsAnyOf<Role, Follower, Candidate>;
/*
all ReplicatedState classes should have an Apply method
that returns our WriteResponseValue after consensus, and
a Read method that returns our ReadResponseValue without
requiring consensus.
ReadResponse Read(ReadOperation);
WriteResponseValue ReplicatedState::Apply(WriteRequest);
For example:
If the state is uint64_t, and WriteRequest is `struct PlusOne {};`,
and WriteResponseValue is also uint64_t (the new value), then
each call to state.Apply(PlusOne{}) will return the new value
after incrementing it. 0, 1, 2, 3... and this will be sent back
to the client that requested the mutation.
In practice, these mutations will usually be predicated on some
previous value, so that they are idempotent, functioning similarly
to a CAS operation.
*/
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
typename ReadResponseValue>
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
};
/// Parameter Purpose
/// --------------------------
/// IoImpl the concrete Io provider - SimulatorTransport, ThriftTransport, etc...
/// ReplicatedState the high-level data structure that is managed by the raft-backed replicated state machine
/// WriteOperation the individual operation type that is applied to the ReplicatedState in identical order
/// across each replica
/// WriteResponseValue the return value of calling ReplicatedState::Apply(WriteOperation), which is executed in
/// identical order across all replicas after an WriteOperation reaches consensus.
/// ReadOperation the type of operations that do not require consensus before executing directly
/// on a const ReplicatedState &
/// ReadResponseValue the return value of calling ReplicatedState::Read(ReadOperation), which is executed directly
/// without going through consensus first
template <typename IoImpl, typename ReplicatedState, typename WriteOperation, typename WriteResponseValue,
typename ReadOperation, typename ReadResponseValue>
requires Rsm<WriteOperation, ReadOperation, ReplicatedState, WriteResponseValue, ReadResponseValue>
class Raft {
CommonState<WriteOperation> state_;
Role role_ = Candidate{};
Io<IoImpl> io_;
std::vector<Address> peers_;
ReplicatedState replicated_state_;
Time next_cron_;
public:
Raft(Io<IoImpl> &&io, std::vector<Address> peers, ReplicatedState &&replicated_state)
: io_(std::forward<Io<IoImpl>>(io)),
peers_(peers),
replicated_state_(std::forward<ReplicatedState>(replicated_state)) {
if (peers.empty()) {
role_ = Leader{};
}
}
/// Periodic protocol maintenance. Returns the time that Cron should be called again
/// in the future.
Time Cron() {
// dispatch periodic logic based on our role to a specific Cron method.
std::optional<Role> new_role = std::visit([&](auto &role) { return Cron(role); }, role_);
if (new_role) {
role_ = std::move(new_role).value();
}
const Duration random_cron_interval = RandomTimeout(kMinimumCronInterval, kMaximumCronInterval);
return io_.Now() + random_cron_interval;
}
/// Returns the Address for our underlying Io implementation
Address GetAddress() { return io_.GetAddress(); }
using ReceiveVariant = std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse>;
void Handle(ReceiveVariant &&message_variant, RequestId request_id, Address from_address) {
// dispatch the message to a handler based on our role,
// which can be specified in the Handle first argument,
// or it can be `auto` if it's a handler for several roles
// or messages.
std::optional<Role> new_role = std::visit(
[&](auto &&msg, auto &role) mutable {
return Handle(role, std::forward<decltype(msg)>(msg), request_id, from_address);
},
std::forward<ReceiveVariant>(message_variant), role_);
// TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
if (new_role) {
role_ = std::move(new_role).value();
}
}
void Run() {
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
if (now >= next_cron_) {
next_cron_ = Cron();
}
const Duration receive_timeout = RandomTimeout(kMinimumReceiveTimeout, kMaximumReceiveTimeout);
auto request_result =
io_.template ReceiveWithTimeout<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse>(receive_timeout);
if (request_result.HasError()) {
continue;
}
auto request = std::move(request_result.GetValue());
Handle(std::move(request.message), request.request_id, request.from_address);
}
}
private:
// Raft paper - 5.3
// When the entry has been safely replicated, the leader applies the
// entry to its state machine and returns the result of that
// execution to the client.
//
// "Safely replicated" is defined as being known to be present
// on at least a majority of all peers (inclusive of the Leader).
void BumpCommitIndexAndReplyToClients(Leader &leader) {
auto confirmed_log_sizes = std::vector<LogSize>{};
// We include our own log size in the calculation of the log
// confirmed log size that is present on at least a majority of all peers.
confirmed_log_sizes.push_back(state_.log.size());
for (const auto &[addr, f] : leader.followers) {
confirmed_log_sizes.push_back(f.confirmed_log_size);
Log("Follower at port ", addr.last_known_port, " has confirmed log size of: ", f.confirmed_log_size);
}
// reverse sort from highest to lowest (using std::ranges::greater)
std::ranges::sort(confirmed_log_sizes, std::ranges::greater());
// This is a particularly correctness-critical calculation because it
// determines the committed log size that will be broadcast in
// the next AppendRequest.
//
// If the following sizes are recorded for clusters of different numbers of peers,
// these are the expected sizes that are considered to have reached consensus:
//
// state | expected value | (confirmed_log_sizes.size() / 2)
// [1] 1 (1 / 2) => 0
// [2, 1] 1 (2 / 2) => 1
// [3, 2, 1] 2 (3 / 2) => 1
// [4, 3, 2, 1] 2 (4 / 2) => 2
// [5, 4, 3, 2, 1] 3 (5 / 2) => 2
const size_t majority_index = confirmed_log_sizes.size() / 2;
const LogSize new_committed_log_size = confirmed_log_sizes[majority_index];
// We never go backwards in history.
MG_ASSERT(state_.committed_log_size <= new_committed_log_size,
"as a Leader, we have previously set our committed_log_size to {}, but our Followers have a majority "
"committed_log_size of {}",
state_.committed_log_size, new_committed_log_size);
state_.committed_log_size = new_committed_log_size;
// For each size between the old size and the new one (inclusive),
// Apply that log's WriteOperation to our replicated_state_,
// and use the specific return value of the ReplicatedState::Apply
// method (WriteResponseValue) to respond to the requester.
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const LogIndex apply_index = state_.applied_size;
const auto &write_request = state_.log[apply_index].second;
const WriteResponseValue write_return = replicated_state_.Apply(write_request);
if (leader.pending_client_requests.contains(apply_index)) {
const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
leader.pending_client_requests.erase(apply_index);
const WriteResponse<WriteResponseValue> resp{
.success = true,
.write_return = std::move(write_return),
.raft_index = apply_index,
};
io_.Send(client_request.address, client_request.request_id, std::move(resp));
}
}
Log("committed_log_size is now ", state_.committed_log_size);
}
// Raft paper - 5.1
// AppendEntries RPCs are initiated by leaders to replicate log entries and to provide a form of heartbeat
void BroadcastAppendEntries(std::map<Address, FollowerTracker> &followers) {
for (auto &[address, follower] : followers) {
const LogIndex next_index = follower.next_index;
const auto missing = state_.log.size() - next_index;
const auto batch_size = std::min(missing, kMaximumAppendBatchSize);
const auto start_index = next_index;
const auto end_index = start_index + batch_size;
// advance follower's next index
follower.next_index += batch_size;
std::vector<std::pair<Term, WriteOperation>> entries;
entries.insert(entries.begin(), state_.log.begin() + start_index, state_.log.begin() + end_index);
const Term previous_term_from_index = PreviousTermFromIndex(start_index);
Log("sending ", entries.size(), " entries to Follower ", address.last_known_port,
" which are above its next_index of ", next_index);
AppendRequest<WriteOperation> ar{
.term = state_.term,
.batch_start_log_index = start_index,
.last_log_term = previous_term_from_index,
.entries = std::move(entries),
.leader_commit = state_.committed_log_size,
};
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr RequestId request_id = 0;
io_.Send(address, request_id, std::move(ar));
}
}
// Raft paper - 5.2
// Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly
Duration RandomTimeout(Duration min, Duration max) {
std::uniform_int_distribution time_distrib(min.count(), max.count());
const auto rand_micros = io_.Rand(time_distrib);
return Duration{rand_micros};
}
Duration RandomTimeout(int min_micros, int max_micros) {
std::uniform_int_distribution time_distrib(min_micros, max_micros);
const int rand_micros = io_.Rand(time_distrib);
return std::chrono::microseconds{rand_micros};
}
Term PreviousTermFromIndex(LogIndex index) const {
if (index == 0 || state_.log.size() + 1 <= index) {
return 0;
}
const auto &[term, data] = state_.log.at(index - 1);
return term;
}
Term CommittedLogTerm() {
MG_ASSERT(state_.log.size() >= state_.committed_log_size);
if (state_.log.empty() || state_.committed_log_size == 0) {
return 0;
}
const auto &[term, data] = state_.log.at(state_.committed_log_size - 1);
return term;
}
Term LastLogTerm() const {
if (state_.log.empty()) {
return 0;
}
const auto &[term, data] = state_.log.back();
return term;
}
template <typename... Ts>
void Log(Ts &&...args) {
const Time now = io_.Now();
const auto micros = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
const Term term = state_.term;
const std::string role_string = std::visit([&](const auto &role) { return role.ToString(); }, role_);
std::ostringstream out;
out << '\t' << static_cast<int>(micros) << "\t" << term << "\t" << io_.GetAddress().last_known_port;
out << role_string;
(out << ... << args);
spdlog::info(out.str());
}
/////////////////////////////////////////////////////////////
/// Raft-related Cron methods
///
/// Cron + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Cron(role) takes as the first argument a reference to its
/// role, and as the second argument, the message that has
/// been received.
/////////////////////////////////////////////////////////////
// Raft paper - 5.2
// Candidates keep sending Vote to peers until:
// 1. receiving Append with a higher term (become Follower)
// 2. receiving Vote with a higher term (become a Follower)
// 3. receiving a quorum of responses to our last batch of Vote (become a Leader)
std::optional<Role> Cron(Candidate &candidate) {
const auto now = io_.Now();
const Duration election_timeout = RandomTimeout(kMinimumElectionTimeout, kMaximumElectionTimeout);
const auto election_timeout_us = std::chrono::duration_cast<std::chrono::milliseconds>(election_timeout).count();
if (now - candidate.election_began > election_timeout) {
state_.term++;
Log("becoming Candidate for term ", state_.term, " after leader timeout of ", election_timeout_us,
"ms elapsed since last election attempt");
const VoteRequest request{
.term = state_.term,
.log_size = state_.log.size(),
.last_log_term = LastLogTerm(),
};
auto outstanding_votes = std::set<Address>();
for (const auto &peer : peers_) {
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr auto request_id = 0;
io_.template Send<VoteRequest>(peer, request_id, request);
outstanding_votes.insert(peer);
}
return Candidate{
.successful_votes = std::map<Address, LogIndex>(),
.election_began = now,
.outstanding_votes = outstanding_votes,
};
}
return std::nullopt;
}
// Raft paper - 5.2
// Followers become candidates if we haven't heard from the leader
// after a randomized timeout.
std::optional<Role> Cron(Follower &follower) {
const auto now = io_.Now();
const auto time_since_last_append_entries = now - follower.last_received_append_entries_timestamp;
// randomized follower timeout
const Duration election_timeout = RandomTimeout(kMinimumElectionTimeout, kMaximumElectionTimeout);
if (time_since_last_append_entries > election_timeout) {
// become a Candidate if we haven't heard from the Leader after this timeout
return Candidate{};
}
return std::nullopt;
}
// Leaders (re)send AppendRequest to followers.
std::optional<Role> Cron(Leader &leader) {
const Time now = io_.Now();
const Duration broadcast_timeout = RandomTimeout(kMinimumBroadcastTimeout, kMaximumBroadcastTimeout);
if (now - leader.last_broadcast > broadcast_timeout) {
BroadcastAppendEntries(leader.followers);
leader.last_broadcast = now;
}
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// Raft-related Handle methods
///
/// Handle + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Handle(role, message, ...)
/// takes as the first argument a reference
/// to its role, and as the second argument, the
/// message that has been received.
/////////////////////////////////////////////////////////////
// all roles can receive Vote and possibly become a follower
template <AllRoles ALL>
std::optional<Role> Handle(ALL & /* variable */, VoteRequest &&req, RequestId request_id, Address from_address) {
Log("received VoteRequest from ", from_address.last_known_port, " with term ", req.term);
const bool last_log_term_dominates = req.last_log_term >= LastLogTerm();
const bool term_dominates = req.term > state_.term;
const bool log_size_dominates = req.log_size >= state_.log.size();
const bool new_leader = last_log_term_dominates && term_dominates && log_size_dominates;
if (new_leader) {
MG_ASSERT(req.term > state_.term);
MG_ASSERT(std::max(req.term, state_.term) == req.term);
}
const VoteResponse res{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
};
io_.Send(from_address, request_id, res);
if (new_leader) {
// become a follower
state_.term = req.term;
return Follower{
.last_received_append_entries_timestamp = io_.Now(),
.leader_address = from_address,
};
}
if (term_dominates) {
Log("received a vote from an inferior candidate. Becoming Candidate");
state_.term = std::max(state_.term, req.term) + 1;
return Candidate{};
}
return std::nullopt;
}
std::optional<Role> Handle(Candidate &candidate, VoteResponse &&res, RequestId /* variable */, Address from_address) {
Log("received VoteResponse");
if (!res.vote_granted || res.term != state_.term) {
Log("received unsuccessful VoteResponse from term ", res.term, " when our candidacy term is ", state_.term);
// we received a delayed VoteResponse from the past, which has to do with an election that is
// no longer valid. We can simply drop this.
return std::nullopt;
}
MG_ASSERT(candidate.outstanding_votes.contains(from_address),
"Received unexpected VoteResponse from server not present in Candidate's outstanding_votes!");
candidate.outstanding_votes.erase(from_address);
MG_ASSERT(!candidate.successful_votes.contains(from_address),
"Received unexpected VoteResponse from server already in Candidate's successful_votes!");
candidate.successful_votes.insert({from_address, res.committed_log_size});
if (candidate.successful_votes.size() >= candidate.outstanding_votes.size()) {
std::map<Address, FollowerTracker> followers{};
for (const auto &[address, committed_log_size] : candidate.successful_votes) {
FollowerTracker follower{
.next_index = committed_log_size,
.confirmed_log_size = committed_log_size,
};
followers.insert({address, follower});
}
for (const auto &address : candidate.outstanding_votes) {
FollowerTracker follower{
.next_index = state_.log.size(),
.confirmed_log_size = 0,
};
followers.insert({address, follower});
}
Log("becoming Leader at term ", state_.term);
BroadcastAppendEntries(followers);
return Leader{
.followers = std::move(followers),
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
};
}
return std::nullopt;
}
template <LeaderOrFollower LOF>
std::optional<Role> Handle(LOF & /* variable */, VoteResponse && /* variable */, RequestId /* variable */,
Address /* variable */) {
Log("non-Candidate received VoteResponse");
return std::nullopt;
}
template <AllRoles ALL>
std::optional<Role> Handle(ALL &role, AppendRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
// log size starts out as state_.committed_log_size and only if everything is successful do we
// switch it to the log length.
AppendResponse res{
.success = false,
.term = state_.term,
.last_log_term = CommittedLogTerm(),
.log_size = state_.log.size(),
};
if constexpr (std::is_same<ALL, Leader>()) {
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
}
const bool is_candidate = std::is_same<ALL, Candidate>();
const bool is_failed_competitor = is_candidate && req.term == state_.term;
const Time now = io_.Now();
// Raft paper - 5.2
// While waiting for votes, a candidate may receive an
// AppendEntries RPC from another server claiming to be leader. If
// the leaders term (included in its RPC) is at least as large as
// the candidates current term, then the candidate recognizes the
// leader as legitimate and returns to follower state.
if (req.term > state_.term || is_failed_competitor) {
// become follower of this leader, reply with our log status
state_.term = req.term;
io_.Send(from_address, request_id, res);
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
return Follower{
.last_received_append_entries_timestamp = now,
.leader_address = from_address,
};
}
if (req.term < state_.term) {
// nack this request from an old leader
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// at this point, we're dealing with our own leader
if constexpr (std::is_same<ALL, Follower>()) {
// small specialization for when we're already a Follower
MG_ASSERT(role.leader_address == from_address, "Multiple Leaders are acting under the same term number!");
role.last_received_append_entries_timestamp = now;
} else {
Log("Somehow entered Follower-specific logic as a non-Follower");
MG_ASSERT(false, "Somehow entered Follower-specific logic as a non-Follower");
}
// Handle steady-state conditions.
if (req.batch_start_log_index != state_.log.size()) {
Log("req.batch_start_log_index of ", req.batch_start_log_index, " does not match our log size of ",
state_.log.size());
} else if (req.last_log_term != LastLogTerm()) {
Log("req.last_log_term differs from our leader term at that slot, expected: ", LastLogTerm(), " but got ",
req.last_log_term);
} else {
// happy path - Apply log
Log("applying batch of ", req.entries.size(), " entries to our log starting at index ",
req.batch_start_log_index);
const auto resize_length = req.batch_start_log_index;
MG_ASSERT(resize_length >= state_.committed_log_size,
"Applied history from Leader which goes back in time from our commit_index");
// possibly chop-off stuff that was replaced by
// things with different terms (we got data that
// hasn't reached consensus yet, which is normal)
state_.log.resize(resize_length);
if (req.entries.size() > 0) {
auto &[first_term, op] = req.entries.at(0);
MG_ASSERT(LastLogTerm() <= first_term);
}
state_.log.insert(state_.log.end(), std::make_move_iterator(req.entries.begin()),
std::make_move_iterator(req.entries.end()));
MG_ASSERT(req.leader_commit >= state_.committed_log_size);
state_.committed_log_size = std::min(req.leader_commit, state_.log.size());
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const auto &write_request = state_.log[state_.applied_size].second;
replicated_state_.Apply(write_request);
}
res.success = true;
}
res.last_log_term = LastLogTerm();
res.log_size = state_.log.size();
Log("returning log_size of ", res.log_size);
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Leader &leader, AppendResponse &&res, RequestId /* variable */, Address from_address) {
if (res.term != state_.term) {
Log("received AppendResponse related to a previous term when we (presumably) were the leader");
return std::nullopt;
}
// TODO(tyler) when we have dynamic membership, this assert will become incorrect, but we should
// keep it in-place until then because it has bug finding value.
MG_ASSERT(leader.followers.contains(from_address), "received AppendResponse from unknown Follower");
// at this point, we know the term matches and we know this Follower
FollowerTracker &follower = leader.followers.at(from_address);
if (res.success) {
Log("got successful AppendResponse from ", from_address.last_known_port, " with log_size of ", res.log_size);
follower.next_index = std::max(follower.next_index, res.log_size);
} else {
Log("got unsuccessful AppendResponse from ", from_address.last_known_port, " with log_size of ", res.log_size);
follower.next_index = res.log_size;
}
follower.confirmed_log_size = std::max(follower.confirmed_log_size, res.log_size);
BumpCommitIndexAndReplyToClients(leader);
return std::nullopt;
}
template <FollowerOrCandidate FOC>
std::optional<Role> Handle(FOC & /* variable */, AppendResponse && /* variable */, RequestId /* variable */,
Address /* variable */) {
// we used to be the leader, and are getting old delayed responses
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// RSM-related handle methods
/////////////////////////////////////////////////////////////
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
Address from_address) {
Log("handling ReadOperation");
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(read_operation);
const ReadResponse<ReadResponseValue> resp{
.success = true,
.read_return = std::move(read_return),
.retry_leader = std::nullopt,
};
io_.Send(from_address, request_id, resp);
return std::nullopt;
}
// Candidates should respond with a failure, similar to the Candidate + WriteRequest failure below
std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
RequestId request_id, Address from_address) {
Log("received ReadOperation - not redirecting because no Leader is known");
const ReadResponse<ReadResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, res);
Cron();
return std::nullopt;
}
// Followers should respond with a redirection, similar to the Follower + WriteRequest response below
std::optional<Role> Handle(Follower &follower, ReadRequest<ReadOperation> && /* variable */, RequestId request_id,
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
const ReadResponse<ReadResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// Raft paper - 8
// When a client first starts up, it connects to a randomly chosen
// server. If the clients first choice is not the leader, that
// server will reject the clients request and supply information
// about the most recent leader it has heard from.
std::optional<Role> Handle(Follower &follower, WriteRequest<WriteOperation> && /* variable */, RequestId request_id,
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
const WriteResponse<WriteResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Candidate & /* variable */, WriteRequest<WriteOperation> && /* variable */,
RequestId request_id, Address from_address) {
Log("received WriteRequest - not redirecting because no Leader is known");
const WriteResponse<WriteResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, res);
Cron();
return std::nullopt;
}
// only leaders actually handle replication requests from clients
std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
Log("handling WriteRequest");
// we are the leader. add item to log and send Append to peers
MG_ASSERT(state_.term >= LastLogTerm());
state_.log.emplace_back(std::pair(state_.term, std::move(req.operation)));
LogIndex log_index = state_.log.size() - 1;
PendingClientRequest pcr{
.request_id = request_id,
.address = from_address,
.received_at = io_.Now(),
};
leader.pending_client_requests.emplace(log_index, pcr);
if (peers_.empty()) {
BumpCommitIndexAndReplyToClients(leader);
} else {
BroadcastAppendEntries(leader.followers);
}
return std::nullopt;
}
};
}; // namespace memgraph::io::rsm

136
src/io/rsm/rsm_client.hpp Normal file
View File

@@ -0,0 +1,136 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iostream>
#include <optional>
#include <vector>
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "utils/result.hpp"
namespace memgraph::io::rsm {
using memgraph::io::Address;
using memgraph::io::Duration;
using memgraph::io::ResponseEnvelope;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::Time;
using memgraph::io::TimedOut;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::ReadResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::utils::BasicResult;
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
typename ReadResponseT>
class RsmClient {
using ServerPool = std::vector<Address>;
Io<IoImpl> io_;
Address leader_;
ServerPool server_addrs_;
template <typename ResponseT>
void PossiblyRedirectLeader(const ResponseT &response) {
if (response.retry_leader) {
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
leader_ = response.retry_leader.value();
spdlog::debug("client redirected to leader server {}", leader_.ToString());
} else if (!response.success) {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = io_.Rand(addr_distrib);
leader_ = server_addrs_[addr_index];
spdlog::debug(
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
"a RSM Candidate) trying a random one at index {} with address {}",
addr_index, leader_.ToString());
}
}
public:
RsmClient(Io<IoImpl> io, Address leader, ServerPool server_addrs)
: io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
RsmClient() = delete;
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
WriteRequest<WriteRequestT> client_req;
client_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
if (response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return response_result.GetError();
}
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
if (write_response.success) {
return std::move(write_response.write_return);
}
PossiblyRedirectLeader(write_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
ReadRequest<ReadRequestT> read_req;
read_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
// receive response
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
if (get_response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return get_response_result.GetError();
}
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
if (read_get_response.success) {
return std::move(read_get_response.read_return);
}
PossiblyRedirectLeader(read_get_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
};
} // namespace memgraph::io::rsm

155
src/io/rsm/shard_rsm.hpp Normal file
View File

@@ -0,0 +1,155 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
/// The ShardRsm is a simple in-memory raft-backed kv store that can be used for simple testing
/// and implementation of some query engine logic before storage engines are fully implemented.
///
/// To implement multiple read and write commands, change the StorageRead* and StorageWrite* requests
/// and responses to a std::variant of the different options, and route them to specific handlers in
/// the ShardRsm's Read and Apply methods. Remember that Read is called immediately when the Raft
/// leader receives the request, and does not replicate anything over Raft. Apply is called only
/// AFTER the StorageWriteRequest is replicated to a majority of Raft peers, and the result of calling
/// ShardRsm::Apply(StorageWriteRequest) is returned to the client that submitted the request.
#include <deque>
#include <iostream>
#include <map>
#include <optional>
#include <set>
#include <thread>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "utils/logging.hpp"
namespace memgraph::io::rsm {
using memgraph::coordinator::Hlc;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyValue;
using ShardRsmKey = std::vector<PropertyValue>;
struct StorageWriteRequest {
LabelId label_id;
Hlc transaction_id;
ShardRsmKey key;
std::optional<int> value;
};
struct StorageWriteResponse {
bool shard_rsm_success;
std::optional<int> last_value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
struct StorageReadRequest {
LabelId label_id;
Hlc transaction_id;
ShardRsmKey key;
};
struct StorageReadResponse {
bool shard_rsm_success;
std::optional<int> value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
class ShardRsm {
std::map<ShardRsmKey, int> state_;
ShardRsmKey minimum_key_;
std::optional<ShardRsmKey> maximum_key_{std::nullopt};
Hlc shard_map_version_;
// The key is not located in this shard
bool IsKeyInRange(const ShardRsmKey &key) const {
if (maximum_key_) [[likely]] {
return (key >= minimum_key_ && key <= maximum_key_);
}
return key >= minimum_key_;
}
public:
StorageReadResponse Read(StorageReadRequest request) const {
StorageReadResponse ret;
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
} else if (state_.contains(request.key)) {
ret.value = state_.at(request.key);
ret.shard_rsm_success = true;
} else {
ret.shard_rsm_success = false;
ret.value = std::nullopt;
}
return ret;
}
StorageWriteResponse Apply(StorageWriteRequest request) {
StorageWriteResponse ret;
// Key is outside the prohibited range
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
}
// Key exist
else if (state_.contains(request.key)) {
auto &val = state_[request.key];
/*
* Delete
*/
if (!request.value) {
ret.shard_rsm_success = true;
ret.last_value = val;
state_.erase(state_.find(request.key));
}
/*
* Update
*/
// Does old_value match?
if (request.value == val) {
ret.last_value = val;
ret.shard_rsm_success = true;
val = request.value.value();
} else {
ret.last_value = val;
ret.shard_rsm_success = false;
}
}
/*
* Create
*/
else {
ret.last_value = std::nullopt;
ret.shard_rsm_success = true;
state_.emplace(request.key, request.value.value());
}
return ret;
}
};
} // namespace memgraph::io::rsm

View File

@@ -0,0 +1,8 @@
set(io_simulator_sources
simulator_handle.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-io-simulator STATIC ${io_simulator_sources})
target_link_libraries(mg-io-simulator stdc++fs Threads::Threads fmt::fmt mg-utils)

Some files were not shown because too many files have changed in this diff Show More