Compare commits

..

239 Commits

Author SHA1 Message Date
Tyler Neely
15cf7794cf Check-in initial RsmEngine class 2022-08-19 14:40:56 +00:00
Tyler Neely
d42fa1fc87 Fix ShardMap::GetShardForKey 2022-08-18 15:14:04 +00:00
Tyler Neely
2a3a3338f0 Clean up coordinator a bit 2022-08-18 14:04:28 +00:00
Tyler Neely
2320f95dd1 Update the coordinator to include request for initializing a new shard map 2022-08-18 14:01:47 +00:00
Tyler Neely
3794693356 Make shard_rsm.hpp a bit more clear 2022-08-18 13:34:11 +00:00
Tyler Neely
d50b6c1abb Move rsm_client to src/io/rsm 2022-08-18 13:20:56 +00:00
Tyler Neely
d292162b00 Use new RsmClient in raft test 2022-08-17 16:40:18 +00:00
Tyler Neely
9c4556618a Rework RsmClient and update sharded_map commit 2022-08-17 16:35:35 +00:00
Tyler Neely
f0dc0d911d Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:51:09 +00:00
Tyler Neely
e545beaf78 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:13 +00:00
Tyler Neely
3c50b68954 Small comment fix 2022-08-16 16:47:13 +00:00
Tyler Neely
fad0c893f1 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:13 +00:00
Tyler Neely
866f1feeeb Added request for edge id batch request to coordinator rsm 2022-08-16 16:47:13 +00:00
gvolfing
16d1fa2c22 Gerneral clean-up 2022-08-16 16:47:13 +00:00
gvolfing
518e8df04c Resolve issues with the sharded_map test 2022-08-16 16:47:13 +00:00
gvolfing
9c453779f5 Add the coordinator part to the sharded_map test 2022-08-16 16:47:13 +00:00
gvolfing
5d66a4e828 Check against cas_succeeded 2022-08-16 16:47:13 +00:00
gvolfing
2553a8fdc3 Remove unfinished statement 2022-08-16 16:47:13 +00:00
gvolfing
9aeca7a4b3 Add test for sharded_map 2022-08-16 16:47:13 +00:00
gvolfing
1b2c8f6b29 Move RsmClient into a separate folder and header 2022-08-16 16:47:13 +00:00
gvolfing
a74a556fcc Remove unnecessary storage pool, general cleanup 2022-08-16 16:47:13 +00:00
gvolfing
66e791d042 Make Coordinator apply splitting on request. 2022-08-16 16:47:13 +00:00
gvolfing
797c76cdfd Add logic to split shards 2022-08-16 16:47:13 +00:00
gvolfing
7af917e408 Add abstraction for RsmClient 2022-08-16 16:47:13 +00:00
gvolfing
c7282e8935 Temporarly remove the stanby server pool 2022-08-16 16:47:13 +00:00
gvolfing
4ed5801588 Change min and max key related shard logic 2022-08-16 16:47:13 +00:00
gvolfing
5963c83a60 Update Coordinator 2022-08-16 16:47:13 +00:00
gvolfing
edf1293274 Add GetShardMap read-only request to coordinator 2022-08-16 16:47:13 +00:00
gvolfing
dd46cc407f Return an error and the latest known ShardMap version if the requested key is not possibly stored in the given shard 2022-08-16 16:47:13 +00:00
Tyler Neely
92d69e080c Add todos to coordinator.hpp 2022-08-16 16:47:13 +00:00
Tyler Neely
629fd231b3 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:09 +00:00
Tyler Neely
95e90e6c2e Check-in in-progress shard test 2022-08-16 16:42:28 +00:00
Tyler Neely
502a9b4823 Check-in coordinator_rsm.hpp 2022-08-16 16:42:28 +00:00
Tyler Neely
c62b0eff93 Add Simulator::RegisterNew helper method 2022-08-16 16:42:28 +00:00
Tyler Neely
6b9311e0b8 Bump shard_rsm to use new method names 2022-08-16 16:42:28 +00:00
Tyler Neely
e3dd404865 Add new test, start to fill out coordinator RSM 2022-08-16 16:42:28 +00:00
Tyler Neely
523e2b9186 Merge branch 'T0912-MG-in-memory-shard-map' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:42:28 +00:00
Tyler Neely
79539d13c9 Use Read/Apply instead of read/apply in Rsm concept 2022-08-16 16:42:28 +00:00
Tyler Neely
6169eb221b Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:41 +02:00
Tyler Neely
342611691a Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:30 +02:00
Tyler Neely
c256dce601 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:03 +02:00
Tyler Neely
2e9cf8f37d Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:12:13 +02:00
Tyler Neely
47186cab18 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:56 +02:00
Tyler Neely
dcc5ec920a Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:41 +02:00
Tyler Neely
c7d96ed5c5 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:11 +02:00
Tyler Neely
7ad9b62968 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:10:37 +02:00
Tyler Neely
bb44aaa188 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:09:51 +02:00
Tyler Neely
9961659103 Use spdlog instead of cout 2022-08-16 14:52:03 +00:00
Tyler Neely
6f906c0488 Use spdlog instead of cout 2022-08-16 14:44:17 +00:00
Tyler Neely
d161163fb4 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-15 13:49:50 +00:00
Tyler Neely
1423da5f51 Improve comments around majority index selection 2022-08-15 11:57:19 +00:00
Tyler Neely
5e4733ac98 Add extra safety checks to commit_index management 2022-08-12 13:20:11 +00:00
Tyler Neely
14c9e68456 Transport prototype (#466) 2022-08-12 08:24:32 +02:00
Tyler Neely
cbdfd54e5f Properly apply state updates in Follower Handle for AppendRequest 2022-08-11 12:30:26 +00:00
Tyler Neely
69a2d73172 Fix bug-prone leader application of write requests to the replicated state 2022-08-11 12:17:33 +00:00
Tyler Neely
5b69f435b6 Small comment cleanup 2022-08-11 08:39:11 +00:00
Tyler Neely
f098d6b331 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-10 10:48:17 +00:00
Tyler Neely
f6edce8f0b Apply Kostas's feedback to transport interface 2022-08-10 09:34:02 +00:00
Tyler Neely
1225230c3a Apply feedback from cpplint 2022-08-09 15:52:24 +00:00
Tyler Neely
5f225a623e align cpp w/ hpp naming 2022-08-09 15:21:52 +00:00
Tyler Neely
d15105455a Clean up comments and TODOs 2022-08-09 14:07:42 +00:00
Tyler Neely
36c133ce4e Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-08 16:29:48 +00:00
Tyler Neely
91d48f05f3 Merge branch 'project-pineapples' into T0879-MG-transport-prototype 2022-08-08 13:34:30 +02:00
Tyler Neely
7bac0d1c61 Use default equality operator for Address 2022-08-08 11:32:42 +00:00
Tyler Neely
a4b1d5efb4 Update src/io/future.hpp
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-08-08 13:27:06 +02:00
Tyler Neely
0a43afdec1 Update raft implementation to use std::chrono like upstream 2022-08-04 13:28:52 +00:00
Tyler Neely
618a3d96b3 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-04 12:07:54 +00:00
jbajic
5012824e05 Address review comments 2022-08-04 11:45:16 +02:00
gvolfing
ee16954641 Add minimum and maximum key for shard rsm prototype 2022-08-04 11:20:19 +02:00
gvolfing
0aab854e34 Add shard rsm prototype 2022-08-04 10:47:00 +02:00
Jure Bajic
a12a1ea358 Move schema to storage v3 and query v2
* Move schema to storage v3

* Remove schema from v2

* Move schema to query v2

* Remove schema from query v1

* Make glue v2

* Move schema related tests to newer versions of query and storage

* Fix typo in CMake

* Fix interpreter test

* Fix clang tidy errors

* Change temp dir name
2022-08-04 09:50:02 +02:00
Tyler Neely
343648f564 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-04 07:49:09 +00:00
gvolfing
726fabd387 Constrain Raft class template with concept 2022-08-04 09:09:18 +02:00
Tyler Neely
d1c5aead61 Implement the rest of the KV-on-Raft RSM test 2022-08-03 15:07:08 +00:00
Tyler Neely
aebac2c519 Hoist read and write requests on the RSM into wrapper structs for futureproofing 2022-08-03 14:21:37 +00:00
Tyler Neely
74b354979c Check in incremental progress on fixing compilation error 2022-08-03 13:42:08 +00:00
gvolfing
b5cff5999b Unfinished rsm changes 2022-08-03 14:39:09 +02:00
Tyler Neely
b2a8063a96 Make more temporaries const 2022-08-02 14:33:05 +00:00
Tyler Neely
54369958d1 Extract BlockedServers functionality from MaybeTickSimulator and IncrementServerCountAndWaitForQuiescentState 2022-08-02 14:30:12 +00:00
Tyler Neely
4f06eb0f2f Address rule-of-five for abstract inhertance in OpaquePromise 2022-08-02 14:24:59 +00:00
Tyler Neely
ad0a8c4942 Use simple virtual inheritance instead of bespoke vtable for OpaquePromise 2022-08-02 13:36:40 +00:00
Tyler Neely
997fdf5a16 Check-in skeleton for RSM logic on top of Raft 2022-08-02 10:18:38 +00:00
Tyler Neely
8be88deee6 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-02 08:48:20 +00:00
Tyler Neely
902a46d14f Make dynamic message Take messages rvalue methods 2022-08-02 07:12:05 +00:00
Tyler Neely
3b44ef70b6 Denest operator< for clarity 2022-08-02 07:06:43 +00:00
Tyler Neely
4d8f9ea821 Remove unnecessary comment 2022-08-02 07:03:12 +00:00
Tyler Neely
e935a9a7b1 Make parenthesization less confusing 2022-08-02 07:02:17 +00:00
Tyler Neely
dd93b594bc Remove dead code 2022-08-02 07:01:25 +00:00
Tyler Neely
997d25d536 Remove buggy usage of std::forward 2022-08-01 15:53:37 +00:00
Tyler Neely
ace5f2b639 Complete clang-tidy cleanup 2022-08-01 15:44:34 +00:00
Tyler Neely
9576eea051 Address some feedback from clang-tidy 2022-08-01 15:37:16 +00:00
Tyler Neely
5bb3361a2d Use std::chrono::microseconds explicitly for Duration. Fix compiler warning related to timeouts 2022-08-01 14:32:07 +00:00
gvolfing
a84f5f6115 Undo the structured binding changes 2022-08-01 16:13:29 +02:00
Tyler Neely
cacb0dac80 Avoid unnamed namespace in future.hpp 2022-08-01 14:12:07 +00:00
gvolfing
918fa7212e Further improve const-correctness and replace dispensable structured bindings with exact return values 2022-08-01 15:30:44 +02:00
gvolfing
e905591372 Add references to the raft paper 2022-08-01 15:17:29 +02:00
Tyler Neely
649b5437b0 Improve docs around Io interface 2022-08-01 12:58:55 +00:00
Tyler Neely
102d997288 Update src/io/simulator/simulator_handle.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-01 14:52:06 +02:00
Tyler Neely
a3f3e05fc2 Include replier address in operator< for PromiseKey 2022-08-01 12:50:48 +00:00
Tyler Neely
4f4eb9ea13 Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-01 14:47:13 +02:00
Tyler Neely
ca638db509 Remove extra blank line 2022-08-01 12:45:45 +00:00
Tyler Neely
7d33bb1937 Update Future unit test to use gtest 2022-08-01 12:42:37 +00:00
Tyler Neely
b8487da392 Move future test to unit tests 2022-08-01 11:58:54 +00:00
Tyler Neely
dbd744470b Add future benchmark 2022-08-01 11:48:21 +00:00
Tyler Neely
b2b11f3a30 Run simulation tests in CI 2022-08-01 11:24:21 +00:00
Tyler Neely
eb1b6c3ac8 Make SimulatorHandle::TimeoutPromisesPastDeadline private 2022-08-01 11:10:00 +00:00
Tyler Neely
0c2cbb5461 Make a couple more things const 2022-08-01 11:08:17 +00:00
Tyler Neely
69ea79a75e Make a couple more things const 2022-08-01 11:07:39 +00:00
Tyler Neely
c0d6cec9ab Split simulator_handle.cpp into a source and header file 2022-08-01 11:06:08 +00:00
Tyler Neely
9b915be1aa Make some methods const. Remove outdated TODO 2022-08-01 10:30:37 +00:00
Tyler Neely
b127f6f345 Use spdlog and create a formatter for Address 2022-08-01 10:23:25 +00:00
jbajic
f57f30c8cf Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-08-01 10:46:11 +02:00
Tyler Neely
66ef5b2072 Make certain temporaries const 2022-08-01 08:26:09 +00:00
Tyler Neely
f877f8e1d3 Add some requests and additional metadata around coordinator operations 2022-07-29 14:40:08 +00:00
gvolfing
0ea9878fd1 CTAD 2022-07-29 14:27:55 +02:00
gvolfing
af94270dc6 Remove redundant else clauses when returning from every branch 2022-07-29 14:22:57 +02:00
gvolfing
8aac3ae7ea Make unchanged variables const 2022-07-29 14:08:37 +02:00
Tyler Neely
507018a630 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-07-29 11:50:21 +00:00
Tyler Neely
e0bbf4766a Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-29 11:47:50 +00:00
Tyler Neely
1a48e0ffa8 Merge github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-07-29 11:46:54 +00:00
Jure Bajic
462daf3a2b Enforce schema on vertex creation
- Separating schema definition from schema validation
- Updating vertex_accessor and db_accessors with necessary methods
- Adding a primary label to Vertex
- Adding schema tests
- Updating existing tests for storage v3, and deprecating old:
  - interpreter => interpreter_v2
  - query_plan_accumulate_aggregate => storage_v3_query_plan_accumulate_aggregate
  - query_plan_create_set_remove_delete => storage_v3_query_plan_create_set_remove_delete
  - query_plan_bag_semantics => storage_v3_query_plan_bag_semantics
  - query_plan_edge_cases => storage_v3_query_plan_edge_cases
  - query_plan_v2_create_set_remove_delete => storage_v3_query_plan_v2_create_set_remove_delete
  - query_plan_match_filter_return => storage_v3_query_plan_match_filter_return
2022-07-29 13:38:17 +02:00
Tyler Neely
6a808ce1fc Delete empty line 2022-07-28 16:27:57 +00:00
Tyler Neely
a89a2d9caa Update src/io/simulator/simulator_transport.hpp
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 18:26:36 +02:00
Tyler Neely
c4f1764fee Rely on default initialization of random number generators 2022-07-28 16:24:45 +00:00
Tyler Neely
48a445f2ed Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-28 16:24:34 +00:00
Tyler Neely
406da4b25c Use uint8_t instead of int for SimulatorConfig::drop_percent 2022-07-28 16:23:10 +00:00
Tyler Neely
8fde05444d Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-28 18:13:50 +02:00
Tyler Neely
6bdfb43ad0 Use rvalue reference requirement for Future::Wait 2022-07-28 16:07:20 +00:00
Tyler Neely
9056e2c97a Make Address's == and < operators consider uuid 2022-07-28 16:06:48 +00:00
Marko Budiselic
2009fefc8a Merge branch 'master' into project-pineapples 2022-07-28 15:36:51 +02:00
Marko Budiselić
eb3f96d1f6 Bring changes from master to project-pineapples (#477)
* Fix aggregation functions on `null` and group-by inputs (#448)
* Upgrade Antrl to 4.10.1 and remove antlr_lock (#441)
* Update clang-tidy job (#476)
* Add parser stress test (#463)

NOTE: Doing this to have buildable comments on the project-pineapples branch

Co-authored-by: gvolfing <107616712+gvolfing@users.noreply.github.com>
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 15:36:17 +02:00
Tyler Neely
51371398ce Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:27:11 +02:00
Tyler Neely
b38dc28e01 Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:26:00 +02:00
Tyler Neely
6cc550719b Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:25:25 +02:00
Tyler Neely
ef70b858e2 Centralize Future's Shared state taking logic. Fix a race condition in quiescence tracking 2022-07-27 15:24:33 +00:00
Tyler Neely
509c12956c Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:09:32 +02:00
Tyler Neely
1527509e36 Use std::chrono instead of raw uint64_t 2022-07-27 15:07:01 +00:00
Tyler Neely
0cfb68bb89 Update src/io/simulator/simulator_handle.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 16:56:14 +02:00
Tyler Neely
e2968c2e21 Use std::chrono instead of raw uint64_t 2022-07-27 14:55:39 +00:00
Tyler Neely
4ee4612a9c Remove removed raft.cpp from CMakeLists.txt for the simulator tests 2022-07-27 14:06:16 +00:00
Tyler Neely
dc38296575 avoid unnecessary std::optional around std::function which can be nullable 2022-07-27 14:05:57 +00:00
Marko Budiselić
80e0e439b7 Add parser stress test (#463) 2022-07-26 20:54:56 +02:00
Tyler Neely
06962a3ec4 Check shard map version 2022-07-26 15:35:37 +00:00
Tyler Neely
c0b0b08d12 Check-in deregistration and shard split things 2022-07-26 15:32:22 +00:00
Tyler Neely
c100a86644 Check-in changes from hangout 2022-07-26 15:23:55 +00:00
Jure Bajic
351258ace8 Update clang-tidy job (#476)
Update clang tidy job to only check the diff against the base branch
2022-07-26 16:53:56 +02:00
Tyler Neely
3ec1ff9ee4 Commit small changes from meeting 2022-07-26 11:49:08 +00:00
Marko Budiselić
74d3663821 Upgrade Antrl to 4.10.1 and remove antlr_lock (#441) 2022-07-26 08:31:38 +02:00
Tyler Neely
98206caf85 Separate split functionality from ShardMap 2022-07-25 10:38:50 +00:00
Tyler Neely
962767ea1c Initial check-in of shard map ideas 2022-07-25 10:34:27 +00:00
Tyler Neely
2a199c9484 Merge branch 'master' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-22 12:14:18 +00:00
Tyler Neely
a85e9fcdd4 Remove raft-related code from transport prototype branch 2022-07-22 12:11:00 +00:00
jbajic
264b233053 Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-07-22 11:48:45 +02:00
Kostas
72b4337864 Added ScanVertices request/response example 2022-07-21 18:37:06 +03:00
János Benjamin Antal
ed71332773 Add dummy storage engine to test 2022-07-21 16:23:27 +02:00
Tyler Neely
fd3d70d847 Move raft-related code to replicated state machine (rsm) namespace 2022-07-21 12:41:02 +00:00
Tyler Neely
f8e5032011 Restructure files into namespaces and subdirectories 2022-07-21 12:10:23 +00:00
Tyler Neely
5b59d890c0 Remove awkward Reply sugar for now 2022-07-21 11:53:51 +00:00
Tyler Neely
0351db2461 Move future.hpp to src/io 2022-07-21 11:45:05 +00:00
Tyler Neely
c379475e12 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-21 11:01:04 +00:00
Tyler Neely
9c5d19bc19 Start to adapt the transport-related code to use namespaces 2022-07-21 11:00:11 +00:00
Tyler Neely
4d85a7e605 Check-in simulator_transport.hpp 2022-07-21 10:49:33 +00:00
Tyler Neely
f6c2202772 Use anonymous namespace to avoid friend declarations in future.hpp 2022-07-21 10:49:19 +00:00
Tyler Neely
498ae97ae9 Clean up log messages 2022-07-21 10:26:08 +00:00
Tyler Neely
fbd015d3c6 Continue codebase clean-up in preparation for merge 2022-07-21 10:24:13 +00:00
Tyler Neely
1ef11a36f4 Apply feedback from cpplint 2022-07-21 09:33:06 +00:00
Tyler Neely
9ae1671e4f Fix bug with MgFuture where its safety check was not correctly initialized 2022-07-20 18:17:06 +00:00
Tyler Neely
b4afe45de5 Handle Request timeouts properly in the simulator 2022-07-20 17:18:07 +00:00
Tyler Neely
581925e660 Fix a few bugs, expose stats 2022-07-20 17:07:18 +00:00
Tyler Neely
afef6dc11b A large batch of bug fixes and clean-ups 2022-07-20 13:18:00 +00:00
Tyler Neely
689336e765 Fix several bugs, clean-ups and tuning 2022-07-19 14:57:01 +00:00
Jure Bajic
2ceaf59767 Create query engine v2 (#444)
Create version v2 of the query engine.
Adjust CMake and lisp files
Connect query engine v2 with storage engine v3
2022-07-19 12:28:19 +02:00
gvolfing
eb0b3141d5 Fix aggregation functions on null and group-by inputs (#448)
The `sum()` and `count()` functions were giving results different from the openCypher specification on null `input.` The aggregation functions also had a problem when they were used in a group-by context and were giving results that were not compliant with the openCypher specification.
2022-07-19 12:16:32 +02:00
Marko Budiselic
5dd0ddc352 Killing bugs 2022-07-18 11:36:56 +02:00
János Benjamin Antal
c0bee760bf git 2022-07-18 08:21:04 +02:00
Marko Budiselic
1480d975a7 Fix the basic_request example, something sill broken 2022-07-17 17:22:39 +02:00
Tyler Neely
9dc37a87f7 Enable more aggressive message scrambling to get a race condition to jump out about 0.2% of the time 2022-07-15 16:46:54 +00:00
Tyler Neely
46a2879ece Fix a number of bugs in raft, and it can now go for around 1400 simulations before hitting a race condition 2022-07-15 16:44:42 +00:00
Tyler Neely
b2142e8d38 Remove unnecessary log message 2022-07-15 16:20:05 +00:00
Tyler Neely
0dc69c180d Start cranking up the testing intensity and fix a variety of correctness and code quality issues 2022-07-15 16:19:00 +00:00
Marko Budiselić
ff2f8031a9 Extend the PR template with the git message guide (#451) 2022-07-15 14:08:10 +02:00
Tyler Neely
ad1d8637e5 Fix race condition in simulator. Make leader election function more reliably in raft code 2022-07-15 09:07:39 +00:00
Tyler Neely
4876b7cd8c Start removing load-bearing sleep statements to get race conditions to jump out in the simulator 2022-07-15 05:43:40 +00:00
Tyler Neely
a6133dab49 Start to tune various timeouts in the raft implementation 2022-07-15 05:38:04 +00:00
Tyler Neely
3466c15f76 Make raft work in the happy path 2022-07-14 19:57:52 +00:00
Tyler Neely
63beeb8771 Send log state along with AppendRequest messages from Leader 2022-07-14 19:06:56 +00:00
Tyler Neely
d14f7705b1 Get most of the raft data path up and running 2022-07-14 18:45:24 +00:00
Tyler Neely
5e98971bb2 Continue filling out AppendEntries state transition logic 2022-07-14 13:06:54 +00:00
Tyler Neely
dc78adde40 Move empty in-flight early exit of simulator tick to before time advancement 2022-07-14 13:06:22 +00:00
Tyler Neely
80970a97f0 Get basic leader election to work and clean some things up 2022-07-14 12:18:09 +00:00
Tyler Neely
57533f2746 A large number of scattered cleanups. Don't use transport Request in state machine code 2022-07-14 10:50:49 +00:00
Tyler Neely
1a12a80af0 Continue to implement raft state machine transitions 2022-07-14 07:09:50 +00:00
Tyler Neely
30c97e658d Implement some more of the raft state machine transitions 2022-07-13 22:31:51 +00:00
Tyler Neely
dd9862d32c Add support for scrambling messages to the simulator 2022-07-13 19:59:23 +00:00
Tyler Neely
dc6548c996 Add support to the simulator to drop messages randomly 2022-07-13 19:51:04 +00:00
Marko Budiselić
094d4f282d Improve error-prone OS setup (#428) 2022-07-13 17:48:01 +02:00
Tyler Neely
881e914b92 Add random number generator support to the IO interface 2022-07-13 13:37:38 +00:00
Tyler Neely
48ee40ce87 Start to fill out more raft logic now that role-specialized Handle and Cron are in place 2022-07-13 06:50:16 +00:00
Tyler Neely
d6742f643c Significant simplification of event-handling logic by using std::visit while matching on both the message type and node role 2022-07-12 19:04:32 +00:00
Tyler Neely
d8f09b59b3 Add MgFuture::Cancel to signal that a future is intentionally dropped 2022-07-12 19:03:12 +00:00
Jure Bajic
3dd2657320 Create mgp python package (#433) 2022-07-12 10:54:23 +02:00
Tyler Neely
2e5d8b7e8c Check-in in-progress work for raft test on top of simulator 2022-07-11 18:01:13 +00:00
Tyler Neely
88eee66258 Small cleanup of basic_request test 2022-07-11 18:00:30 +00:00
Tyler Neely
d4cb259979 Fix issue with recursive parameter pack std::variant conversion of std::any. Improve documentation of the complex functionality 2022-07-11 17:59:52 +00:00
Tyler Neely
1a87dd2497 Add two methods to MgFuture: IsReady and TryGet 2022-07-11 17:59:08 +00:00
gvolfing
6fe474282a Modify logaical operators to conform openCyper regarding checking against NULL in CASE expressions (#432)
* Make `IfOperator` return the `else_expression_` in case of `NULL`

* Add gql_behave tests

* Add gql_behave test to specifically check for the case when the test expression itself is null
2022-07-11 15:00:29 +02:00
gvolfing
7fc0fb6520 Implement ToString function for temporal datatypes (#429)
* Modify `toString` to be able to handle `Date`, `LocalTime`, `LocalDateTime` and `Duration`

* Add unit tests

* Make `operator<<` use the `ToString()` implementations

* Add tests to verify the correctness of negative durations

* Add more tests to look for cases when the individual duration entities overflow.
2022-07-11 13:44:27 +02:00
Jure Bajic
3f4f66b57f Create schema DDL expressions
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Add schema operations in storage

* Add create and show schema queries

* Add privileges for schema

* Add missing keywords into lexer

* Add drop schema query

* Add schema visitors

* Update metadata

* Add PrepareSchemaQuery function

* Implement show schemas

* Add show schema query

* Fix schema visitor

* Add common schema type

* Fix grammar

* Temporary create ddl logic

* Fix naming for schemaproperty type to schema type

* Rename schemaproperty to schemapropertytype

* Enable Create schema ddl

* Override visitPropertyType

* Add initial schema implementation

* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

* Add verification on creation and deletion

* Rename DeleteSchema to DropSchema

* Remove list and map from lexer

* Fix grammar with schemaTypeMap

* Add privilege and cypher visitor tests

* Catch repeating type name in schema definition

* Fix conflicting keywords

* Add notifications

* Drop float support

* Finish interpreter tests

* Fix tests

* Fix clang tidy errors

* Fix GetSchema

* Replace for with transfrom

* Add cloning og schema_property_map

* Address review comments

* Rename SchemaPropertyType to SchemaType

* Remove inline

* Assert of schema properties

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-11 09:20:15 +02:00
Tyler Neely
15d637729e Clean-up basic_request example 2022-07-08 14:10:33 +00:00
Tyler Neely
5ee95a2e70 Add ShutDown method to Simulator and update basic_request test to utilize it 2022-07-08 14:05:02 +00:00
Tyler Neely
597a5d191c Check-in SimulatorConfig 2022-07-08 11:49:29 +00:00
Tyler Neely
eee2a7e019 Fix memory leak caught by LSAN 2022-07-08 11:04:44 +00:00
Jeremy B
063e297e1e Avoid usage of time.sleep (#434)
e2e python: added tooling function around `time.sleep()` that stops as soon as condition is fulfilled and will raise assert if timeout is reached
2022-07-08 10:47:18 +02:00
Jure Bajic
2998f92595 Add initial schema implementation
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-08 10:33:43 +02:00
Tyler Neely
25b0a445b9 Properly fill response promises in the simulator 2022-07-08 08:21:37 +00:00
Tyler Neely
20a6dae047 Get the message receive path up and running. Large number of small refactors 2022-07-07 20:58:16 +00:00
Ante Javor
86b1688192 Rewrite Python API comments and snippets (#420)
* Update comments
2022-07-07 15:05:56 +02:00
Jeremy B
f629de7e60 Save replication settings (#415)
* Storage takes care of the saving of setting when a new replica is added

* Restore replicas at startup

* Modify interactive_mg_runner + memgraph to support that data-directory can be configured in CONTEXT

* Extend e2e test

* Correct typo

* Add flag to config to specify when replication should be stored (true by default when starting Memgraph)

* Remove un-necessary "--" in yaml file

* Make sure Memgraph stops if a replica can't be restored.

* Add UT covering the parsing  of ReplicaStatus to/from json

* Add assert in e2e script to check that a port is free before using it

* Add test covering crash on Jepsen

* Make sure applciaiton crashes if it starts on corrupted replications' info

Starting with a non-reponsive replica is allowed.

* Add temporary startup flag: this is needed so jepsen do not automatically restore replica on startup of main. This will be removed in T0835
2022-07-07 13:30:28 +02:00
Tyler Neely
04dbedc3af Add functionality for converting from std::any to a std::variant. Improve type names. Continue simulator implementation 2022-07-07 11:27:02 +00:00
Tyler Neely
a0058bc10a Add OpaquePromise type with correct dtor. Make a few structs comparable for storage in maps. Fix initialization of Shared 2022-07-06 15:19:16 +00:00
Tyler Neely
f601e83f6f Continue structuring the simulator and filling in its implementation 2022-07-05 21:11:39 +00:00
Tyler Neely
e1aab7065f Use TSAN on futures test. Continue implementation of simulator 2022-07-05 18:02:01 +00:00
Tyler Neely
28516763b9 Fix issue with movement/consumption tracking in MgFuture and MgPromise 2022-07-05 17:36:47 +00:00
Tyler Neely
29e8d8e72c Fix cmake config for new transport 2022-07-05 16:04:03 +00:00
Tyler Neely
6debc9e7d8 Revamp MgFuture, continue threading logic through Simulator 2022-07-05 15:45:59 +00:00
Jeremy B
b737e53456 Remove sync with timeout (#423)
* Remove timout when registering a sync replica

* Simplify jepsen configuration file

* Remove timeout from jepsen configuration

* Add unit test

* Remove TimeoutDispatcher
2022-07-05 09:40:50 +02:00
János Benjamin Antal
1bdc32ba5d Copy storage v2 to create storage v3 (#416)
* Copy storage v2 to v3

* Integrate v3 to cmake

* Fix clang-tidy warnings

* Add dummy unit test for storage-v3 to trigger build for code analysis builds
2022-07-05 08:20:59 +02:00
Tyler Neely
eb4ca543ea Continue to implement the simulated transport 2022-07-04 21:42:10 +00:00
Tyler Neely
38ca430713 Small refactor of cmake for simulator test 2022-07-04 20:44:51 +00:00
Tyler Neely
24128e0bca Check-in new test 2022-07-04 19:42:23 +00:00
Tyler Neely
73719b2120 Continue to bolt-down MgFuture impl 2022-07-04 18:54:09 +00:00
Tyler Neely
20839b0ae0 Add simple test for MgFuture 2022-07-04 15:00:32 +00:00
János Benjamin Antal
10ca68bb2a Remove CODEOWNERS (#427)
Co-authored-by: Marko Budiselić <marko.budiselic@memgraph.com>
2022-07-04 16:42:36 +02:00
Jure Bajic
bfbd8538d4 Update docker release process (#421)
* Fix release directory

* Update release process

* Fix debian arm path
2022-07-04 16:10:33 +02:00
Tyler Neely
6cec9acbb9 Add basic test skeleton 2022-07-04 11:56:41 +00:00
Tyler Neely
cb70431301 Reset optional after moving its value out in Future::Wait 2022-07-04 11:55:18 +00:00
Tyler Neely
0ef1f7eb5d Apply Rule-Of-Five and uniform CamelCase function names 2022-07-01 08:45:19 +00:00
Tyler Neely
4140f3e05e [SQUASH ME] Initial check-in for getting feedback 2022-06-29 15:52:37 +00:00
János Benjamin Antal
21870a0e7e Merge branch 'master' into project-pineapples 2022-06-23 14:49:14 +02:00
378 changed files with 81891 additions and 4164 deletions

View File

@@ -6,6 +6,7 @@ Checks: '*,
-altera-unroll-loops,
-android-*,
-cert-err58-cpp,
-cert-str34-c,
-cppcoreguidelines-avoid-c-arrays,
-cppcoreguidelines-avoid-goto,
-cppcoreguidelines-avoid-magic-numbers,
@@ -49,6 +50,7 @@ Checks: '*,
-misc-non-private-member-variables-in-classes,
-modernize-avoid-c-arrays,
-modernize-concat-nested-namespaces,
-modernize-loop-convert,
-modernize-pass-by-value,
-modernize-use-equals-default,
-modernize-use-nodiscard,

View File

@@ -3,7 +3,9 @@
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Write E2E tests
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
- [ ] Provide the full content or a guide for the final git message
[master < Task] PR
- [ ] Check, and update documentation if necessary
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Provide the full content or a guide for the final git message

View File

@@ -70,6 +70,11 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
# This is also needed if we want do to comparison against other branches
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
- name: Fetch all history for all tags and branches
run: git fetch
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
@@ -110,12 +115,22 @@ jobs:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
@@ -156,7 +171,7 @@ jobs:
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
- name: Run drivers tests
run: |
@@ -247,6 +262,15 @@ jobs:
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run simulation tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__simulation --output-on-failure -j$THREADS
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.

View File

@@ -174,5 +174,5 @@ jobs:
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
name: debian-11-arm
path: build/output/debian-11-arm/memgraph*.deb

View File

@@ -4,8 +4,12 @@ on:
workflow_dispatch:
inputs:
version:
description: "Memgraph binary version to publish on Dockerhub."
description: "Memgraph binary version to publish on DockerHub."
required: true
force_release:
type: boolean
required: false
default: false
jobs:
docker_publish:
@@ -36,6 +40,22 @@ jobs:
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
- name: Check if specified version is already pushed
run: |
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
echo $EXISTS
if [[ ${EXISTS} -eq 0 ]]; then
echo 'The specified version has been already released to DockerHub.'
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
echo 'Forcing the release!'
else
echo 'Stopping the release!'
exit 1
fi
else
echo 'All good the specified version has not been release to DockerHub.'
fi
- name: Build & push docker images
run: |
cd release/docker

9
.gitignore vendored
View File

@@ -9,7 +9,6 @@
*.swn
*.swo
*.swp
*~
.DS_Store
.gdb_history
@@ -24,10 +23,10 @@ cmake-build-*
cmake/DownloadProject/
dist/
src/query/frontend/opencypher/generated/
src/query/v2/frontend/opencypher/generated/
tags
ve/
ve3/
.cache/
perf.data*
TAGS
*.apollo_measurements
@@ -52,15 +51,21 @@ src/distributed/pull_produce_rpc_messages.hpp
src/distributed/storage_gc_rpc_messages.hpp
src/distributed/token_sharing_rpc_messages.hpp
src/distributed/updates_rpc_messages.hpp
src/query/v2/frontend/ast/ast.hpp
src/query/frontend/ast/ast.hpp
src/query/distributed/frontend/ast/ast_serialization.hpp
src/query/v2/distributed/frontend/ast/ast_serialization.hpp
src/durability/distributed/state_delta.hpp
src/durability/single_node/state_delta.hpp
src/durability/single_node_ha/state_delta.hpp
src/query/frontend/semantic/symbol.hpp
src/query/v2/frontend/semantic/symbol.hpp
src/query/distributed/frontend/semantic/symbol_serialization.hpp
src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
src/query/distributed/plan/ops.hpp
src/query/v2/distributed/plan/ops.hpp
src/query/plan/operator.hpp
src/query/v2/plan/operator.hpp
src/raft/log_entry.hpp
src/raft/raft_rpc_messages.hpp
src/raft/snapshot_metadata.hpp

View File

@@ -1 +0,0 @@
* @antaljanosbenjamin @kostasrim

View File

@@ -18,16 +18,14 @@ WIDTH = 80
def wrap_text(s, initial_indent="# "):
return "\n#\n".join(
map(
lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "),
s.split("\n"),
)
)
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent,
subsequent_indent="# "), s.split("\n")))
def extract_flags(binary_path):
ret = {}
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
data = subprocess.run([binary_path, "--help-xml"],
stdout=subprocess.PIPE).stdout.decode("utf-8")
root = ET.fromstring(data)
for child in root:
if child.tag == "usage" and child.text.lower().count("warning"):
@@ -48,7 +46,8 @@ def apply_config_to_flags(config, flags):
for modification in config["modifications"]:
name = modification["name"]
if name not in flags:
print("WARNING: Flag '" + name + "' missing from binary!", file=sys.stderr)
print("WARNING: Flag '" + name + "' missing from binary!",
file=sys.stderr)
continue
flags[name]["default"] = modification["value"]
flags[name]["override"] = modification["override"]
@@ -76,9 +75,8 @@ def extract_sections(flags):
else:
sections.append((current_section, current_flags))
sections.append(("other", other))
assert set(sum(map(lambda x: x[1], sections), [])) == set(
flags.keys()
), "The section extraction algorithm lost some flags!"
assert set(sum(map(lambda x: x[1], sections), [])) == set(flags.keys()), \
"The section extraction algorithm lost some flags!"
return sections
@@ -91,7 +89,8 @@ def generate_config_file(sections, flags):
helpstr = flag["meaning"] + " [" + flag["type"] + "]"
ret += wrap_text(helpstr) + "\n"
prefix = "# " if not flag["override"] else ""
ret += prefix + "--" + flag["name"].replace("_", "-") + "=" + flag["default"] + "\n\n"
ret += prefix + "--" + flag["name"].replace("_", "-") + \
"=" + flag["default"] + "\n\n"
ret += "\n"
ret += wrap_text(config["footer"])
return ret.strip() + "\n"
@@ -99,16 +98,13 @@ def generate_config_file(sections, flags):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("memgraph_binary", help="path to Memgraph binary")
parser.add_argument(
"output_file",
help="path where to store the generated Memgraph " "configuration file",
)
parser.add_argument(
"--config-file",
default=CONFIG_FILE,
help="path to generator configuration file",
)
parser.add_argument("memgraph_binary",
help="path to Memgraph binary")
parser.add_argument("output_file",
help="path where to store the generated Memgraph "
"configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE,
help="path to generator configuration file")
args = parser.parse_args()
flags = extract_flags(args.memgraph_binary)

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "centos-7"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "centos-9"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-10"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "arm64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -5,6 +5,8 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "todo-os-name"
TOOLCHAIN_BUILD_DEPS=(
pkg
)

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-18.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # archive download

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-20.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-22.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -5,10 +5,28 @@ operating_system() {
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
}
check_operating_system() {
if [ "$(operating_system)" != "$1" ]; then
echo "Not the right operating system!"
exit 1
else
echo "The right operating system."
fi
}
architecture() {
uname -m
}
check_architecture() {
if [ "$(architecture)" != "$1" ]; then
echo "Not the right architecture!"
exit 1
else
echo "The right architecture."
fi
}
check_all_yum() {
local missing=""
for pkg in $1; do

File diff suppressed because it is too large Load Diff

3
init
View File

@@ -139,3 +139,6 @@ done;
# Install precommit hook
python3 -m pip install pre-commit
python3 -m pre_commit install
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

1
libs/.gitignore vendored
View File

@@ -5,3 +5,4 @@
!CMakeLists.txt
!__main.cpp
!pulsar.patch
!antlr4.10.1.patch

View File

@@ -106,6 +106,7 @@ import_external_library(antlr4 STATIC
-DWITH_LIBCXX=OFF # because of debian bug
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
-DCMAKE_CXX_STANDARD=20
-DANTLR_BUILD_CPP_TESTS=OFF
BUILD_COMMAND $(MAKE) antlr4_static
INSTALL_COMMAND $(MAKE) install)

13
libs/antlr4.10.1.patch Normal file
View File

@@ -0,0 +1,13 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index baf46cac9..2e7756de8 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -134,7 +134,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
EXPORT antlr4-targets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

View File

@@ -1,43 +0,0 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index a8503bb..11362cf 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
ExternalProject_Add(
utfcpp
- GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
- GIT_TAG "v3.1.1"
+ GIT_REPOSITORY "https://github.com/nemtrif/utfcpp"
+ GIT_TAG "v3.2.1"
SOURCE_DIR ${UTFCPP_DIR}
UPDATE_DISCONNECTED 1
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
DESTINATION lib
EXPORT antlr4-targets)
install(TARGETS antlr4_static
diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
index 468db98..65a473b 100644
--- a/runtime/Cpp/runtime/src/support/Any.h
+++ b/runtime/Cpp/runtime/src/support/Any.h
@@ -122,12 +122,12 @@ private:
}
private:
- template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return new Derived<T>(value);
}
- template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return nullptr;
}

View File

@@ -105,7 +105,7 @@ repo_clone_try_double () {
# Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.10.1-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
@@ -130,7 +130,7 @@ declare -A primary_urls=(
# should fail.
declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
["antlr4-generator"]="https://www.antlr.org/download/antlr-4.10.1-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
@@ -152,10 +152,10 @@ declare -A secondary_urls=(
# antlr
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="4.9.2" # v4.9.2
antlr4_tag="4.10.1" # v4.10.1
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
pushd antlr4
git apply ../antlr4.patch
git apply ../antlr4.10.1.patch
popd
# cppitertools v2.0 2019-12-23
@@ -199,7 +199,7 @@ git apply ../rocksdb.patch
popd
# mgclient
mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08)
mgclient_tag="v1.4.0" # (2022-06-14)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt

View File

@@ -7,11 +7,13 @@ import copy
@mgp.read_proc
def procedure(
context: mgp.ProcCtx,
required_arg: mgp.Nullable[mgp.Any],
optional_arg: mgp.Nullable[mgp.Any] = None,
) -> mgp.Record(args=list, vertex_count=int, avg_degree=mgp.Number, props=mgp.Nullable[mgp.Map]):
def procedure(context: mgp.ProcCtx,
required_arg: mgp.Nullable[mgp.Any],
optional_arg: mgp.Nullable[mgp.Any] = None
) -> mgp.Record(args=list,
vertex_count=int,
avg_degree=mgp.Number,
props=mgp.Nullable[mgp.Map]):
"""
This example procedure returns 4 fields.
@@ -35,7 +37,7 @@ def procedure(
if isinstance(required_arg, (mgp.Edge, mgp.Vertex)):
props = dict(required_arg.properties.items())
elif isinstance(required_arg, mgp.Path):
(start_vertex,) = required_arg.vertices
start_vertex, = required_arg.vertices
props = dict(start_vertex.properties.items())
# Count the vertices and edges in the database; this may take a while.
vertex_count = 0
@@ -49,13 +51,15 @@ def procedure(
# Copy the received arguments to make it equivalent to the C example.
args_copy = [copy.deepcopy(required_arg), copy.deepcopy(optional_arg)]
# Multiple rows can be produced by returning an iterable of mgp.Record.
return mgp.Record(args=args_copy, vertex_count=vertex_count, avg_degree=avg_degree, props=props)
return mgp.Record(args=args_copy, vertex_count=vertex_count,
avg_degree=avg_degree, props=props)
@mgp.write_proc
def write_procedure(
context: mgp.ProcCtx, property_name: str, property_value: mgp.Nullable[mgp.Any]
) -> mgp.Record(created_vertex=mgp.Vertex):
def write_procedure(context: mgp.ProcCtx,
property_name: str,
property_value: mgp.Nullable[mgp.Any]
) -> mgp.Record(created_vertex=mgp.Vertex):
"""
This example procedure creates a new vertex with the specified property
and connects it to all existing vertex which has the same property with

View File

@@ -4,17 +4,15 @@ from collections import OrderedDict
from itertools import chain, repeat
from inspect import cleandoc
from typing import List, Tuple
try:
import networkx as nx
except ImportError as import_error:
sys.stderr.write(
(
"\n"
"NOTE: Please install networkx to be able to use graph_analyzer "
"module. Using Python:\n" + sys.version + "\n"
)
)
sys.stderr.write((
'\n'
'NOTE: Please install networkx to be able to use graph_analyzer '
'module. Using Python:\n'
+ sys.version +
'\n'))
raise import_error
# Imported last because it also depends on networkx.
from mgp_networkx import MemgraphMultiDiGraph # noqa E402
@@ -25,14 +23,16 @@ _MAX_LIST_SIZE = 10
@mgp.read_proc
def help() -> mgp.Record(name=str, value=str):
"""Shows manual page for graph_analyzer."""
'''Shows manual page for graph_analyzer.'''
records = []
def make_records(name, doc):
return (mgp.Record(name=n, value=v) for n, v in zip(chain([name], repeat("")), cleandoc(doc).splitlines()))
return (mgp.Record(name=n, value=v) for n, v in
zip(chain([name], repeat('')), cleandoc(doc).splitlines()))
for func in (help, analyze, analyze_subgraph):
records.extend(make_records("Procedure '{}'".format(func.__name__), func.__doc__))
records.extend(make_records("Procedure '{}'".format(func.__name__),
func.__doc__))
for m, v in _get_analysis_mapping().items():
records.extend(make_records("Analysis '{}'".format(m), v.__doc__))
@@ -41,8 +41,10 @@ def help() -> mgp.Record(name=str, value=str):
@mgp.read_proc
def analyze(context: mgp.ProcCtx, analyses: mgp.Nullable[List[str]] = None) -> mgp.Record(name=str, value=str):
"""
def analyze(context: mgp.ProcCtx,
analyses: mgp.Nullable[List[str]] = None
) -> mgp.Record(name=str, value=str):
'''
Shows graph information.
In case of multiple results, only the first 10 will be shown.
@@ -55,20 +57,19 @@ def analyze(context: mgp.ProcCtx, analyses: mgp.Nullable[List[str]] = None) -> m
Example call (with parameter):
CALL graph_analyzer.analyze(['nodes', 'edges']) YIELD *;
"""
'''
g = MemgraphMultiDiGraph(ctx=context)
recs = _analyze_graph(context, g, analyses)
return [mgp.Record(name=name, value=value) for name, value in recs]
@mgp.read_proc
def analyze_subgraph(
context: mgp.ProcCtx,
vertices: mgp.List[mgp.Vertex],
edges: mgp.List[mgp.Edge],
analyses: mgp.Nullable[List[str]] = None,
) -> mgp.Record(name=str, value=str):
"""
def analyze_subgraph(context: mgp.ProcCtx,
vertices: mgp.List[mgp.Vertex],
edges: mgp.List[mgp.Edge],
analyses: mgp.Nullable[List[str]] = None
) -> mgp.Record(name=str, value=str):
'''
Shows subgraph information.
In case of multiple results, only the first 10 will be shown.
@@ -90,40 +91,36 @@ def analyze_subgraph(
CALL graph_analyzer.analyze_subgraph(nodes, edges, ['nodes', 'edges'])
YIELD *
RETURN name, value;
"""
'''
vertices, edges = map(set, [vertices, edges])
g = nx.subgraph_view(
MemgraphMultiDiGraph(ctx=context),
lambda n: n in vertices,
lambda n1, n2, e: e in edges,
)
lambda n1, n2, e: e in edges)
recs = _analyze_graph(context, g, analyses)
return [mgp.Record(name=name, value=value) for name, value in recs]
def _get_analysis_mapping():
return OrderedDict(
[
("nodes", _number_of_nodes),
("edges", _number_of_edges),
("bridges", _bridges),
("articulation_points", _articulation_points),
("avg_degree", _avg_degree),
("sorted_nodes_degree", _sorted_nodes_degree),
("self_loops", _self_loops),
("is_bipartite", _is_bipartite),
("is_planar", _is_planar),
("is_biconnected: ", _is_biconnected),
("is_weakly_connected", _is_weakly_connected),
("number_of_weakly_components", _weakly_components),
("is_strongly_connected", _is_strongly_connected),
("strongly_components", _strongly_components),
("is_dag", _is_dag),
("is_eulerian", _is_eulerian),
("is_forest", _is_forest),
("is_tree", _is_tree),
]
)
return OrderedDict([
('nodes', _number_of_nodes),
('edges', _number_of_edges),
('bridges', _bridges),
('articulation_points', _articulation_points),
('avg_degree', _avg_degree),
('sorted_nodes_degree', _sorted_nodes_degree),
('self_loops', _self_loops),
('is_bipartite', _is_bipartite),
('is_planar', _is_planar),
('is_biconnected: ', _is_biconnected),
('is_weakly_connected', _is_weakly_connected),
('number_of_weakly_components', _weakly_components),
('is_strongly_connected', _is_strongly_connected),
('strongly_components', _strongly_components),
('is_dag', _is_dag),
('is_eulerian', _is_eulerian),
('is_forest', _is_forest),
('is_tree', _is_tree)])
def _get_analysis_func(name: str):
@@ -135,15 +132,20 @@ def _get_analysis_funcs():
return _get_analysis_mapping().values()
def _analyze_graph(context: mgp.ProcCtx, g: nx.MultiDiGraph, analyses: List[str]) -> List[Tuple[str, str]]:
def _analyze_graph(context: mgp.ProcCtx,
g: nx.MultiDiGraph,
analyses: List[str]
) -> List[Tuple[str, str]]:
functions = _get_analysis_funcs() if analyses is None else [_get_analysis_func(name) for name in analyses]
functions = (_get_analysis_funcs() if analyses is None
else [_get_analysis_func(name) for name in analyses])
records = []
for index, f in enumerate(functions):
context.check_must_abort()
if f is None:
raise KeyError("Graph analysis is not supported: " + analyses[index])
raise KeyError('Graph analysis is not supported: ' +
analyses[index])
name, value = f(g)
if isinstance(value, (list, set, tuple)):
value = list(value)[:_MAX_LIST_SIZE]
@@ -153,120 +155,126 @@ def _analyze_graph(context: mgp.ProcCtx, g: nx.MultiDiGraph, analyses: List[str]
def _number_of_nodes(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of nodes."""
return "Number of nodes", nx.number_of_nodes(g)
'''Returns number of nodes.'''
return 'Number of nodes', nx.number_of_nodes(g)
def _number_of_edges(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of edges."""
return "Number of edges", nx.number_of_edges(g)
'''Returns number of edges.'''
return 'Number of edges', nx.number_of_edges(g)
def _avg_degree(g: nx.MultiDiGraph) -> Tuple[str, float]:
"""Returns average degree."""
'''Returns average degree.'''
_, number_of_nodes = _number_of_nodes(g)
_, number_of_edges = _number_of_edges(g)
avg_degree = 0 if number_of_nodes == 0 else number_of_edges / number_of_nodes
return "Average degree", avg_degree
avg_degree = (0 if number_of_nodes == 0
else number_of_edges / number_of_nodes)
return 'Average degree', avg_degree
def _sorted_nodes_degree(g: nx.MultiDiGraph) -> Tuple[str, List[int]]:
"""Returns list of sorted nodes degree. [(node_id, degree), ...]"""
'''Returns list of sorted nodes degree. [(node_id, degree), ...]'''
nodes_degree = [(n, g.degree(n)) for n in g.nodes()]
nodes_degree.sort(key=lambda x: x[1], reverse=True)
return "Sorted nodes degree", nodes_degree
return 'Sorted nodes degree', nodes_degree
def _self_loops(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of self loops."""
return "Self loops", sum((1 if e[0] == e[1] else 0 for e in g.edges()))
'''Returns number of self loops.'''
return 'Self loops', sum((1 if e[0] == e[1] else 0 for e in g.edges()))
def _is_bipartite(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is bipartite."""
'''Checks if graph is bipartite.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.bipartite.basic.is_bipartite(g)
return "Is bipartite", ret
ret = (False if number_of_nodes == 0
else nx.algorithms.bipartite.basic.is_bipartite(g))
return 'Is bipartite', ret
def _is_planar(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is planar."""
'''Checks if graph is planar.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.planarity.check_planarity(g)[0]
return "Is planar", ret
ret = (False if number_of_nodes == 0
else nx.algorithms.planarity.check_planarity(g)[0])
return 'Is planar', ret
def _is_biconnected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Check if graph is biconnected."""
'''Check if graph is biconnected.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.is_biconnected(nx.MultiDiGraph.to_undirected(g))
return "Is biconnected", ret
ret = (False if number_of_nodes == 0
else nx.is_biconnected(nx.MultiDiGraph.to_undirected(g)))
return 'Is biconnected', ret
def _is_weakly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Check if graph is weakly connected."""
'''Check if graph is weakly connected.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.is_weakly_connected(g)
return "Is weakly connected", ret
return 'Is weakly connected', ret
def _is_strongly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is strongly connected."""
'''Checks if graph is strongly connected.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.is_strongly_connected(g)
return "Is strongly connected", ret
return 'Is strongly connected', ret
def _is_dag(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Check if graph is directed acyclic graph (DAG)"""
'''Check if graph is directed acyclic graph (DAG)'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.dag.is_directed_acyclic_graph(g)
return "Is DAG", ret
ret = (False if number_of_nodes == 0
else nx.algorithms.dag.is_directed_acyclic_graph(g))
return 'Is DAG', ret
def _is_eulerian(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is Eulerian."""
'''Checks if graph is Eulerian.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.euler.is_eulerian(g)
return "Is eulerian", ret
ret = (False if number_of_nodes == 0
else nx.algorithms.euler.is_eulerian(g))
return 'Is eulerian', ret
def _is_forest(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is forest, all components must be trees."""
'''Checks if graph is forest, all components must be trees.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.tree.recognition.is_forest(g)
return "Is forest", ret
ret = (False if number_of_nodes == 0
else nx.algorithms.tree.recognition.is_forest(g))
return 'Is forest', ret
def _is_tree(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is tree."""
'''Checks if graph is tree.'''
_, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.tree.recognition.is_tree(g)
return "Is tree", ret
ret = (False if number_of_nodes == 0
else nx.algorithms.tree.recognition.is_tree(g))
return 'Is tree', ret
def _bridges(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of bridges, multiple edges between same nodes are
mapped to one edge."""
return "Number of bridges", sum(1 for _ in nx.bridges(nx.Graph(g)))
'''Returns number of bridges, multiple edges between same nodes are
mapped to one edge.'''
return 'Number of bridges', sum(1 for _ in nx.bridges(nx.Graph(g)))
def _articulation_points(g: nx.MultiDiGraph):
"""Returns number of articulation points."""
'''Returns number of articulation points.'''
undirected = nx.MultiDiGraph.to_undirected(g)
return (
"Number of articulation points",
sum(1 for _ in nx.articulation_points(undirected)),
)
return ('Number of articulation points',
sum(1 for _ in nx.articulation_points(undirected)))
def _weakly_components(g: nx.MultiDiGraph):
"""Returns number of weakly components."""
'''Returns number of weakly components.'''
comps = nx.algorithms.components.number_weakly_connected_components(g)
return "Number of weakly connected components", comps
return 'Number of weakly connected components', comps
def _strongly_components(g: nx.MultiDiGraph):
"""Returns number of strongly connected components."""
'''Returns number of strongly connected components.'''
comps = nx.algorithms.components.number_strongly_connected_components(g)
return "Number of strongly connected components", comps
return 'Number of strongly connected components', comps

View File

@@ -1,22 +1,20 @@
import sys
import mgp
import collections
try:
import networkx as nx
except ImportError as import_error:
sys.stderr.write(
(
"\n"
"NOTE: Please install networkx to be able to use Memgraph NetworkX "
"wrappers. Using Python:\n" + sys.version + "\n"
)
)
sys.stderr.write((
'\n'
'NOTE: Please install networkx to be able to use Memgraph NetworkX '
'wrappers. Using Python:\n'
+ sys.version +
'\n'))
raise import_error
class MemgraphAdjlistOuterDict(collections.abc.Mapping):
__slots__ = ("_ctx", "_succ", "_multi")
__slots__ = ('_ctx', '_succ', '_multi')
def __init__(self, ctx, succ=True, multi=True):
self._ctx = ctx
@@ -26,7 +24,8 @@ class MemgraphAdjlistOuterDict(collections.abc.Mapping):
def __getitem__(self, key):
if key not in self:
raise KeyError
return MemgraphAdjlistInnerDict(key, succ=self._succ, multi=self._multi)
return MemgraphAdjlistInnerDict(key, succ=self._succ,
multi=self._multi)
def __iter__(self):
return iter(self._ctx.graph.vertices)
@@ -41,7 +40,7 @@ class MemgraphAdjlistOuterDict(collections.abc.Mapping):
class MemgraphAdjlistInnerDict(collections.abc.Mapping):
__slots__ = ("_node", "_succ", "_multi", "_neighbors")
__slots__ = ('_node', '_succ', '_multi', '_neighbors')
def __init__(self, node, succ=True, multi=True):
self._node = node
@@ -72,26 +71,31 @@ class MemgraphAdjlistInnerDict(collections.abc.Mapping):
def _get_neighbors(self):
if not self._neighbors:
if self._succ:
self._neighbors = set(e.to_vertex for e in self._node.out_edges)
self._neighbors = set(
e.to_vertex for e in self._node.out_edges)
else:
self._neighbors = set(e.from_vertex for e in self._node.in_edges)
self._neighbors = set(
e.from_vertex for e in self._node.in_edges)
return self._neighbors
def _get_edge(self, neighbor):
if self._succ:
edge = list(filter(lambda e: e.to_vertex == neighbor, self._node.out_edges))
edge = list(filter(lambda e: e.to_vertex == neighbor,
self._node.out_edges))
else:
edge = list(filter(lambda e: e.from_vertex == neighbor, self._node.in_edges))
edge = list(filter(lambda e: e.from_vertex == neighbor,
self._node.in_edges))
assert len(edge) >= 1
if len(edge) > 1:
raise RuntimeError("Graph contains multiedges but " "is of non-multigraph type: {}".format(edge))
raise RuntimeError('Graph contains multiedges but '
'is of non-multigraph type: {}'.format(edge))
return edge[0]
class MemgraphEdgeKeyDict(collections.abc.Mapping):
__slots__ = ("_node", "_neighbor", "_succ", "_edges")
__slots__ = ('_node', '_neighbor', '_succ', '_edges')
def __init__(self, node, neighbor, succ=True):
self._node = node
@@ -118,14 +122,18 @@ class MemgraphEdgeKeyDict(collections.abc.Mapping):
def _get_edges(self):
if not self._edges:
if self._succ:
self._edges = list(filter(lambda e: e.to_vertex == self._neighbor, self._node.out_edges))
self._edges = list(filter(
lambda e: e.to_vertex == self._neighbor,
self._node.out_edges))
else:
self._edges = list(filter(lambda e: e.from_vertex == self._neighbor, self._node.in_edges))
self._edges = list(filter(
lambda e: e.from_vertex == self._neighbor,
self._node.in_edges))
return self._edges
class UnhashableProperties(collections.abc.Mapping):
__slots__ = "_properties"
__slots__ = ('_properties')
def __init__(self, properties):
self._properties = properties
@@ -147,7 +155,7 @@ class UnhashableProperties(collections.abc.Mapping):
class MemgraphNodeDict(collections.abc.Mapping):
__slots__ = ("_ctx",)
__slots__ = ('_ctx',)
def __init__(self, ctx):
self._ctx = ctx
@@ -179,7 +187,8 @@ class MemgraphNodeDict(collections.abc.Mapping):
class MemgraphDiGraphBase:
def __init__(self, incoming_graph_data=None, ctx=None, multi=True, **kwargs):
def __init__(self, incoming_graph_data=None, ctx=None, multi=True,
**kwargs):
# NOTE: We assume that our graph will never be given any initial data
# because we already pull our data from the Memgraph database. This
# assert is triggered by certain NetworkX procedures because they
@@ -192,30 +201,23 @@ class MemgraphDiGraphBase:
# modify the graph's internal attributes and don't try to populate it
# with initial data or modify it.
self.node_dict_factory = lambda: MemgraphNodeDict(ctx) if ctx else self._error
self.node_dict_factory = lambda: MemgraphNodeDict(ctx) \
if ctx else self._error
self.node_attr_dict_factory = self._error
self.adjlist_outer_dict_factory = lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) if ctx else self._error
self.adjlist_outer_dict_factory = \
lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) \
if ctx else self._error
self.adjlist_inner_dict_factory = self._error
self.edge_key_dict_factory = self._error
self.edge_attr_dict_factory = self._error
# NOTE: We forbid any mutating operations because our graph is
# immutable and pulls its data from the Memgraph database.
for f in [
"add_node",
"add_nodes_from",
"remove_node",
"remove_nodes_from",
"add_edge",
"add_edges_from",
"add_weighted_edges_from",
"new_edge_key",
"remove_edge",
"remove_edges_from",
"update",
"clear",
]:
for f in ['add_node', 'add_nodes_from', 'remove_node',
'remove_nodes_from', 'add_edge', 'add_edges_from',
'add_weighted_edges_from', 'new_edge_key', 'remove_edge',
'remove_edges_from', 'update', 'clear']:
setattr(self, f, lambda *args, **kwargs: self._error())
super().__init__(None, **kwargs)
@@ -229,29 +231,33 @@ class MemgraphDiGraphBase:
self._pred = MemgraphAdjlistOuterDict(ctx, succ=False, multi=multi)
def _error(self):
raise RuntimeError("Modification operations are not supported")
raise RuntimeError('Modification operations are not supported')
class MemgraphMultiDiGraph(MemgraphDiGraphBase, nx.MultiDiGraph):
def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
super().__init__(incoming_graph_data=incoming_graph_data, ctx=ctx, multi=True, **kwargs)
super().__init__(incoming_graph_data=incoming_graph_data,
ctx=ctx, multi=True, **kwargs)
def MemgraphMultiGraph(incoming_graph_data=None, ctx=None, **kwargs):
return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data, ctx=ctx, **kwargs).to_undirected(as_view=True)
return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data,
ctx=ctx, **kwargs).to_undirected(as_view=True)
class MemgraphDiGraph(MemgraphDiGraphBase, nx.DiGraph):
def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
super().__init__(incoming_graph_data=incoming_graph_data, ctx=ctx, multi=False, **kwargs)
super().__init__(incoming_graph_data=incoming_graph_data,
ctx=ctx, multi=False, **kwargs)
def MemgraphGraph(incoming_graph_data=None, ctx=None, **kwargs):
return MemgraphDiGraph(incoming_graph_data=incoming_graph_data, ctx=ctx, **kwargs).to_undirected(as_view=True)
return MemgraphDiGraph(incoming_graph_data=incoming_graph_data,
ctx=ctx, **kwargs).to_undirected(as_view=True)
class PropertiesDictionary(collections.abc.Mapping):
__slots__ = ("_ctx", "_prop", "_len")
__slots__ = ('_ctx', '_prop', '_len')
def __init__(self, ctx, prop):
self._ctx = ctx
@@ -264,7 +270,8 @@ class PropertiesDictionary(collections.abc.Mapping):
try:
return vertex.properties[self._prop]
except KeyError:
raise KeyError(("{} doesn\t have the required " + "property '{}'").format(vertex, self._prop))
raise KeyError(("{} doesn\t have the required " +
"property '{}'").format(vertex, self._prop))
def __iter__(self):
for v in self._ctx.graph.vertices:

File diff suppressed because it is too large Load Diff

View File

@@ -1,20 +1,23 @@
import sys
import mgp
try:
import networkx as nx
except ImportError as import_error:
sys.stderr.write(
"\n" "NOTE: Please install networkx to be able to use wcc module.\n" "Using Python:\n" + sys.version + "\n"
)
'\n'
'NOTE: Please install networkx to be able to use wcc module.\n'
'Using Python:\n'
+ sys.version +
'\n')
raise import_error
@mgp.read_proc
def get_components(
vertices: mgp.List[mgp.Vertex], edges: mgp.List[mgp.Edge]
) -> mgp.Record(n_components=int, components=mgp.List[mgp.List[mgp.Vertex]]):
"""
def get_components(vertices: mgp.List[mgp.Vertex],
edges: mgp.List[mgp.Edge]
) -> mgp.Record(n_components=int,
components=mgp.List[mgp.List[mgp.Vertex]]):
'''
This procedure finds weakly connected components of a given subgraph of a
directed graph.
@@ -38,7 +41,7 @@ def get_components(
WITH collect(n) AS nodes, collect(e) AS edges
CALL wcc.get_components(nodes, edges) YIELD *
RETURN n_components, components;
"""
'''
g = nx.DiGraph()
g.add_nodes_from(vertices)
g.add_edges_from([(edge.from_vertex, edge.to_vertex) for edge in edges])

View File

@@ -104,9 +104,7 @@ def retry(retry_limit, timeout=100):
except Exception:
time.sleep(timeout)
return func(*args, **kwargs)
return wrapper
return inner_func
@@ -165,15 +163,8 @@ def format_version(variant, version, offering, distance=None, shorthash=None, su
# Parse arguments.
parser = argparse.ArgumentParser(description="Get the current version of Memgraph.")
parser.add_argument(
"--open-source",
action="store_true",
help="set the current offering to 'open-source'",
)
parser.add_argument(
"version",
help="manual version override, if supplied the version isn't " "determined using git",
)
parser.add_argument("--open-source", action="store_true", help="set the current offering to 'open-source'")
parser.add_argument("version", help="manual version override, if supplied the version isn't " "determined using git")
parser.add_argument("suffix", help="custom suffix for the current version being built")
parser.add_argument(
"--variant",
@@ -182,9 +173,7 @@ parser.add_argument(
help="which variant of the version string should be generated",
)
parser.add_argument(
"--memgraph-root-dir",
help="The root directory of the checked out " "Memgraph repository.",
default=".",
"--memgraph-root-dir", help="The root directory of the checked out " "Memgraph repository.", default="."
)
args = parser.parse_args()
@@ -267,27 +256,14 @@ for version in versions:
if current_version is None:
raise Exception("You are attempting to determine the version for a very " "old version of Memgraph!")
version, branch, master_branch_merge = current_version
distance = int(
get_output(
"git",
"rev-list",
"--count",
"--first-parent",
master_branch_merge + ".." + current_hash,
)
)
distance = int(get_output("git", "rev-list", "--count", "--first-parent", master_branch_merge + ".." + current_hash))
version_str = ".".join(map(str, version)) + ".0"
if distance == 0:
print(format_version(args.variant, version_str, offering, suffix=args.suffix), end="")
else:
print(
format_version(
args.variant,
version_str,
offering,
distance=distance,
shorthash=current_hash_short,
suffix=args.suffix,
args.variant, version_str, offering, distance=distance, shorthash=current_hash_short, suffix=args.suffix
),
end="",
)

3
release/mgp/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
.venv
dist
mgp.py

201
release/mgp/LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

4
release/mgp/README.md Normal file
View File

@@ -0,0 +1,4 @@
# mgp
PyPi package used for type hinting when creating MAGE modules. The get started
using MAGE repository checkout the repository here: https://github.com/memgraph/mage.

255
release/mgp/_mgp.py Normal file
View File

@@ -0,0 +1,255 @@
from typing import Any
class MgpIterable:
def get() -> Any:
pass
def next() -> Any:
pass
class Vertex:
def is_valid() -> bool: # type: ignore
pass
def underlying_graph_is_mutable() -> bool: # type: ignore
pass
def iter_properties() -> MgpIterable: # type: ignore
pass
def get_property(self, property_name: str) -> "Property": # type: ignore
pass
def set_property(self, property_name: str, value: Any) -> "Property": # type: ignore
pass
def get_id() -> "VertexId": # type: ignore
pass
def label_at(self, index: int) -> "Label": # type: ignore
pass
def labels_count() -> int: # type: ignore
pass
def add_label(self, label: Any):
pass
def remove_label(self, label: Any):
pass
def iter_in_edges() -> MgpIterable: # type: ignore
pass
def iter_out_edges() -> MgpIterable: # type: ignore
pass
class Edge:
def is_valid() -> bool: # type: ignore
pass
def underlying_graph_is_mutable() -> bool: # type: ignore
pass
def iter_properties() -> MgpIterable: # type: ignore
pass
def get_property(self, property_name: str) -> "Property": # type: ignore
pass
def set_property(self, property_name: str, valuse: Any) -> "Property": # type: ignore
pass
def get_type_name() -> str: # type: ignore
pass
def get_id() -> "EdgeId": # type: ignore
pass
def from_vertex() -> Vertex: # type: ignore
pass
def to_vertex() -> Vertex: # type: ignore
pass
class Path:
def is_valid() -> bool: # type: ignore
pass
@staticmethod
def make_with_start(vertex: Vertex) -> "Path": # type: ignore
pass
class Graph:
def is_valid() -> bool: # type: ignore
pass
class CypherType:
pass
class Message:
def is_valid() -> bool: # type: ignore
pass
def source_type() -> str: # type: ignore
pass
def topic_name() -> str: # type: ignore
pass
def key() -> bytes: # type: ignore
pass
def timestamp() -> int: # type: ignore
pass
def offset() -> int: # type: ignore
pass
def payload() -> bytes: # type: ignore
pass
class Messages:
def is_valid() -> bool: # type: ignore
pass
def message_at(self, id: int) -> Message: # type: ignore
pass
def total_messages() -> int: # type: ignore
pass
class UnknownError(Exception):
pass
class UnableToAllocateError(Exception):
pass
class InsufficientBufferError(Exception):
pass
class OutOfRangeError(Exception):
pass
class LogicErrorError(Exception):
pass
class DeletedObjectError(Exception):
pass
class InvalidArgumentError(Exception):
pass
class KeyAlreadyExistsError(Exception):
pass
class ImmutableObjectError(Exception):
pass
class ValueConversionError(Exception):
pass
class SerializationError(Exception):
pass
def type_nullable(elem: Any):
pass
def type_list(elem: Any):
pass
def type_bool():
pass
def type_string():
pass
def type_int():
pass
def type_float():
pass
def type_number():
pass
def type_map():
pass
def type_node():
pass
def type_relationship():
pass
def type_path():
pass
def type_date():
pass
def type_local_time():
pass
def type_local_date_time():
pass
def type_duration():
pass
def type_any():
pass
class _MODULE:
@staticmethod
def add_read_procedure(wrapper):
pass
@staticmethod
def add_write_procedure(wrapper):
pass
@staticmethod
def add_transformation(wrapper):
pass
@staticmethod
def add_function(wrapper):
pass

View File

@@ -0,0 +1,22 @@
# How to publish new versions
## Prerequisites
1. Installed poetry
```
pip install poetry
```
2. Set up [API tokens](https://pypi.org/help/#apitoken)
3. Be a collaborator on [pypi](https://pypi.org/project/mgp/)
## Making changes
1. Make changes to the package
2. Bump version in `pyproject.tml`
3. `poetry build`
4. `poetry publish`
## Why is this not automatized?
Because someone always has to manually bump up the version in `pyproject.toml`
## Why does `_mgp.py` exists?
Because we are mocking here all the types that are created by Memgraph
in order to fix typing errors in `mgp.py`.

View File

@@ -0,0 +1,23 @@
[tool.poetry]
name = "mgp"
version = "1.0.0"
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
authors = [
"MasterMedo <mislav.vuletic@gmail.com>",
"jbajic <jure.bajic@memgraph.io>",
"katarinasupe <katarina.supe@memgraph.io>",
"antejavor <ante.javor@memgraph.io>",
"antaljanosbenjamin <benjamin.antal@memgraph.io>",
]
license = "Apache-2.0"
readme = "README.md"
include = ["mgp.py", "_mgp.py"]
[tool.poetry.dependencies]
python = "^3.7"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@@ -5,16 +5,20 @@ add_subdirectory(lisp)
add_subdirectory(utils)
add_subdirectory(requests)
add_subdirectory(io)
add_subdirectory(io/simulator)
add_subdirectory(kvstore)
add_subdirectory(telemetry)
add_subdirectory(communication)
add_subdirectory(memory)
add_subdirectory(storage/v2)
add_subdirectory(storage/v3)
add_subdirectory(integrations)
add_subdirectory(query)
add_subdirectory(query/v2)
add_subdirectory(slk)
add_subdirectory(rpc)
add_subdirectory(auth)
add_subdirectory(coordinator)
if (MG_ENTERPRISE)
add_subdirectory(audit)

View File

@@ -37,7 +37,7 @@ const std::vector<Permission> kPermissionsAll = {
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET};
Permission::WEBSOCKET, Permission::SCHEMA};
} // namespace
std::string PermissionToString(Permission permission) {
@@ -84,8 +84,8 @@ std::string PermissionToString(Permission permission) {
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
case Permission::LABELS:
return "LABELS";
case Permission::SCHEMA:
return "SCHEMA";
}
}
@@ -185,107 +185,19 @@ bool operator==(const Permissions &first, const Permissions &second) {
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
LabelPermissions::LabelPermissions(const std::unordered_set<std::string> &grants,
const std::unordered_set<std::string> &denies)
: grants_(grants), denies_(denies) {}
PermissionLevel LabelPermissions::Has(const std::string &permission) const {
if (denies_.find(permission) != denies_.end()) {
return PermissionLevel::DENY;
}
if (grants_.find(permission) != denies_.end()) {
return PermissionLevel::GRANT;
}
return PermissionLevel::NEUTRAL;
}
void LabelPermissions::Grant(const std::string &permission) {
auto deniedPermissionIter = denies_.find(permission);
if (deniedPermissionIter != denies_.end()) {
denies_.erase(deniedPermissionIter);
}
if (grants_.find(permission) == grants_.end()) {
grants_.insert(permission);
}
}
void LabelPermissions::Revoke(const std::string &permission) {
auto deniedPermissionIter = denies_.find(permission);
auto grantedPermissionIter = grants_.find(permission);
if (deniedPermissionIter != denies_.end()) {
denies_.erase(deniedPermissionIter);
}
if (grantedPermissionIter != grants_.end()) {
grants_.erase(grantedPermissionIter);
}
}
void LabelPermissions::Deny(const std::string &permission) {
auto grantedPermissionIter = grants_.find(permission);
if (grantedPermissionIter != grants_.end()) {
grants_.erase(grantedPermissionIter);
}
if (denies_.find(permission) == denies_.end()) {
denies_.insert(permission);
}
}
std::unordered_set<std::string> LabelPermissions::GetGrants() const { return grants_; }
std::unordered_set<std::string> LabelPermissions::GetDenies() const { return denies_; }
nlohmann::json LabelPermissions::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["grants"] = grants_;
data["denies"] = denies_;
return data;
}
LabelPermissions LabelPermissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!");
}
return {LabelPermissions(data["grants"], data["denies"])};
}
std::unordered_set<std::string> LabelPermissions::grants() const { return grants_; }
std::unordered_set<std::string> LabelPermissions::denies() const { return denies_; }
bool operator==(const LabelPermissions &first, const LabelPermissions &second) {
return first.grants() == second.grants() && first.denies() == second.denies();
}
bool operator!=(const LabelPermissions &first, const LabelPermissions &second) { return !(first == second); }
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
Role::Role(const std::string &rolename, const Permissions &permissions, const LabelPermissions &labelPermissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions), labelPermissions_(labelPermissions) {}
const std::string &Role::rolename() const { return rolename_; }
const Permissions &Role::permissions() const { return permissions_; }
Permissions &Role::permissions() { return permissions_; }
LabelPermissions &Role::labelPermissions() { return labelPermissions_; }
nlohmann::json Role::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["rolename"] = rolename_;
data["permissions"] = permissions_.Serialize();
data["labelPermissions"] = labelPermissions_.Serialize();
return data;
}
@@ -297,9 +209,7 @@ Role Role::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load role data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
auto labelPermissions = LabelPermissions::Deserialize(data["labelPermissions"]);
return {data["rolename"], permissions, labelPermissions};
return {data["rolename"], permissions};
}
bool operator==(const Role &first, const Role &second) {
@@ -311,13 +221,6 @@ User::User(const std::string &username) : username_(utils::ToLowerCase(username)
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
const LabelPermissions &labelPermissions)
: username_(utils::ToLowerCase(username)),
password_hash_(password_hash),
permissions_(permissions),
labelPermissions_(labelPermissions) {}
bool User::CheckPassword(const std::string &password) {
if (password_hash_.empty()) return true;
return VerifyPassword(password, password_hash_);
@@ -370,8 +273,6 @@ const std::string &User::username() const { return username_; }
const Permissions &User::permissions() const { return permissions_; }
Permissions &User::permissions() { return permissions_; }
LabelPermissions &User::labelPermissions() { return labelPermissions_; }
const Role *User::role() const {
if (role_.has_value()) {
return &role_.value();
@@ -384,7 +285,6 @@ nlohmann::json User::Serialize() const {
data["username"] = username_;
data["password_hash"] = password_hash_;
data["permissions"] = permissions_.Serialize();
data["labelPermissions"] = labelPermissions_.Serialize();
// The role shouldn't be serialized here, it is stored as a foreign key.
return data;
}
@@ -397,14 +297,11 @@ User User::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load user data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
auto labelPermissions = LabelPermissions::Deserialize(data["labelPermissions"]);
return {data["username"], data["password_hash"], permissions, labelPermissions};
return {data["username"], data["password_hash"], permissions};
}
bool operator==(const User &first, const User &second) {
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_;
}
} // namespace memgraph::auth

View File

@@ -12,7 +12,6 @@
#include <string>
#include <json/json.hpp>
#include <unordered_set>
namespace memgraph::auth {
// These permissions must have values that are applicable for usage in a
@@ -40,7 +39,7 @@ enum class Permission : uint64_t {
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U,
LABELS = 1U << 21U
SCHEMA = 1U << 21U
};
// clang-format on
@@ -90,52 +89,16 @@ bool operator==(const Permissions &first, const Permissions &second);
bool operator!=(const Permissions &first, const Permissions &second);
class LabelPermissions final {
public:
LabelPermissions(const std::unordered_set<std::string> &grants = {},
const std::unordered_set<std::string> &denies = {});
PermissionLevel Has(const std::string &permission) const;
void Grant(const std::string &permission);
void Revoke(const std::string &permission);
void Deny(const std::string &permission);
std::unordered_set<std::string> GetGrants() const;
std::unordered_set<std::string> GetDenies() const;
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static LabelPermissions Deserialize(const nlohmann::json &data);
std::unordered_set<std::string> grants() const;
std::unordered_set<std::string> denies() const;
private:
std::unordered_set<std::string> grants_{};
std::unordered_set<std::string> denies_{};
};
bool operator==(const LabelPermissions &first, const LabelPermissions &second);
bool operator!=(const LabelPermissions &first, const LabelPermissions &second);
class Role final {
public:
Role(const std::string &rolename);
Role(const std::string &rolename, const Permissions &permissions);
Role(const std::string &rolename, const Permissions &permissions, const LabelPermissions &labelPermissions);
const std::string &rolename() const;
const Permissions &permissions() const;
Permissions &permissions();
LabelPermissions &labelPermissions();
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
@@ -146,7 +109,6 @@ class Role final {
private:
std::string rolename_;
Permissions permissions_;
LabelPermissions labelPermissions_;
};
bool operator==(const Role &first, const Role &second);
@@ -158,9 +120,6 @@ class User final {
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
const LabelPermissions &labelPermissions);
/// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password);
@@ -180,8 +139,6 @@ class User final {
const Role *role() const;
LabelPermissions &labelPermissions();
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
@@ -194,9 +151,7 @@ class User final {
std::string password_hash_;
Permissions permissions_;
std::optional<Role> role_;
LabelPermissions labelPermissions_;
};
bool operator==(const User &first, const User &second);
} // namespace memgraph::auth

View File

@@ -18,24 +18,19 @@ roles_config = config["roles"]
# Initialize LDAP server.
tls = None
if server_config["encryption"] != "disabled":
cert_file = server_config["cert_file"] if server_config["cert_file"] else None
cert_file = server_config["cert_file"] if server_config["cert_file"] \
else None
key_file = server_config["key_file"] if server_config["key_file"] else None
ca_file = server_config["ca_file"] if server_config["ca_file"] else None
validate = ssl.CERT_REQUIRED if server_config["validate_cert"] else ssl.CERT_NONE
tls = ldap3.Tls(
local_private_key_file=key_file,
local_certificate_file=cert_file,
ca_certs_file=ca_file,
validate=validate,
)
validate = ssl.CERT_REQUIRED if server_config["validate_cert"] \
else ssl.CERT_NONE
tls = ldap3.Tls(local_private_key_file=key_file,
local_certificate_file=cert_file,
ca_certs_file=ca_file,
validate=validate)
use_ssl = server_config["encryption"] == "ssl"
server = ldap3.Server(
server_config["host"],
port=server_config["port"],
tls=tls,
use_ssl=use_ssl,
get_info=ldap3.ALL,
)
server = ldap3.Server(server_config["host"], port=server_config["port"],
tls=tls, use_ssl=use_ssl, get_info=ldap3.ALL)
# Main authentication/authorization function.
@@ -45,12 +40,14 @@ def authenticate(username, password):
return {"authenticated": False, "role": ""}
# Create the DN of the user
dn = users_config["prefix"] + ldap3.utils.dn.escape_rdn(username) + users_config["suffix"]
dn = users_config["prefix"] + ldap3.utils.dn.escape_rdn(username) + \
users_config["suffix"]
# Bind to the server
conn = ldap3.Connection(server, dn, password)
if server_config["encryption"] == "starttls" and not conn.start_tls():
print("ERROR: Couldn't issue STARTTLS to the LDAP server!", file=sys.stderr)
print("ERROR: Couldn't issue STARTTLS to the LDAP server!",
file=sys.stderr)
return {"authenticated": False, "role": ""}
if not conn.bind():
return {"authenticated": False, "role": ""}
@@ -59,32 +56,25 @@ def authenticate(username, password):
if roles_config["root_dn"] != "":
# search for role
search_filter = "(&(objectclass={objclass})({attr}={value}))".format(
objclass=roles_config["root_objectclass"],
attr=roles_config["user_attribute"],
value=ldap3.utils.conv.escape_filter_chars(dn),
)
succ = conn.search(
roles_config["root_dn"],
search_filter,
search_scope=ldap3.LEVEL,
attributes=[roles_config["role_attribute"]],
)
objclass=roles_config["root_objectclass"],
attr=roles_config["user_attribute"],
value=ldap3.utils.conv.escape_filter_chars(dn))
succ = conn.search(roles_config["root_dn"], search_filter,
search_scope=ldap3.LEVEL,
attributes=[roles_config["role_attribute"]])
if not succ or len(conn.entries) == 0:
return {"authenticated": True, "role": ""}
if len(conn.entries) > 1:
roles = list(map(lambda x: x[roles_config["role_attribute"]].value, conn.entries))
roles = list(map(lambda x: x[roles_config["role_attribute"]].value,
conn.entries))
# Because we don't know exactly which role the user should have
# we authorize the user with an empty role.
print(
"WARNING: Found more than one role for " "user '" + username + "':",
", ".join(roles) + "!",
file=sys.stderr,
)
print("WARNING: Found more than one role for "
"user '" + username + "':", ", ".join(roles) + "!",
file=sys.stderr)
return {"authenticated": True, "role": ""}
return {
"authenticated": True,
"role": conn.entries[0][roles_config["role_attribute"]].value,
}
return {"authenticated": True,
"role": conn.entries[0][roles_config["role_attribute"]].value}
else:
return {"authenticated": True, "role": ""}

19
src/common/types.hpp Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
namespace memgraph::common {
enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
} // namespace memgraph::common

View File

@@ -0,0 +1,10 @@
set(coordinator_src_files
coordinator.hpp
shard_map.hpp
hybrid_logical_clock.hpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-coordinator STATIC ${coordinator_src_files})
target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@@ -0,0 +1,243 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <unordered_set>
#include "coordinator/hybrid_logical_clock.hpp"
#include "coordinator/shard_map.hpp"
#include "io/simulator/simulator.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::coordinator {
using Address = memgraph::io::Address;
using SimT = memgraph::io::simulator::SimulatorTransport;
struct HlcRequest {
Hlc last_shard_map_version;
};
struct HlcResponse {
Hlc new_hlc;
std::optional<ShardMap> fresher_shard_map;
};
struct GetShardMapRequest {
// No state
};
struct GetShardMapResponse {
ShardMap shard_map;
};
struct AllocateHlcBatchRequest {
Hlc low;
Hlc high;
};
struct AllocateHlcBatchResponse {
bool success;
Hlc low;
Hlc high;
};
struct AllocateEdgeIdBatchRequest {
size_t batch_size;
};
struct AllocateEdgeIdBatchResponse {
uint64_t low;
uint64_t high;
};
struct SplitShardRequest {
Hlc previous_shard_map_version;
Label label;
CompoundKey split_key;
};
struct SplitShardResponse {
bool success;
};
struct RegisterStorageEngineRequest {
Address address;
};
struct RegisterStorageEngineResponse {
bool success;
};
struct DeregisterStorageEngineRequest {
Address address;
};
struct DeregisterStorageEngineResponse {
bool success;
};
struct InitializeLabelRequest {
std::string label_name;
Hlc last_shard_map_version;
};
struct InitializeLabelResponse {
bool success;
std::optional<ShardMap> fresher_shard_map;
};
using WriteRequests =
std::variant<AllocateHlcBatchRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
DeregisterStorageEngineRequest, InitializeLabelRequest>;
using WriteResponses =
std::variant<AllocateHlcBatchResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
RegisterStorageEngineResponse, DeregisterStorageEngineResponse, InitializeLabelResponse>;
using ReadRequests = std::variant<HlcRequest, GetShardMapRequest>;
using ReadResponses = std::variant<HlcResponse, GetShardMapResponse>;
class Coordinator {
ShardMap shard_map_;
/// The highest reserved timestamp / highest allocated timestamp
/// is a way for minimizing communication involved in query engines
/// reserving Hlc's for their transaction processing.
/// Periodically, the coordinator will allocate a batch of timestamps
/// and this will need to go over consensus. From that point forward,
/// each timestamp in that batch can be given out to "readers" who issue
/// HlcRequest without blocking on consensus first. But if
/// highest_allocated_timestamp_ approaches highest_reserved_timestamp_,
/// it is time to allocate another batch, so that we can keep guaranteeing
/// forward progress.
/// Any time a coordinator becomes a new leader, it will need to issue
/// a new AllocateHlcBatchRequest to create a pool of IDs to allocate.
uint64_t highest_allocated_timestamp_;
uint64_t highest_reserved_timestamp_;
/// Query engines need to periodically request batches of unique edge IDs.
uint64_t highest_allocated_edge_id_;
/// Increment our
ReadResponses HandleRead(HlcRequest &&hlc_request) {
HlcResponse res{};
auto hlc_shard_map = shard_map_.GetHlc();
MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
res.new_hlc = shard_map_.IncrementShardMapVersion();
// res.fresher_shard_map = hlc_request.last_shard_map_version.logical_id < hlc_shard_map.logical_id
// ? std::make_optional(shard_map_)
// : std::nullopt;
// Allways return fresher shard_map for now.
res.fresher_shard_map = std::make_optional(shard_map_);
return res;
}
ReadResponses HandleRead(GetShardMapRequest &&get_shard_map_request) {
GetShardMapResponse res;
res.shard_map = shard_map_;
return res;
}
WriteResponses ApplyWrite(AllocateHlcBatchRequest &&ahr) {
AllocateHlcBatchResponse res{};
return res;
}
WriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
AllocateEdgeIdBatchResponse res{};
uint64_t low = highest_allocated_edge_id_;
highest_allocated_edge_id_ += ahr.batch_size;
uint64_t high = highest_allocated_edge_id_;
res.low = low;
res.high = high;
return res;
}
/// This splits the shard immediately beneath the provided
/// split key, keeping the assigned peers identical for now,
/// but letting them be gradually migrated over time.
WriteResponses ApplyWrite(SplitShardRequest &&split_shard_request) {
SplitShardResponse res{};
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
res.success = false;
} else {
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label,
split_shard_request.split_key);
}
return res;
}
/// This adds the provided storage engine to the standby storage engine pool,
/// which can be used to rebalance storage over time.
WriteResponses ApplyWrite(RegisterStorageEngineRequest &&register_storage_engine_request) {
RegisterStorageEngineResponse res{};
// TODO
return res;
}
/// This begins the process of draining the provided storage engine from all raft
/// clusters that it might be participating in.
WriteResponses ApplyWrite(DeregisterStorageEngineRequest &&register_storage_engine_request) {
DeregisterStorageEngineResponse res{};
// TODO
// const Address &address = register_storage_engine_request.address;
// storage_engine_pool_.erase(address);
// res.success = true;
return res;
}
WriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
InitializeLabelResponse res{};
bool success = shard_map_.InitializeNewLabel(initialize_label_request.label_name,
initialize_label_request.last_shard_map_version);
if (success) {
res.fresher_shard_map = shard_map_;
res.success = false;
} else {
res.fresher_shard_map = std::nullopt;
res.success = true;
}
return res;
}
public:
explicit Coordinator(ShardMap sm) : shard_map_{(sm)} {}
ReadResponses Read(ReadRequests requests) {
return std::visit([&](auto &&requests) { return HandleRead(std::move(requests)); }, std::move(requests));
}
WriteResponses Apply(WriteRequests requests) {
return std::visit([&](auto &&requests) { return ApplyWrite(std::move(requests)); }, std::move(requests));
}
};
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,30 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/time.hpp"
namespace memgraph::coordinator {
using Time = memgraph::io::Time;
/// Hybrid-logical clock
struct Hlc {
uint64_t logical_id;
Time coordinator_wall_clock;
bool operator==(const Hlc &other) const {
return (logical_id == other.logical_id) && (coordinator_wall_clock == other.coordinator_wall_clock);
}
};
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,135 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <map>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::coordinator {
enum class Status : uint8_t {
CONSENSUS_PARTICIPANT,
INITIALIZING,
// TODO(tyler) this will possibly have more states,
// depending on the reconfiguration protocol that we
// implement.
};
struct AddressAndStatus {
memgraph::io::Address address;
Status status;
};
using memgraph::io::Address;
using CompoundKey = std::vector<memgraph::storage::v3::PropertyValue>;
using Shard = std::vector<AddressAndStatus>;
using Shards = std::map<CompoundKey, Shard>;
// use string for intermachine communication and NameIdMapper within the machine
using Label = std::string;
struct ShardMap {
Hlc shard_map_version;
std::map<Label, Shards> shards;
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well
Hlc IncrementShardMapVersion() noexcept {
++shard_map_version.logical_id;
return shard_map_version;
}
Hlc GetHlc() const noexcept { return shard_map_version; }
bool SplitShard(Hlc previous_shard_map_version, Label label, CompoundKey key) {
if (CompareShardMapVersions(previous_shard_map_version, shard_map_version)) {
MG_ASSERT(shards.contains(label));
auto &shards_in_map = shards[label];
MG_ASSERT(!shards_in_map.contains(key));
// Finding the Shard that the new CompoundKey should map to.
Shard shard_to_map_to;
CompoundKey prev_key = ((*shards_in_map.begin()).first);
for (auto iter = std::next(shards_in_map.begin()); iter != shards_in_map.end(); ++iter) {
const auto &current_key = (*iter).first;
if (key > prev_key && key < current_key) {
shard_to_map_to = shards_in_map[prev_key];
}
prev_key = (*iter).first;
}
// Apply the split
shards_in_map[key] = shard_to_map_to;
return true;
}
return false;
}
bool InitializeNewLabel(std::string label_name, Hlc last_shard_map_version) {
if (shard_map_version != last_shard_map_version) {
return false;
}
if (shards.contains(label_name)) {
return false;
}
shards.emplace(label_name, Shards{});
IncrementShardMapVersion();
return true;
}
void AddServer(Address server_address) {
// Find a random place for the server to plug in
}
std::map<Label, Shards> &GetShards() noexcept { return shards; }
Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end);
Shard GetShardForKey(Label label, CompoundKey key) {
auto shard_for_label = shards.at(label);
auto max = (--shard_for_label.end())->first;
if (key > max) {
return shard_for_label[max];
}
for (auto it = shard_for_label.lower_bound(key);; --it) {
MG_ASSERT(it->first <= key);
return it->second;
}
MG_ASSERT(false, "failed to find shard with a key that is less than or equal to the provided key");
}
private:
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well. This function should just be
// replaced with operator== since it is already overloaded for Hlc
// objects.
bool CompareShardMapVersions(Hlc one, Hlc two) { return one.logical_id == two.logical_id; }
};
} // namespace memgraph::coordinator

View File

@@ -57,8 +57,6 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::MODULE_WRITE;
case query::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::AuthQuery::Privilege::LABELS:
return auth::Permission::LABELS;
}
}
} // namespace memgraph::glue

64
src/glue/v2/auth.cpp Normal file
View File

@@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/auth.hpp"
namespace memgraph::glue::v2 {
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege) {
switch (privilege) {
case query::v2::AuthQuery::Privilege::MATCH:
return auth::Permission::MATCH;
case query::v2::AuthQuery::Privilege::CREATE:
return auth::Permission::CREATE;
case query::v2::AuthQuery::Privilege::MERGE:
return auth::Permission::MERGE;
case query::v2::AuthQuery::Privilege::DELETE:
return auth::Permission::DELETE;
case query::v2::AuthQuery::Privilege::SET:
return auth::Permission::SET;
case query::v2::AuthQuery::Privilege::REMOVE:
return auth::Permission::REMOVE;
case query::v2::AuthQuery::Privilege::INDEX:
return auth::Permission::INDEX;
case query::v2::AuthQuery::Privilege::STATS:
return auth::Permission::STATS;
case query::v2::AuthQuery::Privilege::CONSTRAINT:
return auth::Permission::CONSTRAINT;
case query::v2::AuthQuery::Privilege::DUMP:
return auth::Permission::DUMP;
case query::v2::AuthQuery::Privilege::REPLICATION:
return auth::Permission::REPLICATION;
case query::v2::AuthQuery::Privilege::DURABILITY:
return auth::Permission::DURABILITY;
case query::v2::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::v2::AuthQuery::Privilege::FREE_MEMORY:
return auth::Permission::FREE_MEMORY;
case query::v2::AuthQuery::Privilege::TRIGGER:
return auth::Permission::TRIGGER;
case query::v2::AuthQuery::Privilege::CONFIG:
return auth::Permission::CONFIG;
case query::v2::AuthQuery::Privilege::AUTH:
return auth::Permission::AUTH;
case query::v2::AuthQuery::Privilege::STREAM:
return auth::Permission::STREAM;
case query::v2::AuthQuery::Privilege::MODULE_READ:
return auth::Permission::MODULE_READ;
case query::v2::AuthQuery::Privilege::MODULE_WRITE:
return auth::Permission::MODULE_WRITE;
case query::v2::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::v2::AuthQuery::Privilege::SCHEMA:
return auth::Permission::SCHEMA;
}
}
} // namespace memgraph::glue::v2

23
src/glue/v2/auth.hpp Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "auth/models.hpp"
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::glue::v2 {
/**
* This function converts query::AuthQuery::Privilege to its corresponding
* auth::Permission.
*/
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege);
} // namespace memgraph::glue::v2

View File

@@ -0,0 +1,275 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/communication.hpp"
#include <map>
#include <string>
#include <vector>
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/storage.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/temporal.hpp"
using memgraph::communication::bolt::Value;
namespace memgraph::glue::v2 {
query::v2::TypedValue ToTypedValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return query::v2::TypedValue(value.ValueBool());
case Value::Type::Int:
return query::v2::TypedValue(value.ValueInt());
case Value::Type::Double:
return query::v2::TypedValue(value.ValueDouble());
case Value::Type::String:
return query::v2::TypedValue(value.ValueString());
case Value::Type::List: {
std::vector<query::v2::TypedValue> list;
list.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) list.push_back(ToTypedValue(v));
return query::v2::TypedValue(std::move(list));
}
case Value::Type::Map: {
std::map<std::string, query::v2::TypedValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToTypedValue(kv.second));
return query::v2::TypedValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to TypedValue");
case Value::Type::Date:
return query::v2::TypedValue(value.ValueDate());
case Value::Type::LocalTime:
return query::v2::TypedValue(value.ValueLocalTime());
case Value::Type::LocalDateTime:
return query::v2::TypedValue(value.ValueLocalDateTime());
case Value::Type::Duration:
return query::v2::TypedValue(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Storage &db,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
return Value();
case query::v2::TypedValue::Type::Bool:
return Value(value.ValueBool());
case query::v2::TypedValue::Type::Int:
return Value(value.ValueInt());
case query::v2::TypedValue::Type::Double:
return Value(value.ValueDouble());
case query::v2::TypedValue::Type::String:
return Value(std::string(value.ValueString()));
case query::v2::TypedValue::Type::List: {
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
auto maybe_value = ToBoltValue(v, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
values.emplace_back(std::move(*maybe_value));
}
return Value(std::move(values));
}
case query::v2::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return Value(std::move(map));
}
case query::v2::TypedValue::Type::Vertex: {
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::v2::TypedValue::Type::Edge: {
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::v2::TypedValue::Type::Path: {
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::v2::TypedValue::Type::Date:
return Value(value.ValueDate());
case query::v2::TypedValue::Type::LocalTime:
return Value(value.ValueLocalTime());
case query::v2::TypedValue::Type::LocalDateTime:
return Value(value.ValueLocalDateTime());
case query::v2::TypedValue::Type::Duration:
return Value(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view) {
auto id = communication::bolt::Id::FromUint(vertex.Gid().AsUint());
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
std::vector<std::string> labels;
labels.reserve(maybe_labels->size());
for (const auto &label : *maybe_labels) {
labels.push_back(db.LabelToName(label));
}
auto maybe_properties = vertex.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Vertex{id, labels, properties};
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view) {
auto id = communication::bolt::Id::FromUint(edge.Gid().AsUint());
auto from = communication::bolt::Id::FromUint(edge.FromVertex().Gid().AsUint());
auto to = communication::bolt::Id::FromUint(edge.ToVertex().Gid().AsUint());
const auto &type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
storage::v3::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());
for (const auto &v : path.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(std::move(*maybe_vertex));
}
std::vector<communication::bolt::Edge> edges;
edges.reserve(path.edges().size());
for (const auto &e : path.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(std::move(*maybe_edge));
}
return communication::bolt::Path(vertices, edges);
}
storage::v3::PropertyValue ToPropertyValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return storage::v3::PropertyValue(value.ValueBool());
case Value::Type::Int:
return storage::v3::PropertyValue(value.ValueInt());
case Value::Type::Double:
return storage::v3::PropertyValue(value.ValueDouble());
case Value::Type::String:
return storage::v3::PropertyValue(value.ValueString());
case Value::Type::List: {
std::vector<storage::v3::PropertyValue> vec;
vec.reserve(value.ValueList().size());
for (const auto &value : value.ValueList()) vec.emplace_back(ToPropertyValue(value));
return storage::v3::PropertyValue(std::move(vec));
}
case Value::Type::Map: {
std::map<std::string, storage::v3::PropertyValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToPropertyValue(kv.second));
return storage::v3::PropertyValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to PropertyValue");
case Value::Type::Date:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Date, value.ValueDate().MicrosecondsSinceEpoch()));
case Value::Type::LocalTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalTime,
value.ValueLocalTime().MicrosecondsSinceEpoch()));
case Value::Type::LocalDateTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalDateTime,
value.ValueLocalDateTime().MicrosecondsSinceEpoch()));
case Value::Type::Duration:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Duration, value.ValueDuration().microseconds));
}
}
Value ToBoltValue(const storage::v3::PropertyValue &value) {
switch (value.type()) {
case storage::v3::PropertyValue::Type::Null:
return {};
case storage::v3::PropertyValue::Type::Bool:
return {value.ValueBool()};
case storage::v3::PropertyValue::Type::Int:
return {value.ValueInt()};
break;
case storage::v3::PropertyValue::Type::Double:
return {value.ValueDouble()};
case storage::v3::PropertyValue::Type::String:
return {value.ValueString()};
case storage::v3::PropertyValue::Type::List: {
const auto &values = value.ValueList();
std::vector<Value> vec;
vec.reserve(values.size());
for (const auto &v : values) {
vec.push_back(ToBoltValue(v));
}
return {std::move(vec)};
}
case storage::v3::PropertyValue::Type::Map: {
const auto &map = value.ValueMap();
std::map<std::string, Value> dv_map;
for (const auto &kv : map) {
dv_map.emplace(kv.first, ToBoltValue(kv.second));
}
return {std::move(dv_map)};
}
case storage::v3::PropertyValue::Type::TemporalData:
const auto &type = value.ValueTemporalData();
switch (type.type) {
case storage::v3::TemporalType::Date:
return {utils::Date(type.microseconds)};
case storage::v3::TemporalType::LocalTime:
return {utils::LocalTime(type.microseconds)};
case storage::v3::TemporalType::LocalDateTime:
return {utils::LocalDateTime(type.microseconds)};
case storage::v3::TemporalType::Duration:
return {utils::Duration(type.microseconds)};
}
}
}
} // namespace memgraph::glue::v2

View File

@@ -0,0 +1,68 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file Conversion functions between Value and other memgraph types.
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "query/v2/typed_value.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
class Storage;
class VertexAccessor;
} // namespace memgraph::storage::v3
namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param storage::v3::Storage for getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param storage::v3::Storage for getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
const storage::v3::Storage &db, storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
} // namespace memgraph::glue::v2

60
src/io/address.hpp Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <compare>
#include <fmt/format.h>
#include <boost/asio/ip/tcp.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>
namespace memgraph::io {
struct Address {
// It's important for all participants to have a
// unique identifier - IP and port alone are not
// enough, and may change over the lifecycle of
// the nodes. Particularly storage nodes may change
// their IP addresses over time, and the system
// should gracefully update its information
// about them.
boost::uuids::uuid unique_id;
boost::asio::ip::address last_known_ip;
uint16_t last_known_port;
static Address TestAddress(uint16_t port) {
Address ret;
ret.last_known_port = port;
return ret;
}
friend bool operator==(const Address &lhs, const Address &rhs) = default;
/// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
friend bool operator<(const Address &lhs, const Address &rhs) {
if (lhs.unique_id != rhs.unique_id) {
return lhs.unique_id < rhs.unique_id;
}
if (lhs.last_known_ip != rhs.last_known_ip) {
return lhs.last_known_ip < rhs.last_known_ip;
}
return lhs.last_known_port < rhs.last_known_port;
}
std::string ToString() const {
return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
}
};
}; // namespace memgraph::io

26
src/io/errors.hpp Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::io {
// Signifies that a retriable operation was unable to
// complete after a configured number of retries.
struct RetriesExhausted {};
// Signifies that a request was unable to receive a response
// within some configured timeout duration. It is important
// to remember that in distributed systems, a timeout does
// not signify that a request was not received or processed.
// It may be the case that the request was fully processed
// but that the response was not received.
struct TimedOut {};
}; // namespace memgraph::io

262
src/io/future.hpp Normal file
View File

@@ -0,0 +1,262 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <condition_variable>
#include <memory>
#include <mutex>
#include <optional>
#include <thread>
#include <utility>
#include "io/errors.hpp"
#include "utils/logging.hpp"
namespace memgraph::io {
// Shared is in an anonymous namespace, and the only way to
// construct a Promise or Future is to pass a Shared in. This
// ensures that Promises and Futures can only be constructed
// in this translation unit.
namespace details {
template <typename T>
class Shared {
mutable std::condition_variable cv_;
mutable std::mutex mu_;
std::optional<T> item_;
bool consumed_ = false;
bool waiting_ = false;
std::function<bool()> simulator_notifier_ = nullptr;
public:
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
Shared() = default;
Shared(Shared &&) = delete;
Shared &operator=(Shared &&) = delete;
Shared(const Shared &) = delete;
Shared &operator=(const Shared &) = delete;
~Shared() = default;
/// Takes the item out of our optional item_ and returns it.
T Take() {
MG_ASSERT(item_, "Take called without item_ being present");
MG_ASSERT(!consumed_, "Take called on already-consumed Future");
T ret = std::move(item_).value();
item_.reset();
consumed_ = true;
return ret;
}
T Wait() {
std::unique_lock<std::mutex> lock(mu_);
waiting_ = true;
while (!item_) {
bool simulator_progressed = false;
if (simulator_notifier_) [[unlikely]] {
// We can't hold our own lock while notifying
// the simulator because notifying the simulator
// involves acquiring the simulator's mutex
// to guarantee that our notification linearizes
// with the simulator's condition variable.
// However, the simulator may acquire our
// mutex to check if we are being awaited,
// while determining system quiescence,
// so we have to get out of its way to avoid
// a cyclical deadlock.
lock.unlock();
simulator_progressed = std::invoke(simulator_notifier_);
lock.lock();
if (item_) {
// item may have been filled while we
// had dropped our mutex while notifying
// the simulator of our waiting_ status.
break;
}
}
if (!simulator_progressed) [[likely]] {
cv_.wait(lock);
}
MG_ASSERT(!consumed_, "Future consumed twice!");
}
waiting_ = false;
return Take();
}
bool IsReady() const {
std::unique_lock<std::mutex> lock(mu_);
return item_;
}
std::optional<T> TryGet() {
std::unique_lock<std::mutex> lock(mu_);
if (item_) {
return Take();
}
return std::nullopt;
}
void Fill(T item) {
{
std::unique_lock<std::mutex> lock(mu_);
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!item_, "Promise filled twice!");
item_ = item;
} // lock released before condition variable notification
cv_.notify_all();
}
bool IsAwaited() const {
std::unique_lock<std::mutex> lock(mu_);
return waiting_;
}
};
} // namespace details
template <typename T>
class Future {
bool consumed_or_moved_ = false;
std::shared_ptr<details::Shared<T>> shared_;
public:
explicit Future(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Future() = delete;
Future(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
consumed_or_moved_ = old.consumed_or_moved_;
old.consumed_or_moved_ = true;
}
Future &operator=(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
old.consumed_or_moved_ = true;
}
Future(const Future &) = delete;
Future &operator=(const Future &) = delete;
~Future() = default;
/// Returns true if the Future is ready to
/// be consumed using TryGet or Wait (prefer Wait
/// if you know it's ready, because it doesn't
/// return an optional.
bool IsReady() {
MG_ASSERT(!consumed_or_moved_, "Called IsReady after Future already consumed!");
return shared_->IsReady();
}
/// Non-blocking method that returns the inner
/// item if it's already ready, or std::nullopt
/// if it is not ready yet.
std::optional<T> TryGet() {
MG_ASSERT(!consumed_or_moved_, "Called TryGet after Future already consumed!");
std::optional<T> ret = shared_->TryGet();
if (ret) {
consumed_or_moved_ = true;
}
return ret;
}
/// Block on the corresponding promise to be filled,
/// returning the inner item when ready.
T Wait() && {
MG_ASSERT(!consumed_or_moved_, "Future should only be consumed with Wait once!");
T ret = shared_->Wait();
consumed_or_moved_ = true;
return ret;
}
/// Marks this Future as canceled.
void Cancel() {
MG_ASSERT(!consumed_or_moved_, "Future::Cancel called on a future that was already moved or consumed!");
consumed_or_moved_ = true;
}
};
template <typename T>
class Promise {
std::shared_ptr<details::Shared<T>> shared_;
bool filled_or_moved_ = false;
public:
explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Promise() = delete;
Promise(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise &operator=(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise(const Promise &) = delete;
Promise &operator=(const Promise &) = delete;
~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
// Fill the expected item into the Future.
void Fill(T item) {
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
shared_->Fill(item);
filled_or_moved_ = true;
}
bool IsAwaited() { return shared_->IsAwaited(); }
/// Moves this Promise into a unique_ptr.
std::unique_ptr<Promise<T>> ToUnique() && {
std::unique_ptr<Promise<T>> up = std::make_unique<Promise<T>>(std::move(shared_));
filled_or_moved_ = true;
return up;
}
};
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePair() {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>();
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
}; // namespace memgraph::io

View File

@@ -0,0 +1,25 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "coordinator/coordinator.hpp"
#include "io/rsm/raft.hpp"
namespace memgraph::io::rsm {
// TODO(tyler) don't
using namespace memgraph::coordinator;
template <typename IoImpl>
using CoordinatorRsm = io::rsm::Raft<IoImpl, Coordinator, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
} // namespace memgraph::io::rsm

845
src/io/rsm/raft.hpp Normal file
View File

@@ -0,0 +1,845 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// TODO(tyler) buffer out-of-order Append buffers on the Followers to reassemble more quickly
// TODO(tyler) handle granular batch sizes based on simple flow control
#pragma once
#include <deque>
#include <iostream>
#include <map>
#include <set>
#include <thread>
#include <unordered_map>
#include <vector>
#include "io/simulator/simulator.hpp"
#include "io/transport.hpp"
namespace memgraph::io::rsm {
using memgraph::io::Address;
using memgraph::io::Duration;
using memgraph::io::Io;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::Time;
using Term = uint64_t;
using LogIndex = uint64_t;
using RequestId = uint64_t;
template <typename WriteOperation>
struct WriteRequest {
WriteOperation operation;
};
/// WriteResponse is returned to a client after
/// their WriteRequest was entered in to the raft
/// log and it reached consensus.
///
/// WriteReturn is the result of applying the WriteRequest to
/// ReplicatedState, and if the ReplicatedState::write
/// method is deterministic, all replicas will
/// have the same ReplicatedState after applying
/// the submitted WriteRequest.
template <typename WriteReturn>
struct WriteResponse {
bool success;
WriteReturn write_return;
std::optional<Address> retry_leader;
};
template <typename ReadOperation>
struct ReadRequest {
ReadOperation operation;
};
template <typename ReadReturn>
struct ReadResponse {
bool success;
ReadReturn read_return;
std::optional<Address> retry_leader;
};
/// AppendRequest is a raft-level message that the Leader
/// periodically broadcasts to all Follower peers. This
/// serves three main roles:
/// 1. acts as a heartbeat from the Leader to the Follower
/// 2. replicates new data that the Leader has received to the Follower
/// 3. informs Follower peers when the commit index has increased,
/// signalling that it is now safe to apply log items to the
/// replicated state machine
template <typename WriteRequest>
struct AppendRequest {
Term term = 0;
LogIndex last_log_index;
Term last_log_term;
std::vector<std::pair<Term, WriteRequest>> entries;
LogIndex leader_commit;
};
struct AppendResponse {
bool success;
Term term;
Term last_log_term;
// a small optimization over the raft paper, tells
// the leader the offset that we are interested in
// to send log offsets from for us. This will only
// be useful at the beginning of a leader's term.
LogIndex last_log_index;
};
struct VoteRequest {
Term term = 0;
LogIndex last_log_index;
Term last_log_term;
};
struct VoteResponse {
Term term = 0;
LogIndex committed_log_size;
bool vote_granted = false;
};
template <typename WriteRequest>
struct CommonState {
Term term = 0;
std::vector<std::pair<Term, WriteRequest>> log;
LogIndex committed_log_size = 0;
LogIndex applied_size = 0;
};
struct FollowerTracker {
LogIndex next_index = 0;
LogIndex confirmed_contiguous_index = 0;
};
struct PendingClientRequest {
RequestId request_id;
Address address;
Time received_at;
};
struct Leader {
std::map<Address, FollowerTracker> followers;
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
Time last_broadcast = Time::min();
std::string ToString() { return "\tLeader \t"; }
};
struct Candidate {
std::map<Address, LogIndex> successful_votes;
Time election_began = Time::min();
std::set<Address> outstanding_votes;
std::string ToString() { return "\tCandidate\t"; }
};
struct Follower {
Time last_received_append_entries_timestamp;
Address leader_address;
std::string ToString() { return "\tFollower \t"; }
};
using Role = std::variant<Candidate, Leader, Follower>;
/*
all ReplicatedState classes should have an Apply method
that returns our WriteResponseValue after consensus, and
a Read method that returns our ReadResponseValue without
requiring consensus.
ReadResponse Read(ReadOperation);
WriteResponseValue ReplicatedState::Apply(WriteRequest);
For example:
If the state is uint64_t, and WriteRequest is `struct PlusOne {};`,
and WriteResponseValue is also uint64_t (the new value), then
each call to state.Apply(PlusOne{}) will return the new value
after incrementing it. 0, 1, 2, 3... and this will be sent back
to the client that requested the mutation.
In practice, these mutations will usually be predicated on some
previous value, so that they are idempotent, functioning similarly
to a CAS operation.
*/
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
typename ReadResponseValue>
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
};
/// Parameter Purpose
/// --------------------------
/// IoImpl the concrete Io provider - SimulatorTransport, ThriftTransport, etc...
/// ReplicatedState the high-level data structure that is managed by the raft-backed replicated state machine
/// WriteOperation the individual operation type that is applied to the ReplicatedState in identical order
/// across each replica
/// WriteResponseValue the return value of calling ReplicatedState::Apply(WriteOperation), which is executed in
/// identical order across all replicas after an WriteOperation reaches consensus.
/// ReadOperation the type of operations that do not require consensus before executing directly
/// on a const ReplicatedState &
/// ReadResponseValue the return value of calling ReplicatedState::Read(ReadOperation), which is executed directly
/// without going through consensus first
template <typename IoImpl, typename ReplicatedState, typename WriteOperation, typename WriteResponseValue,
typename ReadOperation, typename ReadResponseValue>
requires Rsm<WriteOperation, ReadOperation, ReplicatedState, WriteResponseValue, ReadResponseValue>
class Raft {
CommonState<WriteOperation> state_;
Role role_ = Candidate{};
Io<IoImpl> io_;
std::vector<Address> peers_;
ReplicatedState replicated_state_;
public:
Raft(Io<IoImpl> &&io, std::vector<Address> peers, ReplicatedState &&replicated_state)
: io_(std::move(io)), peers_(peers), replicated_state_(std::move(replicated_state)) {}
void Run() {
Time last_cron = io_.Now();
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
const Duration random_cron_interval = RandomTimeout(1000, 2000);
if (now - last_cron > random_cron_interval) {
Cron();
last_cron = now;
}
Duration receive_timeout = RandomTimeout(10000, 50000);
auto request_result =
io_.template ReceiveWithTimeout<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse>(receive_timeout);
if (request_result.HasError()) {
continue;
}
auto request = std::move(request_result.GetValue());
Handle(std::move(request.message), request.request_id, request.from_address);
}
}
private:
// Raft paper - 5.3
// When the entry has been safely replicated, the leader applies the
// entry to its state machine and returns the result of that
// execution to the client.
//
// "Safely replicated" is defined as being known to be present
// on at least a majority of all peers (inclusive of the Leader).
void BumpCommitIndexAndReplyToClients(Leader &leader) {
auto indices = std::vector<LogIndex>{};
// We include our own log size in the calculation of the log
// index that is present on at least a majority of all peers.
indices.push_back(state_.log.size());
for (const auto &[addr, f] : leader.followers) {
indices.push_back(f.confirmed_contiguous_index);
Log("at port ", addr.last_known_port, " has confirmed contiguous index of: ", f.confirmed_contiguous_index);
}
// reverse sort from highest to lowest (using std::ranges::greater)
std::ranges::sort(indices, std::ranges::greater());
// This is a particularly correctness-critical calculation because it
// determines which index we will consider to be the committed index.
//
// If the following indexes are recorded for clusters of different sizes,
// these are the expected indexes that are considered to have reached
// consensus:
// state | expected value | (indices.size() / 2)
// [1] 1 (1 / 2) => 0
// [2, 1] 1 (2 / 2) => 1
// [3, 2, 1] 2 (3 / 2) => 1
// [4, 3, 2, 1] 2 (4 / 2) => 2
// [5, 4, 3, 2, 1] 3 (5 / 2) => 2
size_t index_present_on_majority = indices.size() / 2;
LogIndex new_committed_log_size = indices[index_present_on_majority];
// We never go backwards in history.
MG_ASSERT(state_.committed_log_size <= new_committed_log_size);
state_.committed_log_size = new_committed_log_size;
// For each index between the old index and the new one (inclusive),
// Apply that log's WriteOperation to our replicated_state_,
// and use the specific return value of the ReplicatedState::Apply
// method (WriteResponseValue) to respond to the requester.
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const LogIndex apply_index = state_.applied_size;
const auto &write_request = state_.log[apply_index].second;
WriteResponseValue write_return = replicated_state_.Apply(write_request);
if (leader.pending_client_requests.contains(apply_index)) {
PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
WriteResponse<WriteResponseValue> resp;
resp.success = true;
resp.write_return = std::move(write_return);
io_.Send(client_request.address, client_request.request_id, std::move(resp));
leader.pending_client_requests.erase(apply_index);
}
}
Log("committed_log_size is now ", state_.committed_log_size);
}
// Raft paper - 5.1
// AppendEntries RPCs are initiated by leaders to replicate log entries and to provide a form of heartbeat
void BroadcastAppendEntries(std::map<Address, FollowerTracker> &followers) {
for (auto &[address, follower] : followers) {
const LogIndex index = follower.confirmed_contiguous_index;
std::vector<std::pair<Term, WriteOperation>> entries;
if (state_.log.size() > index) {
entries.insert(entries.begin(), state_.log.begin() + index, state_.log.end());
}
const Term previous_term_from_index = PreviousTermFromIndex(index);
Log("sending ", entries.size(), " entries to Follower ", address.last_known_port,
" which are above its known index of ", index);
AppendRequest<WriteOperation> ar{
.term = state_.term,
.last_log_index = index,
.last_log_term = previous_term_from_index,
.entries = entries,
.leader_commit = state_.committed_log_size,
};
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr RequestId request_id = 0;
io_.Send(address, request_id, ar);
}
}
// Raft paper - 5.2
// Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly
Duration RandomTimeout(Duration min, Duration max) {
std::uniform_int_distribution time_distrib(min.count(), max.count());
auto rand_micros = io_.Rand(time_distrib);
return std::chrono::microseconds{rand_micros};
}
Duration RandomTimeout(int min_micros, int max_micros) {
std::uniform_int_distribution time_distrib(min_micros, max_micros);
int rand_micros = io_.Rand(time_distrib);
return std::chrono::microseconds{rand_micros};
}
Term PreviousTermFromIndex(LogIndex index) const {
if (index == 0 || state_.log.size() + 1 <= index) {
return 0;
}
const auto &[term, data] = state_.log.at(index - 1);
return term;
}
LogIndex CommittedLogIndex() { return state_.committed_log_size; }
Term CommittedLogTerm() {
MG_ASSERT(state_.log.size() >= state_.committed_log_size);
if (state_.log.empty() || state_.committed_log_size == 0) {
return 0;
}
auto &[term, data] = state_.log.at(state_.committed_log_size - 1);
return term;
}
LogIndex LastLogIndex() { return state_.log.size(); }
Term LastLogTerm() const {
if (state_.log.empty()) {
return 0;
}
const auto &[term, data] = state_.log.back();
return term;
}
template <typename... Ts>
void Log(Ts &&...args) {
const Time now = io_.Now();
auto micros = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
const Term term = state_.term;
std::ostringstream out;
out << '\t' << (int)micros << "\t" << term << "\t" << io_.GetAddress().last_known_port;
std::string role_string = std::visit([&](auto &&role) { return role.ToString(); }, role_);
out << role_string;
(out << ... << args);
spdlog::debug(out.str());
}
/////////////////////////////////////////////////////////////
/// Raft-related Cron methods
///
/// Cron + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Cron(role) takes as the first argument a reference to its
/// role, and as the second argument, the message that has
/// been received.
/////////////////////////////////////////////////////////////
/// Periodic protocol maintenance.
void Cron() {
// dispatch periodic logic based on our role to a specific Cron method.
std::optional<Role> new_role = std::visit([&](auto &&role) { return Cron(role); }, role_);
if (new_role) {
role_ = std::move(new_role).value();
}
}
// Raft paper - 5.2
// Candidates keep sending Vote to peers until:
// 1. receiving Append with a higher term (become Follower)
// 2. receiving Vote with a higher term (become a Follower)
// 3. receiving a quorum of responses to our last batch of Vote (become a Leader)
std::optional<Role> Cron(Candidate &candidate) {
const auto now = io_.Now();
const Duration election_timeout = RandomTimeout(100000, 200000);
const auto election_timeout_us = std::chrono::duration_cast<std::chrono::milliseconds>(election_timeout).count();
if (now - candidate.election_began > election_timeout) {
state_.term++;
Log("becoming Candidate for term ", state_.term, " after leader timeout of ", election_timeout_us,
" elapsed since last election attempt");
const VoteRequest request{
.term = state_.term,
.last_log_index = LastLogIndex(),
.last_log_term = LastLogTerm(),
};
auto outstanding_votes = std::set<Address>();
for (const auto &peer : peers_) {
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr auto request_id = 0;
io_.template Send<VoteRequest>(peer, request_id, request);
outstanding_votes.insert(peer);
}
return Candidate{
.successful_votes = std::map<Address, LogIndex>(),
.election_began = now,
.outstanding_votes = outstanding_votes,
};
}
return std::nullopt;
}
// Raft paper - 5.2
// Followers become candidates if we haven't heard from the leader
// after a randomized timeout.
std::optional<Role> Cron(Follower &follower) {
const auto now = io_.Now();
const auto time_since_last_append_entries = now - follower.last_received_append_entries_timestamp;
Duration election_timeout = RandomTimeout(100000, 200000);
// randomized follower timeout with a range of 100-150ms.
if (time_since_last_append_entries > election_timeout) {
// become a Candidate if we haven't heard from the Leader after this timeout
return Candidate{};
}
return std::nullopt;
}
// Leaders (re)send AppendRequest to followers.
std::optional<Role> Cron(Leader &leader) {
const Time now = io_.Now();
const Duration broadcast_timeout = RandomTimeout(40000, 60000);
if (now - leader.last_broadcast > broadcast_timeout) {
BroadcastAppendEntries(leader.followers);
leader.last_broadcast = now;
}
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// Raft-related Handle methods
///
/// Handle + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Handle(role, message, ...)
/// takes as the first argument a reference
/// to its role, and as the second argument, the
/// message that has been received.
/////////////////////////////////////////////////////////////
void Handle(std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse> &&message_variant,
RequestId request_id, Address from_address) {
// dispatch the message to a handler based on our role,
// which can be specified in the Handle first argument,
// or it can be `auto` if it's a handler for several roles
// or messages.
std::optional<Role> new_role =
std::visit([&](auto &&msg, auto &&role) { return Handle(role, std::move(msg), request_id, from_address); },
std::move(message_variant), role_);
// TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
if (new_role) {
role_ = std::move(new_role).value();
}
}
// all roles can receive Vote and possibly become a follower
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &, VoteRequest &&req, RequestId request_id, Address from_address) {
Log("received Vote from ", from_address.last_known_port, " with term ", req.term);
const bool last_log_term_dominates = req.last_log_term >= LastLogTerm();
const bool term_dominates = req.term > state_.term;
const bool last_log_index_dominates = req.last_log_index >= LastLogIndex();
const bool new_leader = last_log_term_dominates && term_dominates && last_log_index_dominates;
if (new_leader) {
MG_ASSERT(req.term > state_.term);
MG_ASSERT(std::max(req.term, state_.term) == req.term);
}
const VoteResponse res{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
};
io_.Send(from_address, request_id, res);
if (new_leader) {
// become a follower
state_.term = req.term;
return Follower{
.last_received_append_entries_timestamp = io_.Now(),
.leader_address = from_address,
};
} else if (term_dominates) {
Log("received a vote from an inferior candidate. Becoming Candidate");
state_.term = std::max(state_.term, req.term) + 1;
return Candidate{};
}
return std::nullopt;
}
std::optional<Role> Handle(Candidate &candidate, VoteResponse &&res, RequestId, Address from_address) {
Log("received VoteResponse");
if (!res.vote_granted || res.term != state_.term) {
Log("received unsuccessful VoteResponse from term ", res.term, " when our candidacy term is ", state_.term);
// we received a delayed VoteResponse from the past, which has to do with an election that is
// no longer valid. We can simply drop this.
return std::nullopt;
}
MG_ASSERT(candidate.outstanding_votes.contains(from_address),
"Received unexpected VoteResponse from server not present in Candidate's outstanding_votes!");
candidate.outstanding_votes.erase(from_address);
MG_ASSERT(!candidate.successful_votes.contains(from_address),
"Received unexpected VoteResponse from server already in Candidate's successful_votes!");
candidate.successful_votes.insert({from_address, res.committed_log_size});
if (candidate.successful_votes.size() >= candidate.outstanding_votes.size()) {
std::map<Address, FollowerTracker> followers{};
for (const auto &[address, committed_log_size] : candidate.successful_votes) {
FollowerTracker follower{
.next_index = committed_log_size,
.confirmed_contiguous_index = committed_log_size,
};
followers.insert({address, follower});
}
for (const auto &address : candidate.outstanding_votes) {
FollowerTracker follower{
.next_index = state_.log.size(),
.confirmed_contiguous_index = 0,
};
followers.insert({address, follower});
}
Log("becoming Leader at term ", state_.term);
BroadcastAppendEntries(followers);
return Leader{
.followers = std::move(followers),
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
};
}
return std::nullopt;
}
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &, VoteResponse &&, RequestId, Address) {
Log("non-Candidate received VoteResponse");
return std::nullopt;
}
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &role, AppendRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
AppendResponse res{
.success = false,
.term = state_.term,
.last_log_term = CommittedLogTerm(),
.last_log_index = CommittedLogIndex(),
};
if constexpr (std::is_same<AllRoles, Leader>()) {
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
}
const bool is_candidate = std::is_same<AllRoles, Candidate>();
const bool is_failed_competitor = is_candidate && req.term == state_.term;
const Time now = io_.Now();
// Raft paper - 5.2
// While waiting for votes, a candidate may receive an
// AppendEntries RPC from another server claiming to be leader. If
// the leaders term (included in its RPC) is at least as large as
// the candidates current term, then the candidate recognizes the
// leader as legitimate and returns to follower state.
if (req.term > state_.term || is_failed_competitor) {
// become follower of this leader, reply with our log status
state_.term = req.term;
io_.Send(from_address, request_id, res);
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
return Follower{
.last_received_append_entries_timestamp = now,
.leader_address = from_address,
};
} else if (req.term < state_.term) {
// nack this request from an old leader
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// at this point, we're dealing with our own leader
if constexpr (std::is_same<AllRoles, Follower>()) {
// small specialization for when we're already a Follower
MG_ASSERT(role.leader_address == from_address, "Multiple Leaders are acting under the same term number!");
role.last_received_append_entries_timestamp = now;
} else {
Log("Somehow entered Follower-specific logic as a non-Follower");
MG_ASSERT(false, "Somehow entered Follower-specific logic as a non-Follower");
}
res.last_log_term = LastLogTerm();
res.last_log_index = LastLogIndex();
Log("returning last_log_index of ", res.last_log_index);
// Handle steady-state conditions.
if (req.last_log_index != LastLogIndex()) {
Log("req.last_log_index is above our last applied log index");
} else if (req.last_log_term != LastLogTerm()) {
Log("req.last_log_term differs from our leader term at that slot, expected: ", LastLogTerm(), " but got ",
req.last_log_term);
} else {
// happy path - Apply log
Log("applying batch of entries to log of size ", req.entries.size());
MG_ASSERT(req.last_log_index >= state_.committed_log_size,
"Applied history from Leader which goes back in time from our commit_index");
// possibly chop-off stuff that was replaced by
// things with different terms (we got data that
// hasn't reached consensus yet, which is normal)
state_.log.resize(req.last_log_index);
state_.log.insert(state_.log.end(), req.entries.begin(), req.entries.end());
MG_ASSERT(req.leader_commit >= state_.committed_log_size);
state_.committed_log_size = std::min(req.leader_commit, LastLogIndex());
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const auto &write_request = state_.log[state_.applied_size].second;
replicated_state_.Apply(write_request);
}
res.success = true;
}
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Leader &leader, AppendResponse &&res, RequestId, Address from_address) {
if (res.term != state_.term) {
} else if (!leader.followers.contains(from_address)) {
Log("received AppendResponse from unknown Follower");
MG_ASSERT(false, "received AppendResponse from unknown Follower");
} else {
if (res.success) {
Log("got successful AppendResponse from ", from_address.last_known_port, " with last_log_index of ",
res.last_log_index);
} else {
Log("got unsuccessful AppendResponse from ", from_address.last_known_port, " with last_log_index of ",
res.last_log_index);
}
FollowerTracker &follower = leader.followers.at(from_address);
follower.next_index = std::max(follower.next_index, res.last_log_index);
follower.confirmed_contiguous_index = std::max(follower.confirmed_contiguous_index, res.last_log_index);
BumpCommitIndexAndReplyToClients(leader);
}
return std::nullopt;
}
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &, AppendResponse &&, RequestId, Address) {
// we used to be the leader, and are getting old delayed responses
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// RSM-related handle methods
/////////////////////////////////////////////////////////////
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
std::optional<Role> Handle(Leader &, ReadRequest<ReadOperation> &&req, RequestId request_id, Address from_address) {
Log("handling ReadOperation");
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(read_operation);
ReadResponse<ReadResponseValue> resp{
.success = true,
.read_return = std::move(read_return),
.retry_leader = std::nullopt,
};
io_.Send(from_address, request_id, resp);
return std::nullopt;
}
// Candidates should respond with a failure, similar to the Candidate + WriteRequest failure below
std::optional<Role> Handle(Candidate &, ReadRequest<ReadOperation> &&, RequestId request_id, Address from_address) {
Log("received ReadOperation - not redirecting because no Leader is known");
auto res = ReadResponse<ReadResponseValue>{};
res.success = false;
Cron();
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// Followers should respond with a redirection, similar to the Follower + WriteRequest response below
std::optional<Role> Handle(Follower &follower, ReadRequest<ReadOperation> &&, RequestId request_id,
Address from_address) {
auto res = ReadResponse<ReadResponseValue>{};
res.success = false;
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
res.retry_leader = follower.leader_address;
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// Raft paper - 8
// When a client first starts up, it connects to a randomly chosen
// server. If the clients first choice is not the leader, that
// server will reject the clients request and supply information
// about the most recent leader it has heard from.
std::optional<Role> Handle(Follower &follower, WriteRequest<WriteOperation> &&, RequestId request_id,
Address from_address) {
auto res = WriteResponse<WriteResponseValue>{};
res.success = false;
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
res.retry_leader = follower.leader_address;
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Candidate &, WriteRequest<WriteOperation> &&, RequestId request_id, Address from_address) {
Log("received WriteRequest - not redirecting because no Leader is known");
auto res = WriteResponse<WriteResponseValue>{};
res.success = false;
Cron();
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// only leaders actually handle replication requests from clients
std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
Log("handling WriteRequest");
// we are the leader. add item to log and send Append to peers
state_.log.emplace_back(std::pair(state_.term, std::move(req.operation)));
LogIndex log_index = state_.log.size() - 1;
PendingClientRequest pcr{
.request_id = request_id,
.address = from_address,
.received_at = io_.Now(),
};
leader.pending_client_requests.emplace(log_index, pcr);
BroadcastAppendEntries(leader.followers);
return std::nullopt;
}
};
}; // namespace memgraph::io::rsm

137
src/io/rsm/rsm_client.hpp Normal file
View File

@@ -0,0 +1,137 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <iostream>
#include <optional>
#include <vector>
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "utils/result.hpp"
namespace memgraph::io::rsm {
using memgraph::io::Address;
using memgraph::io::Duration;
using memgraph::io::ResponseEnvelope;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::Time;
using memgraph::io::TimedOut;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::ReadResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::utils::BasicResult;
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
typename ReadResponseT>
class RsmClient {
using ServerPool = std::vector<Address>;
IoImpl io_;
Address leader_;
std::mt19937 cli_rng_{0};
ServerPool server_addrs_;
template <typename ResponseT>
void PossiblyRedirectLeader(const ResponseT &response) {
if (response.retry_leader) {
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
leader_ = response.retry_leader.value();
spdlog::debug("client redirected to leader server {}", leader_.ToString());
} else if (!response.success) {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = addr_distrib(cli_rng_);
leader_ = server_addrs_[addr_index];
spdlog::debug(
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
"a RSM Candidate) trying a random one at index {} with address {}",
addr_index, leader_.ToString());
}
}
public:
RsmClient(IoImpl io, Address leader, ServerPool server_addrs)
: io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
RsmClient() = delete;
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
WriteRequest<WriteRequestT> client_req;
client_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
if (response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
// continue;
return response_result.GetError();
}
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
if (write_response.success) {
return std::move(write_response.write_return);
}
PossiblyRedirectLeader(write_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
ReadRequest<ReadRequestT> read_req;
read_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
// receive response
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
if (get_response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return get_response_result.GetError();
}
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
if (read_get_response.success) {
return std::move(read_get_response.read_return);
}
PossiblyRedirectLeader(read_get_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
};
} // namespace memgraph::io::rsm

165
src/io/rsm/shard_rsm.hpp Normal file
View File

@@ -0,0 +1,165 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
/// The StorageRsm is a simple in-memory raft-backed kv store that can be used for simple testing
/// and implementation of some query engine logic before storage engines are fully implemented.
///
/// To implement multiple read and write commands, change the StorageRead* and StorageWrite* requests
/// and responses to a std::variant of the different options, and route them to specific handlers in
/// the StorageRsm's Read and Apply methods. Remember that Read is called immediately when the Raft
/// leader receives the request, and does not replicate anything over Raft. Apply is called only
/// AFTER the StorageWriteRequest is replicated to a majority of Raft peers, and the result of calling
/// StorageRsm::Apply(StorageWriteRequest) is returned to the client that submitted the request.
#include <deque>
#include <iostream>
#include <map>
#include <optional>
#include <set>
#include <thread>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "io/simulator/simulator.hpp"
#include "io/simulator/simulator_transport.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/logging.hpp"
using memgraph::coordinator::Hlc;
using memgraph::io::Address;
using memgraph::io::Io;
using memgraph::io::ResponseEnvelope;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::rsm::Raft;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::ReadResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::io::simulator::Simulator;
using memgraph::io::simulator::SimulatorConfig;
using memgraph::io::simulator::SimulatorStats;
using memgraph::io::simulator::SimulatorTransport;
using memgraph::storage::PropertyValue;
namespace memgraph::io::rsm {
using ShardRsmKey = std::vector<PropertyValue>;
struct StorageWriteRequest {
ShardRsmKey key;
std::optional<int> value;
};
struct StorageWriteResponse {
bool shard_rsm_success;
std::optional<int> last_value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
struct StorageReadRequest {
ShardRsmKey key;
};
struct StorageReadResponse {
bool shard_rsm_success;
std::optional<int> value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
class StorageRsm {
std::map<ShardRsmKey, int> state_;
ShardRsmKey minimum_key_;
std::optional<ShardRsmKey> maximum_key_{std::nullopt};
Hlc shard_map_version_;
// The key is not located in this shard
bool IsKeyInRange(const ShardRsmKey &key) {
if (maximum_key_) [[likely]] {
return (key >= minimum_key_ && key <= maximum_key_);
}
return key >= minimum_key_;
}
public:
StorageReadResponse Read(StorageReadRequest request) {
StorageReadResponse ret;
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
} else if (state_.contains(request.key)) {
ret.value = state_[request.key];
ret.shard_rsm_success = true;
} else {
ret.shard_rsm_success = false;
ret.value = std::nullopt;
}
return ret;
}
StorageWriteResponse Apply(StorageWriteRequest request) {
StorageWriteResponse ret;
// Key is outside the prohibited range
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
}
// Key exist
else if (state_.contains(request.key)) {
auto &val = state_[request.key];
/*
* Delete
*/
if (!request.value) {
ret.shard_rsm_success = true;
ret.last_value = val;
state_.erase(state_.find(request.key));
}
/*
* Update
*/
// Does old_value match?
if (request.value == val) {
ret.last_value = val;
ret.shard_rsm_success = true;
val = request.value.value();
} else {
ret.last_value = val;
ret.shard_rsm_success = false;
}
}
/*
* Create
*/
else {
ret.last_value = std::nullopt;
ret.shard_rsm_success = true;
state_.emplace(request.key, std::move(request.value).value());
}
return ret;
}
};
} // namespace memgraph::io::rsm

View File

@@ -0,0 +1,8 @@
set(io_simulator_sources
simulator_handle.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-io-simulator STATIC ${io_simulator_sources})
target_link_libraries(mg-io-simulator stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@@ -0,0 +1,177 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/transport.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Message;
using memgraph::io::Time;
struct OpaqueMessage {
Address from_address;
uint64_t request_id;
std::any message;
/// Recursively tries to match a specific type from the outer
/// variant's parameter pack against the type of the std::any,
/// and if it matches, make it concrete and return it. Otherwise,
/// move on and compare the any with the next type from the
/// parameter pack.
///
/// Return is the full std::variant<Ts...> type that holds the
/// full parameter pack without interfering with recursive
/// narrowing expansion.
template <typename Return, Message Head, Message... Rest>
std::optional<Return> Unpack(std::any &&a) {
if (typeid(Head) == a.type()) {
Head concrete = std::any_cast<Head>(std::move(a));
return concrete;
}
if constexpr (sizeof...(Rest) > 0) {
return Unpack<Return, Rest...>(std::move(a));
} else {
return std::nullopt;
}
}
/// High level "user-facing" conversion function that lets
/// people interested in conversion only supply a single
/// parameter pack for the types that they want to compare
/// with the any and potentially include in the returned
/// variant.
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
if (m_opt) {
return RequestEnvelope<Ms...>{
.message = std::move(*m_opt),
.request_id = request_id,
.from_address = from_address,
};
}
return std::nullopt;
}
};
class OpaquePromiseTraitBase {
public:
virtual const std::type_info *TypeInfo() const = 0;
virtual bool IsAwaited(void *ptr) const = 0;
virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
virtual void TimeOut(void *ptr) const = 0;
virtual ~OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase &operator=(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase(OpaquePromiseTraitBase &&old) = delete;
OpaquePromiseTraitBase &operator=(OpaquePromiseTraitBase &&) = delete;
};
template <typename T>
class OpaquePromiseTrait : public OpaquePromiseTraitBase {
public:
const std::type_info *TypeInfo() const override { return &typeid(T); };
bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
T message = std::any_cast<T>(std::move(opaque_message.message));
auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
.request_id = opaque_message.request_id,
.from_address = opaque_message.from_address};
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
unique_promise->Fill(std::move(response_envelope));
};
void TimeOut(void *ptr) const override {
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
ResponseResult<T> result = TimedOut{};
unique_promise->Fill(std::move(result));
}
};
class OpaquePromise {
void *ptr_;
std::unique_ptr<OpaquePromiseTraitBase> trait_;
public:
OpaquePromise(OpaquePromise &&old) noexcept : ptr_(old.ptr_), trait_(std::move(old.trait_)) {
MG_ASSERT(old.ptr_ != nullptr);
old.ptr_ = nullptr;
}
OpaquePromise &operator=(OpaquePromise &&old) noexcept {
MG_ASSERT(ptr_ == nullptr);
MG_ASSERT(old.ptr_ != nullptr);
MG_ASSERT(this != &old);
ptr_ = old.ptr_;
trait_ = std::move(old.trait_);
old.ptr_ = nullptr;
return *this;
}
OpaquePromise(const OpaquePromise &) = delete;
OpaquePromise &operator=(const OpaquePromise &) = delete;
template <typename T>
std::unique_ptr<ResponsePromise<T>> Take() && {
MG_ASSERT(typeid(T) == *trait_->TypeInfo());
MG_ASSERT(ptr_ != nullptr);
auto ptr = static_cast<ResponsePromise<T> *>(ptr_);
ptr_ = nullptr;
return std::unique_ptr<T>(ptr);
}
template <typename T>
explicit OpaquePromise(std::unique_ptr<ResponsePromise<T>> promise)
: ptr_(static_cast<void *>(promise.release())), trait_(std::make_unique<OpaquePromiseTrait<T>>()) {}
bool IsAwaited() {
MG_ASSERT(ptr_ != nullptr);
return trait_->IsAwaited(ptr_);
}
void TimeOut() {
MG_ASSERT(ptr_ != nullptr);
trait_->TimeOut(ptr_);
ptr_ = nullptr;
}
void Fill(OpaqueMessage &&opaque_message) {
MG_ASSERT(ptr_ != nullptr);
trait_->Fill(ptr_, std::move(opaque_message));
ptr_ = nullptr;
}
~OpaquePromise() {
MG_ASSERT(ptr_ == nullptr, "OpaquePromise destroyed without being explicitly timed out or filled");
}
};
} // namespace memgraph::io::simulator

View File

@@ -0,0 +1,51 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include <random>
#include "io/address.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/simulator/simulator_transport.hpp"
namespace memgraph::io::simulator {
class Simulator {
std::mt19937 rng_;
std::shared_ptr<SimulatorHandle> simulator_handle_;
uint16_t auto_port_ = 0;
public:
explicit Simulator(SimulatorConfig config)
: rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
void ShutDown() { simulator_handle_->ShutDown(); }
Io<SimulatorTransport> RegisterNew() {
Address address = Address::TestAddress(auto_port_++);
return Register(address);
}
Io<SimulatorTransport> Register(Address address) {
std::uniform_int_distribution<uint64_t> seed_distrib;
uint64_t seed = seed_distrib(rng_);
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
}
void IncrementServerCountAndWaitForQuiescentState(Address address) {
simulator_handle_->IncrementServerCountAndWaitForQuiescentState(address);
}
SimulatorStats Stats() { return simulator_handle_->Stats(); }
};
}; // namespace memgraph::io::simulator

View File

@@ -0,0 +1,30 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include "io/time.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Time;
struct SimulatorConfig {
uint8_t drop_percent = 0;
bool perform_timeouts = false;
bool scramble_messages = true;
uint64_t rng_seed = 0;
Time start_time = Time::min();
Time abort_time = Time::max();
};
}; // namespace memgraph::io::simulator

View File

@@ -0,0 +1,154 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "io/simulator/simulator_handle.hpp"
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
void SimulatorHandle::ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool SimulatorHandle::ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
std::unique_lock<std::mutex> lock(mu_);
server_addresses_.insert(address);
while (true) {
const size_t blocked_servers = BlockedServers();
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
if (all_servers_blocked) {
return;
}
cv_.wait(lock);
}
}
size_t SimulatorHandle::BlockedServers() {
size_t blocked_servers = blocked_on_receive_;
for (auto &[promise_key, opaque_promise] : promises_) {
if (opaque_promise.promise.IsAwaited() && server_addresses_.contains(promise_key.requester_address)) {
blocked_servers++;
}
}
return blocked_servers;
}
bool SimulatorHandle::MaybeTickSimulator() {
std::unique_lock<std::mutex> lock(mu_);
const size_t blocked_servers = BlockedServers();
if (blocked_servers < server_addresses_.size()) {
// we only need to advance the simulator when all
// servers have reached a quiescent state, blocked
// on their own futures or receive methods.
return false;
}
stats_.simulator_ticks++;
cv_.notify_all();
TimeoutPromisesPastDeadline();
if (in_flight_.empty()) {
// return early here because there are no messages to schedule
// We tick the clock forward when all servers are blocked but
// there are no in-flight messages to schedule delivery of.
std::poisson_distribution<> time_distrib(50);
Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
cluster_wide_time_microseconds_ += clock_advance;
MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time.");
return true;
}
if (config_.scramble_messages) {
// scramble messages
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
const size_t swap_index = swap_distrib(rng_);
std::swap(in_flight_[swap_index], in_flight_.back());
}
auto [to_address, opaque_message] = std::move(in_flight_.back());
in_flight_.pop_back();
std::uniform_int_distribution<int> drop_distrib(0, 99);
const int drop_threshold = drop_distrib(rng_);
const bool should_drop = drop_threshold < config_.drop_percent;
if (should_drop) {
stats_.dropped_messages++;
}
PromiseKey promise_key{.requester_address = to_address,
.request_id = opaque_message.request_id,
.replier_address = opaque_message.from_address};
if (promises_.contains(promise_key)) {
// complete waiting promise if it's there
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
const bool normal_timeout = config_.perform_timeouts && (dop.deadline < cluster_wide_time_microseconds_);
if (should_drop || normal_timeout) {
stats_.timed_out_requests++;
dop.promise.TimeOut();
} else {
stats_.total_responses++;
dop.promise.Fill(std::move(opaque_message));
}
} else if (should_drop) {
// don't add it anywhere, let it drop
} else {
// add to can_receive_ if not
const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
om_vec->second.emplace_back(std::move(opaque_message));
}
return true;
}
Time SimulatorHandle::Now() const {
std::unique_lock<std::mutex> lock(mu_);
return cluster_wide_time_microseconds_;
}
SimulatorStats SimulatorHandle::Stats() {
std::unique_lock<std::mutex> lock(mu_);
return stats_;
}
} // namespace memgraph::io::simulator

View File

@@ -0,0 +1,206 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <any>
#include <compare>
#include <iostream>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <utility>
#include <variant>
#include <vector>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/simulator/message_conversion.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
struct PromiseKey {
Address requester_address;
uint64_t request_id;
// TODO(tyler) possibly remove replier_address from promise key
// once we want to support DSR.
Address replier_address;
public:
friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
if (lhs.requester_address != rhs.requester_address) {
return lhs.requester_address < rhs.requester_address;
}
if (lhs.request_id != rhs.request_id) {
return lhs.request_id < rhs.request_id;
}
return lhs.replier_address < rhs.replier_address;
}
};
struct DeadlineAndOpaquePromise {
Time deadline;
OpaquePromise promise;
};
class SimulatorHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
// messages that have not yet been scheduled or dropped
std::vector<std::pair<Address, OpaqueMessage>> in_flight_;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::map<Address, std::vector<OpaqueMessage>> can_receive_;
Time cluster_wide_time_microseconds_;
bool should_shut_down_ = false;
SimulatorStats stats_;
size_t blocked_on_receive_ = 0;
std::set<Address> server_addresses_;
std::mt19937 rng_;
SimulatorConfig config_;
/// Returns the number of servers currently blocked on Receive, plus
/// the servers that are blocked on Futures that were created through
/// SimulatorTransport::Request.
///
/// TODO(tyler) investigate whether avoiding consideration of Futures
/// increases determinism.
size_t BlockedServers();
void TimeoutPromisesPastDeadline() {
const Time now = cluster_wide_time_microseconds_;
for (auto &[promise_key, dop] : promises_) {
if (dop.deadline < now) {
spdlog::debug("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
promise_key.replier_address.ToString());
std::move(dop).promise.TimeOut();
promises_.erase(promise_key);
stats_.timed_out_requests++;
}
}
}
public:
explicit SimulatorHandle(SimulatorConfig config)
: cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
void IncrementServerCountAndWaitForQuiescentState(Address address);
/// This method causes most of the interesting simulation logic to happen, wrt network behavior.
/// It checks to see if all background "server" threads are blocked on new messages, and if so,
/// it will decide whether to drop, reorder, or deliver in-flight messages based on the SimulatorConfig
/// that was used to create the Simulator.
bool MaybeTickSimulator();
void ShutDown();
bool ShouldShutDown() const;
template <Message Request, Message Response>
void SubmitRequest(Address to_address, Address from_address, uint64_t request_id, Request &&request, Duration timeout,
ResponsePromise<Response> &&promise) {
std::unique_lock<std::mutex> lock(mu_);
const Time deadline = cluster_wide_time_microseconds_ + timeout;
std::any message(request);
OpaqueMessage om{.from_address = from_address, .request_id = request_id, .message = std::move(message)};
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
stats_.total_messages++;
stats_.total_requests++;
cv_.notify_all();
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
std::unique_lock<std::mutex> lock(mu_);
blocked_on_receive_ += 1;
const Time deadline = cluster_wide_time_microseconds_ + timeout;
while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
if (can_receive_.contains(receiver)) {
std::vector<OpaqueMessage> &can_rx = can_receive_.at(receiver);
if (!can_rx.empty()) {
OpaqueMessage message = std::move(can_rx.back());
can_rx.pop_back();
// TODO(tyler) search for item in can_receive_ that matches the desired types, rather
// than asserting that the last item in can_rx matches.
auto m_opt = std::move(message).Take<Ms...>();
blocked_on_receive_ -= 1;
return std::move(m_opt).value();
}
}
lock.unlock();
bool made_progress = MaybeTickSimulator();
lock.lock();
if (!should_shut_down_ && !made_progress) {
cv_.wait(lock);
}
}
blocked_on_receive_ -= 1;
return TimedOut{};
}
template <Message M>
void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
std::unique_lock<std::mutex> lock(mu_);
std::any message_any(std::move(message));
OpaqueMessage om{.from_address = from_address, .request_id = request_id, .message = std::move(message_any)};
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
stats_.total_messages++;
cv_.notify_all();
}
Time Now() const;
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::unique_lock<std::mutex> lock(mu_);
return distrib(rng_);
}
SimulatorStats Stats();
};
}; // namespace memgraph::io::simulator

View File

@@ -0,0 +1,25 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
namespace memgraph::io::simulator {
struct SimulatorStats {
uint64_t total_messages = 0;
uint64_t dropped_messages = 0;
uint64_t timed_out_requests = 0;
uint64_t total_requests = 0;
uint64_t total_responses = 0;
uint64_t simulator_ticks = 0;
};
}; // namespace memgraph::io::simulator

View File

@@ -0,0 +1,65 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include <utility>
#include "io/address.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/time.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
class SimulatorTransport {
std::shared_ptr<SimulatorHandle> simulator_handle_;
const Address address_;
std::mt19937 rng_;
public:
SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
template <Message Request, Message Response>
ResponseFuture<Response> Request(Address address, uint64_t request_id, Request request, Duration timeout) {
std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
auto [future, promise] =
memgraph::io::FuturePromisePairWithNotifier<ResponseResult<Response>>(maybe_tick_simulator);
simulator_handle_->SubmitRequest(address, address_, request_id, std::move(request), timeout, std::move(promise));
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Duration timeout) {
return simulator_handle_->template Receive<Ms...>(address_, timeout);
}
template <Message M>
void Send(Address address, uint64_t request_id, M message) {
return simulator_handle_->template Send<M>(address, address_, request_id, message);
}
Time Now() const { return simulator_handle_->Now(); }
bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
return distrib(rng_);
}
};
}; // namespace memgraph::io::simulator

21
src/io/time.hpp Normal file
View File

@@ -0,0 +1,21 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
namespace memgraph::io {
using Duration = std::chrono::microseconds;
using Time = std::chrono::time_point<std::chrono::local_t, Duration>;
} // namespace memgraph::io

133
src/io/transport.hpp Normal file
View File

@@ -0,0 +1,133 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <concepts>
#include <random>
#include <variant>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/future.hpp"
#include "io/time.hpp"
#include "utils/result.hpp"
namespace memgraph::io {
using memgraph::utils::BasicResult;
// TODO(tyler) ensure that Message continues to represent
// reasonable constraints around message types over time,
// as we adapt things to use Thrift-generated message types.
template <typename T>
concept Message = std::same_as<T, std::decay_t<T>>;
template <Message M>
struct ResponseEnvelope {
M message;
uint64_t request_id;
Address from_address;
};
template <Message M>
using ResponseResult = BasicResult<TimedOut, ResponseEnvelope<M>>;
template <Message M>
using ResponseFuture = memgraph::io::Future<ResponseResult<M>>;
template <Message M>
using ResponsePromise = memgraph::io::Promise<ResponseResult<M>>;
template <Message... Ms>
struct RequestEnvelope {
std::variant<Ms...> message;
uint64_t request_id;
Address from_address;
};
template <Message... Ms>
using RequestResult = BasicResult<TimedOut, RequestEnvelope<Ms...>>;
template <typename I>
class Io {
I implementation_;
Address address_;
uint64_t request_id_counter_ = 0;
Duration default_timeout_ = std::chrono::microseconds{50000};
public:
Io(I io, Address address) : implementation_(io), address_(address) {}
/// Set the default timeout for all requests that are issued
/// without an explicit timeout set.
void SetDefaultTimeout(Duration timeout) { default_timeout_ = timeout; }
/// Returns the current default timeout for this Io instance.
Duration GetDefaultTimeout() { return default_timeout_; }
/// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
template <Message Request, Message Response>
ResponseFuture<Response> RequestWithTimeout(Address address, Request request, Duration timeout) {
const uint64_t request_id = ++request_id_counter_;
return implementation_.template Request<Request, Response>(address, request_id, request, timeout);
}
/// Issue a request that times out after the default timeout. This tends
/// to be used by clients.
template <Message Request, Message Response>
ResponseFuture<Response> Request(Address address, Request request) {
const uint64_t request_id = ++request_id_counter_;
const Duration timeout = default_timeout_;
return implementation_.template Request<Request, Response>(address, request_id, std::move(request), timeout);
}
/// Wait for an explicit number of microseconds for a request of one of the
/// provided types to arrive. This tends to be used by servers.
template <Message... Ms>
RequestResult<Ms...> ReceiveWithTimeout(Duration timeout) {
return implementation_.template Receive<Ms...>(timeout);
}
/// Wait the default number of microseconds for a request of one of the
/// provided types to arrive. This tends to be used by servers.
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive() {
const Duration timeout = default_timeout_;
return implementation_.template Receive<Ms...>(timeout);
}
/// Send a message in a best-effort fashion. This is used for messaging where
/// responses are not necessarily expected, and for servers to respond to requests.
/// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
template <Message M>
void Send(Address address, uint64_t request_id, M message) {
return implementation_.template Send<M>(address, request_id, std::move(message));
}
/// The current system time. This time source should be preferred over any other,
/// because it lets us deterministically control clocks from tests for making
/// things like timeouts deterministic.
Time Now() const { return implementation_.Now(); }
/// Returns true if the system should shut-down.
bool ShouldShutDown() const { return implementation_.ShouldShutDown(); }
/// Returns a random number within the specified distribution.
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
return implementation_.template Rand<D, Return>(distrib);
}
Address GetAddress() { return address_; }
};
}; // namespace memgraph::io

View File

@@ -216,6 +216,11 @@ DEFINE_bool(telemetry_enabled, false,
"the database runtime (vertex and edge counts and resource usage) "
"to allow for easier improvement of the product.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_restore_replicas_on_startup, true,
"Controls replicas should be restored automatically."); // TODO(42jeremy) this must be removed once T0835
// is implemented.
// Streams flags
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(
@@ -501,7 +506,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
if (first_user) {
spdlog::info("{} is first created user. Granting all privileges.", username);
GrantPrivilege(username, memgraph::query::kPrivilegesAll, {"*"});
GrantPrivilege(username, memgraph::query::kPrivilegesAll);
}
return user_added;
@@ -747,9 +752,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
}
void GrantPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) override {
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
@@ -758,9 +762,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
}
void DenyPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) override {
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
@@ -769,9 +772,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
}
void RevokePrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) override {
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
@@ -782,8 +784,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
private:
template <class TEditFun>
void EditPermissions(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels, const TEditFun &edit_fun) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges, const TEditFun &edit_fun) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
@@ -803,17 +804,11 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
for (const auto &permission : permissions) {
edit_fun(&user->permissions(), permission);
}
for (const auto &label : labels) {
edit_fun(&user->labelPermissions(), label);
}
locked_auth->SaveUser(*user);
} else {
for (const auto &permission : permissions) {
edit_fun(&role->permissions(), permission);
}
for (const auto &label : labels) {
edit_fun(&role->labelPermissions(), label);
}
locked_auth->SaveRole(*role);
}
} catch (const memgraph::auth::AuthException &e) {
@@ -1205,7 +1200,8 @@ int main(int argc, char **argv) {
.snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit},
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit,
.restore_replicas_on_startup = FLAGS_storage_restore_replicas_on_startup},
.transaction = {.isolation_level = ParseIsolationLevel()}};
if (FLAGS_storage_snapshot_interval_sec == 0) {
if (FLAGS_storage_wal_enabled) {
@@ -1256,9 +1252,8 @@ int main(int argc, char **argv) {
// the triggers
auto storage_accessor = interpreter_context.db->Access();
auto dba = memgraph::query::DbAccessor{&storage_accessor};
interpreter_context.trigger_store.RestoreTriggers(&interpreter_context.ast_cache, &dba,
&interpreter_context.antlr_lock, interpreter_context.config.query,
interpreter_context.auth_checker);
interpreter_context.trigger_store.RestoreTriggers(
&interpreter_context.ast_cache, &dba, interpreter_context.config.query, interpreter_context.auth_checker);
}
// As the Stream transformations are using modules, they have to be restored after the query modules are loaded.

View File

@@ -82,7 +82,7 @@ add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.9.2-complete.jar
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}

View File

@@ -21,8 +21,7 @@ namespace memgraph::query {
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, utils::SpinLock *antlr_lock,
const InterpreterConfig::Query &query_config) {
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config) {
// Strip the query for caching purposes. The process of stripping a query
// "normalizes" it by replacing any literals with new parameters. This
// results in just the *structure* of the query being taken into account for
@@ -63,20 +62,16 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
};
if (it == accessor.end()) {
{
std::unique_lock<utils::SpinLock> guard(*antlr_lock);
try {
parser = std::make_unique<frontend::opencypher::Parser>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<frontend::opencypher::Parser>(query_string);
try {
parser = std::make_unique<frontend::opencypher::Parser>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<frontend::opencypher::Parser>(query_string);
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
}
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
}
// Convert the ANTLR4 parse tree into an AST.

View File

@@ -111,8 +111,7 @@ struct ParsedQuery {
};
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, utils::SpinLock *antlr_lock,
const InterpreterConfig::Query &query_config);
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config);
class SingleNodeLogicalPlan final : public LogicalPlan {
public:

View File

@@ -51,7 +51,6 @@ class EdgeAccessor final {
public:
storage::EdgeAccessor impl_;
public:
explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
@@ -97,7 +96,6 @@ class VertexAccessor final {
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); }
public:
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }

View File

@@ -2239,11 +2239,9 @@ cpp<#
(user "std::string" :scope :public)
(role "std::string" :scope :public)
(user-or-role "std::string" :scope :public)
(password "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(labels "std::vector<std::string>" :scope :public)
(privileges "std::vector<Privilege>" :scope :public))
(:public
(lcp:define-enum action
@@ -2255,7 +2253,7 @@ cpp<#
(lcp:define-enum privilege
(create delete match merge set remove index stats auth constraint
dump replication durability read_file free_memory trigger config stream module_read module_write
websocket labels)
websocket)
(:serialize))
#>cpp
AuthQuery() = default;
@@ -2266,14 +2264,13 @@ cpp<#
#>cpp
AuthQuery(Action action, std::string user, std::string role,
std::string user_or_role, Expression *password,
std::vector<std::string> labels ,std::vector<Privilege> privileges)
std::vector<Privilege> privileges)
: action_(action),
user_(user),
role_(role),
user_or_role_(user_or_role),
password_(password),
labels_(labels),
privileges_(privileges){}
privileges_(privileges) {}
cpp<#)
(:private
#>cpp
@@ -2298,8 +2295,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM,
AuthQuery::Privilege::MODULE_READ, AuthQuery::Privilege::MODULE_WRITE,
AuthQuery::Privilege::WEBSOCKET,
AuthQuery::Privilege::LABELS};
AuthQuery::Privilege::WEBSOCKET};
cpp<#
(lcp:define-class info-query (query)
@@ -2379,10 +2375,7 @@ cpp<#
(port "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(sync_mode "SyncMode" :scope :public)
(timeout "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression")))
(sync_mode "SyncMode" :scope :public))
(:public
(lcp:define-enum action

File diff suppressed because it is too large Load Diff

View File

@@ -115,7 +115,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
auto operators = ExtractOperators(all_children, allowed_operators);
for (auto *expression : _expressions) {
expressions.push_back(expression->accept(this));
expressions.push_back(std::any_cast<Expression *>(expression->accept(this)));
}
Expression *first_operand = expressions[0];
@@ -131,7 +131,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
DMG_ASSERT(_expression, "can't happen");
auto operators = ExtractOperators(all_children, allowed_operators);
Expression *expression = _expression->accept(this);
Expression *expression = std::any_cast<Expression *>(_expression->accept(this));
for (int i = (int)operators.size() - 1; i >= 0; --i) {
expression = CreateUnaryOperatorByToken(operators[i], expression);
}
@@ -473,11 +473,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitPrivilege(MemgraphCypher::PrivilegeContext *ctx) override;
/**
* @return AuthQuery::LabelList
*/
antlrcpp::Any visitLabelList(MemgraphCypher::LabelListContext *ctx) override;
/**
* @return AuthQuery*
*/

View File

@@ -56,7 +56,6 @@ memgraphCypherKeyword : cypherKeyword
| IDENTIFIED
| ISOLATION
| KAFKA
| LABELS
| LEVEL
| LOAD
| LOCK
@@ -255,15 +254,10 @@ privilege : CREATE
| MODULE_READ
| MODULE_WRITE
| WEBSOCKET
| LABELS labels=labelList
;
privilegeList : privilege ( ',' privilege )* ;
labelList : COLON label ( ',' COLON label )* ;
label : ( '*' | symbolicName ) ;
showPrivileges : SHOW PRIVILEGES FOR userOrRole=userOrRoleName ;
showRoleForUser : SHOW ROLE FOR user=userOrRoleName ;
@@ -282,7 +276,6 @@ replicaName : symbolicName ;
socketAddress : literal ;
registerReplica : REGISTER REPLICA replicaName ( SYNC | ASYNC )
( WITH TIMEOUT timeout=literal ) ?
TO socketAddress ;
dropReplica : DROP REPLICA replicaName ;

View File

@@ -66,7 +66,6 @@ IDENTIFIED : I D E N T I F I E D ;
IGNORE : I G N O R E ;
ISOLATION : I S O L A T I O N ;
KAFKA : K A F K A ;
LABELS : L A B E L S ;
LEVEL : L E V E L ;
LOAD : L O A D ;
LOCK : L O C K ;

View File

@@ -205,8 +205,7 @@ const trie::Trie kKeywords = {"union",
"service_url",
"version",
"websocket",
"foreach",
"labels"};
"foreach"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

View File

@@ -882,21 +882,36 @@ TypedValue Id(const TypedValue *args, int64_t nargs, const FunctionContext &ctx)
}
TypedValue ToString(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
FType<Or<Null, String, Number, Bool>>("toString", args, nargs);
FType<Or<Null, String, Number, Date, LocalTime, LocalDateTime, Duration, Bool>>("toString", args, nargs);
const auto &arg = args[0];
if (arg.IsNull()) {
return TypedValue(ctx.memory);
} else if (arg.IsString()) {
}
if (arg.IsString()) {
return TypedValue(arg, ctx.memory);
} else if (arg.IsInt()) {
}
if (arg.IsInt()) {
// TODO: This is making a pointless copy of std::string, we may want to
// use a different conversion to string
return TypedValue(std::to_string(arg.ValueInt()), ctx.memory);
} else if (arg.IsDouble()) {
return TypedValue(std::to_string(arg.ValueDouble()), ctx.memory);
} else {
return TypedValue(arg.ValueBool() ? "true" : "false", ctx.memory);
}
if (arg.IsDouble()) {
return TypedValue(std::to_string(arg.ValueDouble()), ctx.memory);
}
if (arg.IsDate()) {
return TypedValue(arg.ValueDate().ToString(), ctx.memory);
}
if (arg.IsLocalTime()) {
return TypedValue(arg.ValueLocalTime().ToString(), ctx.memory);
}
if (arg.IsLocalDateTime()) {
return TypedValue(arg.ValueLocalDateTime().ToString(), ctx.memory);
}
if (arg.IsDuration()) {
return TypedValue(arg.ValueDuration().ToString(), ctx.memory);
}
return TypedValue(arg.ValueBool() ? "true" : "false", ctx.memory);
}
TypedValue Timestamp(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {

View File

@@ -111,7 +111,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
TypedValue Visit(IfOperator &if_operator) override {
auto condition = if_operator.condition_->Accept(*this);
if (condition.IsNull()) {
return if_operator.then_expression_->Accept(*this);
return if_operator.else_expression_->Accept(*this);
}
if (condition.type() != TypedValue::Type::Bool) {
// At the moment IfOperator is used only in CASE construct.

View File

@@ -44,6 +44,7 @@
#include "query/trigger.hpp"
#include "query/typed_value.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/replication/enums.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
#include "utils/event_counter.hpp"
@@ -160,7 +161,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
void RegisterReplica(const std::string &name, const std::string &socket_address,
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout,
const ReplicationQuery::SyncMode sync_mode,
const std::chrono::seconds replica_check_frequency) override {
if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
// replica can't register another replica
@@ -183,9 +184,9 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::kDefaultReplicationPort);
if (maybe_ip_and_port) {
auto [ip, port] = *maybe_ip_and_port;
auto ret = db_->RegisterReplica(
name, {std::move(ip), port}, repl_mode,
{.timeout = timeout, .replica_check_frequency = replica_check_frequency, .ssl = std::nullopt});
auto ret = db_->RegisterReplica(name, {std::move(ip), port}, repl_mode,
storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID,
{.replica_check_frequency = replica_check_frequency, .ssl = std::nullopt});
if (ret.HasError()) {
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
}
@@ -228,9 +229,6 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
replica.sync_mode = ReplicationQuery::SyncMode::ASYNC;
break;
}
if (repl_info.timeout) {
replica.timeout = *repl_info.timeout;
}
replica.current_timestamp_of_replica = repl_info.timestamp_info.current_timestamp_of_replica;
replica.current_number_of_timestamp_behind_master =
@@ -282,8 +280,6 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
std::string rolename = auth_query->role_;
std::string user_or_role = auth_query->user_or_role_;
std::vector<AuthQuery::Privilege> privileges = auth_query->privileges_;
std::vector<std::string> labels = auth_query->labels_;
// std::vector<storage::LabelId> labels = NamesToLabels(labels, db_accessor);
auto password = EvaluateOptionalExpression(auth_query->password_, &evaluator);
Callback callback;
@@ -298,8 +294,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
if (license_check_result.HasError() && enterprise_only_methods.contains(auth_query->action_)) {
throw utils::BasicException(
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "advanced authentication
features"));
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "advanced authentication features"));
}
switch (auth_query->action_) {
@@ -314,7 +309,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
// If the license is not valid we create users with admin access
if (!valid_enterprise_license) {
spdlog::warn("Granting all the privileges to {}.", username);
auth->GrantPrivilege(username, kPrivilegesAll, {});
auth->GrantPrivilege(username, kPrivilegesAll);
}
return std::vector<std::vector<TypedValue>>();
@@ -389,20 +384,20 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
};
return callback;
case AuthQuery::Action::GRANT_PRIVILEGE:
callback.fn = [auth, user_or_role, privileges, labels] {
auth->GrantPrivilege(user_or_role, privileges, labels);
callback.fn = [auth, user_or_role, privileges] {
auth->GrantPrivilege(user_or_role, privileges);
return std::vector<std::vector<TypedValue>>();
};
return callback;
case AuthQuery::Action::DENY_PRIVILEGE:
callback.fn = [auth, user_or_role, privileges, labels] {
auth->DenyPrivilege(user_or_role, privileges, labels);
callback.fn = [auth, user_or_role, privileges] {
auth->DenyPrivilege(user_or_role, privileges);
return std::vector<std::vector<TypedValue>>();
};
return callback;
case AuthQuery::Action::REVOKE_PRIVILEGE: {
callback.fn = [auth, user_or_role, privileges, labels] {
auth->RevokePrivilege(user_or_role, privileges, labels);
callback.fn = [auth, user_or_role, privileges] {
auth->RevokePrivilege(user_or_role, privileges);
return std::vector<std::vector<TypedValue>>();
};
return callback;
@@ -490,22 +485,11 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
const auto &name = repl_query->replica_name_;
const auto &sync_mode = repl_query->sync_mode_;
auto socket_address = repl_query->socket_address_->Accept(evaluator);
auto timeout = EvaluateOptionalExpression(repl_query->timeout_, &evaluator);
const auto replica_check_frequency = interpreter_context->config.replication_replica_check_frequency;
std::optional<double> maybe_timeout;
if (timeout.IsDouble()) {
maybe_timeout = timeout.ValueDouble();
} else if (timeout.IsInt()) {
maybe_timeout = static_cast<double>(timeout.ValueInt());
}
if (maybe_timeout && *maybe_timeout <= 0.0) {
throw utils::BasicException("Parameter TIMEOUT must be strictly greater than 0.");
}
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name, socket_address, sync_mode,
maybe_timeout, replica_check_frequency]() mutable {
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, maybe_timeout,
replica_check_frequency);
replica_check_frequency]() mutable {
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, replica_check_frequency);
return std::vector<std::vector<TypedValue>>();
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::REGISTER_REPLICA,
@@ -525,13 +509,9 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
}
case ReplicationQuery::Action::SHOW_REPLICAS: {
callback.header = {"name",
"socket_address",
"sync_mode",
"timeout",
"current_timestamp_of_replica",
"number_of_timestamp_behind_master",
"state"};
callback.header = {
"name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master",
"state"};
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, replica_nfields = callback.header.size()] {
const auto &replicas = handler.ShowReplicas();
auto typed_replicas = std::vector<std::vector<TypedValue>>{};
@@ -552,12 +532,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
break;
}
if (replica.timeout) {
typed_replica.emplace_back(TypedValue(*replica.timeout));
} else {
typed_replica.emplace_back(TypedValue());
}
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.current_timestamp_of_replica)));
typed_replica.emplace_back(
TypedValue(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master)));
@@ -1207,7 +1181,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
// full query string) when given just the inner query to execute.
ParsedQuery parsed_inner_query =
ParseQuery(parsed_query.query_string.substr(kExplainQueryStart.size()), parsed_query.user_parameters,
&interpreter_context->ast_cache, &interpreter_context->antlr_lock, interpreter_context->config.query);
&interpreter_context->ast_cache, interpreter_context->config.query);
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
@@ -1274,7 +1248,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
// full query string) when given just the inner query to execute.
ParsedQuery parsed_inner_query =
ParseQuery(parsed_query.query_string.substr(kProfileQueryStart.size()), parsed_query.user_parameters,
&interpreter_context->ast_cache, &interpreter_context->antlr_lock, interpreter_context->config.query);
&interpreter_context->ast_cache, interpreter_context->config.query);
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
@@ -1592,8 +1566,7 @@ Callback CreateTrigger(TriggerQuery *trigger_query,
interpreter_context->trigger_store.AddTrigger(
std::move(trigger_name), trigger_statement, user_parameters, ToTriggerEventType(event_type),
before_commit ? TriggerPhase::BEFORE_COMMIT : TriggerPhase::AFTER_COMMIT, &interpreter_context->ast_cache,
dba, &interpreter_context->antlr_lock, interpreter_context->config.query, std::move(owner),
interpreter_context->auth_checker);
dba, interpreter_context->config.query, std::move(owner), interpreter_context->auth_checker);
return {};
}};
}
@@ -2149,8 +2122,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
query_execution->summary["cost_estimate"] = 0.0;
utils::Timer parsing_timer;
ParsedQuery parsed_query = ParseQuery(query_string, params, &interpreter_context_->ast_cache,
&interpreter_context_->antlr_lock, interpreter_context_->config.query);
ParsedQuery parsed_query =
ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query);
query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count();
// Some queries require an active transaction in order to be prepared.

View File

@@ -99,16 +99,14 @@ class AuthQueryHandler {
virtual std::vector<std::vector<TypedValue>> GetPrivileges(const std::string &user_or_role) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void GrantPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) = 0;
virtual void GrantPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void DenyPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) = 0;
virtual void DenyPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void RevokePrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) = 0;
virtual void RevokePrivilege(const std::string &user_or_role,
const std::vector<AuthQuery::Privilege> &privileges) = 0;
};
enum class QueryHandlerResult { COMMIT, ABORT, NOTHING };
@@ -142,7 +140,7 @@ class ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
virtual void RegisterReplica(const std::string &name, const std::string &socket_address,
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout,
ReplicationQuery::SyncMode sync_mode,
const std::chrono::seconds replica_check_frequency) = 0;
/// @throw QueryRuntimeException if an error ocurred.
@@ -175,13 +173,6 @@ struct InterpreterContext {
storage::Storage *db;
// ANTLR has singleton instance that is shared between threads. It is
// protected by locks inside of ANTLR. Unfortunately, they are not protected
// in a very good way. Once we have ANTLR version without race conditions we
// can remove this lock. This will probably never happen since ANTLR
// developers introduce more bugs in each version. Fortunately, we have
// cache so this lock probably won't impact performance much...
utils::SpinLock antlr_lock;
std::optional<double> tsc_frequency{utils::GetTSCFrequency()};
std::atomic<bool> is_shutting_down{false};

View File

@@ -114,4 +114,4 @@ std::string ExecutionStatsKeyToString(const ExecutionStats::Key key) {
}
}
} // namespace memgraph::query
} // namespace memgraph::query

View File

@@ -368,13 +368,12 @@ VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, Exec
TypedValue &dest_node_value = frame[self_.node_info_.symbol];
ExpectType(self_.node_info_.symbol, dest_node_value, TypedValue::Type::Vertex);
return dest_node_value.ValueVertex();
} else {
auto &created_vertex = CreateLocalVertex(self_.node_info_, &frame, context);
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_vertex);
}
return created_vertex;
}
auto &created_vertex = CreateLocalVertex(self_.node_info_, &frame, context);
if (context.trigger_context_collector) {
context.trigger_context_collector->RegisterCreatedObject(created_vertex);
}
return created_vertex;
}
template <class TVerticesFun>
@@ -2600,13 +2599,13 @@ namespace {
* when there are */
TypedValue DefaultAggregationOpValue(const Aggregate::Element &element, utils::MemoryResource *memory) {
switch (element.op) {
case Aggregation::Op::COUNT:
return TypedValue(0, memory);
case Aggregation::Op::SUM:
case Aggregation::Op::MIN:
case Aggregation::Op::MAX:
case Aggregation::Op::AVG:
return TypedValue(memory);
case Aggregation::Op::COUNT:
case Aggregation::Op::SUM:
return TypedValue(0, memory);
case Aggregation::Op::COLLECT_LIST:
return TypedValue(TypedValue::TVector(memory));
case Aggregation::Op::COLLECT_MAP:
@@ -2628,9 +2627,7 @@ class AggregateCursor : public Cursor {
pulled_all_input_ = true;
aggregation_it_ = aggregation_.begin();
// in case there is no input and no group_bys we need to return true
// just this once
if (aggregation_.empty() && self_.group_by_.empty()) {
if (aggregation_.empty()) {
auto *pull_memory = context.evaluation_context.memory;
// place default aggregation values on the frame
for (const auto &elem : self_.aggregations_)

View File

@@ -619,10 +619,10 @@ void Streams::Drop(const std::string &stream_name) {
// no running Test function for this consumer, therefore it can be erased.
std::visit([&](const auto &stream_data) { stream_data.stream_source->Lock(); }, it->second);
locked_streams->erase(it);
if (!storage_.Delete(stream_name)) {
throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name);
}
locked_streams->erase(it);
// TODO(antaljanosbenjamin) Release the transformation
}

View File

@@ -188,7 +188,7 @@ class Streams final {
void Persist(StreamStatus<TStream> &&status) {
const std::string stream_name = status.name;
if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) {
throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name};
throw StreamsException{"Couldn't persist stream data for stream '{}'", stream_name};
}
}

View File

@@ -153,10 +153,10 @@ std::vector<std::pair<Identifier, TriggerIdentifierTag>> GetPredefinedIdentifier
Trigger::Trigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters,
const TriggerEventType event_type, utils::SkipList<QueryCacheEntry> *query_cache,
DbAccessor *db_accessor, utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
DbAccessor *db_accessor, const InterpreterConfig::Query &query_config,
std::optional<std::string> owner, const query::AuthChecker *auth_checker)
: name_{std::move(name)},
parsed_statements_{ParseQuery(query, user_parameters, query_cache, antlr_lock, query_config)},
parsed_statements_{ParseQuery(query, user_parameters, query_cache, query_config)},
event_type_{event_type},
owner_{std::move(owner)} {
// We check immediately if the query is valid by trying to create a plan.
@@ -257,7 +257,7 @@ inline constexpr uint64_t kVersion{2};
TriggerStore::TriggerStore(std::filesystem::path directory) : storage_{std::move(directory)} {}
void TriggerStore::RestoreTriggers(utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
const InterpreterConfig::Query &query_config,
const query::AuthChecker *auth_checker) {
MG_ASSERT(before_commit_triggers_.size() == 0 && after_commit_triggers_.size() == 0,
"Cannot restore trigger when some triggers already exist!");
@@ -317,8 +317,8 @@ void TriggerStore::RestoreTriggers(utils::SkipList<QueryCacheEntry> *query_cache
std::optional<Trigger> trigger;
try {
trigger.emplace(trigger_name, statement, user_parameters, event_type, query_cache, db_accessor, antlr_lock,
query_config, std::move(owner), auth_checker);
trigger.emplace(trigger_name, statement, user_parameters, event_type, query_cache, db_accessor, query_config,
std::move(owner), auth_checker);
} catch (const utils::BasicException &e) {
spdlog::warn("Failed to create trigger '{}' because: {}", trigger_name, e.what());
continue;
@@ -336,8 +336,8 @@ void TriggerStore::AddTrigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters,
TriggerEventType event_type, TriggerPhase phase,
utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
std::optional<std::string> owner, const query::AuthChecker *auth_checker) {
const InterpreterConfig::Query &query_config, std::optional<std::string> owner,
const query::AuthChecker *auth_checker) {
std::unique_lock store_guard{store_lock_};
if (storage_.Get(name)) {
throw utils::BasicException("Trigger with the same name already exists.");
@@ -345,8 +345,8 @@ void TriggerStore::AddTrigger(std::string name, const std::string &query,
std::optional<Trigger> trigger;
try {
trigger.emplace(std::move(name), query, user_parameters, event_type, query_cache, db_accessor, antlr_lock,
query_config, std::move(owner), auth_checker);
trigger.emplace(std::move(name), query, user_parameters, event_type, query_cache, db_accessor, query_config,
std::move(owner), auth_checker);
} catch (const utils::BasicException &e) {
const auto identifiers = GetPredefinedIdentifiers(event_type);
std::stringstream identifier_names_stream;

View File

@@ -34,7 +34,7 @@ namespace memgraph::query {
struct Trigger {
explicit Trigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters, TriggerEventType event_type,
utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor, utils::SpinLock *antlr_lock,
utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
const InterpreterConfig::Query &query_config, std::optional<std::string> owner,
const query::AuthChecker *auth_checker);
@@ -81,14 +81,13 @@ struct TriggerStore {
explicit TriggerStore(std::filesystem::path directory);
void RestoreTriggers(utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
const query::AuthChecker *auth_checker);
const InterpreterConfig::Query &query_config, const query::AuthChecker *auth_checker);
void AddTrigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters, TriggerEventType event_type,
TriggerPhase phase, utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
std::optional<std::string> owner, const query::AuthChecker *auth_checker);
const InterpreterConfig::Query &query_config, std::optional<std::string> owner,
const query::AuthChecker *auth_checker);
void DropTrigger(const std::string &name);

104
src/query/v2/CMakeLists.txt Normal file
View File

@@ -0,0 +1,104 @@
define_add_lcp(add_lcp_query lcp_query_v2_cpp_files generated_lcp_query_v2_files)
add_lcp_query(frontend/ast/ast.lcp)
add_lcp_query(frontend/semantic/symbol.lcp)
add_lcp_query(plan/operator.lcp)
add_custom_target(generate_lcp_query_v2 DEPENDS ${generated_lcp_query_v2_files})
set(mg_query_v2_sources
${lcp_query_v2_cpp_files}
common.cpp
cypher_query_interpreter.cpp
dump.cpp
frontend/ast/cypher_main_visitor.cpp
frontend/ast/pretty_print.cpp
frontend/parsing.cpp
frontend/semantic/required_privileges.cpp
frontend/semantic/symbol_generator.cpp
frontend/stripped.cpp
interpret/awesome_memgraph_functions.cpp
interpret/eval.cpp
interpreter.cpp
metadata.cpp
plan/operator.cpp
plan/preprocess.cpp
plan/pretty_print.cpp
plan/profile.cpp
plan/read_write_type_checker.cpp
plan/rewrite/index_lookup.cpp
plan/rule_based_planner.cpp
plan/variable_start_planner.cpp
procedure/mg_procedure_impl.cpp
procedure/mg_procedure_helpers.cpp
procedure/module.cpp
procedure/py_module.cpp
serialization/property_value.cpp
stream/streams.cpp
stream/sources.cpp
stream/common.cpp
trigger.cpp
trigger_context.cpp
typed_value.cpp)
find_package(Boost REQUIRED)
add_library(mg-query-v2 STATIC ${mg_query_v2_sources})
add_dependencies(mg-query-v2 generate_lcp_query_v2)
target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()
find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
endif()
target_link_libraries(mg-query-v2 Python3::Python)
# Generate Antlr openCypher parser
set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher)
set(opencypher_generated ${opencypher_frontend}/generated)
set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
set(opencypher_parser_grammar ${opencypher_frontend}/grammar/MemgraphCypher.g4)
set(antlr_opencypher_generated_src
${opencypher_generated}/MemgraphCypherLexer.cpp
${opencypher_generated}/MemgraphCypher.cpp
${opencypher_generated}/MemgraphCypherBaseVisitor.cpp
${opencypher_generated}/MemgraphCypherVisitor.cpp
)
set(antlr_opencypher_generated_include
${opencypher_generated}/MemgraphCypherLexer.h
${opencypher_generated}/MemgraphCypher.h
${opencypher_generated}/MemgraphCypherBaseVisitor.h
${opencypher_generated}/MemgraphCypherVisitor.h
)
add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
DEPENDS
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
${opencypher_frontend}/grammar/CypherLexer.g4
${opencypher_frontend}/grammar/Cypher.g4)
add_custom_target(generate_opencypher_parser_v2
DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})
add_library(antlr_opencypher_parser_lib_v2 STATIC ${antlr_opencypher_generated_src})
add_dependencies(antlr_opencypher_parser_lib_v2 generate_opencypher_parser_v2)
target_link_libraries(antlr_opencypher_parser_lib_v2 antlr4)
target_link_libraries(mg-query-v2 antlr_opencypher_parser_lib_v2)

View File

@@ -0,0 +1,29 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::query::v2 {
class AuthChecker {
public:
virtual bool IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<query::v2::AuthQuery::Privilege> &privileges) const = 0;
};
class AllowEverythingAuthChecker final : public query::v2::AuthChecker {
bool IsUserAuthorized(const std::optional<std::string> & /*username*/,
const std::vector<query::v2::AuthQuery::Privilege> & /*privileges*/) const override {
return true;
}
};
} // namespace memgraph::query::v2

76
src/query/v2/common.cpp Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/common.hpp"
namespace memgraph::query::v2 {
namespace impl {
bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
// in ordering null comes after everything else
// at the same time Null is not less that null
// first deal with Null < Whatever case
if (a.IsNull()) return false;
// now deal with NotNull < Null case
if (b.IsNull()) return true;
// comparisons are from this point legal only between values of
// the same type, or int+float combinations
if ((a.type() != b.type() && !(a.IsNumeric() && b.IsNumeric())))
throw QueryRuntimeException("Can't compare value of type {} to value of type {}.", a.type(), b.type());
switch (a.type()) {
case TypedValue::Type::Bool:
return !a.ValueBool() && b.ValueBool();
case TypedValue::Type::Int:
if (b.type() == TypedValue::Type::Double)
return a.ValueInt() < b.ValueDouble();
else
return a.ValueInt() < b.ValueInt();
case TypedValue::Type::Double:
if (b.type() == TypedValue::Type::Int)
return a.ValueDouble() < b.ValueInt();
else
return a.ValueDouble() < b.ValueDouble();
case TypedValue::Type::String:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueString() < b.ValueString();
case TypedValue::Type::Date:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueDate() < b.ValueDate();
case TypedValue::Type::LocalTime:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueLocalTime() < b.ValueLocalTime();
case TypedValue::Type::LocalDateTime:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueLocalDateTime() < b.ValueLocalDateTime();
case TypedValue::Type::Duration:
// NOLINTNEXTLINE(modernize-use-nullptr)
return a.ValueDuration() < b.ValueDuration();
case TypedValue::Type::List:
case TypedValue::Type::Map:
case TypedValue::Type::Vertex:
case TypedValue::Type::Edge:
case TypedValue::Type::Path:
throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type());
case TypedValue::Type::Null:
LOG_FATAL("Invalid type");
}
}
} // namespace impl
int64_t QueryTimestamp() {
return std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
}
} // namespace memgraph::query::v2

168
src/query/v2/common.hpp Normal file
View File

@@ -0,0 +1,168 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file
#pragma once
#include <concepts>
#include <cstdint>
#include <string>
#include <string_view>
#include <type_traits>
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/frontend/semantic/symbol.hpp"
#include "query/v2/typed_value.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/view.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::query::v2 {
namespace impl {
bool TypedValueCompare(const TypedValue &a, const TypedValue &b);
} // namespace impl
/// Custom Comparator type for comparing vectors of TypedValues.
///
/// Does lexicographical ordering of elements based on the above
/// defined TypedValueCompare, and also accepts a vector of Orderings
/// the define how respective elements compare.
class TypedValueVectorCompare final {
public:
TypedValueVectorCompare() {}
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
template <class TAllocator>
bool operator()(const std::vector<TypedValue, TAllocator> &c1, const std::vector<TypedValue, TAllocator> &c2) const {
// ordering is invalid if there are more elements in the collections
// then there are in the ordering_ vector
MG_ASSERT(c1.size() <= ordering_.size() && c2.size() <= ordering_.size(),
"Collections contain more elements then there are orderings");
auto c1_it = c1.begin();
auto c2_it = c2.begin();
auto ordering_it = ordering_.begin();
for (; c1_it != c1.end() && c2_it != c2.end(); c1_it++, c2_it++, ordering_it++) {
if (impl::TypedValueCompare(*c1_it, *c2_it)) return *ordering_it == Ordering::ASC;
if (impl::TypedValueCompare(*c2_it, *c1_it)) return *ordering_it == Ordering::DESC;
}
// at least one collection is exhausted
// c1 is less then c2 iff c1 reached the end but c2 didn't
return (c1_it == c1.end()) && (c2_it != c2.end());
}
// TODO: Remove this, member is public
const auto &ordering() const { return ordering_; }
std::vector<Ordering> ordering_;
};
/// Raise QueryRuntimeException if the value for symbol isn't of expected type.
inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue::Type expected) {
if (value.type() != expected)
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
template <typename T>
concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
};
inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) {
switch (schema_violation.status) {
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: {
throw SchemaViolationException(
fmt::format("Primary key {} not defined on label :{}",
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type),
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL: {
throw SchemaViolationException(
fmt::format("Label :{} is not a primary label", dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE: {
throw SchemaViolationException(
fmt::format("Wrong type of property {} in schema :{}, should be of type {}",
*schema_violation.violated_property_value, dba.LabelToName(schema_violation.label),
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY: {
throw SchemaViolationException(fmt::format("Updating of primary key {} on schema :{} not supported",
*schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_MODIFY_PRIMARY_LABEL: {
throw SchemaViolationException(fmt::format("Cannot add or remove label :{} since it is a primary label",
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: {
throw SchemaViolationException(
fmt::format("Cannot create vertex with secondary label :{}", dba.LabelToName(schema_violation.label)));
}
}
}
inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
switch (error) {
case storage::v3::Error::SERIALIZATION_ERROR:
throw TransactionSerializationException();
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted object.");
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::NONEXISTENT_OBJECT:
throw QueryRuntimeException("Unexpected error when setting a property.");
}
}
/// Set a property `value` mapped with given `key` on a `record`.
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <AccessorWithSetProperty T>
storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key,
const TypedValue &value) {
try {
if constexpr (std::is_same_v<T, VertexAccessor>) {
const auto maybe_old_value = record->SetPropertyAndValidate(key, storage::v3::PropertyValue(value));
if (maybe_old_value.HasError()) {
std::visit(utils::Overloaded{[](const storage::v3::Error error) { HandleErrorOnPropertyUpdate(error); },
[&dba](const storage::v3::SchemaViolation &schema_violation) {
HandleSchemaViolation(schema_violation, dba);
}},
maybe_old_value.GetError());
}
return std::move(*maybe_old_value);
} else {
// No validation on edge properties
const auto maybe_old_value = record->SetProperty(key, storage::v3::PropertyValue(value));
if (maybe_old_value.HasError()) {
HandleErrorOnPropertyUpdate(maybe_old_value.GetError());
}
return std::move(*maybe_old_value);
}
} catch (const TypedValueException &) {
throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
}
}
int64_t QueryTimestamp();
} // namespace memgraph::query::v2

32
src/query/v2/config.hpp Normal file
View File

@@ -0,0 +1,32 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <string>
namespace memgraph::query::v2 {
struct InterpreterConfig {
struct Query {
bool allow_load_csv{true};
} query;
// The default execution timeout is 10 minutes.
double execution_timeout_sec{600.0};
// The same as \ref memgraph::storage::v3::replication::ReplicationClientConfig
std::chrono::seconds replication_replica_check_frequency{1};
std::string default_kafka_bootstrap_servers;
std::string default_pulsar_service_url;
uint32_t stream_transaction_conflict_retries;
std::chrono::milliseconds stream_transaction_retry_interval;
};
} // namespace memgraph::query::v2

View File

@@ -0,0 +1,19 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
#include <string>
namespace memgraph::query::v2 {
inline constexpr uint16_t kDefaultReplicationPort = 10000;
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
} // namespace memgraph::query::v2

89
src/query/v2/context.hpp Normal file
View File

@@ -0,0 +1,89 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <type_traits>
#include "query/v2/common.hpp"
#include "query/v2/frontend/semantic/symbol_table.hpp"
#include "query/v2/metadata.hpp"
#include "query/v2/parameters.hpp"
#include "query/v2/plan/profile.hpp"
#include "query/v2/trigger.hpp"
#include "utils/async_timer.hpp"
namespace memgraph::query::v2 {
struct EvaluationContext {
/// Memory for allocations during evaluation of a *single* Pull call.
///
/// Although the assigned memory may live longer than the duration of a Pull
/// (e.g. memory is the same as the whole execution memory), you have to treat
/// it as if the lifetime is only valid during the Pull.
utils::MemoryResource *memory{utils::NewDeleteResource()};
int64_t timestamp{-1};
Parameters parameters;
/// All properties indexable via PropertyIx
std::vector<storage::v3::PropertyId> properties;
/// All labels indexable via LabelIx
std::vector<storage::v3::LabelId> labels;
/// All counters generated by `counter` function, mutable because the function
/// modifies the values
mutable std::unordered_map<std::string, int64_t> counters;
};
inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
DbAccessor *dba) {
std::vector<storage::v3::PropertyId> properties;
properties.reserve(property_names.size());
for (const auto &name : property_names) {
properties.push_back(dba->NameToProperty(name));
}
return properties;
}
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names, DbAccessor *dba) {
std::vector<storage::v3::LabelId> labels;
labels.reserve(label_names.size());
for (const auto &name : label_names) {
labels.push_back(dba->NameToLabel(name));
}
return labels;
}
struct ExecutionContext {
DbAccessor *db_accessor{nullptr};
SymbolTable symbol_table;
EvaluationContext evaluation_context;
std::atomic<bool> *is_shutting_down{nullptr};
bool is_profile_query{false};
std::chrono::duration<double> profile_execution_time;
plan::ProfilingStats stats;
plan::ProfilingStats *stats_root{nullptr};
ExecutionStats execution_stats;
TriggerContextCollector *trigger_context_collector{nullptr};
utils::AsyncTimer timer;
};
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");
static_assert(std::is_move_constructible_v<ExecutionContext>, "ExecutionContext must be move constructible!");
inline bool MustAbort(const ExecutionContext &context) noexcept {
return (context.is_shutting_down != nullptr && context.is_shutting_down->load(std::memory_order_acquire)) ||
context.timer.IsExpired();
}
inline plan::ProfilingStatsWithTotalTime GetStatsWithTotalTime(const ExecutionContext &context) {
return plan::ProfilingStatsWithTotalTime{context.stats, context.profile_execution_time};
}
} // namespace memgraph::query::v2

View File

@@ -0,0 +1,153 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/cypher_query_interpreter.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
namespace memgraph::query::v2 {
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::v3::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config) {
// Strip the query for caching purposes. The process of stripping a query
// "normalizes" it by replacing any literals with new parameters. This
// results in just the *structure* of the query being taken into account for
// caching.
frontend::StrippedQuery stripped_query{query_string};
// Copy over the parameters that were introduced during stripping.
Parameters parameters{stripped_query.literals()};
// Check that all user-specified parameters are provided.
for (const auto &param_pair : stripped_query.parameters()) {
auto it = params.find(param_pair.second);
if (it == params.end()) {
throw query::v2::UnprovidedParameterError("Parameter ${} not provided.", param_pair.second);
}
parameters.Add(param_pair.first, it->second);
}
// Cache the query's AST if it isn't already.
auto hash = stripped_query.hash();
auto accessor = cache->access();
auto it = accessor.find(hash);
std::unique_ptr<frontend::opencypher::Parser> parser;
// Return a copy of both the AST storage and the query.
CachedQuery result;
bool is_cacheable = true;
auto get_information_from_cache = [&](const auto &cached_query) {
result.ast_storage.properties_ = cached_query.ast_storage.properties_;
result.ast_storage.labels_ = cached_query.ast_storage.labels_;
result.ast_storage.edge_types_ = cached_query.ast_storage.edge_types_;
result.query = cached_query.query->Clone(&result.ast_storage);
result.required_privileges = cached_query.required_privileges;
};
if (it == accessor.end()) {
try {
parser = std::make_unique<frontend::opencypher::Parser>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<frontend::opencypher::Parser>(query_string);
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
}
// Convert the ANTLR4 parse tree into an AST.
AstStorage ast_storage;
frontend::ParsingContext context{true};
frontend::CypherMainVisitor visitor(context, &ast_storage);
visitor.visit(parser->tree());
if (visitor.GetQueryInfo().has_load_csv && !query_config.allow_load_csv) {
throw utils::BasicException("Load CSV not allowed on this instance because it was disabled by a config.");
}
if (visitor.GetQueryInfo().is_cacheable) {
CachedQuery cached_query{std::move(ast_storage), visitor.query(),
query::v2::GetRequiredPrivileges(visitor.query())};
it = accessor.insert({hash, std::move(cached_query)}).first;
get_information_from_cache(it->second);
} else {
result.ast_storage.properties_ = ast_storage.properties_;
result.ast_storage.labels_ = ast_storage.labels_;
result.ast_storage.edge_types_ = ast_storage.edge_types_;
result.query = visitor.query()->Clone(&result.ast_storage);
result.required_privileges = query::v2::GetRequiredPrivileges(visitor.query());
is_cacheable = false;
}
} else {
get_information_from_cache(it->second);
}
return ParsedQuery{query_string,
params,
std::move(parameters),
std::move(stripped_query),
std::move(result.ast_storage),
result.query,
std::move(result.required_privileges),
is_cacheable};
}
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
auto vertex_counts = plan::MakeVertexCountCache(db_accessor);
auto symbol_table = MakeSymbolTable(query, predefined_identifiers);
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
std::move(symbol_table));
}
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
if (plan_cache) {
plan_cache_access.emplace(plan_cache->access());
auto it = plan_cache_access->find(hash);
if (it != plan_cache_access->end()) {
if (it->second->IsExpired()) {
plan_cache_access->remove(hash);
} else {
return it->second;
}
}
}
auto plan = std::make_shared<CachedPlan>(
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
if (plan_cache_access) {
plan_cache_access->insert({hash, plan});
}
return plan;
}
} // namespace memgraph::query::v2

View File

@@ -0,0 +1,151 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "query/v2/config.hpp"
#include "query/v2/frontend/ast/cypher_main_visitor.hpp"
#include "query/v2/frontend/opencypher/parser.hpp"
#include "query/v2/frontend/semantic/required_privileges.hpp"
#include "query/v2/frontend/semantic/symbol_generator.hpp"
#include "query/v2/frontend/stripped.hpp"
#include "query/v2/plan/planner.hpp"
#include "utils/flag_validation.hpp"
#include "utils/timer.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(query_cost_planner);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int32(query_plan_cache_ttl);
namespace memgraph::query::v2 {
// TODO: Maybe this should move to query/plan/planner.
/// Interface for accessing the root operator of a logical plan.
class LogicalPlan {
public:
explicit LogicalPlan() = default;
virtual ~LogicalPlan() = default;
LogicalPlan(const LogicalPlan &) = default;
LogicalPlan &operator=(const LogicalPlan &) = default;
LogicalPlan(LogicalPlan &&) = default;
LogicalPlan &operator=(LogicalPlan &&) = default;
virtual const plan::LogicalOperator &GetRoot() const = 0;
virtual double GetCost() const = 0;
virtual const SymbolTable &GetSymbolTable() const = 0;
virtual const AstStorage &GetAstStorage() const = 0;
};
class CachedPlan {
public:
explicit CachedPlan(std::unique_ptr<LogicalPlan> plan);
const auto &plan() const { return plan_->GetRoot(); }
double cost() const { return plan_->GetCost(); }
const auto &symbol_table() const { return plan_->GetSymbolTable(); }
const auto &ast_storage() const { return plan_->GetAstStorage(); }
bool IsExpired() const {
// NOLINTNEXTLINE (modernize-use-nullptr)
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
};
private:
std::unique_ptr<LogicalPlan> plan_;
utils::Timer cache_timer_;
};
struct CachedQuery {
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
};
struct QueryCacheEntry {
bool operator==(const QueryCacheEntry &other) const { return first == other.first; }
bool operator<(const QueryCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
CachedQuery second;
};
struct PlanCacheEntry {
bool operator==(const PlanCacheEntry &other) const { return first == other.first; }
bool operator<(const PlanCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
std::shared_ptr<CachedPlan> second;
};
/**
* A container for data related to the parsing of a query.
*/
struct ParsedQuery {
std::string query_string;
std::map<std::string, storage::v3::PropertyValue> user_parameters;
Parameters parameters;
frontend::StrippedQuery stripped_query;
AstStorage ast_storage;
Query *query;
std::vector<AuthQuery::Privilege> required_privileges;
bool is_cacheable{true};
};
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::v3::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config);
class SingleNodeLogicalPlan final : public LogicalPlan {
public:
SingleNodeLogicalPlan(std::unique_ptr<plan::LogicalOperator> root, double cost, AstStorage storage,
const SymbolTable &symbol_table)
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {}
const plan::LogicalOperator &GetRoot() const override { return *root_; }
double GetCost() const override { return cost_; }
const SymbolTable &GetSymbolTable() const override { return symbol_table_; }
const AstStorage &GetAstStorage() const override { return storage_; }
private:
std::unique_ptr<plan::LogicalOperator> root_;
double cost_;
AstStorage storage_;
SymbolTable symbol_table_;
};
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers);
/**
* Return the parsed *Cypher* query's AST cached logical plan, or create and
* cache a fresh one if it doesn't yet exist.
* @param predefined_identifiers optional identifiers you want to inject into a query.
* If an identifier is not defined in a scope, we check the predefined identifiers.
* If an identifier is contained there, we inject it at that place and remove it,
* because a predefined identifier can be used only in one scope.
*/
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers = {});
} // namespace memgraph::query::v2

View File

@@ -0,0 +1,415 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <optional>
#include <vector>
#include <cppitertools/filter.hpp>
#include <cppitertools/imap.hpp>
#include "query/v2/exceptions.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
///////////////////////////////////////////////////////////
// Our communication layer and query engine don't mix
// very well on Centos because OpenSSL version available
// on Centos 7 include libkrb5 which has brilliant macros
// called TRUE and FALSE. For more detailed explanation go
// to memgraph.cpp.
//
// Because of the replication storage now uses some form of
// communication so we have some unwanted macros.
// This cannot be avoided by simple include orderings so we
// simply undefine those macros as we're sure that libkrb5
// won't and can't be used anywhere in the query engine.
#include "storage/v3/storage.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
#undef FALSE
#undef TRUE
///////////////////////////////////////////////////////////
#include "storage/v3/view.hpp"
#include "utils/bound.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::query::v2 {
class VertexAccessor;
class EdgeAccessor final {
public:
storage::v3::EdgeAccessor impl_;
explicit EdgeAccessor(storage::v3::EdgeAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::v3::View view) const { return impl_.IsVisible(view); }
storage::v3::EdgeTypeId EdgeType() const { return impl_.EdgeType(); }
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
return SetProperty(key, storage::v3::PropertyValue());
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
VertexAccessor To() const;
VertexAccessor From() const;
bool IsCycle() const;
int64_t CypherId() const { return impl_.Gid().AsInt(); }
storage::v3::Gid Gid() const noexcept { return impl_.Gid(); }
bool operator==(const EdgeAccessor &e) const noexcept { return impl_ == e.impl_; }
bool operator!=(const EdgeAccessor &e) const noexcept { return !(*this == e); }
};
class VertexAccessor final {
public:
storage::v3::VertexAccessor impl_;
static EdgeAccessor MakeEdgeAccessor(const storage::v3::EdgeAccessor impl) { return EdgeAccessor(impl); }
explicit VertexAccessor(storage::v3::VertexAccessor impl) : impl_(impl) {}
bool IsVisible(storage::v3::View view) const { return impl_.IsVisible(view); }
auto Labels(storage::v3::View view) const { return impl_.Labels(view); }
auto PrimaryLabel(storage::v3::View view) const { return impl_.PrimaryLabel(view); }
storage::v3::Result<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabel(label); }
storage::v3::ResultSchema<bool> AddLabelAndValidate(storage::v3::LabelId label) {
return impl_.AddLabelAndValidate(label);
}
storage::v3::Result<bool> RemoveLabel(storage::v3::LabelId label) { return impl_.RemoveLabel(label); }
storage::v3::ResultSchema<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
return impl_.HasLabel(label, view);
}
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> SetPropertyAndValidate(
storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto InEdges(storage::v3::View view) const { return InEdges(view, {}); }
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto OutEdges(storage::v3::View view) const { return OutEdges(view, {}); }
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
int64_t CypherId() const { return impl_.Gid().AsInt(); }
storage::v3::Gid Gid() const noexcept { return impl_.Gid(); }
bool operator==(const VertexAccessor &v) const noexcept {
static_assert(noexcept(impl_ == v.impl_));
return impl_ == v.impl_;
}
bool operator!=(const VertexAccessor &v) const noexcept { return !(*this == v); }
};
inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
class DbAccessor final {
storage::v3::Storage::Accessor *accessor_;
class VerticesIterable final {
storage::v3::VerticesIterable iterable_;
public:
class Iterator final {
storage::v3::VerticesIterable::Iterator it_;
public:
explicit Iterator(storage::v3::VerticesIterable::Iterator it) : it_(it) {}
VertexAccessor operator*() const { return VertexAccessor(*it_); }
Iterator &operator++() {
++it_;
return *this;
}
bool operator==(const Iterator &other) const { return it_ == other.it_; }
bool operator!=(const Iterator &other) const { return !(other == *this); }
};
explicit VerticesIterable(storage::v3::VerticesIterable iterable) : iterable_(std::move(iterable)) {}
Iterator begin() { return Iterator(iterable_.begin()); }
Iterator end() { return Iterator(iterable_.end()); }
};
public:
explicit DbAccessor(storage::v3::Storage::Accessor *accessor) : accessor_(accessor) {}
std::optional<VertexAccessor> FindVertex(storage::v3::Gid gid, storage::v3::View view) {
auto maybe_vertex = accessor_->FindVertex(gid, view);
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
return std::nullopt;
}
void FinalizeTransaction() { accessor_->FinalizeTransaction(); }
VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); }
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) {
return VerticesIterable(accessor_->Vertices(label, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) {
return VerticesIterable(accessor_->Vertices(label, property, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
const storage::v3::PropertyValue &value) {
return VerticesIterable(accessor_->Vertices(label, property, value, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) {
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
}
// TODO Remove when query modules have been fixed
[[deprecated]] VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex()); }
storage::v3::ResultSchema<VertexAccessor> InsertVertexAndValidate(
const storage::v3::LabelId primary_label, const std::vector<storage::v3::LabelId> &labels,
const std::vector<std::pair<storage::v3::PropertyId, storage::v3::PropertyValue>> &properties) {
auto maybe_vertex_acc = accessor_->CreateVertexAndValidate(primary_label, labels, properties);
if (maybe_vertex_acc.HasError()) {
return {std::move(maybe_vertex_acc.GetError())};
}
return VertexAccessor{maybe_vertex_acc.GetValue()};
}
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::v3::EdgeTypeId &edge_type) {
auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
if (maybe_edge.HasError()) return storage::v3::Result<EdgeAccessor>(maybe_edge.GetError());
return EdgeAccessor(*maybe_edge);
}
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(&edge->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<EdgeAccessor>{};
}
return std::make_optional<EdgeAccessor>(*value);
}
storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<ReturnType>{};
}
const auto &[vertex, edges] = *value;
std::vector<EdgeAccessor> deleted_edges;
deleted_edges.reserve(edges.size());
std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
}
storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<VertexAccessor>{};
}
return {std::make_optional<VertexAccessor>(*value)};
}
storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); }
const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); }
const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); }
void AdvanceCommand() { accessor_->AdvanceCommand(); }
utils::BasicResult<storage::v3::ConstraintViolation, void> Commit() { return accessor_->Commit(); }
void Abort() { accessor_->Abort(); }
bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); }
bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const {
return accessor_->LabelPropertyIndexExists(label, prop);
}
int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); }
int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); }
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const {
return accessor_->ApproximateVertexCount(label, property);
}
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
const storage::v3::PropertyValue &value) const {
return accessor_->ApproximateVertexCount(label, property, value);
}
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) const {
return accessor_->ApproximateVertexCount(label, property, lower, upper);
}
storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); }
storage::v3::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); }
const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); }
storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); }
};
} // namespace memgraph::query::v2
namespace std {
template <>
struct hash<memgraph::query::v2::VertexAccessor> {
size_t operator()(const memgraph::query::v2::VertexAccessor &v) const {
return std::hash<decltype(v.impl_)>{}(v.impl_);
}
};
template <>
struct hash<memgraph::query::v2::EdgeAccessor> {
size_t operator()(const memgraph::query::v2::EdgeAccessor &e) const {
return std::hash<decltype(e.impl_)>{}(e.impl_);
}
};
} // namespace std

View File

@@ -0,0 +1,24 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include "query/v2/typed_value.hpp"
namespace memgraph::query::v2 {
struct DiscardValueResultStream final {
void Result(const std::vector<query::v2::TypedValue> & /*values*/) {
// do nothing
}
};
} // namespace memgraph::query::v2

Some files were not shown because too many files have changed in this diff Show More