Compare commits

..

224 Commits

Author SHA1 Message Date
Tyler Neely
15cf7794cf Check-in initial RsmEngine class 2022-08-19 14:40:56 +00:00
Tyler Neely
d42fa1fc87 Fix ShardMap::GetShardForKey 2022-08-18 15:14:04 +00:00
Tyler Neely
2a3a3338f0 Clean up coordinator a bit 2022-08-18 14:04:28 +00:00
Tyler Neely
2320f95dd1 Update the coordinator to include request for initializing a new shard map 2022-08-18 14:01:47 +00:00
Tyler Neely
3794693356 Make shard_rsm.hpp a bit more clear 2022-08-18 13:34:11 +00:00
Tyler Neely
d50b6c1abb Move rsm_client to src/io/rsm 2022-08-18 13:20:56 +00:00
Tyler Neely
d292162b00 Use new RsmClient in raft test 2022-08-17 16:40:18 +00:00
Tyler Neely
9c4556618a Rework RsmClient and update sharded_map commit 2022-08-17 16:35:35 +00:00
Tyler Neely
f0dc0d911d Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:51:09 +00:00
Tyler Neely
e545beaf78 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:13 +00:00
Tyler Neely
3c50b68954 Small comment fix 2022-08-16 16:47:13 +00:00
Tyler Neely
fad0c893f1 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:13 +00:00
Tyler Neely
866f1feeeb Added request for edge id batch request to coordinator rsm 2022-08-16 16:47:13 +00:00
gvolfing
16d1fa2c22 Gerneral clean-up 2022-08-16 16:47:13 +00:00
gvolfing
518e8df04c Resolve issues with the sharded_map test 2022-08-16 16:47:13 +00:00
gvolfing
9c453779f5 Add the coordinator part to the sharded_map test 2022-08-16 16:47:13 +00:00
gvolfing
5d66a4e828 Check against cas_succeeded 2022-08-16 16:47:13 +00:00
gvolfing
2553a8fdc3 Remove unfinished statement 2022-08-16 16:47:13 +00:00
gvolfing
9aeca7a4b3 Add test for sharded_map 2022-08-16 16:47:13 +00:00
gvolfing
1b2c8f6b29 Move RsmClient into a separate folder and header 2022-08-16 16:47:13 +00:00
gvolfing
a74a556fcc Remove unnecessary storage pool, general cleanup 2022-08-16 16:47:13 +00:00
gvolfing
66e791d042 Make Coordinator apply splitting on request. 2022-08-16 16:47:13 +00:00
gvolfing
797c76cdfd Add logic to split shards 2022-08-16 16:47:13 +00:00
gvolfing
7af917e408 Add abstraction for RsmClient 2022-08-16 16:47:13 +00:00
gvolfing
c7282e8935 Temporarly remove the stanby server pool 2022-08-16 16:47:13 +00:00
gvolfing
4ed5801588 Change min and max key related shard logic 2022-08-16 16:47:13 +00:00
gvolfing
5963c83a60 Update Coordinator 2022-08-16 16:47:13 +00:00
gvolfing
edf1293274 Add GetShardMap read-only request to coordinator 2022-08-16 16:47:13 +00:00
gvolfing
dd46cc407f Return an error and the latest known ShardMap version if the requested key is not possibly stored in the given shard 2022-08-16 16:47:13 +00:00
Tyler Neely
92d69e080c Add todos to coordinator.hpp 2022-08-16 16:47:13 +00:00
Tyler Neely
629fd231b3 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:47:09 +00:00
Tyler Neely
95e90e6c2e Check-in in-progress shard test 2022-08-16 16:42:28 +00:00
Tyler Neely
502a9b4823 Check-in coordinator_rsm.hpp 2022-08-16 16:42:28 +00:00
Tyler Neely
c62b0eff93 Add Simulator::RegisterNew helper method 2022-08-16 16:42:28 +00:00
Tyler Neely
6b9311e0b8 Bump shard_rsm to use new method names 2022-08-16 16:42:28 +00:00
Tyler Neely
e3dd404865 Add new test, start to fill out coordinator RSM 2022-08-16 16:42:28 +00:00
Tyler Neely
523e2b9186 Merge branch 'T0912-MG-in-memory-shard-map' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-16 16:42:28 +00:00
Tyler Neely
79539d13c9 Use Read/Apply instead of read/apply in Rsm concept 2022-08-16 16:42:28 +00:00
Tyler Neely
6169eb221b Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:41 +02:00
Tyler Neely
342611691a Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:30 +02:00
Tyler Neely
c256dce601 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:13:03 +02:00
Tyler Neely
2e9cf8f37d Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:12:13 +02:00
Tyler Neely
47186cab18 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:56 +02:00
Tyler Neely
dcc5ec920a Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:41 +02:00
Tyler Neely
c7d96ed5c5 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:11:11 +02:00
Tyler Neely
7ad9b62968 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:10:37 +02:00
Tyler Neely
bb44aaa188 Update src/io/rsm/raft.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-16 17:09:51 +02:00
Tyler Neely
9961659103 Use spdlog instead of cout 2022-08-16 14:52:03 +00:00
Tyler Neely
6f906c0488 Use spdlog instead of cout 2022-08-16 14:44:17 +00:00
Tyler Neely
d161163fb4 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-15 13:49:50 +00:00
Tyler Neely
1423da5f51 Improve comments around majority index selection 2022-08-15 11:57:19 +00:00
Tyler Neely
5e4733ac98 Add extra safety checks to commit_index management 2022-08-12 13:20:11 +00:00
Tyler Neely
14c9e68456 Transport prototype (#466) 2022-08-12 08:24:32 +02:00
Tyler Neely
cbdfd54e5f Properly apply state updates in Follower Handle for AppendRequest 2022-08-11 12:30:26 +00:00
Tyler Neely
69a2d73172 Fix bug-prone leader application of write requests to the replicated state 2022-08-11 12:17:33 +00:00
Tyler Neely
5b69f435b6 Small comment cleanup 2022-08-11 08:39:11 +00:00
Tyler Neely
f098d6b331 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-10 10:48:17 +00:00
Tyler Neely
f6edce8f0b Apply Kostas's feedback to transport interface 2022-08-10 09:34:02 +00:00
Tyler Neely
1225230c3a Apply feedback from cpplint 2022-08-09 15:52:24 +00:00
Tyler Neely
5f225a623e align cpp w/ hpp naming 2022-08-09 15:21:52 +00:00
Tyler Neely
d15105455a Clean up comments and TODOs 2022-08-09 14:07:42 +00:00
Tyler Neely
36c133ce4e Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-08 16:29:48 +00:00
Tyler Neely
91d48f05f3 Merge branch 'project-pineapples' into T0879-MG-transport-prototype 2022-08-08 13:34:30 +02:00
Tyler Neely
7bac0d1c61 Use default equality operator for Address 2022-08-08 11:32:42 +00:00
Tyler Neely
a4b1d5efb4 Update src/io/future.hpp
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-08-08 13:27:06 +02:00
Tyler Neely
0a43afdec1 Update raft implementation to use std::chrono like upstream 2022-08-04 13:28:52 +00:00
Tyler Neely
618a3d96b3 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-04 12:07:54 +00:00
jbajic
5012824e05 Address review comments 2022-08-04 11:45:16 +02:00
gvolfing
ee16954641 Add minimum and maximum key for shard rsm prototype 2022-08-04 11:20:19 +02:00
gvolfing
0aab854e34 Add shard rsm prototype 2022-08-04 10:47:00 +02:00
Jure Bajic
a12a1ea358 Move schema to storage v3 and query v2
* Move schema to storage v3

* Remove schema from v2

* Move schema to query v2

* Remove schema from query v1

* Make glue v2

* Move schema related tests to newer versions of query and storage

* Fix typo in CMake

* Fix interpreter test

* Fix clang tidy errors

* Change temp dir name
2022-08-04 09:50:02 +02:00
Tyler Neely
343648f564 Merge branch 'T0941-MG-implement-basic-raft-version' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-08-04 07:49:09 +00:00
gvolfing
726fabd387 Constrain Raft class template with concept 2022-08-04 09:09:18 +02:00
Tyler Neely
d1c5aead61 Implement the rest of the KV-on-Raft RSM test 2022-08-03 15:07:08 +00:00
Tyler Neely
aebac2c519 Hoist read and write requests on the RSM into wrapper structs for futureproofing 2022-08-03 14:21:37 +00:00
Tyler Neely
74b354979c Check in incremental progress on fixing compilation error 2022-08-03 13:42:08 +00:00
gvolfing
b5cff5999b Unfinished rsm changes 2022-08-03 14:39:09 +02:00
Tyler Neely
b2a8063a96 Make more temporaries const 2022-08-02 14:33:05 +00:00
Tyler Neely
54369958d1 Extract BlockedServers functionality from MaybeTickSimulator and IncrementServerCountAndWaitForQuiescentState 2022-08-02 14:30:12 +00:00
Tyler Neely
4f06eb0f2f Address rule-of-five for abstract inhertance in OpaquePromise 2022-08-02 14:24:59 +00:00
Tyler Neely
ad0a8c4942 Use simple virtual inheritance instead of bespoke vtable for OpaquePromise 2022-08-02 13:36:40 +00:00
Tyler Neely
997fdf5a16 Check-in skeleton for RSM logic on top of Raft 2022-08-02 10:18:38 +00:00
Tyler Neely
8be88deee6 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0941-MG-implement-basic-raft-version 2022-08-02 08:48:20 +00:00
Tyler Neely
902a46d14f Make dynamic message Take messages rvalue methods 2022-08-02 07:12:05 +00:00
Tyler Neely
3b44ef70b6 Denest operator< for clarity 2022-08-02 07:06:43 +00:00
Tyler Neely
4d8f9ea821 Remove unnecessary comment 2022-08-02 07:03:12 +00:00
Tyler Neely
e935a9a7b1 Make parenthesization less confusing 2022-08-02 07:02:17 +00:00
Tyler Neely
dd93b594bc Remove dead code 2022-08-02 07:01:25 +00:00
Tyler Neely
997d25d536 Remove buggy usage of std::forward 2022-08-01 15:53:37 +00:00
Tyler Neely
ace5f2b639 Complete clang-tidy cleanup 2022-08-01 15:44:34 +00:00
Tyler Neely
9576eea051 Address some feedback from clang-tidy 2022-08-01 15:37:16 +00:00
Tyler Neely
5bb3361a2d Use std::chrono::microseconds explicitly for Duration. Fix compiler warning related to timeouts 2022-08-01 14:32:07 +00:00
gvolfing
a84f5f6115 Undo the structured binding changes 2022-08-01 16:13:29 +02:00
Tyler Neely
cacb0dac80 Avoid unnamed namespace in future.hpp 2022-08-01 14:12:07 +00:00
gvolfing
918fa7212e Further improve const-correctness and replace dispensable structured bindings with exact return values 2022-08-01 15:30:44 +02:00
gvolfing
e905591372 Add references to the raft paper 2022-08-01 15:17:29 +02:00
Tyler Neely
649b5437b0 Improve docs around Io interface 2022-08-01 12:58:55 +00:00
Tyler Neely
102d997288 Update src/io/simulator/simulator_handle.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-01 14:52:06 +02:00
Tyler Neely
a3f3e05fc2 Include replier address in operator< for PromiseKey 2022-08-01 12:50:48 +00:00
Tyler Neely
4f4eb9ea13 Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-08-01 14:47:13 +02:00
Tyler Neely
ca638db509 Remove extra blank line 2022-08-01 12:45:45 +00:00
Tyler Neely
7d33bb1937 Update Future unit test to use gtest 2022-08-01 12:42:37 +00:00
Tyler Neely
b8487da392 Move future test to unit tests 2022-08-01 11:58:54 +00:00
Tyler Neely
dbd744470b Add future benchmark 2022-08-01 11:48:21 +00:00
Tyler Neely
b2b11f3a30 Run simulation tests in CI 2022-08-01 11:24:21 +00:00
Tyler Neely
eb1b6c3ac8 Make SimulatorHandle::TimeoutPromisesPastDeadline private 2022-08-01 11:10:00 +00:00
Tyler Neely
0c2cbb5461 Make a couple more things const 2022-08-01 11:08:17 +00:00
Tyler Neely
69ea79a75e Make a couple more things const 2022-08-01 11:07:39 +00:00
Tyler Neely
c0d6cec9ab Split simulator_handle.cpp into a source and header file 2022-08-01 11:06:08 +00:00
Tyler Neely
9b915be1aa Make some methods const. Remove outdated TODO 2022-08-01 10:30:37 +00:00
Tyler Neely
b127f6f345 Use spdlog and create a formatter for Address 2022-08-01 10:23:25 +00:00
jbajic
f57f30c8cf Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-08-01 10:46:11 +02:00
Tyler Neely
66ef5b2072 Make certain temporaries const 2022-08-01 08:26:09 +00:00
Tyler Neely
f877f8e1d3 Add some requests and additional metadata around coordinator operations 2022-07-29 14:40:08 +00:00
gvolfing
0ea9878fd1 CTAD 2022-07-29 14:27:55 +02:00
gvolfing
af94270dc6 Remove redundant else clauses when returning from every branch 2022-07-29 14:22:57 +02:00
gvolfing
8aac3ae7ea Make unchanged variables const 2022-07-29 14:08:37 +02:00
Tyler Neely
507018a630 Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-07-29 11:50:21 +00:00
Tyler Neely
e0bbf4766a Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-29 11:47:50 +00:00
Tyler Neely
1a48e0ffa8 Merge github.com:memgraph/memgraph into T0912-MG-in-memory-shard-map 2022-07-29 11:46:54 +00:00
Jure Bajic
462daf3a2b Enforce schema on vertex creation
- Separating schema definition from schema validation
- Updating vertex_accessor and db_accessors with necessary methods
- Adding a primary label to Vertex
- Adding schema tests
- Updating existing tests for storage v3, and deprecating old:
  - interpreter => interpreter_v2
  - query_plan_accumulate_aggregate => storage_v3_query_plan_accumulate_aggregate
  - query_plan_create_set_remove_delete => storage_v3_query_plan_create_set_remove_delete
  - query_plan_bag_semantics => storage_v3_query_plan_bag_semantics
  - query_plan_edge_cases => storage_v3_query_plan_edge_cases
  - query_plan_v2_create_set_remove_delete => storage_v3_query_plan_v2_create_set_remove_delete
  - query_plan_match_filter_return => storage_v3_query_plan_match_filter_return
2022-07-29 13:38:17 +02:00
Tyler Neely
6a808ce1fc Delete empty line 2022-07-28 16:27:57 +00:00
Tyler Neely
a89a2d9caa Update src/io/simulator/simulator_transport.hpp
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 18:26:36 +02:00
Tyler Neely
c4f1764fee Rely on default initialization of random number generators 2022-07-28 16:24:45 +00:00
Tyler Neely
48a445f2ed Merge branch 'T0879-MG-transport-prototype' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-28 16:24:34 +00:00
Tyler Neely
406da4b25c Use uint8_t instead of int for SimulatorConfig::drop_percent 2022-07-28 16:23:10 +00:00
Tyler Neely
8fde05444d Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-28 18:13:50 +02:00
Tyler Neely
6bdfb43ad0 Use rvalue reference requirement for Future::Wait 2022-07-28 16:07:20 +00:00
Tyler Neely
9056e2c97a Make Address's == and < operators consider uuid 2022-07-28 16:06:48 +00:00
Marko Budiselic
2009fefc8a Merge branch 'master' into project-pineapples 2022-07-28 15:36:51 +02:00
Marko Budiselić
eb3f96d1f6 Bring changes from master to project-pineapples (#477)
* Fix aggregation functions on `null` and group-by inputs (#448)
* Upgrade Antrl to 4.10.1 and remove antlr_lock (#441)
* Update clang-tidy job (#476)
* Add parser stress test (#463)

NOTE: Doing this to have buildable comments on the project-pineapples branch

Co-authored-by: gvolfing <107616712+gvolfing@users.noreply.github.com>
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-07-28 15:36:17 +02:00
Tyler Neely
51371398ce Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:27:11 +02:00
Tyler Neely
b38dc28e01 Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:26:00 +02:00
Tyler Neely
6cc550719b Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:25:25 +02:00
Tyler Neely
ef70b858e2 Centralize Future's Shared state taking logic. Fix a race condition in quiescence tracking 2022-07-27 15:24:33 +00:00
Tyler Neely
509c12956c Update src/io/future.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 17:09:32 +02:00
Tyler Neely
1527509e36 Use std::chrono instead of raw uint64_t 2022-07-27 15:07:01 +00:00
Tyler Neely
0cfb68bb89 Update src/io/simulator/simulator_handle.hpp
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-07-27 16:56:14 +02:00
Tyler Neely
e2968c2e21 Use std::chrono instead of raw uint64_t 2022-07-27 14:55:39 +00:00
Tyler Neely
4ee4612a9c Remove removed raft.cpp from CMakeLists.txt for the simulator tests 2022-07-27 14:06:16 +00:00
Tyler Neely
dc38296575 avoid unnecessary std::optional around std::function which can be nullable 2022-07-27 14:05:57 +00:00
Tyler Neely
06962a3ec4 Check shard map version 2022-07-26 15:35:37 +00:00
Tyler Neely
c0b0b08d12 Check-in deregistration and shard split things 2022-07-26 15:32:22 +00:00
Tyler Neely
c100a86644 Check-in changes from hangout 2022-07-26 15:23:55 +00:00
Tyler Neely
3ec1ff9ee4 Commit small changes from meeting 2022-07-26 11:49:08 +00:00
Tyler Neely
98206caf85 Separate split functionality from ShardMap 2022-07-25 10:38:50 +00:00
Tyler Neely
962767ea1c Initial check-in of shard map ideas 2022-07-25 10:34:27 +00:00
Tyler Neely
2a199c9484 Merge branch 'master' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-22 12:14:18 +00:00
Tyler Neely
a85e9fcdd4 Remove raft-related code from transport prototype branch 2022-07-22 12:11:00 +00:00
jbajic
264b233053 Merge branch 'project-pineapples' into E112-MG-implement-partial-schema 2022-07-22 11:48:45 +02:00
Kostas
72b4337864 Added ScanVertices request/response example 2022-07-21 18:37:06 +03:00
János Benjamin Antal
ed71332773 Add dummy storage engine to test 2022-07-21 16:23:27 +02:00
Tyler Neely
fd3d70d847 Move raft-related code to replicated state machine (rsm) namespace 2022-07-21 12:41:02 +00:00
Tyler Neely
f8e5032011 Restructure files into namespaces and subdirectories 2022-07-21 12:10:23 +00:00
Tyler Neely
5b59d890c0 Remove awkward Reply sugar for now 2022-07-21 11:53:51 +00:00
Tyler Neely
0351db2461 Move future.hpp to src/io 2022-07-21 11:45:05 +00:00
Tyler Neely
c379475e12 Merge branch 'project-pineapples' of github.com:memgraph/memgraph into T0879-MG-transport-prototype 2022-07-21 11:01:04 +00:00
Tyler Neely
9c5d19bc19 Start to adapt the transport-related code to use namespaces 2022-07-21 11:00:11 +00:00
Tyler Neely
4d85a7e605 Check-in simulator_transport.hpp 2022-07-21 10:49:33 +00:00
Tyler Neely
f6c2202772 Use anonymous namespace to avoid friend declarations in future.hpp 2022-07-21 10:49:19 +00:00
Tyler Neely
498ae97ae9 Clean up log messages 2022-07-21 10:26:08 +00:00
Tyler Neely
fbd015d3c6 Continue codebase clean-up in preparation for merge 2022-07-21 10:24:13 +00:00
Tyler Neely
1ef11a36f4 Apply feedback from cpplint 2022-07-21 09:33:06 +00:00
Tyler Neely
9ae1671e4f Fix bug with MgFuture where its safety check was not correctly initialized 2022-07-20 18:17:06 +00:00
Tyler Neely
b4afe45de5 Handle Request timeouts properly in the simulator 2022-07-20 17:18:07 +00:00
Tyler Neely
581925e660 Fix a few bugs, expose stats 2022-07-20 17:07:18 +00:00
Tyler Neely
afef6dc11b A large batch of bug fixes and clean-ups 2022-07-20 13:18:00 +00:00
Tyler Neely
689336e765 Fix several bugs, clean-ups and tuning 2022-07-19 14:57:01 +00:00
Jure Bajic
2ceaf59767 Create query engine v2 (#444)
Create version v2 of the query engine.
Adjust CMake and lisp files
Connect query engine v2 with storage engine v3
2022-07-19 12:28:19 +02:00
Marko Budiselic
5dd0ddc352 Killing bugs 2022-07-18 11:36:56 +02:00
János Benjamin Antal
c0bee760bf git 2022-07-18 08:21:04 +02:00
Marko Budiselic
1480d975a7 Fix the basic_request example, something sill broken 2022-07-17 17:22:39 +02:00
Tyler Neely
9dc37a87f7 Enable more aggressive message scrambling to get a race condition to jump out about 0.2% of the time 2022-07-15 16:46:54 +00:00
Tyler Neely
46a2879ece Fix a number of bugs in raft, and it can now go for around 1400 simulations before hitting a race condition 2022-07-15 16:44:42 +00:00
Tyler Neely
b2142e8d38 Remove unnecessary log message 2022-07-15 16:20:05 +00:00
Tyler Neely
0dc69c180d Start cranking up the testing intensity and fix a variety of correctness and code quality issues 2022-07-15 16:19:00 +00:00
Tyler Neely
ad1d8637e5 Fix race condition in simulator. Make leader election function more reliably in raft code 2022-07-15 09:07:39 +00:00
Tyler Neely
4876b7cd8c Start removing load-bearing sleep statements to get race conditions to jump out in the simulator 2022-07-15 05:43:40 +00:00
Tyler Neely
a6133dab49 Start to tune various timeouts in the raft implementation 2022-07-15 05:38:04 +00:00
Tyler Neely
3466c15f76 Make raft work in the happy path 2022-07-14 19:57:52 +00:00
Tyler Neely
63beeb8771 Send log state along with AppendRequest messages from Leader 2022-07-14 19:06:56 +00:00
Tyler Neely
d14f7705b1 Get most of the raft data path up and running 2022-07-14 18:45:24 +00:00
Tyler Neely
5e98971bb2 Continue filling out AppendEntries state transition logic 2022-07-14 13:06:54 +00:00
Tyler Neely
dc78adde40 Move empty in-flight early exit of simulator tick to before time advancement 2022-07-14 13:06:22 +00:00
Tyler Neely
80970a97f0 Get basic leader election to work and clean some things up 2022-07-14 12:18:09 +00:00
Tyler Neely
57533f2746 A large number of scattered cleanups. Don't use transport Request in state machine code 2022-07-14 10:50:49 +00:00
Tyler Neely
1a12a80af0 Continue to implement raft state machine transitions 2022-07-14 07:09:50 +00:00
Tyler Neely
30c97e658d Implement some more of the raft state machine transitions 2022-07-13 22:31:51 +00:00
Tyler Neely
dd9862d32c Add support for scrambling messages to the simulator 2022-07-13 19:59:23 +00:00
Tyler Neely
dc6548c996 Add support to the simulator to drop messages randomly 2022-07-13 19:51:04 +00:00
Tyler Neely
881e914b92 Add random number generator support to the IO interface 2022-07-13 13:37:38 +00:00
Tyler Neely
48ee40ce87 Start to fill out more raft logic now that role-specialized Handle and Cron are in place 2022-07-13 06:50:16 +00:00
Tyler Neely
d6742f643c Significant simplification of event-handling logic by using std::visit while matching on both the message type and node role 2022-07-12 19:04:32 +00:00
Tyler Neely
d8f09b59b3 Add MgFuture::Cancel to signal that a future is intentionally dropped 2022-07-12 19:03:12 +00:00
Tyler Neely
2e5d8b7e8c Check-in in-progress work for raft test on top of simulator 2022-07-11 18:01:13 +00:00
Tyler Neely
88eee66258 Small cleanup of basic_request test 2022-07-11 18:00:30 +00:00
Tyler Neely
d4cb259979 Fix issue with recursive parameter pack std::variant conversion of std::any. Improve documentation of the complex functionality 2022-07-11 17:59:52 +00:00
Tyler Neely
1a87dd2497 Add two methods to MgFuture: IsReady and TryGet 2022-07-11 17:59:08 +00:00
Jure Bajic
3f4f66b57f Create schema DDL expressions
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Add schema operations in storage

* Add create and show schema queries

* Add privileges for schema

* Add missing keywords into lexer

* Add drop schema query

* Add schema visitors

* Update metadata

* Add PrepareSchemaQuery function

* Implement show schemas

* Add show schema query

* Fix schema visitor

* Add common schema type

* Fix grammar

* Temporary create ddl logic

* Fix naming for schemaproperty type to schema type

* Rename schemaproperty to schemapropertytype

* Enable Create schema ddl

* Override visitPropertyType

* Add initial schema implementation

* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

* Add verification on creation and deletion

* Rename DeleteSchema to DropSchema

* Remove list and map from lexer

* Fix grammar with schemaTypeMap

* Add privilege and cypher visitor tests

* Catch repeating type name in schema definition

* Fix conflicting keywords

* Add notifications

* Drop float support

* Finish interpreter tests

* Fix tests

* Fix clang tidy errors

* Fix GetSchema

* Replace for with transfrom

* Add cloning og schema_property_map

* Address review comments

* Rename SchemaPropertyType to SchemaType

* Remove inline

* Assert of schema properties

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-11 09:20:15 +02:00
Tyler Neely
15d637729e Clean-up basic_request example 2022-07-08 14:10:33 +00:00
Tyler Neely
5ee95a2e70 Add ShutDown method to Simulator and update basic_request test to utilize it 2022-07-08 14:05:02 +00:00
Tyler Neely
597a5d191c Check-in SimulatorConfig 2022-07-08 11:49:29 +00:00
Tyler Neely
eee2a7e019 Fix memory leak caught by LSAN 2022-07-08 11:04:44 +00:00
Jure Bajic
2998f92595 Add initial schema implementation
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-07-08 10:33:43 +02:00
Tyler Neely
25b0a445b9 Properly fill response promises in the simulator 2022-07-08 08:21:37 +00:00
Tyler Neely
20a6dae047 Get the message receive path up and running. Large number of small refactors 2022-07-07 20:58:16 +00:00
Tyler Neely
04dbedc3af Add functionality for converting from std::any to a std::variant. Improve type names. Continue simulator implementation 2022-07-07 11:27:02 +00:00
Tyler Neely
a0058bc10a Add OpaquePromise type with correct dtor. Make a few structs comparable for storage in maps. Fix initialization of Shared 2022-07-06 15:19:16 +00:00
Tyler Neely
f601e83f6f Continue structuring the simulator and filling in its implementation 2022-07-05 21:11:39 +00:00
Tyler Neely
e1aab7065f Use TSAN on futures test. Continue implementation of simulator 2022-07-05 18:02:01 +00:00
Tyler Neely
28516763b9 Fix issue with movement/consumption tracking in MgFuture and MgPromise 2022-07-05 17:36:47 +00:00
Tyler Neely
29e8d8e72c Fix cmake config for new transport 2022-07-05 16:04:03 +00:00
Tyler Neely
6debc9e7d8 Revamp MgFuture, continue threading logic through Simulator 2022-07-05 15:45:59 +00:00
János Benjamin Antal
1bdc32ba5d Copy storage v2 to create storage v3 (#416)
* Copy storage v2 to v3

* Integrate v3 to cmake

* Fix clang-tidy warnings

* Add dummy unit test for storage-v3 to trigger build for code analysis builds
2022-07-05 08:20:59 +02:00
Tyler Neely
eb4ca543ea Continue to implement the simulated transport 2022-07-04 21:42:10 +00:00
Tyler Neely
38ca430713 Small refactor of cmake for simulator test 2022-07-04 20:44:51 +00:00
Tyler Neely
24128e0bca Check-in new test 2022-07-04 19:42:23 +00:00
Tyler Neely
73719b2120 Continue to bolt-down MgFuture impl 2022-07-04 18:54:09 +00:00
Tyler Neely
20839b0ae0 Add simple test for MgFuture 2022-07-04 15:00:32 +00:00
Tyler Neely
6cec9acbb9 Add basic test skeleton 2022-07-04 11:56:41 +00:00
Tyler Neely
cb70431301 Reset optional after moving its value out in Future::Wait 2022-07-04 11:55:18 +00:00
Tyler Neely
0ef1f7eb5d Apply Rule-Of-Five and uniform CamelCase function names 2022-07-01 08:45:19 +00:00
Tyler Neely
4140f3e05e [SQUASH ME] Initial check-in for getting feedback 2022-06-29 15:52:37 +00:00
János Benjamin Antal
21870a0e7e Merge branch 'master' into project-pineapples 2022-06-23 14:49:14 +02:00
618 changed files with 81058 additions and 49795 deletions

View File

@@ -6,6 +6,7 @@ Checks: '*,
-altera-unroll-loops,
-android-*,
-cert-err58-cpp,
-cert-str34-c,
-cppcoreguidelines-avoid-c-arrays,
-cppcoreguidelines-avoid-goto,
-cppcoreguidelines-avoid-magic-numbers,
@@ -49,6 +50,7 @@ Checks: '*,
-misc-non-private-member-variables-in-classes,
-modernize-avoid-c-arrays,
-modernize-concat-nested-namespaces,
-modernize-loop-convert,
-modernize-pass-by-value,
-modernize-use-equals-default,
-modernize-use-nodiscard,

View File

@@ -1,14 +1,11 @@
[master < Epic] PR
- [ ] Check, and update documentation if necessary
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Write E2E tests
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
- [ ] Provide the full content or a guide for the final git message
[master < Task] PR
- [ ] Check, and update documentation if necessary
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Provide the full content or a guide for the final git message
To keep docs changelog up to date, one more thing to do:
- [ ] Write a release note here
- [ ] Tag someone from docs team in the comments

View File

@@ -3,7 +3,7 @@ name: Daily Benchmark
on:
workflow_dispatch:
schedule:
- cron: "0 22 * * *"
- cron: "0 1 * * *"
jobs:
release_benchmarks:
@@ -16,7 +16,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -14,7 +14,6 @@ on:
- "**/*.md"
- ".clang-format"
- "CODEOWNERS"
- "licenses/*"
jobs:
community_build:
@@ -27,7 +26,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -65,7 +64,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -76,7 +75,7 @@ jobs:
- name: Fetch all history for all tags and branches
run: git fetch
- name: Initialize deps
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
@@ -84,32 +83,6 @@ jobs:
# Initialize dependencies.
./init
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Python code analysis
run: |
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
for file in ${CHANGED_FILES}; do
echo ${file}
if [[ ${file} == *.py ]]; then
python3 -m black --check --diff ${file}
python3 -m isort --check-only --diff ${file}
fi
done
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
@@ -137,17 +110,27 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
@@ -162,7 +145,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -188,7 +171,7 @@ jobs:
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
- name: Run drivers tests
run: |
@@ -223,7 +206,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -238,7 +221,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -263,7 +246,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
@@ -279,6 +262,15 @@ jobs:
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run simulation tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run unit tests.
cd build
ctest -R memgraph__simulation --output-on-failure -j$THREADS
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
@@ -320,13 +312,13 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
@@ -345,7 +337,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -370,7 +362,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: "Jepsen Report"
@@ -386,7 +378,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -437,7 +429,7 @@ jobs:
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -2,12 +2,7 @@ name: Package All
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'vX.Y.Z'"
required: false
on: workflow_dispatch
jobs:
centos-7:
@@ -22,7 +17,7 @@ jobs:
run: |
./release/package/run.sh package centos-7
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
@@ -39,7 +34,7 @@ jobs:
run: |
./release/package/run.sh package centos-9
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
@@ -56,7 +51,7 @@ jobs:
run: |
./release/package/run.sh package debian-10
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
@@ -73,7 +68,7 @@ jobs:
run: |
./release/package/run.sh package debian-11
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
@@ -92,7 +87,7 @@ jobs:
./run.sh package debian-11 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: docker
path: build/output/docker/memgraph*.tar.gz
@@ -109,9 +104,9 @@ jobs:
run: |
./release/package/run.sh package ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-18.04
name: ubuntu-1804
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004:
@@ -126,9 +121,9 @@ jobs:
run: |
./release/package/run.sh package ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-20.04
name: ubuntu-2004
path: build/output/ubuntu-20.04/memgraph*.deb
ubuntu-2204:
@@ -143,9 +138,9 @@ jobs:
run: |
./release/package/run.sh package ubuntu-22.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-22.04
name: ubuntu-2204
path: build/output/ubuntu-22.04/memgraph*.deb
debian-11-platform:
@@ -160,45 +155,11 @@ jobs:
run: |
./release/package/run.sh package debian-11 --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
fedora-36:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package fedora-36
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: fedora-36
path: build/output/fedora-36/memgraph*.rpm
amzn-2:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: amzn-2
path: build/output/amzn-2/memgraph*.rpm
debian-11-arm:
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 60
@@ -211,46 +172,7 @@ jobs:
run: |
./release/package/run.sh package debian-11-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11-aarch64
name: debian-11-arm
path: build/output/debian-11-arm/memgraph*.deb
ubuntu-2204-arm:
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04-aarch64
path: build/output/ubuntu-22.04-arm/memgraph*.deb
upload-to-s3:
# only run upload if we specified version. Allows for runs without upload
if: "${{ github.event.inputs.memgraph_version != '' }}"
needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm]
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v3
with:
# name: # if name input parameter is not provided, all artifacts are downloaded
# and put in directories named after each one.
path: build/output/release
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: "download.memgraph.com"
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph/${{ github.event.inputs.memgraph_version }}/"

View File

@@ -3,7 +3,7 @@ name: Release CentOS 8
on:
workflow_dispatch:
schedule:
- cron: "0 22 * * *"
- cron: "0 1 * * *"
jobs:
community_build:
@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -225,7 +225,7 @@ jobs:
rpmlint memgraph*.rpm
- name: Save enterprise RPM package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise RPM package"
path: build/output/memgraph*.rpm
@@ -262,7 +262,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |

View File

@@ -3,7 +3,7 @@ name: Release Debian 10
on:
workflow_dispatch:
schedule:
- cron: "0 22 * * *"
- cron: "0 1 * * *"
jobs:
community_build:
@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
@@ -324,7 +324,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -349,7 +349,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: "Jepsen Report"

View File

@@ -19,20 +19,20 @@ jobs:
DOCKER_REPOSITORY_NAME: memgraph
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v1
- name: Log in to Docker Hub
uses: docker/login-action@v2
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Download memgraph binary
run: |

View File

@@ -1,63 +0,0 @@
name: "Mgbench Bolt Client Publish Docker Image"
on:
workflow_dispatch:
inputs:
version:
description: "Mgbench bolt client version to publish on Dockerhub."
required: true
force_release:
type: boolean
required: false
default: false
jobs:
mgbench_docker_publish:
runs-on: ubuntu-latest
env:
DOCKER_ORGANIZATION_NAME: memgraph
DOCKER_REPOSITORY_NAME: mgbench-client
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check if specified version is already pushed
run: |
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
echo $EXISTS
if [[ ${EXISTS} -eq 0 ]]; then
echo 'The specified version has been already released to DockerHub.'
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
echo 'Forcing the release!'
else
echo 'Stopping the release!'
exit 1
fi
else
echo 'All good the specified version has not been release to DockerHub.'
fi
- name: Build & push docker images
run: |
cd tests/mgbench
docker buildx build \
--build-arg TOOLCHAIN_VERSION=toolchain-v4 \
--platform linux/amd64,linux/arm64 \
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
--file Dockerfile.mgbench_client \
--push .

View File

@@ -3,7 +3,7 @@ name: Release Ubuntu 20.04
on:
workflow_dispatch:
schedule:
- cron: "0 22 * * *"
- cron: "0 1 * * *"
jobs:
community_build:
@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |

14
.gitignore vendored
View File

@@ -23,6 +23,7 @@ cmake-build-*
cmake/DownloadProject/
dist/
src/query/frontend/opencypher/generated/
src/query/v2/frontend/opencypher/generated/
tags
ve/
ve3/
@@ -34,6 +35,9 @@ TAGS
*.fas
*.fasl
# LCP generated C++ files
*.lcp.cpp
src/database/distributed/serialization.hpp
src/database/single_node_ha/serialization.hpp
src/distributed/bfs_rpc_messages.hpp
@@ -47,11 +51,21 @@ src/distributed/pull_produce_rpc_messages.hpp
src/distributed/storage_gc_rpc_messages.hpp
src/distributed/token_sharing_rpc_messages.hpp
src/distributed/updates_rpc_messages.hpp
src/query/v2/frontend/ast/ast.hpp
src/query/frontend/ast/ast.hpp
src/query/distributed/frontend/ast/ast_serialization.hpp
src/query/v2/distributed/frontend/ast/ast_serialization.hpp
src/durability/distributed/state_delta.hpp
src/durability/single_node/state_delta.hpp
src/durability/single_node_ha/state_delta.hpp
src/query/frontend/semantic/symbol.hpp
src/query/v2/frontend/semantic/symbol.hpp
src/query/distributed/frontend/semantic/symbol_serialization.hpp
src/query/v2/distributed/frontend/semantic/symbol_serialization.hpp
src/query/distributed/plan/ops.hpp
src/query/v2/distributed/plan/ops.hpp
src/query/plan/operator.hpp
src/query/v2/plan/operator.hpp
src/raft/log_entry.hpp
src/raft/raft_rpc_messages.hpp
src/raft/snapshot_metadata.hpp

View File

@@ -1,19 +1,23 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v2.3.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 23.1.0
rev: 22.3.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.12.0
hooks:
- id: isort
name: isort (python)
args: # arguments to configure black
- --line-length=120
- --include='\.pyi?$'
# these folders wont be formatted by black
- --exclude="""\.git |
\.__pycache__|
build|
libs|
.cache"""
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: v13.0.0
hooks:

View File

@@ -231,15 +231,7 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
if (NOT MG_ARCH)
set(MG_ARCH_DESCR "Host architecture to build Memgraph on. Supported values are x86_64, ARM64.")
if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(MG_ARCH "ARM64" CACHE STRING ${MG_ARCH_DESCR})
else()
set(MG_ARCH "x86_64" CACHE STRING ${MG_ARCH_DESCR})
endif()
endif()
message(STATUS "MG_ARCH: ${MG_ARCH}")
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
# setup external dependencies -------------------------------------------------

View File

@@ -4,6 +4,10 @@
---
<p align="center">
Build modern, graph-based applications on top of your streaming data in minutes.
</p>
<p align="center">
<a href="https://github.com/memgraph/memgraph/blob/master/licenses/APL.txt">
<img src="https://img.shields.io/badge/license-APL-green" alt="license" title="license"/>
@@ -18,7 +22,7 @@
<p align="center">
<a href="https://github.com/memgraph/memgraph">
<img src="https://img.shields.io/github/actions/workflow/status/memgraph/memgraph/release_debian10.yaml?branch=master&label=build%20and%20test&logo=github"/>
<img src="https://img.shields.io/github/workflow/status/memgraph/memgraph/Release%20Ubuntu%2020.04/master" alt="build" title="build"/>
</a>
<a href="https://memgraph.com/docs/" alt="Documentation">
<img src="https://img.shields.io/badge/documentation-Memgraph-orange" />
@@ -33,10 +37,9 @@
## :clipboard: Description
Memgraph is an open source graph database built for real-time streaming and
compatible with Neo4j. Whether you're a developer or a data scientist with
interconnected data, Memgraph will get you the immediate actionable insights
fast.
Memgraph is a streaming graph application platform that helps you wrangle your
streaming data, build sophisticated models that you can query in real-time, and
develop graph applications.
Memgraph directly connects to your streaming infrastructure. You can ingest data
from sources like Kafka, SQL, or plain CSV files. Memgraph provides a standard
@@ -48,20 +51,8 @@ natural and effective way to model many real-world problems without relying on
complex SQL schemas.
Memgraph is implemented in C/C++ and leverages an in-memory first architecture
to ensure that youre getting the [best possible
performance](http://memgraph.com/benchgraph) consistently and without surprises.
Its also ACID-compliant and highly available.
## :zap: Features
- Run Python, Rust, and C/C++ code natively, check out the
[MAGE](https://github.com/memgraph/mage) graph algorithm library
- Native support for machine learning
- Streaming support
- Replication
- Authentication and authorization
- ACID compliance
to ensure that youre getting the best possible performance consistently and
without surprises. Its also ACID-compliant and highly available.
## :video_game: Memgraph Playground
@@ -85,49 +76,28 @@ your browser.
### macOS
[![macOS](https://img.shields.io/badge/macOS-Docker-000000?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-macos-docker)
[![macOS](https://img.shields.io/badge/lima-AACF41?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu)
### Linux
[![Linux](https://img.shields.io/badge/Linux-Docker-FCC624?style=for-the-badge&logo=linux&logoColor=black)](https://memgraph.com/docs/memgraph/install-memgraph-on-linux-docker)
[![Debian](https://img.shields.io/badge/Debian-D70A53?style=for-the-badge&logo=debian&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-debian)
[![Ubuntu](https://img.shields.io/badge/Ubuntu-E95420?style=for-the-badge&logo=ubuntu&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu)
[![Cent OS](https://img.shields.io/badge/cent%20os-002260?style=for-the-badge&logo=centos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
[![Fedora](https://img.shields.io/badge/fedora-0B57A4?style=for-the-badge&logo=fedora&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
[![RedHat](https://img.shields.io/badge/redhat-EE0000?style=for-the-badge&logo=redhat&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
[![Cent
OS](https://img.shields.io/badge/cent%20os-002260?style=for-the-badge&logo=centos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
You can find the binaries and Docker images on the [Download
Hub](https://memgraph.com/download) and the installation instructions in the
[official documentation](https://memgraph.com/docs/memgraph/installation).
## :zap: Features
## :cloud: Memgraph Cloud
Check out [Memgraph Cloud](https://memgraph.com/docs/memgraph-cloud) - a cloud service fully managed on AWS and available in 6 geographic regions around the world. Memgraph Cloud allows you to create projects with Enterprise instances of MemgraphDB from your browser.
<p align="left">
<a href="https://memgraph.com/docs/memgraph-cloud">
<img width="450px" alt="Memgraph Cloud" src="https://public-assets.memgraph.com/memgraph-gifs%2Fcloud.gif">
</a>
</p>
## :link: Connect to Memgraph
[Connect to the database](https://memgraph.com/docs/memgraph/connect-to-memgraph) using Memgraph Lab, mgconsole, various drivers (Python, C/C++ and others) and WebSocket.
### :microscope: Memgraph Lab
Visualize graphs and play with queries to understand your data. [Memgraph Lab](https://memgraph.com/docs/memgraph-lab) is a user interface that helps you explore and manipulate the data stored in Memgraph. Visualize graphs, execute ad hoc queries, and optimize their performance.
<p align="left">
<a href="https://memgraph.com/docs/memgraph-lab">
<img width="450px" alt="Memgraph Cloud" src="https://public-assets.memgraph.com/memgraph-gifs%2Flab.gif">
</a>
</p>
## :file_folder: Import data
[Import data](https://memgraph.com/docs/memgraph/import-data) into Memgraph using Kafka, RedPanda or Pulsar streams, CSV and JSON files, or Cypher commands.
- Run Python, Rust, and C/C++ code natively, check out the
[MAGE](https://github.com/memgraph/mage) graph algorithm library
- Native support for machine learning
- Streaming support
- Replication
- Authentication and authorization
- ACID compliance
## :bookmark_tabs: Documentation
@@ -171,17 +141,8 @@ Memgraph Community is available under the [BSL
license](./licenses/BSL.txt).</br> Memgraph Enterprise is available under the
[MEL license](./licenses/MEL.txt).
## :busts_in_silhouette: Community
- :purple_heart: [**Discord**](https://discord.gg/memgraph)
- :ocean: [**Stack Overflow**](https://stackoverflow.com/questions/tagged/memgraphdb)
- :busts_in_silhouette: [**Discourse forum**](https://discourse.memgraph.com/)
- :bird: [**Twitter**](https://twitter.com/memgraphdb)
- :movie_camera:
[**YouTube**](https://www.youtube.com/channel/UCZ3HOJvHGxtQ_JHxOselBYg)
<p align="center">
<a href="#">
<img src="https://img.shields.io/badge/⬆️ back_to_top_⬆-white" alt="Back to top" title="Back to top"/>
<img src="https://img.shields.io/badge/⬆back_to_top_⬆-white" alt="Back to top" title="Back to top"/>
</a>
</p>

View File

@@ -5,10 +5,12 @@ import os
import subprocess
import sys
import textwrap
import xml.etree.ElementTree as ET
import yaml
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(SCRIPT_DIR, "flags.yaml")
WIDTH = 80
@@ -16,13 +18,14 @@ WIDTH = 80
def wrap_text(s, initial_indent="# "):
return "\n#\n".join(
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "), s.split("\n"))
)
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent,
subsequent_indent="# "), s.split("\n")))
def extract_flags(binary_path):
ret = {}
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
data = subprocess.run([binary_path, "--help-xml"],
stdout=subprocess.PIPE).stdout.decode("utf-8")
root = ET.fromstring(data)
for child in root:
if child.tag == "usage" and child.text.lower().count("warning"):
@@ -43,7 +46,8 @@ def apply_config_to_flags(config, flags):
for modification in config["modifications"]:
name = modification["name"]
if name not in flags:
print("WARNING: Flag '" + name + "' missing from binary!", file=sys.stderr)
print("WARNING: Flag '" + name + "' missing from binary!",
file=sys.stderr)
continue
flags[name]["default"] = modification["value"]
flags[name]["override"] = modification["override"]
@@ -71,9 +75,8 @@ def extract_sections(flags):
else:
sections.append((current_section, current_flags))
sections.append(("other", other))
assert set(sum(map(lambda x: x[1], sections), [])) == set(
flags.keys()
), "The section extraction algorithm lost some flags!"
assert set(sum(map(lambda x: x[1], sections), [])) == set(flags.keys()), \
"The section extraction algorithm lost some flags!"
return sections
@@ -86,7 +89,8 @@ def generate_config_file(sections, flags):
helpstr = flag["meaning"] + " [" + flag["type"] + "]"
ret += wrap_text(helpstr) + "\n"
prefix = "# " if not flag["override"] else ""
ret += prefix + "--" + flag["name"].replace("_", "-") + "=" + flag["default"] + "\n\n"
ret += prefix + "--" + flag["name"].replace("_", "-") + \
"=" + flag["default"] + "\n\n"
ret += "\n"
ret += wrap_text(config["footer"])
return ret.strip() + "\n"
@@ -94,9 +98,13 @@ def generate_config_file(sections, flags):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("memgraph_binary", help="path to Memgraph binary")
parser.add_argument("output_file", help="path where to store the generated Memgraph " "configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE, help="path to generator configuration file")
parser.add_argument("memgraph_binary",
help="path to Memgraph binary")
parser.add_argument("output_file",
help="path where to store the generated Memgraph "
"configuration file")
parser.add_argument("--config-file", default=CONFIG_FILE,
help="path to generator configuration file")
args = parser.parse_args()
flags = extract_flags(args.memgraph_binary)

View File

@@ -1,204 +0,0 @@
# CSV Import Tool Documentation
CSV is a universal and very versatile data format used to store large quantities
of data. Each Memgraph database instance has a CSV import tool installed called
`mg_import_csv`. The CSV import tool should be used for initial bulk ingestion
of data into the database. Upon ingestion, the CSV importer creates a snapshot
that will be used by the database to recover its state on its next startup.
If you are already familiar with the Neo4j bulk import tool, then using the
`mg_import_csv` tool should be easy. The CSV import tool is fully compatible
with the [Neo4j CSV
format](https://neo4j.com/docs/operations-manual/current/tools/import/). If you
already have a pipeline set-up for Neo4j, you should only replace `neo4j-admin
import` with `mg_import_csv`.
## CSV File Format
Each row of a CSV file represents a single entry that should be imported into
the database. Both nodes and relationships can be imported into the database
using CSV files.
Each set of CSV files must have a header that describes the data that is stored
in the CSV files. Each field in the CSV header is in the format
`<name>[:<type>]` which identifies the name that should be used for that column
and the type that should be used for that column. The type is optional and
defaults to `string` (see the following chapter).
Each CSV field must be divided using the delimiter and each CSV field can either
be quoted or unquoted. When the field is quoted, the first and last character in
the field *must* be the quote character. If the field isn't quoted, and a quote
character appears in it, it is treated as a regular character. If a quote
character appears inside a quoted string then the quote character must be
doubled in order to escape it. Line feeds and carriage returns are ignored in
the CSV file, also, the file can't contain a NULL character.
## Properties
Both nodes and relationships can have properties added to them. When importing
properties, the CSV importer uses the name specified in the header of the
corresponding CSV column for the name of the property. A property is designated
by specifying one of the following types in the header:
- `integer`, `int`, `long`, `byte`, `short`: creates an integer property
- `float`, `double`: creates a float property
- `boolean`, `bool`: creates a boolean property
- `string`, `char`: creates a string property
When importing a boolean value, the CSV field should contain exactly the text
`true` to import a `True` boolean value. All other text values are treated as a
boolean value `False`.
If you want to import an array of values, you can do so by appending `[]` to any
of the above types. The values of the array are then determined by splitting
the raw CSV value using the array delimiter character.
Assuming that the array delimiter is `;`, the following example:
```plaintext
first_name,last_name:string,number:integer,aliases:string[]
John,Doe,1,Johnny;Jo;J-man
Melissa,Doe,2,Mel
```
Will yield these results:
```plaintext
CREATE ({first_name: "John", last_name: "Doe", number: 1, aliases: ["Johnny", "Jo", "J-man"]});
CREATE ({first_name: "Melissa", last_name: "Doe", number: 2, aliases: ["Mel"]});
```
### Nodes
When importing nodes, several more types can be specified in the header of the
CSV file (along with all property types):
- `ID`: id of the node that should be used as the node ID when importing
relationships
- `LABEL`: designates that the field contains additional labels for the node
- `IGNORE`: designates that the field should be ignored
The `ID` field type sets the internal ID that will be used for the node when
creating relationships. It is optional and nodes that don't have an ID value
specified will be imported, but can't be connected to any relationships. If you
want to save the ID value as a property in the database, just specify a name for
the ID (`user_id:ID`). If you just want to use the ID during the import, leave
out the name of the field (`:ID`). The `ID` field also supports creating
separate ID spaces. The ID space is specified with the ID space name appended
to the `ID` type in parentheses (`ID(user)`). That allows you to have the same
IDs (by value) for multiple different node files (for example, numbers from 1 to
N). The IDs in each ID space will be treated as an independent set of IDs that
don't interfere with IDs in another ID space.
The `LABEL` field type adds additional labels to the node. The value is treated
as an array type so that multiple additional labels can be specified for each
node. The value is split using the array delimiter (`--array-delimiter` flag).
### Relationships
In order to be able to import relationships, you must import the nodes in the
same invocation of `mg_import_csv` that is used to import the relationships.
When importing relationships, several more types can be specified in the header
of the CSV file (along with all property types):
- `START_ID`: id of the start node that should be connected with the
relationship
- `END_ID`: id of the end node that should be connected with the relationship
- `TYPE`: designates the type of the relationship
- `IGNORE`: designates that the field should be ignored
The `START_ID` field type sets the start node that should be connected with the
relationship to the end node. The field *must* be specified and the node ID
must be one of the node IDs that were specified in the node CSV files. The name
of this field is ignored. If the node ID is in an ID space, you can specify the
ID space for the in the same way as for the node ID (`START_ID(user)`).
The `END_ID` field type sets the end node that should be connected with the
relationship to the start node. The field *must* be specified and the node ID
must be one of the node IDs that were specified in the node CSV files. The name
of this field is ignored. If the node ID is in an ID space, you can specify the
ID space for the in the same way as for the node ID (`END_ID(user)`).
The `TYPE` field type sets the type of the relationship. Each relationship
*must* have a relationship type, but it doesn't necessarily need to be specified
in the CSV file, it can also be set externally for the whole CSV file. The name
of this field is ignored.
## CSV Importer Flags
The importer has many command line options that allow you to customize the way
the importer loads your data.
The two main flags that are used to specify the input CSV files are `--nodes`
and `--relationships`. Basic description of these flags is provided in the table
and more detailed explainion can be found further down bellow.
| Flag | Description |
|-----------------------| -------------- |
|`--nodes` | Used to specify CSV files that contain the nodes to the importer. |
|`--relationships` | Used to specify CSV files that contain the relationships to the importer.|
|`--delimiter` | Sets the delimiter that should be used when splitting the CSV fields (default `,`)|
|`--quote` | Sets the quote character that should be used to quote a CSV field (default `"`)|
|`--array-delimiter` | Sets the delimiter that should be used when splitting array values (default `;`)|
|`--id-type` | Specifies which data type should be used to store the supplied <br /> node IDs when storing them as properties (if the field name is supplied). <br /> The supported values are either `STRING` or `INTEGER`. (default `STRING`)|
|`--ignore-empty-strings` | Instructs the importer to treat all empty strings as `Null` values <br /> instead of an empty string value (default `false`)|
|`--ignore-extra-columns` | Instructs the importer to ignore all columns (instead of raising an error) <br /> that aren't specified after the last specified column in the CSV header. (default `false`) |
| `--skip-bad-relationships`| Instructs the importer to ignore all relationships (instead of raising an error) <br /> that refer to nodes that don't exist in the node files. (default `false`) |
|`--skip-duplicate-nodes` | Instructs the importer to ignore all duplicate nodes (instead of raising an error). <br /> Duplicate nodes are nodes that have an ID that is the same as another node that was already imported. (default `false`) |
| `--trim-strings`| Instructs the importer to trim all of the loaded CSV field values before processing them further. <br /> Trimming the fields removes all leading and trailing whitespace from them. (default `false`) |
The `--nodes` and `--relationships` flags are used to specify CSV files that
contain the nodes and relationships to the importer. Multiple files can be
specified in each supplied `--nodes` or `--relationships` flag. Files that are
supplied in one `--nodes` or `--relationships` flag are treated by the CSV
parser as one big CSV file. Only the first line of the first file is parsed for
the CSV header, all other files (and rows) are treated as data. This is useful
when you have a very large CSV file and don't want to edit its first line just
to add a CSV header. Instead, you can specify the header in a separate file
(e.g. `users_header.csv` or `friendships_header.csv`) and have the data intact
in the large file (e.g. `users.csv` or `friendships.csv`). Also, you can supply
additional labels for each set of node files.
The format of `--nodes` flag is:
`[<label>[:<label>]...=]<file>[,<file>][,<file>]...`. Take note that only the
first `<file>` part is mandatory, all other parts of the flag value are
optional. Multiple `--nodes` flags can be supplied to describe multiple sets of
different node files. For the importer to work, at least one `--nodes` flag
*must* be supplied.
The format of `--relationships` flag is: `[<type>=]<file>[,<file>][,<file>]...`.
Take note that only the first `<file>` part is mandatory, all other parts of the
flag value are optional. Multiple `--relationships` flags can be supplied to
describe multiple sets of different relationship files. The `--relationships`
flag isn't mandatory.
## CSV Parser Logic
The CSV parser uses the same logic as the standard Python CSV parser. The data
is parsed in the same way as the following snippet:
```python
import csv
for row in csv.reader(stream, strict=True):
# process 'row'
```
Python uses 'excel' as the default dialect when parsing CSV files and the
default settings for the CSV parser are:
- delimiter: `','`
- doublequote: `True`
- escapechar: `None`
- lineterminator: `'\r\n'`
- quotechar: `'"'`
- skipinitialspace: `False`
The above snippet can be expanded to:
```python
import csv
for row in csv.reader(stream, delimiter=',', doublequote=True,
escapechar=None, lineterminator='\r\n',
quotechar='"', skipinitialspace=False,
strict=True):
# process 'row'
```
For more information about the meaning of the above values, see:
https://docs.python.org/3/library/csv.html#csv.Dialect

View File

@@ -1,156 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "amzn-2"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
patch
libipt libipt-devel # intel
perl # for openssl
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip nmap-ncat # for tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""}
LD_LIBRARY_PATH=""
for pkg in $1; do
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
for pkg in $1; do
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libipt-devel ]; then
if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == nodejs ]; then
curl -sL https://rpm.nodesource.com/setup_16.x | bash -
if ! yum list installed nodejs >/dev/null 2>/dev/null; then
yum install -y nodejs
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
if [ "$pkg" == nodejs ]; then
curl -sL https://rpm.nodesource.com/setup_16.x | bash -
if ! yum list installed nodejs >/dev/null 2>/dev/null; then
yum install -y nodejs
fi
continue
fi
if [ "$pkg" == java-11-openjdk ]; then
amazon-linux-extras install -y java-openjdk11
continue
fi
if [ "$pkg" == java-11-openjdk-devel ]; then
amazon-linux-extras install -y java-openjdk11
yum install -y java-11-openjdk-devel
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -6,7 +6,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "arm64" "aarch64"
check_architecture "arm64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools

View File

@@ -1,103 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "fedora-36"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
perl # for openssl
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1,96 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-22.04"
check_architecture "arm64" "aarch64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 libbabeltrace1 liblzma5 python3 # for gdb
libcurl4 # for cmake
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
netcat # tests are using nc to wait for memgraph
python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-6.0 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cd "$DIR"
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb
apt-get update
apt-get install -y apt-transport-https dotnet-sdk-6.0
fi
continue
fi
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1 +0,0 @@
*.tar.gz

View File

@@ -10,14 +10,6 @@ cd "$DIR"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
function log_tool_name () {
echo ""
echo ""
echo "#### $1 ####"
echo ""
echo ""
}
for_arm=false
if [[ "$#" -eq 1 ]]; then
if [[ "$1" == "--for-arm" ]]; then
@@ -28,11 +20,9 @@ if [[ "$#" -eq 1 ]]; then
fi
fi
TOOLCHAIN_STDCXX="${TOOLCHAIN_STDCXX:-libstdc++}"
if [[ "$TOOLCHAIN_STDCXX" != "libstdc++" && "$TOOLCHAIN_STDCXX" != "libc++" ]]; then
echo "Only GCC (libstdc++) or LLVM (libc++) C++ standard library implementations are supported."
exit 1
fi
os="$1"
# toolchain version
TOOLCHAIN_VERSION=4
# package versions used
@@ -51,19 +41,11 @@ CPPCHECK_VERSION=2.6
LLVM_VERSION=13.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Set the right env script
ENV_SCRIPT="$DIR/../os/$DISTRO.sh"
if [[ "$for_arm" = true ]]; then
ENV_SCRIPT="$DIR/../os/$DISTRO-arm.sh"
fi
# Check for the toolchain build dependencies.
echo "ALL BUILD PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_BUILD_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_BUILD_DEPS
# Check for the toolchain run dependencies.
echo "ALL RUN PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_RUN_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_RUN_DEPS
# Check for the dependencies.
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
# check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -117,8 +99,6 @@ if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxx-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxxabi-$LLVM_VERSION.src.tar.xz
fi
if [ ! -f pahole-gdb-master.zip ]; then
wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
@@ -176,8 +156,6 @@ if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxx-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxxabi-$LLVM_VERSION.src.tar.xz.sig
fi
# list of valid llvm gnupg keys: https://releases.llvm.org/download.html
$GPG --keyserver $KEYSERVER --recv-keys 0x474E22316ABF4785A88C6E8EA2C794A986419D8A
@@ -187,8 +165,6 @@ $GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
$GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
$GPG --verify libunwind-$LLVM_VERSION.src.tar.xz.sig libunwind-$LLVM_VERSION.src.tar.xz
$GPG --verify libcxx-$LLVM_VERSION.src.tar.xz.sig libcxx-$LLVM_VERSION.src.tar.xz
$GPG --verify libcxxabi-$LLVM_VERSION.src.tar.xz.sig libcxxabi-$LLVM_VERSION.src.tar.xz
popd
@@ -196,7 +172,7 @@ popd
mkdir -p build
pushd build
log_tool_name "GCC $GCC_VERSION"
# compile gcc
if [ ! -f $PREFIX/bin/gcc ]; then
if [ -d gcc-$GCC_VERSION ]; then
rm -rf gcc-$GCC_VERSION
@@ -287,7 +263,7 @@ fi
export PATH=$PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$PREFIX/lib64
log_tool_name "binutils $BINUTILS_VERSION"
# compile binutils
if [ ! -f $PREFIX/bin/ld.gold ]; then
if [ -d binutils-$BINUTILS_VERSION ]; then
rm -rf binutils-$BINUTILS_VERSION
@@ -351,7 +327,7 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
popd && popd
fi
log_tool_name "GDB $GDB_VERSION"
# compile gdb
if [ ! -f $PREFIX/bin/gdb ]; then
if [ -d gdb-$GDB_VERSION ]; then
rm -rf gdb-$GDB_VERSION
@@ -387,62 +363,6 @@ if [ ! -f $PREFIX/bin/gdb ]; then
--without-babeltrace \
--enable-tui \
--with-python=python3
elif [[ "${DISTRO}" == fedora* ]]; then
# Remove readline, gdb does not compile
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
elif [[ "${DISTRO}" == "amzn-2" ]]; then
# Remove readline, gdb does not compile
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
else
# https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
env \
@@ -478,13 +398,13 @@ if [ ! -f $PREFIX/bin/gdb ]; then
popd && popd
fi
log_tool_name "install pahole"
# install pahole
if [ ! -d $PREFIX/share/pahole-gdb ]; then
unzip ../archives/pahole-gdb-master.zip
mv pahole-gdb-master $PREFIX/share/pahole-gdb
fi
log_tool_name "setup system gdbinit"
# setup system gdbinit
if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
mkdir -p $PREFIX/etc/gdb
cat >$PREFIX/etc/gdb/gdbinit <<EOF
@@ -510,7 +430,7 @@ end
EOF
fi
log_tool_name "cmake $CMAKE_VERSION"
# compile cmake
if [ ! -f $PREFIX/bin/cmake ]; then
if [ -d cmake-$CMAKE_VERSION ]; then
rm -rf cmake-$CMAKE_VERSION
@@ -536,7 +456,7 @@ if [ ! -f $PREFIX/bin/cmake ]; then
popd && popd
fi
log_tool_name "cppcheck $CPPCHECK_VERSION"
# compile cppcheck
if [ ! -f $PREFIX/bin/cppcheck ]; then
if [ -d cppcheck-$CPPCHECK_VERSION ]; then
rm -rf cppcheck-$CPPCHECK_VERSION
@@ -560,7 +480,7 @@ if [ ! -f $PREFIX/bin/cppcheck ]; then
popd
fi
log_tool_name "swig $SWIG_VERSION"
# compile swig
if [ ! -d swig-$SWIG_VERSION/install ]; then
if [ -d swig-$SWIG_VERSION ]; then
rm -rf swig-$SWIG_VERSION
@@ -576,7 +496,7 @@ if [ ! -d swig-$SWIG_VERSION/install ]; then
popd && popd
fi
log_tool_name "LLVM $LLVM_VERSION"
# compile llvm
if [ ! -f $PREFIX/bin/clang ]; then
if [ -d llvm-$LLVM_VERSION ]; then
rm -rf llvm-$LLVM_VERSION
@@ -593,19 +513,8 @@ if [ ! -f $PREFIX/bin/clang ]; then
mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz
mv libunwind-$LLVM_VERSION.src/include/mach-o llvm-$LLVM_VERSION/tools/lld/include
# The following is required because of libc++
tar -xvf ../archives/libcxx-$LLVM_VERSION.src.tar.xz
mv libcxx-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libcxx
tar -xvf ../archives/libcxxabi-$LLVM_VERSION.src.tar.xz
mv libcxxabi-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libcxxabi
# NOTE: We moved part of the libunwind in one of the previous step.
rm -r libunwind-$LLVM_VERSION.src
tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz
mv libunwind-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libunwind
pushd llvm-$LLVM_VERSION
mkdir -p build && pushd build
mkdir build && pushd build
# activate swig
export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
@@ -658,7 +567,7 @@ In order to be able to run all of these tools you should install the following
packages:
\`\`\`
$($DIR/../os/$ENV_SCRIPT.sh list TOOLCHAIN_RUN_DEPS)
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
\`\`\`
## Usage
@@ -715,7 +624,6 @@ export PS1="($NAME) \$PS1"
export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64
export CXXFLAGS=-isystem\ $PREFIX/include\ \$CXXFLAGS
export CFLAGS=-isystem\ $PREFIX/include\ \$CFLAGS
export VENV=$PREFIX
# disable root
function su () {
@@ -767,7 +675,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
SNAPPY_VERSION=1.1.9
XZ_VERSION=5.2.5 # for LZMA
ZLIB_VERSION=1.2.13
ZLIB_VERSION=1.2.12
ZSTD_VERSION=1.5.0
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
@@ -912,11 +820,7 @@ source $PREFIX/activate
export CC=$PREFIX/bin/clang
export CXX=$PREFIX/bin/clang++
export CFLAGS="$CFLAGS -fPIC"
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
export CXXFLAGS="$CXXFLAGS -fPIC"
else
export CXXFLAGS="$CXXFLAGS -fPIC -stdlib=libc++"
fi
export CXXFLAGS="$CXXFLAGS -fPIC"
COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
-DCMAKE_PREFIX_PATH=$PREFIX
-DCMAKE_BUILD_TYPE=Release
@@ -930,7 +834,7 @@ COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
COMMON_CONFIGURE_FLAGS="--enable-shared=no --prefix=$PREFIX"
COMMON_MAKE_INSTALL_FLAGS="-j$CPUS BUILD_SHARED=no PREFIX=$PREFIX install"
log_tool_name "bzip2 $BZIP2_VERSION"
# install bzip2
if [ ! -f $PREFIX/include/bzlib.h ]; then
if [ -d bzip2-$BZIP2_VERSION ]; then
rm -rf bzip2-$BZIP2_VERSION
@@ -941,7 +845,7 @@ if [ ! -f $PREFIX/include/bzlib.h ]; then
popd
fi
log_tool_name "fmt $FMT_VERSION"
# install fmt
if [ ! -d $PREFIX/include/fmt ]; then
if [ -d fmt-$FMT_VERSION ]; then
rm -rf fmt-$FMT_VERSION
@@ -954,7 +858,7 @@ if [ ! -d $PREFIX/include/fmt ]; then
popd && popd
fi
log_tool_name "lz4 $LZ4_VERSION"
# install lz4
if [ ! -f $PREFIX/include/lz4.h ]; then
if [ -d lz4-$LZ4_VERSION ]; then
rm -rf lz4-$LZ4_VERSION
@@ -965,7 +869,7 @@ if [ ! -f $PREFIX/include/lz4.h ]; then
popd
fi
log_tool_name "xz $XZ_VERSION"
# install xz
if [ ! -f $PREFIX/include/lzma.h ]; then
if [ -d xz-$XZ_VERSION ]; then
rm -rf xz-$XZ_VERSION
@@ -977,7 +881,7 @@ if [ ! -f $PREFIX/include/lzma.h ]; then
popd
fi
log_tool_name "zlib $ZLIB_VERSION"
# install zlib
if [ ! -f $PREFIX/include/zlib.h ]; then
if [ -d zlib-$ZLIB_VERSION ]; then
rm -rf zlib-$ZLIB_VERSION
@@ -991,7 +895,7 @@ if [ ! -f $PREFIX/include/zlib.h ]; then
popd && popd
fi
log_tool_name "zstd $ZSTD_VERSION"
# install zstd
if [ ! -f $PREFIX/include/zstd.h ]; then
if [ -d zstd-$ZSTD_VERSION ]; then
rm -rf zstd-$ZSTD_VERSION
@@ -1006,8 +910,7 @@ if [ ! -f $PREFIX/include/zstd.h ]; then
popd && popd
fi
# TODO(gitbuda): Freeze jmalloc version.
log_tool_name "jmalloc"
#install jemalloc
if [ ! -d $PREFIX/include/jemalloc ]; then
if [ -d jemalloc ]; then
rm -rf jemalloc
@@ -1024,7 +927,7 @@ if [ ! -d $PREFIX/include/jemalloc ]; then
popd
fi
log_tool_name "BOOST $BOOST_VERSION"
# install boost
if [ ! -d $PREFIX/include/boost ]; then
if [ -d boost_$BOOST_VERSION_UNDERSCORES ]; then
rm -rf boost_$BOOST_VERSION_UNDERSCORES
@@ -1032,24 +935,15 @@ if [ ! -d $PREFIX/include/boost ]; then
tar -xzf ../archives/boost_$BOOST_VERSION_UNDERSCORES.tar.gz
pushd boost_$BOOST_VERSION_UNDERSCORES
./bootstrap.sh --prefix=$PREFIX --with-toolset=clang --with-python=python3 --without-icu
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
else
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
fi
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
popd
fi
log_tool_name "double-conversion $DOUBLE_CONVERSION_VERSION"
# install double-conversion
if [ ! -d $PREFIX/include/double-conversion ]; then
if [ -d double-conversion-$DOUBLE_CONVERSION_VERSION ]; then
rm -rf double-conversion-$DOUBLE_CONVERSION_VERSION
@@ -1064,8 +958,7 @@ if [ ! -d $PREFIX/include/double-conversion ]; then
popd && popd
fi
# TODO(gitbuda): Freeze gflags version.
log_tool_name "gflags"
# install gflags
if [ ! -d $PREFIX/include/gflags ]; then
if [ -d gflags ]; then
rm -rf gflags
@@ -1084,7 +977,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
popd && popd
fi
log_tool_name "libunwind $LIBUNWIND_VERSION"
# install libunwind
if [ ! -f $PREFIX/include/libunwind.h ]; then
if [ -d libunwind-$LIBUNWIND_VERSION ]; then
rm -rf libunwind-$LIBUNWIND_VERSION
@@ -1097,7 +990,7 @@ if [ ! -f $PREFIX/include/libunwind.h ]; then
popd
fi
log_tool_name "glog $GLOG_VERSION"
# install glog
if [ ! -d $PREFIX/include/glog ]; then
if [ -d glog-$GLOG_VERSION ]; then
rm -rf glog-$GLOG_VERSION
@@ -1111,7 +1004,7 @@ if [ ! -d $PREFIX/include/glog ]; then
popd && popd
fi
log_tool_name "libevent $LIBEVENT_VERSION"
# install libevent
if [ ! -d $PREFIX/include/event2 ]; then
if [ -d libevent-$LIBEVENT_VERSION ]; then
rm -rf libevent-$LIBEVENT_VERSION
@@ -1130,7 +1023,7 @@ if [ ! -d $PREFIX/include/event2 ]; then
popd && popd
fi
log_tool_name "snappy $SNAPPY_VERSION"
# install snappy
if [ ! -f $PREFIX/include/snappy.h ]; then
if [ -d snappy-$SNAPPY_VERSION ]; then
rm -rf snappy-$SNAPPY_VERSION
@@ -1148,7 +1041,7 @@ if [ ! -f $PREFIX/include/snappy.h ]; then
popd && popd
fi
log_tool_name "libsodium $LIBSODIUM_VERSION"
# install libsodium
if [ ! -f $PREFIX/include/sodium.h ]; then
if [ -d libsodium-$LIBSODIUM_VERSION ]; then
rm -rf libsodium-$LIBSODIUM_VERSION
@@ -1160,7 +1053,7 @@ if [ ! -f $PREFIX/include/sodium.h ]; then
popd
fi
log_tool_name "libaio $LIBAIO_VERSION"
# install libaio
if [ ! -f $PREFIX/include/libaio.h ]; then
if [ -d libaio-$LIBAIO_VERSION ]; then
rm -rf libaio-$LIBAIO_VERSION
@@ -1171,121 +1064,114 @@ if [ ! -f $PREFIX/include/libaio.h ]; then
popd
fi
if [[ "${DISTRO}" != "amzn-2" ]]; then
log_tool_name "folly $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/folly ]; then
if [ -d folly-$FBLIBS_VERSION ]; then
rm -rf folly-$FBLIBS_VERSION
fi
mkdir folly-$FBLIBS_VERSION
tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
pushd folly-$FBLIBS_VERSION
patch -p1 < ../../folly.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBOOST_LINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCXX_STD="c++20"
make -j$CPUS install
popd && popd
# install folly
if [ ! -d $PREFIX/include/folly ]; then
if [ -d folly-$FBLIBS_VERSION ]; then
rm -rf folly-$FBLIBS_VERSION
fi
mkdir folly-$FBLIBS_VERSION
tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
pushd folly-$FBLIBS_VERSION
patch -p1 < ../../folly.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBOOST_LINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCXX_STD="c++20"
make -j$CPUS install
popd && popd
fi
log_tool_name "fizz $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/fizz ]; then
if [ -d fizz-$FBLIBS_VERSION ]; then
rm -rf fizz-$FBLIBS_VERSION
fi
mkdir fizz-$FBLIBS_VERSION
tar -xzf ../archives/fizz-$FBLIBS_VERSION.tar.gz -C fizz-$FBLIBS_VERSION
pushd fizz-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../fizz $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
# install fizz
if [ ! -d $PREFIX/include/fizz ]; then
if [ -d fizz-$FBLIBS_VERSION ]; then
rm -rf fizz-$FBLIBS_VERSION
fi
mkdir fizz-$FBLIBS_VERSION
tar -xzf ../archives/fizz-$FBLIBS_VERSION.tar.gz -C fizz-$FBLIBS_VERSION
pushd fizz-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../fizz $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
fi
log_tool_name "wangle FBLIBS_VERSION"
if [ ! -d $PREFIX/include/wangle ]; then
if [ -d wangle-$FBLIBS_VERSION ]; then
rm -rf wangle-$FBLIBS_VERSION
fi
mkdir wangle-$FBLIBS_VERSION
tar -xzf ../archives/wangle-$FBLIBS_VERSION.tar.gz -C wangle-$FBLIBS_VERSION
pushd wangle-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../wangle $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
# install wangle
if [ ! -d $PREFIX/include/wangle ]; then
if [ -d wangle-$FBLIBS_VERSION ]; then
rm -rf wangle-$FBLIBS_VERSION
fi
mkdir wangle-$FBLIBS_VERSION
tar -xzf ../archives/wangle-$FBLIBS_VERSION.tar.gz -C wangle-$FBLIBS_VERSION
pushd wangle-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../wangle $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
fi
log_tool_name "proxygen $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/proxygen ]; then
if [ -d proxygen-$FBLIBS_VERSION ]; then
rm -rf proxygen-$FBLIBS_VERSION
fi
mkdir proxygen-$FBLIBS_VERSION
tar -xzf ../archives/proxygen-$FBLIBS_VERSION.tar.gz -C proxygen-$FBLIBS_VERSION
pushd proxygen-$FBLIBS_VERSION
patch -p1 < ../../proxygen.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_SAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DBUILD_QUIC=OFF
make -j$CPUS install
popd && popd
# install proxygen
if [ ! -d $PREFIX/include/proxygen ]; then
if [ -d proxygen-$FBLIBS_VERSION ]; then
rm -rf proxygen-$FBLIBS_VERSION
fi
mkdir proxygen-$FBLIBS_VERSION
tar -xzf ../archives/proxygen-$FBLIBS_VERSION.tar.gz -C proxygen-$FBLIBS_VERSION
pushd proxygen-$FBLIBS_VERSION
patch -p1 < ../../proxygen.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_SAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DBUILD_QUIC=OFF
make -j$CPUS install
popd && popd
fi
log_tool_name "flex $FBLIBS_VERSION"
if [ ! -f $PREFIX/include/FlexLexer.h ]; then
if [ -d flex-$FLEX_VERSION ]; then
rm -rf flex-$FLEX_VERSION
fi
tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
pushd flex-$FLEX_VERSION
./configure $COMMON_CONFIGURE_FLAGS
make -j$CPUS install
popd
# install flex
if [ ! -f $PREFIX/include/FlexLexer.h ]; then
if [ -d flex-$FLEX_VERSION ]; then
rm -rf flex-$FLEX_VERSION
fi
tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
pushd flex-$FLEX_VERSION
./configure $COMMON_CONFIGURE_FLAGS
make -j$CPUS install
popd
fi
log_tool_name "fbthrift $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/thrift ]; then
if [ -d fbthrift-$FBLIBS_VERSION ]; then
rm -rf fbthrift-$FBLIBS_VERSION
fi
git clone --depth 1 --branch v$FBLIBS_VERSION https://github.com/facebook/fbthrift.git fbthrift-$FBLIBS_VERSION
pushd fbthrift-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
CMAKE_CXX_FLAGS="-fsized-deallocation"
else
CMAKE_CXX_FLAGS="-fsized-deallocation -stdlib=libc++"
fi
cmake .. $COMMON_CMAKE_FLAGS \
-Denable_tests=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"
make -j$CPUS install
popd
# install fbthrift
if [ ! -d $PREFIX/include/thrift ]; then
if [ -d fbthrift-$FBLIBS_VERSION ]; then
rm -rf fbthrift-$FBLIBS_VERSION
fi
git clone --depth 1 --branch v$FBLIBS_VERSION https://github.com/facebook/fbthrift.git fbthrift-$FBLIBS_VERSION
pushd fbthrift-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-Denable_tests=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCMAKE_CXX_FLAGS=-fsized-deallocation
make -j$CPUS install
popd
fi
popd
@@ -1293,7 +1179,7 @@ popd
# create toolchain archive
if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
DISTRO_FULL_NAME=${DISTRO}
if [[ "${DISTRO}" == centos* ]] || [[ "${DISTRO}" == fedora* ]]; then
if [[ "${DISTRO}" == centos* ]]; then
if [[ "$for_arm" = "true" ]]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
else
@@ -1306,12 +1192,7 @@ if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
fi
fi
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
# Pass because infra scripts assume there is not C++ standard lib in the name.
echo "NOTE: Not adding anything to the archive name that GCC C++ standard lib is used."
else
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-libc++"
fi
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
fi

View File

@@ -1,11 +1,11 @@
#!/bin/bash
function operating_system() {
operating_system() {
grep -E '^(VERSION_)?ID=' /etc/os-release | \
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
}
function check_operating_system() {
check_operating_system() {
if [ "$(operating_system)" != "$1" ]; then
echo "Not the right operating system!"
exit 1
@@ -14,25 +14,20 @@ function check_operating_system() {
fi
}
function architecture() {
architecture() {
uname -m
}
check_architecture() {
local ARCH=$(architecture)
for arch in "$@"; do
if [ "${ARCH}" = "$arch" ]; then
echo "The right architecture!"
return 0
fi
done
echo "Not the right architecture!"
echo "Expected: $@"
echo "Actual: ${ARCH}"
exit 1
if [ "$(architecture)" != "$1" ]; then
echo "Not the right architecture!"
exit 1
else
echo "The right architecture."
fi
}
function check_all_yum() {
check_all_yum() {
local missing=""
for pkg in $1; do
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
@@ -45,7 +40,7 @@ function check_all_yum() {
fi
}
function check_all_dpkg() {
check_all_dpkg() {
local missing=""
for pkg in $1; do
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
@@ -58,7 +53,7 @@ function check_all_dpkg() {
fi
}
function check_all_dnf() {
check_all_dnf() {
local missing=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
@@ -70,8 +65,7 @@ function check_all_dnf() {
exit 1
fi
}
function install_all_apt() {
install_all_apt() {
for pkg in $1; do
apt install -y "$pkg"
done

View File

@@ -1,734 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file _mgp.hpp
///
/// The file contains methods that connect mg procedures and the outside code
/// Methods like mapping a graph into memory or assigning new mg results or
/// their properties are implemented.
#pragma once
#include "mg_exceptions.hpp"
#include "mg_procedure.h"
namespace mgp {
namespace {
inline void MgExceptionHandle(mgp_error result_code) {
switch (result_code) {
case mgp_error::MGP_ERROR_UNKNOWN_ERROR:
throw mg_exception::UnknownException();
case mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE:
throw mg_exception::AllocationException();
case mgp_error::MGP_ERROR_INSUFFICIENT_BUFFER:
throw mg_exception::InsufficientBufferException();
case mgp_error::MGP_ERROR_OUT_OF_RANGE:
throw mg_exception::OutOfRangeException();
case mgp_error::MGP_ERROR_LOGIC_ERROR:
throw mg_exception::LogicException();
case mgp_error::MGP_ERROR_DELETED_OBJECT:
throw mg_exception::DeletedObjectException();
case mgp_error::MGP_ERROR_INVALID_ARGUMENT:
throw mg_exception::InvalidArgumentException();
case mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS:
throw mg_exception::KeyAlreadyExistsException();
case mgp_error::MGP_ERROR_IMMUTABLE_OBJECT:
throw mg_exception::ImmutableObjectException();
case mgp_error::MGP_ERROR_VALUE_CONVERSION:
throw mg_exception::ValueConversionException();
case mgp_error::MGP_ERROR_SERIALIZATION_ERROR:
throw mg_exception::SerializationException();
default:
return;
}
}
template <typename TResult, typename TFunc, typename... TArgs>
TResult MgInvoke(TFunc func, TArgs... args) {
TResult result{};
auto result_code = func(args..., &result);
MgExceptionHandle(result_code);
return result;
}
template <typename TFunc, typename... TArgs>
inline void MgInvokeVoid(TFunc func, TArgs... args) {
auto result_code = func(args...);
MgExceptionHandle(result_code);
}
} // namespace
// mgp_value
// Make value
inline mgp_value *value_make_null(mgp_memory *memory) { return MgInvoke<mgp_value *>(mgp_value_make_null, memory); }
inline mgp_value *value_make_bool(int val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_bool, val, memory);
}
inline mgp_value *value_make_int(int64_t val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_int, val, memory);
}
inline mgp_value *value_make_double(double val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_double, val, memory);
}
inline mgp_value *value_make_string(const char *val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_string, val, memory);
}
inline mgp_value *value_make_list(mgp_list *val) { return MgInvoke<mgp_value *>(mgp_value_make_list, val); }
inline mgp_value *value_make_map(mgp_map *val) { return MgInvoke<mgp_value *>(mgp_value_make_map, val); }
inline mgp_value *value_make_vertex(mgp_vertex *val) { return MgInvoke<mgp_value *>(mgp_value_make_vertex, val); }
inline mgp_value *value_make_edge(mgp_edge *val) { return MgInvoke<mgp_value *>(mgp_value_make_edge, val); }
inline mgp_value *value_make_path(mgp_path *val) { return MgInvoke<mgp_value *>(mgp_value_make_path, val); }
inline mgp_value *value_make_date(mgp_date *val) { return MgInvoke<mgp_value *>(mgp_value_make_date, val); }
inline mgp_value *value_make_local_time(mgp_local_time *val) {
return MgInvoke<mgp_value *>(mgp_value_make_local_time, val);
}
inline mgp_value *value_make_local_date_time(mgp_local_date_time *val) {
return MgInvoke<mgp_value *>(mgp_value_make_local_date_time, val);
}
inline mgp_value *value_make_duration(mgp_duration *val) { return MgInvoke<mgp_value *>(mgp_value_make_duration, val); }
// Copy value
// TODO: implement within MGP API
// with primitive types ({bool, int, double, string}), create a new identical value
// otherwise call mgp_##TYPE_copy and convert tpye
inline mgp_value *value_copy(mgp_value *val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_copy, val, memory);
}
// Destroy value
inline void value_destroy(mgp_value *val) { mgp_value_destroy(val); }
// Get value of type
inline mgp_value_type value_get_type(mgp_value *val) { return MgInvoke<mgp_value_type>(mgp_value_get_type, val); }
inline bool value_get_bool(mgp_value *val) { return MgInvoke<int>(mgp_value_get_bool, val); }
inline int64_t value_get_int(mgp_value *val) { return MgInvoke<int64_t>(mgp_value_get_int, val); }
inline double value_get_double(mgp_value *val) { return MgInvoke<double>(mgp_value_get_double, val); }
inline const char *value_get_string(mgp_value *val) { return MgInvoke<const char *>(mgp_value_get_string, val); }
inline mgp_list *value_get_list(mgp_value *val) { return MgInvoke<mgp_list *>(mgp_value_get_list, val); }
inline mgp_map *value_get_map(mgp_value *val) { return MgInvoke<mgp_map *>(mgp_value_get_map, val); }
inline mgp_vertex *value_get_vertex(mgp_value *val) { return MgInvoke<mgp_vertex *>(mgp_value_get_vertex, val); }
inline mgp_edge *value_get_edge(mgp_value *val) { return MgInvoke<mgp_edge *>(mgp_value_get_edge, val); }
inline mgp_path *value_get_path(mgp_value *val) { return MgInvoke<mgp_path *>(mgp_value_get_path, val); }
inline mgp_date *value_get_date(mgp_value *val) { return MgInvoke<mgp_date *>(mgp_value_get_date, val); }
inline mgp_local_time *value_get_local_time(mgp_value *val) {
return MgInvoke<mgp_local_time *>(mgp_value_get_local_time, val);
}
inline mgp_local_date_time *value_get_local_date_time(mgp_value *val) {
return MgInvoke<mgp_local_date_time *>(mgp_value_get_local_date_time, val);
}
inline mgp_duration *value_get_duration(mgp_value *val) {
return MgInvoke<mgp_duration *>(mgp_value_get_duration, val);
}
// Check type of value
inline bool value_is_null(mgp_value *val) { return MgInvoke<int>(mgp_value_is_null, val); }
inline bool value_is_bool(mgp_value *val) { return MgInvoke<int>(mgp_value_is_bool, val); }
inline bool value_is_int(mgp_value *val) { return MgInvoke<int>(mgp_value_is_int, val); }
inline bool value_is_double(mgp_value *val) { return MgInvoke<int>(mgp_value_is_double, val); }
inline bool value_is_string(mgp_value *val) { return MgInvoke<int>(mgp_value_is_string, val); }
inline bool value_is_list(mgp_value *val) { return MgInvoke<int>(mgp_value_is_list, val); }
inline bool value_is_map(mgp_value *val) { return MgInvoke<int>(mgp_value_is_map, val); }
inline bool value_is_vertex(mgp_value *val) { return MgInvoke<int>(mgp_value_is_vertex, val); }
inline bool value_is_edge(mgp_value *val) { return MgInvoke<int>(mgp_value_is_edge, val); }
inline bool value_is_path(mgp_value *val) { return MgInvoke<int>(mgp_value_is_path, val); }
inline bool value_is_date(mgp_value *val) { return MgInvoke<int>(mgp_value_is_date, val); }
inline bool value_is_local_time(mgp_value *val) { return MgInvoke<int>(mgp_value_is_local_time, val); }
inline bool value_is_local_date_time(mgp_value *val) { return MgInvoke<int>(mgp_value_is_local_date_time, val); }
inline bool value_is_duration(mgp_value *val) { return MgInvoke<int>(mgp_value_is_duration, val); }
// Get type
inline mgp_type *type_any() { return MgInvoke<mgp_type *>(mgp_type_any); }
inline mgp_type *type_bool() { return MgInvoke<mgp_type *>(mgp_type_bool); }
inline mgp_type *type_string() { return MgInvoke<mgp_type *>(mgp_type_string); }
inline mgp_type *type_int() { return MgInvoke<mgp_type *>(mgp_type_int); }
inline mgp_type *type_float() { return MgInvoke<mgp_type *>(mgp_type_float); }
inline mgp_type *type_number() { return MgInvoke<mgp_type *>(mgp_type_number); }
inline mgp_type *type_list(mgp_type *element_type) { return MgInvoke<mgp_type *>(mgp_type_list, element_type); }
inline mgp_type *type_map() { return MgInvoke<mgp_type *>(mgp_type_map); }
inline mgp_type *type_node() { return MgInvoke<mgp_type *>(mgp_type_node); }
inline mgp_type *type_relationship() { return MgInvoke<mgp_type *>(mgp_type_relationship); }
inline mgp_type *type_path() { return MgInvoke<mgp_type *>(mgp_type_path); }
inline mgp_type *type_date() { return MgInvoke<mgp_type *>(mgp_type_date); }
inline mgp_type *type_local_time() { return MgInvoke<mgp_type *>(mgp_type_local_time); }
inline mgp_type *type_local_date_time() { return MgInvoke<mgp_type *>(mgp_type_local_date_time); }
inline mgp_type *type_duration() { return MgInvoke<mgp_type *>(mgp_type_duration); }
inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke<mgp_type *>(mgp_type_nullable, type); }
// mgp_graph
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_graph_create_vertex, graph, memory);
}
inline void graph_delete_vertex(mgp_graph *graph, mgp_vertex *vertex) {
MgInvokeVoid(mgp_graph_delete_vertex, graph, vertex);
}
inline void graph_detach_delete_vertex(mgp_graph *graph, mgp_vertex *vertex) {
MgInvokeVoid(mgp_graph_detach_delete_vertex, graph, vertex);
}
inline mgp_edge *graph_create_edge(mgp_graph *graph, mgp_vertex *from, mgp_vertex *to, mgp_edge_type type,
mgp_memory *memory) {
return MgInvoke<mgp_edge *>(mgp_graph_create_edge, graph, from, to, type, memory);
}
inline void graph_delete_edge(mgp_graph *graph, mgp_edge *edge) { MgInvokeVoid(mgp_graph_delete_edge, graph, edge); }
inline mgp_vertex *graph_get_vertex_by_id(mgp_graph *g, mgp_vertex_id id, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_graph_get_vertex_by_id, g, id, memory);
}
inline mgp_vertices_iterator *graph_iter_vertices(mgp_graph *g, mgp_memory *memory) {
return MgInvoke<mgp_vertices_iterator *>(mgp_graph_iter_vertices, g, memory);
}
// mgp_vertices_iterator
inline void vertices_iterator_destroy(mgp_vertices_iterator *it) { mgp_vertices_iterator_destroy(it); }
inline mgp_vertex *vertices_iterator_get(mgp_vertices_iterator *it) {
return MgInvoke<mgp_vertex *>(mgp_vertices_iterator_get, it);
}
inline mgp_vertex *vertices_iterator_next(mgp_vertices_iterator *it) {
return MgInvoke<mgp_vertex *>(mgp_vertices_iterator_next, it);
}
// mgp_edges_iterator
inline void edges_iterator_destroy(mgp_edges_iterator *it) { mgp_edges_iterator_destroy(it); }
inline mgp_edge *edges_iterator_get(mgp_edges_iterator *it) { return MgInvoke<mgp_edge *>(mgp_edges_iterator_get, it); }
inline mgp_edge *edges_iterator_next(mgp_edges_iterator *it) {
return MgInvoke<mgp_edge *>(mgp_edges_iterator_next, it);
}
// mgp_properties_iterator
inline void properties_iterator_destroy(mgp_properties_iterator *it) { mgp_properties_iterator_destroy(it); }
inline mgp_property *properties_iterator_get(mgp_properties_iterator *it) {
return MgInvoke<mgp_property *>(mgp_properties_iterator_get, it);
}
inline mgp_property *properties_iterator_next(mgp_properties_iterator *it) {
return MgInvoke<mgp_property *>(mgp_properties_iterator_next, it);
}
// Container {mgp_list, mgp_map} methods
// mgp_list
inline mgp_list *list_make_empty(size_t capacity, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_make_empty, capacity, memory);
}
inline mgp_list *list_copy(mgp_list *list, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_copy, list, memory);
}
inline void list_destroy(mgp_list *list) { mgp_list_destroy(list); }
inline void list_append(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append, list, val); }
inline void list_append_extend(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append_extend, list, val); }
inline size_t list_size(mgp_list *list) { return MgInvoke<size_t>(mgp_list_size, list); }
inline size_t list_capacity(mgp_list *list) { return MgInvoke<size_t>(mgp_list_capacity, list); }
inline mgp_value *list_at(mgp_list *list, size_t index) { return MgInvoke<mgp_value *>(mgp_list_at, list, index); }
// mgp_map
inline mgp_map *map_make_empty(mgp_memory *memory) { return MgInvoke<mgp_map *>(mgp_map_make_empty, memory); }
inline mgp_map *map_copy(mgp_map *map, mgp_memory *memory) { return MgInvoke<mgp_map *>(mgp_map_copy, map, memory); }
inline void map_destroy(mgp_map *map) { mgp_map_destroy(map); }
inline void map_insert(mgp_map *map, const char *key, mgp_value *value) {
MgInvokeVoid(mgp_map_insert, map, key, value);
}
inline size_t map_size(mgp_map *map) { return MgInvoke<size_t>(mgp_map_size, map); }
inline mgp_value *map_at(mgp_map *map, const char *key) { return MgInvoke<mgp_value *>(mgp_map_at, map, key); }
inline const char *map_item_key(mgp_map_item *item) { return MgInvoke<const char *>(mgp_map_item_key, item); }
inline mgp_value *map_item_value(mgp_map_item *item) { return MgInvoke<mgp_value *>(mgp_map_item_value, item); }
inline mgp_map_items_iterator *map_iter_items(mgp_map *map, mgp_memory *memory) {
return MgInvoke<mgp_map_items_iterator *>(mgp_map_iter_items, map, memory);
}
inline void map_items_iterator_destroy(mgp_map_items_iterator *it) { mgp_map_items_iterator_destroy(it); }
inline mgp_map_item *map_items_iterator_get(mgp_map_items_iterator *it) {
return MgInvoke<mgp_map_item *>(mgp_map_items_iterator_get, it);
}
inline mgp_map_item *map_items_iterator_next(mgp_map_items_iterator *it) {
return MgInvoke<mgp_map_item *>(mgp_map_items_iterator_next, it);
}
// mgp_vertex
inline mgp_vertex_id vertex_get_id(mgp_vertex *v) { return MgInvoke<mgp_vertex_id>(mgp_vertex_get_id, v); }
inline mgp_vertex *vertex_copy(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_vertex_copy, v, memory);
}
inline void vertex_destroy(mgp_vertex *v) { mgp_vertex_destroy(v); }
inline bool vertex_equal(mgp_vertex *v1, mgp_vertex *v2) { return MgInvoke<int>(mgp_vertex_equal, v1, v2); }
inline size_t vertex_labels_count(mgp_vertex *v) { return MgInvoke<size_t>(mgp_vertex_labels_count, v); }
inline mgp_label vertex_label_at(mgp_vertex *v, size_t index) {
return MgInvoke<mgp_label>(mgp_vertex_label_at, v, index);
}
inline bool vertex_has_label(mgp_vertex *v, mgp_label label) { return MgInvoke<int>(mgp_vertex_has_label, v, label); }
inline bool vertex_has_label_named(mgp_vertex *v, const char *label_name) {
return MgInvoke<int>(mgp_vertex_has_label_named, v, label_name);
}
inline void vertex_add_label(mgp_vertex *vertex, mgp_label label) { MgInvokeVoid(mgp_vertex_add_label, vertex, label); }
inline mgp_value *vertex_get_property(mgp_vertex *v, const char *property_name, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_vertex_get_property, v, property_name, memory);
}
inline void vertex_set_property(mgp_vertex *v, const char *property_name, mgp_value *property_value) {
MgInvokeVoid(mgp_vertex_set_property, v, property_name, property_value);
}
inline mgp_properties_iterator *vertex_iter_properties(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_properties_iterator *>(mgp_vertex_iter_properties, v, memory);
}
inline mgp_edges_iterator *vertex_iter_in_edges(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_edges_iterator *>(mgp_vertex_iter_in_edges, v, memory);
}
inline mgp_edges_iterator *vertex_iter_out_edges(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_edges_iterator *>(mgp_vertex_iter_out_edges, v, memory);
}
// mgp_edge
inline mgp_edge_id edge_get_id(mgp_edge *e) { return MgInvoke<mgp_edge_id>(mgp_edge_get_id, e); }
inline mgp_edge *edge_copy(mgp_edge *e, mgp_memory *memory) { return MgInvoke<mgp_edge *>(mgp_edge_copy, e, memory); }
inline void edge_destroy(mgp_edge *e) { mgp_edge_destroy(e); }
inline bool edge_equal(mgp_edge *e1, mgp_edge *e2) { return MgInvoke<int>(mgp_edge_equal, e1, e2); }
inline mgp_edge_type edge_get_type(mgp_edge *e) { return MgInvoke<mgp_edge_type>(mgp_edge_get_type, e); }
inline mgp_vertex *edge_get_from(mgp_edge *e) { return MgInvoke<mgp_vertex *>(mgp_edge_get_from, e); }
inline mgp_vertex *edge_get_to(mgp_edge *e) { return MgInvoke<mgp_vertex *>(mgp_edge_get_to, e); }
inline mgp_value *edge_get_property(mgp_edge *e, const char *property_name, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_edge_get_property, e, property_name, memory);
}
inline void edge_set_property(mgp_edge *e, const char *property_name, mgp_value *property_value) {
MgInvokeVoid(mgp_edge_set_property, e, property_name, property_value);
}
inline mgp_properties_iterator *edge_iter_properties(mgp_edge *e, mgp_memory *memory) {
return MgInvoke<mgp_properties_iterator *>(mgp_edge_iter_properties, e, memory);
}
// mgp_path
inline mgp_path *path_make_with_start(mgp_vertex *vertex, mgp_memory *memory) {
return MgInvoke<mgp_path *>(mgp_path_make_with_start, vertex, memory);
}
inline mgp_path *path_copy(mgp_path *path, mgp_memory *memory) {
return MgInvoke<mgp_path *>(mgp_path_copy, path, memory);
}
inline void path_destroy(mgp_path *path) { mgp_path_destroy(path); }
inline void path_expand(mgp_path *path, mgp_edge *edge) { MgInvokeVoid(mgp_path_expand, path, edge); }
inline size_t path_size(mgp_path *path) { return MgInvoke<size_t>(mgp_path_size, path); }
inline mgp_vertex *path_vertex_at(mgp_path *path, size_t index) {
return MgInvoke<mgp_vertex *>(mgp_path_vertex_at, path, index);
}
inline mgp_edge *path_edge_at(mgp_path *path, size_t index) {
return MgInvoke<mgp_edge *>(mgp_path_edge_at, path, index);
}
inline bool path_equal(mgp_path *p1, mgp_path *p2) { return MgInvoke<int>(mgp_path_equal, p1, p2); }
// Temporal type {mgp_date, mgp_local_time, mgp_local_date_time, mgp_duration} methods
// mgp_date
inline mgp_date *date_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_from_string, string, memory);
}
inline mgp_date *date_from_parameters(mgp_date_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_from_parameters, parameters, memory);
}
inline mgp_date *date_copy(mgp_date *date, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_copy, date, memory);
}
inline void date_destroy(mgp_date *date) { mgp_date_destroy(date); }
inline bool date_equal(mgp_date *first, mgp_date *second) { return MgInvoke<int>(mgp_date_equal, first, second); }
inline int date_get_year(mgp_date *date) { return MgInvoke<int>(mgp_date_get_year, date); }
inline int date_get_month(mgp_date *date) { return MgInvoke<int>(mgp_date_get_month, date); }
inline int date_get_day(mgp_date *date) { return MgInvoke<int>(mgp_date_get_day, date); }
inline int64_t date_timestamp(mgp_date *date) { return MgInvoke<int64_t>(mgp_date_timestamp, date); }
inline mgp_date *date_now(mgp_memory *memory) { return MgInvoke<mgp_date *>(mgp_date_now, memory); }
inline mgp_date *date_add_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_add_duration, date, dur, memory);
}
inline mgp_date *date_sub_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_sub_duration, date, dur, memory);
}
inline mgp_duration *date_diff(mgp_date *first, mgp_date *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_date_diff, first, second, memory);
}
// mgp_local_time
inline mgp_local_time *local_time_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_from_string, string, memory);
}
inline mgp_local_time *local_time_from_parameters(mgp_local_time_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_from_parameters, parameters, memory);
}
inline mgp_local_time *local_time_copy(mgp_local_time *local_time, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_copy, local_time, memory);
}
inline void local_time_destroy(mgp_local_time *local_time) { mgp_local_time_destroy(local_time); }
inline bool local_time_equal(mgp_local_time *first, mgp_local_time *second) {
return MgInvoke<int>(mgp_local_time_equal, first, second);
}
inline int local_time_get_hour(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_hour, local_time);
}
inline int local_time_get_minute(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_minute, local_time);
}
inline int local_time_get_second(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_second, local_time);
}
inline int local_time_get_millisecond(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_millisecond, local_time);
}
inline int local_time_get_microsecond(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_microsecond, local_time);
}
inline int64_t local_time_timestamp(mgp_local_time *local_time) {
return MgInvoke<int64_t>(mgp_local_time_timestamp, local_time);
}
inline mgp_local_time *local_time_now(mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_now, memory);
}
inline mgp_local_time *local_time_add_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_add_duration, local_time, dur, memory);
}
inline mgp_local_time *local_time_sub_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_sub_duration, local_time, dur, memory);
}
inline mgp_duration *local_time_diff(mgp_local_time *first, mgp_local_time *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_local_time_diff, first, second, memory);
}
// mgp_local_date_time
inline mgp_local_date_time *local_date_time_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_from_string, string, memory);
}
inline mgp_local_date_time *local_date_time_from_parameters(mgp_local_date_time_parameters *parameters,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_from_parameters, parameters, memory);
}
inline mgp_local_date_time *local_date_time_copy(mgp_local_date_time *local_date_time, mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_copy, local_date_time, memory);
}
inline void local_date_time_destroy(mgp_local_date_time *local_date_time) {
mgp_local_date_time_destroy(local_date_time);
}
inline bool local_date_time_equal(mgp_local_date_time *first, mgp_local_date_time *second) {
return MgInvoke<int>(mgp_local_date_time_equal, first, second);
}
inline int local_date_time_get_year(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_year, local_date_time);
}
inline int local_date_time_get_month(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_month, local_date_time);
}
inline int local_date_time_get_day(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_day, local_date_time);
}
inline int local_date_time_get_hour(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_hour, local_date_time);
}
inline int local_date_time_get_minute(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_minute, local_date_time);
}
inline int local_date_time_get_second(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_second, local_date_time);
}
inline int local_date_time_get_millisecond(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_millisecond, local_date_time);
}
inline int local_date_time_get_microsecond(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_microsecond, local_date_time);
}
inline int64_t local_date_time_timestamp(mgp_local_date_time *local_date_time) {
return MgInvoke<int64_t>(mgp_local_date_time_timestamp, local_date_time);
}
inline mgp_local_date_time *local_date_time_now(mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_now, memory);
}
inline mgp_local_date_time *local_date_time_add_duration(mgp_local_date_time *local_date_time, mgp_duration *dur,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_add_duration, local_date_time, dur, memory);
}
inline mgp_local_date_time *local_date_time_sub_duration(mgp_local_date_time *local_date_time, mgp_duration *dur,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_sub_duration, local_date_time, dur, memory);
}
inline mgp_duration *local_date_time_diff(mgp_local_date_time *first, mgp_local_date_time *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_local_date_time_diff, first, second, memory);
}
// mgp_duration
inline mgp_duration *duration_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_string, string, memory);
}
inline mgp_duration *duration_from_parameters(mgp_duration_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_parameters, parameters, memory);
}
inline mgp_duration *duration_from_microseconds(int64_t microseconds, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_microseconds, microseconds, memory);
}
inline mgp_duration *duration_copy(mgp_duration *duration, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_copy, duration, memory);
}
inline void duration_destroy(mgp_duration *duration) { mgp_duration_destroy(duration); }
inline int64_t duration_get_microseconds(mgp_duration *duration) {
return MgInvoke<int64_t>(mgp_duration_get_microseconds, duration);
}
inline bool duration_equal(mgp_duration *first, mgp_duration *second) {
return MgInvoke<int>(mgp_duration_equal, first, second);
}
inline mgp_duration *duration_neg(mgp_duration *duration, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_neg, duration, memory);
}
inline mgp_duration *duration_add(mgp_duration *first, mgp_duration *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_add, first, second, memory);
}
inline mgp_duration *duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_sub, first, second, memory);
}
// Procedure
inline mgp_proc *module_add_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb) {
return MgInvoke<mgp_proc *>(mgp_module_add_read_procedure, module, name, cb);
}
inline mgp_proc *module_add_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb) {
return MgInvoke<mgp_proc *>(mgp_module_add_write_procedure, module, name, cb);
}
inline void proc_add_arg(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_arg, proc, name, type);
}
inline void proc_add_opt_arg(mgp_proc *proc, const char *name, mgp_type *type, mgp_value *default_value) {
MgInvokeVoid(mgp_proc_add_opt_arg, proc, name, type, default_value);
}
inline void proc_add_result(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_result, proc, name, type);
}
inline void proc_add_deprecated_result(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_deprecated_result, proc, name, type);
}
inline bool must_abort(mgp_graph *graph) { return mgp_must_abort(graph); }
// mgp_result
inline void result_set_error_msg(mgp_result *res, const char *error_msg) {
MgInvokeVoid(mgp_result_set_error_msg, res, error_msg);
}
inline mgp_result_record *result_new_record(mgp_result *res) {
return MgInvoke<mgp_result_record *>(mgp_result_new_record, res);
}
inline void result_record_insert(mgp_result_record *record, const char *field_name, mgp_value *val) {
MgInvokeVoid(mgp_result_record_insert, record, field_name, val);
}
// Function
inline mgp_func *module_add_function(mgp_module *module, const char *name, mgp_func_cb cb) {
return MgInvoke<mgp_func *>(mgp_module_add_function, module, name, cb);
}
inline void func_add_arg(mgp_func *func, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_func_add_arg, func, name, type);
}
inline void func_add_opt_arg(mgp_func *func, const char *name, mgp_type *type, mgp_value *default_value) {
MgInvokeVoid(mgp_func_add_opt_arg, func, name, type, default_value);
}
inline void func_result_set_error_msg(mgp_func_result *res, const char *msg, mgp_memory *memory) {
MgInvokeVoid(mgp_func_result_set_error_msg, res, msg, memory);
}
inline void func_result_set_value(mgp_func_result *res, mgp_value *value, mgp_memory *memory) {
MgInvokeVoid(mgp_func_result_set_value, res, value, memory);
}
} // namespace mgp

View File

@@ -1,343 +0,0 @@
import typing
from enum import Enum
import networkx as nx
NX_LABEL_ATTR = "labels"
NX_TYPE_ATTR = "type"
SOURCE_TYPE_KAFKA = "SOURCE_TYPE_KAFKA"
SOURCE_TYPE_PULSAR = "SOURCE_TYPE_PULSAR"
"""
This module provides helpers for the mock Python API, much like _mgp.py does for mgp.py.
"""
class InvalidArgumentError(Exception):
"""
Signals that some of the arguments have invalid values.
"""
pass
class ImmutableObjectError(Exception):
pass
class LogicErrorError(Exception):
pass
class DeletedObjectError(Exception):
pass
class EdgeConstants(Enum):
I_START = 0
I_END = 1
I_KEY = 2
class Graph:
"""Wrapper around a NetworkX MultiDiGraph instance."""
__slots__ = ("nx", "_highest_vertex_id", "_highest_edge_id", "_valid")
def __init__(self, graph: nx.MultiDiGraph) -> None:
if not isinstance(graph, nx.MultiDiGraph):
raise TypeError(f"Expected 'networkx.classes.multidigraph.MultiDiGraph', got '{type(graph)}'")
self.nx = graph
self._highest_vertex_id = None
self._highest_edge_id = None
self._valid = True
@property
def vertex_ids(self):
return self.nx.nodes
def vertex_is_isolate(self, vertex_id: int) -> bool:
return nx.is_isolate(self.nx, vertex_id)
@property
def vertices(self):
return (Vertex(node_id, self) for node_id in self.nx.nodes)
def has_node(self, node_id):
return self.nx.has_node(node_id)
@property
def edges(self):
return self.nx.edges
def is_valid(self) -> bool:
return self._valid
def get_vertex_by_id(self, vertex_id: int) -> "Vertex":
return Vertex(vertex_id, self)
def invalidate(self):
self._valid = False
def is_immutable(self) -> bool:
return nx.is_frozen(self.nx)
def make_immutable(self):
self.nx = nx.freeze(self.nx)
def _new_vertex_id(self):
if self._highest_vertex_id is None:
self._highest_vertex_id = max(vertex_id for vertex_id in self.nx.nodes)
return self._highest_vertex_id + 1
def _new_edge_id(self):
if self._highest_edge_id is None:
self._highest_edge_id = max(edge[EdgeConstants.I_KEY.value] for edge in self.nx.edges(keys=True))
return self._highest_edge_id + 1
def create_vertex(self) -> "Vertex":
vertex_id = self._new_vertex_id()
self.nx.add_node(vertex_id)
self._highest_vertex_id = vertex_id
return Vertex(vertex_id, self)
def create_edge(self, from_vertex: "Vertex", to_vertex: "Vertex", edge_type: str) -> "Edge":
if from_vertex.is_deleted() or to_vertex.is_deleted():
raise DeletedObjectError("Accessing deleted object.")
edge_id = self._new_edge_id()
from_id = from_vertex.id
to_id = to_vertex.id
self.nx.add_edge(from_id, to_id, key=edge_id, type=edge_type)
self._highest_edge_id = edge_id
return Edge((from_id, to_id, edge_id), self)
def delete_vertex(self, vertex_id: int):
self.nx.remove_node(vertex_id)
def delete_edge(self, from_vertex_id: int, to_vertex_id: int, edge_id: int):
self.nx.remove_edge(from_vertex_id, to_vertex_id, edge_id)
@property
def highest_vertex_id(self) -> int:
if self._highest_vertex_id is None:
self._highest_vertex_id = max(vertex_id for vertex_id in self.nx.nodes) + 1
return self._highest_vertex_id
@property
def highest_edge_id(self) -> int:
if self._highest_edge_id is None:
self._highest_edge_id = max(edge[EdgeConstants.I_KEY.value] for edge in self.nx.edges(keys=True))
return self._highest_edge_id + 1
class Vertex:
"""Represents a graph vertex."""
__slots__ = ("_id", "_graph")
def __init__(self, id: int, graph: Graph) -> None:
if not isinstance(id, int):
raise TypeError(f"Expected 'int', got '{type(id)}'")
if not isinstance(graph, Graph):
raise TypeError(f"Expected '_mgp_mock.Graph', got '{type(graph)}'")
if not graph.nx.has_node(id):
raise IndexError(f"Unable to find vertex with ID {id}.")
self._id = id
self._graph = graph
def is_valid(self) -> bool:
return self._graph.is_valid()
def is_deleted(self) -> bool:
return not self._graph.nx.has_node(self._id) and self._id <= self._graph.highest_vertex_id
@property
def underlying_graph(self) -> Graph:
return self._graph
def underlying_graph_is_mutable(self) -> bool:
return not nx.is_frozen(self._graph.nx)
@property
def labels(self) -> typing.List[int]:
return self._graph.nx.nodes[self._id][NX_LABEL_ATTR].split(":")
def add_label(self, label: str) -> None:
if nx.is_frozen(self._graph.nx):
raise ImmutableObjectError("Cannot modify immutable object.")
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] += f":{label}"
def remove_label(self, label: str) -> None:
if nx.is_frozen(self._graph.nx):
raise ImmutableObjectError("Cannot modify immutable object.")
labels = self._graph.nx.nodes[self._id][NX_LABEL_ATTR]
if labels.startswith(f"{label}:"):
labels = "\n" + labels # pseudo-string starter
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] = labels.replace(f"\n{label}:", "")
elif labels.endswith(f":{label}"):
labels += "\n" # pseudo-string terminator
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] = labels.replace(f":{label}\n", "")
else:
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] = labels.replace(f":{label}:", ":")
@property
def id(self) -> int:
return self._id
@property
def properties(self):
return (
(key, value)
for key, value in self._graph.nx.nodes[self._id].items()
if key not in (NX_LABEL_ATTR, NX_TYPE_ATTR)
)
def get_property(self, property_name: str):
return self._graph.nx.nodes[self._id][property_name]
def set_property(self, property_name: str, value: object):
self._graph.nx.nodes[self._id][property_name] = value
@property
def in_edges(self) -> typing.Iterable["Edge"]:
return [Edge(edge, self._graph) for edge in self._graph.nx.in_edges(self._id, keys=True)]
@property
def out_edges(self) -> typing.Iterable["Edge"]:
return [Edge(edge, self._graph) for edge in self._graph.nx.out_edges(self._id, keys=True)]
class Edge:
"""Represents a graph edge."""
__slots__ = ("_edge", "_graph")
def __init__(self, edge: typing.Tuple[int, int, int], graph: Graph) -> None:
if not isinstance(edge, typing.Tuple):
raise TypeError(f"Expected 'Tuple', got '{type(edge)}'")
if not isinstance(graph, Graph):
raise TypeError(f"Expected '_mgp_mock.Graph', got '{type(graph)}'")
if not graph.nx.has_edge(*edge):
raise IndexError(f"Unable to find edge with ID {edge[EdgeConstants.I_KEY.value]}.")
self._edge = edge
self._graph = graph
def is_valid(self) -> bool:
return self._graph.is_valid()
def is_deleted(self) -> bool:
return (
not self._graph.nx.has_edge(*self._edge)
and self._edge[EdgeConstants.I_KEY.value] <= self._graph.highest_edge_id
)
def underlying_graph_is_mutable(self) -> bool:
return not nx.is_frozen(self._graph.nx)
@property
def id(self) -> int:
return self._edge[EdgeConstants.I_KEY.value]
@property
def edge(self) -> typing.Tuple[int, int, int]:
return self._edge
@property
def start_id(self) -> int:
return self._edge[EdgeConstants.I_START.value]
@property
def end_id(self) -> int:
return self._edge[EdgeConstants.I_END.value]
def get_type_name(self):
return self._graph.nx.get_edge_data(*self._edge)[NX_TYPE_ATTR]
def from_vertex(self) -> Vertex:
return Vertex(self.start_id, self._graph)
def to_vertex(self) -> Vertex:
return Vertex(self.end_id, self._graph)
@property
def properties(self):
return (
(key, value)
for key, value in self._graph.nx.edges[self._edge].items()
if key not in (NX_LABEL_ATTR, NX_TYPE_ATTR)
)
def get_property(self, property_name: str):
return self._graph.nx.edges[self._edge][property_name]
def set_property(self, property_name: str, value: object):
self._graph.nx.edges[self._edge][property_name] = value
class Path:
"""Represents a path comprised of `Vertex` and `Edge` instances."""
__slots__ = ("_vertices", "_edges", "_graph")
__create_key = object()
def __init__(self, create_key, vertex_id: int, graph: Graph) -> None:
assert create_key == Path.__create_key, "Path objects must be created using Path.make_with_start"
self._vertices = [vertex_id]
self._edges = []
self._graph = graph
@classmethod
def make_with_start(cls, vertex: Vertex) -> "Path":
if not isinstance(vertex, Vertex):
raise TypeError(f"Expected 'Vertex', got '{type(vertex)}'")
if not isinstance(vertex.underlying_graph, Graph):
raise TypeError(f"Expected '_mgp_mock.Graph', got '{type(vertex.underlying_graph)}'")
if not vertex.underlying_graph.nx.has_node(vertex._id):
raise IndexError(f"Unable to find vertex with ID {vertex._id}.")
return Path(cls.__create_key, vertex._id, vertex.underlying_graph)
def is_valid(self) -> bool:
return self._graph.is_valid()
def underlying_graph_is_mutable(self) -> bool:
return not nx.is_frozen(self._graph.nx)
def expand(self, edge: Edge):
if edge.start_id != self._vertices[-1]:
raise LogicErrorError("Logic error.")
self._vertices.append(edge.end_id)
self._edges.append((edge.start_id, edge.end_id, edge.id))
def vertex_at(self, index: int) -> Vertex:
return Vertex(self._vertices[index], self._graph)
def edge_at(self, index: int) -> Edge:
return Edge(self._edges[index], self._graph)
def size(self) -> int:
return len(self._edges)

View File

@@ -1,57 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <exception>
#include <iostream>
namespace mg_exception {
struct NotEnoughMemoryException : public std::exception {
const char *what() const throw() { return "Not enough memory!"; }
};
struct UnknownException : public std::exception {
const char *what() const throw() { return "Unknown exception!"; }
};
struct AllocationException : public std::exception {
const char *what() const throw() { return "Could not allocate memory!"; }
};
struct InsufficientBufferException : public std::exception {
const char *what() const throw() { return "Buffer is not sufficient to process procedure!"; }
};
struct OutOfRangeException : public std::exception {
const char *what() const throw() { return "Index out of range!"; }
};
struct LogicException : public std::exception {
const char *what() const throw() { return "Logic exception, check the procedure signature!"; }
};
struct DeletedObjectException : public std::exception {
const char *what() const throw() { return "Object is deleted!"; }
};
struct InvalidArgumentException : public std::exception {
const char *what() const throw() { return "Invalid argument!"; }
};
struct InvalidIDException : public std::exception {
const char *what() const throw() { return "Invalid ID!"; }
};
struct KeyAlreadyExistsException : public std::exception {
const char *what() const throw() { return "Key you are trying to set already exists!"; }
};
struct ImmutableObjectException : public std::exception {
const char *what() const throw() { return "Object you are trying to change is immutable!"; }
};
struct ValueConversionException : public std::exception {
const char *what() const throw() { return "Error in value conversion!"; }
};
struct SerializationException : public std::exception {
const char *what() const throw() { return "Error in serialization!"; }
};
} // namespace mg_exception

View File

@@ -37,19 +37,12 @@ extern "C" {
/// All functions return an error code that can be used to figure out whether the API call was successful or not. In
/// case of failure, the specific error code can be used to identify the reason of the failure.
MGP_ENUM_CLASS MGP_NODISCARD mgp_error{
MGP_ERROR_NO_ERROR,
MGP_ERROR_UNKNOWN_ERROR,
MGP_ERROR_UNABLE_TO_ALLOCATE,
MGP_ERROR_INSUFFICIENT_BUFFER,
MGP_ERROR_OUT_OF_RANGE,
MGP_ERROR_LOGIC_ERROR,
MGP_ERROR_DELETED_OBJECT,
MGP_ERROR_INVALID_ARGUMENT,
MGP_ERROR_KEY_ALREADY_EXISTS,
MGP_ERROR_IMMUTABLE_OBJECT,
MGP_ERROR_VALUE_CONVERSION,
MGP_ERROR_SERIALIZATION_ERROR,
MGP_ERROR_AUTHORIZATION_ERROR,
MGP_ERROR_NO_ERROR, MGP_ERROR_UNKNOWN_ERROR,
MGP_ERROR_UNABLE_TO_ALLOCATE, MGP_ERROR_INSUFFICIENT_BUFFER,
MGP_ERROR_OUT_OF_RANGE, MGP_ERROR_LOGIC_ERROR,
MGP_ERROR_DELETED_OBJECT, MGP_ERROR_INVALID_ARGUMENT,
MGP_ERROR_KEY_ALREADY_EXISTS, MGP_ERROR_IMMUTABLE_OBJECT,
MGP_ERROR_VALUE_CONVERSION, MGP_ERROR_SERIALIZATION_ERROR,
};
///@}
@@ -171,8 +164,6 @@ enum mgp_value_type {
MGP_VALUE_TYPE_DURATION,
};
enum mgp_error mgp_value_copy(struct mgp_value *val, struct mgp_memory *memory, struct mgp_value **result);
/// Free the memory used by the given mgp_value instance.
void mgp_value_destroy(struct mgp_value *val);
@@ -408,8 +399,6 @@ enum mgp_error mgp_value_get_duration(struct mgp_value *val, struct mgp_duration
/// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_list.
enum mgp_error mgp_list_make_empty(size_t capacity, struct mgp_memory *memory, struct mgp_list **result);
enum mgp_error mgp_list_copy(struct mgp_list *list, struct mgp_memory *memory, struct mgp_list **result);
/// Free the memory used by the given mgp_list and contained elements.
void mgp_list_destroy(struct mgp_list *list);
@@ -448,8 +437,6 @@ enum mgp_error mgp_list_at(struct mgp_list *list, size_t index, struct mgp_value
/// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_map.
enum mgp_error mgp_map_make_empty(struct mgp_memory *memory, struct mgp_map **result);
enum mgp_error mgp_map_copy(struct mgp_map *map, struct mgp_memory *memory, struct mgp_map **result);
/// Free the memory used by the given mgp_map and contained items.
void mgp_map_destroy(struct mgp_map *map);
@@ -1305,12 +1292,6 @@ struct mgp_proc;
/// Describes a Memgraph magic function.
struct mgp_func;
/// All available log levels that can be used in mgp_log function
MGP_ENUM_CLASS mgp_log_level{
MGP_LOG_LEVEL_TRACE, MGP_LOG_LEVEL_DEBUG, MGP_LOG_LEVEL_INFO,
MGP_LOG_LEVEL_WARN, MGP_LOG_LEVEL_ERROR, MGP_LOG_LEVEL_CRITICAL,
};
/// Entry-point for a query module read procedure, invoked through openCypher.
///
/// Passed in arguments will not live longer than the callback's execution.
@@ -1405,9 +1386,6 @@ enum mgp_error mgp_proc_add_result(struct mgp_proc *proc, const char *name, stru
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid result name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a result field with the same name was already added.
enum mgp_error mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char *name, struct mgp_type *type);
/// Log a message on a certain level.
enum mgp_error mgp_log(enum mgp_log_level log_level, const char *output);
///@}
/// @name Execution
@@ -1534,10 +1512,6 @@ enum mgp_error mgp_module_add_transformation(struct mgp_module *module, const ch
///
///@{
/// State of the database that is exposed to magic functions. Currently it is unused, but it enables extending the
/// functionalities of magic functions in future without breaking the API.
struct mgp_func_context;
/// Add a required argument to a function.
///
/// The order of the added arguments corresponds to the signature of the openCypher function.

File diff suppressed because it is too large Load Diff

View File

@@ -134,15 +134,6 @@ class SerializationError(_mgp.SerializationError):
pass
class AuthorizationError(_mgp.AuthorizationError):
"""
Signals that the user doesn't have sufficient permissions to perform
procedure call.
"""
pass
class Label:
"""Label of a `Vertex`."""
@@ -155,7 +146,7 @@ class Label:
def name(self) -> str:
"""
Get the name of the label.
Returns:
A string that represents the name of the label.
@@ -204,20 +195,20 @@ class Properties:
def get(self, property_name: str, default=None) -> object:
"""
Get the value of a property with the given name or return default value.
Args:
Args:
property_name: String that represents property name.
default: Default value return if there is no property.
Returns:
Any object value that property under `property_name` has or default value otherwise.
Raises:
InvalidContextError: If `edge` or `vertex` is out of context.
Returns:
Any object value that property under `property_name` has or default value otherwise.
Raises:
InvalidContextError: If `edge` or `vertex` is out of context.
UnableToAllocateError: If unable to allocate a `mgp.Value`.
DeletedObjectError: If the `object` has been deleted.
Examples:
Examples:
```
vertex.properties.get(property_name)
edge.properties.get(property_name)
@@ -236,23 +227,23 @@ class Properties:
Set the value of the property. When the value is `None`, then the
property is removed.
Args:
property_name: String that represents property name.
Args:
property_name: String that represents property name.
value: Object that represents value to be set.
Raises:
Raises:
UnableToAllocateError: If unable to allocate memory for storing the property.
ImmutableObjectError: If the object is immutable.
DeletedObjectError: If the object has been deleted.
SerializationError: If the object has been modified by another transaction.
ValueConversionError: If `value` is vertex, edge or path.
Examples:
```
vertex.properties.set(property_name, value)
edge.properties.set(property_name, value)
```
"""
self[property_name] = value
@@ -261,15 +252,15 @@ class Properties:
Iterate over the properties. Doesnt return a dynamic view of the properties but copies the
current properties.
Returns:
Returns:
Iterable `Property` of names and values.
Raises:
InvalidContextError: If edge or vertex is out of context.
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
items = vertex.properties.items()
for it in items:
@@ -299,15 +290,15 @@ class Properties:
Iterate over property names. Doesnt return a dynamic view of the property names but copies the
name of the current properties.
Returns:
Returns:
Iterable list of strings that represent names/keys of properties.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
graph.vertex.properties.keys()
graph.edge.properties.keys()
@@ -323,20 +314,20 @@ class Properties:
Iterate over property values. Doesnt return a dynamic view of the property values but copies the
value of the current properties.
Returns:
Returns:
Iterable list of property values.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
vertex.properties.values()
edge.properties.values()
```
"""
if not self._vertex_or_edge.is_valid():
raise InvalidContextError()
@@ -347,15 +338,15 @@ class Properties:
"""
Get the number of properties.
Returns:
Returns:
A number of properties on vertex or edge.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
len(vertex.properties)
len(edge.properties)
@@ -372,15 +363,15 @@ class Properties:
"""
Iterate over property names.
Returns:
Returns:
Iterable list of strings that represent names of properties.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
iter(vertex.properties)
iter(edge.properties)
@@ -395,24 +386,24 @@ class Properties:
def __getitem__(self, property_name: str) -> object:
"""
Get the value of a property with the given name or raise KeyError.
Args:
Args:
property_name: String that represents property name.
Returns:
Returns:
Any value that property under property_name have.
Raises:
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate a mgp.Value.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
vertex.properties[property_name]
edge.properties[property_name]
```
"""
if not self._vertex_or_edge.is_valid():
raise InvalidContextError()
@@ -426,18 +417,18 @@ class Properties:
Set the value of the property. When the value is `None`, then the
property is removed.
Args:
Args:
property_name: String that represents property name.
value: Object that represents value to be set.
Raises:
UnableToAllocateError: If unable to allocate memory for storing the property.
ImmutableObjectError: If the object is immutable.
DeletedObjectError: If the object has been deleted.
SerializationError: If the object has been modified by another transaction.
ValueConversionError: If `value` is vertex, edge or path.
Examples:
Examples:
```
vertex.properties[property_name] = value
edge.properties[property_name] = value
@@ -452,18 +443,18 @@ class Properties:
"""
Check if there is a property with the given name.
Args:
Args:
property_name: String that represents property name
Returns:
Bool value that depends if there is with a given name.
Raises:
Bool value that depends if there is with a given name.
Raises:
InvalidContextError: If edge or vertex is out of context.
UnableToAllocateError: If unable to allocate a mgp.Value.
DeletedObjectError: If the object has been deleted.
Examples:
Examples:
```
if property_name in vertex.properties:
```
@@ -492,7 +483,7 @@ class EdgeType:
def name(self) -> str:
"""
Get the name of EdgeType.
Returns:
A string that represents the name of EdgeType.
@@ -521,7 +512,7 @@ class Edge:
Access to an Edge is only valid during a single execution of a procedure in
a query. You should not globally store an instance of an Edge. Using an
invalid Edge instance will raise InvalidContextError.
"""
__slots__ = ("_edge",)
@@ -541,10 +532,10 @@ class Edge:
def is_valid(self) -> bool:
"""
Check if `edge` is in a valid context and may be used.
Returns:
A `bool` value depends on if the `edge` is in a valid context.
A `bool` value depends on if the `edge` is in a valid context.
Examples:
```edge.is_valid()```
@@ -552,15 +543,15 @@ class Edge:
return self._edge.is_valid()
def underlying_graph_is_mutable(self) -> bool:
"""
Check if the `graph` can be modified.
"""
Check if the `graph` can be modified.
Returns:
Returns:
A `bool` value depends on if the `graph` is mutable.
Examples:
Examples:
```edge.underlying_graph_is_mutable()```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -573,10 +564,10 @@ class Edge:
Returns:
`EdgeId` represents ID of the edge.
Raises:
Raises:
InvalidContextError: If edge is out of context.
Examples:
```edge.id```
"""
@@ -590,12 +581,12 @@ class Edge:
Get the type of edge.
Returns:
`EdgeType` describing the type of edge.
`EdgeType` describing the type of edge.
Raises:
InvalidContextError: If edge is out of context.
Examples:
Examples:
```edge.type```
"""
if not self.is_valid():
@@ -607,10 +598,10 @@ class Edge:
"""
Get the source vertex.
Returns:
Returns:
`Vertex` from where the edge is directed.
Raises:
Raises:
InvalidContextError: If edge is out of context.
Examples:
@@ -624,14 +615,14 @@ class Edge:
def to_vertex(self) -> "Vertex":
"""
Get the destination vertex.
Returns:
Returns:
`Vertex` to where the edge is directed.
Raises:
Raises:
InvalidContextError: If edge is out of context.
Examples:
Examples:
```edge.to_vertex```
"""
if not self.is_valid():
@@ -644,7 +635,7 @@ class Edge:
Get the properties of the edge.
Returns:
All `Properties` of edge.
All `Properties` of edge.
Raises:
InvalidContextError: If edge is out of context.
@@ -701,9 +692,9 @@ class Vertex:
Checks if `Vertex` is in valid context and may be used.
Returns:
A `bool` value depends on if the `Vertex` is in a valid context.
Examples:
A `bool` value depends on if the `Vertex` is in a valid context.
Examples:
```vertex.is_valid()```
"""
@@ -711,14 +702,14 @@ class Vertex:
def underlying_graph_is_mutable(self) -> bool:
"""
Check if the `graph` is mutable.
Check if the `graph` is mutable.
Returns:
Returns:
A `bool` value depends on if the `graph` is mutable.
Examples:
Examples:
```vertex.underlying_graph_is_mutable()```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -731,10 +722,10 @@ class Vertex:
Returns:
`VertexId` represents ID of the vertex.
Raises:
Raises:
InvalidContextError: If vertex is out of context.
Examples:
```vertex.id```
"""
@@ -747,15 +738,15 @@ class Vertex:
"""
Get the labels of the vertex.
Returns:
Returns:
A tuple of `Label` representing vertex Labels
Raises:
Raises:
InvalidContextError: If vertex is out of context.
OutOfRangeError: If some of the labels are removed while collecting the labels.
DeletedObjectError: If `Vertex` has been deleted.
Examples:
Examples:
```vertex.labels```
"""
if not self.is_valid():
@@ -766,17 +757,17 @@ class Vertex:
"""
Add the label to the vertex.
Args:
label: String label to be added.
Raises:
Args:
label: String label to be added.
Raises:
InvalidContextError: If `Vertex` is out of context.
UnableToAllocateError: If unable to allocate memory for storing the label.
ImmutableObjectError: If `Vertex` is immutable.
DeletedObjectError: If `Vertex` has been deleted.
SerializationError: If `Vertex` has been modified by another transaction.
Examples:
Examples:
```vertex.add_label(label)```
"""
if not self.is_valid():
@@ -787,15 +778,15 @@ class Vertex:
"""
Remove the label from the vertex.
Args:
label: String label to be deleted
Raises:
Args:
label: String label to be deleted
Raises:
InvalidContextError: If `Vertex` is out of context.
ImmutableObjectError: If `Vertex` is immutable.
DeletedObjectError: If `Vertex` has been deleted.
SerializationError: If `Vertex` has been modified by another transaction.
Examples:
Examples:
```vertex.remove_label(label)```
"""
if not self.is_valid():
@@ -807,13 +798,13 @@ class Vertex:
"""
Get the properties of the vertex.
Returns:
Returns:
`Properties` on a current vertex.
Raises:
Raises:
InvalidContextError: If `Vertex` is out of context.
Examples:
Examples:
```vertex.properties```
"""
if not self.is_valid():
@@ -829,13 +820,13 @@ class Vertex:
Returns:
Iterable list of `Edge` objects that are directed in towards the current vertex.
Raises:
Raises:
InvalidContextError: If `Vertex` is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If `Vertex` has been deleted.
Examples:
Examples:
```for edge in vertex.in_edges:```
"""
if not self.is_valid():
@@ -859,12 +850,12 @@ class Vertex:
Returns:
Iterable list of `Edge` objects that are directed out of the current vertex.
Raises:
Raises:
InvalidContextError: If `Vertex` is out of context.
UnableToAllocateError: If unable to allocate an iterator.
DeletedObjectError: If `Vertex` has been deleted.
Examples:
Examples:
```for edge in vertex.out_edges:```
"""
if not self.is_valid():
@@ -897,7 +888,7 @@ class Path:
def __init__(self, starting_vertex_or_path: typing.Union[_mgp.Path, Vertex]):
"""Initialize with a starting Vertex.
Raises:
Raises:
InvalidContextError: If passed in Vertex is invalid.
UnableToAllocateError: If cannot allocate a path.
"""
@@ -941,10 +932,10 @@ class Path:
def is_valid(self) -> bool:
"""
Check if `Path` is in valid context and may be used.
Returns:
A `bool` value depends on if the `Path` is in a valid context.
A `bool` value depends on if the `Path` is in a valid context.
Examples:
```path.is_valid()```
"""
@@ -957,15 +948,15 @@ class Path:
The last vertex on the path will become the other endpoint of the given
edge, as continued from the current last vertex.
Args:
Args:
edge: `Edge` that is added to the path
Raises:
Raises:
InvalidContextError: If using an invalid `Path` instance or if passed in `Edge` is invalid.
LogicErrorError: If the current last vertex in the path is not part of the given edge.
UnableToAllocateError: If unable to allocate memory for path extension.
Examples:
Examples:
```path.expand(edge)```
"""
if not isinstance(edge, Edge):
@@ -982,14 +973,14 @@ class Path:
"""
Vertices are ordered from the start to the end of the path.
Returns:
A tuple of `Vertex` objects order from start to end of the path.
Returns:
A tuple of `Vertex` objects order from start to end of the path.
Raises:
InvalidContextError: If using an invalid Path instance.
Examples:
```path.vertices```
```path.vertices```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -1003,11 +994,11 @@ class Path:
"""
Edges are ordered from the start to the end of the path.
Returns:
Returns:
A tuple of `Edge` objects order from start to end of the path
Raises:
Raises:
InvalidContextError: If using an invalid `Path` instance.
Examples:
Examples:
```path.edges```
"""
if not self.is_valid():
@@ -1048,10 +1039,10 @@ class Vertices:
def is_valid(self) -> bool:
"""
Check if `Vertices` is in valid context and may be used.
Returns:
A `bool` value depends on if the `Vertices` is in valid context.
A `bool` value depends on if the `Vertices` is in valid context.
Examples:
```vertices.is_valid()```
"""
@@ -1061,14 +1052,14 @@ class Vertices:
"""
Iterate over vertices.
Returns:
Iterable list of `Vertex` objects.
Returns:
Iterable list of `Vertex` objects.
Raises:
Raises:
InvalidContextError: If context is invalid.
UnableToAllocateError: If unable to allocate an iterator or a vertex.
Examples:
Examples:
```
for vertex in graph.vertices:
```
@@ -1089,18 +1080,18 @@ class Vertices:
def __contains__(self, vertex):
"""
Check if Vertices contain the given vertex.
Check if Vertices contain the given vertex.
Args:
Args:
vertex: `Vertex` to be checked if it is a part of graph `Vertices`.
Returns:
Bool value depends if there is `Vertex` in graph `Vertices`.
Bool value depends if there is `Vertex` in graph `Vertices`.
Raises:
UnableToAllocateError: If unable to allocate the vertex.
Examples:
Examples:
```if vertex in graph.vertices:```
"""
try:
@@ -1113,14 +1104,14 @@ class Vertices:
"""
Get the number of vertices.
Returns:
Returns:
A number of vertices in the graph.
Raises:
Raises:
InvalidContextError: If context is invalid.
UnableToAllocateError: If unable to allocate an iterator or a vertex.
Examples:
Examples:
```len(graph.vertices)```
"""
if not self._len:
@@ -1149,9 +1140,9 @@ class Graph:
Check if `graph` is in a valid context and may be used.
Returns:
A `bool` value depends on if the `graph` is in a valid context.
Examples:
A `bool` value depends on if the `graph` is in a valid context.
Examples:
```graph.is_valid()```
"""
@@ -1178,7 +1169,7 @@ class Graph:
Examples:
```graph.get_vertex_by_id(vertex_id)```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -1216,11 +1207,11 @@ class Graph:
def is_mutable(self) -> bool:
"""
Check if the graph is mutable. Thus it can be used to modify vertices and edges.
Returns:
A `bool` value that depends if the graph is mutable or not.
Returns:
A `bool` value that depends if the graph is mutable or not.
Examples:
Examples:
```graph.is_mutable()```
"""
if not self.is_valid():
@@ -1231,14 +1222,14 @@ class Graph:
"""
Create an empty vertex.
Returns:
Created `Vertex`.
Returns:
Created `Vertex`.
Raises:
Raises:
ImmutableObjectError: If `graph` is immutable.
UnableToAllocateError: If unable to allocate a vertex.
Examples:
Examples:
Creating an empty vertex.
```vertex = graph.create_vertex()```
@@ -1258,7 +1249,7 @@ class Graph:
LogicErrorError: If `vertex` has edges.
SerializationError: If `vertex` has been modified by
another transaction.
Examples:
Examples:
```graph.delete_vertex(vertex)```
"""
@@ -1269,39 +1260,36 @@ class Graph:
def detach_delete_vertex(self, vertex: Vertex) -> None:
"""
Delete a vertex and all of its edges.
Args:
Args:
vertex: `Vertex` to be deleted with all of its edges
Raises:
Raises:
ImmutableObjectError: If `graph` is immutable.
SerializationError: If `vertex` has been modified by another transaction.
Examples:
Examples:
```graph.detach_delete_vertex(vertex)```
"""
if not self.is_valid():
raise InvalidContextError()
self._graph.detach_delete_vertex(vertex._vertex)
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> Edge:
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> None:
"""
Create an edge.
Args:
from_vertex: `Vertex` from where edge is directed.
to_vertex: `Vertex' to where edge is directed.
edge_type: `EdgeType` defines the type of edge.
Returns:
Created `Edge`.
Args:
from_vertex: `Vertex` from where edge is directed.
to_vertex: `Vertex' to where edge is directed.
edge_type: `EdgeType` defines the type of edge.
Raises:
ImmutableObjectError: If `graph` is immutable.
UnableToAllocateError: If unable to allocate an edge.
DeletedObjectError: If `from_vertex` or `to_vertex` has been deleted.
SerializationError: If `from_vertex` or `to_vertex` has been modified by another transaction.
Examples:
```edge = graph.create_edge(from_vertex, vertex, edge_type)```
Examples:
```graph.create_edge(from_vertex, vertex, edge_type)```
"""
if not self.is_valid():
raise InvalidContextError()
@@ -1313,8 +1301,8 @@ class Graph:
Args:
edge: `Edge` to be deleted
Raises:
Raises:
ImmutableObjectError if `graph` is immutable.
Raise SerializationError if `edge`, its source or destination vertex has been modified by another transaction.
"""
@@ -1349,15 +1337,15 @@ class ProcCtx:
@property
def graph(self) -> Graph:
"""
Access to `Graph` object.
Access to `Graph` object.
Returns:
Graph object.
Returns:
Graph object.
Raises:
Raises:
InvalidContextError: If context is invalid.
Examples:
Examples:
```context.graph```
"""
if not self.is_valid():
@@ -1981,8 +1969,6 @@ def _wrap_exceptions():
raise ValueConversionError(e)
except _mgp.SerializationError as e:
raise SerializationError(e)
except _mgp.AuthorizationError as e:
raise AuthorizationError(e)
return wrapped_func
@@ -2018,82 +2004,4 @@ def _wrap_exceptions():
setattr(module, name, wrap_function(obj))
class Logger:
"""Represents a Logger through which it is possible
to send logs via API to the graph database.
The best way to use this Logger is to have one per query module."""
__slots__ = ("_logger",)
def __init__(self):
self._logger = _mgp._LOGGER
def info(self, out: str) -> None:
"""
Log message on INFO level..
Args:
out: String message to be logged.
Examples:
```logger.info("Hello from query module.")```
"""
self._logger.info(out)
def warning(self, out: str) -> None:
"""
Log message on WARNING level..
Args:
out: String message to be logged.
Examples:
```logger.warning("Hello from query module.")```
"""
self._logger.warning(out)
def critical(self, out: str) -> None:
"""
Log message on CRITICAL level..
Args:
out: String message to be logged.
Examples:
```logger.critical("Hello from query module.")```
"""
self._logger.critical(out)
def error(self, out: str) -> None:
"""
Log message on ERROR level..
Args:
out: String message to be logged.
Examples:
```logger.error("Hello from query module.")```
"""
self._logger.error(out)
def trace(self, out: str) -> None:
"""
Log message on TRACE level..
Args:
out: String message to be logged.
Examples:
```logger.trace("Hello from query module.")```
"""
self._logger.trace(out)
def debug(self, out: str) -> None:
"""
Log message on DEBUG level..
Args:
out: String message to be logged.
Examples:
```logger.debug("Hello from query module.")```
"""
self._logger.debug(out)
_wrap_exceptions()

File diff suppressed because it is too large Load Diff

71
init
View File

@@ -5,15 +5,13 @@ cd "$DIR"
source "$DIR/environment/util.sh"
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
function print_help () {
echo "Usage: $0 [OPTION]"
echo -e "Check for missing packages and setup the project.\n"
echo "Optional arguments:"
echo -e " -h\tdisplay this help and exit"
echo -e " --without-libs-setup\tskip the step for setting up libs"
echo -e " --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)"
}
function setup_virtualenv () {
@@ -34,6 +32,7 @@ function setup_virtualenv () {
popd > /dev/null
}
wsl_quicklisp_proxy=""
setup_libs=true
if [[ $# -eq 1 && "$1" == "-h" ]]; then
print_help
@@ -41,6 +40,16 @@ if [[ $# -eq 1 && "$1" == "-h" ]]; then
else
while(($#)); do
case "$1" in
--wsl-quicklisp-proxy)
shift
if [[ $# -eq 0 ]]; then
echo "Missing proxy URL"
print_help
exit 1
fi
wsl_quicklisp_proxy=":proxy \"http://$1/\""
shift
;;
--without-libs-setup)
shift
setup_libs=false
@@ -55,6 +64,8 @@ else
done
fi
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
else
@@ -67,18 +78,37 @@ echo "All packages are in-place..."
# create a default build directory
mkdir -p ./build
if [[ "$setup_libs" == "true" ]]; then
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
# quicklisp package manager for Common Lisp
quicklisp_install_dir="$HOME/quicklisp"
if [[ -v QUICKLISP_HOME ]]; then
quicklisp_install_dir="${QUICKLISP_HOME}"
fi
# Fix for centos 7 during release
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then
python3 -m pip uninstall -y virtualenv
python3 -m pip install virtualenv
if [[ ! -f "${quicklisp_install_dir}/setup.lisp" ]]; then
wget -nv https://beta.quicklisp.org/quicklisp.lisp -O quicklisp.lisp || exit 1
echo \
"
(load \"${DIR}/quicklisp.lisp\")
(quicklisp-quickstart:install $wsl_quicklisp_proxy :path \"${quicklisp_install_dir}\")
" | sbcl --script || exit 1
rm -rf quicklisp.lisp || exit 1
fi
ln -Tfs "$DIR/src/lisp" "${quicklisp_install_dir}/local-projects/lcp"
# Install LCP dependencies
# TODO: We should at some point cache or have a mirror of packages we use.
# TODO: move the installation of LCP's dependencies into ./setup.sh
echo \
"
(load \"${quicklisp_install_dir}/setup.lisp\")
(ql:quickload '(:lcp :lcp/test) :silent t)
" | sbcl --script
if [[ "$setup_libs" == "true" ]]; then
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
fi
# setup gql_behave dependencies
@@ -106,18 +136,9 @@ for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
echo "Added $hook hook"
done;
# Install precommit hook except on old operating systems because we don't
# develop on them -> pre-commit hook not required -> we can use latest
# packages.
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ] && [ "${DISTRO}" != "amzn-2" ]; then
python3 -m pip install pre-commit
python3 -m pre_commit install
# Install py format tools for usage during the development.
echo "Install black formatter"
python3 -m pip install black==23.1.*
echo "Install isort"
python3 -m pip install isort==5.12.*
fi
# Install precommit hook
python3 -m pip install pre-commit
python3 -m pre_commit install
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

View File

@@ -117,7 +117,7 @@ declare -A primary_urls=(
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-5.6.0-unix.tar.gz"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
["protobuf"]="http://$local_cache_host/git/protobuf.git"
["pulsar"]="http://$local_cache_host/git/pulsar.git"
@@ -142,7 +142,7 @@ declare -A secondary_urls=(
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
["spdlog"]="https://github.com/gabime/spdlog"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="https://dist.neo4j.org/neo4j-community-5.6.0-unix.tar.gz"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
["pulsar"]="https://github.com/apache/pulsar.git"
@@ -180,9 +180,9 @@ repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}
# neo4j
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
tar -xzf neo4j-community-5.6.0-unix.tar.gz
mv neo4j-community-5.6.0 neo4j
rm neo4j-community-5.6.0-unix.tar.gz
tar -xzf neo4j-community-3.2.3-unix.tar.gz
mv neo4j-community-3.2.3 neo4j
rm neo4j-community-3.2.3-unix.tar.gz
# nlohmann json
# We wget header instead of cloning repo since repo is huge (lots of test data).
@@ -208,7 +208,7 @@ pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
# mgconsole
mgconsole_tag="v1.3.0" # (2022-11-20)
mgconsole_tag="v1.1.0" # (2021-10-07)
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
spdlog_tag="v1.9.2" # (2021-08-12)

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2027-05-04
CHANGE DATE: 2026-27-04
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -2,8 +2,8 @@ MEMGRAPH
ENTERPRISE LICENCE AGREEMENT
Memgraph Limited is registered in England under registration 10195084 and has its registered office at 90a High Street,
Hertfordshire, Berkhamsted, HP4 2BL United Kingdom ("Memgraph").
Memgraph Limited is registered in England under registration 10195084 and has its registered office at Suite 4,
Ironstone House, Ironstone Way, Brixworth, Northampton, NN6 9UD (Memgraph).
Memgraph agrees to license and/or grant you (the “Customer”) access to the Software ( as defined below) and provide

View File

@@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@@ -1,12 +0,0 @@
[tool.black]
line-length = 120
include = '\.pyi?$'
extend-exclude = '''
/(
| .git
| .__pycache__
| build
| libs
| .cache
)/
'''

View File

@@ -8,40 +8,29 @@ disallow_in_source_build()
# Everything that is installed here, should be under the "query_modules" component.
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules")
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
add_library(example_c SHARED example.c)
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_c PRIVATE -Wall)
# Strip C example in release build.
add_library(example SHARED example.c)
target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example PRIVATE -Wall)
# Strip the library in release build.
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_c POST_BUILD
COMMAND strip -s $<TARGET_FILE:example_c>
COMMENT "Stripping symbols and sections from the C example module")
add_custom_command(TARGET example POST_BUILD
COMMAND strip -s $<TARGET_FILE:example>
COMMENT "Stripping symbols and sections from example module")
endif()
install(PROGRAMS $<TARGET_FILE:example_c>
install(PROGRAMS $<TARGET_FILE:example>
DESTINATION lib/memgraph/query_modules
RENAME example_c.so)
RENAME example.so)
# Also install the source of the example, so user can read it.
install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
add_library(example_cpp SHARED example.cpp)
target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_cpp PRIVATE -Wall)
# Strip C++ example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_cpp POST_BUILD
COMMAND strip -s $<TARGET_FILE:example_cpp>
COMMENT "Stripping symbols and sections from the C++ example module")
endif()
install(PROGRAMS $<TARGET_FILE:example_cpp>
DESTINATION lib/memgraph/query_modules
RENAME example_cpp.so)
# Also install the source of the example, so user can read it.
install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
# Install the Python example and modules
# Install the Python example
install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
# Install the Python modules
install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)

View File

@@ -1,103 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <mgp.hpp>
void ProcImpl(std::vector<mgp::Value> arguments, mgp::Graph graph, mgp::RecordFactory record_factory) {
auto record = record_factory.NewRecord();
record.Insert("out", true);
}
void SampleReadProc(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
try {
mgp::memory = memory;
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
ProcImpl(arguments, mgp::Graph(memgraph_graph), mgp::RecordFactory(result));
} catch (const std::exception &e) {
mgp::result_set_error_msg(result, e.what());
return;
}
}
void AddXNodes(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::memory = memory;
auto graph = mgp::Graph(memgraph_graph);
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
for (int i = 0; i < arguments[0].ValueInt(); i++) {
graph.CreateNode();
}
}
void Multiply(mgp_list *args, mgp_func_context *ctx, mgp_func_result *res, mgp_memory *memory) {
mgp::memory = memory;
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
auto result = mgp::Result(res);
auto first = arguments[0].ValueInt();
auto second = arguments[1].ValueInt();
result.SetValue(first * second);
}
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
try {
mgp::memory = memory;
AddProcedure(SampleReadProc, "return_true", mgp::ProcedureType::Read,
{mgp::Parameter("param_1", mgp::Type::Int), mgp::Parameter("param_2", mgp::Type::Double, 2.3)},
{mgp::Return("out", mgp::Type::Bool)}, module, memory);
} catch (const std::exception &e) {
return 1;
}
try {
mgp::memory = memory;
mgp::AddProcedure(AddXNodes, "add_x_nodes", mgp::ProcedureType::Write, {mgp::Parameter("param_1", mgp::Type::Int)},
{}, module, memory);
} catch (const std::exception &e) {
return 1;
}
try {
mgp::memory = memory;
mgp::AddFunction(Multiply, "multiply",
{mgp::Parameter("int", mgp::Type::Int), mgp::Parameter("int", mgp::Type::Int, (int64_t)3)}, module,
memory);
} catch (const std::exception &e) {
return 1;
}
return 0;
}
extern "C" int mgp_shutdown_module() { return 0; }

View File

@@ -1,10 +1,6 @@
# Install systemd service (must use absolute path).
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/memgraph.service
DESTINATION /lib/systemd/system)
# Set parameters to recognize the host distro
cmake_host_system_information(RESULT DISTRO QUERY DISTRIB_NAME)
cmake_host_system_information(RESULT DISTRO_VERSION QUERY DISTRIB_VERSION)
DESTINATION /lib/systemd/system)
# ---- Setup CPack --------
@@ -16,11 +12,10 @@ set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
# Setting arhitecture extension for deb packages
set(MG_ARCH_EXTENSION_DEB "all")
if(${MG_ARCH} STREQUAL "x86_64")
if (${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION_DEB "amd64")
elseif(${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION_DEB "arm64")
elseif (${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION_DEB "arm64")
endif()
# DEB specific
@@ -39,24 +34,21 @@ set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
"${CMAKE_CURRENT_SOURCE_DIR}/debian/postrm;"
"${CMAKE_CURRENT_SOURCE_DIR}/debian/postinst;")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
# Description formatting is important, summary must be followed with a newline and 1 space.
set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
Contains Memgraph, the graph database. It aims to deliver developers the
speed, simplicity and scale required to build the next generation of
applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6")
# Setting arhitecture extension for rpm packages
set(MG_ARCH_EXTENSION_RPM "noarch")
if(${MG_ARCH} STREQUAL "x86_64")
if (${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION_RPM "x86_64")
elseif(${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION_RPM "aarch64")
elseif (${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION_RPM "aarch64")
endif()
# RPM specific
@@ -64,26 +56,18 @@ set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION_RPM}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/var /var/lib /var/log /etc/logrotate.d
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
/var /var/lib /var/log /etc/logrotate.d
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/rpm/memgraph.spec.in")
set(CPACK_RPM_PACKAGE_LICENSE "Memgraph License")
# Description formatting is important, no line must be greater than 80 characters.
set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
It aims to deliver developers the speed, simplicity and scale required to build
the next generation of applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc++ >= 3.4.29, logrotate")
# If amzn-2
if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")
# It causes issues with glibcxx 2.4
set(CPACK_RPM_PACKAGE_AUTOREQ " no")
endif()
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate")
# All variables must be set before including.
include(CPack)

View File

@@ -1,4 +1,3 @@
.venv
dist
mgp.py
poetry.lock

View File

@@ -1,11 +1,4 @@
# mgp
PyPi package used for type hinting when creating query modules. Repository of already available query modules is called [MAGE](https://github.com/memgraph/mage).
## 🎬 Get started
To learn more, head over to the [docs for the query modules Python API](https://memgraph.com/docs/memgraph/reference-guide/query-modules/api/python-api). To get started with query modules, check out the [how-to guide](https://memgraph.com/docs/memgraph/how-to-guides/query-modules) on Memgraph docs.
## 🔢 Versioning
- mgp v1.1 is compatible with Memgraph >= 2.4.0
PyPi package used for type hinting when creating MAGE modules. The get started
using MAGE repository checkout the repository here: https://github.com/memgraph/mage.

View File

@@ -173,10 +173,6 @@ class SerializationError(Exception):
pass
class AuthorizationError(Exception):
pass
def type_nullable(elem: Any):
pass
@@ -257,11 +253,3 @@ class _MODULE:
@staticmethod
def add_function(wrapper):
pass
class SOURCE_TYPE_KAFKA:
pass
class SOURCE_TYPE_PULSAR:
pass

View File

@@ -1,13 +1,13 @@
[tool.poetry]
name = "mgp"
version = "1.1.1"
version = "1.0.0"
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
authors = [
"katarinasupe <katarina.supe@memgraph.io>",
"MasterMedo <mislav.vuletic@gmail.com>",
"jbajic <jure.bajic@memgraph.io>",
"katarinasupe <katarina.supe@memgraph.io>",
"antejavor <ante.javor@memgraph.io>",
"antaljanosbenjamin <benjamin.antal@memgraph.io>",
"MasterMedo <mislav.vuletic@gmail.com>",
]
license = "Apache-2.0"
readme = "README.md"

View File

@@ -1,14 +0,0 @@
FROM amazonlinux:2
ARG TOOLCHAIN_VERSION
RUN yum -y update \
&& yum install -y wget git tar
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,11 +0,0 @@
version: "3"
services:
debian-11-arm:
build:
context: debian-11-arm
container_name: "mgbuild_debian-11-arm"
ubuntu-2204-arm:
build:
context: ubuntu-22.04-arm
container_name: "mgbuild_ubuntu-22.04-arm"

View File

@@ -28,11 +28,3 @@ services:
build:
context: ubuntu-22.04
container_name: "mgbuild_ubuntu-22.04"
mgbuild_fedora-36:
build:
context: fedora-36
container_name: "mgbuild_fedora-36"
mgbuild_amzn-2:
build:
context: amzn-2
container_name: "mgbuild_amzn-2"

View File

@@ -1,15 +0,0 @@
FROM fedora:36
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
RUN yum -y update \
&& yum install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -3,14 +3,7 @@
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OS=(
centos-7 centos-9
debian-10 debian-11 debian-11-arm
ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
fedora-36
amzn-2
)
SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm)
PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
@@ -30,9 +23,9 @@ make_package () {
echo "Building Memgraph for $os on $build_container..."
package_command=""
if [[ "$os" =~ ^"centos".* ]] || [[ "$os" =~ ^"fedora".* ]] || [[ "$os" =~ ^"amzn".* ]]; then
if [[ "$os" =~ ^"centos".* ]]; then
docker exec "$build_container" bash -c "yum -y update"
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
docker exec "$build_container" bash -c "apt update"
@@ -79,11 +72,9 @@ make_package () {
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
# Fix issue with git marking directory as not safe
docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'"
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
if [[ "$os" =~ "-arm" ]]; then
if [[ "$os" == "debian-11-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."

View File

@@ -1,17 +0,0 @@
FROM ubuntu:22.04
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -30,7 +30,7 @@ BuildRequires: systemd
# This is needed to prevent Python compilation errors when building the RPM
# package
# https://github.com/scylladb/scylla/issues/2235
%if 0%{?rhel} && 0%{?rhel} < 8
%if 0%{?rhel} < 8
%global __os_install_post \
/usr/lib/rpm/redhat/brp-compress \
%{!?__debug_package:\
@@ -40,9 +40,7 @@ BuildRequires: systemd
/usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
%{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \
%{nil}
%endif
%if 0%{?fedora} && 0%{?fedora} < 35
%else
%global __os_install_post \
/usr/lib/rpm/brp-compress \
%{!?__debug_package:\

View File

@@ -1,4 +0,0 @@
# from https://github.com/google/earthenterprise/blob/master/earth_enterprise/rpmlintrc
# We are not packaging log dir
addFilter("E: logrotate-log-dir-not-packaged")

View File

@@ -1,24 +1,27 @@
# CMake configuration for the main memgraph library and executable
# add memgraph sub libraries, ordered by dependency
add_subdirectory(lisp)
add_subdirectory(utils)
add_subdirectory(requests)
add_subdirectory(io)
add_subdirectory(io/simulator)
add_subdirectory(kvstore)
add_subdirectory(telemetry)
add_subdirectory(communication)
add_subdirectory(memory)
add_subdirectory(storage/v2)
add_subdirectory(storage/v3)
add_subdirectory(integrations)
add_subdirectory(query)
add_subdirectory(glue)
add_subdirectory(query/v2)
add_subdirectory(slk)
add_subdirectory(rpc)
add_subdirectory(license)
add_subdirectory(auth)
add_subdirectory(coordinator)
if(MG_ENTERPRISE)
add_subdirectory(audit)
if (MG_ENTERPRISE)
add_subdirectory(audit)
endif()
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -32,98 +35,80 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
# Memgraph Single Node v2 Executable
# ----------------------------------------------------------------------------
set(mg_single_node_v2_sources
memgraph.cpp
glue/communication.cpp
memgraph.cpp
glue/auth.cpp
)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
mg-telemetry mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-glue)
if(MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
endif()
# memgraph main executable
add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph ${mg_single_node_v2_libs})
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the
# symbols in custom procedure module libraries.
target_link_libraries(memgraph "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
set_target_properties(memgraph PROPERTIES
# Set the executable output name to include version information.
OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
# Output the executable in main binary dir.
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Set the executable output name to include version information.
OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
# Output the executable in main binary dir.
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Create symlink to the built executable.
add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph
BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph
COMMENT "Creating symlink to memgraph executable")
COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph
BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph
COMMENT "Creating symlink to memgraph executable")
# Emulate the installed python_support, by creating a symlink
add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support
BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support
COMMENT "Creating symlink for python_support")
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support
BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support
COMMENT "Creating symlink for python_support")
# Strip the executable in release build.
if(lower_build_type STREQUAL "release")
add_custom_command(TARGET memgraph POST_BUILD
COMMAND strip -s $<TARGET_FILE:memgraph>
COMMENT "Stripping symbols and sections from memgraph")
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET memgraph POST_BUILD
COMMAND strip -s $<TARGET_FILE:memgraph>
COMMENT "Stripping symbols and sections from memgraph")
endif()
# Generate the configuration file.
add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_BINARY_DIR}/memgraph
${CMAKE_BINARY_DIR}/config/memgraph.conf
DEPENDS ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_SOURCE_DIR}/config/flags.yaml
BYPRODUCTS ${CMAKE_BINARY_DIR}/config/memgraph.conf
COMMENT "Generating memgraph configuration file")
COMMAND ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_BINARY_DIR}/memgraph
${CMAKE_BINARY_DIR}/config/memgraph.conf
DEPENDS ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_SOURCE_DIR}/config/flags.yaml
BYPRODUCTS ${CMAKE_BINARY_DIR}/config/memgraph.conf
COMMENT "Generating memgraph configuration file")
# Everything here is under "memgraph" install component.
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph")
# TODO: Default directory permissions to 755
# NOTE: This is added in CMake 3.11, so enable it then
# set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
# OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ)
#set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
# OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ)
# Install and rename executable to just 'memgraph' Since we have to rename,
# we cannot use the recommended `install(TARGETS ...)`.
install(PROGRAMS $<TARGET_FILE:memgraph>
DESTINATION lib/memgraph RENAME memgraph)
# Install Python source for supporting our embedded Python.
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.py
DESTINATION lib/memgraph/python_support)
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp_mock.py
DESTINATION lib/memgraph/python_support)
install(FILES ${CMAKE_SOURCE_DIR}/include/_mgp_mock.py
DESTINATION lib/memgraph/python_support)
# Install the includes file for writing custom procedures in C and C++>
# Install the include file for writing custom procedures.
install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/_mgp.hpp
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/mg_exceptions.hpp
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.hpp
DESTINATION include/memgraph)
# Install the config file (must use absolute path).
install(FILES ${CMAKE_BINARY_DIR}/config/memgraph.conf
DESTINATION /etc/memgraph RENAME memgraph.conf)
# Install logrotate configuration (must use absolute path).
install(FILES ${CMAKE_SOURCE_DIR}/release/logrotate.conf
DESTINATION /etc/logrotate.d RENAME memgraph)
@@ -139,14 +124,15 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
# ----------------------------------------------------------------------------
# Memgraph CSV Import Tool Executable
# ----------------------------------------------------------------------------
add_executable(mg_import_csv mg_import_csv.cpp)
target_link_libraries(mg_import_csv mg-storage-v2)
# Strip the executable in release build.
if(lower_build_type STREQUAL "release")
add_custom_command(TARGET mg_import_csv POST_BUILD
COMMAND strip -s mg_import_csv
COMMENT "Stripping symbols and sections from mg_import_csv")
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET mg_import_csv POST_BUILD
COMMAND strip -s mg_import_csv
COMMENT "Stripping symbols and sections from mg_import_csv")
endif()
install(TARGETS mg_import_csv RUNTIME DESTINATION bin)

View File

@@ -16,10 +16,6 @@ target_link_libraries(mg-auth mg-utils mg-kvstore mg-license )
target_link_libraries(mg-auth ${Seccomp_LIBRARIES})
target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS})
find_package(OpenSSL REQUIRED)
target_link_libraries(mg-auth ${OPENSSL_LIBRARIES})
target_include_directories(mg-auth SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
# Install reference auth modules and their configuration files.
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/reference_modules/example.py
DESTINATION lib/memgraph/auth_module)

View File

@@ -16,8 +16,8 @@
#include <fmt/format.h>
#include "auth/exceptions.hpp"
#include "license/license.hpp"
#include "utils/flag_validation.hpp"
#include "utils/license.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/settings.hpp"
@@ -68,9 +68,10 @@ Auth::Auth(const std::string &storage_directory) : storage_(storage_directory),
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
if (module_.IsUsed()) {
const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings);
if (license_check_result.HasError()) {
spdlog::warn(license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
spdlog::warn(
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
return std::nullopt;
}
@@ -225,7 +226,7 @@ std::vector<auth::User> Auth::AllUsers() const {
if (username != utils::ToLowerCase(username)) continue;
auto user = GetUser(username);
if (user) {
ret.push_back(std::move(*user));
ret.push_back(*user);
}
}
return ret;
@@ -305,7 +306,7 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
if (it->second == rolename) {
auto user = GetUser(username);
if (user) {
ret.push_back(std::move(*user));
ret.push_back(*user);
} else {
throw AuthException("Couldn't load user '{}'!", username);
}

View File

@@ -21,7 +21,7 @@
namespace memgraph::auth {
/**
* This class serves as the main Authentication/Authorization storage.
* It provides functions for managing Users, Roles, Permissions and FineGrainedAccessPermissions.
* It provides functions for managing Users, Roles and Permissions.
* NOTE: The non-const functions in this class aren't thread safe.
* TODO (mferencevic): Disable user/role modification functions when they are
* being managed by the auth module.

View File

@@ -1,64 +1,19 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
// this file except in compliance with the License. You may obtain a copy of the License at https://memgraph.com/legal.
//
//
#include "auth/crypto.hpp"
#include <iomanip>
#include <sstream>
#include <gflags/gflags.h>
#include <libbcrypt/bcrypt.h>
#include <openssl/evp.h>
#include <openssl/opensslv.h>
#include <openssl/sha.h>
#include "auth/exceptions.hpp"
#include "utils/enum.hpp"
#include "utils/flag_validation.hpp"
namespace {
using namespace std::literals;
inline constexpr std::array password_encryption_mappings{
std::pair{"bcrypt"sv, memgraph::auth::PasswordEncryptionAlgorithm::BCRYPT},
std::pair{"sha256"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256},
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256_MULTIPLE}};
inline constexpr uint64_t ONE_SHA_ITERATION = 1;
inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
} // namespace
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,misc-unused-parameters)
DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
"The password encryption algorithm used for authentication.", {
if (const auto result =
memgraph::utils::IsValidEnumValueString(value, password_encryption_mappings);
result.HasError()) {
const auto error = result.GetError();
switch (error) {
case memgraph::utils::ValidationError::EmptyValue: {
std::cout << "Password encryption algorithm cannot be empty." << std::endl;
break;
}
case memgraph::utils::ValidationError::InvalidValue: {
std::cout << "Invalid value for password encryption algorithm. Allowed values: "
<< memgraph::utils::GetAllowedEnumValuesString(password_encryption_mappings)
<< std::endl;
break;
}
}
return false;
}
return true;
});
namespace memgraph::auth {
namespace BCrypt {
std::string EncryptPassword(const std::string &password) {
const std::string EncryptPassword(const std::string &password) {
char salt[BCRYPT_HASHSIZE];
char hash[BCRYPT_HASHSIZE];
@@ -73,7 +28,7 @@ std::string EncryptPassword(const std::string &password) {
throw AuthException("Couldn't hash password!");
}
return {hash};
return std::string(hash);
}
bool VerifyPassword(const std::string &password, const std::string &hash) {
@@ -83,102 +38,5 @@ bool VerifyPassword(const std::string &password, const std::string &hash) {
}
return ret == 0;
}
} // namespace BCrypt
namespace SHA {
#if OPENSSL_VERSION_MAJOR >= 3
std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t number_of_iterations) {
unsigned char hash[SHA256_DIGEST_LENGTH];
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
EVP_MD *md = EVP_MD_fetch(nullptr, "SHA2-256", nullptr);
EVP_DigestInit_ex(ctx, md, nullptr);
for (auto i = 0; i < number_of_iterations; i++) {
EVP_DigestUpdate(ctx, password.c_str(), password.size());
}
EVP_DigestFinal_ex(ctx, hash, nullptr);
EVP_MD_free(md);
EVP_MD_CTX_free(ctx);
std::stringstream result_stream;
for (auto hash_char : hash) {
result_stream << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
}
return result_stream.str();
}
#else
std::string EncryptPasswordOpenSSL1_1(const std::string &password, const uint64_t number_of_iterations) {
unsigned char hash[SHA256_DIGEST_LENGTH];
SHA256_CTX sha256;
SHA256_Init(&sha256);
for (auto i = 0; i < number_of_iterations; i++) {
SHA256_Update(&sha256, password.c_str(), password.size());
}
SHA256_Final(hash, &sha256);
std::stringstream ss;
for (auto hash_char : hash) {
ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
}
return ss.str();
}
#endif
std::string EncryptPassword(const std::string &password, const uint64_t number_of_iterations) {
#if OPENSSL_VERSION_MAJOR >= 3
return EncryptPasswordOpenSSL3(password, number_of_iterations);
#else
return EncryptPasswordOpenSSL1_1(password, number_of_iterations);
#endif
}
bool VerifyPassword(const std::string &password, const std::string &hash, const uint64_t number_of_iterations) {
auto password_hash = EncryptPassword(password, number_of_iterations);
return password_hash == hash;
}
} // namespace SHA
bool VerifyPassword(const std::string &password, const std::string &hash) {
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
FLAGS_password_encryption_algorithm, password_encryption_mappings);
if (!password_encryption_algorithm.has_value()) {
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
switch (password_encryption_algorithm.value()) {
case PasswordEncryptionAlgorithm::BCRYPT:
return BCrypt::VerifyPassword(password, hash);
case PasswordEncryptionAlgorithm::SHA256:
return SHA::VerifyPassword(password, hash, ONE_SHA_ITERATION);
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
return SHA::VerifyPassword(password, hash, MULTIPLE_SHA_ITERATIONS);
}
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
std::string EncryptPassword(const std::string &password) {
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
FLAGS_password_encryption_algorithm, password_encryption_mappings);
if (!password_encryption_algorithm.has_value()) {
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
switch (password_encryption_algorithm.value()) {
case PasswordEncryptionAlgorithm::BCRYPT:
return BCrypt::EncryptPassword(password);
case PasswordEncryptionAlgorithm::SHA256:
return SHA::EncryptPassword(password, ONE_SHA_ITERATION);
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
return SHA::EncryptPassword(password, MULTIPLE_SHA_ITERATIONS);
}
}
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -11,11 +11,10 @@
#include <string>
namespace memgraph::auth {
enum class PasswordEncryptionAlgorithm : uint8_t { BCRYPT, SHA256, SHA256_MULTIPLE };
/// @throw AuthException if unable to encrypt the password.
std::string EncryptPassword(const std::string &password);
const std::string EncryptPassword(const std::string &password);
/// @throw AuthException if unable to verify the password.
bool VerifyPassword(const std::string &password, const std::string &hash);
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -8,17 +8,14 @@
#include "auth/models.hpp"
#include <cstdint>
#include <regex>
#include <gflags/gflags.h>
#include "auth/crypto.hpp"
#include "auth/exceptions.hpp"
#include "license/license.hpp"
#include "query/constants.hpp"
#include "utils/cast.hpp"
#include "utils/logging.hpp"
#include "utils/license.hpp"
#include "utils/settings.hpp"
#include "utils/string.hpp"
@@ -33,21 +30,14 @@ DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
namespace memgraph::auth {
namespace {
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permission::CREATE,
Permission::MERGE, Permission::DELETE,
Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS,
Permission::CONSTRAINT, Permission::DUMP,
Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE,
Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM,
Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET, Permission::TRANSACTION_MANAGEMENT,
Permission::STORAGE_MODE};
const std::vector<Permission> kPermissionsAll = {
Permission::MATCH, Permission::CREATE, Permission::MERGE, Permission::DELETE,
Permission::SET, Permission::REMOVE, Permission::INDEX, Permission::STATS,
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET, Permission::SCHEMA};
} // namespace
std::string PermissionToString(Permission permission) {
@@ -94,10 +84,8 @@ std::string PermissionToString(Permission permission) {
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
case Permission::TRANSACTION_MANAGEMENT:
return "TRANSACTION_MANAGEMENT";
case Permission::STORAGE_MODE:
return "STORAGE_MODE";
case Permission::SCHEMA:
return "SCHEMA";
}
}
@@ -112,55 +100,6 @@ std::string PermissionLevelToString(PermissionLevel level) {
}
}
#ifdef MG_ENTERPRISE
FineGrainedPermission PermissionToFineGrainedPermission(const uint64_t permission) {
if (permission & FineGrainedPermission::CREATE_DELETE) {
return FineGrainedPermission::CREATE_DELETE;
}
if (permission & FineGrainedPermission::UPDATE) {
return FineGrainedPermission::UPDATE;
}
if (permission & FineGrainedPermission::READ) {
return FineGrainedPermission::READ;
}
return FineGrainedPermission::NOTHING;
}
std::string FineGrainedPermissionToString(const FineGrainedPermission level) {
switch (level) {
case FineGrainedPermission::CREATE_DELETE:
return "CREATE_DELETE";
case FineGrainedPermission::UPDATE:
return "UPDATE";
case FineGrainedPermission::READ:
return "READ";
case FineGrainedPermission::NOTHING:
return "NOTHING";
}
}
FineGrainedAccessPermissions Merge(const FineGrainedAccessPermissions &first,
const FineGrainedAccessPermissions &second) {
std::unordered_map<std::string, uint64_t> permissions{first.GetPermissions()};
std::optional<uint64_t> global_permission;
if (second.GetGlobalPermission().has_value()) {
global_permission = *second.GetGlobalPermission();
} else if (first.GetGlobalPermission().has_value()) {
global_permission = *first.GetGlobalPermission();
}
for (const auto &[label_name, permission] : second.GetPermissions()) {
permissions[label_name] = permission;
}
return FineGrainedAccessPermissions(permissions, global_permission);
}
#endif
Permissions::Permissions(uint64_t grants, uint64_t denies) {
// The deny bitmask has higher priority than the grant bitmask.
denies_ = denies;
@@ -234,7 +173,7 @@ Permissions Permissions::Deserialize(const nlohmann::json &data) {
if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) {
throw AuthException("Couldn't load permissions data!");
}
return Permissions{data["grants"], data["denies"]};
return {data["grants"], data["denies"]};
}
uint64_t Permissions::grants() const { return grants_; }
@@ -246,191 +185,19 @@ bool operator==(const Permissions &first, const Permissions &second) {
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions::FineGrainedAccessPermissions(const std::unordered_map<std::string, uint64_t> &permissions,
const std::optional<uint64_t> &global_permission)
: permissions_(permissions), global_permission_(global_permission) {}
PermissionLevel FineGrainedAccessPermissions::Has(const std::string &permission,
const FineGrainedPermission fine_grained_permission) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return PermissionLevel::GRANT;
}
const auto concrete_permission = std::invoke([&]() -> uint64_t {
if (permissions_.contains(permission)) {
return permissions_.at(permission);
}
if (global_permission_.has_value()) {
return global_permission_.value();
}
return 0;
});
const auto temp_permission = concrete_permission & fine_grained_permission;
return temp_permission > 0 ? PermissionLevel::GRANT : PermissionLevel::DENY;
}
void FineGrainedAccessPermissions::Grant(const std::string &permission,
const FineGrainedPermission fine_grained_permission) {
if (permission == query::kAsterisk) {
global_permission_ = CalculateGrant(fine_grained_permission);
} else {
permissions_[permission] = CalculateGrant(fine_grained_permission);
}
}
void FineGrainedAccessPermissions::Revoke(const std::string &permission) {
if (permission == query::kAsterisk) {
permissions_.clear();
global_permission_ = std::nullopt;
} else {
permissions_.erase(permission);
}
}
nlohmann::json FineGrainedAccessPermissions::Serialize() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
nlohmann::json data = nlohmann::json::object();
data["permissions"] = permissions_;
data["global_permission"] = global_permission_.has_value() ? global_permission_.value() : -1;
return data;
}
FineGrainedAccessPermissions FineGrainedAccessPermissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!");
}
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
std::optional<uint64_t> global_permission;
if (data["global_permission"].empty() || data["global_permission"] == -1) {
global_permission = std::nullopt;
} else {
global_permission = data["global_permission"];
}
return FineGrainedAccessPermissions(data["permissions"], global_permission);
}
const std::unordered_map<std::string, uint64_t> &FineGrainedAccessPermissions::GetPermissions() const {
return permissions_;
}
const std::optional<uint64_t> &FineGrainedAccessPermissions::GetGlobalPermission() const { return global_permission_; };
uint64_t FineGrainedAccessPermissions::CalculateGrant(FineGrainedPermission fine_grained_permission) {
uint64_t shift{1};
uint64_t result{0};
auto uint_fine_grained_permission = static_cast<uint64_t>(fine_grained_permission);
while (uint_fine_grained_permission > 0) {
result |= uint_fine_grained_permission;
uint_fine_grained_permission >>= shift;
}
return result;
}
bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return first.GetPermissions() == second.GetPermissions() &&
first.GetGlobalPermission() == second.GetGlobalPermission();
}
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return !(first == second);
}
FineGrainedAccessHandler::FineGrainedAccessHandler(FineGrainedAccessPermissions labelPermissions,
FineGrainedAccessPermissions edgeTypePermissions)
: label_permissions_(std::move(labelPermissions)), edge_type_permissions_(std::move(edgeTypePermissions)) {}
const FineGrainedAccessPermissions &FineGrainedAccessHandler::label_permissions() const { return label_permissions_; }
FineGrainedAccessPermissions &FineGrainedAccessHandler::label_permissions() { return label_permissions_; }
const FineGrainedAccessPermissions &FineGrainedAccessHandler::edge_type_permissions() const {
return edge_type_permissions_;
}
FineGrainedAccessPermissions &FineGrainedAccessHandler::edge_type_permissions() { return edge_type_permissions_; }
nlohmann::json FineGrainedAccessHandler::Serialize() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
nlohmann::json data = nlohmann::json::object();
data["label_permissions"] = label_permissions_.Serialize();
data["edge_type_permissions"] = edge_type_permissions_.Serialize();
return data;
}
FineGrainedAccessHandler FineGrainedAccessHandler::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load role data!");
}
if (!data["label_permissions"].is_object() || !data["edge_type_permissions"].is_object()) {
throw AuthException("Couldn't load label_permissions or edge_type_permissions data!");
}
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessHandler{};
}
auto label_permissions = FineGrainedAccessPermissions::Deserialize(data["label_permissions"]);
auto edge_type_permissions = FineGrainedAccessPermissions::Deserialize(data["edge_type_permissions"]);
return FineGrainedAccessHandler(std::move(label_permissions), std::move(edge_type_permissions));
}
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second) {
return first.label_permissions_ == second.label_permissions_ &&
first.edge_type_permissions_ == second.edge_type_permissions_;
}
bool operator!=(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second) {
return !(first == second);
}
#endif
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
Role::Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler)
: rolename_(utils::ToLowerCase(rolename)),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
#endif
const std::string &Role::rolename() const { return rolename_; }
const Permissions &Role::permissions() const { return permissions_; }
Permissions &Role::permissions() { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &Role::fine_grained_access_handler() const { return fine_grained_access_handler_; }
FineGrainedAccessHandler &Role::fine_grained_access_handler() { return fine_grained_access_handler_; }
const FineGrainedAccessPermissions &Role::GetFineGrainedAccessLabelPermissions() const {
return fine_grained_access_handler_.label_permissions();
}
const FineGrainedAccessPermissions &Role::GetFineGrainedAccessEdgeTypePermissions() const {
return fine_grained_access_handler_.edge_type_permissions();
}
#endif
nlohmann::json Role::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["rolename"] = rolename_;
data["permissions"] = permissions_.Serialize();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
} else {
data["fine_grained_access_handler"] = {};
}
#endif
return data;
}
@@ -442,43 +209,18 @@ Role Role::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load role data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
if (!data["fine_grained_access_handler"].is_object()) {
throw AuthException("Couldn't load user data!");
}
auto fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
return {data["rolename"], permissions, std::move(fine_grained_access_handler)};
}
#endif
return {data["rolename"], permissions};
}
bool operator==(const Role &first, const Role &second) {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_ &&
first.fine_grained_access_handler_ == second.fine_grained_access_handler_;
}
#endif
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_;
}
User::User() {}
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler)
: username_(utils::ToLowerCase(username)),
password_hash_(password_hash),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
#endif
bool User::CheckPassword(const std::string &password) {
if (password_hash_.empty()) return true;
return VerifyPassword(password, password_hash_);
@@ -494,13 +236,13 @@ void User::UpdatePassword(const std::optional<std::string> &password) {
}
if (FLAGS_auth_password_strength_regex != default_password_regex) {
if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
if (const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings);
license_check_result.HasError()) {
throw AuthException(
"Custom password regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
default_password_regex,
license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
}
}
std::regex re(FLAGS_auth_password_strength_regex);
@@ -520,47 +262,17 @@ void User::ClearRole() { role_ = std::nullopt; }
Permissions User::GetPermissions() const {
if (role_) {
return Permissions{permissions_.grants() | role_->permissions().grants(),
permissions_.denies() | role_->permissions().denies()};
return Permissions(permissions_.grants() | role_->permissions().grants(),
permissions_.denies() | role_->permissions().denies());
}
return permissions_;
}
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions User::GetFineGrainedAccessLabelPermissions() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
if (role_) {
return Merge(role()->fine_grained_access_handler().label_permissions(),
fine_grained_access_handler_.label_permissions());
}
return fine_grained_access_handler_.label_permissions();
}
FineGrainedAccessPermissions User::GetFineGrainedAccessEdgeTypePermissions() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
if (role_) {
return Merge(role()->fine_grained_access_handler().edge_type_permissions(),
fine_grained_access_handler_.edge_type_permissions());
}
return fine_grained_access_handler_.edge_type_permissions();
}
#endif
const std::string &User::username() const { return username_; }
const Permissions &User::permissions() const { return permissions_; }
Permissions &User::permissions() { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &User::fine_grained_access_handler() const { return fine_grained_access_handler_; }
FineGrainedAccessHandler &User::fine_grained_access_handler() { return fine_grained_access_handler_; }
#endif
const Role *User::role() const {
if (role_.has_value()) {
return &role_.value();
@@ -573,13 +285,6 @@ nlohmann::json User::Serialize() const {
data["username"] = username_;
data["password_hash"] = password_hash_;
data["permissions"] = permissions_.Serialize();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
} else {
data["fine_grained_access_handler"] = {};
}
#endif
// The role shouldn't be serialized here, it is stored as a foreign key.
return data;
}
@@ -592,28 +297,11 @@ User User::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load user data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
if (!data["fine_grained_access_handler"].is_object()) {
throw AuthException("Couldn't load user data!");
}
auto fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
return {data["username"], data["password_hash"], permissions, fine_grained_access_handler};
}
#endif
return {data["username"], data["password_hash"], permissions};
}
bool operator==(const User &first, const User &second) {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_ &&
first.fine_grained_access_handler_ == second.fine_grained_access_handler_;
}
#endif
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_;
}
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -10,7 +10,6 @@
#include <optional>
#include <string>
#include <unordered_map>
#include <json/json.hpp>
@@ -40,66 +39,26 @@ enum class Permission : uint64_t {
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U,
TRANSACTION_MANAGEMENT = 1U << 21U,
STORAGE_MODE = 1U << 22U
SCHEMA = 1U << 21U
};
// clang-format on
#ifdef MG_ENTERPRISE
// clang-format off
enum class FineGrainedPermission : uint64_t {
NOTHING = 0,
READ = 1,
UPDATE = 1U << 1U,
CREATE_DELETE = 1U << 2U
};
// clang-format on
constexpr inline uint64_t operator|(FineGrainedPermission lhs, FineGrainedPermission rhs) {
return static_cast<uint64_t>(lhs) | static_cast<uint64_t>(rhs);
}
constexpr inline uint64_t operator|(uint64_t lhs, FineGrainedPermission rhs) {
return lhs | static_cast<uint64_t>(rhs);
}
constexpr inline uint64_t operator&(uint64_t lhs, FineGrainedPermission rhs) {
return (lhs & static_cast<uint64_t>(rhs)) != 0;
}
constexpr uint64_t kLabelPermissionAll = memgraph::auth::FineGrainedPermission::CREATE_DELETE |
memgraph::auth::FineGrainedPermission::UPDATE |
memgraph::auth::FineGrainedPermission::READ;
constexpr uint64_t kLabelPermissionMax = static_cast<uint64_t>(memgraph::auth::FineGrainedPermission::CREATE_DELETE);
constexpr uint64_t kLabelPermissionMin = static_cast<uint64_t>(memgraph::auth::FineGrainedPermission::READ);
#endif
// Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission);
// Class that indicates what permission level the user/role has.
enum class PermissionLevel : uint8_t { GRANT, NEUTRAL, DENY };
enum class PermissionLevel {
GRANT,
NEUTRAL,
DENY,
};
// Function that converts a permission level to its string representation.
std::string PermissionLevelToString(PermissionLevel level);
#ifdef MG_ENTERPRISE
// Function that converts a label permission level to its string representation.
std::string FineGrainedPermissionToString(FineGrainedPermission level);
// Constructs a label permission from a permission
FineGrainedPermission PermissionToFineGrainedPermission(uint64_t permission);
#endif
class Permissions final {
public:
explicit Permissions(uint64_t grants = 0, uint64_t denies = 0);
Permissions(const Permissions &) = default;
Permissions &operator=(const Permissions &) = default;
Permissions(Permissions &&) noexcept = default;
Permissions &operator=(Permissions &&) noexcept = default;
~Permissions() = default;
Permissions(uint64_t grants = 0, uint64_t denies = 0);
PermissionLevel Has(Permission permission) const;
@@ -130,96 +89,16 @@ bool operator==(const Permissions &first, const Permissions &second);
bool operator!=(const Permissions &first, const Permissions &second);
#ifdef MG_ENTERPRISE
class FineGrainedAccessPermissions final {
public:
explicit FineGrainedAccessPermissions(const std::unordered_map<std::string, uint64_t> &permissions = {},
const std::optional<uint64_t> &global_permission = std::nullopt);
FineGrainedAccessPermissions(const FineGrainedAccessPermissions &) = default;
FineGrainedAccessPermissions &operator=(const FineGrainedAccessPermissions &) = default;
FineGrainedAccessPermissions(FineGrainedAccessPermissions &&) = default;
FineGrainedAccessPermissions &operator=(FineGrainedAccessPermissions &&) = default;
~FineGrainedAccessPermissions() = default;
PermissionLevel Has(const std::string &permission, FineGrainedPermission fine_grained_permission) const;
void Grant(const std::string &permission, FineGrainedPermission fine_grained_permission);
void Revoke(const std::string &permission);
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static FineGrainedAccessPermissions Deserialize(const nlohmann::json &data);
const std::unordered_map<std::string, uint64_t> &GetPermissions() const;
const std::optional<uint64_t> &GetGlobalPermission() const;
private:
std::unordered_map<std::string, uint64_t> permissions_{};
std::optional<uint64_t> global_permission_;
static uint64_t CalculateGrant(FineGrainedPermission fine_grained_permission);
};
bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
class FineGrainedAccessHandler final {
public:
explicit FineGrainedAccessHandler(FineGrainedAccessPermissions labelPermissions = FineGrainedAccessPermissions(),
FineGrainedAccessPermissions edgeTypePermissions = FineGrainedAccessPermissions());
FineGrainedAccessHandler(const FineGrainedAccessHandler &) = default;
FineGrainedAccessHandler &operator=(const FineGrainedAccessHandler &) = default;
FineGrainedAccessHandler(FineGrainedAccessHandler &&) noexcept = default;
FineGrainedAccessHandler &operator=(FineGrainedAccessHandler &&) noexcept = default;
~FineGrainedAccessHandler() = default;
const FineGrainedAccessPermissions &label_permissions() const;
FineGrainedAccessPermissions &label_permissions();
const FineGrainedAccessPermissions &edge_type_permissions() const;
FineGrainedAccessPermissions &edge_type_permissions();
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static FineGrainedAccessHandler Deserialize(const nlohmann::json &data);
friend bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
private:
FineGrainedAccessPermissions label_permissions_;
FineGrainedAccessPermissions edge_type_permissions_;
};
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
#endif
class Role final {
public:
explicit Role(const std::string &rolename);
Role(const std::string &rolename);
Role(const std::string &rolename, const Permissions &permissions);
#ifdef MG_ENTERPRISE
Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler);
#endif
Role(const Role &) = default;
Role &operator=(const Role &) = default;
Role(Role &&) noexcept = default;
Role &operator=(Role &&) noexcept = default;
~Role() = default;
const std::string &rolename() const;
const Permissions &permissions() const;
Permissions &permissions();
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
const FineGrainedAccessPermissions &GetFineGrainedAccessLabelPermissions() const;
const FineGrainedAccessPermissions &GetFineGrainedAccessEdgeTypePermissions() const;
#endif
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
@@ -230,9 +109,6 @@ class Role final {
private:
std::string rolename_;
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
#endif
};
bool operator==(const Role &first, const Role &second);
@@ -240,19 +116,9 @@ bool operator==(const Role &first, const Role &second);
// TODO (mferencevic): Implement password expiry.
class User final {
public:
User();
User(const std::string &username);
explicit User(const std::string &username);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
#ifdef MG_ENTERPRISE
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler);
#endif
User(const User &) = default;
User &operator=(const User &) = default;
User(User &&) noexcept = default;
User &operator=(User &&) noexcept = default;
~User() = default;
/// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password);
@@ -266,12 +132,6 @@ class User final {
Permissions GetPermissions() const;
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions GetFineGrainedAccessLabelPermissions() const;
FineGrainedAccessPermissions GetFineGrainedAccessEdgeTypePermissions() const;
const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
#endif
const std::string &username() const;
const Permissions &permissions() const;
@@ -290,16 +150,8 @@ class User final {
std::string username_;
std::string password_hash_;
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
#endif
std::optional<Role> role_;
};
bool operator==(const User &first, const User &second);
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions Merge(const FineGrainedAccessPermissions &first,
const FineGrainedAccessPermissions &second);
#endif
} // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -9,10 +9,11 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/frontend/semantic/symbol.hpp"
#include "utils/typeinfo.hpp"
#pragma once
namespace memgraph {
#include <cstdint>
constexpr utils::TypeInfo query::Symbol::kType{utils::TypeId::SYMBOL, "Symbol", nullptr};
} // namespace memgraph
namespace memgraph::common {
enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
} // namespace memgraph::common

View File

@@ -7,7 +7,6 @@ set(communication_src_files
websocket/listener.cpp
websocket/session.cpp
bolt/v1/value.cpp
bolt/client.cpp
buffer.cpp
client.cpp
context.cpp

View File

@@ -1,267 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "communication/bolt/client.hpp"
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/value.hpp"
#include "utils/logging.hpp"
namespace {
constexpr uint8_t kBoltV43Version[4] = {0x00, 0x00, 0x03, 0x04};
constexpr uint8_t kEmptyBoltVersion[4] = {0x00, 0x00, 0x00, 0x00};
} // namespace
namespace memgraph::communication::bolt {
Client::Client(communication::ClientContext &context) : client_{&context} {}
void Client::Connect(const io::network::Endpoint &endpoint, const std::string &username, const std::string &password,
const std::string &client_name) {
if (!client_.Connect(endpoint)) {
throw ClientFatalException("Couldn't connect to {}!", endpoint);
}
if (!client_.Write(kPreamble, sizeof(kPreamble), true)) {
spdlog::error("Couldn't send preamble!");
throw ServerCommunicationException();
}
if (!client_.Write(kBoltV43Version, sizeof(kBoltV43Version), true)) {
spdlog::error("Couldn't send protocol version!");
throw ServerCommunicationException();
}
for (int i = 0; i < 3; ++i) {
if (!client_.Write(kEmptyBoltVersion, sizeof(kEmptyBoltVersion), i != 2)) {
spdlog::error("Couldn't send protocol version!");
throw ServerCommunicationException();
}
}
if (!client_.Read(sizeof(kBoltV43Version))) {
spdlog::error("Couldn't get negotiated protocol version!");
throw ServerCommunicationException();
}
if (memcmp(kBoltV43Version, client_.GetData(), sizeof(kBoltV43Version)) != 0) {
spdlog::error("Server negotiated unsupported protocol version!");
throw ClientFatalException("The server negotiated an usupported protocol version!");
}
client_.ShiftData(sizeof(kBoltV43Version));
if (!encoder_.MessageInit({{"user_agent", client_name},
{"scheme", "basic"},
{"principal", username},
{"credentials", password},
{"routing", {}}})) {
spdlog::error("Couldn't send init message!");
throw ServerCommunicationException();
}
Signature signature{};
Value metadata;
if (!ReadMessage(signature, metadata)) {
spdlog::error("Couldn't read init message response!");
throw ServerCommunicationException();
}
if (signature != Signature::Success) {
spdlog::error("Handshake failed!");
throw ClientFatalException("Handshake with the server failed!");
}
spdlog::debug("Metadata of init message response: {}", metadata);
}
QueryData Client::Execute(const std::string &query, const std::map<std::string, Value> &parameters) {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
spdlog::debug("Sending run message with statement: '{}'; parameters: {}", query, parameters);
// It is super critical from performance point of view to send the pull message right after the run message. Otherwise
// the performance will degrade multiple magnitudes.
encoder_.MessageRun(query, parameters, {});
encoder_.MessagePull({{"n", Value(-1)}});
spdlog::debug("Reading run message response");
Signature signature{};
Value fields;
if (!ReadMessage(signature, fields)) {
throw ServerCommunicationException();
}
if (fields.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
if (signature == Signature::Failure) {
HandleFailure<ClientQueryException>(fields.ValueMap());
}
if (signature != Signature::Success) {
throw ServerMalformedDataException();
}
spdlog::debug("Reading pull_all message response");
Marker marker{};
Value metadata;
std::vector<std::vector<Value>> records;
while (true) {
if (!GetMessage()) {
throw ServerCommunicationException();
}
if (!decoder_.ReadMessageHeader(&signature, &marker)) {
throw ServerCommunicationException();
}
if (signature == Signature::Record) {
Value record;
if (!decoder_.ReadValue(&record, Value::Type::List)) {
throw ServerCommunicationException();
}
records.emplace_back(std::move(record.ValueList()));
} else if (signature == Signature::Success) {
if (!decoder_.ReadValue(&metadata)) {
throw ServerCommunicationException();
}
break;
} else if (signature == Signature::Failure) {
Value data;
if (!decoder_.ReadValue(&data)) {
throw ServerCommunicationException();
}
HandleFailure<ClientQueryException>(data.ValueMap());
} else {
throw ServerMalformedDataException();
}
}
if (metadata.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
auto &header = fields.ValueMap();
QueryData ret{{}, std::move(records), std::move(metadata.ValueMap())};
if (header.find("fields") == header.end()) {
throw ServerMalformedDataException();
}
if (header["fields"].type() != Value::Type::List) {
throw ServerMalformedDataException();
}
auto &field_vector = header["fields"].ValueList();
for (auto &field_item : field_vector) {
if (field_item.type() != Value::Type::String) {
throw ServerMalformedDataException();
}
ret.fields.emplace_back(std::move(field_item.ValueString()));
}
if (header.contains("qid")) {
ret.metadata["qid"] = header["qid"];
}
return ret;
}
void Client::Reset() {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
spdlog::debug("Sending reset message");
encoder_.MessageReset();
Signature signature{};
Value fields;
// In Execute the pull message is sent right after the run message without reading the answer for the run message.
// That means some of the messages sent might get ignored.
while (true) {
if (!ReadMessage(signature, fields)) {
throw ServerCommunicationException();
}
if (signature == Signature::Success) {
break;
}
if (signature != Signature::Ignored) {
throw ServerMalformedDataException();
}
}
}
std::optional<std::map<std::string, Value>> Client::Route(const std::map<std::string, Value> &routing,
const std::vector<Value> &bookmarks,
const std::optional<std::string> &db) {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
spdlog::debug("Sending route message with routing: {}; bookmarks: {}; db: {}", routing, bookmarks,
db.has_value() ? *db : Value());
encoder_.MessageRoute(routing, bookmarks, db);
spdlog::debug("Reading route message response");
Signature signature{};
Value fields;
if (!ReadMessage(signature, fields)) {
throw ServerCommunicationException();
}
if (signature == Signature::Ignored) {
return std::nullopt;
}
if (signature == Signature::Failure) {
HandleFailure(fields.ValueMap());
}
if (signature != Signature::Success) {
throw ServerMalformedDataException{};
}
return fields.ValueMap();
}
void Client::Close() { client_.Close(); };
bool Client::GetMessage() {
client_.ClearData();
while (true) {
if (!client_.Read(kChunkHeaderSize)) return false;
size_t chunk_size = client_.GetData()[0];
chunk_size <<= 8U;
chunk_size += client_.GetData()[1];
if (chunk_size == 0) return true;
if (!client_.Read(chunk_size)) return false;
if (decoder_buffer_.GetChunk() != ChunkState::Whole) return false;
client_.ClearData();
}
return true;
}
bool Client::ReadMessage(Signature &signature, Value &ret) {
Marker marker{};
if (!GetMessage()) return false;
if (!decoder_.ReadMessageHeader(&signature, &marker)) return false;
return ReadMessageData(marker, ret);
}
bool Client::ReadMessageData(Marker marker, Value &ret) {
if (marker == Marker::TinyStruct) {
ret = Value();
return true;
}
if (marker == Marker::TinyStruct1) {
return decoder_.ReadValue(&ret);
}
return false;
}
} // namespace memgraph::communication::bolt

View File

@@ -11,12 +11,6 @@
#pragma once
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/decoder/chunked_decoder_buffer.hpp"
#include "communication/bolt/v1/decoder/decoder.hpp"
#include "communication/bolt/v1/encoder/chunked_encoder_buffer.hpp"
@@ -25,17 +19,22 @@
#include "communication/context.hpp"
#include "io/network/endpoint.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
namespace memgraph::communication::bolt {
class FailureResponseException : public utils::BasicException {
/// This exception is thrown whenever an error occurs during query execution
/// that isn't fatal (eg. mistyped query or some transient error occurred).
/// It should be handled by everyone who uses the client.
class ClientQueryException : public utils::BasicException {
public:
FailureResponseException() : utils::BasicException{"Couldn't execute query!"} {}
using utils::BasicException::BasicException;
explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {}
ClientQueryException() : utils::BasicException("Couldn't execute query!") {}
FailureResponseException(const std::string &code, const std::string &message)
: utils::BasicException{message}, code_{code} {}
template <class... Args>
ClientQueryException(const std::string &code, Args &&...args)
: utils::BasicException(std::forward<Args>(args)...), code_(code) {}
const std::string &code() const { return code_; }
@@ -43,14 +42,6 @@ class FailureResponseException : public utils::BasicException {
std::string code_;
};
/// This exception is thrown whenever an error occurs during query execution
/// that isn't fatal (eg. mistyped query or some transient error occurred).
/// It should be handled by everyone who uses the client.
class ClientQueryException : public FailureResponseException {
public:
using FailureResponseException::FailureResponseException;
};
/// This exception is thrown whenever a fatal error occurs during query
/// execution and/or connecting to the server.
/// It should be handled by everyone who uses the client.
@@ -85,13 +76,12 @@ struct QueryData {
/// server. It supports both SSL and plaintext connections.
class Client final {
public:
explicit Client(communication::ClientContext &context);
explicit Client(communication::ClientContext *context) : client_(context) {}
Client(const Client &) = delete;
Client(Client &&) = delete;
Client &operator=(const Client &) = delete;
Client &operator=(Client &&) = delete;
~Client() = default;
/// Method used to connect to the server. Before executing queries this method
/// should be called to set-up the connection to the server. After the
@@ -99,7 +89,50 @@ class Client final {
/// established connection.
/// @throws ClientFatalException when we couldn't connect to the server
void Connect(const io::network::Endpoint &endpoint, const std::string &username, const std::string &password,
const std::string &client_name = "memgraph-bolt");
const std::string &client_name = "memgraph-bolt") {
if (!client_.Connect(endpoint)) {
throw ClientFatalException("Couldn't connect to {}!", endpoint);
}
if (!client_.Write(kPreamble, sizeof(kPreamble), true)) {
SPDLOG_ERROR("Couldn't send preamble!");
throw ServerCommunicationException();
}
for (int i = 0; i < 4; ++i) {
if (!client_.Write(kProtocol, sizeof(kProtocol), i != 3)) {
SPDLOG_ERROR("Couldn't send protocol version!");
throw ServerCommunicationException();
}
}
if (!client_.Read(sizeof(kProtocol))) {
SPDLOG_ERROR("Couldn't get negotiated protocol version!");
throw ServerCommunicationException();
}
if (memcmp(kProtocol, client_.GetData(), sizeof(kProtocol)) != 0) {
SPDLOG_ERROR("Server negotiated unsupported protocol version!");
throw ClientFatalException("The server negotiated an usupported protocol version!");
}
client_.ShiftData(sizeof(kProtocol));
if (!encoder_.MessageInit(client_name, {{"scheme", "basic"}, {"principal", username}, {"credentials", password}})) {
SPDLOG_ERROR("Couldn't send init message!");
throw ServerCommunicationException();
}
Signature signature;
Value metadata;
if (!ReadMessage(&signature, &metadata)) {
SPDLOG_ERROR("Couldn't read init message response!");
throw ServerCommunicationException();
}
if (signature != Signature::Success) {
SPDLOG_ERROR("Handshake failed!");
throw ClientFatalException("Handshake with the server failed!");
}
SPDLOG_INFO("Metadata of init message response: {}", metadata);
}
/// Function used to execute queries against the server. Before you can
/// execute queries you must connect the client to the server.
@@ -107,40 +140,167 @@ class Client final {
/// executing the query (eg. mistyped query,
/// etc.)
/// @throws ClientFatalException when we couldn't communicate with the server
QueryData Execute(const std::string &query, const std::map<std::string, Value> &parameters);
/// Close the active client connection.
void Close();
/// Can be used to reset the active client connection. Reset is automatically sent after receiving a failure message
/// from the server, which result in throwing an FailureResponseException or any exception derived from it.
void Reset();
/// Can be used to send a route message.
std::optional<std::map<std::string, Value>> Route(const std::map<std::string, Value> &routing,
const std::vector<Value> &bookmarks,
const std::optional<std::string> &db);
private:
using ClientEncoder = ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>>;
template <typename TException = FailureResponseException>
[[noreturn]] void HandleFailure(const std::map<std::string, Value> &response_map) {
Reset();
auto it = response_map.find("message");
if (it != response_map.end()) {
auto it_code = response_map.find("code");
if (it_code != response_map.end()) {
throw TException(it_code->second.ValueString(), it->second.ValueString());
}
throw TException("", it->second.ValueString());
QueryData Execute(const std::string &query, const std::map<std::string, Value> &parameters) {
if (!client_.IsConnected()) {
throw ClientFatalException("You must first connect to the server before using the client!");
}
throw TException();
SPDLOG_INFO("Sending run message with statement: '{}'; parameters: {}", query, parameters);
encoder_.MessageRun(query, parameters);
encoder_.MessagePullAll();
SPDLOG_INFO("Reading run message response");
Signature signature;
Value fields;
if (!ReadMessage(&signature, &fields)) {
throw ServerCommunicationException();
}
if (fields.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
if (signature == Signature::Failure) {
HandleFailure();
auto &tmp = fields.ValueMap();
auto it = tmp.find("message");
if (it != tmp.end()) {
auto it_code = tmp.find("code");
if (it_code != tmp.end()) {
throw ClientQueryException(it_code->second.ValueString(), it->second.ValueString());
} else {
throw ClientQueryException("", it->second.ValueString());
}
}
throw ClientQueryException();
} else if (signature != Signature::Success) {
throw ServerMalformedDataException();
}
SPDLOG_INFO("Reading pull_all message response");
Marker marker;
Value metadata;
std::vector<std::vector<Value>> records;
while (true) {
if (!GetMessage()) {
throw ServerCommunicationException();
}
if (!decoder_.ReadMessageHeader(&signature, &marker)) {
throw ServerCommunicationException();
}
if (signature == Signature::Record) {
Value record;
if (!decoder_.ReadValue(&record, Value::Type::List)) {
throw ServerCommunicationException();
}
records.emplace_back(std::move(record.ValueList()));
} else if (signature == Signature::Success) {
if (!decoder_.ReadValue(&metadata)) {
throw ServerCommunicationException();
}
break;
} else if (signature == Signature::Failure) {
Value data;
if (!decoder_.ReadValue(&data)) {
throw ServerCommunicationException();
}
HandleFailure();
auto &tmp = data.ValueMap();
auto it = tmp.find("message");
if (it != tmp.end()) {
auto it_code = tmp.find("code");
if (it_code != tmp.end()) {
throw ClientQueryException(it_code->second.ValueString(), it->second.ValueString());
} else {
throw ClientQueryException("", it->second.ValueString());
}
}
throw ClientQueryException();
} else {
throw ServerMalformedDataException();
}
}
if (metadata.type() != Value::Type::Map) {
throw ServerMalformedDataException();
}
QueryData ret{{}, std::move(records), std::move(metadata.ValueMap())};
auto &header = fields.ValueMap();
if (header.find("fields") == header.end()) {
throw ServerMalformedDataException();
}
if (header["fields"].type() != Value::Type::List) {
throw ServerMalformedDataException();
}
auto &field_vector = header["fields"].ValueList();
for (auto &field_item : field_vector) {
if (field_item.type() != Value::Type::String) {
throw ServerMalformedDataException();
}
ret.fields.emplace_back(std::move(field_item.ValueString()));
}
return ret;
}
bool GetMessage();
bool ReadMessage(Signature &signature, Value &ret);
bool ReadMessageData(Marker marker, Value &ret);
/// Close the active client connection.
void Close() { client_.Close(); };
private:
bool GetMessage() {
client_.ClearData();
while (true) {
if (!client_.Read(kChunkHeaderSize)) return false;
size_t chunk_size = client_.GetData()[0];
chunk_size <<= 8;
chunk_size += client_.GetData()[1];
if (chunk_size == 0) return true;
if (!client_.Read(chunk_size)) return false;
if (decoder_buffer_.GetChunk() != ChunkState::Whole) return false;
client_.ClearData();
}
return true;
}
bool ReadMessage(Signature *signature, Value *ret) {
Marker marker;
if (!GetMessage()) return false;
if (!decoder_.ReadMessageHeader(signature, &marker)) return false;
return ReadMessageData(marker, ret);
}
bool ReadMessageData(Marker marker, Value *ret) {
if (marker == Marker::TinyStruct) {
*ret = Value();
return true;
} else if (marker == Marker::TinyStruct1) {
return decoder_.ReadValue(ret);
}
return false;
}
void HandleFailure() {
if (!encoder_.MessageAckFailure()) {
throw ServerCommunicationException();
}
while (true) {
Signature signature;
Value data;
if (!ReadMessage(&signature, &data)) {
throw ServerCommunicationException();
}
if (signature == Signature::Success) {
break;
} else if (signature != Signature::Ignored) {
throw ServerMalformedDataException();
}
}
}
// client
communication::Client client_;
@@ -153,6 +313,6 @@ class Client final {
// encoder objects
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_};
ClientEncoder encoder_{encoder_buffer_};
ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>> encoder_{encoder_buffer_};
};
} // namespace memgraph::communication::bolt

View File

@@ -16,6 +16,7 @@
namespace memgraph::communication::bolt {
inline constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
inline constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
enum class Signature : uint8_t {
Noop = 0x00,

View File

@@ -11,11 +11,6 @@
#pragma once
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/encoder/base_encoder.hpp"
@@ -35,7 +30,6 @@ class ClientEncoder : private BaseEncoder<Buffer> {
using BaseEncoder<Buffer>::WriteList;
using BaseEncoder<Buffer>::WriteMap;
using BaseEncoder<Buffer>::WriteString;
using BaseEncoder<Buffer>::WriteNull;
using BaseEncoder<Buffer>::buffer_;
public:
@@ -44,9 +38,10 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a Init message.
*
* From the Bolt v4.3 documentation:
* HelloMess (signature=0x01) {
* Map<String,Value> extra
* From the Bolt v1 documentation:
* InitMessage (signature=0x01) {
* String clientName
* Map<String,Value> authToken
* }
*
* @param client_name the name of the connected client
@@ -54,10 +49,11 @@ class ClientEncoder : private BaseEncoder<Buffer> {
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageInit(const std::map<std::string, Value> &extra) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
bool MessageInit(const std::string client_name, const std::map<std::string, Value> &auth_token) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
WriteRAW(utils::UnderlyingCast(Signature::Init));
WriteMap(extra);
WriteString(client_name);
WriteMap(auth_token);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -68,11 +64,10 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a Run message.
*
* From the Bolt v4.3 documentation:
* From the Bolt v1 documentation:
* RunMessage (signature=0x10) {
* String statement
* Map<String,Value> parameters
* Map<String,Value> extra
* String statement
* Map<String,Value> parameters
* }
*
* @param statement the statement that should be executed
@@ -80,13 +75,11 @@ class ClientEncoder : private BaseEncoder<Buffer> {
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageRun(const std::string &statement, const std::map<std::string, Value> &parameters,
const std::map<std::string, Value> &extra, bool have_more = true) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct3));
bool MessageRun(const std::string &statement, const std::map<std::string, Value> &parameters, bool have_more = true) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
WriteRAW(utils::UnderlyingCast(Signature::Run));
WriteString(statement);
WriteMap(parameters);
WriteMap(extra);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -97,20 +90,18 @@ class ClientEncoder : private BaseEncoder<Buffer> {
}
/**
* Writes a Discard message.
* Writes a DiscardAll message.
*
* From the Bolt v4.3 documentation:
* From the Bolt v1 documentation:
* DiscardMessage (signature=0x2F) {
* Map<String,Value> extra
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageDiscard(const std::map<std::string, Value> &extra) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
bool MessageDiscardAll() {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
WriteRAW(utils::UnderlyingCast(Signature::Discard));
WriteMap(extra);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -121,18 +112,36 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a PullAll message.
*
* From the Bolt v4.3 documentation:
* PullMessage (signature=0x3F) {
* Map<String,Value> extra
* From the Bolt v1 documentation:
* PullAllMessage (signature=0x3F) {
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessagePull(const std::map<std::string, Value> &extra) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
bool MessagePullAll() {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
WriteRAW(utils::UnderlyingCast(Signature::Pull));
WriteMap(extra);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
/**
* Writes a AckFailure message.
*
* From the Bolt v1 documentation:
* AckFailureMessage (signature=0x0E) {
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageAckFailure() {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct));
WriteRAW(utils::UnderlyingCast(Signature::AckFailure));
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
@@ -143,7 +152,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
/**
* Writes a Reset message.
*
* From the Bolt v4.3 documentation:
* From the Bolt v1 documentation:
* ResetMessage (signature=0x0F) {
* }
*
@@ -159,36 +168,5 @@ class ClientEncoder : private BaseEncoder<Buffer> {
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
/**
* Writes a Route message.
*
* From the Bolt v4.3 documentation:
* RouteMessage (signature=0x0F) {
* Map<String,Value> routing
* List<String> bookmarks
* String db
* }
*
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageRoute(const std::map<std::string, Value> &routing, const std::vector<Value> &bookmarks,
const std::optional<std::string> &db) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct3));
WriteRAW(utils::UnderlyingCast(Signature::Route));
WriteMap(routing);
WriteList(bookmarks);
if (db.has_value()) {
WriteString(*db);
} else {
WriteNull();
}
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
};
} // namespace memgraph::communication::bolt

View File

@@ -117,6 +117,29 @@ class Encoder : private BaseEncoder<Buffer> {
return buffer_.Flush();
}
/**
* Sends an Ignored message.
*
* From the bolt v1 documentation:
* IgnoredMessage (signature=0x7E) {
* Map<String,Value> metadata
* }
*
* @param metadata the metadata map object that should be sent
* @returns true if the data was successfully sent to the client,
* false otherwise
*/
bool MessageIgnored(const std::map<std::string, Value> &metadata) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct1));
WriteRAW(utils::UnderlyingCast(Signature::Ignored));
WriteMap(metadata);
// Try to flush all remaining data in the buffer, but tell it that we will
// send more data (the end of message chunk).
if (!buffer_.Flush(true)) return false;
// Flush an empty chunk to indicate that the message is done.
return buffer_.Flush();
}
/**
* Sends an Ignored message.
*

View File

@@ -15,7 +15,6 @@
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/states/handlers.hpp"
#include "communication/bolt/v1/value.hpp"
#include "utils/cast.hpp"
#include "utils/likely.hpp"
@@ -31,8 +30,8 @@ namespace memgraph::communication::bolt {
*/
template <typename TSession>
State StateErrorRun(TSession &session, State state) {
Marker marker{};
Signature signature{};
Marker marker;
Signature signature;
if (!session.decoder_.ReadMessageHeader(&signature, &marker)) {
spdlog::trace("Missing header data!");
return State::Close;
@@ -46,49 +45,54 @@ State StateErrorRun(TSession &session, State state) {
// Clear the data buffer if it has any leftover data.
session.encoder_buffer_.Clear();
if (session.version_.major == 1 && signature == Signature::AckFailure) {
spdlog::trace("AckFailure received");
if ((session.version_.major == 1 && signature == Signature::AckFailure) || signature == Signature::Reset) {
if (signature == Signature::AckFailure) {
spdlog::trace("AckFailure received");
} else {
spdlog::trace("Reset received");
}
if (!session.encoder_.MessageSuccess()) {
spdlog::trace("Couldn't send success message!");
return State::Close;
}
if (signature == Signature::Reset) {
session.Abort();
return State::Idle;
}
// We got AckFailure get back to right state.
MG_ASSERT(state == State::Error, "Shouldn't happen");
return State::Idle;
}
if (signature == Signature::Reset) {
spdlog::trace("Reset received");
return HandleReset(session, marker);
}
} else {
uint8_t value = utils::UnderlyingCast(marker);
uint8_t value = utils::UnderlyingCast(marker);
// All bolt client messages have less than 15 parameters so if we receive
// anything than a TinyStruct it's an error.
if ((value & 0xF0U) != utils::UnderlyingCast(Marker::TinyStruct)) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", value);
return State::Close;
}
// We need to clean up all parameters from this command.
value &= 0x0FU; // The length is stored in the lower nibble.
Value dv;
for (int i = 0; i < value; ++i) {
if (!session.decoder_.ReadValue(&dv)) {
spdlog::trace("Couldn't clean up parameter {} / {}!", i, value);
// All bolt client messages have less than 15 parameters so if we receive
// anything than a TinyStruct it's an error.
if ((value & 0xF0) != utils::UnderlyingCast(Marker::TinyStruct)) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", value);
return State::Close;
}
}
// Ignore this message.
if (!session.encoder_.MessageIgnored()) {
spdlog::trace("Couldn't send ignored message!");
return State::Close;
}
// We need to clean up all parameters from this command.
value &= 0x0F; // The length is stored in the lower nibble.
Value dv;
for (int i = 0; i < value; ++i) {
if (!session.decoder_.ReadValue(&dv)) {
spdlog::trace("Couldn't clean up parameter {} / {}!", i, value);
return State::Close;
}
}
// Cleanup done, command ignored, stay in error state.
return state;
// Ignore this message.
if (!session.encoder_.MessageIgnored()) {
spdlog::trace("Couldn't send ignored message!");
return State::Close;
}
// Cleanup done, command ignored, stay in error state.
return state;
}
}
} // namespace memgraph::communication::bolt

View File

@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
}
case Signature::Route: {
if constexpr (bolt_minor >= 3) {
if (signature == Signature::Route) return HandleRoute<TSession>(session, marker);
if (signature == Signature::Route) return HandleRoute<TSession>(session);
} else {
spdlog::trace("Supported only in bolt v4.3");
return State::Close;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,6 @@
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/bolt/v1/exceptions.hpp"
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
@@ -73,6 +72,40 @@ inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::ex
namespace details {
template <typename TSession>
State HandleRun(TSession &session, const State state, const Value &query, const Value &params) {
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <bool is_pull, typename TSession>
State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<int> qid) {
try {
@@ -103,7 +136,7 @@ template <bool is_pull, typename TSession>
State HandlePullDiscardV1(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct;
if (marker != expected_marker) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -124,7 +157,7 @@ template <bool is_pull, typename TSession>
State HandlePullDiscardV4(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct1;
if (marker != expected_marker) {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct1", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -183,8 +216,7 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query;
Value params;
Value query, params;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
@@ -195,48 +227,17 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
return State::Close;
}
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
return details::HandleRun(session, state, query, params);
}
template <typename TSession>
State HandleRunV4(TSession &session, const State state, const Marker marker) {
const auto expected_marker = Marker::TinyStruct3;
if (marker != expected_marker) {
spdlog::trace("Expected TinyStruct3 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace("Expected {} marker, but received 0x{:02X}!", "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
Value query;
Value params;
Value extra;
Value query, params, extra;
if (!session.decoder_.ReadValue(&query, Value::Type::String)) {
spdlog::trace("Couldn't read query string!");
return State::Close;
@@ -252,40 +253,7 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
spdlog::trace("Couldn't read extra field!");
}
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
if (qid.has_value()) {
data.emplace("qid", Value{*qid});
}
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
return details::HandleRun(session, state, query, params);
}
template <typename TSession>
@@ -324,6 +292,9 @@ State HandleReset(TSession &session, const Marker marker) {
return State::Close;
}
// Clear all pending data and send a success message.
session.encoder_buffer_.Clear();
if (!session.encoder_.MessageSuccess()) {
spdlog::trace("Couldn't send success message!");
return State::Close;
@@ -432,33 +403,12 @@ State HandleGoodbye() {
}
template <typename TSession>
State HandleRoute(TSession &session, const Marker marker) {
// Route message is not implemented since it is Neo4j specific, therefore we will receive it and inform user that
// there is no implementation. Before that, we have to read out the fields from the buffer to leave it in a clean
// state.
if (marker != Marker::TinyStruct3) {
spdlog::trace("Expected TinyStruct3 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
Value routing;
if (!session.decoder_.ReadValue(&routing, Value::Type::Map)) {
spdlog::trace("Couldn't read routing field!");
return State::Close;
}
Value bookmarks;
if (!session.decoder_.ReadValue(&bookmarks, Value::Type::List)) {
spdlog::trace("Couldn't read bookmarks field!");
return State::Close;
}
Value db;
if (!session.decoder_.ReadValue(&db)) {
spdlog::trace("Couldn't read db field!");
return State::Close;
}
State HandleRoute(TSession &session) {
// Route message is not implemented since it is neo4j specific, therefore we
// will receive it an inform user that there is no implementation.
session.encoder_buffer_.Clear();
bool fail_sent =
session.encoder_.MessageFailure({{"code", "66"}, {"message", "Route message is not supported in Memgraph!"}});
session.encoder_.MessageFailure({{"code", 66}, {"message", "Route message not supported in Memgraph!"}});
if (!fail_sent) {
spdlog::trace("Couldn't send failure message!");
return State::Close;

View File

@@ -0,0 +1,10 @@
set(coordinator_src_files
coordinator.hpp
shard_map.hpp
hybrid_logical_clock.hpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-coordinator STATIC ${coordinator_src_files})
target_link_libraries(mg-coordinator stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@@ -0,0 +1,243 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <unordered_set>
#include "coordinator/hybrid_logical_clock.hpp"
#include "coordinator/shard_map.hpp"
#include "io/simulator/simulator.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::coordinator {
using Address = memgraph::io::Address;
using SimT = memgraph::io::simulator::SimulatorTransport;
struct HlcRequest {
Hlc last_shard_map_version;
};
struct HlcResponse {
Hlc new_hlc;
std::optional<ShardMap> fresher_shard_map;
};
struct GetShardMapRequest {
// No state
};
struct GetShardMapResponse {
ShardMap shard_map;
};
struct AllocateHlcBatchRequest {
Hlc low;
Hlc high;
};
struct AllocateHlcBatchResponse {
bool success;
Hlc low;
Hlc high;
};
struct AllocateEdgeIdBatchRequest {
size_t batch_size;
};
struct AllocateEdgeIdBatchResponse {
uint64_t low;
uint64_t high;
};
struct SplitShardRequest {
Hlc previous_shard_map_version;
Label label;
CompoundKey split_key;
};
struct SplitShardResponse {
bool success;
};
struct RegisterStorageEngineRequest {
Address address;
};
struct RegisterStorageEngineResponse {
bool success;
};
struct DeregisterStorageEngineRequest {
Address address;
};
struct DeregisterStorageEngineResponse {
bool success;
};
struct InitializeLabelRequest {
std::string label_name;
Hlc last_shard_map_version;
};
struct InitializeLabelResponse {
bool success;
std::optional<ShardMap> fresher_shard_map;
};
using WriteRequests =
std::variant<AllocateHlcBatchRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
DeregisterStorageEngineRequest, InitializeLabelRequest>;
using WriteResponses =
std::variant<AllocateHlcBatchResponse, AllocateEdgeIdBatchResponse, SplitShardResponse,
RegisterStorageEngineResponse, DeregisterStorageEngineResponse, InitializeLabelResponse>;
using ReadRequests = std::variant<HlcRequest, GetShardMapRequest>;
using ReadResponses = std::variant<HlcResponse, GetShardMapResponse>;
class Coordinator {
ShardMap shard_map_;
/// The highest reserved timestamp / highest allocated timestamp
/// is a way for minimizing communication involved in query engines
/// reserving Hlc's for their transaction processing.
/// Periodically, the coordinator will allocate a batch of timestamps
/// and this will need to go over consensus. From that point forward,
/// each timestamp in that batch can be given out to "readers" who issue
/// HlcRequest without blocking on consensus first. But if
/// highest_allocated_timestamp_ approaches highest_reserved_timestamp_,
/// it is time to allocate another batch, so that we can keep guaranteeing
/// forward progress.
/// Any time a coordinator becomes a new leader, it will need to issue
/// a new AllocateHlcBatchRequest to create a pool of IDs to allocate.
uint64_t highest_allocated_timestamp_;
uint64_t highest_reserved_timestamp_;
/// Query engines need to periodically request batches of unique edge IDs.
uint64_t highest_allocated_edge_id_;
/// Increment our
ReadResponses HandleRead(HlcRequest &&hlc_request) {
HlcResponse res{};
auto hlc_shard_map = shard_map_.GetHlc();
MG_ASSERT(!(hlc_request.last_shard_map_version.logical_id > hlc_shard_map.logical_id));
res.new_hlc = shard_map_.IncrementShardMapVersion();
// res.fresher_shard_map = hlc_request.last_shard_map_version.logical_id < hlc_shard_map.logical_id
// ? std::make_optional(shard_map_)
// : std::nullopt;
// Allways return fresher shard_map for now.
res.fresher_shard_map = std::make_optional(shard_map_);
return res;
}
ReadResponses HandleRead(GetShardMapRequest &&get_shard_map_request) {
GetShardMapResponse res;
res.shard_map = shard_map_;
return res;
}
WriteResponses ApplyWrite(AllocateHlcBatchRequest &&ahr) {
AllocateHlcBatchResponse res{};
return res;
}
WriteResponses ApplyWrite(AllocateEdgeIdBatchRequest &&ahr) {
AllocateEdgeIdBatchResponse res{};
uint64_t low = highest_allocated_edge_id_;
highest_allocated_edge_id_ += ahr.batch_size;
uint64_t high = highest_allocated_edge_id_;
res.low = low;
res.high = high;
return res;
}
/// This splits the shard immediately beneath the provided
/// split key, keeping the assigned peers identical for now,
/// but letting them be gradually migrated over time.
WriteResponses ApplyWrite(SplitShardRequest &&split_shard_request) {
SplitShardResponse res{};
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
res.success = false;
} else {
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label,
split_shard_request.split_key);
}
return res;
}
/// This adds the provided storage engine to the standby storage engine pool,
/// which can be used to rebalance storage over time.
WriteResponses ApplyWrite(RegisterStorageEngineRequest &&register_storage_engine_request) {
RegisterStorageEngineResponse res{};
// TODO
return res;
}
/// This begins the process of draining the provided storage engine from all raft
/// clusters that it might be participating in.
WriteResponses ApplyWrite(DeregisterStorageEngineRequest &&register_storage_engine_request) {
DeregisterStorageEngineResponse res{};
// TODO
// const Address &address = register_storage_engine_request.address;
// storage_engine_pool_.erase(address);
// res.success = true;
return res;
}
WriteResponses ApplyWrite(InitializeLabelRequest &&initialize_label_request) {
InitializeLabelResponse res{};
bool success = shard_map_.InitializeNewLabel(initialize_label_request.label_name,
initialize_label_request.last_shard_map_version);
if (success) {
res.fresher_shard_map = shard_map_;
res.success = false;
} else {
res.fresher_shard_map = std::nullopt;
res.success = true;
}
return res;
}
public:
explicit Coordinator(ShardMap sm) : shard_map_{(sm)} {}
ReadResponses Read(ReadRequests requests) {
return std::visit([&](auto &&requests) { return HandleRead(std::move(requests)); }, std::move(requests));
}
WriteResponses Apply(WriteRequests requests) {
return std::visit([&](auto &&requests) { return ApplyWrite(std::move(requests)); }, std::move(requests));
}
};
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,30 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/time.hpp"
namespace memgraph::coordinator {
using Time = memgraph::io::Time;
/// Hybrid-logical clock
struct Hlc {
uint64_t logical_id;
Time coordinator_wall_clock;
bool operator==(const Hlc &other) const {
return (logical_id == other.logical_id) && (coordinator_wall_clock == other.coordinator_wall_clock);
}
};
} // namespace memgraph::coordinator

View File

@@ -0,0 +1,135 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <map>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::coordinator {
enum class Status : uint8_t {
CONSENSUS_PARTICIPANT,
INITIALIZING,
// TODO(tyler) this will possibly have more states,
// depending on the reconfiguration protocol that we
// implement.
};
struct AddressAndStatus {
memgraph::io::Address address;
Status status;
};
using memgraph::io::Address;
using CompoundKey = std::vector<memgraph::storage::v3::PropertyValue>;
using Shard = std::vector<AddressAndStatus>;
using Shards = std::map<CompoundKey, Shard>;
// use string for intermachine communication and NameIdMapper within the machine
using Label = std::string;
struct ShardMap {
Hlc shard_map_version;
std::map<Label, Shards> shards;
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well
Hlc IncrementShardMapVersion() noexcept {
++shard_map_version.logical_id;
return shard_map_version;
}
Hlc GetHlc() const noexcept { return shard_map_version; }
bool SplitShard(Hlc previous_shard_map_version, Label label, CompoundKey key) {
if (CompareShardMapVersions(previous_shard_map_version, shard_map_version)) {
MG_ASSERT(shards.contains(label));
auto &shards_in_map = shards[label];
MG_ASSERT(!shards_in_map.contains(key));
// Finding the Shard that the new CompoundKey should map to.
Shard shard_to_map_to;
CompoundKey prev_key = ((*shards_in_map.begin()).first);
for (auto iter = std::next(shards_in_map.begin()); iter != shards_in_map.end(); ++iter) {
const auto &current_key = (*iter).first;
if (key > prev_key && key < current_key) {
shard_to_map_to = shards_in_map[prev_key];
}
prev_key = (*iter).first;
}
// Apply the split
shards_in_map[key] = shard_to_map_to;
return true;
}
return false;
}
bool InitializeNewLabel(std::string label_name, Hlc last_shard_map_version) {
if (shard_map_version != last_shard_map_version) {
return false;
}
if (shards.contains(label_name)) {
return false;
}
shards.emplace(label_name, Shards{});
IncrementShardMapVersion();
return true;
}
void AddServer(Address server_address) {
// Find a random place for the server to plug in
}
std::map<Label, Shards> &GetShards() noexcept { return shards; }
Shards GetShardsForRange(Label label, CompoundKey start, CompoundKey end);
Shard GetShardForKey(Label label, CompoundKey key) {
auto shard_for_label = shards.at(label);
auto max = (--shard_for_label.end())->first;
if (key > max) {
return shard_for_label[max];
}
for (auto it = shard_for_label.lower_bound(key);; --it) {
MG_ASSERT(it->first <= key);
return it->second;
}
MG_ASSERT(false, "failed to find shard with a key that is less than or equal to the provided key");
}
private:
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well. This function should just be
// replaced with operator== since it is already overloaded for Hlc
// objects.
bool CompareShardMapVersions(Hlc one, Hlc two) { return one.logical_id == two.logical_id; }
};
} // namespace memgraph::coordinator

View File

@@ -1,4 +0,0 @@
set(mg_glue_sources auth.cpp auth_checker.cpp auth_handler.cpp communication.cpp)
add_library(mg-glue STATIC ${mg_glue_sources})
target_link_libraries(mg-glue mg-query mg-auth)

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,7 +10,6 @@
// licenses/APL.txt.
#include "glue/auth.hpp"
#include "auth/models.hpp"
namespace memgraph::glue {
@@ -58,26 +57,6 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::MODULE_WRITE;
case query::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::AuthQuery::Privilege::STORAGE_MODE:
return auth::Permission::STORAGE_MODE;
case query::AuthQuery::Privilege::TRANSACTION_MANAGEMENT:
return auth::Permission::TRANSACTION_MANAGEMENT;
}
}
#ifdef MG_ENTERPRISE
auth::FineGrainedPermission FineGrainedPrivilegeToFineGrainedPermission(
const query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
switch (fine_grained_privilege) {
case query::AuthQuery::FineGrainedPrivilege::NOTHING:
return auth::FineGrainedPermission::NOTHING;
case query::AuthQuery::FineGrainedPrivilege::READ:
return auth::FineGrainedPermission::READ;
case query::AuthQuery::FineGrainedPrivilege::UPDATE:
return auth::FineGrainedPermission::UPDATE;
case query::AuthQuery::FineGrainedPrivilege::CREATE_DELETE:
return auth::FineGrainedPermission::CREATE_DELETE;
}
}
#endif
} // namespace memgraph::glue

View File

@@ -20,12 +20,4 @@ namespace memgraph::glue {
*/
auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege);
#ifdef MG_ENTERPRISE
/**
* Converts query::AuthQuery::FineGrainedPrivilege to its corresponding
* auth::EntityPermission.
*/
auth::FineGrainedPermission FineGrainedPrivilegeToFineGrainedPermission(
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege);
#endif
} // namespace memgraph::glue

View File

@@ -1,173 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/auth_checker.hpp"
#include "auth/auth.hpp"
#include "auth/models.hpp"
#include "glue/auth.hpp"
#include "license/license.hpp"
#include "query/constants.hpp"
#include "query/frontend/ast/ast.hpp"
#include "utils/synchronized.hpp"
#ifdef MG_ENTERPRISE
namespace {
bool IsUserAuthorizedLabels(const memgraph::auth::User &user, const memgraph::query::DbAccessor *dba,
const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return std::all_of(labels.begin(), labels.end(), [dba, &user, fine_grained_privilege](const auto &label) {
return user.GetFineGrainedAccessLabelPermissions().Has(
dba->LabelToName(label), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) == memgraph::auth::PermissionLevel::GRANT;
});
}
bool IsUserAuthorizedGloballyLabels(const memgraph::auth::User &user,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return user.GetFineGrainedAccessLabelPermissions().Has(memgraph::query::kAsterisk, fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}
bool IsUserAuthorizedGloballyEdges(const memgraph::auth::User &user,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return user.GetFineGrainedAccessEdgeTypePermissions().Has(memgraph::query::kAsterisk, fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}
bool IsUserAuthorizedEdgeType(const memgraph::auth::User &user, const memgraph::query::DbAccessor *dba,
const memgraph::storage::EdgeTypeId &edgeType,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return user.GetFineGrainedAccessEdgeTypePermissions().Has(
dba->EdgeTypeToName(edgeType), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) == memgraph::auth::PermissionLevel::GRANT;
}
} // namespace
#endif
namespace memgraph::glue {
AuthChecker::AuthChecker(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth)
: auth_(auth) {}
bool AuthChecker::IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) const {
std::optional<memgraph::auth::User> maybe_user;
{
auto locked_auth = auth_->ReadLock();
if (!locked_auth->HasUsers()) {
return true;
}
if (username.has_value()) {
maybe_user = locked_auth->GetUser(*username);
}
}
return maybe_user.has_value() && IsUserAuthorized(*maybe_user, privileges);
}
#ifdef MG_ENTERPRISE
std::unique_ptr<memgraph::query::FineGrainedAuthChecker> AuthChecker::GetFineGrainedAuthChecker(
const std::string &username, const memgraph::query::DbAccessor *dba) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
return std::make_unique<memgraph::glue::FineGrainedAuthChecker>(std::move(*user), dba);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
#endif
bool AuthChecker::IsUserAuthorized(const memgraph::auth::User &user,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) {
const auto user_permissions = user.GetPermissions();
return std::all_of(privileges.begin(), privileges.end(), [&user_permissions](const auto privilege) {
return user_permissions.Has(memgraph::glue::PrivilegeToPermission(privilege)) ==
memgraph::auth::PermissionLevel::GRANT;
});
}
#ifdef MG_ENTERPRISE
FineGrainedAuthChecker::FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba)
: user_{std::move(user)}, dba_(dba){};
bool FineGrainedAuthChecker::Has(const memgraph::query::VertexAccessor &vertex, const memgraph::storage::View view,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) {
switch (maybe_labels.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:
throw memgraph::query::QueryRuntimeException("Trying to get labels from a deleted node.");
case memgraph::storage::Error::NONEXISTENT_OBJECT:
throw memgraph::query::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
case memgraph::storage::Error::SERIALIZATION_ERROR:
case memgraph::storage::Error::VERTEX_HAS_EDGES:
case memgraph::storage::Error::PROPERTIES_DISABLED:
throw memgraph::query::QueryRuntimeException("Unexpected error when getting labels.");
}
}
return IsUserAuthorizedLabels(user_, dba_, *maybe_labels, fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const memgraph::query::EdgeAccessor &edge,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsUserAuthorizedEdgeType(user_, dba_, edge.EdgeType(), fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsUserAuthorizedLabels(user_, dba_, labels, fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const memgraph::storage::EdgeTypeId &edge_type,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsUserAuthorizedEdgeType(user_, dba_, edge_type, fine_grained_privilege);
}
bool FineGrainedAuthChecker::HasGlobalPrivilegeOnVertices(
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return IsUserAuthorizedGloballyLabels(user_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
}
bool FineGrainedAuthChecker::HasGlobalPrivilegeOnEdges(
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return IsUserAuthorizedGloballyEdges(user_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
};
#endif
} // namespace memgraph::glue

View File

@@ -1,69 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "auth/auth.hpp"
#include "glue/auth.hpp"
#include "query/auth_checker.hpp"
#include "query/db_accessor.hpp"
#include "query/frontend/ast/ast.hpp"
namespace memgraph::glue {
class AuthChecker : public query::AuthChecker {
public:
explicit AuthChecker(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth);
bool IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<query::AuthQuery::Privilege> &privileges) const override;
#ifdef MG_ENTERPRISE
std::unique_ptr<memgraph::query::FineGrainedAuthChecker> GetFineGrainedAuthChecker(
const std::string &username, const memgraph::query::DbAccessor *dba) const override;
#endif
[[nodiscard]] static bool IsUserAuthorized(const memgraph::auth::User &user,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges);
private:
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
};
#ifdef MG_ENTERPRISE
class FineGrainedAuthChecker : public query::FineGrainedAuthChecker {
public:
explicit FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba);
bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const query::EdgeAccessor &edge,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const std::vector<memgraph::storage::LabelId> &labels,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const memgraph::storage::EdgeTypeId &edge_type,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnVertices(
memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnEdges(
memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
private:
auth::User user_;
const memgraph::query::DbAccessor *dba_;
};
#endif
} // namespace memgraph::glue

View File

@@ -1,664 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/auth_handler.hpp"
#include <sstream>
#include <fmt/format.h>
#include "auth/models.hpp"
#include "glue/auth.hpp"
#include "license/license.hpp"
#include "query/constants.hpp"
namespace {
struct PermissionForPrivilegeResult {
std::string permission;
memgraph::auth::PermissionLevel permission_level;
std::string description;
};
struct FineGrainedPermissionForPrivilegeResult {
std::string permission;
#ifdef MG_ENTERPRISE
memgraph::auth::FineGrainedPermission permission_level;
#endif
std::string description;
};
PermissionForPrivilegeResult GetPermissionForPrivilegeForUserOrRole(
const memgraph::auth::Permissions &permissions, const memgraph::query::AuthQuery::Privilege &privilege,
const std::string &user_or_role) {
PermissionForPrivilegeResult container;
const auto permission = memgraph::glue::PrivilegeToPermission(privilege);
container.permission = memgraph::auth::PermissionToString(permission);
container.permission_level = permissions.Has(permission);
switch (container.permission_level) {
case memgraph::auth::PermissionLevel::GRANT:
container.description = "GRANTED TO " + user_or_role;
break;
case memgraph::auth::PermissionLevel::DENY:
container.description = "DENIED TO " + user_or_role;
break;
case memgraph::auth::PermissionLevel::NEUTRAL:
break;
}
return container;
}
std::vector<std::vector<memgraph::query::TypedValue>> ConstructPrivilegesResult(
const std::vector<PermissionForPrivilegeResult> &privileges) {
std::vector<std::vector<memgraph::query::TypedValue>> grants;
grants.reserve(privileges.size());
for (const auto &permission : privileges) {
grants.push_back({memgraph::query::TypedValue(permission.permission),
memgraph::query::TypedValue(memgraph::auth::PermissionLevelToString(permission.permission_level)),
memgraph::query::TypedValue(permission.description)});
}
return grants;
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowUserPrivileges(
const std::optional<memgraph::auth::User> &user) {
std::vector<PermissionForPrivilegeResult> privilege_results;
const auto &permissions = user->GetPermissions();
const auto &user_level_permissions = user->permissions();
for (const auto &privilege : memgraph::query::kPrivilegesAll) {
auto user_permission_result = GetPermissionForPrivilegeForUserOrRole(permissions, privilege, "USER");
auto user_only_permissions_result =
GetPermissionForPrivilegeForUserOrRole(user_level_permissions, privilege, "USER");
if (user_permission_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
std::vector<std::string> full_description;
if (user_only_permissions_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
full_description.emplace_back(user_only_permissions_result.description);
}
if (const auto *role = user->role(); role != nullptr) {
auto role_permission_result = GetPermissionForPrivilegeForUserOrRole(role->permissions(), privilege, "ROLE");
if (role_permission_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
full_description.emplace_back(role_permission_result.description);
}
}
privilege_results.push_back(PermissionForPrivilegeResult{user_permission_result.permission,
user_permission_result.permission_level,
memgraph::utils::Join(full_description, ", ")});
}
}
return ConstructPrivilegesResult(privilege_results);
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowRolePrivileges(
const std::optional<memgraph::auth::Role> &role) {
std::vector<PermissionForPrivilegeResult> privilege_results;
const auto &permissions = role->permissions();
for (const auto &privilege : memgraph::query::kPrivilegesAll) {
auto role_permission_result = GetPermissionForPrivilegeForUserOrRole(permissions, privilege, "ROLE");
if (role_permission_result.permission_level != memgraph::auth::PermissionLevel::NEUTRAL) {
privilege_results.push_back(role_permission_result);
}
}
return ConstructPrivilegesResult(privilege_results);
}
#ifdef MG_ENTERPRISE
std::vector<FineGrainedPermissionForPrivilegeResult> GetFineGrainedPermissionForPrivilegeForUserOrRole(
const memgraph::auth::FineGrainedAccessPermissions &permissions, const std::string &permission_type,
const std::string &user_or_role) {
std::vector<FineGrainedPermissionForPrivilegeResult> fine_grained_permissions;
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return fine_grained_permissions;
}
const auto global_permission = permissions.GetGlobalPermission();
if (global_permission.has_value()) {
const auto &permission_level = memgraph::auth::PermissionToFineGrainedPermission(global_permission.value());
std::stringstream permission_representation;
permission_representation << "ALL " << permission_type << "S";
const auto &permission_level_representation =
permission_level == memgraph::auth::FineGrainedPermission::NOTHING ? "DENIED" : "GRANTED";
const auto permission_description =
fmt::format("GLOBAL {0} PERMISSION {1} TO {2}", permission_type, permission_level_representation, user_or_role);
fine_grained_permissions.push_back(FineGrainedPermissionForPrivilegeResult{
permission_representation.str(), permission_level, permission_description});
}
for (const auto &[label, permission] : permissions.GetPermissions()) {
auto permission_level = memgraph::auth::PermissionToFineGrainedPermission(permission);
std::stringstream permission_representation;
permission_representation << permission_type << " :" << label;
const auto &permission_level_representation =
permission_level == memgraph::auth::FineGrainedPermission::NOTHING ? "DENIED" : "GRANTED";
const auto permission_description =
fmt::format("{0} PERMISSION {1} TO {2}", permission_type, permission_level_representation, user_or_role);
fine_grained_permissions.push_back(FineGrainedPermissionForPrivilegeResult{
permission_representation.str(), permission_level, permission_description});
}
return fine_grained_permissions;
}
std::vector<std::vector<memgraph::query::TypedValue>> ConstructFineGrainedPrivilegesResult(
const std::vector<FineGrainedPermissionForPrivilegeResult> &privileges) {
std::vector<std::vector<memgraph::query::TypedValue>> grants;
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
grants.reserve(privileges.size());
for (const auto &permission : privileges) {
grants.push_back(
{memgraph::query::TypedValue(permission.permission),
memgraph::query::TypedValue(memgraph::auth::FineGrainedPermissionToString(permission.permission_level)),
memgraph::query::TypedValue(permission.description)});
}
return grants;
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowFineGrainedUserPrivileges(
const std::optional<memgraph::auth::User> &user) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
const auto &label_permissions = user->GetFineGrainedAccessLabelPermissions();
const auto &edge_type_permissions = user->GetFineGrainedAccessEdgeTypePermissions();
auto all_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(label_permissions, "LABEL", "USER");
auto edge_type_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(edge_type_permissions, "EDGE_TYPE", "USER");
all_fine_grained_permissions.insert(all_fine_grained_permissions.end(), edge_type_fine_grained_permissions.begin(),
edge_type_fine_grained_permissions.end());
return ConstructFineGrainedPrivilegesResult(all_fine_grained_permissions);
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowFineGrainedRolePrivileges(
const std::optional<memgraph::auth::Role> &role) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
const auto &label_permissions = role->GetFineGrainedAccessLabelPermissions();
const auto &edge_type_permissions = role->GetFineGrainedAccessEdgeTypePermissions();
auto all_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(label_permissions, "LABEL", "USER");
auto edge_type_fine_grained_permissions =
GetFineGrainedPermissionForPrivilegeForUserOrRole(edge_type_permissions, "EDGE_TYPE", "USER");
all_fine_grained_permissions.insert(all_fine_grained_permissions.end(), edge_type_fine_grained_permissions.begin(),
edge_type_fine_grained_permissions.end());
return ConstructFineGrainedPrivilegesResult(all_fine_grained_permissions);
}
#endif
} // namespace
namespace memgraph::glue {
AuthQueryHandler::AuthQueryHandler(
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::string name_regex_string)
: auth_(auth), name_regex_string_(std::move(name_regex_string)), name_regex_(name_regex_string_) {}
bool AuthQueryHandler::CreateUser(const std::string &username, const std::optional<std::string> &password) {
if (name_regex_string_ != kDefaultUserRoleRegex) {
if (const auto license_check_result =
memgraph::license::global_license_checker.IsEnterpriseValid(memgraph::utils::global_settings);
license_check_result.HasError()) {
throw memgraph::auth::AuthException(
"Custom user/role regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-user-or-role-name-regex\") to its default value (\"{}\") or remove the flag.\n{}",
kDefaultUserRoleRegex,
memgraph::license::LicenseCheckErrorToString(license_check_result.GetError(), "user/role regex"));
}
}
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
try {
const auto [first_user, user_added] = std::invoke([&, this] {
auto locked_auth = auth_->Lock();
const auto first_user = !locked_auth->HasUsers();
const auto user_added = locked_auth->AddUser(username, password).has_value();
return std::make_pair(first_user, user_added);
});
if (first_user) {
spdlog::info("{} is first created user. Granting all privileges.", username);
GrantPrivilege(
username, memgraph::query::kPrivilegesAll
#ifdef MG_ENTERPRISE
,
{{{memgraph::query::AuthQuery::FineGrainedPrivilege::CREATE_DELETE, {memgraph::query::kAsterisk}}}},
{
{
{
memgraph::query::AuthQuery::FineGrainedPrivilege::CREATE_DELETE, { memgraph::query::kAsterisk }
}
}
}
#endif
);
}
return user_added;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
bool AuthQueryHandler::DropUser(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) return false;
return locked_auth->RemoveUser(username);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::SetPassword(const std::string &username, const std::optional<std::string> &password) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist.", username);
}
user->UpdatePassword(password);
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
bool AuthQueryHandler::CreateRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
return locked_auth->AddRole(rolename).has_value();
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
bool AuthQueryHandler::DropRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
auto role = locked_auth->GetRole(rolename);
if (!role) {
return false;
};
return locked_auth->RemoveRole(rolename);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetUsernames() {
try {
auto locked_auth = auth_->ReadLock();
std::vector<memgraph::query::TypedValue> usernames;
const auto &users = locked_auth->AllUsers();
usernames.reserve(users.size());
for (const auto &user : users) {
usernames.emplace_back(user.username());
}
return usernames;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetRolenames() {
try {
auto locked_auth = auth_->ReadLock();
std::vector<memgraph::query::TypedValue> rolenames;
const auto &roles = locked_auth->AllRoles();
rolenames.reserve(roles.size());
for (const auto &role : roles) {
rolenames.emplace_back(role.rolename());
}
return rolenames;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::optional<std::string> AuthQueryHandler::GetRolenameForUser(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
if (const auto *role = user->role(); role != nullptr) {
return role->rolename();
}
return std::nullopt;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<memgraph::query::TypedValue> AuthQueryHandler::GetUsernamesForRole(const std::string &rolename) {
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->ReadLock();
auto role = locked_auth->GetRole(rolename);
if (!role) {
throw memgraph::query::QueryRuntimeException("Role '{}' doesn't exist.", rolename);
}
std::vector<memgraph::query::TypedValue> usernames;
const auto &users = locked_auth->AllUsersForRole(rolename);
usernames.reserve(users.size());
for (const auto &user : users) {
usernames.emplace_back(user.username());
}
return usernames;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::SetRole(const std::string &username, const std::string &rolename) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
if (!std::regex_match(rolename, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid role name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
auto role = locked_auth->GetRole(rolename);
if (!role) {
throw memgraph::query::QueryRuntimeException("Role '{}' doesn't exist .", rolename);
}
if (const auto *current_role = user->role(); current_role != nullptr) {
throw memgraph::query::QueryRuntimeException("User '{}' is already a member of role '{}'.", username,
current_role->rolename());
}
user->SetRole(*role);
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::ClearRole(const std::string &username) {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
user->ClearRole();
locked_auth->SaveUser(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<std::vector<memgraph::query::TypedValue>> AuthQueryHandler::GetPrivileges(const std::string &user_or_role) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
auto locked_auth = auth_->ReadLock();
std::vector<std::vector<memgraph::query::TypedValue>> grants;
#ifdef MG_ENTERPRISE
std::vector<std::vector<memgraph::query::TypedValue>> fine_grained_grants;
#endif
auto user = locked_auth->GetUser(user_or_role);
auto role = locked_auth->GetRole(user_or_role);
if (!user && !role) {
throw memgraph::query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role);
}
if (user) {
grants = ShowUserPrivileges(user);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
fine_grained_grants = ShowFineGrainedUserPrivileges(user);
}
#endif
} else {
grants = ShowRolePrivileges(role);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
fine_grained_grants = ShowFineGrainedRolePrivileges(role);
}
#endif
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
grants.insert(grants.end(), fine_grained_grants.begin(), fine_grained_grants.end());
}
#endif
return grants;
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::GrantPrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) {
EditPermissions(
user_or_role, privileges,
#ifdef MG_ENTERPRISE
label_privileges, edge_type_privileges,
#endif
[](auto &permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions.Grant(permission);
}
#ifdef MG_ENTERPRISE
,
[](auto &fine_grained_permissions, const auto &privilege_collection) {
for (const auto &[privilege, entities] : privilege_collection) {
const auto &permission = memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(privilege);
for (const auto &entity : entities) {
fine_grained_permissions.Grant(entity, permission);
}
}
}
#endif
);
} // namespace memgraph::glue
void AuthQueryHandler::DenyPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) {
EditPermissions(
user_or_role, privileges,
#ifdef MG_ENTERPRISE
{}, {},
#endif
[](auto &permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions.Deny(permission);
}
#ifdef MG_ENTERPRISE
,
[](auto &fine_grained_permissions, const auto &privilege_collection) {}
#endif
);
}
void AuthQueryHandler::RevokePrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) {
EditPermissions(
user_or_role, privileges,
#ifdef MG_ENTERPRISE
label_privileges, edge_type_privileges,
#endif
[](auto &permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions.Revoke(permission);
}
#ifdef MG_ENTERPRISE
,
[](auto &fine_grained_permissions, const auto &privilege_collection) {
for ([[maybe_unused]] const auto &[privilege, entities] : privilege_collection) {
for (const auto &entity : entities) {
fine_grained_permissions.Revoke(entity);
}
}
}
#endif
);
} // namespace memgraph::glue
template <class TEditPermissionsFun
#ifdef MG_ENTERPRISE
,
class TEditFineGrainedPermissionsFun
#endif
>
void AuthQueryHandler::EditPermissions(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
,
const TEditPermissionsFun &edit_permissions_fun
#ifdef MG_ENTERPRISE
,
const TEditFineGrainedPermissionsFun &edit_fine_grained_permissions_fun
#endif
) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
try {
std::vector<memgraph::auth::Permission> permissions;
permissions.reserve(privileges.size());
for (const auto &privilege : privileges) {
permissions.push_back(memgraph::glue::PrivilegeToPermission(privilege));
}
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(user_or_role);
auto role = locked_auth->GetRole(user_or_role);
if (!user && !role) {
throw memgraph::query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role);
}
if (user) {
for (const auto &permission : permissions) {
edit_permissions_fun(user->permissions(), permission);
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
for (const auto &label_privilege_collection : label_privileges) {
edit_fine_grained_permissions_fun(user->fine_grained_access_handler().label_permissions(),
label_privilege_collection);
}
for (const auto &edge_type_privilege_collection : edge_type_privileges) {
edit_fine_grained_permissions_fun(user->fine_grained_access_handler().edge_type_permissions(),
edge_type_privilege_collection);
}
}
#endif
locked_auth->SaveUser(*user);
} else {
for (const auto &permission : permissions) {
edit_permissions_fun(role->permissions(), permission);
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
for (const auto &label_privilege : label_privileges) {
edit_fine_grained_permissions_fun(role->fine_grained_access_handler().label_permissions(), label_privilege);
}
for (const auto &edge_type_privilege : edge_type_privileges) {
edit_fine_grained_permissions_fun(role->fine_grained_access_handler().edge_type_permissions(),
edge_type_privilege);
}
}
#endif
locked_auth->SaveRole(*role);
}
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
} // namespace memgraph::glue

View File

@@ -1,109 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <regex>
#include "auth/auth.hpp"
#include "glue/auth.hpp"
#include "license/license.hpp"
#include "query/interpreter.hpp"
#include "utils/string.hpp"
namespace memgraph::glue {
inline constexpr std::string_view kDefaultUserRoleRegex = "[a-zA-Z0-9_.+-@]+";
class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
std::string name_regex_string_;
std::regex name_regex_;
public:
AuthQueryHandler(memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
std::string name_regex_string);
bool CreateUser(const std::string &username, const std::optional<std::string> &password) override;
bool DropUser(const std::string &username) override;
void SetPassword(const std::string &username, const std::optional<std::string> &password) override;
bool CreateRole(const std::string &rolename) override;
bool DropRole(const std::string &rolename) override;
std::vector<memgraph::query::TypedValue> GetUsernames() override;
std::vector<memgraph::query::TypedValue> GetRolenames() override;
std::optional<std::string> GetRolenameForUser(const std::string &username) override;
std::vector<memgraph::query::TypedValue> GetUsernamesForRole(const std::string &rolename) override;
void SetRole(const std::string &username, const std::string &rolename) override;
void ClearRole(const std::string &username) override;
std::vector<std::vector<memgraph::query::TypedValue>> GetPrivileges(const std::string &user_or_role) override;
void GrantPrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) override;
void DenyPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override;
void RevokePrivilege(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
) override;
private:
template <class TEditPermissionsFun
#ifdef MG_ENTERPRISE
,
class TEditFineGrainedPermissionsFun
#endif
>
void EditPermissions(
const std::string &user_or_role, const std::vector<memgraph::query::AuthQuery::Privilege> &privileges
#ifdef MG_ENTERPRISE
,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&label_privileges,
const std::vector<std::unordered_map<memgraph::query::AuthQuery::FineGrainedPrivilege, std::vector<std::string>>>
&edge_type_privileges
#endif
,
const TEditPermissionsFun &edit_permissions_fun
#ifdef MG_ENTERPRISE
,
const TEditFineGrainedPermissionsFun &edit_fine_grained_permissions_fun
#endif
);
};
} // namespace memgraph::glue

View File

@@ -127,10 +127,6 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage
return Value(value.ValueLocalDateTime());
case query::TypedValue::Type::Duration:
return Value(value.ValueDuration());
case query::TypedValue::Type::Graph:
auto maybe_graph = ToBoltGraph(value.ValueGraph(), db, view);
if (maybe_graph.HasError()) return maybe_graph.GetError();
return Value(std::move(*maybe_graph));
}
}
@@ -187,30 +183,6 @@ storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, c
return communication::bolt::Path(vertices, edges);
}
storage::Result<std::map<std::string, Value>> ToBoltGraph(const query::Graph &graph, const storage::Storage &db,
storage::View view) {
std::map<std::string, Value> map;
std::vector<Value> vertices;
vertices.reserve(graph.vertices().size());
for (const auto &v : graph.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(Value(std::move(*maybe_vertex)));
}
map.emplace("nodes", Value(vertices));
std::vector<Value> edges;
edges.reserve(graph.edges().size());
for (const auto &e : graph.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(Value(std::move(*maybe_edge)));
}
map.emplace("edges", Value(edges));
return std::move(map);
}
storage::PropertyValue ToPropertyValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:

View File

@@ -51,15 +51,6 @@ storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccesso
storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, const storage::Storage &db,
storage::View view);
/// @param query::Graph for converting to communication::bolt::Map.
/// @param storage::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::Result<std::map<std::string, communication::bolt::Value>> ToBoltGraph(const query::Graph &graph,
const storage::Storage &db,
storage::View view);
/// @param query::TypedValue for converting to communication::bolt::Value.
/// @param storage::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::View for ToBoltVertex and ToBoltEdge.

64
src/glue/v2/auth.cpp Normal file
View File

@@ -0,0 +1,64 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/auth.hpp"
namespace memgraph::glue::v2 {
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege) {
switch (privilege) {
case query::v2::AuthQuery::Privilege::MATCH:
return auth::Permission::MATCH;
case query::v2::AuthQuery::Privilege::CREATE:
return auth::Permission::CREATE;
case query::v2::AuthQuery::Privilege::MERGE:
return auth::Permission::MERGE;
case query::v2::AuthQuery::Privilege::DELETE:
return auth::Permission::DELETE;
case query::v2::AuthQuery::Privilege::SET:
return auth::Permission::SET;
case query::v2::AuthQuery::Privilege::REMOVE:
return auth::Permission::REMOVE;
case query::v2::AuthQuery::Privilege::INDEX:
return auth::Permission::INDEX;
case query::v2::AuthQuery::Privilege::STATS:
return auth::Permission::STATS;
case query::v2::AuthQuery::Privilege::CONSTRAINT:
return auth::Permission::CONSTRAINT;
case query::v2::AuthQuery::Privilege::DUMP:
return auth::Permission::DUMP;
case query::v2::AuthQuery::Privilege::REPLICATION:
return auth::Permission::REPLICATION;
case query::v2::AuthQuery::Privilege::DURABILITY:
return auth::Permission::DURABILITY;
case query::v2::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::v2::AuthQuery::Privilege::FREE_MEMORY:
return auth::Permission::FREE_MEMORY;
case query::v2::AuthQuery::Privilege::TRIGGER:
return auth::Permission::TRIGGER;
case query::v2::AuthQuery::Privilege::CONFIG:
return auth::Permission::CONFIG;
case query::v2::AuthQuery::Privilege::AUTH:
return auth::Permission::AUTH;
case query::v2::AuthQuery::Privilege::STREAM:
return auth::Permission::STREAM;
case query::v2::AuthQuery::Privilege::MODULE_READ:
return auth::Permission::MODULE_READ;
case query::v2::AuthQuery::Privilege::MODULE_WRITE:
return auth::Permission::MODULE_WRITE;
case query::v2::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::v2::AuthQuery::Privilege::SCHEMA:
return auth::Permission::SCHEMA;
}
}
} // namespace memgraph::glue::v2

23
src/glue/v2/auth.hpp Normal file
View File

@@ -0,0 +1,23 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "auth/models.hpp"
#include "query/v2/frontend/ast/ast.hpp"
namespace memgraph::glue::v2 {
/**
* This function converts query::AuthQuery::Privilege to its corresponding
* auth::Permission.
*/
auth::Permission PrivilegeToPermission(query::v2::AuthQuery::Privilege privilege);
} // namespace memgraph::glue::v2

View File

@@ -0,0 +1,275 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/v2/communication.hpp"
#include <map>
#include <string>
#include <vector>
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/storage.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/temporal.hpp"
using memgraph::communication::bolt::Value;
namespace memgraph::glue::v2 {
query::v2::TypedValue ToTypedValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return query::v2::TypedValue(value.ValueBool());
case Value::Type::Int:
return query::v2::TypedValue(value.ValueInt());
case Value::Type::Double:
return query::v2::TypedValue(value.ValueDouble());
case Value::Type::String:
return query::v2::TypedValue(value.ValueString());
case Value::Type::List: {
std::vector<query::v2::TypedValue> list;
list.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) list.push_back(ToTypedValue(v));
return query::v2::TypedValue(std::move(list));
}
case Value::Type::Map: {
std::map<std::string, query::v2::TypedValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToTypedValue(kv.second));
return query::v2::TypedValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to TypedValue");
case Value::Type::Date:
return query::v2::TypedValue(value.ValueDate());
case Value::Type::LocalTime:
return query::v2::TypedValue(value.ValueLocalTime());
case Value::Type::LocalDateTime:
return query::v2::TypedValue(value.ValueLocalDateTime());
case Value::Type::Duration:
return query::v2::TypedValue(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Storage &db,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
return Value();
case query::v2::TypedValue::Type::Bool:
return Value(value.ValueBool());
case query::v2::TypedValue::Type::Int:
return Value(value.ValueInt());
case query::v2::TypedValue::Type::Double:
return Value(value.ValueDouble());
case query::v2::TypedValue::Type::String:
return Value(std::string(value.ValueString()));
case query::v2::TypedValue::Type::List: {
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
auto maybe_value = ToBoltValue(v, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
values.emplace_back(std::move(*maybe_value));
}
return Value(std::move(values));
}
case query::v2::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return Value(std::move(map));
}
case query::v2::TypedValue::Type::Vertex: {
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::v2::TypedValue::Type::Edge: {
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::v2::TypedValue::Type::Path: {
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::v2::TypedValue::Type::Date:
return Value(value.ValueDate());
case query::v2::TypedValue::Type::LocalTime:
return Value(value.ValueLocalTime());
case query::v2::TypedValue::Type::LocalDateTime:
return Value(value.ValueLocalDateTime());
case query::v2::TypedValue::Type::Duration:
return Value(value.ValueDuration());
}
}
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view) {
auto id = communication::bolt::Id::FromUint(vertex.Gid().AsUint());
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
std::vector<std::string> labels;
labels.reserve(maybe_labels->size());
for (const auto &label : *maybe_labels) {
labels.push_back(db.LabelToName(label));
}
auto maybe_properties = vertex.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Vertex{id, labels, properties};
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view) {
auto id = communication::bolt::Id::FromUint(edge.Gid().AsUint());
auto from = communication::bolt::Id::FromUint(edge.FromVertex().Gid().AsUint());
auto to = communication::bolt::Id::FromUint(edge.ToVertex().Gid().AsUint());
const auto &type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
storage::v3::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());
for (const auto &v : path.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(std::move(*maybe_vertex));
}
std::vector<communication::bolt::Edge> edges;
edges.reserve(path.edges().size());
for (const auto &e : path.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(std::move(*maybe_edge));
}
return communication::bolt::Path(vertices, edges);
}
storage::v3::PropertyValue ToPropertyValue(const Value &value) {
switch (value.type()) {
case Value::Type::Null:
return {};
case Value::Type::Bool:
return storage::v3::PropertyValue(value.ValueBool());
case Value::Type::Int:
return storage::v3::PropertyValue(value.ValueInt());
case Value::Type::Double:
return storage::v3::PropertyValue(value.ValueDouble());
case Value::Type::String:
return storage::v3::PropertyValue(value.ValueString());
case Value::Type::List: {
std::vector<storage::v3::PropertyValue> vec;
vec.reserve(value.ValueList().size());
for (const auto &value : value.ValueList()) vec.emplace_back(ToPropertyValue(value));
return storage::v3::PropertyValue(std::move(vec));
}
case Value::Type::Map: {
std::map<std::string, storage::v3::PropertyValue> map;
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToPropertyValue(kv.second));
return storage::v3::PropertyValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException("Unsupported conversion from Value to PropertyValue");
case Value::Type::Date:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Date, value.ValueDate().MicrosecondsSinceEpoch()));
case Value::Type::LocalTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalTime,
value.ValueLocalTime().MicrosecondsSinceEpoch()));
case Value::Type::LocalDateTime:
return storage::v3::PropertyValue(storage::v3::TemporalData(storage::v3::TemporalType::LocalDateTime,
value.ValueLocalDateTime().MicrosecondsSinceEpoch()));
case Value::Type::Duration:
return storage::v3::PropertyValue(
storage::v3::TemporalData(storage::v3::TemporalType::Duration, value.ValueDuration().microseconds));
}
}
Value ToBoltValue(const storage::v3::PropertyValue &value) {
switch (value.type()) {
case storage::v3::PropertyValue::Type::Null:
return {};
case storage::v3::PropertyValue::Type::Bool:
return {value.ValueBool()};
case storage::v3::PropertyValue::Type::Int:
return {value.ValueInt()};
break;
case storage::v3::PropertyValue::Type::Double:
return {value.ValueDouble()};
case storage::v3::PropertyValue::Type::String:
return {value.ValueString()};
case storage::v3::PropertyValue::Type::List: {
const auto &values = value.ValueList();
std::vector<Value> vec;
vec.reserve(values.size());
for (const auto &v : values) {
vec.push_back(ToBoltValue(v));
}
return {std::move(vec)};
}
case storage::v3::PropertyValue::Type::Map: {
const auto &map = value.ValueMap();
std::map<std::string, Value> dv_map;
for (const auto &kv : map) {
dv_map.emplace(kv.first, ToBoltValue(kv.second));
}
return {std::move(dv_map)};
}
case storage::v3::PropertyValue::Type::TemporalData:
const auto &type = value.ValueTemporalData();
switch (type.type) {
case storage::v3::TemporalType::Date:
return {utils::Date(type.microseconds)};
case storage::v3::TemporalType::LocalTime:
return {utils::LocalTime(type.microseconds)};
case storage::v3::TemporalType::LocalDateTime:
return {utils::LocalDateTime(type.microseconds)};
case storage::v3::TemporalType::Duration:
return {utils::Duration(type.microseconds)};
}
}
}
} // namespace memgraph::glue::v2

View File

@@ -0,0 +1,68 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file Conversion functions between Value and other memgraph types.
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "query/v2/typed_value.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
class Storage;
class VertexAccessor;
} // namespace memgraph::storage::v3
namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param storage::v3::Storage for getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Storage &db, storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param storage::v3::Storage for getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Storage &db, storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Storage &db,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param storage::v3::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
const storage::v3::Storage &db, storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
} // namespace memgraph::glue::v2

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,7 +12,6 @@
#pragma once
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
@@ -62,42 +61,3 @@ inline void LoadConfig(const std::string &product_name) {
for (int i = 0; i < custom_argc; ++i) free(custom_argv[i]);
delete[] custom_argv;
}
std::pair<std::string, std::string> LoadUsernameAndPassword(const std::string &pass_file) {
std::ifstream file(pass_file);
if (file.fail()) {
spdlog::warn("Problem with opening MG_PASSFILE, memgraph server will start without user");
return {};
}
std::vector<std::string> result;
std::string line;
std::getline(file, line);
size_t pos = 0;
std::string token;
static constexpr std::string_view delimiter{":"};
while ((pos = line.find(delimiter)) != std::string::npos) {
if (line[pos - 1] == '\\') {
line.erase(pos - 1, 1);
token += line.substr(0, pos);
line.erase(0, pos);
} else {
token += line.substr(0, pos);
result.push_back(token);
line.erase(0, pos + delimiter.length());
token = "";
}
}
result.push_back(line);
file.close();
if (result.size() != 2) {
spdlog::warn(
"Wrong data format. Data should be store in format: username:password, memgraph server will start without "
"user");
return {};
}
return {result[0], result[1]};
}

60
src/io/address.hpp Normal file
View File

@@ -0,0 +1,60 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <compare>
#include <fmt/format.h>
#include <boost/asio/ip/tcp.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>
namespace memgraph::io {
struct Address {
// It's important for all participants to have a
// unique identifier - IP and port alone are not
// enough, and may change over the lifecycle of
// the nodes. Particularly storage nodes may change
// their IP addresses over time, and the system
// should gracefully update its information
// about them.
boost::uuids::uuid unique_id;
boost::asio::ip::address last_known_ip;
uint16_t last_known_port;
static Address TestAddress(uint16_t port) {
Address ret;
ret.last_known_port = port;
return ret;
}
friend bool operator==(const Address &lhs, const Address &rhs) = default;
/// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
friend bool operator<(const Address &lhs, const Address &rhs) {
if (lhs.unique_id != rhs.unique_id) {
return lhs.unique_id < rhs.unique_id;
}
if (lhs.last_known_ip != rhs.last_known_ip) {
return lhs.last_known_ip < rhs.last_known_ip;
}
return lhs.last_known_port < rhs.last_known_port;
}
std::string ToString() const {
return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
}
};
}; // namespace memgraph::io

26
src/io/errors.hpp Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::io {
// Signifies that a retriable operation was unable to
// complete after a configured number of retries.
struct RetriesExhausted {};
// Signifies that a request was unable to receive a response
// within some configured timeout duration. It is important
// to remember that in distributed systems, a timeout does
// not signify that a request was not received or processed.
// It may be the case that the request was fully processed
// but that the response was not received.
struct TimedOut {};
}; // namespace memgraph::io

262
src/io/future.hpp Normal file
View File

@@ -0,0 +1,262 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <condition_variable>
#include <memory>
#include <mutex>
#include <optional>
#include <thread>
#include <utility>
#include "io/errors.hpp"
#include "utils/logging.hpp"
namespace memgraph::io {
// Shared is in an anonymous namespace, and the only way to
// construct a Promise or Future is to pass a Shared in. This
// ensures that Promises and Futures can only be constructed
// in this translation unit.
namespace details {
template <typename T>
class Shared {
mutable std::condition_variable cv_;
mutable std::mutex mu_;
std::optional<T> item_;
bool consumed_ = false;
bool waiting_ = false;
std::function<bool()> simulator_notifier_ = nullptr;
public:
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
Shared() = default;
Shared(Shared &&) = delete;
Shared &operator=(Shared &&) = delete;
Shared(const Shared &) = delete;
Shared &operator=(const Shared &) = delete;
~Shared() = default;
/// Takes the item out of our optional item_ and returns it.
T Take() {
MG_ASSERT(item_, "Take called without item_ being present");
MG_ASSERT(!consumed_, "Take called on already-consumed Future");
T ret = std::move(item_).value();
item_.reset();
consumed_ = true;
return ret;
}
T Wait() {
std::unique_lock<std::mutex> lock(mu_);
waiting_ = true;
while (!item_) {
bool simulator_progressed = false;
if (simulator_notifier_) [[unlikely]] {
// We can't hold our own lock while notifying
// the simulator because notifying the simulator
// involves acquiring the simulator's mutex
// to guarantee that our notification linearizes
// with the simulator's condition variable.
// However, the simulator may acquire our
// mutex to check if we are being awaited,
// while determining system quiescence,
// so we have to get out of its way to avoid
// a cyclical deadlock.
lock.unlock();
simulator_progressed = std::invoke(simulator_notifier_);
lock.lock();
if (item_) {
// item may have been filled while we
// had dropped our mutex while notifying
// the simulator of our waiting_ status.
break;
}
}
if (!simulator_progressed) [[likely]] {
cv_.wait(lock);
}
MG_ASSERT(!consumed_, "Future consumed twice!");
}
waiting_ = false;
return Take();
}
bool IsReady() const {
std::unique_lock<std::mutex> lock(mu_);
return item_;
}
std::optional<T> TryGet() {
std::unique_lock<std::mutex> lock(mu_);
if (item_) {
return Take();
}
return std::nullopt;
}
void Fill(T item) {
{
std::unique_lock<std::mutex> lock(mu_);
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!item_, "Promise filled twice!");
item_ = item;
} // lock released before condition variable notification
cv_.notify_all();
}
bool IsAwaited() const {
std::unique_lock<std::mutex> lock(mu_);
return waiting_;
}
};
} // namespace details
template <typename T>
class Future {
bool consumed_or_moved_ = false;
std::shared_ptr<details::Shared<T>> shared_;
public:
explicit Future(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Future() = delete;
Future(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
consumed_or_moved_ = old.consumed_or_moved_;
old.consumed_or_moved_ = true;
}
Future &operator=(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
old.consumed_or_moved_ = true;
}
Future(const Future &) = delete;
Future &operator=(const Future &) = delete;
~Future() = default;
/// Returns true if the Future is ready to
/// be consumed using TryGet or Wait (prefer Wait
/// if you know it's ready, because it doesn't
/// return an optional.
bool IsReady() {
MG_ASSERT(!consumed_or_moved_, "Called IsReady after Future already consumed!");
return shared_->IsReady();
}
/// Non-blocking method that returns the inner
/// item if it's already ready, or std::nullopt
/// if it is not ready yet.
std::optional<T> TryGet() {
MG_ASSERT(!consumed_or_moved_, "Called TryGet after Future already consumed!");
std::optional<T> ret = shared_->TryGet();
if (ret) {
consumed_or_moved_ = true;
}
return ret;
}
/// Block on the corresponding promise to be filled,
/// returning the inner item when ready.
T Wait() && {
MG_ASSERT(!consumed_or_moved_, "Future should only be consumed with Wait once!");
T ret = shared_->Wait();
consumed_or_moved_ = true;
return ret;
}
/// Marks this Future as canceled.
void Cancel() {
MG_ASSERT(!consumed_or_moved_, "Future::Cancel called on a future that was already moved or consumed!");
consumed_or_moved_ = true;
}
};
template <typename T>
class Promise {
std::shared_ptr<details::Shared<T>> shared_;
bool filled_or_moved_ = false;
public:
explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
Promise() = delete;
Promise(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise &operator=(Promise &&old) noexcept {
MG_ASSERT(!old.filled_or_moved_, "Promise moved from after already being moved from or filled.");
shared_ = std::move(old.shared_);
old.filled_or_moved_ = true;
}
Promise(const Promise &) = delete;
Promise &operator=(const Promise &) = delete;
~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
// Fill the expected item into the Future.
void Fill(T item) {
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
shared_->Fill(item);
filled_or_moved_ = true;
}
bool IsAwaited() { return shared_->IsAwaited(); }
/// Moves this Promise into a unique_ptr.
std::unique_ptr<Promise<T>> ToUnique() && {
std::unique_ptr<Promise<T>> up = std::make_unique<Promise<T>>(std::move(shared_));
filled_or_moved_ = true;
return up;
}
};
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePair() {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>();
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);
return std::make_pair(std::move(future), std::move(promise));
}
}; // namespace memgraph::io

View File

@@ -0,0 +1,25 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "coordinator/coordinator.hpp"
#include "io/rsm/raft.hpp"
namespace memgraph::io::rsm {
// TODO(tyler) don't
using namespace memgraph::coordinator;
template <typename IoImpl>
using CoordinatorRsm = io::rsm::Raft<IoImpl, Coordinator, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
} // namespace memgraph::io::rsm

845
src/io/rsm/raft.hpp Normal file
View File

@@ -0,0 +1,845 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
// TODO(tyler) buffer out-of-order Append buffers on the Followers to reassemble more quickly
// TODO(tyler) handle granular batch sizes based on simple flow control
#pragma once
#include <deque>
#include <iostream>
#include <map>
#include <set>
#include <thread>
#include <unordered_map>
#include <vector>
#include "io/simulator/simulator.hpp"
#include "io/transport.hpp"
namespace memgraph::io::rsm {
using memgraph::io::Address;
using memgraph::io::Duration;
using memgraph::io::Io;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::Time;
using Term = uint64_t;
using LogIndex = uint64_t;
using RequestId = uint64_t;
template <typename WriteOperation>
struct WriteRequest {
WriteOperation operation;
};
/// WriteResponse is returned to a client after
/// their WriteRequest was entered in to the raft
/// log and it reached consensus.
///
/// WriteReturn is the result of applying the WriteRequest to
/// ReplicatedState, and if the ReplicatedState::write
/// method is deterministic, all replicas will
/// have the same ReplicatedState after applying
/// the submitted WriteRequest.
template <typename WriteReturn>
struct WriteResponse {
bool success;
WriteReturn write_return;
std::optional<Address> retry_leader;
};
template <typename ReadOperation>
struct ReadRequest {
ReadOperation operation;
};
template <typename ReadReturn>
struct ReadResponse {
bool success;
ReadReturn read_return;
std::optional<Address> retry_leader;
};
/// AppendRequest is a raft-level message that the Leader
/// periodically broadcasts to all Follower peers. This
/// serves three main roles:
/// 1. acts as a heartbeat from the Leader to the Follower
/// 2. replicates new data that the Leader has received to the Follower
/// 3. informs Follower peers when the commit index has increased,
/// signalling that it is now safe to apply log items to the
/// replicated state machine
template <typename WriteRequest>
struct AppendRequest {
Term term = 0;
LogIndex last_log_index;
Term last_log_term;
std::vector<std::pair<Term, WriteRequest>> entries;
LogIndex leader_commit;
};
struct AppendResponse {
bool success;
Term term;
Term last_log_term;
// a small optimization over the raft paper, tells
// the leader the offset that we are interested in
// to send log offsets from for us. This will only
// be useful at the beginning of a leader's term.
LogIndex last_log_index;
};
struct VoteRequest {
Term term = 0;
LogIndex last_log_index;
Term last_log_term;
};
struct VoteResponse {
Term term = 0;
LogIndex committed_log_size;
bool vote_granted = false;
};
template <typename WriteRequest>
struct CommonState {
Term term = 0;
std::vector<std::pair<Term, WriteRequest>> log;
LogIndex committed_log_size = 0;
LogIndex applied_size = 0;
};
struct FollowerTracker {
LogIndex next_index = 0;
LogIndex confirmed_contiguous_index = 0;
};
struct PendingClientRequest {
RequestId request_id;
Address address;
Time received_at;
};
struct Leader {
std::map<Address, FollowerTracker> followers;
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
Time last_broadcast = Time::min();
std::string ToString() { return "\tLeader \t"; }
};
struct Candidate {
std::map<Address, LogIndex> successful_votes;
Time election_began = Time::min();
std::set<Address> outstanding_votes;
std::string ToString() { return "\tCandidate\t"; }
};
struct Follower {
Time last_received_append_entries_timestamp;
Address leader_address;
std::string ToString() { return "\tFollower \t"; }
};
using Role = std::variant<Candidate, Leader, Follower>;
/*
all ReplicatedState classes should have an Apply method
that returns our WriteResponseValue after consensus, and
a Read method that returns our ReadResponseValue without
requiring consensus.
ReadResponse Read(ReadOperation);
WriteResponseValue ReplicatedState::Apply(WriteRequest);
For example:
If the state is uint64_t, and WriteRequest is `struct PlusOne {};`,
and WriteResponseValue is also uint64_t (the new value), then
each call to state.Apply(PlusOne{}) will return the new value
after incrementing it. 0, 1, 2, 3... and this will be sent back
to the client that requested the mutation.
In practice, these mutations will usually be predicated on some
previous value, so that they are idempotent, functioning similarly
to a CAS operation.
*/
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
typename ReadResponseValue>
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
};
/// Parameter Purpose
/// --------------------------
/// IoImpl the concrete Io provider - SimulatorTransport, ThriftTransport, etc...
/// ReplicatedState the high-level data structure that is managed by the raft-backed replicated state machine
/// WriteOperation the individual operation type that is applied to the ReplicatedState in identical order
/// across each replica
/// WriteResponseValue the return value of calling ReplicatedState::Apply(WriteOperation), which is executed in
/// identical order across all replicas after an WriteOperation reaches consensus.
/// ReadOperation the type of operations that do not require consensus before executing directly
/// on a const ReplicatedState &
/// ReadResponseValue the return value of calling ReplicatedState::Read(ReadOperation), which is executed directly
/// without going through consensus first
template <typename IoImpl, typename ReplicatedState, typename WriteOperation, typename WriteResponseValue,
typename ReadOperation, typename ReadResponseValue>
requires Rsm<WriteOperation, ReadOperation, ReplicatedState, WriteResponseValue, ReadResponseValue>
class Raft {
CommonState<WriteOperation> state_;
Role role_ = Candidate{};
Io<IoImpl> io_;
std::vector<Address> peers_;
ReplicatedState replicated_state_;
public:
Raft(Io<IoImpl> &&io, std::vector<Address> peers, ReplicatedState &&replicated_state)
: io_(std::move(io)), peers_(peers), replicated_state_(std::move(replicated_state)) {}
void Run() {
Time last_cron = io_.Now();
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
const Duration random_cron_interval = RandomTimeout(1000, 2000);
if (now - last_cron > random_cron_interval) {
Cron();
last_cron = now;
}
Duration receive_timeout = RandomTimeout(10000, 50000);
auto request_result =
io_.template ReceiveWithTimeout<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse>(receive_timeout);
if (request_result.HasError()) {
continue;
}
auto request = std::move(request_result.GetValue());
Handle(std::move(request.message), request.request_id, request.from_address);
}
}
private:
// Raft paper - 5.3
// When the entry has been safely replicated, the leader applies the
// entry to its state machine and returns the result of that
// execution to the client.
//
// "Safely replicated" is defined as being known to be present
// on at least a majority of all peers (inclusive of the Leader).
void BumpCommitIndexAndReplyToClients(Leader &leader) {
auto indices = std::vector<LogIndex>{};
// We include our own log size in the calculation of the log
// index that is present on at least a majority of all peers.
indices.push_back(state_.log.size());
for (const auto &[addr, f] : leader.followers) {
indices.push_back(f.confirmed_contiguous_index);
Log("at port ", addr.last_known_port, " has confirmed contiguous index of: ", f.confirmed_contiguous_index);
}
// reverse sort from highest to lowest (using std::ranges::greater)
std::ranges::sort(indices, std::ranges::greater());
// This is a particularly correctness-critical calculation because it
// determines which index we will consider to be the committed index.
//
// If the following indexes are recorded for clusters of different sizes,
// these are the expected indexes that are considered to have reached
// consensus:
// state | expected value | (indices.size() / 2)
// [1] 1 (1 / 2) => 0
// [2, 1] 1 (2 / 2) => 1
// [3, 2, 1] 2 (3 / 2) => 1
// [4, 3, 2, 1] 2 (4 / 2) => 2
// [5, 4, 3, 2, 1] 3 (5 / 2) => 2
size_t index_present_on_majority = indices.size() / 2;
LogIndex new_committed_log_size = indices[index_present_on_majority];
// We never go backwards in history.
MG_ASSERT(state_.committed_log_size <= new_committed_log_size);
state_.committed_log_size = new_committed_log_size;
// For each index between the old index and the new one (inclusive),
// Apply that log's WriteOperation to our replicated_state_,
// and use the specific return value of the ReplicatedState::Apply
// method (WriteResponseValue) to respond to the requester.
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const LogIndex apply_index = state_.applied_size;
const auto &write_request = state_.log[apply_index].second;
WriteResponseValue write_return = replicated_state_.Apply(write_request);
if (leader.pending_client_requests.contains(apply_index)) {
PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
WriteResponse<WriteResponseValue> resp;
resp.success = true;
resp.write_return = std::move(write_return);
io_.Send(client_request.address, client_request.request_id, std::move(resp));
leader.pending_client_requests.erase(apply_index);
}
}
Log("committed_log_size is now ", state_.committed_log_size);
}
// Raft paper - 5.1
// AppendEntries RPCs are initiated by leaders to replicate log entries and to provide a form of heartbeat
void BroadcastAppendEntries(std::map<Address, FollowerTracker> &followers) {
for (auto &[address, follower] : followers) {
const LogIndex index = follower.confirmed_contiguous_index;
std::vector<std::pair<Term, WriteOperation>> entries;
if (state_.log.size() > index) {
entries.insert(entries.begin(), state_.log.begin() + index, state_.log.end());
}
const Term previous_term_from_index = PreviousTermFromIndex(index);
Log("sending ", entries.size(), " entries to Follower ", address.last_known_port,
" which are above its known index of ", index);
AppendRequest<WriteOperation> ar{
.term = state_.term,
.last_log_index = index,
.last_log_term = previous_term_from_index,
.entries = entries,
.leader_commit = state_.committed_log_size,
};
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr RequestId request_id = 0;
io_.Send(address, request_id, ar);
}
}
// Raft paper - 5.2
// Raft uses randomized election timeouts to ensure that split votes are rare and that they are resolved quickly
Duration RandomTimeout(Duration min, Duration max) {
std::uniform_int_distribution time_distrib(min.count(), max.count());
auto rand_micros = io_.Rand(time_distrib);
return std::chrono::microseconds{rand_micros};
}
Duration RandomTimeout(int min_micros, int max_micros) {
std::uniform_int_distribution time_distrib(min_micros, max_micros);
int rand_micros = io_.Rand(time_distrib);
return std::chrono::microseconds{rand_micros};
}
Term PreviousTermFromIndex(LogIndex index) const {
if (index == 0 || state_.log.size() + 1 <= index) {
return 0;
}
const auto &[term, data] = state_.log.at(index - 1);
return term;
}
LogIndex CommittedLogIndex() { return state_.committed_log_size; }
Term CommittedLogTerm() {
MG_ASSERT(state_.log.size() >= state_.committed_log_size);
if (state_.log.empty() || state_.committed_log_size == 0) {
return 0;
}
auto &[term, data] = state_.log.at(state_.committed_log_size - 1);
return term;
}
LogIndex LastLogIndex() { return state_.log.size(); }
Term LastLogTerm() const {
if (state_.log.empty()) {
return 0;
}
const auto &[term, data] = state_.log.back();
return term;
}
template <typename... Ts>
void Log(Ts &&...args) {
const Time now = io_.Now();
auto micros = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();
const Term term = state_.term;
std::ostringstream out;
out << '\t' << (int)micros << "\t" << term << "\t" << io_.GetAddress().last_known_port;
std::string role_string = std::visit([&](auto &&role) { return role.ToString(); }, role_);
out << role_string;
(out << ... << args);
spdlog::debug(out.str());
}
/////////////////////////////////////////////////////////////
/// Raft-related Cron methods
///
/// Cron + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Cron(role) takes as the first argument a reference to its
/// role, and as the second argument, the message that has
/// been received.
/////////////////////////////////////////////////////////////
/// Periodic protocol maintenance.
void Cron() {
// dispatch periodic logic based on our role to a specific Cron method.
std::optional<Role> new_role = std::visit([&](auto &&role) { return Cron(role); }, role_);
if (new_role) {
role_ = std::move(new_role).value();
}
}
// Raft paper - 5.2
// Candidates keep sending Vote to peers until:
// 1. receiving Append with a higher term (become Follower)
// 2. receiving Vote with a higher term (become a Follower)
// 3. receiving a quorum of responses to our last batch of Vote (become a Leader)
std::optional<Role> Cron(Candidate &candidate) {
const auto now = io_.Now();
const Duration election_timeout = RandomTimeout(100000, 200000);
const auto election_timeout_us = std::chrono::duration_cast<std::chrono::milliseconds>(election_timeout).count();
if (now - candidate.election_began > election_timeout) {
state_.term++;
Log("becoming Candidate for term ", state_.term, " after leader timeout of ", election_timeout_us,
" elapsed since last election attempt");
const VoteRequest request{
.term = state_.term,
.last_log_index = LastLogIndex(),
.last_log_term = LastLogTerm(),
};
auto outstanding_votes = std::set<Address>();
for (const auto &peer : peers_) {
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr auto request_id = 0;
io_.template Send<VoteRequest>(peer, request_id, request);
outstanding_votes.insert(peer);
}
return Candidate{
.successful_votes = std::map<Address, LogIndex>(),
.election_began = now,
.outstanding_votes = outstanding_votes,
};
}
return std::nullopt;
}
// Raft paper - 5.2
// Followers become candidates if we haven't heard from the leader
// after a randomized timeout.
std::optional<Role> Cron(Follower &follower) {
const auto now = io_.Now();
const auto time_since_last_append_entries = now - follower.last_received_append_entries_timestamp;
Duration election_timeout = RandomTimeout(100000, 200000);
// randomized follower timeout with a range of 100-150ms.
if (time_since_last_append_entries > election_timeout) {
// become a Candidate if we haven't heard from the Leader after this timeout
return Candidate{};
}
return std::nullopt;
}
// Leaders (re)send AppendRequest to followers.
std::optional<Role> Cron(Leader &leader) {
const Time now = io_.Now();
const Duration broadcast_timeout = RandomTimeout(40000, 60000);
if (now - leader.last_broadcast > broadcast_timeout) {
BroadcastAppendEntries(leader.followers);
leader.last_broadcast = now;
}
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// Raft-related Handle methods
///
/// Handle + std::visit is how events are dispatched
/// to certain code based on Raft role.
///
/// Handle(role, message, ...)
/// takes as the first argument a reference
/// to its role, and as the second argument, the
/// message that has been received.
/////////////////////////////////////////////////////////////
void Handle(std::variant<ReadRequest<ReadOperation>, AppendRequest<WriteOperation>, AppendResponse,
WriteRequest<WriteOperation>, VoteRequest, VoteResponse> &&message_variant,
RequestId request_id, Address from_address) {
// dispatch the message to a handler based on our role,
// which can be specified in the Handle first argument,
// or it can be `auto` if it's a handler for several roles
// or messages.
std::optional<Role> new_role =
std::visit([&](auto &&msg, auto &&role) { return Handle(role, std::move(msg), request_id, from_address); },
std::move(message_variant), role_);
// TODO(tyler) (M3) maybe replace std::visit with get_if for explicit prioritized matching, [[likely]] etc...
if (new_role) {
role_ = std::move(new_role).value();
}
}
// all roles can receive Vote and possibly become a follower
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &, VoteRequest &&req, RequestId request_id, Address from_address) {
Log("received Vote from ", from_address.last_known_port, " with term ", req.term);
const bool last_log_term_dominates = req.last_log_term >= LastLogTerm();
const bool term_dominates = req.term > state_.term;
const bool last_log_index_dominates = req.last_log_index >= LastLogIndex();
const bool new_leader = last_log_term_dominates && term_dominates && last_log_index_dominates;
if (new_leader) {
MG_ASSERT(req.term > state_.term);
MG_ASSERT(std::max(req.term, state_.term) == req.term);
}
const VoteResponse res{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
};
io_.Send(from_address, request_id, res);
if (new_leader) {
// become a follower
state_.term = req.term;
return Follower{
.last_received_append_entries_timestamp = io_.Now(),
.leader_address = from_address,
};
} else if (term_dominates) {
Log("received a vote from an inferior candidate. Becoming Candidate");
state_.term = std::max(state_.term, req.term) + 1;
return Candidate{};
}
return std::nullopt;
}
std::optional<Role> Handle(Candidate &candidate, VoteResponse &&res, RequestId, Address from_address) {
Log("received VoteResponse");
if (!res.vote_granted || res.term != state_.term) {
Log("received unsuccessful VoteResponse from term ", res.term, " when our candidacy term is ", state_.term);
// we received a delayed VoteResponse from the past, which has to do with an election that is
// no longer valid. We can simply drop this.
return std::nullopt;
}
MG_ASSERT(candidate.outstanding_votes.contains(from_address),
"Received unexpected VoteResponse from server not present in Candidate's outstanding_votes!");
candidate.outstanding_votes.erase(from_address);
MG_ASSERT(!candidate.successful_votes.contains(from_address),
"Received unexpected VoteResponse from server already in Candidate's successful_votes!");
candidate.successful_votes.insert({from_address, res.committed_log_size});
if (candidate.successful_votes.size() >= candidate.outstanding_votes.size()) {
std::map<Address, FollowerTracker> followers{};
for (const auto &[address, committed_log_size] : candidate.successful_votes) {
FollowerTracker follower{
.next_index = committed_log_size,
.confirmed_contiguous_index = committed_log_size,
};
followers.insert({address, follower});
}
for (const auto &address : candidate.outstanding_votes) {
FollowerTracker follower{
.next_index = state_.log.size(),
.confirmed_contiguous_index = 0,
};
followers.insert({address, follower});
}
Log("becoming Leader at term ", state_.term);
BroadcastAppendEntries(followers);
return Leader{
.followers = std::move(followers),
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
};
}
return std::nullopt;
}
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &, VoteResponse &&, RequestId, Address) {
Log("non-Candidate received VoteResponse");
return std::nullopt;
}
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &role, AppendRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
AppendResponse res{
.success = false,
.term = state_.term,
.last_log_term = CommittedLogTerm(),
.last_log_index = CommittedLogIndex(),
};
if constexpr (std::is_same<AllRoles, Leader>()) {
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
}
const bool is_candidate = std::is_same<AllRoles, Candidate>();
const bool is_failed_competitor = is_candidate && req.term == state_.term;
const Time now = io_.Now();
// Raft paper - 5.2
// While waiting for votes, a candidate may receive an
// AppendEntries RPC from another server claiming to be leader. If
// the leaders term (included in its RPC) is at least as large as
// the candidates current term, then the candidate recognizes the
// leader as legitimate and returns to follower state.
if (req.term > state_.term || is_failed_competitor) {
// become follower of this leader, reply with our log status
state_.term = req.term;
io_.Send(from_address, request_id, res);
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
return Follower{
.last_received_append_entries_timestamp = now,
.leader_address = from_address,
};
} else if (req.term < state_.term) {
// nack this request from an old leader
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// at this point, we're dealing with our own leader
if constexpr (std::is_same<AllRoles, Follower>()) {
// small specialization for when we're already a Follower
MG_ASSERT(role.leader_address == from_address, "Multiple Leaders are acting under the same term number!");
role.last_received_append_entries_timestamp = now;
} else {
Log("Somehow entered Follower-specific logic as a non-Follower");
MG_ASSERT(false, "Somehow entered Follower-specific logic as a non-Follower");
}
res.last_log_term = LastLogTerm();
res.last_log_index = LastLogIndex();
Log("returning last_log_index of ", res.last_log_index);
// Handle steady-state conditions.
if (req.last_log_index != LastLogIndex()) {
Log("req.last_log_index is above our last applied log index");
} else if (req.last_log_term != LastLogTerm()) {
Log("req.last_log_term differs from our leader term at that slot, expected: ", LastLogTerm(), " but got ",
req.last_log_term);
} else {
// happy path - Apply log
Log("applying batch of entries to log of size ", req.entries.size());
MG_ASSERT(req.last_log_index >= state_.committed_log_size,
"Applied history from Leader which goes back in time from our commit_index");
// possibly chop-off stuff that was replaced by
// things with different terms (we got data that
// hasn't reached consensus yet, which is normal)
state_.log.resize(req.last_log_index);
state_.log.insert(state_.log.end(), req.entries.begin(), req.entries.end());
MG_ASSERT(req.leader_commit >= state_.committed_log_size);
state_.committed_log_size = std::min(req.leader_commit, LastLogIndex());
for (; state_.applied_size < state_.committed_log_size; state_.applied_size++) {
const auto &write_request = state_.log[state_.applied_size].second;
replicated_state_.Apply(write_request);
}
res.success = true;
}
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Leader &leader, AppendResponse &&res, RequestId, Address from_address) {
if (res.term != state_.term) {
} else if (!leader.followers.contains(from_address)) {
Log("received AppendResponse from unknown Follower");
MG_ASSERT(false, "received AppendResponse from unknown Follower");
} else {
if (res.success) {
Log("got successful AppendResponse from ", from_address.last_known_port, " with last_log_index of ",
res.last_log_index);
} else {
Log("got unsuccessful AppendResponse from ", from_address.last_known_port, " with last_log_index of ",
res.last_log_index);
}
FollowerTracker &follower = leader.followers.at(from_address);
follower.next_index = std::max(follower.next_index, res.last_log_index);
follower.confirmed_contiguous_index = std::max(follower.confirmed_contiguous_index, res.last_log_index);
BumpCommitIndexAndReplyToClients(leader);
}
return std::nullopt;
}
template <typename AllRoles>
std::optional<Role> Handle(AllRoles &, AppendResponse &&, RequestId, Address) {
// we used to be the leader, and are getting old delayed responses
return std::nullopt;
}
/////////////////////////////////////////////////////////////
/// RSM-related handle methods
/////////////////////////////////////////////////////////////
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
std::optional<Role> Handle(Leader &, ReadRequest<ReadOperation> &&req, RequestId request_id, Address from_address) {
Log("handling ReadOperation");
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(read_operation);
ReadResponse<ReadResponseValue> resp{
.success = true,
.read_return = std::move(read_return),
.retry_leader = std::nullopt,
};
io_.Send(from_address, request_id, resp);
return std::nullopt;
}
// Candidates should respond with a failure, similar to the Candidate + WriteRequest failure below
std::optional<Role> Handle(Candidate &, ReadRequest<ReadOperation> &&, RequestId request_id, Address from_address) {
Log("received ReadOperation - not redirecting because no Leader is known");
auto res = ReadResponse<ReadResponseValue>{};
res.success = false;
Cron();
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// Followers should respond with a redirection, similar to the Follower + WriteRequest response below
std::optional<Role> Handle(Follower &follower, ReadRequest<ReadOperation> &&, RequestId request_id,
Address from_address) {
auto res = ReadResponse<ReadResponseValue>{};
res.success = false;
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
res.retry_leader = follower.leader_address;
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// Raft paper - 8
// When a client first starts up, it connects to a randomly chosen
// server. If the clients first choice is not the leader, that
// server will reject the clients request and supply information
// about the most recent leader it has heard from.
std::optional<Role> Handle(Follower &follower, WriteRequest<WriteOperation> &&, RequestId request_id,
Address from_address) {
auto res = WriteResponse<WriteResponseValue>{};
res.success = false;
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
res.retry_leader = follower.leader_address;
io_.Send(from_address, request_id, res);
return std::nullopt;
}
std::optional<Role> Handle(Candidate &, WriteRequest<WriteOperation> &&, RequestId request_id, Address from_address) {
Log("received WriteRequest - not redirecting because no Leader is known");
auto res = WriteResponse<WriteResponseValue>{};
res.success = false;
Cron();
io_.Send(from_address, request_id, res);
return std::nullopt;
}
// only leaders actually handle replication requests from clients
std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
Log("handling WriteRequest");
// we are the leader. add item to log and send Append to peers
state_.log.emplace_back(std::pair(state_.term, std::move(req.operation)));
LogIndex log_index = state_.log.size() - 1;
PendingClientRequest pcr{
.request_id = request_id,
.address = from_address,
.received_at = io_.Now(),
};
leader.pending_client_requests.emplace(log_index, pcr);
BroadcastAppendEntries(leader.followers);
return std::nullopt;
}
};
}; // namespace memgraph::io::rsm

137
src/io/rsm/rsm_client.hpp Normal file
View File

@@ -0,0 +1,137 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <iostream>
#include <optional>
#include <vector>
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "utils/result.hpp"
namespace memgraph::io::rsm {
using memgraph::io::Address;
using memgraph::io::Duration;
using memgraph::io::ResponseEnvelope;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::Time;
using memgraph::io::TimedOut;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::ReadResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::utils::BasicResult;
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
typename ReadResponseT>
class RsmClient {
using ServerPool = std::vector<Address>;
IoImpl io_;
Address leader_;
std::mt19937 cli_rng_{0};
ServerPool server_addrs_;
template <typename ResponseT>
void PossiblyRedirectLeader(const ResponseT &response) {
if (response.retry_leader) {
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
leader_ = response.retry_leader.value();
spdlog::debug("client redirected to leader server {}", leader_.ToString());
} else if (!response.success) {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = addr_distrib(cli_rng_);
leader_ = server_addrs_[addr_index];
spdlog::debug(
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
"a RSM Candidate) trying a random one at index {} with address {}",
addr_index, leader_.ToString());
}
}
public:
RsmClient(IoImpl io, Address leader, ServerPool server_addrs)
: io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
RsmClient() = delete;
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
WriteRequest<WriteRequestT> client_req;
client_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
if (response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
// continue;
return response_result.GetError();
}
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
if (write_response.success) {
return std::move(write_response.write_return);
}
PossiblyRedirectLeader(write_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
ReadRequest<ReadRequestT> read_req;
read_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
// receive response
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
if (get_response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return get_response_result.GetError();
}
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
if (read_get_response.success) {
return std::move(read_get_response.read_return);
}
PossiblyRedirectLeader(read_get_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
};
} // namespace memgraph::io::rsm

165
src/io/rsm/shard_rsm.hpp Normal file
View File

@@ -0,0 +1,165 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
/// The StorageRsm is a simple in-memory raft-backed kv store that can be used for simple testing
/// and implementation of some query engine logic before storage engines are fully implemented.
///
/// To implement multiple read and write commands, change the StorageRead* and StorageWrite* requests
/// and responses to a std::variant of the different options, and route them to specific handlers in
/// the StorageRsm's Read and Apply methods. Remember that Read is called immediately when the Raft
/// leader receives the request, and does not replicate anything over Raft. Apply is called only
/// AFTER the StorageWriteRequest is replicated to a majority of Raft peers, and the result of calling
/// StorageRsm::Apply(StorageWriteRequest) is returned to the client that submitted the request.
#include <deque>
#include <iostream>
#include <map>
#include <optional>
#include <set>
#include <thread>
#include <vector>
#include "coordinator/hybrid_logical_clock.hpp"
#include "io/address.hpp"
#include "io/rsm/raft.hpp"
#include "io/simulator/simulator.hpp"
#include "io/simulator/simulator_transport.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/logging.hpp"
using memgraph::coordinator::Hlc;
using memgraph::io::Address;
using memgraph::io::Io;
using memgraph::io::ResponseEnvelope;
using memgraph::io::ResponseFuture;
using memgraph::io::ResponseResult;
using memgraph::io::rsm::Raft;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::ReadResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::io::simulator::Simulator;
using memgraph::io::simulator::SimulatorConfig;
using memgraph::io::simulator::SimulatorStats;
using memgraph::io::simulator::SimulatorTransport;
using memgraph::storage::PropertyValue;
namespace memgraph::io::rsm {
using ShardRsmKey = std::vector<PropertyValue>;
struct StorageWriteRequest {
ShardRsmKey key;
std::optional<int> value;
};
struct StorageWriteResponse {
bool shard_rsm_success;
std::optional<int> last_value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
struct StorageReadRequest {
ShardRsmKey key;
};
struct StorageReadResponse {
bool shard_rsm_success;
std::optional<int> value;
// Only has a value if the given shard does not contain the requested key
std::optional<Hlc> latest_known_shard_map_version{std::nullopt};
};
class StorageRsm {
std::map<ShardRsmKey, int> state_;
ShardRsmKey minimum_key_;
std::optional<ShardRsmKey> maximum_key_{std::nullopt};
Hlc shard_map_version_;
// The key is not located in this shard
bool IsKeyInRange(const ShardRsmKey &key) {
if (maximum_key_) [[likely]] {
return (key >= minimum_key_ && key <= maximum_key_);
}
return key >= minimum_key_;
}
public:
StorageReadResponse Read(StorageReadRequest request) {
StorageReadResponse ret;
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
} else if (state_.contains(request.key)) {
ret.value = state_[request.key];
ret.shard_rsm_success = true;
} else {
ret.shard_rsm_success = false;
ret.value = std::nullopt;
}
return ret;
}
StorageWriteResponse Apply(StorageWriteRequest request) {
StorageWriteResponse ret;
// Key is outside the prohibited range
if (!IsKeyInRange(request.key)) {
ret.latest_known_shard_map_version = shard_map_version_;
ret.shard_rsm_success = false;
}
// Key exist
else if (state_.contains(request.key)) {
auto &val = state_[request.key];
/*
* Delete
*/
if (!request.value) {
ret.shard_rsm_success = true;
ret.last_value = val;
state_.erase(state_.find(request.key));
}
/*
* Update
*/
// Does old_value match?
if (request.value == val) {
ret.last_value = val;
ret.shard_rsm_success = true;
val = request.value.value();
} else {
ret.last_value = val;
ret.shard_rsm_success = false;
}
}
/*
* Create
*/
else {
ret.last_value = std::nullopt;
ret.shard_rsm_success = true;
state_.emplace(request.key, std::move(request.value).value());
}
return ret;
}
};
} // namespace memgraph::io::rsm

View File

@@ -0,0 +1,8 @@
set(io_simulator_sources
simulator_handle.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-io-simulator STATIC ${io_simulator_sources})
target_link_libraries(mg-io-simulator stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@@ -0,0 +1,177 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/transport.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Message;
using memgraph::io::Time;
struct OpaqueMessage {
Address from_address;
uint64_t request_id;
std::any message;
/// Recursively tries to match a specific type from the outer
/// variant's parameter pack against the type of the std::any,
/// and if it matches, make it concrete and return it. Otherwise,
/// move on and compare the any with the next type from the
/// parameter pack.
///
/// Return is the full std::variant<Ts...> type that holds the
/// full parameter pack without interfering with recursive
/// narrowing expansion.
template <typename Return, Message Head, Message... Rest>
std::optional<Return> Unpack(std::any &&a) {
if (typeid(Head) == a.type()) {
Head concrete = std::any_cast<Head>(std::move(a));
return concrete;
}
if constexpr (sizeof...(Rest) > 0) {
return Unpack<Return, Rest...>(std::move(a));
} else {
return std::nullopt;
}
}
/// High level "user-facing" conversion function that lets
/// people interested in conversion only supply a single
/// parameter pack for the types that they want to compare
/// with the any and potentially include in the returned
/// variant.
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<std::variant<Ms...>> VariantFromAny(std::any &&a) {
return Unpack<std::variant<Ms...>, Ms...>(std::move(a));
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) std::optional<RequestEnvelope<Ms...>> Take() && {
std::optional<std::variant<Ms...>> m_opt = VariantFromAny<Ms...>(std::move(message));
if (m_opt) {
return RequestEnvelope<Ms...>{
.message = std::move(*m_opt),
.request_id = request_id,
.from_address = from_address,
};
}
return std::nullopt;
}
};
class OpaquePromiseTraitBase {
public:
virtual const std::type_info *TypeInfo() const = 0;
virtual bool IsAwaited(void *ptr) const = 0;
virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
virtual void TimeOut(void *ptr) const = 0;
virtual ~OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase() = default;
OpaquePromiseTraitBase(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase &operator=(const OpaquePromiseTraitBase &) = delete;
OpaquePromiseTraitBase(OpaquePromiseTraitBase &&old) = delete;
OpaquePromiseTraitBase &operator=(OpaquePromiseTraitBase &&) = delete;
};
template <typename T>
class OpaquePromiseTrait : public OpaquePromiseTraitBase {
public:
const std::type_info *TypeInfo() const override { return &typeid(T); };
bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
T message = std::any_cast<T>(std::move(opaque_message.message));
auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
.request_id = opaque_message.request_id,
.from_address = opaque_message.from_address};
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
unique_promise->Fill(std::move(response_envelope));
};
void TimeOut(void *ptr) const override {
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
ResponseResult<T> result = TimedOut{};
unique_promise->Fill(std::move(result));
}
};
class OpaquePromise {
void *ptr_;
std::unique_ptr<OpaquePromiseTraitBase> trait_;
public:
OpaquePromise(OpaquePromise &&old) noexcept : ptr_(old.ptr_), trait_(std::move(old.trait_)) {
MG_ASSERT(old.ptr_ != nullptr);
old.ptr_ = nullptr;
}
OpaquePromise &operator=(OpaquePromise &&old) noexcept {
MG_ASSERT(ptr_ == nullptr);
MG_ASSERT(old.ptr_ != nullptr);
MG_ASSERT(this != &old);
ptr_ = old.ptr_;
trait_ = std::move(old.trait_);
old.ptr_ = nullptr;
return *this;
}
OpaquePromise(const OpaquePromise &) = delete;
OpaquePromise &operator=(const OpaquePromise &) = delete;
template <typename T>
std::unique_ptr<ResponsePromise<T>> Take() && {
MG_ASSERT(typeid(T) == *trait_->TypeInfo());
MG_ASSERT(ptr_ != nullptr);
auto ptr = static_cast<ResponsePromise<T> *>(ptr_);
ptr_ = nullptr;
return std::unique_ptr<T>(ptr);
}
template <typename T>
explicit OpaquePromise(std::unique_ptr<ResponsePromise<T>> promise)
: ptr_(static_cast<void *>(promise.release())), trait_(std::make_unique<OpaquePromiseTrait<T>>()) {}
bool IsAwaited() {
MG_ASSERT(ptr_ != nullptr);
return trait_->IsAwaited(ptr_);
}
void TimeOut() {
MG_ASSERT(ptr_ != nullptr);
trait_->TimeOut(ptr_);
ptr_ = nullptr;
}
void Fill(OpaqueMessage &&opaque_message) {
MG_ASSERT(ptr_ != nullptr);
trait_->Fill(ptr_, std::move(opaque_message));
ptr_ = nullptr;
}
~OpaquePromise() {
MG_ASSERT(ptr_ == nullptr, "OpaquePromise destroyed without being explicitly timed out or filled");
}
};
} // namespace memgraph::io::simulator

View File

@@ -0,0 +1,51 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <memory>
#include <random>
#include "io/address.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/simulator/simulator_transport.hpp"
namespace memgraph::io::simulator {
class Simulator {
std::mt19937 rng_;
std::shared_ptr<SimulatorHandle> simulator_handle_;
uint16_t auto_port_ = 0;
public:
explicit Simulator(SimulatorConfig config)
: rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
void ShutDown() { simulator_handle_->ShutDown(); }
Io<SimulatorTransport> RegisterNew() {
Address address = Address::TestAddress(auto_port_++);
return Register(address);
}
Io<SimulatorTransport> Register(Address address) {
std::uniform_int_distribution<uint64_t> seed_distrib;
uint64_t seed = seed_distrib(rng_);
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
}
void IncrementServerCountAndWaitForQuiescentState(Address address) {
simulator_handle_->IncrementServerCountAndWaitForQuiescentState(address);
}
SimulatorStats Stats() { return simulator_handle_->Stats(); }
};
}; // namespace memgraph::io::simulator

View File

@@ -0,0 +1,30 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include "io/time.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Time;
struct SimulatorConfig {
uint8_t drop_percent = 0;
bool perform_timeouts = false;
bool scramble_messages = true;
uint64_t rng_seed = 0;
Time start_time = Time::min();
Time abort_time = Time::max();
};
}; // namespace memgraph::io::simulator

View File

@@ -0,0 +1,154 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "io/simulator/simulator_handle.hpp"
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
void SimulatorHandle::ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool SimulatorHandle::ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
std::unique_lock<std::mutex> lock(mu_);
server_addresses_.insert(address);
while (true) {
const size_t blocked_servers = BlockedServers();
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
if (all_servers_blocked) {
return;
}
cv_.wait(lock);
}
}
size_t SimulatorHandle::BlockedServers() {
size_t blocked_servers = blocked_on_receive_;
for (auto &[promise_key, opaque_promise] : promises_) {
if (opaque_promise.promise.IsAwaited() && server_addresses_.contains(promise_key.requester_address)) {
blocked_servers++;
}
}
return blocked_servers;
}
bool SimulatorHandle::MaybeTickSimulator() {
std::unique_lock<std::mutex> lock(mu_);
const size_t blocked_servers = BlockedServers();
if (blocked_servers < server_addresses_.size()) {
// we only need to advance the simulator when all
// servers have reached a quiescent state, blocked
// on their own futures or receive methods.
return false;
}
stats_.simulator_ticks++;
cv_.notify_all();
TimeoutPromisesPastDeadline();
if (in_flight_.empty()) {
// return early here because there are no messages to schedule
// We tick the clock forward when all servers are blocked but
// there are no in-flight messages to schedule delivery of.
std::poisson_distribution<> time_distrib(50);
Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
cluster_wide_time_microseconds_ += clock_advance;
MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time.");
return true;
}
if (config_.scramble_messages) {
// scramble messages
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
const size_t swap_index = swap_distrib(rng_);
std::swap(in_flight_[swap_index], in_flight_.back());
}
auto [to_address, opaque_message] = std::move(in_flight_.back());
in_flight_.pop_back();
std::uniform_int_distribution<int> drop_distrib(0, 99);
const int drop_threshold = drop_distrib(rng_);
const bool should_drop = drop_threshold < config_.drop_percent;
if (should_drop) {
stats_.dropped_messages++;
}
PromiseKey promise_key{.requester_address = to_address,
.request_id = opaque_message.request_id,
.replier_address = opaque_message.from_address};
if (promises_.contains(promise_key)) {
// complete waiting promise if it's there
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
const bool normal_timeout = config_.perform_timeouts && (dop.deadline < cluster_wide_time_microseconds_);
if (should_drop || normal_timeout) {
stats_.timed_out_requests++;
dop.promise.TimeOut();
} else {
stats_.total_responses++;
dop.promise.Fill(std::move(opaque_message));
}
} else if (should_drop) {
// don't add it anywhere, let it drop
} else {
// add to can_receive_ if not
const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
om_vec->second.emplace_back(std::move(opaque_message));
}
return true;
}
Time SimulatorHandle::Now() const {
std::unique_lock<std::mutex> lock(mu_);
return cluster_wide_time_microseconds_;
}
SimulatorStats SimulatorHandle::Stats() {
std::unique_lock<std::mutex> lock(mu_);
return stats_;
}
} // namespace memgraph::io::simulator

Some files were not shown because too many files have changed in this diff Show More