Compare commits

..

231 Commits

Author SHA1 Message Date
Josip Mrden
59c05d83e9 Add rewrite of set property into set properties 2023-08-08 10:45:28 +02:00
Aidar Samerkhanov
271b1a5ddb Fix bug with on-disk triggers (#1134)
* Fix TriggerContext adaptation for accessors.
* Fix edge deserialization in case of the deleted vertex.
2023-08-08 10:37:14 +02:00
gvolfing
260660f1dd Fix sequential label-property index recovery (#1135)
The parallel_exec_info should have been passed to this function before,
otherwise, the recovery of label-property indices would never have been
parallelized.
2023-08-05 23:20:15 +02:00
Marko Budiselić
e5350a011c Upgrade to mgconsole v1.4.0 (#1144) 2023-08-05 15:52:31 +02:00
Kruno Golubic
7bf827bb1e Remove link to Discourse forum from README (#1138) 2023-08-05 14:37:07 +02:00
Marko Budiselić
5d13c281fa Update v2.10 license date (#1133) 2023-08-02 15:13:15 +02:00
Marko Budiselic
020273f475 Increase package all ARM timeout 2023-08-02 07:53:31 +00:00
andrejtonev
4a99625287 Fix throwing if user was created without a license (#1067) 2023-08-01 23:36:12 +02:00
andrejtonev
5bbed6ef9a Implement user caching to speed up PullPlan (#1109) 2023-08-01 23:04:35 +02:00
Andi
f0bac53e7b Improve restore replication role (#1089) 2023-08-01 21:51:52 +02:00
Matija Pintarić
514fed51c4 Add implementation of C++ API Node::RemoveProperty (#1128) 2023-08-01 20:11:38 +02:00
imilinovic
2877c343e8 Add implementation of << operator for mgp::Value (#1127) 2023-08-01 19:30:23 +02:00
ind1xa
50a1d1abb3 Add remove label to the query modules C API (#1126) 2023-08-01 19:24:11 +02:00
andrejtonev
e8850549d2 Add multi-tenancy v1 (#952)
* Decouple BoltSession and communication::bolt::Session
* Add CREATE/USE/DROP DATABASE
* Add SHOW DATABASES
* Cover WebSocket session
* Simple session safety implemented via RWLock
* Storage symlinks for backward. compatibility
* Extend the audit log with the DB info
* Add auth part
* Add tenant recovery
2023-08-01 18:49:11 +02:00
andrejtonev
fd819cd099 Add custom e2e test activation to include the toolchain libs (#1130) 2023-08-01 17:13:47 +02:00
Andi
60f4ffc6a1 Improve logging if replica cannot recover using curr WAL file (#1086) 2023-08-01 10:33:46 +02:00
Andi
bd2ec6374a Remove node argument from start-stop Jepsen functions (#1091) 2023-08-01 00:06:57 +02:00
Marko Budiselić
c501f59a09 Add more info to the query module error messages (#771) 2023-08-01 00:05:44 +02:00
gvolfing
210bea83d4 Add GraphQL transpilation compatibility (#1018)
* Add callable mappings feature
* Implement mgps.validate (void procedure)
* Make '_' a valid variable name
2023-07-31 14:48:12 +02:00
Josh Soref
57fe3463f2 Fix a bunch of spelling mistakes (1/n) (#1112) 2023-07-30 14:05:05 +02:00
Tyler Neely
53fcd8ac4d Add manual .py test verifying isolation levels (#407) 2023-07-30 14:04:26 +02:00
Tyler Neely
259cba5d43 Add manual .py test asserting forward progress across reconnects (#408) 2023-07-30 14:02:59 +02:00
Antonio Filipovic
285b409927 Improve all shortest paths memory usage (#981)
* Change allocator to PoolResource
2023-07-30 12:58:07 +02:00
Gareth Andrew Lloyd
8ebab84324 Add handling of partial results on timeout (#1046) 2023-07-30 10:48:11 +02:00
imilinovic
3fd9ce4a33 Add mgp::map erase and update (#1103) 2023-07-30 08:36:50 +02:00
Andi
be4eb95a98 Fix Jepsen replication pause (#1082) 2023-07-30 02:36:11 +02:00
Andi
903a9f4636 Set explicit node config for jepsen tests (#1088) 2023-07-30 00:49:48 +02:00
Andi
18bd02423a Fix PropertyStore buffer serialization (#1111) 2023-07-29 19:14:27 +02:00
andrejtonev
58c0c4cebb Add missing-field-initialization warning flag (#1113) 2023-07-29 17:59:11 +02:00
andrejtonev
110ca3968c Fix path generation ignores edge's element_id (#1108) 2023-07-29 15:51:51 +02:00
Andi
9072fb7703 Fix flaky transactional queue e2e test (#1102) 2023-07-29 11:11:27 +02:00
Matija Pintarić
2b7707a2f1 Add non-const return value for mgp::Value subtypes (#1099) 2023-07-28 12:26:10 +02:00
Matija Pintarić
76ca019f31 Add overload of operator< for mgp::Value (#1090) 2023-07-28 11:35:14 +02:00
imilinovic
609b9a20f1 Add hash on mgp::Value (#1093) 2023-07-28 09:08:36 +02:00
Matija Pintarić
e489e4f3e7 Extend insert on Record to accept mgp::Value (#1094) 2023-07-27 10:33:32 +02:00
Matija Pintarić
ab4d1efe0b Overload << operator for mgp::Value::Type (#1080) 2023-07-25 13:30:02 +02:00
Bruno Sačarić
036da58d30 Add standalone upload to S3 workflow (#866) 2023-07-22 19:21:17 +02:00
Marko Budiselić
919f07fae1 Add macOS support under environment/util.sh:operating_system (#1098)
* Fix release/package ubuntu-22.04 amd64 Dockerfile
2023-07-22 19:20:10 +02:00
Marko Budiselić
ca1e98ad94 Add package/release/run.sh build (#1060)
* Optimize `memgraph/memgraph-builder` (on Dockerhub) image size
* Optimize `mgbuild_{{ os }}` (under CI) image size
2023-07-22 18:00:38 +02:00
Marko Budiselić
f0aca2d23b Improve init script logging (#1096) 2023-07-22 13:08:39 +02:00
Marko Budiselić
ec9840fff2 Fix custom maven setup (#1044) 2023-07-22 12:29:36 +02:00
Marko Budiselić
992e718a97 Fix v2.9 release build issues (#1087)
* Fix include order inside `tests/unit/query_variable_start_planner.cpp` fails on an Rhel7 based OS
2023-07-20 16:15:27 +02:00
Marko Budiselić
2ec6b7f40b Update v2.9 license date (#1081) 2023-07-20 10:56:31 +02:00
Andi
05eca46267 Fix flaky transaction_queue_multiple unit test (#1077) 2023-07-19 22:58:02 +02:00
Vlasta
fae039c215 Improve logging for the on-disk storage (#1079) 2023-07-19 22:54:42 +02:00
Marko Budiselić
3b9133fd5a Improve e2e and replication testing setup (#1061)
* Add `--replication-restore-state-on-startup` with `false` as default

Co-authored-by: Aidar Samerkhanov <aidar.samerkhanov@memgraph.io>
Co-authored-by: Andi Skrgat <andi8647@gmail.com>
2023-07-19 21:18:43 +02:00
Marko Budiselić
9d056e7649 Add experimental/v1 of ON_DISK_TRANSACTIONAL storage (#850)
Co-authored-by: Andi Skrgat <andi8647@gmail.com>
Co-authored-by: Aidar Samerkhanov <aidar.samerkhanov@memgraph.io>
2023-06-29 11:44:55 +02:00
Antonio Filipovic
aa4f68a37d Add error handling on py batched module init (#1052) 2023-06-28 17:23:42 +02:00
Josipmrden
84721f7e0a Add vertex degree to index statistics (#1026)
Add graph analysis of vertex degrees when doing ANALYZE GRAPH.
2023-06-27 18:06:20 +02:00
Ante Javor
261aa4f49b Improve replication logging (#1030) 2023-06-27 17:57:51 +02:00
Josipmrden
5ce1526995 Fix map at operator returning null from a map (#1039)
Fix map returning null from a map and not throwing exception if key is not present
2023-06-27 15:04:12 +02:00
Antonio Filipovic
cb843ee664 Fix multiple include error on mgp.hpp (#1043) 2023-06-27 10:07:33 +02:00
Andi
0f1ca745e5 Improve connection handling in tests/e2e (#1012) 2023-06-26 22:43:34 +02:00
Gareth Andrew Lloyd
3b781bf525 Add HTTP+GZIP support to LOAD CSV (#1027) 2023-06-26 19:10:48 +02:00
Antonio Filipovic
d573eda8bb Add python & cpp batching option in procedures
* Add API for batching from the procedure 
* Use PoolResource for batched procedures
2023-06-26 15:46:13 +02:00
Marko Budiselić
00226dee24 Improve setup when memgraph is a git submodule (#1038) 2023-06-26 12:27:58 +02:00
Marko Budiselić
546bfc0ede Fix package workflow on ARM (#1040) 2023-06-26 11:19:13 +02:00
Gareth Andrew Lloyd
5b1ba10183 Fix IN_MEMORY_ANALYTICAL storage GC (#1025) 2023-06-23 12:50:03 +02:00
Vlasta
b25e9968ee Update links inside CSV import tool (#834) 2023-06-22 16:00:22 +02:00
Vlasta
bcd23fe3cb Update CSV import tool error docs 2023-06-22 14:44:46 +02:00
Katarina Supe
68e5610566 Fix replica exception message (#930) 2023-06-22 14:41:59 +02:00
Ante Javor
0ea96663ba Add check for opening snapshots (#966) 2023-06-22 13:29:49 +02:00
Marko Budiselić
da17fe92d6 Update package_docker by adding --pull (#1032)
Each release will pull the latest base image because we want to include any new
security patches.
2023-06-22 12:12:31 +02:00
Marko Budiselić
e73eac77a9 Improve libstdc++ dependency on RPM systems (#863) 2023-06-22 10:14:59 +02:00
Marko Budiselić
d51a61fc5f Update dependencies under environment/os (#862) 2023-06-21 23:14:37 +02:00
Josipmrden
b875649270 Add restoring of replication roles upon database startup (#791)
Fix replica node restoration on startup so it is restored as replica and not as main.
2023-06-21 19:08:58 +02:00
Josipmrden
05cc35bf93 Add command NULLIF for identifying nulls in LOAD CSV (#914)
Add NULLIF command which turns all row values corresponding to the string to the nullif character sequence.
2023-06-21 14:50:46 +02:00
Josipmrden
63f8298033 Fix MATCH + LOAD CSV to load CSV only once (#916)
* update profile query to use poolresource
* Optimize update of indexes
* Add ignore empty strings to load csv
* Add operator changes to support handling of nulls
* Store chunks in memory pools ordered
* Use same max block per chunks number
* Remove redundant return statement
* add hacky cached solution
* change map to set
* remove memory
* Add match load csv invalid behaviour commit
* Accept input on LOAD CSV
* Ommit changes not tied to the PR
* Add tests for match + load csv
* Add gqlalchemy installation for e2e tests
* Modify setup script to update packages
* Revert gqlalchemy to 1.3.3
* Revert gqlalchemy to 1.3.3
* Address PR review comments
* Ommit semicolon
---------

Co-authored-by: antoniofilipovic <filipovicantonio1998@gmail.com>
Co-authored-by: János Benjamin Antal <benjamin.antal@memgraph.io>
2023-06-21 11:13:40 +02:00
Josipmrden
df95775222 Fix init file startup in community edition (#974)
* Fix init file startup in community edition

* Add possibility to build binary without MG_ENTERPRISE

* Added trace spdlog for when init file is not present

* Add gqlalchemy and unit tests

* Add init data files which correspond to the right directory by the github actions
2023-06-20 17:54:50 +02:00
Josipmrden
eb22edfd35 Add any type to C++ mgp wrapper 2023-06-20 09:33:14 +02:00
Marko Budiselić
1b85d77e9e Add cypherl transform scripts under import/ (#773)
* Add `mglogs2cypherl.sh`
* Add `n2mg_cypherl.sh`
2023-06-18 11:57:46 +02:00
Marko Budiselić
cf1a86ed13 Refactor tests/integration/run.sh (#1016) 2023-06-15 23:10:52 +02:00
Marko Budiselić
7fb3f62703 Upgrade to RocksDB 8.1.1 (#1013) 2023-06-15 11:54:24 +02:00
Marko Budiselić
cb4b71bdbd Update pull_request_template.md 2023-06-14 16:04:35 +02:00
andrejtonev
30ec570bb9 Add Bolt v5 support (#938) 2023-06-12 18:55:15 +02:00
Antonio Filipovic
d917c3f0fd Fix slow IN LIST evaluation (#901) 2023-05-29 17:52:20 +02:00
andrejtonev
d842adbed3 Handle user-defined metadata and expose it with SHOW TRANSACTIONS(#945) 2023-05-29 11:40:14 +02:00
Bruno Sačarić
cdfcbc106c Update license date (#941) 2023-05-18 11:42:12 +02:00
Josipmrden
651b6f3a5a Expose system metrics over HTTP Endpoint (#940) 2023-05-18 05:10:57 +00:00
Ante Pušić
0d9bd74a8a Add support for map projection (#892) 2023-05-16 20:05:35 +02:00
andrejtonev
802f8aceda Add data directory status and (un)lock query (#933) 2023-05-16 18:36:04 +02:00
gvolfing
7ddce539fa Add return build type command (#894) 2023-05-16 16:02:03 +02:00
gvolfing
c3e4f81026 Include additional info inside storage mode info query (#883) 2023-05-16 14:25:41 +02:00
Antonio Filipovic
208705f296 Reduce memory consumption on return from python procedures (#932) 2023-05-16 10:33:09 +02:00
Ante Javor
69634a5354 Fix typo in mgbench 2023-05-10 14:02:46 +02:00
Aidar Samerkhanov
b8f282468d Update pulsar client for e2e tests 2023-05-09 12:23:28 +02:00
Ante Javor
ab38161cd2 FIix methodology links (#903)
Co-authored-by: Josip Mrden <josip.mrden@memgraph.io>
2023-05-03 16:37:36 +02:00
János Benjamin Antal
3a5f140c2b Order chunks in utils::Pool to speed up deallocation (#898) 2023-05-02 13:08:20 +02:00
Ante Javor
eead0f79fc Fix missing argument in daily benchmark (#907) 2023-05-02 11:00:44 +02:00
Antonio Filipovic
91017b7f36 Update profile query to use PoolResource for LOAD CSV (#885) 2023-04-26 18:04:13 +02:00
gvolfing
00f8d54249 Parallelize index creation (#882) 2023-04-26 16:28:02 +02:00
János Benjamin Antal
4fcdd52f88 Use correct memory resource (#900) 2023-04-26 10:02:55 +02:00
János Benjamin Antal
6c947947eb Parallelize recovery (#868)
* Parallelize edge recovery

* Load vertex labels and properties parallel

* Add parallel connectivity loading

* Add batches information to snapshot

* Introduce `items_per_batch` and `recovery_thread_count` flags

* Make possible to load snapshots with old version

* Add vertex batches to `RecoveryInfo`

* Extend durability integration tests with v15 test cases

* Add `std::vector` based `InitProperties`

* Use `InitProperties` in snapshot loading
2023-04-25 16:25:25 +02:00
Ante Javor
64fd281b2e Update benchgraph methodology (#899) 2023-04-25 09:45:25 +02:00
János Benjamin Antal
97e250129e Change AccumulateCursor to use utils::pmr::deque (#888)
* Increase performance by eliminating unnecessary `TypedValue` copies
2023-04-24 16:22:22 +02:00
Marko Budiselić
b02b201129 Improve Jepsen setup (#893) 2023-04-23 16:16:49 +02:00
Antonio Filipovic
2c6a55775d Fix max block size bug on LOAD CSV(#877) 2023-04-19 16:10:20 +02:00
Ante Javor
940bf6722c Add mgbench tutorial (#836)
* Add Docker runner
* Add Docker client
* Add benchgraph.sh script
* Add package script
2023-04-19 08:21:55 +02:00
Bruno Sačarić
49b5343238 Update license year (#867) 2023-04-05 11:06:32 +02:00
Andi
26a0866938 Fix index optimization bug (#860) 2023-04-04 23:43:13 +02:00
Bruno Sačarić
6545283dac Add upload to S3 job to Package All workflow (#845)
* add to package_all

* add separate workflow

* make reduced jobs version for testing

* typo

* exclude amzn-2 in init script, because isort 5.12. fails to install

* change dir name

* move env var for release version

* bugfix

* Revert "make reduced jobs version for testing"

This reverts commit 7bb75f34a4.

* remove releaes folder

* extend timeout for arm builds

* increase timeout limit to dangerous levels

* revert timeouts, fix upload naming

* remove untested workflow
2023-04-04 21:54:25 +02:00
Bruno Sačarić
69c735934c Update docker credentials (#853) 2023-04-04 21:47:04 +02:00
Antonio Filipovic
64e837b355 Introduce analytics mode (#772) 2023-04-04 18:46:26 +02:00
Antonio Filipovic
a586f2f98d Change EvalContext and QueryExecution to use PoolResource on LOAD CSV (#825)
* Change PullPlan to use specific PoolResource for LOAD CSV
2023-04-04 16:54:08 +02:00
Josipmrden
9fc51f74a0 Skip label based auth on user with global visibility on graph (#837) 2023-04-04 11:13:25 +02:00
Josipmrden
128771a6ec Add SHA-256 password encryption (#839) 2023-04-03 16:29:21 +02:00
Josipmrden
f5a49ed29f Add Cypher subqueries (#794) (#851)
Co-authored-by: Bruno Sačarić <bruno.sacaric@gmail.com>
2023-03-31 13:49:10 +00:00
Josipmrden
398503da7a Add index statistics for better query planning (#812) 2023-03-30 15:34:34 +02:00
Bruno Sačarić
0819b40202 Fix bug on AllShortest with multiple edges between nodes (#832) 2023-03-29 16:39:41 +02:00
Andi
029be10f1d Add queries to show or terminate active transactions (#790) 2023-03-27 15:46:00 +02:00
Aidar Samerkhanov
a9dc344b49 Add automatic CPU architecture detection to CMake (#838) 2023-03-27 11:26:10 +02:00
Marko Budiselić
8b0dca9eab Upgrade pre-commit hook to use isort 5.12 (#840) 2023-03-26 17:34:51 +02:00
Ante Javor
cb813c3070 Add bigger LDBC dataset to mgbench (#747) 2023-03-21 21:44:11 +01:00
Ante Javor
6349fc9501 Add time-depended execution to the mgbench client (#805) 2023-03-18 20:18:58 +01:00
Jure Bajic
c4167bafdd Add support for Amazon Linux 2 and stop generating C++ using Lisp/LCP (#814) 2023-03-14 19:24:55 +01:00
Bruno Sačarić
6f51141148 Update license year(#821) 2023-03-08 10:46:21 +01:00
Ante Pušić
97d45ab1d8 Add Python query module API mock (#757) 2023-03-07 15:41:19 +01:00
Josipmrden
6abd356d01 [master < E214] WHERE Exists feature (#818)
Add WHERE exists() to filter based on neighbouring pattern expressions
2023-03-07 00:28:41 +01:00
Vlasta
99a6c72bba Change message on incompatible epoch_id error (#786) 2023-03-06 20:01:02 +01:00
Ante Pušić
173f5430aa Remove noexcept from functions that may throw (#819) 2023-03-06 17:34:34 +01:00
Jure Bajic
362dc95e27 Add support for Ubuntu 22.04 ARM (#810) 2023-03-01 18:44:56 +01:00
Kruno Golubic
eead24a562 Add Lima badge to README (#802) 2023-02-24 17:08:07 +01:00
Antonio Filipovic
d79dd69607 Improve performance with props init on node|edge creation (#788) 2023-02-24 15:40:35 +01:00
Katarina Supe
024bf0c578 Merge pull request #806 from memgraph/vpavicic-patch-1
Update README.md
2023-02-23 15:06:40 +01:00
Vlasta
d7443a5558 Update README.md 2023-02-23 15:05:03 +01:00
Kruno Golubic
2df357b012 Add RedHat badge to README (#796) 2023-02-20 18:18:29 +01:00
Kruno Golubic
b2b5a6e2a0 Add Fedora badge to README (#795) 2023-02-20 17:59:57 +01:00
Ante Javor
5e2ee6c817 Improve mgbench C++ client (#760) 2023-02-17 17:54:05 +01:00
Antonio Filipovic
862a1afdf1 Improve Visit performance (#774) 2023-02-17 13:09:25 +01:00
Antonio Filipovic
bbce21e78f Update pull request template (#775) 2023-02-17 11:50:17 +01:00
Jure Bajic
15c8662023 Add support for Fedora 36 (#787) 2023-02-17 10:47:36 +01:00
Katarina Supe
beaba0fc16 Update README with Cloud, Lab and import info (#768) 2023-02-07 15:43:41 +01:00
Josipmrden
8f70c5f2a5 Fix label-based auth using OLD view instead of NEW when merging nodes (#755) 2023-02-01 13:20:26 +01:00
Bruno Sačarić
14c651d3ba edit workflows (#756) 2023-01-31 23:20:49 +01:00
Andi
04efc7a4a6 Remove torch and igraph from sys cache (#720) 2023-01-31 13:11:59 +01:00
Bruno Sačarić
19832b5838 Merge pull request #749 from memgraph/update-license-year-2.5.2
[master < MG] Update license year
2023-01-26 13:08:47 +01:00
Bruno Sačarić
e63fae2d5b Update license year 2023-01-26 13:02:33 +01:00
Bruno Sačarić
34dd47ef07 Fix nested FOREACH shadowing bug (#725) 2023-01-25 20:06:05 +01:00
Jure Bajic
d3f275b231 Exclude license dir from CI trigger (#740) 2023-01-25 18:49:02 +01:00
Ante Pušić
aad4bcb7a0 Fix C++ API memory leak on Relationships() (#743) 2023-01-25 17:23:46 +01:00
Bruno Sačarić
034b54cb72 Fix bug on all shortest paths with an upper bound (#737) 2023-01-25 15:32:00 +01:00
Josipmrden
8cf51d9f68 Fix bug in query plan to use indexes on optional match and foreach (#736)
* Add fix in query plan to use indexes on optional match and foreach
2023-01-25 12:53:33 +01:00
Antonio Filipovic
1cd1da84fd Fix bug on (vertex|edge) properties in C++ API (#732) 2023-01-23 12:57:17 +01:00
Jure Bajic
128a6cd522 Update license year (#739) 2023-01-19 12:31:59 +01:00
niko4299
d9eeedb9ee Adding qid in bolt (#721) 2023-01-18 16:33:03 +01:00
Andi
156e2cd095 On delete triggers invalid edge reference (#717)
* Added check if there is invalid reference to the underlying edge

* Added fix and e2e tests

* Isolation levels tracking based on from_vertex_

* Added explicit transaction test + edge accessor changes based on the vertex_edge

* Autocommit on tests, initialize deleted by checking out_edges

Co-authored-by: Marko Budiselić <marko.budiselic@memgraph.com>
2023-01-18 15:05:10 +01:00
Ante Javor
8b834c702c Update mgbench to run Diff workflow under 30mins (#730) 2023-01-14 16:11:49 +01:00
Katarina Supe
eda5213d95 Release pypi mgp 1.1.1 (#727) 2022-12-24 09:33:53 +01:00
Bruno Sačarić
1f2a15e7c8 Fix MATCH not allowed on replica (#709) 2022-12-23 14:47:12 +01:00
Ante Javor
d72e7fa38d Fix mgp.py create edge type hint and comment (#724) 2022-12-23 10:08:52 +01:00
Antonio Filipovic
e5e37bc14a Fix LOAD CSV large memory usage (#712) 2022-12-22 19:38:48 +01:00
Katarina Supe
3ee068bbf9 Update build badge (#722) 2022-12-20 21:56:05 +01:00
Jure Bajic
68e846b182 Update license year (#711) 2022-12-13 13:54:09 +01:00
Andi
310e305cfb Fix python module reloading (#706) 2022-12-12 21:11:13 +01:00
Marko Budiselić
9d6a23b6bd Add init-file and init-data-file capabilities (#696) 2022-12-09 18:50:33 +01:00
Andi
f2d5ab61c4 Fix Python submodules reloading (#653) 2022-12-09 14:30:41 +01:00
Andi
0f77c85824 Fix cursor exhaustion by adding EmptyResult operator (#667) 2022-12-09 11:44:07 +01:00
niko4299
d6d4153fb7 Fix graph projection bug (#697) 2022-12-08 13:45:20 +01:00
Tyler Neely
7d6a5e5b9c Add support for -h to show help in addition to --help (#682) 2022-12-07 16:51:32 +01:00
Vlasta
c529d52664 Add community links to the README (#693) 2022-12-07 12:42:48 +01:00
Ante Pušić
45451bae3b Fix C++ query modules API bugs (#688) 2022-12-06 16:57:50 +01:00
niko4299
3e11f38548 Add aggregation distinct (#654) (#665) 2022-12-03 13:48:44 +02:00
Jure Bajic
6e4047a847 Bump mgconsole version (#660) 2022-12-01 13:10:08 +01:00
Ante Javor
8febdc12fb Update tests/mgbench README (#679) 2022-11-30 12:43:57 +01:00
Kruno Golubic
3f23a10f44 Update README with one new paragraph and link (#675) 2022-11-29 14:24:02 +01:00
Ante Javor
11300960de Add mixed workload and Neo4j client to mgbench (#566)
* Fix bolt bug inside the C++ client
* Add tail latency stats
* Add hot run option
* Add query caching
* Add jcmd memory tracking
2022-11-28 08:47:22 +01:00
Jure Bajic
1d5f387ddd Add python check (#643) 2022-11-09 11:48:34 +02:00
Marko Budiselić
c4c3a254bf Remove tools/check-build-system (#642) 2022-11-07 18:54:22 +01:00
Jure Bajic
b4beb0fc86 Update license for release 2.4.2 (#641) 2022-11-07 13:07:20 +01:00
Bruno Sačarić
58e6097664 Fix ALLSHORTEST combined with id function (#636) 2022-11-04 19:36:03 +01:00
Jure Bajic
ff21c0705c Add multiple license support (#618)
Make license info available through LicenseChecker
Add LicenseInfoSender
Move license library from utils
Rename telemetry_lib to mg-telemetry
2022-11-04 15:23:43 +01:00
Katarina Supe
2a2b99b02a Release mgp 1.1.0 (#620)
- Updated its README, authors, and published it with the added
   AuthorizationError that was a part of Memgraph 2.4.0 release.
 - Added missing SOURCE_TYPE_KAFKA and SOURCE_TYPE_PULSAR variables in _mgp.
2022-11-02 14:14:12 +01:00
Antonio Filipovic
3daab6ce97 Fix bug in the C API in-edges iterator (#613) 2022-10-25 19:18:44 +02:00
Marko Budiselić
fbd7274c95 Add Fedora 36 as an OS (#599) 2022-10-21 15:22:50 +02:00
Marko Budiselić
6efc84f022 Add libc++ option to the toolchain (#567) 2022-10-20 07:52:59 +02:00
Jure Bajic
287c2e94d1 Update license date (#586) 2022-10-07 14:42:52 +02:00
Antonio Filipovic
417cf4b30b Fix bug related to EdgeType and Label getters in query modules (#582)
Co-authored-by: Kostas Kyrimis <kostaskyrim@gmail.com>
2022-10-06 21:21:11 +02:00
Jure Bajic
68e7fd3d36 Fix architecture check (#583) 2022-10-06 15:55:23 +02:00
Bruno Sačarić
5261d82063 Fix passing user's fine_grained_access_handler instead of role's (#579)
Co-authored-by: Jure Bajic <jure.bajic@memgraph.com>
2022-09-30 18:27:47 +02:00
Jure Bajic
9eb87bcf3e Update release process (#564)
* Fix centos 7 virtualenv issue
* Fix ubuntu release issue
* Fix architecture check
2022-09-20 18:42:15 +02:00
Jure Bajic
a9491d3e68 Update license date (#561) 2022-09-20 14:23:24 +02:00
Marko Budiselić
b42e47b0be Reduce the size of TypedValue (#560)
- Reduce the size of TypedValue
- Fix double allocation
- Add `Graph` to `TypedValue` unit tests
- Fix allocator usage in `TypedValue`
- Add graph projection to `long_running.cpp` stress test
2022-09-20 14:21:34 +02:00
Marko Budiselić
898c894a48 Merge pull request #484 from memgraph/E129-MG-label-based-authorization
* Add label-based authorization

Co-authored-by: Boris Taševski <boris.tasevski@memgraph.com>
Co-authored-by: Josip Mrden <josip.mrden@memgraph.com>
Co-authored-by: Niko Krvavica <niko.krvavica@memgraph.com>
Co-authored-by: Bruno Sacaric <bruno.sacaric@memgraph.com>
2022-09-16 15:12:54 +02:00
niko4299
a3c2492672 Add fine grained access control to mgbench (#522) 2022-09-15 21:33:15 +02:00
Boris Taševski
a0b8871b36 Fix cland tidy errors and other warning (#555) 2022-09-15 15:51:35 +02:00
Marko Budiselic
bb6cf35441 Merge master cpp module API 2022-09-15 11:29:52 +02:00
Ante Pušić
5bc301d21d Add C++ query modules API (#546)
Co-authored-by: Ante Pusic <ante.pusic@memgraph.com>
Co-authored-by: Josip Mrden <josip.mrden@memgraph.com>
2022-09-15 11:26:26 +02:00
Marko Budiselic
1b89e679df Merge master 2022-09-15 08:32:41 +02:00
Boris Taševski
43e0520bc8 Merge master (#554) 2022-09-15 07:25:36 +02:00
Bruno Sačarić
2c8e45e889 Add run_id to the query summary (#548) 2022-09-14 20:21:06 +02:00
Boris Taševski
0876a8848d Merge master to epic and fix differences (#552) 2022-09-14 18:36:21 +02:00
Boris Taševski
fb4641a6be Fix logic in fine grained permissions (#551) 2022-09-14 12:39:23 +02:00
niko4299
201f75e809 Add MG_ENTERPRISE and license checks (#547) 2022-09-14 01:10:28 +02:00
niko4299
dc8dad9794 Add authorization in SetLabels, RemoveLabels, Allshortestpath cursor (#537) 2022-09-13 17:14:23 +02:00
Boris Taševski
aa02745915 [E129-MG < T1030-MG] Tech debts (#540)
* renamed parameters (#539)

* added variable declarations in ifs; minor code improvements; (#541)

* dba parameter removed (#543)

* Accept -> Has rename; HasGlobalPermissionOnVertices/Edges -> HasGlobalPrivilegeOnVertices/Edges (#545)

* replaced passing dba from reference to pointer
2022-09-13 11:37:17 +02:00
Josipmrden
b2d5a8eeca [E129-MG < T1040-MG] Add exceptions in LBA cursors (#536)
Exceptions added in update and create delete operators instead of logging
2022-09-12 14:04:40 +02:00
Boris Taševski
c09b175c76 [E129-MG < T1006-MG] Expand C API with LBA checks (#527)
* [T1006-MG < T1017-MG] Add LBA checks to all read procedures in C API (#515)

* Initial Impl

* NextPermittedEdge introduced

* revert moving constructor to cpp

* edge from and edge to methods expanded with lba check

* minor fix

* added check to path expand procedure

* Added integration tests for read query procedures

* additional check

* changed iterator type to reference

* comments from pr

Co-authored-by: Josip Mrden <josip.mrden@memgraph.io>

* [T1006-MG < T1018-MG] Add LBA checks to all update procedures in C API (#516)

* Initial Impl

* NextPermittedEdge introduced

* revert moving constructor to cpp

* edge from and edge to methods expanded with lba check

* minor fix

* extended update methods

* added check to path expand procedure

* Added integration tests for read query procedures

* Added integration tests for update query modules

* additional check

* changed iterator type to reference

* fixed bug in Update property for node; fixed 2 e2e tests

* replaced enum

Co-authored-by: Josip Mrden <josip.mrden@memgraph.io>

* [T1006-MG < T1019-MG] Add LBA checks to all Create and Delete procedures in C API (#517)

* Initial Impl

* NextPermittedEdge introduced

* revert moving constructor to cpp

* edge from and edge to methods expanded with lba check

* minor fix

* extended update methods

* initial implementation

* added check to path expand procedure

* Added integration tests for read query procedures

* Added integration tests for update query modules

* Added unit tests for creation of vertex, adding and removing vertex label

* additional check

* changed iterator type to reference

* Added unit tests for create edge

* Corrected query module in create edge

* fixed bug in Update property for node; fixed 2 e2e tests

* fixed merge errors

* Expanded FineGrainedAuthChecker with HasGlobalPermissionOnVertices and HasGlobalPermissionOnEdges

* Removed two wrong checks; Added two global checks

* return null added

* introduced new mgp_error value

* fixed endless loop

* replaced enum

* intermediate

* tests updated

* PermissionDeniedError -> AuthorizationError rename

* rename in enum permission_denied error -> authorization error

* mgp_vertex_remove_label check improved

* quotes changed; order of imports fixed

* string constant introduced

* import fixed

* yaml format

Co-authored-by: Josip Mrden <josip.mrden@memgraph.io>

Co-authored-by: Josip Mrden <josip.mrden@memgraph.io>
2022-09-08 17:48:34 +02:00
Kostas Kyrimis
f1fe77adfb Graph project feature implementation (#508) (#535) 2022-09-07 16:00:49 +03:00
Josip Mrden
35f8978560 Merge branch 'master' into E129-MG-label-based-authorization 2022-09-07 09:28:32 +02:00
Josip Matak
9e8fb2516b Add all shortest path algorithm (#409) 2022-09-06 16:21:32 +02:00
Josip Mrden
0a66feccff Merge branch 'master' into E129-MG-label-based-authorization 2022-09-06 11:14:27 +02:00
Boris Taševski
d008a2ad8d [E129-MG < T1007-MG] Expand Cursors with LBA checks (#524)
* [T1007-MG < T0997-MG] Authorization on paths (#501)

* Added read authorization in paths operators

* [T1007-MG < T1016-MG] Added authorization in create and delete operators (#513)

* Added authorization in RemoveNodeCursor, RemoveExpandCursor, CreateNodeCursor, CreateExpandCursor,MergeCursor

* [T1007-MG < T1014-MG] Add authorization to read operators (#520)

Added label based access control to read operators (ScanAll).

* [T1007-MG < T1015-MG] Add authorization to update operators (SetProperty, SetProperties, RemoveProperty) (#521)

Added label based authorization to update operators

Co-authored-by: niko4299 <51059248+niko4299@users.noreply.github.com>
Co-authored-by: Josip Mrden <josip.mrden@memgraph.io>
2022-09-02 17:12:07 +02:00
Josipmrden
7478300762 [E129-MG < T997-MG] Show label privileges (#506)
Added showing of label privileges functionality to fine grained access control.
2022-08-31 12:14:16 +02:00
János Benjamin Antal
0bc298c3ad Fix handling of the ROUTE Bolt message (#475)
The fields of ROUTE message were not read from the input buffer, thus the
input buffer got corrupted. Sending a new message to the server would result
reading the remaining fields from the buffer, which means reading some values
instead of message signature. Because of this unmet expectation, Memgraph closed
the connection. With this fix, the fields of the ROUTE message are properly
read and ignored.
2022-08-26 13:19:27 +02:00
Boris Taševski
05f120b7d4 [E129-MG < T1004-MG] Expand cypher with more granular label permissions (#500)
* Added enum for more granular access control; Expanded functionality of fine grained access checker; Propagated changes to Edit, Deny and Revoke permissions methods in interpreter

* Introduced Merge method for merging two colle with permissions

* e2e tests implementation started

* Expanded cypher to support fine grained permissions

* ast.lcp::AuthQuery removed labels, added support for label permissions

* promoted label permissions to vector

* removed unnecesary enum value

* expanded glue/auth with LabelPrivilegeToLabelPermission

* added const

* extended Grant Deny and Revoke Privileges with new label privileges

* extended Edit Grant Deny and Revoke Privileges to properly use new model

* Fixed unit tests

* FineGrainedAccessChecker Grant and Deny methods reworked

* Revoke cypher slightly reworked; Revoke for labels works without label permissions

* EditPermission's label_permission lambda now takes two parameters

* constants naming enforced; replaced asterisks with string constant

* removed faulty test addition

* Naming fixes; FineGrainedAccessChecker unit tests introduced

* unnecessary includes removed; minor code improvements

* minor fix

* Access checker reworked; denies and grant merged into single permission object; Created global_permission that applies to all non-created permissions. Grant, Deny and Revoke reworked; Merge method reworked

* Fixed wrong check;

* Fix after merge; renamed constants; removed unused constant

* Fix after merge; workloads.yaml for lbaprocedures e2e tests updated with new grammar

* Fixes after merge

* Fixes after merge

* fixed Revoke that was not fixed after the merge

* updated cypher main visitor tests

* PR review changes; Naming and const fixed, replaced double tertiary with lambda

* unwrapping the iterator fix

* merge 1003 minor fix

* minor spelling fixes

* Introduced visitPrivilegesList because of the doubled code

* const added

* string const to enum

* redundant braces

* added const

* minor code improvement

* e2e tests expanded

* if -> switch

* enum class inherits uint8_t now

* LabelPrililege::EDIT -> LabelPrivilege::UPDATE

* LabelPermission -> EntityPermission; LabelPrivilege -> EntityPrivilege

* EntityPrivilege -> FineGrainedPrivilege; EntityPermission -> FineGrainedPermission
2022-08-22 14:11:43 +02:00
antoniofilipovic
d73d153978 Add logging API (#417) 2022-08-22 14:47:52 +03:00
Boris Taševski
b489ac7cff [E129-MG < T1003-MG] Expand fine grained access checker with more granular permissions (#496)
* Added enum for more granular access control; Expanded functionality of fine grained access checker; Propagated changes to Edit, Deny and Revoke permissions methods in interpreter

* Introduced Merge method for merging two colle with permissions

* e2e tests implementation started

* FineGrainedAccessChecker Grant and Deny methods reworked

* removed faulty test addition

* Naming fixes; FineGrainedAccessChecker unit tests introduced

* unnecessary includes removed; minor code improvements

* Access checker reworked; denies and grant merged into single permission object; Created global_permission that applies to all non-created permissions. Grant, Deny and Revoke reworked; Merge method reworked

* Fixed wrong check;

* PR review changes; Naming and const fixed, replaced double tertiary with lambda

* unwrapping the iterator fix

* minor spelling fixes
2022-08-18 16:59:38 +02:00
niko4299
e15576f56c [E129-MG <-T0982-MG] implement edge type filtering (#489)
* GRANT, REVOKE, DENY and access_checker DONE

* Added AccessChecker to ExecutionContext

* grammar expanded; (#462)

* current

* T0954 mg expand user and role to hold permissions on labels (#465)

* added FineGrainedAccessPermissions class to model

* expanded user and role with fine grained access permissions

* fixed grammar

* [E129 < T0953-MG] GRANT, DENY, REVOKE added in interpreter and mainVisitor (#464)

* GRANT, DENY, REVOKE added in interpreter and mainVisitor

* Commented labelPermissons

* remove labelsPermission adding

* Fixed

* Removed extra lambda

* fixed

* [E129<-T0955-MG] Expand ExecutionContext with label related information (#467)

* added

* Added FineGrainedAccessChecker to Context

* fixed

* Added filtering

* testing

* Added edge filtering to storage, need to add filtering in simple Expand in operator.cpp

* Removed storage changes

* MATCH filtering working

* EdgeTypeFiltering working, just need to test everything again

* Removed FineGrainedAccessChecker

* Removed Expand Path

* Fix

* Tested FineGrainedAccessHandler, need to test AuthChecker

* Added integration test for lba

* Fixed merge conflicts

* PR fix

* fixed

* PR fix

* Fix test

* removed .vscode, .cache, .githooks

* githooks

* added tests

* fixed build

* Changed ast.lcp and User pointer to value in context.hpp

* Fixed test

* Remove denies on grant all

* AuthChecker

* Pr fix, auth_checker still not fixed

* Create mg-glue and extract UserBasedAuthChecker from AuthChecker

* Build fixed, need to fix test

* e2e tests

* e2e test working

* Added unit test, e2e and FineGrainedChecker

* Mege E129, auth_checker tests

* Fixed test

* e2e fix

Co-authored-by: Boris Taševski <36607228+BorisTasevski@users.noreply.github.com>
Co-authored-by: josipmrden <josip.mrden@external-basf.com>
Co-authored-by: János Benjamin Antal <benjamin.antal@memgraph.io>
2022-08-16 15:57:23 +02:00
Boris Taševski
a98463b0bd [E129 < T0996] C-API: Implement using Fine Grained Access Checker in iterator over vertices (#494)
* implemented skipping vertices in Constructor and mgp_vertices_iterator_next

* Added utility function for moving iterator to next permitted vertex

* removed ifdef directive

* NextPermitted parameter type changed from mgp_vertices_iterator* to mgp_vertices_iterator&

* created support for lba-procedures e2e testing; Added test for vertex iterator skipping unauthorized vertices

* removed fixture from tests; converted generator to regular function;
2022-08-12 19:34:47 +02:00
Kruno Golubic
705631a35d Create README file for CSV Import Tools (#493)
Co-authored-by: Marko Budiselić <marko.budiselic@memgraph.com>
2022-08-11 16:10:36 +02:00
Jeremy B
d4f0bb0e38 Correct inconsistencies w.r.t. sync replication (#435)
Add a report for the case where a sync replica does not confirm within a timeout:
-Add a new exception: ReplicationException to be returned when one sync replica does not confirm the reception of messages (new data, new constraint/index, or for triggers)
-Update the logic to throw the ReplicationException when needed for insertion of new data, triggers, or creation of new constraint/index
-Add end-to-end tests to cover the loss of connection with sync/async replicas when adding new data, adding new constraint/indexes, and triggers

Add end-to-end tests to cover the creation and drop of indexes, existence constraints, and uniqueness constraints

Improved tooling function mg_sleep_and_assert to also show the last result when duration is exceeded
2022-08-09 11:29:55 +02:00
Jure Bajic
531db2d47c Fix WebSocket test (#485)
* Fix websocket unit tests hanging
* Rename websocket to monitoring unit test
2022-08-08 14:49:48 +02:00
Boris Taševski
116262d9a0 [E129 < T0956] Filtering nodes in ScanAll cursor [Niko] (#492)
* implemented scanall filtering

* minor code refactor

* FindNextNode -> FindNextVertex
2022-08-04 19:20:17 +02:00
gvolfing
bbfef45b37 Add command to return startup config (#459)
Add a new command that is able to return the set of configurations that that the
given instance of memgraph was started up with. The returned information
currently consists of the name, the default and the current value of each flag.
The hidden property of three flags were removed, namely --query-cost-planner,
--query-vertex-count-to-expand-existing and --query-max-plans. The flag
--log-link-basename was completely removed since it is not used.
2022-08-03 18:08:44 +02:00
János Benjamin Antal
05b00edfd4 Declare mgp_func_context outside the callback function (#481) 2022-08-03 15:15:53 +02:00
Boris Taševski
480df4ed69 Merge old Label Based Auth Epic branch into new one because of commits with bad checks on the old epic branch (#478)
* grammar expanded; (#462)

* T0954 mg expand user and role to hold permissions on labels (#465)

* added FineGrainedAccessPermissions class to model

* expanded user and role with fine grained access permissions

* fixed grammar

* [E129 < T0953-MG] GRANT, DENY, REVOKE added in interpreter and mainVisitor (#464)

* GRANT, DENY, REVOKE added in interpreter and mainVisitor

* Commented labelPermissons

* remove labelsPermission adding

* Removed extra lambda

* [E129<-T0955-MG] Expand ExecutionContext with label related information (#467)

* Added FineGrainedAccessChecker to Context

* fixed failing tests for label based authorization (#480)

* Marked FineGrainedAccessChecker ctor explicit; Introduced change to clang-tidy; (#483)

Co-authored-by: niko4299 <51059248+niko4299@users.noreply.github.com>
2022-08-02 12:51:22 +02:00
Marko Budiselić
80e0e439b7 Add parser stress test (#463) 2022-07-26 20:54:56 +02:00
Jure Bajic
351258ace8 Update clang-tidy job (#476)
Update clang tidy job to only check the diff against the base branch
2022-07-26 16:53:56 +02:00
Marko Budiselić
74d3663821 Upgrade Antrl to 4.10.1 and remove antlr_lock (#441) 2022-07-26 08:31:38 +02:00
gvolfing
eb0b3141d5 Fix aggregation functions on null and group-by inputs (#448)
The `sum()` and `count()` functions were giving results different from the openCypher specification on null `input.` The aggregation functions also had a problem when they were used in a group-by context and were giving results that were not compliant with the openCypher specification.
2022-07-19 12:16:32 +02:00
Marko Budiselić
ff2f8031a9 Extend the PR template with the git message guide (#451) 2022-07-15 14:08:10 +02:00
Marko Budiselić
094d4f282d Improve error-prone OS setup (#428) 2022-07-13 17:48:01 +02:00
Jure Bajic
3dd2657320 Create mgp python package (#433) 2022-07-12 10:54:23 +02:00
gvolfing
6fe474282a Modify logaical operators to conform openCyper regarding checking against NULL in CASE expressions (#432)
* Make `IfOperator` return the `else_expression_` in case of `NULL`

* Add gql_behave tests

* Add gql_behave test to specifically check for the case when the test expression itself is null
2022-07-11 15:00:29 +02:00
gvolfing
7fc0fb6520 Implement ToString function for temporal datatypes (#429)
* Modify `toString` to be able to handle `Date`, `LocalTime`, `LocalDateTime` and `Duration`

* Add unit tests

* Make `operator<<` use the `ToString()` implementations

* Add tests to verify the correctness of negative durations

* Add more tests to look for cases when the individual duration entities overflow.
2022-07-11 13:44:27 +02:00
Jeremy B
063e297e1e Avoid usage of time.sleep (#434)
e2e python: added tooling function around `time.sleep()` that stops as soon as condition is fulfilled and will raise assert if timeout is reached
2022-07-08 10:47:18 +02:00
Ante Javor
86b1688192 Rewrite Python API comments and snippets (#420)
* Update comments
2022-07-07 15:05:56 +02:00
Jeremy B
f629de7e60 Save replication settings (#415)
* Storage takes care of the saving of setting when a new replica is added

* Restore replicas at startup

* Modify interactive_mg_runner + memgraph to support that data-directory can be configured in CONTEXT

* Extend e2e test

* Correct typo

* Add flag to config to specify when replication should be stored (true by default when starting Memgraph)

* Remove un-necessary "--" in yaml file

* Make sure Memgraph stops if a replica can't be restored.

* Add UT covering the parsing  of ReplicaStatus to/from json

* Add assert in e2e script to check that a port is free before using it

* Add test covering crash on Jepsen

* Make sure applciaiton crashes if it starts on corrupted replications' info

Starting with a non-reponsive replica is allowed.

* Add temporary startup flag: this is needed so jepsen do not automatically restore replica on startup of main. This will be removed in T0835
2022-07-07 13:30:28 +02:00
Jeremy B
b737e53456 Remove sync with timeout (#423)
* Remove timout when registering a sync replica

* Simplify jepsen configuration file

* Remove timeout from jepsen configuration

* Add unit test

* Remove TimeoutDispatcher
2022-07-05 09:40:50 +02:00
János Benjamin Antal
10ca68bb2a Remove CODEOWNERS (#427)
Co-authored-by: Marko Budiselić <marko.budiselic@memgraph.com>
2022-07-04 16:42:36 +02:00
Jure Bajic
bfbd8538d4 Update docker release process (#421)
* Fix release directory

* Update release process

* Fix debian arm path
2022-07-04 16:10:33 +02:00
823 changed files with 97056 additions and 19771 deletions

View File

@@ -1,6 +1,7 @@
--- ---
BasedOnStyle: Google
---
Language: Cpp Language: Cpp
BasedOnStyle: Google
Standard: "c++20" Standard: "c++20"
UseTab: Never UseTab: Never
DerivePointerAlignment: false DerivePointerAlignment: false

View File

@@ -1,9 +1,14 @@
[master < Epic] PR [master < Epic] PR
- [ ] Check, and update documentation if necessary - [ ] Check, and update documentation if necessary
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Write E2E tests - [ ] Write E2E tests
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch - [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
- [ ] Provide the full content or a guide for the final git message
[master < Task] PR [master < Task] PR
- [ ] Check, and update documentation if necessary - [ ] Check, and update documentation if necessary
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog) - [ ] Provide the full content or a guide for the final git message
To keep docs changelog up to date, one more thing to do:
- [ ] Write a release note here, including added/changed clauses
- [ ] Tag someone from docs team in the comments

View File

@@ -3,7 +3,7 @@ name: Daily Benchmark
on: on:
workflow_dispatch: workflow_dispatch:
schedule: schedule:
- cron: "0 1 * * *" - cron: "0 22 * * *"
jobs: jobs:
release_benchmarks: release_benchmarks:
@@ -16,7 +16,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -67,7 +67,7 @@ jobs:
- name: Run mgbench - name: Run mgbench
run: | run: |
cd tests/mgbench cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/* ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results - name: Upload mgbench results
run: | run: |

View File

@@ -14,6 +14,7 @@ on:
- "**/*.md" - "**/*.md"
- ".clang-format" - ".clang-format"
- "CODEOWNERS" - "CODEOWNERS"
- "licenses/*"
jobs: jobs:
community_build: community_build:
@@ -26,7 +27,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -64,13 +65,18 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
fetch-depth: 0 fetch-depth: 0
- name: Build combined ASAN, UBSAN and coverage binaries # This is also needed if we want do to comparison against other branches
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
- name: Fetch all history for all tags and branches
run: git fetch
- name: Initialize deps
run: | run: |
# Activate toolchain. # Activate toolchain.
source /opt/toolchain-v4/activate source /opt/toolchain-v4/activate
@@ -78,6 +84,32 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Python code analysis
run: |
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
for file in ${CHANGED_FILES}; do
echo ${file}
if [[ ${file} == *.py ]]; then
python3 -m black --check --diff ${file}
python3 -m isort --check-only --diff ${file}
fi
done
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build cd build
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON .. cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit make -j$THREADS memgraph__unit
@@ -105,7 +137,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage - name: Save code coverage
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz path: tools/github/generated/code_coverage.tar.gz
@@ -115,7 +147,7 @@ jobs:
source /opt/toolchain-v4/activate source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts # Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported # Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null ! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
@@ -130,7 +162,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -164,22 +196,7 @@ jobs:
- name: Run integration tests - name: Run integration tests
run: | run: |
cd tests/integration tests/integration/run.sh
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format - name: Run cppcheck and clang-format
run: | run: |
@@ -191,7 +208,7 @@ jobs:
./cppcheck_and_clang_format diff ./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors - name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt path: tools/github/cppcheck_and_clang_format.txt
@@ -206,7 +223,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -231,7 +248,7 @@ jobs:
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "GQL Behave Status" name: "GQL Behave Status"
path: | path: |
@@ -249,12 +266,11 @@ jobs:
- name: Run e2e tests - name: Run e2e tests
run: | run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests cd tests
./setup.sh ./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate source ve3/bin/activate_e2e
cd e2e cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory . ./run.sh
- name: Run stress test (plain) - name: Run stress test (plain)
run: | run: |
@@ -276,7 +292,6 @@ jobs:
run: | run: |
# Activate toolchain. # Activate toolchain.
source /opt/toolchain-v4/activate source /opt/toolchain-v4/activate
cd build cd build
# create mgconsole # create mgconsole
@@ -288,13 +303,13 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package - name: Save enterprise DEB package
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Enterprise DEB package" name: "Enterprise DEB package"
path: build/output/memgraph*.deb path: build/output/memgraph*.deb
- name: Save test data - name: Save test data
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
if: always() if: always()
with: with:
name: "Test data" name: "Test data"
@@ -313,7 +328,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -323,10 +338,8 @@ jobs:
run: | run: |
# Activate toolchain. # Activate toolchain.
source /opt/toolchain-v4/activate source /opt/toolchain-v4/activate
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Build only memgraph release binarie. # Build only memgraph release binarie.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=release ..
@@ -335,10 +348,10 @@ jobs:
- name: Run Jepsen tests - name: Run Jepsen tests
run: | run: |
cd tests/jepsen cd tests/jepsen
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs ./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report - name: Save Jepsen report
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
if: ${{ always() }} if: ${{ always() }}
with: with:
name: "Jepsen Report" name: "Jepsen Report"
@@ -354,7 +367,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -405,7 +418,7 @@ jobs:
- name: Run mgbench - name: Run mgbench
run: | run: |
cd tests/mgbench cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/* ./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results - name: Upload mgbench results
run: | run: |

View File

@@ -14,7 +14,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)

View File

@@ -2,7 +2,12 @@ name: Package All
# TODO(gitbuda): Cleanup docker container if GHA job was canceled. # TODO(gitbuda): Cleanup docker container if GHA job was canceled.
on: workflow_dispatch on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
required: false
jobs: jobs:
centos-7: centos-7:
@@ -17,7 +22,7 @@ jobs:
run: | run: |
./release/package/run.sh package centos-7 ./release/package/run.sh package centos-7
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: centos-7 name: centos-7
path: build/output/centos-7/memgraph*.rpm path: build/output/centos-7/memgraph*.rpm
@@ -34,7 +39,7 @@ jobs:
run: | run: |
./release/package/run.sh package centos-9 ./release/package/run.sh package centos-9
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: centos-9 name: centos-9
path: build/output/centos-9/memgraph*.rpm path: build/output/centos-9/memgraph*.rpm
@@ -51,7 +56,7 @@ jobs:
run: | run: |
./release/package/run.sh package debian-10 ./release/package/run.sh package debian-10
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: debian-10 name: debian-10
path: build/output/debian-10/memgraph*.deb path: build/output/debian-10/memgraph*.deb
@@ -68,7 +73,7 @@ jobs:
run: | run: |
./release/package/run.sh package debian-11 ./release/package/run.sh package debian-11
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: debian-11 name: debian-11
path: build/output/debian-11/memgraph*.deb path: build/output/debian-11/memgraph*.deb
@@ -87,7 +92,7 @@ jobs:
./run.sh package debian-11 --for-docker ./run.sh package debian-11 --for-docker
./run.sh docker ./run.sh docker
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: docker name: docker
path: build/output/docker/memgraph*.tar.gz path: build/output/docker/memgraph*.tar.gz
@@ -104,9 +109,9 @@ jobs:
run: | run: |
./release/package/run.sh package ubuntu-18.04 ./release/package/run.sh package ubuntu-18.04
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: ubuntu-1804 name: ubuntu-18.04
path: build/output/ubuntu-18.04/memgraph*.deb path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004: ubuntu-2004:
@@ -121,9 +126,9 @@ jobs:
run: | run: |
./release/package/run.sh package ubuntu-20.04 ./release/package/run.sh package ubuntu-20.04
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: ubuntu-2004 name: ubuntu-20.04
path: build/output/ubuntu-20.04/memgraph*.deb path: build/output/ubuntu-20.04/memgraph*.deb
ubuntu-2204: ubuntu-2204:
@@ -138,9 +143,9 @@ jobs:
run: | run: |
./release/package/run.sh package ubuntu-22.04 ./release/package/run.sh package ubuntu-22.04
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: ubuntu-2204 name: ubuntu-22.04
path: build/output/ubuntu-22.04/memgraph*.deb path: build/output/ubuntu-22.04/memgraph*.deb
debian-11-platform: debian-11-platform:
@@ -155,14 +160,48 @@ jobs:
run: | run: |
./release/package/run.sh package debian-11 --for-platform ./release/package/run.sh package debian-11 --for-platform
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: debian-11-platform name: debian-11-platform
path: build/output/debian-11/memgraph*.deb path: build/output/debian-11/memgraph*.deb
fedora-36:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package fedora-36
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: fedora-36
path: build/output/fedora-36/memgraph*.rpm
amzn-2:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: amzn-2
path: build/output/amzn-2/memgraph*.rpm
debian-11-arm: debian-11-arm:
runs-on: [self-hosted, DockerMgBuild, ARM64, strange] runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 60 timeout-minutes: 120
steps: steps:
- name: "Set up repository" - name: "Set up repository"
uses: actions/checkout@v3 uses: actions/checkout@v3
@@ -172,7 +211,46 @@ jobs:
run: | run: |
./release/package/run.sh package debian-11-arm ./release/package/run.sh package debian-11-arm
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: debian-11 name: debian-11-aarch64
path: build/output/debian-11/memgraph*.deb path: build/output/debian-11-arm/memgraph*.deb
ubuntu-2204-arm:
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 120
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04-aarch64
path: build/output/ubuntu-22.04-arm/memgraph*.deb
upload-to-s3:
# only run upload if we specified version. Allows for runs without upload
if: "${{ github.event.inputs.memgraph_version != '' }}"
needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm]
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v3
with:
# name: # if name input parameter is not provided, all artifacts are downloaded
# and put in directories named after each one.
path: build/output/release
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: "download.memgraph.com"
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/"

View File

@@ -3,7 +3,7 @@ name: Release CentOS 8
on: on:
workflow_dispatch: workflow_dispatch:
schedule: schedule:
- cron: "0 1 * * *" - cron: "0 22 * * *"
jobs: jobs:
community_build: community_build:
@@ -17,7 +17,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage - name: Save code coverage
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -146,22 +146,7 @@ jobs:
- name: Run integration tests - name: Run integration tests
run: | run: |
cd tests/integration tests/integration/run.sh
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format - name: Run cppcheck and clang-format
run: | run: |
@@ -173,7 +158,7 @@ jobs:
./cppcheck_and_clang_format diff ./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors - name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +174,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -225,7 +210,7 @@ jobs:
rpmlint memgraph*.rpm rpmlint memgraph*.rpm
- name: Save enterprise RPM package - name: Save enterprise RPM package
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Enterprise RPM package" name: "Enterprise RPM package"
path: build/output/memgraph*.rpm path: build/output/memgraph*.rpm
@@ -262,7 +247,7 @@ jobs:
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "GQL Behave Status" name: "GQL Behave Status"
path: | path: |
@@ -280,12 +265,11 @@ jobs:
- name: Run e2e tests - name: Run e2e tests
run: | run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests cd tests
./setup.sh ./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate source ve3/bin/activate_e2e
cd e2e cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory . ./run.sh
- name: Run stress test (plain) - name: Run stress test (plain)
run: | run: |

View File

@@ -3,7 +3,7 @@ name: Release Debian 10
on: on:
workflow_dispatch: workflow_dispatch:
schedule: schedule:
- cron: "0 1 * * *" - cron: "0 22 * * *"
jobs: jobs:
community_build: community_build:
@@ -17,7 +17,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage - name: Save code coverage
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -146,22 +146,7 @@ jobs:
- name: Run integration tests - name: Run integration tests
run: | run: |
cd tests/integration tests/integration/run.sh
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format - name: Run cppcheck and clang-format
run: | run: |
@@ -173,7 +158,7 @@ jobs:
./cppcheck_and_clang_format diff ./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors - name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +174,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -224,7 +209,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package - name: Save enterprise DEB package
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Enterprise DEB package" name: "Enterprise DEB package"
path: build/output/memgraph*.deb path: build/output/memgraph*.deb
@@ -261,7 +246,7 @@ jobs:
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "GQL Behave Status" name: "GQL Behave Status"
path: | path: |
@@ -279,12 +264,11 @@ jobs:
- name: Run e2e tests - name: Run e2e tests
run: | run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests cd tests
./setup.sh ./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate source ve3/bin/activate_e2e
cd e2e cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory . ./run.sh
- name: Run stress test (plain) - name: Run stress test (plain)
run: | run: |
@@ -324,7 +308,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -334,10 +318,8 @@ jobs:
run: | run: |
# Activate toolchain. # Activate toolchain.
source /opt/toolchain-v4/activate source /opt/toolchain-v4/activate
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Build only memgraph release binary. # Build only memgraph release binary.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=release ..
@@ -346,10 +328,10 @@ jobs:
- name: Run Jepsen tests - name: Run Jepsen tests
run: | run: |
cd tests/jepsen cd tests/jepsen
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs ./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report - name: Save Jepsen report
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
if: ${{ always() }} if: ${{ always() }}
with: with:
name: "Jepsen Report" name: "Jepsen Report"

View File

@@ -4,8 +4,12 @@ on:
workflow_dispatch: workflow_dispatch:
inputs: inputs:
version: version:
description: "Memgraph binary version to publish on Dockerhub." description: "Memgraph binary version to publish on DockerHub."
required: true required: true
force_release:
type: boolean
required: false
default: false
jobs: jobs:
docker_publish: docker_publish:
@@ -15,20 +19,20 @@ jobs:
DOCKER_REPOSITORY_NAME: memgraph DOCKER_REPOSITORY_NAME: memgraph
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v2 uses: actions/checkout@v3
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@v1 uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx - name: Set up Docker Buildx
id: buildx id: buildx
uses: docker/setup-buildx-action@v1 uses: docker/setup-buildx-action@v2
- name: Log in to Docker Hub - name: Log in to Docker Hub
uses: docker/login-action@v1 uses: docker/login-action@v2
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Download memgraph binary - name: Download memgraph binary
run: | run: |
@@ -36,6 +40,22 @@ jobs:
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
- name: Check if specified version is already pushed
run: |
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
echo $EXISTS
if [[ ${EXISTS} -eq 0 ]]; then
echo 'The specified version has been already released to DockerHub.'
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
echo 'Forcing the release!'
else
echo 'Stopping the release!'
exit 1
fi
else
echo 'All good the specified version has not been release to DockerHub.'
fi
- name: Build & push docker images - name: Build & push docker images
run: | run: |
cd release/docker cd release/docker

View File

@@ -0,0 +1,63 @@
name: "Mgbench Bolt Client Publish Docker Image"
on:
workflow_dispatch:
inputs:
version:
description: "Mgbench bolt client version to publish on Dockerhub."
required: true
force_release:
type: boolean
required: false
default: false
jobs:
mgbench_docker_publish:
runs-on: ubuntu-latest
env:
DOCKER_ORGANIZATION_NAME: memgraph
DOCKER_REPOSITORY_NAME: mgbench-client
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
- name: Log in to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Check if specified version is already pushed
run: |
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
echo $EXISTS
if [[ ${EXISTS} -eq 0 ]]; then
echo 'The specified version has been already released to DockerHub.'
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
echo 'Forcing the release!'
else
echo 'Stopping the release!'
exit 1
fi
else
echo 'All good the specified version has not been release to DockerHub.'
fi
- name: Build & push docker images
run: |
cd tests/mgbench
docker buildx build \
--build-arg TOOLCHAIN_VERSION=toolchain-v4 \
--platform linux/amd64,linux/arm64 \
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
--file Dockerfile.mgbench_client \
--push .

View File

@@ -3,7 +3,7 @@ name: Release Ubuntu 20.04
on: on:
workflow_dispatch: workflow_dispatch:
schedule: schedule:
- cron: "0 1 * * *" - cron: "0 22 * * *"
jobs: jobs:
community_build: community_build:
@@ -17,7 +17,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage - name: Save code coverage
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -146,22 +146,7 @@ jobs:
- name: Run integration tests - name: Run integration tests
run: | run: |
cd tests/integration tests/integration/run.sh
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format - name: Run cppcheck and clang-format
run: | run: |
@@ -173,7 +158,7 @@ jobs:
./cppcheck_and_clang_format diff ./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors - name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Code coverage" name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +174,7 @@ jobs:
steps: steps:
- name: Set up repository - name: Set up repository
uses: actions/checkout@v2 uses: actions/checkout@v3
with: with:
# Number of commits to fetch. `0` indicates all history for all # Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1) # branches and tags. (default: 1)
@@ -224,7 +209,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package - name: Save enterprise DEB package
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "Enterprise DEB package" name: "Enterprise DEB package"
path: build/output/memgraph*.deb path: build/output/memgraph*.deb
@@ -261,7 +246,7 @@ jobs:
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status
uses: actions/upload-artifact@v2 uses: actions/upload-artifact@v3
with: with:
name: "GQL Behave Status" name: "GQL Behave Status"
path: | path: |
@@ -279,12 +264,11 @@ jobs:
- name: Run e2e tests - name: Run e2e tests
run: | run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests cd tests
./setup.sh ./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate source ve3/bin/activate_e2e
cd e2e cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory . ./run.sh
- name: Run stress test (plain) - name: Run stress test (plain)
run: | run: |

32
.github/workflows/upload_to_s3.yaml vendored Normal file
View File

@@ -0,0 +1,32 @@
name: Upload Package All artifacts to S3
on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. Format: 'X.Y.Z'"
required: true
run_number:
description: "# of the package_all workflow run to upload artifacts from. Format: '#XYZ'"
required: true
jobs:
upload-to-s3:
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: package_all.yaml
workflow_conclusion: success
run_number: "${{ github.event.inputs.run_number }}"
path: build/output/release
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: "download.memgraph.com"
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/"

11
.gitignore vendored
View File

@@ -9,7 +9,6 @@
*.swn *.swn
*.swo *.swo
*.swp *.swp
*~ *~
.DS_Store .DS_Store
.gdb_history .gdb_history
@@ -27,7 +26,6 @@ src/query/frontend/opencypher/generated/
tags tags
ve/ ve/
ve3/ ve3/
.cache/
perf.data* perf.data*
TAGS TAGS
*.apollo_measurements *.apollo_measurements
@@ -36,9 +34,6 @@ TAGS
*.fas *.fas
*.fasl *.fasl
# LCP generated C++ files
*.lcp.cpp
src/database/distributed/serialization.hpp src/database/distributed/serialization.hpp
src/database/single_node_ha/serialization.hpp src/database/single_node_ha/serialization.hpp
src/distributed/bfs_rpc_messages.hpp src/distributed/bfs_rpc_messages.hpp
@@ -52,15 +47,11 @@ src/distributed/pull_produce_rpc_messages.hpp
src/distributed/storage_gc_rpc_messages.hpp src/distributed/storage_gc_rpc_messages.hpp
src/distributed/token_sharing_rpc_messages.hpp src/distributed/token_sharing_rpc_messages.hpp
src/distributed/updates_rpc_messages.hpp src/distributed/updates_rpc_messages.hpp
src/query/frontend/ast/ast.hpp
src/query/distributed/frontend/ast/ast_serialization.hpp
src/durability/distributed/state_delta.hpp src/durability/distributed/state_delta.hpp
src/durability/single_node/state_delta.hpp src/durability/single_node/state_delta.hpp
src/durability/single_node_ha/state_delta.hpp src/durability/single_node_ha/state_delta.hpp
src/query/frontend/semantic/symbol.hpp
src/query/distributed/frontend/semantic/symbol_serialization.hpp src/query/distributed/frontend/semantic/symbol_serialization.hpp
src/query/distributed/plan/ops.hpp src/query/distributed/plan/ops.hpp
src/query/plan/operator.hpp
src/raft/log_entry.hpp src/raft/log_entry.hpp
src/raft/raft_rpc_messages.hpp src/raft/raft_rpc_messages.hpp
src/raft/snapshot_metadata.hpp src/raft/snapshot_metadata.hpp
@@ -68,3 +59,5 @@ src/raft/storage_info_rpc_messages.hpp
src/stats/stats_rpc_messages.hpp src/stats/stats_rpc_messages.hpp
src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp
src/transactions/distributed/engine_rpc_messages.hpp src/transactions/distributed/engine_rpc_messages.hpp
/tests/manual/js/transaction_timeout/package-lock.json
/tests/manual/js/transaction_timeout/node_modules/

View File

@@ -1,23 +1,20 @@
repos: repos:
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0 rev: v4.4.0
hooks: hooks:
- id: check-yaml - id: check-yaml
args: [--allow-multiple-documents]
- id: end-of-file-fixer - id: end-of-file-fixer
- id: trailing-whitespace - id: trailing-whitespace
- repo: https://github.com/psf/black - repo: https://github.com/psf/black
rev: 22.3.0 rev: 23.1.0
hooks: hooks:
- id: black - id: black
args: # arguments to configure black - repo: https://github.com/pycqa/isort
- --line-length=120 rev: 5.12.0
- --include='\.pyi?$' hooks:
# these folders wont be formatted by black - id: isort
- --exclude="""\.git | name: isort (python)
\.__pycache__|
build|
libs|
.cache"""
- repo: https://github.com/pre-commit/mirrors-clang-format - repo: https://github.com/pre-commit/mirrors-clang-format
rev: v13.0.0 rev: v13.0.0
hooks: hooks:

View File

@@ -1,6 +1,7 @@
# MemGraph CMake configuration # MemGraph CMake configuration
cmake_minimum_required(VERSION 3.8) cmake_minimum_required(VERSION 3.12)
cmake_policy(SET CMP0076 NEW)
# !! IMPORTANT !! run ./project_root/init.sh before cmake command # !! IMPORTANT !! run ./project_root/init.sh before cmake command
# to download dependencies # to download dependencies
@@ -18,10 +19,12 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM TRUE)
# during the code coverage process # during the code coverage process
find_program(CCACHE_FOUND ccache) find_program(CCACHE_FOUND ccache)
option(USE_CCACHE "ccache:" ON) option(USE_CCACHE "ccache:" ON)
message(STATUS "CCache: ${USE_CCACHE}")
if(CCACHE_FOUND AND USE_CCACHE) if(CCACHE_FOUND AND USE_CCACHE)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
message(STATUS "CCache: Used")
else ()
message(STATUS "CCache: Not used")
endif(CCACHE_FOUND AND USE_CCACHE) endif(CCACHE_FOUND AND USE_CCACHE)
# choose a compiler # choose a compiler
@@ -37,7 +40,14 @@ endif()
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
project(memgraph) project(memgraph LANGUAGES C CXX)
#TODO: upgrade to cmake 3.24 + CheckIPOSupported
#cmake_policy(SET CMP0138 NEW)
#include(CheckIPOSupported)
#check_ipo_supported()
#set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_Release TRUE)
#set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RelWithDebInfo TRUE)
# Install licenses. # Install licenses.
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
@@ -143,7 +153,9 @@ endif()
# files used can be seen here: # files used can be seen here:
# https://git-scm.com/book/en/v2/Git-Internals-Git-References # https://git-scm.com/book/en/v2/Git-Internals-Git-References
set(git_directory "${CMAKE_SOURCE_DIR}/.git") set(git_directory "${CMAKE_SOURCE_DIR}/.git")
if (EXISTS "${git_directory}") # Check for directory because if the repo is cloned as a git submodule, .git is
# a file and below code doesn't work.
if (IS_DIRECTORY "${git_directory}")
set_property(DIRECTORY APPEND PROPERTY set_property(DIRECTORY APPEND PROPERTY
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD") CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
file(STRINGS "${git_directory}/HEAD" git_head_data) file(STRINGS "${git_directory}/HEAD" git_head_data)
@@ -158,7 +170,7 @@ endif()
# setup CMake module path, defines path for include() and find_package() # setup CMake module path, defines path for include() and find_package()
# https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html # https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
# custom function definitions # custom function definitions
include(functions) include(functions)
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@@ -184,7 +196,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-Werror=switch -Werror=switch-bool -Werror=return-type \ -Werror=switch -Werror=switch-bool -Werror=return-type \
-Werror=return-stack-address \ -Werror=return-stack-address \
-Wno-c99-designator \ -Wno-c99-designator -Wmissing-field-initializers \
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT") -DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug. # Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
@@ -231,7 +243,17 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}") message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.") add_definitions( -DCMAKE_BUILD_TYPE_NAME="${CMAKE_BUILD_TYPE}")
if (NOT MG_ARCH)
set(MG_ARCH_DESCR "Host architecture to build Memgraph on. Supported values are x86_64, ARM64.")
if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(MG_ARCH "ARM64" CACHE STRING ${MG_ARCH_DESCR})
else()
set(MG_ARCH "x86_64" CACHE STRING ${MG_ARCH_DESCR})
endif()
endif()
message(STATUS "MG_ARCH: ${MG_ARCH}")
# setup external dependencies ------------------------------------------------- # setup external dependencies -------------------------------------------------
@@ -276,7 +298,7 @@ set(ENABLE_JEMALLOC ON)
if (ASAN) if (ASAN)
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN") message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
set(ENABLE_JEMALLOC OFF) set(ENABLE_JEMALLOC OFF)
# Enable Addres sanitizer and get nicer stack traces in error messages. # Enable Address sanitizer and get nicer stack traces in error messages.
# NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang # NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
# distribution to symbolize the stack traces (note that ideally the # distribution to symbolize the stack traces (note that ideally the
# llvm-symbolizer version must match the version of ASan runtime library). # llvm-symbolizer version must match the version of ASan runtime library).

View File

@@ -1 +0,0 @@
* @antaljanosbenjamin @kostasrim

View File

@@ -4,10 +4,6 @@
--- ---
<p align="center">
Build modern, graph-based applications on top of your streaming data in minutes.
</p>
<p align="center"> <p align="center">
<a href="https://github.com/memgraph/memgraph/blob/master/licenses/APL.txt"> <a href="https://github.com/memgraph/memgraph/blob/master/licenses/APL.txt">
<img src="https://img.shields.io/badge/license-APL-green" alt="license" title="license"/> <img src="https://img.shields.io/badge/license-APL-green" alt="license" title="license"/>
@@ -22,7 +18,7 @@ Build modern, graph-based applications on top of your streaming data in minutes.
<p align="center"> <p align="center">
<a href="https://github.com/memgraph/memgraph"> <a href="https://github.com/memgraph/memgraph">
<img src="https://img.shields.io/github/workflow/status/memgraph/memgraph/Release%20Ubuntu%2020.04/master" alt="build" title="build"/> <img src="https://img.shields.io/github/actions/workflow/status/memgraph/memgraph/release_debian10.yaml?branch=master&label=build%20and%20test&logo=github"/>
</a> </a>
<a href="https://memgraph.com/docs/" alt="Documentation"> <a href="https://memgraph.com/docs/" alt="Documentation">
<img src="https://img.shields.io/badge/documentation-Memgraph-orange" /> <img src="https://img.shields.io/badge/documentation-Memgraph-orange" />
@@ -37,9 +33,10 @@ Build modern, graph-based applications on top of your streaming data in minutes.
## :clipboard: Description ## :clipboard: Description
Memgraph is a streaming graph application platform that helps you wrangle your Memgraph is an open source graph database built for real-time streaming and
streaming data, build sophisticated models that you can query in real-time, and compatible with Neo4j. Whether you're a developer or a data scientist with
develop graph applications. interconnected data, Memgraph will get you the immediate actionable insights
fast.
Memgraph directly connects to your streaming infrastructure. You can ingest data Memgraph directly connects to your streaming infrastructure. You can ingest data
from sources like Kafka, SQL, or plain CSV files. Memgraph provides a standard from sources like Kafka, SQL, or plain CSV files. Memgraph provides a standard
@@ -51,8 +48,20 @@ natural and effective way to model many real-world problems without relying on
complex SQL schemas. complex SQL schemas.
Memgraph is implemented in C/C++ and leverages an in-memory first architecture Memgraph is implemented in C/C++ and leverages an in-memory first architecture
to ensure that youre getting the best possible performance consistently and to ensure that youre getting the [best possible
without surprises. Its also ACID-compliant and highly available. performance](http://memgraph.com/benchgraph) consistently and without surprises.
Its also ACID-compliant and highly available.
## :zap: Features
- Run Python, Rust, and C/C++ code natively, check out the
[MAGE](https://github.com/memgraph/mage) graph algorithm library
- Native support for machine learning
- Streaming support
- Replication
- Authentication and authorization
- ACID compliance
## :video_game: Memgraph Playground ## :video_game: Memgraph Playground
@@ -76,28 +85,49 @@ your browser.
### macOS ### macOS
[![macOS](https://img.shields.io/badge/macOS-Docker-000000?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-macos-docker) [![macOS](https://img.shields.io/badge/macOS-Docker-000000?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-macos-docker)
[![macOS](https://img.shields.io/badge/lima-AACF41?style=for-the-badge&logo=macos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu)
### Linux ### Linux
[![Linux](https://img.shields.io/badge/Linux-Docker-FCC624?style=for-the-badge&logo=linux&logoColor=black)](https://memgraph.com/docs/memgraph/install-memgraph-on-linux-docker) [![Linux](https://img.shields.io/badge/Linux-Docker-FCC624?style=for-the-badge&logo=linux&logoColor=black)](https://memgraph.com/docs/memgraph/install-memgraph-on-linux-docker)
[![Debian](https://img.shields.io/badge/Debian-D70A53?style=for-the-badge&logo=debian&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-debian) [![Debian](https://img.shields.io/badge/Debian-D70A53?style=for-the-badge&logo=debian&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-debian)
[![Ubuntu](https://img.shields.io/badge/Ubuntu-E95420?style=for-the-badge&logo=ubuntu&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu) [![Ubuntu](https://img.shields.io/badge/Ubuntu-E95420?style=for-the-badge&logo=ubuntu&logoColor=white)](https://memgraph.com/docs/memgraph/install-memgraph-on-ubuntu)
[![Cent [![Cent OS](https://img.shields.io/badge/cent%20os-002260?style=for-the-badge&logo=centos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
OS](https://img.shields.io/badge/cent%20os-002260?style=for-the-badge&logo=centos&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm) [![Fedora](https://img.shields.io/badge/fedora-0B57A4?style=for-the-badge&logo=fedora&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
[![RedHat](https://img.shields.io/badge/redhat-EE0000?style=for-the-badge&logo=redhat&logoColor=F0F0F0)](https://memgraph.com/docs/memgraph/install-memgraph-from-rpm)
You can find the binaries and Docker images on the [Download You can find the binaries and Docker images on the [Download
Hub](https://memgraph.com/download) and the installation instructions in the Hub](https://memgraph.com/download) and the installation instructions in the
[official documentation](https://memgraph.com/docs/memgraph/installation). [official documentation](https://memgraph.com/docs/memgraph/installation).
## :zap: Features
- Run Python, Rust, and C/C++ code natively, check out the ## :cloud: Memgraph Cloud
[MAGE](https://github.com/memgraph/mage) graph algorithm library
- Native support for machine learning Check out [Memgraph Cloud](https://memgraph.com/docs/memgraph-cloud) - a cloud service fully managed on AWS and available in 6 geographic regions around the world. Memgraph Cloud allows you to create projects with Enterprise instances of MemgraphDB from your browser.
- Streaming support
- Replication <p align="left">
- Authentication and authorization <a href="https://memgraph.com/docs/memgraph-cloud">
- ACID compliance <img width="450px" alt="Memgraph Cloud" src="https://public-assets.memgraph.com/memgraph-gifs%2Fcloud.gif">
</a>
</p>
## :link: Connect to Memgraph
[Connect to the database](https://memgraph.com/docs/memgraph/connect-to-memgraph) using Memgraph Lab, mgconsole, various drivers (Python, C/C++ and others) and WebSocket.
### :microscope: Memgraph Lab
Visualize graphs and play with queries to understand your data. [Memgraph Lab](https://memgraph.com/docs/memgraph-lab) is a user interface that helps you explore and manipulate the data stored in Memgraph. Visualize graphs, execute ad hoc queries, and optimize their performance.
<p align="left">
<a href="https://memgraph.com/docs/memgraph-lab">
<img width="450px" alt="Memgraph Cloud" src="https://public-assets.memgraph.com/memgraph-gifs%2Flab.gif">
</a>
</p>
## :file_folder: Import data
[Import data](https://memgraph.com/docs/memgraph/import-data) into Memgraph using Kafka, RedPanda or Pulsar streams, CSV and JSON files, or Cypher commands.
## :bookmark_tabs: Documentation ## :bookmark_tabs: Documentation
@@ -141,8 +171,16 @@ Memgraph Community is available under the [BSL
license](./licenses/BSL.txt).</br> Memgraph Enterprise is available under the license](./licenses/BSL.txt).</br> Memgraph Enterprise is available under the
[MEL license](./licenses/MEL.txt). [MEL license](./licenses/MEL.txt).
## :busts_in_silhouette: Community
- :purple_heart: [**Discord**](https://discord.gg/memgraph)
- :ocean: [**Stack Overflow**](https://stackoverflow.com/questions/tagged/memgraphdb)
- :bird: [**Twitter**](https://twitter.com/memgraphdb)
- :movie_camera:
[**YouTube**](https://www.youtube.com/channel/UCZ3HOJvHGxtQ_JHxOselBYg)
<p align="center"> <p align="center">
<a href="#"> <a href="#">
<img src="https://img.shields.io/badge/⬆back_to_top_⬆-white" alt="Back to top" title="Back to top"/> <img src="https://img.shields.io/badge/⬆️ back_to_top_⬆-white" alt="Back to top" title="Back to top"/>
</a> </a>
</p> </p>

View File

@@ -103,6 +103,10 @@ modifications:
value: "true" value: "true"
override: false override: false
- name: "storage_parallel_index_recovery"
value: "false"
override: true
undocumented: undocumented:
- "flag_file" - "flag_file"
- "also_log_to_stderr" - "also_log_to_stderr"

View File

@@ -5,12 +5,10 @@ import os
import subprocess import subprocess
import sys import sys
import textwrap import textwrap
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
import yaml import yaml
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
CONFIG_FILE = os.path.join(SCRIPT_DIR, "flags.yaml") CONFIG_FILE = os.path.join(SCRIPT_DIR, "flags.yaml")
WIDTH = 80 WIDTH = 80
@@ -18,16 +16,21 @@ WIDTH = 80
def wrap_text(s, initial_indent="# "): def wrap_text(s, initial_indent="# "):
return "\n#\n".join( return "\n#\n".join(
map( map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "), s.split("\n"))
lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "),
s.split("\n"),
)
) )
def extract_flags(binary_path): def extract_flags(binary_path):
ret = {} ret = {}
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8") data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
# If something is printed out before the help output, it will break the the
# XML parsing -> filter out if something is not XML line because something
# can be logged before gflags output (e.g. during the global objects init).
# This gets called during memgraph build phase to generate default config
# file later installed under /etc/memgraph/memgraph.conf
# NOTE: Don't use \n in the gflags description strings.
# NOTE: Check here if gflags version changes because of the XML format.
data = "\n".join([line for line in data.split("\n") if line.startswith("<")])
root = ET.fromstring(data) root = ET.fromstring(data)
for child in root: for child in root:
if child.tag == "usage" and child.text.lower().count("warning"): if child.tag == "usage" and child.text.lower().count("warning"):
@@ -100,15 +103,8 @@ def generate_config_file(sections, flags):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("memgraph_binary", help="path to Memgraph binary") parser.add_argument("memgraph_binary", help="path to Memgraph binary")
parser.add_argument( parser.add_argument("output_file", help="path where to store the generated Memgraph " "configuration file")
"output_file", parser.add_argument("--config-file", default=CONFIG_FILE, help="path to generator configuration file")
help="path where to store the generated Memgraph " "configuration file",
)
parser.add_argument(
"--config-file",
default=CONFIG_FILE,
help="path to generator configuration file",
)
args = parser.parse_args() args = parser.parse_args()
flags = extract_flags(args.memgraph_binary) flags = extract_flags(args.memgraph_binary)

View File

@@ -0,0 +1,230 @@
# CSV Import Tool Documentation
CSV is a universal and very versatile data format used to store large quantities
of data. Each Memgraph database instance has a CSV import tool installed called
`mg_import_csv`. The CSV import tool should be used for initial bulk ingestion
of data into the database. Upon ingestion, the CSV importer creates a snapshot
that will be used by the database to recover its state on its next startup.
If you are already familiar with the Neo4j bulk import tool, then using the
`mg_import_csv` tool should be easy. The CSV import tool is fully compatible
with the [Neo4j CSV
format](https://neo4j.com/docs/operations-manual/current/tools/import/). If you
already have a pipeline set-up for Neo4j, you should only replace `neo4j-admin
import` with `mg_import_csv`.
## CSV File Format
Each row of a CSV file represents a single entry that should be imported into
the database. Both nodes and relationships can be imported into the database
using CSV files.
Each set of CSV files must have a header that describes the data that is stored
in the CSV files. Each field in the CSV header is in the format
`<name>[:<type>]` which identifies the name that should be used for that column
and the type that should be used for that column. The type is optional and
defaults to `string` (see the following chapter).
Each CSV field must be divided using the delimiter and each CSV field can either
be quoted or unquoted. When the field is quoted, the first and last character in
the field *must* be the quote character. If the field isn't quoted, and a quote
character appears in it, it is treated as a regular character. If a quote
character appears inside a quoted string then the quote character must be
doubled in order to escape it. Line feeds and carriage returns are ignored in
the CSV file, also, the file can't contain a NULL character.
## Properties
Both nodes and relationships can have properties added to them. When importing
properties, the CSV importer uses the name specified in the header of the
corresponding CSV column for the name of the property. A property is designated
by specifying one of the following types in the header:
- `integer`, `int`, `long`, `byte`, `short`: creates an integer property
- `float`, `double`: creates a float property
- `boolean`, `bool`: creates a boolean property
- `string`, `char`: creates a string property
When importing a boolean value, the CSV field should contain exactly the text
`true` to import a `True` boolean value. All other text values are treated as a
boolean value `False`.
If you want to import an array of values, you can do so by appending `[]` to any
of the above types. The values of the array are then determined by splitting
the raw CSV value using the array delimiter character.
Assuming that the array delimiter is `;`, the following example:
```plaintext
first_name,last_name:string,number:integer,aliases:string[]
John,Doe,1,Johnny;Jo;J-man
Melissa,Doe,2,Mel
```
Will yield these results:
```plaintext
CREATE ({first_name: "John", last_name: "Doe", number: 1, aliases: ["Johnny", "Jo", "J-man"]});
CREATE ({first_name: "Melissa", last_name: "Doe", number: 2, aliases: ["Mel"]});
```
### Nodes
When importing nodes, several more types can be specified in the header of the
CSV file (along with all property types):
- `ID`: id of the node that should be used as the node ID when importing
relationships
- `LABEL`: designates that the field contains additional labels for the node
- `IGNORE`: designates that the field should be ignored
The `ID` field type sets the internal ID that will be used for the node when
creating relationships. It is optional and nodes that don't have an ID value
specified will be imported, but can't be connected to any relationships. If you
want to save the ID value as a property in the database, just specify a name for
the ID (`user_id:ID`). If you just want to use the ID during the import, leave
out the name of the field (`:ID`). The `ID` field also supports creating
separate ID spaces. The ID space is specified with the ID space name appended
to the `ID` type in parentheses (`ID(user)`). That allows you to have the same
IDs (by value) for multiple different node files (for example, numbers from 1 to
N). The IDs in each ID space will be treated as an independent set of IDs that
don't interfere with IDs in another ID space.
The `LABEL` field type adds additional labels to the node. The value is treated
as an array type so that multiple additional labels can be specified for each
node. The value is split using the array delimiter (`--array-delimiter` flag).
### Relationships
In order to be able to import relationships, you must import the nodes in the
same invocation of `mg_import_csv` that is used to import the relationships.
When importing relationships, several more types can be specified in the header
of the CSV file (along with all property types):
- `START_ID`: id of the start node that should be connected with the
relationship
- `END_ID`: id of the end node that should be connected with the relationship
- `TYPE`: designates the type of the relationship
- `IGNORE`: designates that the field should be ignored
The `START_ID` field type sets the start node that should be connected with the
relationship to the end node. The field *must* be specified and the node ID
must be one of the node IDs that were specified in the node CSV files. The name
of this field is ignored. If the node ID is in an ID space, you can specify the
ID space for the in the same way as for the node ID (`START_ID(user)`).
The `END_ID` field type sets the end node that should be connected with the
relationship to the start node. The field *must* be specified and the node ID
must be one of the node IDs that were specified in the node CSV files. The name
of this field is ignored. If the node ID is in an ID space, you can specify the
ID space for the in the same way as for the node ID (`END_ID(user)`).
The `TYPE` field type sets the type of the relationship. Each relationship
*must* have a relationship type, but it doesn't necessarily need to be specified
in the CSV file, it can also be set externally for the whole CSV file. The name
of this field is ignored.
## CSV Importer Flags
The importer has many command line options that allow you to customize the way
the importer loads your data.
The two main flags that are used to specify the input CSV files are `--nodes`
and `--relationships`. Basic description of these flags is provided in the table
and more detailed explainion can be found further down bellow.
| Flag | Description |
|-----------------------| -------------- |
|`--nodes` | Used to specify CSV files that contain the nodes to the importer. |
|`--relationships` | Used to specify CSV files that contain the relationships to the importer.|
|`--delimiter` | Sets the delimiter that should be used when splitting the CSV fields (default `,`)|
|`--quote` | Sets the quote character that should be used to quote a CSV field (default `"`)|
|`--array-delimiter` | Sets the delimiter that should be used when splitting array values (default `;`)|
|`--id-type` | Specifies which data type should be used to store the supplied <br /> node IDs when storing them as properties (if the field name is supplied). <br /> The supported values are either `STRING` or `INTEGER`. (default `STRING`)|
|`--ignore-empty-strings` | Instructs the importer to treat all empty strings as `Null` values <br /> instead of an empty string value (default `false`)|
|`--ignore-extra-columns` | Instructs the importer to ignore all columns (instead of raising an error) <br /> that aren't specified after the last specified column in the CSV header. (default `false`) |
| `--skip-bad-relationships`| Instructs the importer to ignore all relationships (instead of raising an error) <br /> that refer to nodes that don't exist in the node files. (default `false`) |
|`--skip-duplicate-nodes` | Instructs the importer to ignore all duplicate nodes (instead of raising an error). <br /> Duplicate nodes are nodes that have an ID that is the same as another node that was already imported. (default `false`) |
| `--trim-strings`| Instructs the importer to trim all of the loaded CSV field values before processing them further. <br /> Trimming the fields removes all leading and trailing whitespace from them. (default `false`) |
The `--nodes` and `--relationships` flags are used to specify CSV files that
contain the nodes and relationships to the importer. Multiple files can be
specified in each supplied `--nodes` or `--relationships` flag. Files that are
supplied in one `--nodes` or `--relationships` flag are treated by the CSV
parser as one big CSV file. Only the first line of the first file is parsed for
the CSV header, all other files (and rows) are treated as data. This is useful
when you have a very large CSV file and don't want to edit its first line just
to add a CSV header. Instead, you can specify the header in a separate file
(e.g. `users_header.csv` or `friendships_header.csv`) and have the data intact
in the large file (e.g. `users.csv` or `friendships.csv`). Also, you can supply
additional labels for each set of node files.
The format of `--nodes` flag is:
`[<label>[:<label>]...=]<file>[,<file>][,<file>]...`. Take note that only the
first `<file>` part is mandatory, all other parts of the flag value are
optional. Multiple `--nodes` flags can be supplied to describe multiple sets of
different node files. For the importer to work, at least one `--nodes` flag
*must* be supplied.
The format of `--relationships` flag is: `[<type>=]<file>[,<file>][,<file>]...`.
Take note that only the first `<file>` part is mandatory, all other parts of the
flag value are optional. Multiple `--relationships` flags can be supplied to
describe multiple sets of different relationship files. The `--relationships`
flag isn't mandatory.
## CSV Parser Logic
The CSV parser uses the same logic as the standard Python CSV parser. The data
is parsed in the same way as the following snippet:
```python
import csv
for row in csv.reader(stream, strict=True):
# process 'row'
```
Python uses 'excel' as the default dialect when parsing CSV files and the
default settings for the CSV parser are:
- delimiter: `','`
- doublequote: `True`
- escapechar: `None`
- lineterminator: `'\r\n'`
- quotechar: `'"'`
- skipinitialspace: `False`
The above snippet can be expanded to:
```python
import csv
for row in csv.reader(stream, delimiter=',', doublequote=True,
escapechar=None, lineterminator='\r\n',
quotechar='"', skipinitialspace=False,
strict=True):
# process 'row'
```
For more information about the meaning of the above values, see:
https://docs.python.org/3/library/csv.html#csv.Dialect
## Errors
1. [Skipping duplicate node with ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-1)
2. [Skipping bad relationship with START_ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-2)
3. [Skipping bad relationship with END_ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-3)
## Skipping duplicate node with ID {} {#error-1}
Duplicate nodes are nodes that have an ID that is the same as another node that
was already imported. You can instruct the importer to ignore all duplicate
nodes (instead of raising an error) by using the `--skip-duplicate-nodes` flag.
## Skipping bad relationship with START_ID {} {#error-2}
A node with the id `START_ID` doesn't exist. You can instruct the importer to
ignore all bad relationships (instead of raising an error) that refer to nodes
that don't exist in the node files by using the `--skip-bad-relationships` flag.
## Skipping bad relationship with END_ID {} {#error-3}
A node with the id `END_ID` doesn't exist. You can instruct the importer to
ignore all bad relationships (instead of raising an error) that refer to nodes
that don't exist in the node files by using the `--skip-bad-relationships` flag.

10
environment/README.md Normal file
View File

@@ -0,0 +1,10 @@
# Memgraph Operating Environments
## os
Under the `os` directory, you can find scripts to install all required system
dependencies on operating systems where Memgraph natively builds. The testing
script helps to see how to install all packages (in the case of a new package),
or make any adjustments in the overall system setup. Also, the testing script
helps check if Memgraph runs on a freshly installed operating system (with no
packages installed).

View File

@@ -1,3 +1,6 @@
*.deb *.deb
*.deb.*
*.rpm *.rpm
*.rpm.*
*.tar.gz *.tar.gz
*.tar.gz.*

196
environment/os/amzn-2.sh Executable file
View File

@@ -0,0 +1,196 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "amzn-2"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
patch
libipt libipt-devel # intel
perl # for openssl
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip nmap-ncat # for tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang custom-golang1.18.9 zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.3 # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
local missing=""
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""}
LD_LIBRARY_PATH=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == jdk-17 ]; then
if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then
wget --no-check-certificate -c --header "Cookie: oraclelicense=accept-securebackup-cookie" https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
rpm -Uvh jdk-17_linux-x64_bin.rpm
# NOTE: Set Java 11 as default.
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libipt-devel ]; then
if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == nodejs ]; then
curl -sL https://rpm.nodesource.com/setup_16.x | bash -
if ! yum list installed nodejs >/dev/null 2>/dev/null; then
yum install -y nodejs
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
if [ "$pkg" == nodejs ]; then
curl -sL https://rpm.nodesource.com/setup_16.x | bash -
if ! yum list installed nodejs >/dev/null 2>/dev/null; then
yum install -y nodejs
fi
continue
fi
if [ "$pkg" == java-11-openjdk ]; then
amazon-linux-extras install -y java-openjdk11
continue
fi
if [ "$pkg" == java-11-openjdk-devel ]; then
amazon-linux-extras install -y java-openjdk11
yum install -y java-11-openjdk-devel
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "centos-7"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download wget # used for archive download
@@ -36,7 +39,7 @@ TOOLCHAIN_RUN_DEPS=(
) )
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
make pkgconfig # build system make cmake pkgconfig # build system
curl wget # for downloading libs curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console readline-devel # for memgraph console
@@ -53,9 +56,19 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
rpm-build rpmlint # for RPM package building rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests which mono-complete dotnet-sdk-3.1 golang custom-golang1.18.9 # for driver tests
nodejs zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.3 # for driver tests
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -65,6 +78,18 @@ list() {
check() { check() {
local missing="" local missing=""
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == git ]; then if [ "$pkg" == git ]; then
if ! which "git" >/dev/null; then if ! which "git" >/dev/null; then
missing="git $missing" missing="git $missing"
@@ -107,7 +132,25 @@ install() {
yum update -y yum update -y
yum install -y wget python3 python3-pip yum install -y wget python3 python3-pip
yum install -y git yum install -y git
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == jdk-17 ]; then
if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then
wget https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
rpm -ivh jdk-17_linux-x64_bin.rpm
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
if [ "$pkg" == libipt ]; then if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "centos-9"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download wget # used for archive download
@@ -37,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
git # source code control git # source code control
make pkgconf-pkg-config # build system make cmake pkgconf-pkg-config # build system
wget # for downloading libs wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console readline-devel # for memgraph console
@@ -53,10 +56,20 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests which nodejs golang custom-golang1.18.9 # for driver tests
zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.3 # for driver tests
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -66,6 +79,18 @@ list() {
check() { check() {
local missing="" local missing=""
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing" missing="$pkg $missing"
@@ -100,7 +125,16 @@ install() {
fi fi
yum update -y yum update -y
yum install -y wget git python3 python3-pip yum install -y wget git python3 python3-pip
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
# Since there is no support for libipt-devel for CentOS 9 we install # Since there is no support for libipt-devel for CentOS 9 we install
# Fedoras version of same libs, they are the same version but released # Fedoras version of same libs, they are the same version but released
# for different OS # for different OS

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "debian-10"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download wget # used for archive download
@@ -37,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
git # source code control git # source code control
make pkg-config # build system make cmake pkg-config # build system
curl wget # for downloading libs curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console libreadline-dev # for memgraph console
@@ -50,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests mono-runtime mono-mcs zip unzip default-jdk-headless oracle-java17-installer custom-maven3.9.3 # for driver tests
dotnet-sdk-3.1 golang nodejs npm dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -61,7 +73,28 @@ list() {
} }
check() { check() {
check_all_dpkg "$1" local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
} }
install() { install() {
@@ -72,8 +105,15 @@ deb http://deb.debian.org/debian/ buster-updates main contrib non-free
deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free
deb http://security.debian.org/debian-security buster/updates main contrib non-free deb http://security.debian.org/debian-security buster/updates main contrib non-free
deb-src http://security.debian.org/debian-security buster/updates main contrib non-free deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
EOF
apt --allow-releaseinfo-change update
cat >/etc/apt/sources.list.d/java.list << EOF
deb http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main
deb-src http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main
EOF EOF
cd "$DIR" cd "$DIR"
apt install -y gnupg
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EA8CACC073C3DB2A
apt --allow-releaseinfo-change update apt --allow-releaseinfo-change update
# If GitHub Actions runner is installed, append LANG to the environment. # If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export. # Python related tests doesn't work the LANG export.
@@ -82,8 +122,26 @@ EOF
else else
echo "NOTE: export LANG=en_US.utf8" echo "NOTE: export LANG=en_US.utf8"
fi fi
apt install -y wget
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == oracle-java17-installer ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
echo oracle-java17-installer shared/accepted-oracle-license-v1-3 select true | /usr/bin/debconf-set-selections
echo oracle-java17-installer shared/accepted-oracle-license-v1-3 seen true | /usr/bin/debconf-set-selections
apt install -y "$pkg"
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "arm64" "aarch64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download wget # used for archive download
@@ -51,10 +54,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.3 # for driver tests
golang nodejs npm golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -62,7 +74,28 @@ list() {
} }
check() { check() {
check_all_dpkg "$1" local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
} }
install() { install() {
@@ -86,7 +119,25 @@ EOF
echo "NOTE: export LANG=en_US.utf8" echo "NOTE: export LANG=en_US.utf8"
fi fi
apt install -y wget apt install -y wget
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-arm64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-arm64/bin/javac
fi
continue
fi
apt install -y "$pkg" apt install -y "$pkg"
done done
} }

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download wget # used for archive download
@@ -38,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
git # source code control git # source code control
make pkg-config # build system make cmake pkg-config # build system
curl wget # for downloading libs curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console libreadline-dev # for memgraph console
@@ -51,10 +54,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.3 # for driver tests
dotnet-sdk-3.1 golang nodejs npm dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -62,7 +74,28 @@ list() {
} }
check() { check() {
check_all_dpkg "$1" local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
} }
install() { install() {
@@ -86,7 +119,25 @@ EOF
echo "NOTE: export LANG=en_US.utf8" echo "NOTE: export LANG=en_US.utf8"
fi fi
apt install -y wget apt install -y wget
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

149
environment/os/fedora-36.sh Executable file
View File

@@ -0,0 +1,149 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "fedora-36"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
perl # for openssl
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
java-11-openjdk-devel java-17-openjdk-devel custom-maven3.9.3 # for driver tests
which zip unzip
nodejs golang custom-golang1.18.9 # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
local missing=""
if [ -v LD_LIBRARY_PATH ]; then
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
fi
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
if [ -v OLD_LD_LIBRARY_PATH ]; then
echo "Restoring LD_LIBRARY_PATH..."
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == java-17-openjdk-devel ]; then
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
dnf install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

145
environment/os/run.sh Executable file
View File

@@ -0,0 +1,145 @@
#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
IFS=' '
# NOTE: docker_image_name could be local image build based on release/package images.
# NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space.
# "docker_container_type script_name docker_image_name"
OPERATING_SYSTEMS=(
"mgrun amzn-2 amazonlinux:2"
"mgrun centos-7 centos:7"
"mgrun centos-9 dokken/centos-stream-9"
"mgrun debian-10 debian:10"
"mgrun debian-11 debian:11"
"mgrun fedora-36 fedora:36"
"mgrun ubuntu-18.04 ubuntu:18.04"
"mgrun ubuntu-20.04 ubuntu:20.04"
"mgrun ubuntu-22.04 ubuntu:22.04"
# "mgbuild centos-7 package-mgbuild_centos-7"
)
if [ ! "$(docker info)" ]; then
echo "ERROR: Docker is required"
exit 1
fi
print_help () {
echo -e "$0 all\t\t\t\t => start + init all containers in the background"
echo -e "$0 check\t\t\t\t => check all containers"
echo -e "$0 delete\t\t\t\t => stop + remove all containers"
echo -e "$0 copy src_container dst_container => copy build package from src to dst container"
exit 1
}
# NOTE: This is an idempotent operation!
# TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function.
docker_run () {
cnt_name="$1"
cnt_image="$2"
if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
echo "Cleanup of the old exited container..."
docker rm $cnt_name
fi
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
fi
echo "The $cnt_image container is active under $cnt_name name!"
}
docker_exec () {
cnt_name="$1"
cnt_cmd="$2"
docker exec -it "$cnt_name" bash -c "$cnt_cmd"
}
docker_stop_and_rm () {
cnt_name="$1"
if [ "$(docker ps -q -f name=$cnt_name)" ]; then
docker stop "$1"
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
docker rm "$1"
fi
fi
}
# TODO(gitbuda): Make the call to `install NEW_DEPS` configurable, the question what else is useful?
start_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_run "$docker_name" "$docker_image"
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""
done
}
check_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh check NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""
done
}
delete_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
docker_stop_and_rm "$docker_name"
echo "~~~~ $docker_image as $docker_name DELETED"
done
}
# TODO(gitbuda): Copy file between containers is a useful util, also delete, + consider copying of a whole folder.
# TODO(gitbuda): Add args: src_cnt dst_cnt abs_path; both file and recursive folder, always delete + copy.
copy_build_package () {
src_container="$1"
dst_container="$2"
src="$src_container:/memgraph/build/output"
tmp_dst="$SCRIPT_DIR/../../build"
mkdir -p "$tmp_dst"
rm -rf "$tmp_dst/output"
dst="$dst_container:/"
docker cp "$src" "$tmp_dst"
docker cp "$tmp_dst/output" "$dst"
}
if [ "$#" -eq 0 ]; then
print_help
else
case $1 in
all)
start_all
;;
check)
check_all
;;
delete)
delete_all
;;
copy) # src_container dst_container
if [ "$#" -ne 3 ]; then
print_help
fi
copy_build_package "$2" "$3"
;;
*)
print_help
;;
esac
fi

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "todo-os-name"
check_architecture "todo-arch-name"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
pkg pkg
) )
@@ -17,6 +20,16 @@ MEMGRAPH_BUILD_DEPS=(
pkg pkg
) )
MEMGRAPH_RUN_DEPS=(
pkg
)
# NEW_DEPS is useful when you won't to test the installation of a new package.
# During the test you can put here packages like wget curl tar gzip
NEW_DEPS=(
pkg
)
list() { list() {
echo "$1" echo "$1"
} }

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "ubuntu-18.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools coreutils gcc g++ build-essential make # generic build tools
wget # archive download wget # archive download
@@ -38,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
git # source code control git # source code control
make pkg-config # build system make cmake pkg-config # build system
curl wget # downloading libs curl wget # downloading libs
uuid-dev default-jre-headless # required by antlr uuid-dev default-jre-headless # required by antlr
libreadline-dev # memgraph console libreadline-dev # memgraph console
@@ -50,9 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests libcurl4-openssl-dev # mg-requests
sbcl # custom Lisp C++ preprocessing sbcl # custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests mono-runtime mono-mcs nodejs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # driver tests
custom-golang1.18.9 # for driver tests
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -60,11 +73,53 @@ list() {
} }
check() { check() {
check_all_dpkg "$1" local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
} }
install() { install() {
apt install -y $1 apt update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
} }
deps=$2"[*]" deps=$2"[*]"

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "ubuntu-20.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download wget # used for archive download
@@ -37,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
git # source code control git # source code control
make pkg-config # build system make cmake pkg-config # build system
curl wget # for downloading libs curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console libreadline-dev # for memgraph console
@@ -50,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-3.1 golang nodejs npm dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -61,12 +73,35 @@ list() {
} }
check() { check() {
check_all_dpkg "$1" local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
} }
install() { install() {
cd "$DIR" cd "$DIR"
apt update export DEBIAN_FRONTEND=noninteractive
apt update -y
apt install -y wget
# If GitHub Actions runner is installed, append LANG to the environment. # If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export. # Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then if [ -d "/home/gh/actions-runner" ]; then
@@ -74,8 +109,16 @@ install() {
else else
echo "NOTE: export LANG=en_US.utf8" echo "NOTE: export LANG=en_US.utf8"
fi fi
apt install -y wget
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -85,6 +128,15 @@ install() {
fi fi
continue continue
fi fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg" apt install -y "$pkg"
done done
} }

View File

@@ -0,0 +1,144 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-22.04"
check_architecture "arm64" "aarch64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 libbabeltrace1 liblzma5 python3 # for gdb
libcurl4 # for cmake
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
netcat # tests are using nc to wait for memgraph
python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb
apt-get update
apt-get install -y apt-transport-https dotnet-sdk-6.0
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-arm64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-arm64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -5,6 +5,9 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh" source "$DIR/../util.sh"
check_operating_system "ubuntu-22.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=( TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download wget # used for archive download
@@ -37,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=( MEMGRAPH_BUILD_DEPS=(
git # source code control git # source code control
make pkg-config # build system make cmake pkg-config # build system
curl wget # for downloading libs curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console libreadline-dev # for memgraph console
@@ -50,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-6.0 golang nodejs npm dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm # for driver tests
autoconf # for jemalloc code generation autoconf # for jemalloc code generation
libtool # for protobuf code generation libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
) )
list() { list() {
@@ -61,12 +73,34 @@ list() {
} }
check() { check() {
check_all_dpkg "$1" local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
} }
install() { install() {
cd "$DIR" cd "$DIR"
apt update apt update -y
apt install -y wget
# If GitHub Actions runner is installed, append LANG to the environment. # If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export. # Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then if [ -d "/home/gh/actions-runner" ]; then
@@ -74,8 +108,16 @@ install() {
else else
echo "NOTE: export LANG=en_US.utf8" echo "NOTE: export LANG=en_US.utf8"
fi fi
apt install -y wget
for pkg in $1; do for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-6.0 ]; then if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -85,6 +127,15 @@ install() {
fi fi
continue continue
fi fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg" apt install -y "$pkg"
done done
} }

1
environment/toolchain/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*.tar.gz

View File

@@ -10,6 +10,14 @@ cd "$DIR"
source "$DIR/../util.sh" source "$DIR/../util.sh"
DISTRO="$(operating_system)" DISTRO="$(operating_system)"
function log_tool_name () {
echo ""
echo ""
echo "#### $1 ####"
echo ""
echo ""
}
for_arm=false for_arm=false
if [[ "$#" -eq 1 ]]; then if [[ "$#" -eq 1 ]]; then
if [[ "$1" == "--for-arm" ]]; then if [[ "$1" == "--for-arm" ]]; then
@@ -20,9 +28,11 @@ if [[ "$#" -eq 1 ]]; then
fi fi
fi fi
os="$1" TOOLCHAIN_STDCXX="${TOOLCHAIN_STDCXX:-libstdc++}"
if [[ "$TOOLCHAIN_STDCXX" != "libstdc++" && "$TOOLCHAIN_STDCXX" != "libc++" ]]; then
# toolchain version echo "Only GCC (libstdc++) or LLVM (libc++) C++ standard library implementations are supported."
exit 1
fi
TOOLCHAIN_VERSION=4 TOOLCHAIN_VERSION=4
# package versions used # package versions used
@@ -41,11 +51,19 @@ CPPCHECK_VERSION=2.6
LLVM_VERSION=13.0.0 LLVM_VERSION=13.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Check for the dependencies. # Set the right env script
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)" ENV_SCRIPT="$DIR/../os/$DISTRO.sh"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS if [[ "$for_arm" = true ]]; then
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)" ENV_SCRIPT="$DIR/../os/$DISTRO-arm.sh"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS fi
# Check for the toolchain build dependencies.
echo "ALL BUILD PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_BUILD_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_BUILD_DEPS
# Check for the toolchain run dependencies.
echo "ALL RUN PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_RUN_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_RUN_DEPS
# check installation directory # check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -99,6 +117,8 @@ if [ ! -f llvm-$LLVM_VERSION.src.tar.xz ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxx-$LLVM_VERSION.src.tar.xz
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxxabi-$LLVM_VERSION.src.tar.xz
fi fi
if [ ! -f pahole-gdb-master.zip ]; then if [ ! -f pahole-gdb-master.zip ]; then
wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip wget https://github.com/PhilArmstrong/pahole-gdb/archive/master.zip -O pahole-gdb-master.zip
@@ -156,6 +176,8 @@ if [ ! -f llvm-$LLVM_VERSION.src.tar.xz.sig ]; then
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/compiler-rt-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz.sig wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libunwind-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxx-$LLVM_VERSION.src.tar.xz.sig
wget https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/libcxxabi-$LLVM_VERSION.src.tar.xz.sig
fi fi
# list of valid llvm gnupg keys: https://releases.llvm.org/download.html # list of valid llvm gnupg keys: https://releases.llvm.org/download.html
$GPG --keyserver $KEYSERVER --recv-keys 0x474E22316ABF4785A88C6E8EA2C794A986419D8A $GPG --keyserver $KEYSERVER --recv-keys 0x474E22316ABF4785A88C6E8EA2C794A986419D8A
@@ -165,6 +187,8 @@ $GPG --verify lld-$LLVM_VERSION.src.tar.xz.sig lld-$LLVM_VERSION.src.tar.xz
$GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz $GPG --verify clang-tools-extra-$LLVM_VERSION.src.tar.xz.sig clang-tools-extra-$LLVM_VERSION.src.tar.xz
$GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz $GPG --verify compiler-rt-$LLVM_VERSION.src.tar.xz.sig compiler-rt-$LLVM_VERSION.src.tar.xz
$GPG --verify libunwind-$LLVM_VERSION.src.tar.xz.sig libunwind-$LLVM_VERSION.src.tar.xz $GPG --verify libunwind-$LLVM_VERSION.src.tar.xz.sig libunwind-$LLVM_VERSION.src.tar.xz
$GPG --verify libcxx-$LLVM_VERSION.src.tar.xz.sig libcxx-$LLVM_VERSION.src.tar.xz
$GPG --verify libcxxabi-$LLVM_VERSION.src.tar.xz.sig libcxxabi-$LLVM_VERSION.src.tar.xz
popd popd
@@ -172,7 +196,7 @@ popd
mkdir -p build mkdir -p build
pushd build pushd build
# compile gcc log_tool_name "GCC $GCC_VERSION"
if [ ! -f $PREFIX/bin/gcc ]; then if [ ! -f $PREFIX/bin/gcc ]; then
if [ -d gcc-$GCC_VERSION ]; then if [ -d gcc-$GCC_VERSION ]; then
rm -rf gcc-$GCC_VERSION rm -rf gcc-$GCC_VERSION
@@ -263,7 +287,7 @@ fi
export PATH=$PREFIX/bin:$PATH export PATH=$PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$PREFIX/lib64 export LD_LIBRARY_PATH=$PREFIX/lib64
# compile binutils log_tool_name "binutils $BINUTILS_VERSION"
if [ ! -f $PREFIX/bin/ld.gold ]; then if [ ! -f $PREFIX/bin/ld.gold ]; then
if [ -d binutils-$BINUTILS_VERSION ]; then if [ -d binutils-$BINUTILS_VERSION ]; then
rm -rf binutils-$BINUTILS_VERSION rm -rf binutils-$BINUTILS_VERSION
@@ -327,7 +351,7 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
popd && popd popd && popd
fi fi
# compile gdb log_tool_name "GDB $GDB_VERSION"
if [ ! -f $PREFIX/bin/gdb ]; then if [ ! -f $PREFIX/bin/gdb ]; then
if [ -d gdb-$GDB_VERSION ]; then if [ -d gdb-$GDB_VERSION ]; then
rm -rf gdb-$GDB_VERSION rm -rf gdb-$GDB_VERSION
@@ -363,6 +387,62 @@ if [ ! -f $PREFIX/bin/gdb ]; then
--without-babeltrace \ --without-babeltrace \
--enable-tui \ --enable-tui \
--with-python=python3 --with-python=python3
elif [[ "${DISTRO}" == fedora* ]]; then
# Remove readline, gdb does not compile
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
elif [[ "${DISTRO}" == "amzn-2" ]]; then
# Remove readline, gdb does not compile
env \
CC=gcc \
CXX=g++ \
CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
LDFLAGS="-Wl,-z,relro" \
PYTHON="" \
../configure \
--build=x86_64-linux-gnu \
--host=x86_64-linux-gnu \
--prefix=$PREFIX \
--disable-maintainer-mode \
--disable-dependency-tracking \
--disable-silent-rules \
--disable-gdbtk \
--disable-shared \
--without-guile \
--with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
--with-expat \
--with-system-zlib \
--with-lzma \
--with-babeltrace \
--with-intel-pt \
--enable-tui \
--with-python=python3
else else
# https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0 # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
env \ env \
@@ -398,13 +478,13 @@ if [ ! -f $PREFIX/bin/gdb ]; then
popd && popd popd && popd
fi fi
# install pahole log_tool_name "install pahole"
if [ ! -d $PREFIX/share/pahole-gdb ]; then if [ ! -d $PREFIX/share/pahole-gdb ]; then
unzip ../archives/pahole-gdb-master.zip unzip ../archives/pahole-gdb-master.zip
mv pahole-gdb-master $PREFIX/share/pahole-gdb mv pahole-gdb-master $PREFIX/share/pahole-gdb
fi fi
# setup system gdbinit log_tool_name "setup system gdbinit"
if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then if [ ! -f $PREFIX/etc/gdb/gdbinit ]; then
mkdir -p $PREFIX/etc/gdb mkdir -p $PREFIX/etc/gdb
cat >$PREFIX/etc/gdb/gdbinit <<EOF cat >$PREFIX/etc/gdb/gdbinit <<EOF
@@ -430,7 +510,7 @@ end
EOF EOF
fi fi
# compile cmake log_tool_name "cmake $CMAKE_VERSION"
if [ ! -f $PREFIX/bin/cmake ]; then if [ ! -f $PREFIX/bin/cmake ]; then
if [ -d cmake-$CMAKE_VERSION ]; then if [ -d cmake-$CMAKE_VERSION ]; then
rm -rf cmake-$CMAKE_VERSION rm -rf cmake-$CMAKE_VERSION
@@ -456,7 +536,7 @@ if [ ! -f $PREFIX/bin/cmake ]; then
popd && popd popd && popd
fi fi
# compile cppcheck log_tool_name "cppcheck $CPPCHECK_VERSION"
if [ ! -f $PREFIX/bin/cppcheck ]; then if [ ! -f $PREFIX/bin/cppcheck ]; then
if [ -d cppcheck-$CPPCHECK_VERSION ]; then if [ -d cppcheck-$CPPCHECK_VERSION ]; then
rm -rf cppcheck-$CPPCHECK_VERSION rm -rf cppcheck-$CPPCHECK_VERSION
@@ -480,7 +560,7 @@ if [ ! -f $PREFIX/bin/cppcheck ]; then
popd popd
fi fi
# compile swig log_tool_name "swig $SWIG_VERSION"
if [ ! -d swig-$SWIG_VERSION/install ]; then if [ ! -d swig-$SWIG_VERSION/install ]; then
if [ -d swig-$SWIG_VERSION ]; then if [ -d swig-$SWIG_VERSION ]; then
rm -rf swig-$SWIG_VERSION rm -rf swig-$SWIG_VERSION
@@ -496,7 +576,7 @@ if [ ! -d swig-$SWIG_VERSION/install ]; then
popd && popd popd && popd
fi fi
# compile llvm log_tool_name "LLVM $LLVM_VERSION"
if [ ! -f $PREFIX/bin/clang ]; then if [ ! -f $PREFIX/bin/clang ]; then
if [ -d llvm-$LLVM_VERSION ]; then if [ -d llvm-$LLVM_VERSION ]; then
rm -rf llvm-$LLVM_VERSION rm -rf llvm-$LLVM_VERSION
@@ -513,8 +593,19 @@ if [ ! -f $PREFIX/bin/clang ]; then
mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt mv compiler-rt-$LLVM_VERSION.src/ llvm-$LLVM_VERSION/projects/compiler-rt
tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz
mv libunwind-$LLVM_VERSION.src/include/mach-o llvm-$LLVM_VERSION/tools/lld/include mv libunwind-$LLVM_VERSION.src/include/mach-o llvm-$LLVM_VERSION/tools/lld/include
# The following is required because of libc++
tar -xvf ../archives/libcxx-$LLVM_VERSION.src.tar.xz
mv libcxx-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libcxx
tar -xvf ../archives/libcxxabi-$LLVM_VERSION.src.tar.xz
mv libcxxabi-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libcxxabi
# NOTE: We moved part of the libunwind in one of the previous step.
rm -r libunwind-$LLVM_VERSION.src
tar -xvf ../archives/libunwind-$LLVM_VERSION.src.tar.xz
mv libunwind-$LLVM_VERSION.src llvm-$LLVM_VERSION/projects/libunwind
pushd llvm-$LLVM_VERSION pushd llvm-$LLVM_VERSION
mkdir build && pushd build mkdir -p build && pushd build
# activate swig # activate swig
export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH export PATH=$DIR/build/swig-$SWIG_VERSION/install/bin:$PATH
# influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0 # influenced by: https://buildd.debian.org/status/fetch.php?pkg=llvm-toolchain-7&arch=amd64&ver=1%3A7.0.1%7E%2Brc2-1%7Eexp1&stamp=1541506173&raw=0
@@ -567,7 +658,7 @@ In order to be able to run all of these tools you should install the following
packages: packages:
\`\`\` \`\`\`
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS) $($DIR/../os/$ENV_SCRIPT.sh list TOOLCHAIN_RUN_DEPS)
\`\`\` \`\`\`
## Usage ## Usage
@@ -624,6 +715,7 @@ export PS1="($NAME) \$PS1"
export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64 export LD_LIBRARY_PATH=$PREFIX/lib:$PREFIX/lib64
export CXXFLAGS=-isystem\ $PREFIX/include\ \$CXXFLAGS export CXXFLAGS=-isystem\ $PREFIX/include\ \$CXXFLAGS
export CFLAGS=-isystem\ $PREFIX/include\ \$CFLAGS export CFLAGS=-isystem\ $PREFIX/include\ \$CFLAGS
export VENV=$PREFIX
# disable root # disable root
function su () { function su () {
@@ -675,7 +767,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7 SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
SNAPPY_VERSION=1.1.9 SNAPPY_VERSION=1.1.9
XZ_VERSION=5.2.5 # for LZMA XZ_VERSION=5.2.5 # for LZMA
ZLIB_VERSION=1.2.12 ZLIB_VERSION=1.2.13
ZSTD_VERSION=1.5.0 ZSTD_VERSION=1.5.0
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
@@ -820,7 +912,11 @@ source $PREFIX/activate
export CC=$PREFIX/bin/clang export CC=$PREFIX/bin/clang
export CXX=$PREFIX/bin/clang++ export CXX=$PREFIX/bin/clang++
export CFLAGS="$CFLAGS -fPIC" export CFLAGS="$CFLAGS -fPIC"
export CXXFLAGS="$CXXFLAGS -fPIC" if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
export CXXFLAGS="$CXXFLAGS -fPIC"
else
export CXXFLAGS="$CXXFLAGS -fPIC -stdlib=libc++"
fi
COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
-DCMAKE_PREFIX_PATH=$PREFIX -DCMAKE_PREFIX_PATH=$PREFIX
-DCMAKE_BUILD_TYPE=Release -DCMAKE_BUILD_TYPE=Release
@@ -834,7 +930,7 @@ COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
COMMON_CONFIGURE_FLAGS="--enable-shared=no --prefix=$PREFIX" COMMON_CONFIGURE_FLAGS="--enable-shared=no --prefix=$PREFIX"
COMMON_MAKE_INSTALL_FLAGS="-j$CPUS BUILD_SHARED=no PREFIX=$PREFIX install" COMMON_MAKE_INSTALL_FLAGS="-j$CPUS BUILD_SHARED=no PREFIX=$PREFIX install"
# install bzip2 log_tool_name "bzip2 $BZIP2_VERSION"
if [ ! -f $PREFIX/include/bzlib.h ]; then if [ ! -f $PREFIX/include/bzlib.h ]; then
if [ -d bzip2-$BZIP2_VERSION ]; then if [ -d bzip2-$BZIP2_VERSION ]; then
rm -rf bzip2-$BZIP2_VERSION rm -rf bzip2-$BZIP2_VERSION
@@ -845,7 +941,7 @@ if [ ! -f $PREFIX/include/bzlib.h ]; then
popd popd
fi fi
# install fmt log_tool_name "fmt $FMT_VERSION"
if [ ! -d $PREFIX/include/fmt ]; then if [ ! -d $PREFIX/include/fmt ]; then
if [ -d fmt-$FMT_VERSION ]; then if [ -d fmt-$FMT_VERSION ]; then
rm -rf fmt-$FMT_VERSION rm -rf fmt-$FMT_VERSION
@@ -858,7 +954,7 @@ if [ ! -d $PREFIX/include/fmt ]; then
popd && popd popd && popd
fi fi
# install lz4 log_tool_name "lz4 $LZ4_VERSION"
if [ ! -f $PREFIX/include/lz4.h ]; then if [ ! -f $PREFIX/include/lz4.h ]; then
if [ -d lz4-$LZ4_VERSION ]; then if [ -d lz4-$LZ4_VERSION ]; then
rm -rf lz4-$LZ4_VERSION rm -rf lz4-$LZ4_VERSION
@@ -869,7 +965,7 @@ if [ ! -f $PREFIX/include/lz4.h ]; then
popd popd
fi fi
# install xz log_tool_name "xz $XZ_VERSION"
if [ ! -f $PREFIX/include/lzma.h ]; then if [ ! -f $PREFIX/include/lzma.h ]; then
if [ -d xz-$XZ_VERSION ]; then if [ -d xz-$XZ_VERSION ]; then
rm -rf xz-$XZ_VERSION rm -rf xz-$XZ_VERSION
@@ -881,7 +977,7 @@ if [ ! -f $PREFIX/include/lzma.h ]; then
popd popd
fi fi
# install zlib log_tool_name "zlib $ZLIB_VERSION"
if [ ! -f $PREFIX/include/zlib.h ]; then if [ ! -f $PREFIX/include/zlib.h ]; then
if [ -d zlib-$ZLIB_VERSION ]; then if [ -d zlib-$ZLIB_VERSION ]; then
rm -rf zlib-$ZLIB_VERSION rm -rf zlib-$ZLIB_VERSION
@@ -895,7 +991,7 @@ if [ ! -f $PREFIX/include/zlib.h ]; then
popd && popd popd && popd
fi fi
# install zstd log_tool_name "zstd $ZSTD_VERSION"
if [ ! -f $PREFIX/include/zstd.h ]; then if [ ! -f $PREFIX/include/zstd.h ]; then
if [ -d zstd-$ZSTD_VERSION ]; then if [ -d zstd-$ZSTD_VERSION ]; then
rm -rf zstd-$ZSTD_VERSION rm -rf zstd-$ZSTD_VERSION
@@ -910,7 +1006,8 @@ if [ ! -f $PREFIX/include/zstd.h ]; then
popd && popd popd && popd
fi fi
#install jemalloc # TODO(gitbuda): Freeze jmalloc version.
log_tool_name "jmalloc"
if [ ! -d $PREFIX/include/jemalloc ]; then if [ ! -d $PREFIX/include/jemalloc ]; then
if [ -d jemalloc ]; then if [ -d jemalloc ]; then
rm -rf jemalloc rm -rf jemalloc
@@ -927,7 +1024,7 @@ if [ ! -d $PREFIX/include/jemalloc ]; then
popd popd
fi fi
# install boost log_tool_name "BOOST $BOOST_VERSION"
if [ ! -d $PREFIX/include/boost ]; then if [ ! -d $PREFIX/include/boost ]; then
if [ -d boost_$BOOST_VERSION_UNDERSCORES ]; then if [ -d boost_$BOOST_VERSION_UNDERSCORES ]; then
rm -rf boost_$BOOST_VERSION_UNDERSCORES rm -rf boost_$BOOST_VERSION_UNDERSCORES
@@ -935,15 +1032,24 @@ if [ ! -d $PREFIX/include/boost ]; then
tar -xzf ../archives/boost_$BOOST_VERSION_UNDERSCORES.tar.gz tar -xzf ../archives/boost_$BOOST_VERSION_UNDERSCORES.tar.gz
pushd boost_$BOOST_VERSION_UNDERSCORES pushd boost_$BOOST_VERSION_UNDERSCORES
./bootstrap.sh --prefix=$PREFIX --with-toolset=clang --with-python=python3 --without-icu ./bootstrap.sh --prefix=$PREFIX --with-toolset=clang --with-python=python3 --without-icu
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \ if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \ ./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \ -sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \ -sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib" -sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
else
./b2 toolset=clang -j$CPUS install variant=release link=static cxxstd=20 --disable-icu \
cxxflags="-stdlib=libc++" linkflags="-stdlib=libc++" \
-sZLIB_SOURCE="$PREFIX" -sZLIB_INCLUDE="$PREFIX/include" -sZLIB_LIBPATH="$PREFIX/lib" \
-sBZIP2_SOURCE="$PREFIX" -sBZIP2_INCLUDE="$PREFIX/include" -sBZIP2_LIBPATH="$PREFIX/lib" \
-sLZMA_SOURCE="$PREFIX" -sLZMA_INCLUDE="$PREFIX/include" -sLZMA_LIBPATH="$PREFIX/lib" \
-sZSTD_SOURCE="$PREFIX" -sZSTD_INCLUDE="$PREFIX/include" -sZSTD_LIBPATH="$PREFIX/lib"
fi
popd popd
fi fi
# install double-conversion log_tool_name "double-conversion $DOUBLE_CONVERSION_VERSION"
if [ ! -d $PREFIX/include/double-conversion ]; then if [ ! -d $PREFIX/include/double-conversion ]; then
if [ -d double-conversion-$DOUBLE_CONVERSION_VERSION ]; then if [ -d double-conversion-$DOUBLE_CONVERSION_VERSION ]; then
rm -rf double-conversion-$DOUBLE_CONVERSION_VERSION rm -rf double-conversion-$DOUBLE_CONVERSION_VERSION
@@ -958,7 +1064,8 @@ if [ ! -d $PREFIX/include/double-conversion ]; then
popd && popd popd && popd
fi fi
# install gflags # TODO(gitbuda): Freeze gflags version.
log_tool_name "gflags"
if [ ! -d $PREFIX/include/gflags ]; then if [ ! -d $PREFIX/include/gflags ]; then
if [ -d gflags ]; then if [ -d gflags ]; then
rm -rf gflags rm -rf gflags
@@ -977,7 +1084,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
popd && popd popd && popd
fi fi
# install libunwind log_tool_name "libunwind $LIBUNWIND_VERSION"
if [ ! -f $PREFIX/include/libunwind.h ]; then if [ ! -f $PREFIX/include/libunwind.h ]; then
if [ -d libunwind-$LIBUNWIND_VERSION ]; then if [ -d libunwind-$LIBUNWIND_VERSION ]; then
rm -rf libunwind-$LIBUNWIND_VERSION rm -rf libunwind-$LIBUNWIND_VERSION
@@ -990,7 +1097,7 @@ if [ ! -f $PREFIX/include/libunwind.h ]; then
popd popd
fi fi
# install glog log_tool_name "glog $GLOG_VERSION"
if [ ! -d $PREFIX/include/glog ]; then if [ ! -d $PREFIX/include/glog ]; then
if [ -d glog-$GLOG_VERSION ]; then if [ -d glog-$GLOG_VERSION ]; then
rm -rf glog-$GLOG_VERSION rm -rf glog-$GLOG_VERSION
@@ -1004,7 +1111,7 @@ if [ ! -d $PREFIX/include/glog ]; then
popd && popd popd && popd
fi fi
# install libevent log_tool_name "libevent $LIBEVENT_VERSION"
if [ ! -d $PREFIX/include/event2 ]; then if [ ! -d $PREFIX/include/event2 ]; then
if [ -d libevent-$LIBEVENT_VERSION ]; then if [ -d libevent-$LIBEVENT_VERSION ]; then
rm -rf libevent-$LIBEVENT_VERSION rm -rf libevent-$LIBEVENT_VERSION
@@ -1023,7 +1130,7 @@ if [ ! -d $PREFIX/include/event2 ]; then
popd && popd popd && popd
fi fi
# install snappy log_tool_name "snappy $SNAPPY_VERSION"
if [ ! -f $PREFIX/include/snappy.h ]; then if [ ! -f $PREFIX/include/snappy.h ]; then
if [ -d snappy-$SNAPPY_VERSION ]; then if [ -d snappy-$SNAPPY_VERSION ]; then
rm -rf snappy-$SNAPPY_VERSION rm -rf snappy-$SNAPPY_VERSION
@@ -1041,7 +1148,7 @@ if [ ! -f $PREFIX/include/snappy.h ]; then
popd && popd popd && popd
fi fi
# install libsodium log_tool_name "libsodium $LIBSODIUM_VERSION"
if [ ! -f $PREFIX/include/sodium.h ]; then if [ ! -f $PREFIX/include/sodium.h ]; then
if [ -d libsodium-$LIBSODIUM_VERSION ]; then if [ -d libsodium-$LIBSODIUM_VERSION ]; then
rm -rf libsodium-$LIBSODIUM_VERSION rm -rf libsodium-$LIBSODIUM_VERSION
@@ -1053,7 +1160,7 @@ if [ ! -f $PREFIX/include/sodium.h ]; then
popd popd
fi fi
# install libaio log_tool_name "libaio $LIBAIO_VERSION"
if [ ! -f $PREFIX/include/libaio.h ]; then if [ ! -f $PREFIX/include/libaio.h ]; then
if [ -d libaio-$LIBAIO_VERSION ]; then if [ -d libaio-$LIBAIO_VERSION ]; then
rm -rf libaio-$LIBAIO_VERSION rm -rf libaio-$LIBAIO_VERSION
@@ -1064,114 +1171,121 @@ if [ ! -f $PREFIX/include/libaio.h ]; then
popd popd
fi fi
# install folly if [[ "${DISTRO}" != "amzn-2" ]]; then
if [ ! -d $PREFIX/include/folly ]; then log_tool_name "folly $FBLIBS_VERSION"
if [ -d folly-$FBLIBS_VERSION ]; then if [ ! -d $PREFIX/include/folly ]; then
rm -rf folly-$FBLIBS_VERSION if [ -d folly-$FBLIBS_VERSION ]; then
rm -rf folly-$FBLIBS_VERSION
fi
mkdir folly-$FBLIBS_VERSION
tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
pushd folly-$FBLIBS_VERSION
patch -p1 < ../../folly.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBOOST_LINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCXX_STD="c++20"
make -j$CPUS install
popd && popd
fi fi
mkdir folly-$FBLIBS_VERSION
tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
pushd folly-$FBLIBS_VERSION
patch -p1 < ../../folly.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBOOST_LINK_STATIC=ON \
-DBUILD_TESTS=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCXX_STD="c++20"
make -j$CPUS install
popd && popd
fi
# install fizz log_tool_name "fizz $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/fizz ]; then if [ ! -d $PREFIX/include/fizz ]; then
if [ -d fizz-$FBLIBS_VERSION ]; then if [ -d fizz-$FBLIBS_VERSION ]; then
rm -rf fizz-$FBLIBS_VERSION rm -rf fizz-$FBLIBS_VERSION
fi
mkdir fizz-$FBLIBS_VERSION
tar -xzf ../archives/fizz-$FBLIBS_VERSION.tar.gz -C fizz-$FBLIBS_VERSION
pushd fizz-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../fizz $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
fi fi
mkdir fizz-$FBLIBS_VERSION
tar -xzf ../archives/fizz-$FBLIBS_VERSION.tar.gz -C fizz-$FBLIBS_VERSION
pushd fizz-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../fizz $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
fi
# install wangle log_tool_name "wangle FBLIBS_VERSION"
if [ ! -d $PREFIX/include/wangle ]; then if [ ! -d $PREFIX/include/wangle ]; then
if [ -d wangle-$FBLIBS_VERSION ]; then if [ -d wangle-$FBLIBS_VERSION ]; then
rm -rf wangle-$FBLIBS_VERSION rm -rf wangle-$FBLIBS_VERSION
fi
mkdir wangle-$FBLIBS_VERSION
tar -xzf ../archives/wangle-$FBLIBS_VERSION.tar.gz -C wangle-$FBLIBS_VERSION
pushd wangle-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../wangle $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
fi fi
mkdir wangle-$FBLIBS_VERSION
tar -xzf ../archives/wangle-$FBLIBS_VERSION.tar.gz -C wangle-$FBLIBS_VERSION
pushd wangle-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake ../wangle $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_EXAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF
make -j$CPUS install
popd && popd
fi
# install proxygen log_tool_name "proxygen $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/proxygen ]; then if [ ! -d $PREFIX/include/proxygen ]; then
if [ -d proxygen-$FBLIBS_VERSION ]; then if [ -d proxygen-$FBLIBS_VERSION ]; then
rm -rf proxygen-$FBLIBS_VERSION rm -rf proxygen-$FBLIBS_VERSION
fi
mkdir proxygen-$FBLIBS_VERSION
tar -xzf ../archives/proxygen-$FBLIBS_VERSION.tar.gz -C proxygen-$FBLIBS_VERSION
pushd proxygen-$FBLIBS_VERSION
patch -p1 < ../../proxygen.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_SAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DBUILD_QUIC=OFF
make -j$CPUS install
popd && popd
fi fi
mkdir proxygen-$FBLIBS_VERSION
tar -xzf ../archives/proxygen-$FBLIBS_VERSION.tar.gz -C proxygen-$FBLIBS_VERSION
pushd proxygen-$FBLIBS_VERSION
patch -p1 < ../../proxygen.patch
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-DBUILD_TESTS=OFF \
-DBUILD_SAMPLES=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DBUILD_QUIC=OFF
make -j$CPUS install
popd && popd
fi
# install flex log_tool_name "flex $FBLIBS_VERSION"
if [ ! -f $PREFIX/include/FlexLexer.h ]; then if [ ! -f $PREFIX/include/FlexLexer.h ]; then
if [ -d flex-$FLEX_VERSION ]; then if [ -d flex-$FLEX_VERSION ]; then
rm -rf flex-$FLEX_VERSION rm -rf flex-$FLEX_VERSION
fi
tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
pushd flex-$FLEX_VERSION
./configure $COMMON_CONFIGURE_FLAGS
make -j$CPUS install
popd
fi fi
tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
pushd flex-$FLEX_VERSION
./configure $COMMON_CONFIGURE_FLAGS
make -j$CPUS install
popd
fi
# install fbthrift log_tool_name "fbthrift $FBLIBS_VERSION"
if [ ! -d $PREFIX/include/thrift ]; then if [ ! -d $PREFIX/include/thrift ]; then
if [ -d fbthrift-$FBLIBS_VERSION ]; then if [ -d fbthrift-$FBLIBS_VERSION ]; then
rm -rf fbthrift-$FBLIBS_VERSION rm -rf fbthrift-$FBLIBS_VERSION
fi
git clone --depth 1 --branch v$FBLIBS_VERSION https://github.com/facebook/fbthrift.git fbthrift-$FBLIBS_VERSION
pushd fbthrift-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
CMAKE_CXX_FLAGS="-fsized-deallocation"
else
CMAKE_CXX_FLAGS="-fsized-deallocation -stdlib=libc++"
fi
cmake .. $COMMON_CMAKE_FLAGS \
-Denable_tests=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"
make -j$CPUS install
popd
fi fi
git clone --depth 1 --branch v$FBLIBS_VERSION https://github.com/facebook/fbthrift.git fbthrift-$FBLIBS_VERSION
pushd fbthrift-$FBLIBS_VERSION
# build is used by facebook builder
mkdir _build
pushd _build
cmake .. $COMMON_CMAKE_FLAGS \
-Denable_tests=OFF \
-DGFLAGS_NOTHREADS=OFF \
-DCMAKE_CXX_FLAGS=-fsized-deallocation
make -j$CPUS install
popd
fi fi
popd popd
@@ -1179,7 +1293,7 @@ popd
# create toolchain archive # create toolchain archive
if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
DISTRO_FULL_NAME=${DISTRO} DISTRO_FULL_NAME=${DISTRO}
if [[ "${DISTRO}" == centos* ]]; then if [[ "${DISTRO}" == centos* ]] || [[ "${DISTRO}" == fedora* ]]; then
if [[ "$for_arm" = "true" ]]; then if [[ "$for_arm" = "true" ]]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64" DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
else else
@@ -1192,7 +1306,12 @@ if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64" DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
fi fi
fi fi
if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
# Pass because infra scripts assume there is not C++ standard lib in the name.
echo "NOTE: Not adding anything to the archive name that GCC C++ standard lib is used."
else
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-libc++"
fi
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
fi fi

View File

@@ -1,15 +1,45 @@
#!/bin/bash #!/bin/bash
operating_system() { function operating_system() {
grep -E '^(VERSION_)?ID=' /etc/os-release | \ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-' grep -E '^(VERSION_)?ID=' /etc/os-release | \
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
elif [[ "$OSTYPE" == "darwin"* ]]; then
echo "$(sw_vers -productName)-$(sw_vers -productVersion | cut -d '.' -f 1)"
else
echo "operating_system called on an unknown OS"
exit 1
fi
} }
architecture() { function check_operating_system() {
if [ "$(operating_system)" != "$1" ]; then
echo "Not the right operating system!"
exit 1
else
echo "The right operating system."
fi
}
function architecture() {
uname -m uname -m
} }
check_all_yum() { check_architecture() {
local ARCH=$(architecture)
for arch in "$@"; do
if [ "${ARCH}" = "$arch" ]; then
echo "The right architecture!"
return 0
fi
done
echo "Not the right architecture!"
echo "Expected: $@"
echo "Actual: ${ARCH}"
exit 1
}
function check_all_yum() {
local missing="" local missing=""
for pkg in $1; do for pkg in $1; do
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
@@ -22,7 +52,7 @@ check_all_yum() {
fi fi
} }
check_all_dpkg() { function check_all_dpkg() {
local missing="" local missing=""
for pkg in $1; do for pkg in $1; do
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
@@ -35,7 +65,7 @@ check_all_dpkg() {
fi fi
} }
check_all_dnf() { function check_all_dnf() {
local missing="" local missing=""
for pkg in $1; do for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
@@ -47,8 +77,34 @@ check_all_dnf() {
exit 1 exit 1
fi fi
} }
install_all_apt() {
function install_all_apt() {
for pkg in $1; do for pkg in $1; do
apt install -y "$pkg" apt install -y "$pkg"
done done
} }
function install_custom_golang() {
# NOTE: The official https://go.dev/doc/manage-install doesn't seem to be working.
GOVERSION="$1"
GOINSTALLDIR="/opt/go$GOVERSION"
GOROOT="$GOINSTALLDIR/go" # GOPATH=$HOME/go
if [ ! -f "$GOROOT/bin/go" ]; then
curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
mkdir -p "$GOINSTALLDIR"
tar -C "$GOINSTALLDIR" -xzf go$GOVERSION.linux-amd64.tar.gz
fi
echo "go $GOVERSION installed under $GOROOT"
}
function install_custom_maven() {
MVNVERSION="$1"
MVNINSTALLDIR="/opt/apache-maven-$MVNVERSION"
MVNURL="https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/maven/apache-maven-$MVNVERSION-bin.tar.gz"
if [ ! -f "$MVNINSTALLDIR/bin/mvn" ]; then
echo "Downloading maven from $MVNURL"
curl -LO "$MVNURL"
tar -C "/opt" -xzf "apache-maven-$MVNVERSION-bin.tar.gz"
fi
echo "maven $MVNVERSION installed under $MVNINSTALLDIR"
}

26
import/mglogs2cypherl.sh Executable file
View File

@@ -0,0 +1,26 @@
#!/bin/bash -e
COLOR_ORANGE="\e[38;5;208m"
COLOR_GREEN="\e[38;5;35m"
COLOR_RED="\e[0;31m"
COLOR_NULL="\e[0m"
print_help() {
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 memgraph_logs_file_path cypherl_output_path"
exit 1
}
if [ "$#" -ne 2 ]; then
print_help
fi
INPUT="$1"
OUTPUT="$2"
if [ ! -f "$INPUT" ]; then
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} memgraph_logs_file_path is not a file!"
print_help
fi
awk -v RS="Run] '" 'NR>1 { print $0 }' < "$INPUT" | sed -e "/^\[/d;" -e "s/'\([^']*\)$/;/g" > "$OUTPUT"
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl file under $OUTPUT"
echo ""
echo "Import can be done by executing => \`cat $OUTPUT | mgconsole\`"

35
import/n2mg_cypherl.sh Executable file
View File

@@ -0,0 +1,35 @@
#!/bin/bash -e
COLOR_ORANGE="\e[38;5;208m"
COLOR_GREEN="\e[38;5;35m"
COLOR_RED="\e[0;31m"
COLOR_NULL="\e[0m"
print_help() {
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 input_file_path output_file_path"
exit 1
}
if [ "$#" -ne 2 ]; then
print_help
fi
INPUT="$1"
OUTPUT="$2"
if [ ! -f "$INPUT" ]; then
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
print_help
fi
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create consraints manually if needed${COLOR_NULL}"
sed -e 's/^:begin/BEGIN/g; s/^BEGIN$/BEGIN;/g;' \
-e 's/^:commit/COMMIT/g; s/^COMMIT$/COMMIT;/g;' \
-e '/^CALL/d; /^SCHEMA AWAIT/d;' \
-e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
-e 's/) ON (n./(/g;' \
-e '/^CREATE CONSTRAINT/d; /^DROP CONSTRAINT/d;' "$INPUT" > "$OUTPUT"
echo ""
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher file under $OUTPUT"
echo ""
echo "Please import data by executing => \`cat $OUTPUT | mgconsole\`"

763
include/_mgp.hpp Normal file
View File

@@ -0,0 +1,763 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
/// @file _mgp.hpp
///
/// The file contains methods that connect mg procedures and the outside code
/// Methods like mapping a graph into memory or assigning new mg results or
/// their properties are implemented.
#pragma once
#include "mg_exceptions.hpp"
#include "mg_procedure.h"
namespace mgp {
namespace {
inline void MgExceptionHandle(mgp_error result_code) {
switch (result_code) {
case mgp_error::MGP_ERROR_UNKNOWN_ERROR:
throw mg_exception::UnknownException();
case mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE:
throw mg_exception::AllocationException();
case mgp_error::MGP_ERROR_INSUFFICIENT_BUFFER:
throw mg_exception::InsufficientBufferException();
case mgp_error::MGP_ERROR_OUT_OF_RANGE:
throw mg_exception::OutOfRangeException();
case mgp_error::MGP_ERROR_LOGIC_ERROR:
throw mg_exception::LogicException();
case mgp_error::MGP_ERROR_DELETED_OBJECT:
throw mg_exception::DeletedObjectException();
case mgp_error::MGP_ERROR_INVALID_ARGUMENT:
throw mg_exception::InvalidArgumentException();
case mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS:
throw mg_exception::KeyAlreadyExistsException();
case mgp_error::MGP_ERROR_IMMUTABLE_OBJECT:
throw mg_exception::ImmutableObjectException();
case mgp_error::MGP_ERROR_VALUE_CONVERSION:
throw mg_exception::ValueConversionException();
case mgp_error::MGP_ERROR_SERIALIZATION_ERROR:
throw mg_exception::SerializationException();
default:
return;
}
}
template <typename TResult, typename TFunc, typename... TArgs>
TResult MgInvoke(TFunc func, TArgs... args) {
TResult result{};
auto result_code = func(args..., &result);
MgExceptionHandle(result_code);
return result;
}
template <typename TFunc, typename... TArgs>
inline void MgInvokeVoid(TFunc func, TArgs... args) {
auto result_code = func(args...);
MgExceptionHandle(result_code);
}
} // namespace
// mgp_value
// Make value
inline mgp_value *value_make_null(mgp_memory *memory) { return MgInvoke<mgp_value *>(mgp_value_make_null, memory); }
inline mgp_value *value_make_bool(int val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_bool, val, memory);
}
inline mgp_value *value_make_int(int64_t val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_int, val, memory);
}
inline mgp_value *value_make_double(double val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_double, val, memory);
}
inline mgp_value *value_make_string(const char *val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_make_string, val, memory);
}
inline mgp_value *value_make_list(mgp_list *val) { return MgInvoke<mgp_value *>(mgp_value_make_list, val); }
inline mgp_value *value_make_map(mgp_map *val) { return MgInvoke<mgp_value *>(mgp_value_make_map, val); }
inline mgp_value *value_make_vertex(mgp_vertex *val) { return MgInvoke<mgp_value *>(mgp_value_make_vertex, val); }
inline mgp_value *value_make_edge(mgp_edge *val) { return MgInvoke<mgp_value *>(mgp_value_make_edge, val); }
inline mgp_value *value_make_path(mgp_path *val) { return MgInvoke<mgp_value *>(mgp_value_make_path, val); }
inline mgp_value *value_make_date(mgp_date *val) { return MgInvoke<mgp_value *>(mgp_value_make_date, val); }
inline mgp_value *value_make_local_time(mgp_local_time *val) {
return MgInvoke<mgp_value *>(mgp_value_make_local_time, val);
}
inline mgp_value *value_make_local_date_time(mgp_local_date_time *val) {
return MgInvoke<mgp_value *>(mgp_value_make_local_date_time, val);
}
inline mgp_value *value_make_duration(mgp_duration *val) { return MgInvoke<mgp_value *>(mgp_value_make_duration, val); }
// Copy value
// TODO: implement within MGP API
// with primitive types ({bool, int, double, string}), create a new identical value
// otherwise call mgp_##TYPE_copy and convert tpye
inline mgp_value *value_copy(mgp_value *val, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_value_copy, val, memory);
}
// Destroy value
inline void value_destroy(mgp_value *val) { mgp_value_destroy(val); }
// Get value of type
inline mgp_value_type value_get_type(mgp_value *val) { return MgInvoke<mgp_value_type>(mgp_value_get_type, val); }
inline bool value_get_bool(mgp_value *val) { return MgInvoke<int>(mgp_value_get_bool, val); }
inline int64_t value_get_int(mgp_value *val) { return MgInvoke<int64_t>(mgp_value_get_int, val); }
inline double value_get_double(mgp_value *val) { return MgInvoke<double>(mgp_value_get_double, val); }
inline double value_get_numeric(mgp_value *val) {
if (MgInvoke<int>(mgp_value_is_int, val)) {
return static_cast<double>(value_get_int(val));
}
return value_get_double(val);
}
inline const char *value_get_string(mgp_value *val) { return MgInvoke<const char *>(mgp_value_get_string, val); }
inline mgp_list *value_get_list(mgp_value *val) { return MgInvoke<mgp_list *>(mgp_value_get_list, val); }
inline mgp_map *value_get_map(mgp_value *val) { return MgInvoke<mgp_map *>(mgp_value_get_map, val); }
inline mgp_vertex *value_get_vertex(mgp_value *val) { return MgInvoke<mgp_vertex *>(mgp_value_get_vertex, val); }
inline mgp_edge *value_get_edge(mgp_value *val) { return MgInvoke<mgp_edge *>(mgp_value_get_edge, val); }
inline mgp_path *value_get_path(mgp_value *val) { return MgInvoke<mgp_path *>(mgp_value_get_path, val); }
inline mgp_date *value_get_date(mgp_value *val) { return MgInvoke<mgp_date *>(mgp_value_get_date, val); }
inline mgp_local_time *value_get_local_time(mgp_value *val) {
return MgInvoke<mgp_local_time *>(mgp_value_get_local_time, val);
}
inline mgp_local_date_time *value_get_local_date_time(mgp_value *val) {
return MgInvoke<mgp_local_date_time *>(mgp_value_get_local_date_time, val);
}
inline mgp_duration *value_get_duration(mgp_value *val) {
return MgInvoke<mgp_duration *>(mgp_value_get_duration, val);
}
// Check type of value
inline bool value_is_null(mgp_value *val) { return MgInvoke<int>(mgp_value_is_null, val); }
inline bool value_is_bool(mgp_value *val) { return MgInvoke<int>(mgp_value_is_bool, val); }
inline bool value_is_int(mgp_value *val) { return MgInvoke<int>(mgp_value_is_int, val); }
inline bool value_is_double(mgp_value *val) { return MgInvoke<int>(mgp_value_is_double, val); }
inline bool value_is_numeric(mgp_value *val) { return value_is_int(val) || value_is_double(val); }
inline bool value_is_string(mgp_value *val) { return MgInvoke<int>(mgp_value_is_string, val); }
inline bool value_is_list(mgp_value *val) { return MgInvoke<int>(mgp_value_is_list, val); }
inline bool value_is_map(mgp_value *val) { return MgInvoke<int>(mgp_value_is_map, val); }
inline bool value_is_vertex(mgp_value *val) { return MgInvoke<int>(mgp_value_is_vertex, val); }
inline bool value_is_edge(mgp_value *val) { return MgInvoke<int>(mgp_value_is_edge, val); }
inline bool value_is_path(mgp_value *val) { return MgInvoke<int>(mgp_value_is_path, val); }
inline bool value_is_date(mgp_value *val) { return MgInvoke<int>(mgp_value_is_date, val); }
inline bool value_is_local_time(mgp_value *val) { return MgInvoke<int>(mgp_value_is_local_time, val); }
inline bool value_is_local_date_time(mgp_value *val) { return MgInvoke<int>(mgp_value_is_local_date_time, val); }
inline bool value_is_duration(mgp_value *val) { return MgInvoke<int>(mgp_value_is_duration, val); }
// Get type
inline mgp_type *type_any() { return MgInvoke<mgp_type *>(mgp_type_any); }
inline mgp_type *type_bool() { return MgInvoke<mgp_type *>(mgp_type_bool); }
inline mgp_type *type_string() { return MgInvoke<mgp_type *>(mgp_type_string); }
inline mgp_type *type_int() { return MgInvoke<mgp_type *>(mgp_type_int); }
inline mgp_type *type_float() { return MgInvoke<mgp_type *>(mgp_type_float); }
inline mgp_type *type_number() { return MgInvoke<mgp_type *>(mgp_type_number); }
inline mgp_type *type_list(mgp_type *element_type) { return MgInvoke<mgp_type *>(mgp_type_list, element_type); }
inline mgp_type *type_map() { return MgInvoke<mgp_type *>(mgp_type_map); }
inline mgp_type *type_node() { return MgInvoke<mgp_type *>(mgp_type_node); }
inline mgp_type *type_relationship() { return MgInvoke<mgp_type *>(mgp_type_relationship); }
inline mgp_type *type_path() { return MgInvoke<mgp_type *>(mgp_type_path); }
inline mgp_type *type_date() { return MgInvoke<mgp_type *>(mgp_type_date); }
inline mgp_type *type_local_time() { return MgInvoke<mgp_type *>(mgp_type_local_time); }
inline mgp_type *type_local_date_time() { return MgInvoke<mgp_type *>(mgp_type_local_date_time); }
inline mgp_type *type_duration() { return MgInvoke<mgp_type *>(mgp_type_duration); }
inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke<mgp_type *>(mgp_type_nullable, type); }
// mgp_graph
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_graph_create_vertex, graph, memory);
}
inline void graph_delete_vertex(mgp_graph *graph, mgp_vertex *vertex) {
MgInvokeVoid(mgp_graph_delete_vertex, graph, vertex);
}
inline void graph_detach_delete_vertex(mgp_graph *graph, mgp_vertex *vertex) {
MgInvokeVoid(mgp_graph_detach_delete_vertex, graph, vertex);
}
inline mgp_edge *graph_create_edge(mgp_graph *graph, mgp_vertex *from, mgp_vertex *to, mgp_edge_type type,
mgp_memory *memory) {
return MgInvoke<mgp_edge *>(mgp_graph_create_edge, graph, from, to, type, memory);
}
inline void graph_delete_edge(mgp_graph *graph, mgp_edge *edge) { MgInvokeVoid(mgp_graph_delete_edge, graph, edge); }
inline mgp_vertex *graph_get_vertex_by_id(mgp_graph *g, mgp_vertex_id id, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_graph_get_vertex_by_id, g, id, memory);
}
inline mgp_vertices_iterator *graph_iter_vertices(mgp_graph *g, mgp_memory *memory) {
return MgInvoke<mgp_vertices_iterator *>(mgp_graph_iter_vertices, g, memory);
}
// mgp_vertices_iterator
inline void vertices_iterator_destroy(mgp_vertices_iterator *it) { mgp_vertices_iterator_destroy(it); }
inline mgp_vertex *vertices_iterator_get(mgp_vertices_iterator *it) {
return MgInvoke<mgp_vertex *>(mgp_vertices_iterator_get, it);
}
inline mgp_vertex *vertices_iterator_next(mgp_vertices_iterator *it) {
return MgInvoke<mgp_vertex *>(mgp_vertices_iterator_next, it);
}
// mgp_edges_iterator
inline void edges_iterator_destroy(mgp_edges_iterator *it) { mgp_edges_iterator_destroy(it); }
inline mgp_edge *edges_iterator_get(mgp_edges_iterator *it) { return MgInvoke<mgp_edge *>(mgp_edges_iterator_get, it); }
inline mgp_edge *edges_iterator_next(mgp_edges_iterator *it) {
return MgInvoke<mgp_edge *>(mgp_edges_iterator_next, it);
}
// mgp_properties_iterator
inline void properties_iterator_destroy(mgp_properties_iterator *it) { mgp_properties_iterator_destroy(it); }
inline mgp_property *properties_iterator_get(mgp_properties_iterator *it) {
return MgInvoke<mgp_property *>(mgp_properties_iterator_get, it);
}
inline mgp_property *properties_iterator_next(mgp_properties_iterator *it) {
return MgInvoke<mgp_property *>(mgp_properties_iterator_next, it);
}
// Container {mgp_list, mgp_map} methods
// mgp_list
inline mgp_list *list_make_empty(size_t capacity, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_make_empty, capacity, memory);
}
inline mgp_list *list_copy(mgp_list *list, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_copy, list, memory);
}
inline void list_destroy(mgp_list *list) { mgp_list_destroy(list); }
inline void list_append(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append, list, val); }
inline void list_append_extend(mgp_list *list, mgp_value *val) { MgInvokeVoid(mgp_list_append_extend, list, val); }
inline size_t list_size(mgp_list *list) { return MgInvoke<size_t>(mgp_list_size, list); }
inline size_t list_capacity(mgp_list *list) { return MgInvoke<size_t>(mgp_list_capacity, list); }
inline mgp_value *list_at(mgp_list *list, size_t index) { return MgInvoke<mgp_value *>(mgp_list_at, list, index); }
// mgp_map
inline mgp_map *map_make_empty(mgp_memory *memory) { return MgInvoke<mgp_map *>(mgp_map_make_empty, memory); }
inline mgp_map *map_copy(mgp_map *map, mgp_memory *memory) { return MgInvoke<mgp_map *>(mgp_map_copy, map, memory); }
inline void map_destroy(mgp_map *map) { mgp_map_destroy(map); }
inline void map_insert(mgp_map *map, const char *key, mgp_value *value) {
MgInvokeVoid(mgp_map_insert, map, key, value);
}
inline void map_update(mgp_map *map, const char *key, mgp_value *value) {
MgInvokeVoid(mgp_map_update, map, key, value);
}
inline void map_erase(mgp_map *map, const char *key) { MgInvokeVoid(mgp_map_erase, map, key); }
inline size_t map_size(mgp_map *map) { return MgInvoke<size_t>(mgp_map_size, map); }
inline mgp_value *map_at(mgp_map *map, const char *key) { return MgInvoke<mgp_value *>(mgp_map_at, map, key); }
inline const char *map_item_key(mgp_map_item *item) { return MgInvoke<const char *>(mgp_map_item_key, item); }
inline mgp_value *map_item_value(mgp_map_item *item) { return MgInvoke<mgp_value *>(mgp_map_item_value, item); }
inline mgp_map_items_iterator *map_iter_items(mgp_map *map, mgp_memory *memory) {
return MgInvoke<mgp_map_items_iterator *>(mgp_map_iter_items, map, memory);
}
inline void map_items_iterator_destroy(mgp_map_items_iterator *it) { mgp_map_items_iterator_destroy(it); }
inline mgp_map_item *map_items_iterator_get(mgp_map_items_iterator *it) {
return MgInvoke<mgp_map_item *>(mgp_map_items_iterator_get, it);
}
inline mgp_map_item *map_items_iterator_next(mgp_map_items_iterator *it) {
return MgInvoke<mgp_map_item *>(mgp_map_items_iterator_next, it);
}
// mgp_vertex
inline mgp_vertex_id vertex_get_id(mgp_vertex *v) { return MgInvoke<mgp_vertex_id>(mgp_vertex_get_id, v); }
inline mgp_vertex *vertex_copy(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_vertex *>(mgp_vertex_copy, v, memory);
}
inline void vertex_destroy(mgp_vertex *v) { mgp_vertex_destroy(v); }
inline bool vertex_equal(mgp_vertex *v1, mgp_vertex *v2) { return MgInvoke<int>(mgp_vertex_equal, v1, v2); }
inline size_t vertex_labels_count(mgp_vertex *v) { return MgInvoke<size_t>(mgp_vertex_labels_count, v); }
inline mgp_label vertex_label_at(mgp_vertex *v, size_t index) {
return MgInvoke<mgp_label>(mgp_vertex_label_at, v, index);
}
inline bool vertex_has_label(mgp_vertex *v, mgp_label label) { return MgInvoke<int>(mgp_vertex_has_label, v, label); }
inline bool vertex_has_label_named(mgp_vertex *v, const char *label_name) {
return MgInvoke<int>(mgp_vertex_has_label_named, v, label_name);
}
inline void vertex_add_label(mgp_vertex *vertex, mgp_label label) { MgInvokeVoid(mgp_vertex_add_label, vertex, label); }
inline void vertex_remove_label(mgp_vertex *vertex, mgp_label label) {
MgInvokeVoid(mgp_vertex_remove_label, vertex, label);
}
inline mgp_value *vertex_get_property(mgp_vertex *v, const char *property_name, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_vertex_get_property, v, property_name, memory);
}
inline void vertex_set_property(mgp_vertex *v, const char *property_name, mgp_value *property_value) {
MgInvokeVoid(mgp_vertex_set_property, v, property_name, property_value);
}
inline mgp_properties_iterator *vertex_iter_properties(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_properties_iterator *>(mgp_vertex_iter_properties, v, memory);
}
inline mgp_edges_iterator *vertex_iter_in_edges(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_edges_iterator *>(mgp_vertex_iter_in_edges, v, memory);
}
inline mgp_edges_iterator *vertex_iter_out_edges(mgp_vertex *v, mgp_memory *memory) {
return MgInvoke<mgp_edges_iterator *>(mgp_vertex_iter_out_edges, v, memory);
}
// mgp_edge
inline mgp_edge_id edge_get_id(mgp_edge *e) { return MgInvoke<mgp_edge_id>(mgp_edge_get_id, e); }
inline mgp_edge *edge_copy(mgp_edge *e, mgp_memory *memory) { return MgInvoke<mgp_edge *>(mgp_edge_copy, e, memory); }
inline void edge_destroy(mgp_edge *e) { mgp_edge_destroy(e); }
inline bool edge_equal(mgp_edge *e1, mgp_edge *e2) { return MgInvoke<int>(mgp_edge_equal, e1, e2); }
inline mgp_edge_type edge_get_type(mgp_edge *e) { return MgInvoke<mgp_edge_type>(mgp_edge_get_type, e); }
inline mgp_vertex *edge_get_from(mgp_edge *e) { return MgInvoke<mgp_vertex *>(mgp_edge_get_from, e); }
inline mgp_vertex *edge_get_to(mgp_edge *e) { return MgInvoke<mgp_vertex *>(mgp_edge_get_to, e); }
inline mgp_value *edge_get_property(mgp_edge *e, const char *property_name, mgp_memory *memory) {
return MgInvoke<mgp_value *>(mgp_edge_get_property, e, property_name, memory);
}
inline void edge_set_property(mgp_edge *e, const char *property_name, mgp_value *property_value) {
MgInvokeVoid(mgp_edge_set_property, e, property_name, property_value);
}
inline mgp_properties_iterator *edge_iter_properties(mgp_edge *e, mgp_memory *memory) {
return MgInvoke<mgp_properties_iterator *>(mgp_edge_iter_properties, e, memory);
}
// mgp_path
inline mgp_path *path_make_with_start(mgp_vertex *vertex, mgp_memory *memory) {
return MgInvoke<mgp_path *>(mgp_path_make_with_start, vertex, memory);
}
inline mgp_path *path_copy(mgp_path *path, mgp_memory *memory) {
return MgInvoke<mgp_path *>(mgp_path_copy, path, memory);
}
inline void path_destroy(mgp_path *path) { mgp_path_destroy(path); }
inline void path_expand(mgp_path *path, mgp_edge *edge) { MgInvokeVoid(mgp_path_expand, path, edge); }
inline size_t path_size(mgp_path *path) { return MgInvoke<size_t>(mgp_path_size, path); }
inline mgp_vertex *path_vertex_at(mgp_path *path, size_t index) {
return MgInvoke<mgp_vertex *>(mgp_path_vertex_at, path, index);
}
inline mgp_edge *path_edge_at(mgp_path *path, size_t index) {
return MgInvoke<mgp_edge *>(mgp_path_edge_at, path, index);
}
inline bool path_equal(mgp_path *p1, mgp_path *p2) { return MgInvoke<int>(mgp_path_equal, p1, p2); }
// Temporal type {mgp_date, mgp_local_time, mgp_local_date_time, mgp_duration} methods
// mgp_date
inline mgp_date *date_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_from_string, string, memory);
}
inline mgp_date *date_from_parameters(mgp_date_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_from_parameters, parameters, memory);
}
inline mgp_date *date_copy(mgp_date *date, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_copy, date, memory);
}
inline void date_destroy(mgp_date *date) { mgp_date_destroy(date); }
inline bool date_equal(mgp_date *first, mgp_date *second) { return MgInvoke<int>(mgp_date_equal, first, second); }
inline int date_get_year(mgp_date *date) { return MgInvoke<int>(mgp_date_get_year, date); }
inline int date_get_month(mgp_date *date) { return MgInvoke<int>(mgp_date_get_month, date); }
inline int date_get_day(mgp_date *date) { return MgInvoke<int>(mgp_date_get_day, date); }
inline int64_t date_timestamp(mgp_date *date) { return MgInvoke<int64_t>(mgp_date_timestamp, date); }
inline mgp_date *date_now(mgp_memory *memory) { return MgInvoke<mgp_date *>(mgp_date_now, memory); }
inline mgp_date *date_add_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_add_duration, date, dur, memory);
}
inline mgp_date *date_sub_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_date *>(mgp_date_sub_duration, date, dur, memory);
}
inline mgp_duration *date_diff(mgp_date *first, mgp_date *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_date_diff, first, second, memory);
}
// mgp_local_time
inline mgp_local_time *local_time_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_from_string, string, memory);
}
inline mgp_local_time *local_time_from_parameters(mgp_local_time_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_from_parameters, parameters, memory);
}
inline mgp_local_time *local_time_copy(mgp_local_time *local_time, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_copy, local_time, memory);
}
inline void local_time_destroy(mgp_local_time *local_time) { mgp_local_time_destroy(local_time); }
inline bool local_time_equal(mgp_local_time *first, mgp_local_time *second) {
return MgInvoke<int>(mgp_local_time_equal, first, second);
}
inline int local_time_get_hour(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_hour, local_time);
}
inline int local_time_get_minute(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_minute, local_time);
}
inline int local_time_get_second(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_second, local_time);
}
inline int local_time_get_millisecond(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_millisecond, local_time);
}
inline int local_time_get_microsecond(mgp_local_time *local_time) {
return MgInvoke<int>(mgp_local_time_get_microsecond, local_time);
}
inline int64_t local_time_timestamp(mgp_local_time *local_time) {
return MgInvoke<int64_t>(mgp_local_time_timestamp, local_time);
}
inline mgp_local_time *local_time_now(mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_now, memory);
}
inline mgp_local_time *local_time_add_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_add_duration, local_time, dur, memory);
}
inline mgp_local_time *local_time_sub_duration(mgp_local_time *local_time, mgp_duration *dur, mgp_memory *memory) {
return MgInvoke<mgp_local_time *>(mgp_local_time_sub_duration, local_time, dur, memory);
}
inline mgp_duration *local_time_diff(mgp_local_time *first, mgp_local_time *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_local_time_diff, first, second, memory);
}
// mgp_local_date_time
inline mgp_local_date_time *local_date_time_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_from_string, string, memory);
}
inline mgp_local_date_time *local_date_time_from_parameters(mgp_local_date_time_parameters *parameters,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_from_parameters, parameters, memory);
}
inline mgp_local_date_time *local_date_time_copy(mgp_local_date_time *local_date_time, mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_copy, local_date_time, memory);
}
inline void local_date_time_destroy(mgp_local_date_time *local_date_time) {
mgp_local_date_time_destroy(local_date_time);
}
inline bool local_date_time_equal(mgp_local_date_time *first, mgp_local_date_time *second) {
return MgInvoke<int>(mgp_local_date_time_equal, first, second);
}
inline int local_date_time_get_year(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_year, local_date_time);
}
inline int local_date_time_get_month(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_month, local_date_time);
}
inline int local_date_time_get_day(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_day, local_date_time);
}
inline int local_date_time_get_hour(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_hour, local_date_time);
}
inline int local_date_time_get_minute(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_minute, local_date_time);
}
inline int local_date_time_get_second(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_second, local_date_time);
}
inline int local_date_time_get_millisecond(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_millisecond, local_date_time);
}
inline int local_date_time_get_microsecond(mgp_local_date_time *local_date_time) {
return MgInvoke<int>(mgp_local_date_time_get_microsecond, local_date_time);
}
inline int64_t local_date_time_timestamp(mgp_local_date_time *local_date_time) {
return MgInvoke<int64_t>(mgp_local_date_time_timestamp, local_date_time);
}
inline mgp_local_date_time *local_date_time_now(mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_now, memory);
}
inline mgp_local_date_time *local_date_time_add_duration(mgp_local_date_time *local_date_time, mgp_duration *dur,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_add_duration, local_date_time, dur, memory);
}
inline mgp_local_date_time *local_date_time_sub_duration(mgp_local_date_time *local_date_time, mgp_duration *dur,
mgp_memory *memory) {
return MgInvoke<mgp_local_date_time *>(mgp_local_date_time_sub_duration, local_date_time, dur, memory);
}
inline mgp_duration *local_date_time_diff(mgp_local_date_time *first, mgp_local_date_time *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_local_date_time_diff, first, second, memory);
}
// mgp_duration
inline mgp_duration *duration_from_string(const char *string, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_string, string, memory);
}
inline mgp_duration *duration_from_parameters(mgp_duration_parameters *parameters, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_parameters, parameters, memory);
}
inline mgp_duration *duration_from_microseconds(int64_t microseconds, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_from_microseconds, microseconds, memory);
}
inline mgp_duration *duration_copy(mgp_duration *duration, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_copy, duration, memory);
}
inline void duration_destroy(mgp_duration *duration) { mgp_duration_destroy(duration); }
inline int64_t duration_get_microseconds(mgp_duration *duration) {
return MgInvoke<int64_t>(mgp_duration_get_microseconds, duration);
}
inline bool duration_equal(mgp_duration *first, mgp_duration *second) {
return MgInvoke<int>(mgp_duration_equal, first, second);
}
inline mgp_duration *duration_neg(mgp_duration *duration, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_neg, duration, memory);
}
inline mgp_duration *duration_add(mgp_duration *first, mgp_duration *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_add, first, second, memory);
}
inline mgp_duration *duration_sub(mgp_duration *first, mgp_duration *second, mgp_memory *memory) {
return MgInvoke<mgp_duration *>(mgp_duration_sub, first, second, memory);
}
// Procedure
inline mgp_proc *module_add_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb) {
return MgInvoke<mgp_proc *>(mgp_module_add_read_procedure, module, name, cb);
}
inline mgp_proc *module_add_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb) {
return MgInvoke<mgp_proc *>(mgp_module_add_write_procedure, module, name, cb);
}
inline mgp_proc *module_add_batch_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup) {
return MgInvoke<mgp_proc *>(mgp_module_add_batch_read_procedure, module, name, cb, initializer, cleanup);
}
inline mgp_proc *module_add_batch_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup) {
return MgInvoke<mgp_proc *>(mgp_module_add_batch_write_procedure, module, name, cb, initializer, cleanup);
}
inline void proc_add_arg(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_arg, proc, name, type);
}
inline void proc_add_opt_arg(mgp_proc *proc, const char *name, mgp_type *type, mgp_value *default_value) {
MgInvokeVoid(mgp_proc_add_opt_arg, proc, name, type, default_value);
}
inline void proc_add_result(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_result, proc, name, type);
}
inline void proc_add_deprecated_result(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_deprecated_result, proc, name, type);
}
inline int must_abort(mgp_graph *graph) { return mgp_must_abort(graph); }
// mgp_result
inline void result_set_error_msg(mgp_result *res, const char *error_msg) {
MgInvokeVoid(mgp_result_set_error_msg, res, error_msg);
}
inline mgp_result_record *result_new_record(mgp_result *res) {
return MgInvoke<mgp_result_record *>(mgp_result_new_record, res);
}
inline void result_record_insert(mgp_result_record *record, const char *field_name, mgp_value *val) {
MgInvokeVoid(mgp_result_record_insert, record, field_name, val);
}
// Function
inline mgp_func *module_add_function(mgp_module *module, const char *name, mgp_func_cb cb) {
return MgInvoke<mgp_func *>(mgp_module_add_function, module, name, cb);
}
inline void func_add_arg(mgp_func *func, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_func_add_arg, func, name, type);
}
inline void func_add_opt_arg(mgp_func *func, const char *name, mgp_type *type, mgp_value *default_value) {
MgInvokeVoid(mgp_func_add_opt_arg, func, name, type, default_value);
}
inline void func_result_set_error_msg(mgp_func_result *res, const char *msg, mgp_memory *memory) {
MgInvokeVoid(mgp_func_result_set_error_msg, res, msg, memory);
}
inline void func_result_set_value(mgp_func_result *res, mgp_value *value, mgp_memory *memory) {
MgInvokeVoid(mgp_func_result_set_value, res, value, memory);
}
} // namespace mgp

343
include/_mgp_mock.py Normal file
View File

@@ -0,0 +1,343 @@
import typing
from enum import Enum
import networkx as nx
NX_LABEL_ATTR = "labels"
NX_TYPE_ATTR = "type"
SOURCE_TYPE_KAFKA = "SOURCE_TYPE_KAFKA"
SOURCE_TYPE_PULSAR = "SOURCE_TYPE_PULSAR"
"""
This module provides helpers for the mock Python API, much like _mgp.py does for mgp.py.
"""
class InvalidArgumentError(Exception):
"""
Signals that some of the arguments have invalid values.
"""
pass
class ImmutableObjectError(Exception):
pass
class LogicErrorError(Exception):
pass
class DeletedObjectError(Exception):
pass
class EdgeConstants(Enum):
I_START = 0
I_END = 1
I_KEY = 2
class Graph:
"""Wrapper around a NetworkX MultiDiGraph instance."""
__slots__ = ("nx", "_highest_vertex_id", "_highest_edge_id", "_valid")
def __init__(self, graph: nx.MultiDiGraph) -> None:
if not isinstance(graph, nx.MultiDiGraph):
raise TypeError(f"Expected 'networkx.classes.multidigraph.MultiDiGraph', got '{type(graph)}'")
self.nx = graph
self._highest_vertex_id = None
self._highest_edge_id = None
self._valid = True
@property
def vertex_ids(self):
return self.nx.nodes
def vertex_is_isolate(self, vertex_id: int) -> bool:
return nx.is_isolate(self.nx, vertex_id)
@property
def vertices(self):
return (Vertex(node_id, self) for node_id in self.nx.nodes)
def has_node(self, node_id):
return self.nx.has_node(node_id)
@property
def edges(self):
return self.nx.edges
def is_valid(self) -> bool:
return self._valid
def get_vertex_by_id(self, vertex_id: int) -> "Vertex":
return Vertex(vertex_id, self)
def invalidate(self):
self._valid = False
def is_immutable(self) -> bool:
return nx.is_frozen(self.nx)
def make_immutable(self):
self.nx = nx.freeze(self.nx)
def _new_vertex_id(self):
if self._highest_vertex_id is None:
self._highest_vertex_id = max(vertex_id for vertex_id in self.nx.nodes)
return self._highest_vertex_id + 1
def _new_edge_id(self):
if self._highest_edge_id is None:
self._highest_edge_id = max(edge[EdgeConstants.I_KEY.value] for edge in self.nx.edges(keys=True))
return self._highest_edge_id + 1
def create_vertex(self) -> "Vertex":
vertex_id = self._new_vertex_id()
self.nx.add_node(vertex_id)
self._highest_vertex_id = vertex_id
return Vertex(vertex_id, self)
def create_edge(self, from_vertex: "Vertex", to_vertex: "Vertex", edge_type: str) -> "Edge":
if from_vertex.is_deleted() or to_vertex.is_deleted():
raise DeletedObjectError("Accessing deleted object.")
edge_id = self._new_edge_id()
from_id = from_vertex.id
to_id = to_vertex.id
self.nx.add_edge(from_id, to_id, key=edge_id, type=edge_type)
self._highest_edge_id = edge_id
return Edge((from_id, to_id, edge_id), self)
def delete_vertex(self, vertex_id: int):
self.nx.remove_node(vertex_id)
def delete_edge(self, from_vertex_id: int, to_vertex_id: int, edge_id: int):
self.nx.remove_edge(from_vertex_id, to_vertex_id, edge_id)
@property
def highest_vertex_id(self) -> int:
if self._highest_vertex_id is None:
self._highest_vertex_id = max(vertex_id for vertex_id in self.nx.nodes) + 1
return self._highest_vertex_id
@property
def highest_edge_id(self) -> int:
if self._highest_edge_id is None:
self._highest_edge_id = max(edge[EdgeConstants.I_KEY.value] for edge in self.nx.edges(keys=True))
return self._highest_edge_id + 1
class Vertex:
"""Represents a graph vertex."""
__slots__ = ("_id", "_graph")
def __init__(self, id: int, graph: Graph) -> None:
if not isinstance(id, int):
raise TypeError(f"Expected 'int', got '{type(id)}'")
if not isinstance(graph, Graph):
raise TypeError(f"Expected '_mgp_mock.Graph', got '{type(graph)}'")
if not graph.nx.has_node(id):
raise IndexError(f"Unable to find vertex with ID {id}.")
self._id = id
self._graph = graph
def is_valid(self) -> bool:
return self._graph.is_valid()
def is_deleted(self) -> bool:
return not self._graph.nx.has_node(self._id) and self._id <= self._graph.highest_vertex_id
@property
def underlying_graph(self) -> Graph:
return self._graph
def underlying_graph_is_mutable(self) -> bool:
return not nx.is_frozen(self._graph.nx)
@property
def labels(self) -> typing.List[int]:
return self._graph.nx.nodes[self._id][NX_LABEL_ATTR].split(":")
def add_label(self, label: str) -> None:
if nx.is_frozen(self._graph.nx):
raise ImmutableObjectError("Cannot modify immutable object.")
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] += f":{label}"
def remove_label(self, label: str) -> None:
if nx.is_frozen(self._graph.nx):
raise ImmutableObjectError("Cannot modify immutable object.")
labels = self._graph.nx.nodes[self._id][NX_LABEL_ATTR]
if labels.startswith(f"{label}:"):
labels = "\n" + labels # pseudo-string starter
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] = labels.replace(f"\n{label}:", "")
elif labels.endswith(f":{label}"):
labels += "\n" # pseudo-string terminator
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] = labels.replace(f":{label}\n", "")
else:
self._graph.nx.nodes[self._id][NX_LABEL_ATTR] = labels.replace(f":{label}:", ":")
@property
def id(self) -> int:
return self._id
@property
def properties(self):
return (
(key, value)
for key, value in self._graph.nx.nodes[self._id].items()
if key not in (NX_LABEL_ATTR, NX_TYPE_ATTR)
)
def get_property(self, property_name: str):
return self._graph.nx.nodes[self._id][property_name]
def set_property(self, property_name: str, value: object):
self._graph.nx.nodes[self._id][property_name] = value
@property
def in_edges(self) -> typing.Iterable["Edge"]:
return [Edge(edge, self._graph) for edge in self._graph.nx.in_edges(self._id, keys=True)]
@property
def out_edges(self) -> typing.Iterable["Edge"]:
return [Edge(edge, self._graph) for edge in self._graph.nx.out_edges(self._id, keys=True)]
class Edge:
"""Represents a graph edge."""
__slots__ = ("_edge", "_graph")
def __init__(self, edge: typing.Tuple[int, int, int], graph: Graph) -> None:
if not isinstance(edge, typing.Tuple):
raise TypeError(f"Expected 'Tuple', got '{type(edge)}'")
if not isinstance(graph, Graph):
raise TypeError(f"Expected '_mgp_mock.Graph', got '{type(graph)}'")
if not graph.nx.has_edge(*edge):
raise IndexError(f"Unable to find edge with ID {edge[EdgeConstants.I_KEY.value]}.")
self._edge = edge
self._graph = graph
def is_valid(self) -> bool:
return self._graph.is_valid()
def is_deleted(self) -> bool:
return (
not self._graph.nx.has_edge(*self._edge)
and self._edge[EdgeConstants.I_KEY.value] <= self._graph.highest_edge_id
)
def underlying_graph_is_mutable(self) -> bool:
return not nx.is_frozen(self._graph.nx)
@property
def id(self) -> int:
return self._edge[EdgeConstants.I_KEY.value]
@property
def edge(self) -> typing.Tuple[int, int, int]:
return self._edge
@property
def start_id(self) -> int:
return self._edge[EdgeConstants.I_START.value]
@property
def end_id(self) -> int:
return self._edge[EdgeConstants.I_END.value]
def get_type_name(self):
return self._graph.nx.get_edge_data(*self._edge)[NX_TYPE_ATTR]
def from_vertex(self) -> Vertex:
return Vertex(self.start_id, self._graph)
def to_vertex(self) -> Vertex:
return Vertex(self.end_id, self._graph)
@property
def properties(self):
return (
(key, value)
for key, value in self._graph.nx.edges[self._edge].items()
if key not in (NX_LABEL_ATTR, NX_TYPE_ATTR)
)
def get_property(self, property_name: str):
return self._graph.nx.edges[self._edge][property_name]
def set_property(self, property_name: str, value: object):
self._graph.nx.edges[self._edge][property_name] = value
class Path:
"""Represents a path comprised of `Vertex` and `Edge` instances."""
__slots__ = ("_vertices", "_edges", "_graph")
__create_key = object()
def __init__(self, create_key, vertex_id: int, graph: Graph) -> None:
assert create_key == Path.__create_key, "Path objects must be created using Path.make_with_start"
self._vertices = [vertex_id]
self._edges = []
self._graph = graph
@classmethod
def make_with_start(cls, vertex: Vertex) -> "Path":
if not isinstance(vertex, Vertex):
raise TypeError(f"Expected 'Vertex', got '{type(vertex)}'")
if not isinstance(vertex.underlying_graph, Graph):
raise TypeError(f"Expected '_mgp_mock.Graph', got '{type(vertex.underlying_graph)}'")
if not vertex.underlying_graph.nx.has_node(vertex._id):
raise IndexError(f"Unable to find vertex with ID {vertex._id}.")
return Path(cls.__create_key, vertex._id, vertex.underlying_graph)
def is_valid(self) -> bool:
return self._graph.is_valid()
def underlying_graph_is_mutable(self) -> bool:
return not nx.is_frozen(self._graph.nx)
def expand(self, edge: Edge):
if edge.start_id != self._vertices[-1]:
raise LogicErrorError("Logic error.")
self._vertices.append(edge.end_id)
self._edges.append((edge.start_id, edge.end_id, edge.id))
def vertex_at(self, index: int) -> Vertex:
return Vertex(self._vertices[index], self._graph)
def edge_at(self, index: int) -> Edge:
return Edge(self._edges[index], self._graph)
def size(self) -> int:
return len(self._edges)

108
include/mg_exceptions.hpp Normal file
View File

@@ -0,0 +1,108 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <exception>
#include <iostream>
#include <sstream>
#include <string>
namespace mg_exception {
// Instead of writing this utility function, we could have used `fmt::format`, but that's not an ideal option here
// because that would introduce dependency that would be propagated to the client code (if exceptions here would be
// used). Since the functionality here is not complex + the code is not on a critical path, we opted for a pure C++
// solution.
template <typename FirstArg, typename... Args>
std::string StringSerialize(FirstArg &&firstArg, Args &&...args) {
std::stringstream stream;
stream << firstArg;
((stream << " " << args), ...);
return stream.str();
}
struct UnknownException : public std::exception {
const char *what() const noexcept override { return "Unknown exception!"; }
};
struct NotEnoughMemoryException : public std::exception {
NotEnoughMemoryException()
: message_{
StringSerialize("Not enough memory! For more details please visit", "https://memgr.ph/memory-control")} {}
const char *what() const noexcept override { return message_.c_str(); }
private:
std::string message_;
};
struct AllocationException : public std::exception {
AllocationException()
: message_{StringSerialize("Could not allocate memory. For more details please visit",
"https://memgr.ph/memory-control")} {}
const char *what() const noexcept override { return message_.c_str(); }
private:
std::string message_;
};
struct InsufficientBufferException : public std::exception {
const char *what() const noexcept override { return "Buffer is not sufficient to process procedure!"; }
};
struct OutOfRangeException : public std::exception {
const char *what() const noexcept override { return "Index out of range!"; }
};
struct LogicException : public std::exception {
const char *what() const noexcept override { return "Logic exception, check the procedure signature!"; }
};
struct DeletedObjectException : public std::exception {
const char *what() const noexcept override { return "Object is deleted!"; }
};
struct InvalidArgumentException : public std::exception {
const char *what() const noexcept override { return "Invalid argument!"; }
};
struct InvalidIDException : public std::exception {
InvalidIDException() : message_{"Invalid ID!"} {}
explicit InvalidIDException(std::uint64_t identifier) : message_{StringSerialize("Invalid ID =", identifier)} {}
const char *what() const noexcept override { return message_.c_str(); }
private:
std::string message_;
};
struct KeyAlreadyExistsException : public std::exception {
KeyAlreadyExistsException() : message_{"Key you are trying to set already exists!"} {}
explicit KeyAlreadyExistsException(const std::string &key)
: message_{StringSerialize("Key you are trying to set already exists! KEY = ", key)} {}
const char *what() const noexcept override { return message_.c_str(); }
private:
std::string message_;
};
struct ImmutableObjectException : public std::exception {
const char *what() const noexcept override { return "Object you are trying to change is immutable!"; }
};
struct ValueConversionException : public std::exception {
const char *what() const noexcept override { return "Error in value conversion!"; }
};
struct SerializationException : public std::exception {
const char *what() const noexcept override { return "Error in serialization!"; }
};
} // namespace mg_exception

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -37,12 +37,19 @@ extern "C" {
/// All functions return an error code that can be used to figure out whether the API call was successful or not. In /// All functions return an error code that can be used to figure out whether the API call was successful or not. In
/// case of failure, the specific error code can be used to identify the reason of the failure. /// case of failure, the specific error code can be used to identify the reason of the failure.
MGP_ENUM_CLASS MGP_NODISCARD mgp_error{ MGP_ENUM_CLASS MGP_NODISCARD mgp_error{
MGP_ERROR_NO_ERROR, MGP_ERROR_UNKNOWN_ERROR, MGP_ERROR_NO_ERROR,
MGP_ERROR_UNABLE_TO_ALLOCATE, MGP_ERROR_INSUFFICIENT_BUFFER, MGP_ERROR_UNKNOWN_ERROR,
MGP_ERROR_OUT_OF_RANGE, MGP_ERROR_LOGIC_ERROR, MGP_ERROR_UNABLE_TO_ALLOCATE,
MGP_ERROR_DELETED_OBJECT, MGP_ERROR_INVALID_ARGUMENT, MGP_ERROR_INSUFFICIENT_BUFFER,
MGP_ERROR_KEY_ALREADY_EXISTS, MGP_ERROR_IMMUTABLE_OBJECT, MGP_ERROR_OUT_OF_RANGE,
MGP_ERROR_VALUE_CONVERSION, MGP_ERROR_SERIALIZATION_ERROR, MGP_ERROR_LOGIC_ERROR,
MGP_ERROR_DELETED_OBJECT,
MGP_ERROR_INVALID_ARGUMENT,
MGP_ERROR_KEY_ALREADY_EXISTS,
MGP_ERROR_IMMUTABLE_OBJECT,
MGP_ERROR_VALUE_CONVERSION,
MGP_ERROR_SERIALIZATION_ERROR,
MGP_ERROR_AUTHORIZATION_ERROR,
}; };
///@} ///@}
@@ -164,6 +171,8 @@ enum mgp_value_type {
MGP_VALUE_TYPE_DURATION, MGP_VALUE_TYPE_DURATION,
}; };
enum mgp_error mgp_value_copy(struct mgp_value *val, struct mgp_memory *memory, struct mgp_value **result);
/// Free the memory used by the given mgp_value instance. /// Free the memory used by the given mgp_value instance.
void mgp_value_destroy(struct mgp_value *val); void mgp_value_destroy(struct mgp_value *val);
@@ -399,6 +408,8 @@ enum mgp_error mgp_value_get_duration(struct mgp_value *val, struct mgp_duration
/// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_list. /// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_list.
enum mgp_error mgp_list_make_empty(size_t capacity, struct mgp_memory *memory, struct mgp_list **result); enum mgp_error mgp_list_make_empty(size_t capacity, struct mgp_memory *memory, struct mgp_list **result);
enum mgp_error mgp_list_copy(struct mgp_list *list, struct mgp_memory *memory, struct mgp_list **result);
/// Free the memory used by the given mgp_list and contained elements. /// Free the memory used by the given mgp_list and contained elements.
void mgp_list_destroy(struct mgp_list *list); void mgp_list_destroy(struct mgp_list *list);
@@ -437,6 +448,8 @@ enum mgp_error mgp_list_at(struct mgp_list *list, size_t index, struct mgp_value
/// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_map. /// mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate a mgp_map.
enum mgp_error mgp_map_make_empty(struct mgp_memory *memory, struct mgp_map **result); enum mgp_error mgp_map_make_empty(struct mgp_memory *memory, struct mgp_map **result);
enum mgp_error mgp_map_copy(struct mgp_map *map, struct mgp_memory *memory, struct mgp_map **result);
/// Free the memory used by the given mgp_map and contained items. /// Free the memory used by the given mgp_map and contained items.
void mgp_map_destroy(struct mgp_map *map); void mgp_map_destroy(struct mgp_map *map);
@@ -449,6 +462,18 @@ void mgp_map_destroy(struct mgp_map *map);
/// Return mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS if a previous mapping already exists. /// Return mgp_error::MGP_ERROR_KEY_ALREADY_EXISTS if a previous mapping already exists.
enum mgp_error mgp_map_insert(struct mgp_map *map, const char *key, struct mgp_value *value); enum mgp_error mgp_map_insert(struct mgp_map *map, const char *key, struct mgp_value *value);
/// Insert a mapping from a NULL terminated character string to a value.
/// If a mapping with the same key already exists, it is replaced.
/// In case of update, both the string and the value are copied into the map.
/// Therefore, the map does not take ownership of the original key nor value, so
/// you still need to free their memory explicitly.
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE is returned if unable to allocate for insertion.
enum mgp_error mgp_map_update(struct mgp_map *map, const char *key, struct mgp_value *value);
// Erase a mapping by key.
// If the key doesn't exist in the map nothing happens
enum mgp_error mgp_map_erase(struct mgp_map *map, const char *key);
/// Get the number of items stored in mgp_map. /// Get the number of items stored in mgp_map.
/// Current implementation always returns without errors. /// Current implementation always returns without errors.
enum mgp_error mgp_map_size(struct mgp_map *map, size_t *result); enum mgp_error mgp_map_size(struct mgp_map *map, size_t *result);
@@ -1292,6 +1317,12 @@ struct mgp_proc;
/// Describes a Memgraph magic function. /// Describes a Memgraph magic function.
struct mgp_func; struct mgp_func;
/// All available log levels that can be used in mgp_log function
MGP_ENUM_CLASS mgp_log_level{
MGP_LOG_LEVEL_TRACE, MGP_LOG_LEVEL_DEBUG, MGP_LOG_LEVEL_INFO,
MGP_LOG_LEVEL_WARN, MGP_LOG_LEVEL_ERROR, MGP_LOG_LEVEL_CRITICAL,
};
/// Entry-point for a query module read procedure, invoked through openCypher. /// Entry-point for a query module read procedure, invoked through openCypher.
/// ///
/// Passed in arguments will not live longer than the callback's execution. /// Passed in arguments will not live longer than the callback's execution.
@@ -1299,6 +1330,13 @@ struct mgp_func;
/// to allocate global resources. /// to allocate global resources.
typedef void (*mgp_proc_cb)(struct mgp_list *, struct mgp_graph *, struct mgp_result *, struct mgp_memory *); typedef void (*mgp_proc_cb)(struct mgp_list *, struct mgp_graph *, struct mgp_result *, struct mgp_memory *);
/// Cleanup for a query module read procedure. Can't be invoked through OpenCypher. Cleans batched stream.
typedef void (*mgp_proc_cleanup)();
/// Initializer for a query module batched read procedure. Can't be invoked through OpenCypher. Initializes batched
/// stream.
typedef void (*mgp_proc_initializer)(struct mgp_list *, struct mgp_graph *, struct mgp_memory *);
/// Register a read-only procedure to a module. /// Register a read-only procedure to a module.
/// ///
/// The `name` must be a sequence of digits, underscores, lowercase and /// The `name` must be a sequence of digits, underscores, lowercase and
@@ -1323,6 +1361,30 @@ enum mgp_error mgp_module_add_read_procedure(struct mgp_module *module, const ch
enum mgp_error mgp_module_add_write_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb, enum mgp_error mgp_module_add_write_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb,
struct mgp_proc **result); struct mgp_proc **result);
/// Register a readable batched procedure to a module.
///
/// The `name` must be a valid identifier, following the same rules as the
/// procedure`name` in mgp_module_add_read_procedure.
///
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if unable to allocate memory for mgp_proc.
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid procedure name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a procedure with the same name was already registered.
enum mgp_error mgp_module_add_batch_read_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup,
struct mgp_proc **result);
/// Register a writeable batched procedure to a module.
///
/// The `name` must be a valid identifier, following the same rules as the
/// procedure`name` in mgp_module_add_read_procedure.
///
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if unable to allocate memory for mgp_proc.
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid procedure name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a procedure with the same name was already registered.
enum mgp_error mgp_module_add_batch_write_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup,
struct mgp_proc **result);
/// Add a required argument to a procedure. /// Add a required argument to a procedure.
/// ///
/// The order of adding arguments will correspond to the order the procedure /// The order of adding arguments will correspond to the order the procedure
@@ -1386,6 +1448,9 @@ enum mgp_error mgp_proc_add_result(struct mgp_proc *proc, const char *name, stru
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid result name. /// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid result name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a result field with the same name was already added. /// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a result field with the same name was already added.
enum mgp_error mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char *name, struct mgp_type *type); enum mgp_error mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char *name, struct mgp_type *type);
/// Log a message on a certain level.
enum mgp_error mgp_log(enum mgp_log_level log_level, const char *output);
///@} ///@}
/// @name Execution /// @name Execution
@@ -1395,7 +1460,10 @@ enum mgp_error mgp_proc_add_deprecated_result(struct mgp_proc *proc, const char
/// @{ /// @{
/// Return non-zero if the currently executing procedure should abort as soon as /// Return non-zero if the currently executing procedure should abort as soon as
/// possible. /// possible. If non-zero the reasons are:
/// (1) The transaction was requested to be terminated
/// (2) The server is gracefully shutting down
/// (3) The transaction has hit its timeout threshold
/// ///
/// Procedures which perform heavyweight processing run the risk of running too /// Procedures which perform heavyweight processing run the risk of running too
/// long and going over the query execution time limit. To prevent this, such /// long and going over the query execution time limit. To prevent this, such
@@ -1512,6 +1580,10 @@ enum mgp_error mgp_module_add_transformation(struct mgp_module *module, const ch
/// ///
///@{ ///@{
/// State of the database that is exposed to magic functions. Currently it is unused, but it enables extending the
/// functionalities of magic functions in future without breaking the API.
struct mgp_func_context;
/// Add a required argument to a function. /// Add a required argument to a function.
/// ///
/// The order of the added arguments corresponds to the signature of the openCypher function. /// The order of the added arguments corresponds to the signature of the openCypher function.

4057
include/mgp.hpp Normal file

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

1655
include/mgp_mock.py Normal file

File diff suppressed because it is too large Load Diff

96
init
View File

@@ -5,13 +5,15 @@ cd "$DIR"
source "$DIR/environment/util.sh" source "$DIR/environment/util.sh"
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
function print_help () { function print_help () {
echo "Usage: $0 [OPTION]" echo "Usage: $0 [OPTION]"
echo -e "Check for missing packages and setup the project.\n" echo -e "Check for missing packages and setup the project.\n"
echo "Optional arguments:" echo "Optional arguments:"
echo -e " -h\tdisplay this help and exit" echo -e " -h\tdisplay this help and exit"
echo -e " --without-libs-setup\tskip the step for setting up libs" echo -e " --without-libs-setup\tskip the step for setting up libs"
echo -e " --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)"
} }
function setup_virtualenv () { function setup_virtualenv () {
@@ -32,7 +34,6 @@ function setup_virtualenv () {
popd > /dev/null popd > /dev/null
} }
wsl_quicklisp_proxy=""
setup_libs=true setup_libs=true
if [[ $# -eq 1 && "$1" == "-h" ]]; then if [[ $# -eq 1 && "$1" == "-h" ]]; then
print_help print_help
@@ -40,16 +41,6 @@ if [[ $# -eq 1 && "$1" == "-h" ]]; then
else else
while(($#)); do while(($#)); do
case "$1" in case "$1" in
--wsl-quicklisp-proxy)
shift
if [[ $# -eq 0 ]]; then
echo "Missing proxy URL"
print_help
exit 1
fi
wsl_quicklisp_proxy=":proxy \"http://$1/\""
shift
;;
--without-libs-setup) --without-libs-setup)
shift shift
setup_libs=false setup_libs=false
@@ -64,8 +55,6 @@ else
done done
fi fi
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
else else
@@ -78,37 +67,20 @@ echo "All packages are in-place..."
# create a default build directory # create a default build directory
mkdir -p ./build mkdir -p ./build
# quicklisp package manager for Common Lisp
quicklisp_install_dir="$HOME/quicklisp"
if [[ -v QUICKLISP_HOME ]]; then
quicklisp_install_dir="${QUICKLISP_HOME}"
fi
if [[ ! -f "${quicklisp_install_dir}/setup.lisp" ]]; then
wget -nv https://beta.quicklisp.org/quicklisp.lisp -O quicklisp.lisp || exit 1
echo \
"
(load \"${DIR}/quicklisp.lisp\")
(quicklisp-quickstart:install $wsl_quicklisp_proxy :path \"${quicklisp_install_dir}\")
" | sbcl --script || exit 1
rm -rf quicklisp.lisp || exit 1
fi
ln -Tfs "$DIR/src/lisp" "${quicklisp_install_dir}/local-projects/lcp"
# Install LCP dependencies
# TODO: We should at some point cache or have a mirror of packages we use.
# TODO: move the installation of LCP's dependencies into ./setup.sh
echo \
"
(load \"${quicklisp_install_dir}/setup.lisp\")
(ql:quickload '(:lcp :lcp/test) :silent t)
" | sbcl --script
if [[ "$setup_libs" == "true" ]]; then if [[ "$setup_libs" == "true" ]]; then
# Setup libs (download). # Setup libs (download).
cd libs cd libs
./cleanup.sh ./cleanup.sh
./setup.sh ./setup.sh
cd .. cd ..
fi
# Fix for centos 7 during release
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then
if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then
python3 -m pip uninstall -y virtualenv
fi
python3 -m pip install virtualenv
fi fi
# setup gql_behave dependencies # setup gql_behave dependencies
@@ -121,6 +93,10 @@ setup_virtualenv tests/stress
setup_virtualenv tests/integration/ldap setup_virtualenv tests/integration/ldap
# Setup tests dependencies. # Setup tests dependencies.
# NOTE: This is commented out because of the build order (at the time of
# execution mgclient is not built yet) which makes this setup to fail. mgclient
# is built during the make phase. The tests/setup.sh is called under GHA CI
# jobs.
# cd tests # cd tests
# ./setup.sh # ./setup.sh
# cd .. # cd ..
@@ -130,12 +106,28 @@ setup_virtualenv tests/integration/ldap
echo "Done installing dependencies for Memgraph" echo "Done installing dependencies for Memgraph"
echo "Linking git hooks" echo "Linking git hooks OR skip if .git folder is not there"
for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do if [ -d "$DIR/.git" ]; then
ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook" for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
echo "Added $hook hook" ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook"
done; echo "Added $hook hook"
done;
else
echo "WARNING: .git folder not present, skip adding hooks"
fi
# Install precommit hook # Install precommit hook except on old operating systems because we don't
python3 -m pip install pre-commit # develop on them -> pre-commit hook not required -> we can use latest
python3 -m pre_commit install # packages.
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ] && [ "${DISTRO}" != "amzn-2" ]; then
python3 -m pip install pre-commit
python3 -m pre_commit install
# Install py format tools for usage during the development.
echo "Install black formatter"
python3 -m pip install black==23.1.*
echo "Install isort"
python3 -m pip install isort==5.12.*
fi
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

2
libs/.gitignore vendored
View File

@@ -5,3 +5,5 @@
!CMakeLists.txt !CMakeLists.txt
!__main.cpp !__main.cpp
!pulsar.patch !pulsar.patch
!antlr4.10.1.patch
!rocksdb8.1.1.patch

View File

@@ -12,6 +12,7 @@ find_package(Boost 1.78 REQUIRED)
find_package(BZip2 1.0.6 REQUIRED) find_package(BZip2 1.0.6 REQUIRED)
find_package(Threads REQUIRED) find_package(Threads REQUIRED)
set(GFLAGS_NOTHREADS OFF) set(GFLAGS_NOTHREADS OFF)
# NOTE: config/generate.py depends on the gflags help XML format.
find_package(gflags REQUIRED) find_package(gflags REQUIRED)
find_package(fmt 8.0.1) find_package(fmt 8.0.1)
find_package(Jemalloc REQUIRED) find_package(Jemalloc REQUIRED)
@@ -28,6 +29,7 @@ function(import_header_library name include_dir)
set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
"Path to ${name} include directory" FORCE) "Path to ${name} include directory" FORCE)
mark_as_advanced(${_upper_name}_INCLUDE_DIR) mark_as_advanced(${_upper_name}_INCLUDE_DIR)
add_library(lib::${name} ALIAS ${name})
endfunction(import_header_library) endfunction(import_header_library)
function(import_library name type location include_dir) function(import_library name type location include_dir)
@@ -106,6 +108,7 @@ import_external_library(antlr4 STATIC
-DWITH_LIBCXX=OFF # because of debian bug -DWITH_LIBCXX=OFF # because of debian bug
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
-DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD=20
-DANTLR_BUILD_CPP_TESTS=OFF
BUILD_COMMAND $(MAKE) antlr4_static BUILD_COMMAND $(MAKE) antlr4_static
INSTALL_COMMAND $(MAKE) install) INSTALL_COMMAND $(MAKE) install)
@@ -255,3 +258,6 @@ import_external_library(librdtsc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include ${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include
CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS} CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS}
BUILD_COMMAND $(MAKE) rdtsc) BUILD_COMMAND $(MAKE) rdtsc)
# setup ctre
import_header_library(ctre ${CMAKE_CURRENT_SOURCE_DIR})

13
libs/antlr4.10.1.patch Normal file
View File

@@ -0,0 +1,13 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index baf46cac9..2e7756de8 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -134,7 +134,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
EXPORT antlr4-targets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

View File

@@ -1,43 +0,0 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index a8503bb..11362cf 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
ExternalProject_Add(
utfcpp
- GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
- GIT_TAG "v3.1.1"
+ GIT_REPOSITORY "https://github.com/nemtrif/utfcpp"
+ GIT_TAG "v3.2.1"
SOURCE_DIR ${UTFCPP_DIR}
UPDATE_DISCONNECTED 1
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
DESTINATION lib
EXPORT antlr4-targets)
install(TARGETS antlr4_static
diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
index 468db98..65a473b 100644
--- a/runtime/Cpp/runtime/src/support/Any.h
+++ b/runtime/Cpp/runtime/src/support/Any.h
@@ -122,12 +122,12 @@ private:
}
private:
- template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return new Derived<T>(value);
}
- template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return nullptr;
}

13
libs/rocksdb8.1.1.patch Normal file
View File

@@ -0,0 +1,13 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 598c728..816c705 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1242,7 +1242,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(ROCKSDB_BUILD_SHARED)
install(
- TARGETS ${ROCKSDB_SHARED_LIB}
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
EXPORT RocksDBTargets
COMPONENT runtime
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"

View File

@@ -71,8 +71,8 @@ file_get_try_double () {
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
filename="$(basename "$secondary_url")" filename="$(basename "$secondary_url")"
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1 # Redirect primary/cache to /dev/null to make it less confusing for a new contributor because only CI has access to the cache.
echo "" wget -nv "$primary_url" -O "$filename" >/dev/null 2>&1 || wget -nv "$secondary_url" -O "$filename" || exit 1
} }
repo_clone_try_double () { repo_clone_try_double () {
@@ -86,8 +86,8 @@ repo_clone_try_double () {
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
clone "$primary_url" "$folder_name" "$ref" "$shallow" || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1 # Redirect primary/cache to /dev/null to make it less confusing for a new contributor because only CI has access to the cache.
echo "" clone "$primary_url" "$folder_name" "$ref" "$shallow" >/dev/null 2>&1 || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1
} }
# List all dependencies. # List all dependencies.
@@ -105,7 +105,7 @@ repo_clone_try_double () {
# Download from primary_urls might fail because the cache is not installed. # Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=( declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git" ["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar" ["antlr4-generator"]="http://$local_cache_host/file/antlr-4.10.1-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git" ["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git" ["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git" ["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
@@ -117,11 +117,12 @@ declare -A primary_urls=(
["mgconsole"]="http://$local_cache_host/git/mgconsole.git" ["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
["spdlog"]="http://$local_cache_host/git/spdlog" ["spdlog"]="http://$local_cache_host/git/spdlog"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp" ["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz" ["neo4j"]="http://$local_cache_host/file/neo4j-community-5.6.0-unix.tar.gz"
["librdkafka"]="http://$local_cache_host/git/librdkafka.git" ["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
["protobuf"]="http://$local_cache_host/git/protobuf.git" ["protobuf"]="http://$local_cache_host/git/protobuf.git"
["pulsar"]="http://$local_cache_host/git/pulsar.git" ["pulsar"]="http://$local_cache_host/git/pulsar.git"
["librdtsc"]="http://$local_cache_host/git/librdtsc.git" ["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
) )
# The goal of secondary urls is to have links to the "source of truth" of # The goal of secondary urls is to have links to the "source of truth" of
@@ -130,7 +131,7 @@ declare -A primary_urls=(
# should fail. # should fail.
declare -A secondary_urls=( declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git" ["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar" ["antlr4-generator"]="https://www.antlr.org/download/antlr-4.10.1-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git" ["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git" ["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git" ["gbenchmark"]="https://github.com/google/benchmark.git"
@@ -142,20 +143,21 @@ declare -A secondary_urls=(
["mgconsole"]="http://github.com/memgraph/mgconsole.git" ["mgconsole"]="http://github.com/memgraph/mgconsole.git"
["spdlog"]="https://github.com/gabime/spdlog" ["spdlog"]="https://github.com/gabime/spdlog"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp" ["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz" ["neo4j"]="https://dist.neo4j.org/neo4j-community-5.6.0-unix.tar.gz"
["librdkafka"]="https://github.com/edenhill/librdkafka.git" ["librdkafka"]="https://github.com/edenhill/librdkafka.git"
["protobuf"]="https://github.com/protocolbuffers/protobuf.git" ["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
["pulsar"]="https://github.com/apache/pulsar.git" ["pulsar"]="https://github.com/apache/pulsar.git"
["librdtsc"]="https://github.com/gabrieleara/librdtsc.git" ["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
["ctre"]="https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
) )
# antlr # antlr
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}" file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="4.9.2" # v4.9.2 antlr4_tag="4.10.1" # v4.10.1
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
pushd antlr4 pushd antlr4
git apply ../antlr4.patch git apply ../antlr4.10.1.patch
popd popd
# cppitertools v2.0 2019-12-23 # cppitertools v2.0 2019-12-23
@@ -180,9 +182,9 @@ repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}
# neo4j # neo4j
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}" file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
tar -xzf neo4j-community-3.2.3-unix.tar.gz tar -xzf neo4j-community-5.6.0-unix.tar.gz
mv neo4j-community-3.2.3 neo4j mv neo4j-community-5.6.0 neo4j
rm neo4j-community-3.2.3-unix.tar.gz rm neo4j-community-5.6.0-unix.tar.gz
# nlohmann json # nlohmann json
# We wget header instead of cloning repo since repo is huge (lots of test data). # We wget header instead of cloning repo since repo is huge (lots of test data).
@@ -192,14 +194,14 @@ cd json
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}" file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd .. cd ..
rocksdb_tag="v6.14.6" # (2020-10-14) rocksdb_tag="v8.1.1" # (2023-04-21)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
pushd rocksdb pushd rocksdb
git apply ../rocksdb.patch git apply ../rocksdb8.1.1.patch
popd popd
# mgclient # mgclient
mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08) mgclient_tag="v1.4.0" # (2022-06-14)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag" repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
@@ -208,7 +210,7 @@ pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag" repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
# mgconsole # mgconsole
mgconsole_tag="v1.1.0" # (2021-10-07) mgconsole_tag="v1.4.0" # (2023-05-21)
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
spdlog_tag="v1.9.2" # (2021-08-12) spdlog_tag="v1.9.2" # (2021-08-12)
@@ -238,3 +240,9 @@ repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}"
pushd librdtsc pushd librdtsc
git apply ../librdtsc.patch git apply ../librdtsc.patch
popd popd
#ctre
mkdir -p ctre
cd ctre
file_get_try_double "${primary_urls[ctre]}" "${secondary_urls[ctre]}"
cd ..

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution 3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to which competes (or might reasonably be expected to
compete) with the Licensed Work. compete) with the Licensed Work.
CHANGE DATE: 2026-27-04 CHANGE DATE: 2027-02-08
CHANGE LICENSE: Apache License, Version 2.0 CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal. For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -2,8 +2,8 @@ MEMGRAPH
ENTERPRISE LICENCE AGREEMENT ENTERPRISE LICENCE AGREEMENT
Memgraph Limited is registered in England under registration 10195084 and has its registered office at Suite 4, Memgraph Limited is registered in England under registration 10195084 and has its registered office at 90a High Street,
Ironstone House, Ironstone Way, Brixworth, Northampton, NN6 9UD (Memgraph). Hertfordshire, Berkhamsted, HP4 2BL United Kingdom ("Memgraph").
Memgraph agrees to license and/or grant you (the “Customer”) access to the Software ( as defined below) and provide Memgraph agrees to license and/or grant you (the “Customer”) access to the Software ( as defined below) and provide

202
licenses/third-party/ldbc/LICENSE vendored Normal file
View File

@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

12
pyproject.toml Normal file
View File

@@ -0,0 +1,12 @@
[tool.black]
line-length = 120
include = '\.pyi?$'
extend-exclude = '''
/(
| .git
| .__pycache__
| build
| libs
| .cache
)/
'''

View File

@@ -8,29 +8,40 @@ disallow_in_source_build()
# Everything that is installed here, should be under the "query_modules" component. # Everything that is installed here, should be under the "query_modules" component.
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules") set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "query_modules")
add_library(example SHARED example.c)
target_include_directories(example PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example PRIVATE -Wall)
# Strip the library in release build.
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example POST_BUILD
COMMAND strip -s $<TARGET_FILE:example>
COMMENT "Stripping symbols and sections from example module")
endif()
install(PROGRAMS $<TARGET_FILE:example> add_library(example_c SHARED example.c)
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_c PRIVATE -Wall)
# Strip C example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_c POST_BUILD
COMMAND strip -s $<TARGET_FILE:example_c>
COMMENT "Stripping symbols and sections from the C example module")
endif()
install(PROGRAMS $<TARGET_FILE:example_c>
DESTINATION lib/memgraph/query_modules DESTINATION lib/memgraph/query_modules
RENAME example.so) RENAME example_c.so)
# Also install the source of the example, so user can read it. # Also install the source of the example, so user can read it.
install(FILES example.c DESTINATION lib/memgraph/query_modules/src) install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
# Install the Python example add_library(example_cpp SHARED example.cpp)
install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py) target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_cpp PRIVATE -Wall)
# Strip C++ example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_cpp POST_BUILD
COMMAND strip -s $<TARGET_FILE:example_cpp>
COMMENT "Stripping symbols and sections from the C++ example module")
endif()
install(PROGRAMS $<TARGET_FILE:example_cpp>
DESTINATION lib/memgraph/query_modules
RENAME example_cpp.so)
# Also install the source of the example, so user can read it.
install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
# Install the Python modules # Install the Python example and modules
install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules) install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules) install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
install(FILES nxalg.py DESTINATION lib/memgraph/query_modules) install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)

103
query_modules/example.cpp Normal file
View File

@@ -0,0 +1,103 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <mgp.hpp>
void ProcImpl(std::vector<mgp::Value> arguments, mgp::Graph graph, mgp::RecordFactory record_factory) {
auto record = record_factory.NewRecord();
record.Insert("out", true);
}
void SampleReadProc(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
try {
mgp::memory = memory;
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
ProcImpl(arguments, mgp::Graph(memgraph_graph), mgp::RecordFactory(result));
} catch (const std::exception &e) {
mgp::result_set_error_msg(result, e.what());
return;
}
}
void AddXNodes(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::memory = memory;
auto graph = mgp::Graph(memgraph_graph);
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
for (int i = 0; i < arguments[0].ValueInt(); i++) {
graph.CreateNode();
}
}
void Multiply(mgp_list *args, mgp_func_context *ctx, mgp_func_result *res, mgp_memory *memory) {
mgp::memory = memory;
std::vector<mgp::Value> arguments;
for (size_t i = 0; i < mgp::list_size(args); i++) {
auto arg = mgp::Value(mgp::list_at(args, i));
arguments.push_back(arg);
}
auto result = mgp::Result(res);
auto first = arguments[0].ValueInt();
auto second = arguments[1].ValueInt();
result.SetValue(first * second);
}
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
try {
mgp::memory = memory;
AddProcedure(SampleReadProc, "return_true", mgp::ProcedureType::Read,
{mgp::Parameter("param_1", mgp::Type::Int), mgp::Parameter("param_2", mgp::Type::Double, 2.3)},
{mgp::Return("out", mgp::Type::Bool)}, module, memory);
} catch (const std::exception &e) {
return 1;
}
try {
mgp::memory = memory;
mgp::AddProcedure(AddXNodes, "add_x_nodes", mgp::ProcedureType::Write, {mgp::Parameter("param_1", mgp::Type::Int)},
{}, module, memory);
} catch (const std::exception &e) {
return 1;
}
try {
mgp::memory = memory;
mgp::AddFunction(Multiply, "multiply",
{mgp::Parameter("int", mgp::Type::Int), mgp::Parameter("int", mgp::Type::Int, (int64_t)3)}, module,
memory);
} catch (const std::exception &e) {
return 1;
}
return 0;
}
extern "C" int mgp_shutdown_module() { return 0; }

View File

@@ -7,11 +7,13 @@ import copy
@mgp.read_proc @mgp.read_proc
def procedure( def procedure(context: mgp.ProcCtx,
context: mgp.ProcCtx, required_arg: mgp.Nullable[mgp.Any],
required_arg: mgp.Nullable[mgp.Any], optional_arg: mgp.Nullable[mgp.Any] = None
optional_arg: mgp.Nullable[mgp.Any] = None, ) -> mgp.Record(args=list,
) -> mgp.Record(args=list, vertex_count=int, avg_degree=mgp.Number, props=mgp.Nullable[mgp.Map]): vertex_count=int,
avg_degree=mgp.Number,
props=mgp.Nullable[mgp.Map]):
""" """
This example procedure returns 4 fields. This example procedure returns 4 fields.
@@ -35,7 +37,7 @@ def procedure(
if isinstance(required_arg, (mgp.Edge, mgp.Vertex)): if isinstance(required_arg, (mgp.Edge, mgp.Vertex)):
props = dict(required_arg.properties.items()) props = dict(required_arg.properties.items())
elif isinstance(required_arg, mgp.Path): elif isinstance(required_arg, mgp.Path):
(start_vertex,) = required_arg.vertices start_vertex, = required_arg.vertices
props = dict(start_vertex.properties.items()) props = dict(start_vertex.properties.items())
# Count the vertices and edges in the database; this may take a while. # Count the vertices and edges in the database; this may take a while.
vertex_count = 0 vertex_count = 0
@@ -49,13 +51,15 @@ def procedure(
# Copy the received arguments to make it equivalent to the C example. # Copy the received arguments to make it equivalent to the C example.
args_copy = [copy.deepcopy(required_arg), copy.deepcopy(optional_arg)] args_copy = [copy.deepcopy(required_arg), copy.deepcopy(optional_arg)]
# Multiple rows can be produced by returning an iterable of mgp.Record. # Multiple rows can be produced by returning an iterable of mgp.Record.
return mgp.Record(args=args_copy, vertex_count=vertex_count, avg_degree=avg_degree, props=props) return mgp.Record(args=args_copy, vertex_count=vertex_count,
avg_degree=avg_degree, props=props)
@mgp.write_proc @mgp.write_proc
def write_procedure( def write_procedure(context: mgp.ProcCtx,
context: mgp.ProcCtx, property_name: str, property_value: mgp.Nullable[mgp.Any] property_name: str,
) -> mgp.Record(created_vertex=mgp.Vertex): property_value: mgp.Nullable[mgp.Any]
) -> mgp.Record(created_vertex=mgp.Vertex):
""" """
This example procedure creates a new vertex with the specified property This example procedure creates a new vertex with the specified property
and connects it to all existing vertex which has the same property with and connects it to all existing vertex which has the same property with

View File

@@ -4,17 +4,15 @@ from collections import OrderedDict
from itertools import chain, repeat from itertools import chain, repeat
from inspect import cleandoc from inspect import cleandoc
from typing import List, Tuple from typing import List, Tuple
try: try:
import networkx as nx import networkx as nx
except ImportError as import_error: except ImportError as import_error:
sys.stderr.write( sys.stderr.write((
( '\n'
"\n" 'NOTE: Please install networkx to be able to use graph_analyzer '
"NOTE: Please install networkx to be able to use graph_analyzer " 'module. Using Python:\n'
"module. Using Python:\n" + sys.version + "\n" + sys.version +
) '\n'))
)
raise import_error raise import_error
# Imported last because it also depends on networkx. # Imported last because it also depends on networkx.
from mgp_networkx import MemgraphMultiDiGraph # noqa E402 from mgp_networkx import MemgraphMultiDiGraph # noqa E402
@@ -25,14 +23,16 @@ _MAX_LIST_SIZE = 10
@mgp.read_proc @mgp.read_proc
def help() -> mgp.Record(name=str, value=str): def help() -> mgp.Record(name=str, value=str):
"""Shows manual page for graph_analyzer.""" '''Shows manual page for graph_analyzer.'''
records = [] records = []
def make_records(name, doc): def make_records(name, doc):
return (mgp.Record(name=n, value=v) for n, v in zip(chain([name], repeat("")), cleandoc(doc).splitlines())) return (mgp.Record(name=n, value=v) for n, v in
zip(chain([name], repeat('')), cleandoc(doc).splitlines()))
for func in (help, analyze, analyze_subgraph): for func in (help, analyze, analyze_subgraph):
records.extend(make_records("Procedure '{}'".format(func.__name__), func.__doc__)) records.extend(make_records("Procedure '{}'".format(func.__name__),
func.__doc__))
for m, v in _get_analysis_mapping().items(): for m, v in _get_analysis_mapping().items():
records.extend(make_records("Analysis '{}'".format(m), v.__doc__)) records.extend(make_records("Analysis '{}'".format(m), v.__doc__))
@@ -41,8 +41,10 @@ def help() -> mgp.Record(name=str, value=str):
@mgp.read_proc @mgp.read_proc
def analyze(context: mgp.ProcCtx, analyses: mgp.Nullable[List[str]] = None) -> mgp.Record(name=str, value=str): def analyze(context: mgp.ProcCtx,
""" analyses: mgp.Nullable[List[str]] = None
) -> mgp.Record(name=str, value=str):
'''
Shows graph information. Shows graph information.
In case of multiple results, only the first 10 will be shown. In case of multiple results, only the first 10 will be shown.
@@ -55,20 +57,19 @@ def analyze(context: mgp.ProcCtx, analyses: mgp.Nullable[List[str]] = None) -> m
Example call (with parameter): Example call (with parameter):
CALL graph_analyzer.analyze(['nodes', 'edges']) YIELD *; CALL graph_analyzer.analyze(['nodes', 'edges']) YIELD *;
""" '''
g = MemgraphMultiDiGraph(ctx=context) g = MemgraphMultiDiGraph(ctx=context)
recs = _analyze_graph(context, g, analyses) recs = _analyze_graph(context, g, analyses)
return [mgp.Record(name=name, value=value) for name, value in recs] return [mgp.Record(name=name, value=value) for name, value in recs]
@mgp.read_proc @mgp.read_proc
def analyze_subgraph( def analyze_subgraph(context: mgp.ProcCtx,
context: mgp.ProcCtx, vertices: mgp.List[mgp.Vertex],
vertices: mgp.List[mgp.Vertex], edges: mgp.List[mgp.Edge],
edges: mgp.List[mgp.Edge], analyses: mgp.Nullable[List[str]] = None
analyses: mgp.Nullable[List[str]] = None, ) -> mgp.Record(name=str, value=str):
) -> mgp.Record(name=str, value=str): '''
"""
Shows subgraph information. Shows subgraph information.
In case of multiple results, only the first 10 will be shown. In case of multiple results, only the first 10 will be shown.
@@ -90,40 +91,36 @@ def analyze_subgraph(
CALL graph_analyzer.analyze_subgraph(nodes, edges, ['nodes', 'edges']) CALL graph_analyzer.analyze_subgraph(nodes, edges, ['nodes', 'edges'])
YIELD * YIELD *
RETURN name, value; RETURN name, value;
""" '''
vertices, edges = map(set, [vertices, edges]) vertices, edges = map(set, [vertices, edges])
g = nx.subgraph_view( g = nx.subgraph_view(
MemgraphMultiDiGraph(ctx=context), MemgraphMultiDiGraph(ctx=context),
lambda n: n in vertices, lambda n: n in vertices,
lambda n1, n2, e: e in edges, lambda n1, n2, e: e in edges)
)
recs = _analyze_graph(context, g, analyses) recs = _analyze_graph(context, g, analyses)
return [mgp.Record(name=name, value=value) for name, value in recs] return [mgp.Record(name=name, value=value) for name, value in recs]
def _get_analysis_mapping(): def _get_analysis_mapping():
return OrderedDict( return OrderedDict([
[ ('nodes', _number_of_nodes),
("nodes", _number_of_nodes), ('edges', _number_of_edges),
("edges", _number_of_edges), ('bridges', _bridges),
("bridges", _bridges), ('articulation_points', _articulation_points),
("articulation_points", _articulation_points), ('avg_degree', _avg_degree),
("avg_degree", _avg_degree), ('sorted_nodes_degree', _sorted_nodes_degree),
("sorted_nodes_degree", _sorted_nodes_degree), ('self_loops', _self_loops),
("self_loops", _self_loops), ('is_bipartite', _is_bipartite),
("is_bipartite", _is_bipartite), ('is_planar', _is_planar),
("is_planar", _is_planar), ('is_biconnected: ', _is_biconnected),
("is_biconnected: ", _is_biconnected), ('is_weakly_connected', _is_weakly_connected),
("is_weakly_connected", _is_weakly_connected), ('number_of_weakly_components', _weakly_components),
("number_of_weakly_components", _weakly_components), ('is_strongly_connected', _is_strongly_connected),
("is_strongly_connected", _is_strongly_connected), ('strongly_components', _strongly_components),
("strongly_components", _strongly_components), ('is_dag', _is_dag),
("is_dag", _is_dag), ('is_eulerian', _is_eulerian),
("is_eulerian", _is_eulerian), ('is_forest', _is_forest),
("is_forest", _is_forest), ('is_tree', _is_tree)])
("is_tree", _is_tree),
]
)
def _get_analysis_func(name: str): def _get_analysis_func(name: str):
@@ -135,15 +132,20 @@ def _get_analysis_funcs():
return _get_analysis_mapping().values() return _get_analysis_mapping().values()
def _analyze_graph(context: mgp.ProcCtx, g: nx.MultiDiGraph, analyses: List[str]) -> List[Tuple[str, str]]: def _analyze_graph(context: mgp.ProcCtx,
g: nx.MultiDiGraph,
analyses: List[str]
) -> List[Tuple[str, str]]:
functions = _get_analysis_funcs() if analyses is None else [_get_analysis_func(name) for name in analyses] functions = (_get_analysis_funcs() if analyses is None
else [_get_analysis_func(name) for name in analyses])
records = [] records = []
for index, f in enumerate(functions): for index, f in enumerate(functions):
context.check_must_abort() context.check_must_abort()
if f is None: if f is None:
raise KeyError("Graph analysis is not supported: " + analyses[index]) raise KeyError('Graph analysis is not supported: ' +
analyses[index])
name, value = f(g) name, value = f(g)
if isinstance(value, (list, set, tuple)): if isinstance(value, (list, set, tuple)):
value = list(value)[:_MAX_LIST_SIZE] value = list(value)[:_MAX_LIST_SIZE]
@@ -153,120 +155,126 @@ def _analyze_graph(context: mgp.ProcCtx, g: nx.MultiDiGraph, analyses: List[str]
def _number_of_nodes(g: nx.MultiDiGraph) -> Tuple[str, int]: def _number_of_nodes(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of nodes.""" '''Returns number of nodes.'''
return "Number of nodes", nx.number_of_nodes(g) return 'Number of nodes', nx.number_of_nodes(g)
def _number_of_edges(g: nx.MultiDiGraph) -> Tuple[str, int]: def _number_of_edges(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of edges.""" '''Returns number of edges.'''
return "Number of edges", nx.number_of_edges(g) return 'Number of edges', nx.number_of_edges(g)
def _avg_degree(g: nx.MultiDiGraph) -> Tuple[str, float]: def _avg_degree(g: nx.MultiDiGraph) -> Tuple[str, float]:
"""Returns average degree.""" '''Returns average degree.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
_, number_of_edges = _number_of_edges(g) _, number_of_edges = _number_of_edges(g)
avg_degree = 0 if number_of_nodes == 0 else number_of_edges / number_of_nodes avg_degree = (0 if number_of_nodes == 0
return "Average degree", avg_degree else number_of_edges / number_of_nodes)
return 'Average degree', avg_degree
def _sorted_nodes_degree(g: nx.MultiDiGraph) -> Tuple[str, List[int]]: def _sorted_nodes_degree(g: nx.MultiDiGraph) -> Tuple[str, List[int]]:
"""Returns list of sorted nodes degree. [(node_id, degree), ...]""" '''Returns list of sorted nodes degree. [(node_id, degree), ...]'''
nodes_degree = [(n, g.degree(n)) for n in g.nodes()] nodes_degree = [(n, g.degree(n)) for n in g.nodes()]
nodes_degree.sort(key=lambda x: x[1], reverse=True) nodes_degree.sort(key=lambda x: x[1], reverse=True)
return "Sorted nodes degree", nodes_degree return 'Sorted nodes degree', nodes_degree
def _self_loops(g: nx.MultiDiGraph) -> Tuple[str, int]: def _self_loops(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of self loops.""" '''Returns number of self loops.'''
return "Self loops", sum((1 if e[0] == e[1] else 0 for e in g.edges())) return 'Self loops', sum((1 if e[0] == e[1] else 0 for e in g.edges()))
def _is_bipartite(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_bipartite(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is bipartite.""" '''Checks if graph is bipartite.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.bipartite.basic.is_bipartite(g) ret = (False if number_of_nodes == 0
return "Is bipartite", ret else nx.algorithms.bipartite.basic.is_bipartite(g))
return 'Is bipartite', ret
def _is_planar(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_planar(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is planar.""" '''Checks if graph is planar.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.planarity.check_planarity(g)[0] ret = (False if number_of_nodes == 0
return "Is planar", ret else nx.algorithms.planarity.check_planarity(g)[0])
return 'Is planar', ret
def _is_biconnected(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_biconnected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Check if graph is biconnected.""" '''Check if graph is biconnected.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.is_biconnected(nx.MultiDiGraph.to_undirected(g)) ret = (False if number_of_nodes == 0
return "Is biconnected", ret else nx.is_biconnected(nx.MultiDiGraph.to_undirected(g)))
return 'Is biconnected', ret
def _is_weakly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_weakly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Check if graph is weakly connected.""" '''Check if graph is weakly connected.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.is_weakly_connected(g) ret = False if number_of_nodes == 0 else nx.is_weakly_connected(g)
return "Is weakly connected", ret return 'Is weakly connected', ret
def _is_strongly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_strongly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is strongly connected.""" '''Checks if graph is strongly connected.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.is_strongly_connected(g) ret = False if number_of_nodes == 0 else nx.is_strongly_connected(g)
return "Is strongly connected", ret return 'Is strongly connected', ret
def _is_dag(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_dag(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Check if graph is directed acyclic graph (DAG)""" '''Check if graph is directed acyclic graph (DAG)'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.dag.is_directed_acyclic_graph(g) ret = (False if number_of_nodes == 0
return "Is DAG", ret else nx.algorithms.dag.is_directed_acyclic_graph(g))
return 'Is DAG', ret
def _is_eulerian(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_eulerian(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is Eulerian.""" '''Checks if graph is Eulerian.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.euler.is_eulerian(g) ret = (False if number_of_nodes == 0
return "Is eulerian", ret else nx.algorithms.euler.is_eulerian(g))
return 'Is eulerian', ret
def _is_forest(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_forest(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is forest, all components must be trees.""" '''Checks if graph is forest, all components must be trees.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.tree.recognition.is_forest(g) ret = (False if number_of_nodes == 0
return "Is forest", ret else nx.algorithms.tree.recognition.is_forest(g))
return 'Is forest', ret
def _is_tree(g: nx.MultiDiGraph) -> Tuple[str, bool]: def _is_tree(g: nx.MultiDiGraph) -> Tuple[str, bool]:
"""Checks if graph is tree.""" '''Checks if graph is tree.'''
_, number_of_nodes = _number_of_nodes(g) _, number_of_nodes = _number_of_nodes(g)
ret = False if number_of_nodes == 0 else nx.algorithms.tree.recognition.is_tree(g) ret = (False if number_of_nodes == 0
return "Is tree", ret else nx.algorithms.tree.recognition.is_tree(g))
return 'Is tree', ret
def _bridges(g: nx.MultiDiGraph) -> Tuple[str, int]: def _bridges(g: nx.MultiDiGraph) -> Tuple[str, int]:
"""Returns number of bridges, multiple edges between same nodes are '''Returns number of bridges, multiple edges between same nodes are
mapped to one edge.""" mapped to one edge.'''
return "Number of bridges", sum(1 for _ in nx.bridges(nx.Graph(g))) return 'Number of bridges', sum(1 for _ in nx.bridges(nx.Graph(g)))
def _articulation_points(g: nx.MultiDiGraph): def _articulation_points(g: nx.MultiDiGraph):
"""Returns number of articulation points.""" '''Returns number of articulation points.'''
undirected = nx.MultiDiGraph.to_undirected(g) undirected = nx.MultiDiGraph.to_undirected(g)
return ( return ('Number of articulation points',
"Number of articulation points", sum(1 for _ in nx.articulation_points(undirected)))
sum(1 for _ in nx.articulation_points(undirected)),
)
def _weakly_components(g: nx.MultiDiGraph): def _weakly_components(g: nx.MultiDiGraph):
"""Returns number of weakly components.""" '''Returns number of weakly components.'''
comps = nx.algorithms.components.number_weakly_connected_components(g) comps = nx.algorithms.components.number_weakly_connected_components(g)
return "Number of weakly connected components", comps return 'Number of weakly connected components', comps
def _strongly_components(g: nx.MultiDiGraph): def _strongly_components(g: nx.MultiDiGraph):
"""Returns number of strongly connected components.""" '''Returns number of strongly connected components.'''
comps = nx.algorithms.components.number_strongly_connected_components(g) comps = nx.algorithms.components.number_strongly_connected_components(g)
return "Number of strongly connected components", comps return 'Number of strongly connected components', comps

View File

@@ -1,22 +1,20 @@
import sys import sys
import mgp import mgp
import collections import collections
try: try:
import networkx as nx import networkx as nx
except ImportError as import_error: except ImportError as import_error:
sys.stderr.write( sys.stderr.write((
( '\n'
"\n" 'NOTE: Please install networkx to be able to use Memgraph NetworkX '
"NOTE: Please install networkx to be able to use Memgraph NetworkX " 'wrappers. Using Python:\n'
"wrappers. Using Python:\n" + sys.version + "\n" + sys.version +
) '\n'))
)
raise import_error raise import_error
class MemgraphAdjlistOuterDict(collections.abc.Mapping): class MemgraphAdjlistOuterDict(collections.abc.Mapping):
__slots__ = ("_ctx", "_succ", "_multi") __slots__ = ('_ctx', '_succ', '_multi')
def __init__(self, ctx, succ=True, multi=True): def __init__(self, ctx, succ=True, multi=True):
self._ctx = ctx self._ctx = ctx
@@ -26,7 +24,8 @@ class MemgraphAdjlistOuterDict(collections.abc.Mapping):
def __getitem__(self, key): def __getitem__(self, key):
if key not in self: if key not in self:
raise KeyError raise KeyError
return MemgraphAdjlistInnerDict(key, succ=self._succ, multi=self._multi) return MemgraphAdjlistInnerDict(key, succ=self._succ,
multi=self._multi)
def __iter__(self): def __iter__(self):
return iter(self._ctx.graph.vertices) return iter(self._ctx.graph.vertices)
@@ -41,7 +40,7 @@ class MemgraphAdjlistOuterDict(collections.abc.Mapping):
class MemgraphAdjlistInnerDict(collections.abc.Mapping): class MemgraphAdjlistInnerDict(collections.abc.Mapping):
__slots__ = ("_node", "_succ", "_multi", "_neighbors") __slots__ = ('_node', '_succ', '_multi', '_neighbors')
def __init__(self, node, succ=True, multi=True): def __init__(self, node, succ=True, multi=True):
self._node = node self._node = node
@@ -72,26 +71,31 @@ class MemgraphAdjlistInnerDict(collections.abc.Mapping):
def _get_neighbors(self): def _get_neighbors(self):
if not self._neighbors: if not self._neighbors:
if self._succ: if self._succ:
self._neighbors = set(e.to_vertex for e in self._node.out_edges) self._neighbors = set(
e.to_vertex for e in self._node.out_edges)
else: else:
self._neighbors = set(e.from_vertex for e in self._node.in_edges) self._neighbors = set(
e.from_vertex for e in self._node.in_edges)
return self._neighbors return self._neighbors
def _get_edge(self, neighbor): def _get_edge(self, neighbor):
if self._succ: if self._succ:
edge = list(filter(lambda e: e.to_vertex == neighbor, self._node.out_edges)) edge = list(filter(lambda e: e.to_vertex == neighbor,
self._node.out_edges))
else: else:
edge = list(filter(lambda e: e.from_vertex == neighbor, self._node.in_edges)) edge = list(filter(lambda e: e.from_vertex == neighbor,
self._node.in_edges))
assert len(edge) >= 1 assert len(edge) >= 1
if len(edge) > 1: if len(edge) > 1:
raise RuntimeError("Graph contains multiedges but " "is of non-multigraph type: {}".format(edge)) raise RuntimeError('Graph contains multiedges but '
'is of non-multigraph type: {}'.format(edge))
return edge[0] return edge[0]
class MemgraphEdgeKeyDict(collections.abc.Mapping): class MemgraphEdgeKeyDict(collections.abc.Mapping):
__slots__ = ("_node", "_neighbor", "_succ", "_edges") __slots__ = ('_node', '_neighbor', '_succ', '_edges')
def __init__(self, node, neighbor, succ=True): def __init__(self, node, neighbor, succ=True):
self._node = node self._node = node
@@ -118,14 +122,18 @@ class MemgraphEdgeKeyDict(collections.abc.Mapping):
def _get_edges(self): def _get_edges(self):
if not self._edges: if not self._edges:
if self._succ: if self._succ:
self._edges = list(filter(lambda e: e.to_vertex == self._neighbor, self._node.out_edges)) self._edges = list(filter(
lambda e: e.to_vertex == self._neighbor,
self._node.out_edges))
else: else:
self._edges = list(filter(lambda e: e.from_vertex == self._neighbor, self._node.in_edges)) self._edges = list(filter(
lambda e: e.from_vertex == self._neighbor,
self._node.in_edges))
return self._edges return self._edges
class UnhashableProperties(collections.abc.Mapping): class UnhashableProperties(collections.abc.Mapping):
__slots__ = "_properties" __slots__ = ('_properties')
def __init__(self, properties): def __init__(self, properties):
self._properties = properties self._properties = properties
@@ -147,7 +155,7 @@ class UnhashableProperties(collections.abc.Mapping):
class MemgraphNodeDict(collections.abc.Mapping): class MemgraphNodeDict(collections.abc.Mapping):
__slots__ = ("_ctx",) __slots__ = ('_ctx',)
def __init__(self, ctx): def __init__(self, ctx):
self._ctx = ctx self._ctx = ctx
@@ -179,7 +187,8 @@ class MemgraphNodeDict(collections.abc.Mapping):
class MemgraphDiGraphBase: class MemgraphDiGraphBase:
def __init__(self, incoming_graph_data=None, ctx=None, multi=True, **kwargs): def __init__(self, incoming_graph_data=None, ctx=None, multi=True,
**kwargs):
# NOTE: We assume that our graph will never be given any initial data # NOTE: We assume that our graph will never be given any initial data
# because we already pull our data from the Memgraph database. This # because we already pull our data from the Memgraph database. This
# assert is triggered by certain NetworkX procedures because they # assert is triggered by certain NetworkX procedures because they
@@ -192,30 +201,23 @@ class MemgraphDiGraphBase:
# modify the graph's internal attributes and don't try to populate it # modify the graph's internal attributes and don't try to populate it
# with initial data or modify it. # with initial data or modify it.
self.node_dict_factory = lambda: MemgraphNodeDict(ctx) if ctx else self._error self.node_dict_factory = lambda: MemgraphNodeDict(ctx) \
if ctx else self._error
self.node_attr_dict_factory = self._error self.node_attr_dict_factory = self._error
self.adjlist_outer_dict_factory = lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) if ctx else self._error self.adjlist_outer_dict_factory = \
lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) \
if ctx else self._error
self.adjlist_inner_dict_factory = self._error self.adjlist_inner_dict_factory = self._error
self.edge_key_dict_factory = self._error self.edge_key_dict_factory = self._error
self.edge_attr_dict_factory = self._error self.edge_attr_dict_factory = self._error
# NOTE: We forbid any mutating operations because our graph is # NOTE: We forbid any mutating operations because our graph is
# immutable and pulls its data from the Memgraph database. # immutable and pulls its data from the Memgraph database.
for f in [ for f in ['add_node', 'add_nodes_from', 'remove_node',
"add_node", 'remove_nodes_from', 'add_edge', 'add_edges_from',
"add_nodes_from", 'add_weighted_edges_from', 'new_edge_key', 'remove_edge',
"remove_node", 'remove_edges_from', 'update', 'clear']:
"remove_nodes_from",
"add_edge",
"add_edges_from",
"add_weighted_edges_from",
"new_edge_key",
"remove_edge",
"remove_edges_from",
"update",
"clear",
]:
setattr(self, f, lambda *args, **kwargs: self._error()) setattr(self, f, lambda *args, **kwargs: self._error())
super().__init__(None, **kwargs) super().__init__(None, **kwargs)
@@ -229,29 +231,33 @@ class MemgraphDiGraphBase:
self._pred = MemgraphAdjlistOuterDict(ctx, succ=False, multi=multi) self._pred = MemgraphAdjlistOuterDict(ctx, succ=False, multi=multi)
def _error(self): def _error(self):
raise RuntimeError("Modification operations are not supported") raise RuntimeError('Modification operations are not supported')
class MemgraphMultiDiGraph(MemgraphDiGraphBase, nx.MultiDiGraph): class MemgraphMultiDiGraph(MemgraphDiGraphBase, nx.MultiDiGraph):
def __init__(self, incoming_graph_data=None, ctx=None, **kwargs): def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
super().__init__(incoming_graph_data=incoming_graph_data, ctx=ctx, multi=True, **kwargs) super().__init__(incoming_graph_data=incoming_graph_data,
ctx=ctx, multi=True, **kwargs)
def MemgraphMultiGraph(incoming_graph_data=None, ctx=None, **kwargs): def MemgraphMultiGraph(incoming_graph_data=None, ctx=None, **kwargs):
return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data, ctx=ctx, **kwargs).to_undirected(as_view=True) return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data,
ctx=ctx, **kwargs).to_undirected(as_view=True)
class MemgraphDiGraph(MemgraphDiGraphBase, nx.DiGraph): class MemgraphDiGraph(MemgraphDiGraphBase, nx.DiGraph):
def __init__(self, incoming_graph_data=None, ctx=None, **kwargs): def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
super().__init__(incoming_graph_data=incoming_graph_data, ctx=ctx, multi=False, **kwargs) super().__init__(incoming_graph_data=incoming_graph_data,
ctx=ctx, multi=False, **kwargs)
def MemgraphGraph(incoming_graph_data=None, ctx=None, **kwargs): def MemgraphGraph(incoming_graph_data=None, ctx=None, **kwargs):
return MemgraphDiGraph(incoming_graph_data=incoming_graph_data, ctx=ctx, **kwargs).to_undirected(as_view=True) return MemgraphDiGraph(incoming_graph_data=incoming_graph_data,
ctx=ctx, **kwargs).to_undirected(as_view=True)
class PropertiesDictionary(collections.abc.Mapping): class PropertiesDictionary(collections.abc.Mapping):
__slots__ = ("_ctx", "_prop", "_len") __slots__ = ('_ctx', '_prop', '_len')
def __init__(self, ctx, prop): def __init__(self, ctx, prop):
self._ctx = ctx self._ctx = ctx
@@ -264,7 +270,8 @@ class PropertiesDictionary(collections.abc.Mapping):
try: try:
return vertex.properties[self._prop] return vertex.properties[self._prop]
except KeyError: except KeyError:
raise KeyError(("{} doesn\t have the required " + "property '{}'").format(vertex, self._prop)) raise KeyError(("{} doesn\t have the required " +
"property '{}'").format(vertex, self._prop))
def __iter__(self): def __iter__(self):
for v in self._ctx.graph.vertices: for v in self._ctx.graph.vertices:

File diff suppressed because it is too large Load Diff

View File

@@ -1,20 +1,23 @@
import sys import sys
import mgp import mgp
try: try:
import networkx as nx import networkx as nx
except ImportError as import_error: except ImportError as import_error:
sys.stderr.write( sys.stderr.write(
"\n" "NOTE: Please install networkx to be able to use wcc module.\n" "Using Python:\n" + sys.version + "\n" '\n'
) 'NOTE: Please install networkx to be able to use wcc module.\n'
'Using Python:\n'
+ sys.version +
'\n')
raise import_error raise import_error
@mgp.read_proc @mgp.read_proc
def get_components( def get_components(vertices: mgp.List[mgp.Vertex],
vertices: mgp.List[mgp.Vertex], edges: mgp.List[mgp.Edge] edges: mgp.List[mgp.Edge]
) -> mgp.Record(n_components=int, components=mgp.List[mgp.List[mgp.Vertex]]): ) -> mgp.Record(n_components=int,
""" components=mgp.List[mgp.List[mgp.Vertex]]):
'''
This procedure finds weakly connected components of a given subgraph of a This procedure finds weakly connected components of a given subgraph of a
directed graph. directed graph.
@@ -38,7 +41,7 @@ def get_components(
WITH collect(n) AS nodes, collect(e) AS edges WITH collect(n) AS nodes, collect(e) AS edges
CALL wcc.get_components(nodes, edges) YIELD * CALL wcc.get_components(nodes, edges) YIELD *
RETURN n_components, components; RETURN n_components, components;
""" '''
g = nx.DiGraph() g = nx.DiGraph()
g.add_nodes_from(vertices) g.add_nodes_from(vertices)
g.add_edges_from([(edge.from_vertex, edge.to_vertex) for edge in edges]) g.add_edges_from([(edge.from_vertex, edge.to_vertex) for edge in edges])

View File

@@ -1,6 +1,10 @@
# Install systemd service (must use absolute path). # Install systemd service (must use absolute path).
install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/memgraph.service install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/memgraph.service
DESTINATION /lib/systemd/system) DESTINATION /lib/systemd/system)
# Set parameters to recognize the host distro
cmake_host_system_information(RESULT DISTRO QUERY DISTRIB_NAME)
cmake_host_system_information(RESULT DISTRO_VERSION QUERY DISTRIB_VERSION)
# ---- Setup CPack -------- # ---- Setup CPack --------
@@ -10,12 +14,13 @@ set(CPACK_PACKAGE_VENDOR "Memgraph Ltd.")
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
"High performance, in-memory, transactional graph database") "High performance, in-memory, transactional graph database")
# Setting arhitecture extension for deb packages # Setting architecture extension for deb packages
set(MG_ARCH_EXTENSION_DEB "all") set(MG_ARCH_EXTENSION_DEB "all")
if (${MG_ARCH} STREQUAL "x86_64")
if(${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION_DEB "amd64") set(MG_ARCH_EXTENSION_DEB "amd64")
elseif (${MG_ARCH} STREQUAL "ARM64") elseif(${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION_DEB "arm64") set(MG_ARCH_EXTENSION_DEB "arm64")
endif() endif()
# DEB specific # DEB specific
@@ -34,21 +39,24 @@ set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
"${CMAKE_CURRENT_SOURCE_DIR}/debian/postrm;" "${CMAKE_CURRENT_SOURCE_DIR}/debian/postrm;"
"${CMAKE_CURRENT_SOURCE_DIR}/debian/postinst;") "${CMAKE_CURRENT_SOURCE_DIR}/debian/postinst;")
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
# Description formatting is important, summary must be followed with a newline and 1 space. # Description formatting is important, summary must be followed with a newline and 1 space.
set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY} set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
Contains Memgraph, the graph database. It aims to deliver developers the Contains Memgraph, the graph database. It aims to deliver developers the
speed, simplicity and scale required to build the next generation of speed, simplicity and scale required to build the next generation of
applications driver by real-time connected data.") applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates. # Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph. # We also depend on `python3` because we embed it in Memgraph.
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6") set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
# Setting arhitecture extension for rpm packages # Setting architecture extension for rpm packages
set(MG_ARCH_EXTENSION_RPM "noarch") set(MG_ARCH_EXTENSION_RPM "noarch")
if (${MG_ARCH} STREQUAL "x86_64")
if(${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION_RPM "x86_64") set(MG_ARCH_EXTENSION_RPM "x86_64")
elseif (${MG_ARCH} STREQUAL "ARM64") elseif(${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION_RPM "aarch64") set(MG_ARCH_EXTENSION_RPM "aarch64")
endif() endif()
# RPM specific # RPM specific
@@ -56,18 +64,26 @@ set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}") set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION_RPM}.rpm") set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION_RPM}.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/var /var/lib /var/log /etc/logrotate.d /var /var/lib /var/log /etc/logrotate.d
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service) /lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils") set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/rpm/memgraph.spec.in") set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/rpm/memgraph.spec.in")
set(CPACK_RPM_PACKAGE_LICENSE "Memgraph License") set(CPACK_RPM_PACKAGE_LICENSE "Memgraph License")
# CPACK deduces dependency to libstdc++ which:
# * can't be easily installed on Centos 7 (the one from the toolchain,
# only required to avoid printing issue within query modules)
# * it causes issues with glibcxx 2.4
# `if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")`
set(CPACK_RPM_PACKAGE_AUTOREQ " no")
# Description formatting is important, no line must be greater than 80 characters. # Description formatting is important, no line must be greater than 80 characters.
set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database. set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
It aims to deliver developers the speed, simplicity and scale required to build It aims to deliver developers the speed, simplicity and scale required to build
the next generation of applications driver by real-time connected data.") the next generation of applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates. # Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph. # We also depend on `python3` because we embed it in Memgraph.
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate") set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, logrotate")
# All variables must be set before including. # All variables must be set before including.
include(CPack) include(CPack)

View File

@@ -55,7 +55,8 @@ image_name="memgraph:${version}"
image_package_name="memgraph-${version}-docker.tar.gz" image_package_name="memgraph-${version}-docker.tar.gz"
# Build docker image. # Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \ # NOTE: --pull is here to always pull that latest base image because of security patches.
docker build --pull -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
--build-arg BINARY_NAME=${package_name} \ --build-arg BINARY_NAME=${package_name} \
--build-arg EXTENSION=${extension} \ --build-arg EXTENSION=${extension} \
--build-arg TARGETARCH="" . --build-arg TARGETARCH="" .

View File

@@ -104,9 +104,7 @@ def retry(retry_limit, timeout=100):
except Exception: except Exception:
time.sleep(timeout) time.sleep(timeout)
return func(*args, **kwargs) return func(*args, **kwargs)
return wrapper return wrapper
return inner_func return inner_func
@@ -165,15 +163,8 @@ def format_version(variant, version, offering, distance=None, shorthash=None, su
# Parse arguments. # Parse arguments.
parser = argparse.ArgumentParser(description="Get the current version of Memgraph.") parser = argparse.ArgumentParser(description="Get the current version of Memgraph.")
parser.add_argument( parser.add_argument("--open-source", action="store_true", help="set the current offering to 'open-source'")
"--open-source", parser.add_argument("version", help="manual version override, if supplied the version isn't " "determined using git")
action="store_true",
help="set the current offering to 'open-source'",
)
parser.add_argument(
"version",
help="manual version override, if supplied the version isn't " "determined using git",
)
parser.add_argument("suffix", help="custom suffix for the current version being built") parser.add_argument("suffix", help="custom suffix for the current version being built")
parser.add_argument( parser.add_argument(
"--variant", "--variant",
@@ -182,9 +173,7 @@ parser.add_argument(
help="which variant of the version string should be generated", help="which variant of the version string should be generated",
) )
parser.add_argument( parser.add_argument(
"--memgraph-root-dir", "--memgraph-root-dir", help="The root directory of the checked out " "Memgraph repository.", default="."
help="The root directory of the checked out " "Memgraph repository.",
default=".",
) )
args = parser.parse_args() args = parser.parse_args()
@@ -267,27 +256,14 @@ for version in versions:
if current_version is None: if current_version is None:
raise Exception("You are attempting to determine the version for a very " "old version of Memgraph!") raise Exception("You are attempting to determine the version for a very " "old version of Memgraph!")
version, branch, master_branch_merge = current_version version, branch, master_branch_merge = current_version
distance = int( distance = int(get_output("git", "rev-list", "--count", "--first-parent", master_branch_merge + ".." + current_hash))
get_output(
"git",
"rev-list",
"--count",
"--first-parent",
master_branch_merge + ".." + current_hash,
)
)
version_str = ".".join(map(str, version)) + ".0" version_str = ".".join(map(str, version)) + ".0"
if distance == 0: if distance == 0:
print(format_version(args.variant, version_str, offering, suffix=args.suffix), end="") print(format_version(args.variant, version_str, offering, suffix=args.suffix), end="")
else: else:
print( print(
format_version( format_version(
args.variant, args.variant, version_str, offering, distance=distance, shorthash=current_hash_short, suffix=args.suffix
version_str,
offering,
distance=distance,
shorthash=current_hash_short,
suffix=args.suffix,
), ),
end="", end="",
) )

4
release/mgp/.gitignore vendored Normal file
View File

@@ -0,0 +1,4 @@
.venv
dist
mgp.py
poetry.lock

201
release/mgp/LICENSE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

11
release/mgp/README.md Normal file
View File

@@ -0,0 +1,11 @@
# mgp
PyPi package used for type hinting when creating query modules. Repository of already available query modules is called [MAGE](https://github.com/memgraph/mage).
## 🎬 Get started
To learn more, head over to the [docs for the query modules Python API](https://memgraph.com/docs/memgraph/reference-guide/query-modules/api/python-api). To get started with query modules, check out the [how-to guide](https://memgraph.com/docs/memgraph/how-to-guides/query-modules) on Memgraph docs.
## 🔢 Versioning
- mgp v1.1 is compatible with Memgraph >= 2.4.0

267
release/mgp/_mgp.py Normal file
View File

@@ -0,0 +1,267 @@
from typing import Any
class MgpIterable:
def get() -> Any:
pass
def next() -> Any:
pass
class Vertex:
def is_valid() -> bool: # type: ignore
pass
def underlying_graph_is_mutable() -> bool: # type: ignore
pass
def iter_properties() -> MgpIterable: # type: ignore
pass
def get_property(self, property_name: str) -> "Property": # type: ignore
pass
def set_property(self, property_name: str, value: Any) -> "Property": # type: ignore
pass
def get_id() -> "VertexId": # type: ignore
pass
def label_at(self, index: int) -> "Label": # type: ignore
pass
def labels_count() -> int: # type: ignore
pass
def add_label(self, label: Any):
pass
def remove_label(self, label: Any):
pass
def iter_in_edges() -> MgpIterable: # type: ignore
pass
def iter_out_edges() -> MgpIterable: # type: ignore
pass
class Edge:
def is_valid() -> bool: # type: ignore
pass
def underlying_graph_is_mutable() -> bool: # type: ignore
pass
def iter_properties() -> MgpIterable: # type: ignore
pass
def get_property(self, property_name: str) -> "Property": # type: ignore
pass
def set_property(self, property_name: str, valuse: Any) -> "Property": # type: ignore
pass
def get_type_name() -> str: # type: ignore
pass
def get_id() -> "EdgeId": # type: ignore
pass
def from_vertex() -> Vertex: # type: ignore
pass
def to_vertex() -> Vertex: # type: ignore
pass
class Path:
def is_valid() -> bool: # type: ignore
pass
@staticmethod
def make_with_start(vertex: Vertex) -> "Path": # type: ignore
pass
class Graph:
def is_valid() -> bool: # type: ignore
pass
class CypherType:
pass
class Message:
def is_valid() -> bool: # type: ignore
pass
def source_type() -> str: # type: ignore
pass
def topic_name() -> str: # type: ignore
pass
def key() -> bytes: # type: ignore
pass
def timestamp() -> int: # type: ignore
pass
def offset() -> int: # type: ignore
pass
def payload() -> bytes: # type: ignore
pass
class Messages:
def is_valid() -> bool: # type: ignore
pass
def message_at(self, id: int) -> Message: # type: ignore
pass
def total_messages() -> int: # type: ignore
pass
class UnknownError(Exception):
pass
class UnableToAllocateError(Exception):
pass
class InsufficientBufferError(Exception):
pass
class OutOfRangeError(Exception):
pass
class LogicErrorError(Exception):
pass
class DeletedObjectError(Exception):
pass
class InvalidArgumentError(Exception):
pass
class KeyAlreadyExistsError(Exception):
pass
class ImmutableObjectError(Exception):
pass
class ValueConversionError(Exception):
pass
class SerializationError(Exception):
pass
class AuthorizationError(Exception):
pass
def type_nullable(elem: Any):
pass
def type_list(elem: Any):
pass
def type_bool():
pass
def type_string():
pass
def type_int():
pass
def type_float():
pass
def type_number():
pass
def type_map():
pass
def type_node():
pass
def type_relationship():
pass
def type_path():
pass
def type_date():
pass
def type_local_time():
pass
def type_local_date_time():
pass
def type_duration():
pass
def type_any():
pass
class _MODULE:
@staticmethod
def add_read_procedure(wrapper):
pass
@staticmethod
def add_write_procedure(wrapper):
pass
@staticmethod
def add_transformation(wrapper):
pass
@staticmethod
def add_function(wrapper):
pass
class SOURCE_TYPE_KAFKA:
pass
class SOURCE_TYPE_PULSAR:
pass

View File

@@ -0,0 +1,22 @@
# How to publish new versions
## Prerequisites
1. Installed poetry
```
pip install poetry
```
2. Set up [API tokens](https://pypi.org/help/#apitoken)
3. Be a collaborator on [pypi](https://pypi.org/project/mgp/)
## Making changes
1. Make changes to the package
2. Bump version in `pyproject.tml`
3. `poetry build`
4. `poetry publish`
## Why is this not automatized?
Because someone always has to manually bump up the version in `pyproject.toml`
## Why does `_mgp.py` exists?
Because we are mocking here all the types that are created by Memgraph
in order to fix typing errors in `mgp.py`.

View File

@@ -0,0 +1,23 @@
[tool.poetry]
name = "mgp"
version = "1.1.1"
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
authors = [
"katarinasupe <katarina.supe@memgraph.io>",
"jbajic <jure.bajic@memgraph.io>",
"antejavor <ante.javor@memgraph.io>",
"antaljanosbenjamin <benjamin.antal@memgraph.io>",
"MasterMedo <mislav.vuletic@gmail.com>",
]
license = "Apache-2.0"
readme = "README.md"
include = ["mgp.py", "_mgp.py"]
[tool.poetry.dependencies]
python = "^3.7"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@@ -0,0 +1,15 @@
FROM amazonlinux:2
ARG TOOLCHAIN_VERSION
RUN yum -y update \
&& yum install -y wget git tar
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,11 @@
version: "3"
services:
debian-11-arm:
build:
context: debian-11-arm
container_name: "mgbuild_debian-11-arm"
ubuntu-2204-arm:
build:
context: ubuntu-22.04-arm
container_name: "mgbuild_ubuntu-22.04-arm"

View File

@@ -9,6 +9,7 @@ RUN yum -y update \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -9,6 +9,7 @@ RUN yum -y update \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,6 +12,7 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,6 +12,7 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,6 +12,7 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -28,3 +28,11 @@ services:
build: build:
context: ubuntu-22.04 context: ubuntu-22.04
container_name: "mgbuild_ubuntu-22.04" container_name: "mgbuild_ubuntu-22.04"
mgbuild_fedora-36:
build:
context: fedora-36
container_name: "mgbuild_fedora-36"
mgbuild_amzn-2:
build:
context: amzn-2
container_name: "mgbuild_amzn-2"

View File

@@ -0,0 +1,16 @@
FROM fedora:36
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
RUN yum -y update \
&& yum install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -3,14 +3,22 @@
set -Eeuo pipefail set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm) SUPPORTED_OS=(
centos-7 centos-9
debian-10 debian-11 debian-11-arm
ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 ubuntu-22.04-arm
fedora-36
amzn-2
)
PROJECT_ROOT="$SCRIPT_DIR/../.." PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v4" TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate" ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output" HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () { print_help () {
echo "$0 init|package {os} [--for-docker|--for-platform]|docker|test" # TODO(gitbuda): Update the release/package/run.sh help
echo "$0 init|package|docker|test {os} [--for-docker|--for-platform]"
echo "" echo ""
echo " OSs: ${SUPPORTED_OS[*]}" echo " OSs: ${SUPPORTED_OS[*]}"
exit 1 exit 1
@@ -23,12 +31,12 @@ make_package () {
echo "Building Memgraph for $os on $build_container..." echo "Building Memgraph for $os on $build_container..."
package_command="" package_command=""
if [[ "$os" =~ ^"centos".* ]]; then if [[ "$os" =~ ^"centos".* ]] || [[ "$os" =~ ^"fedora".* ]] || [[ "$os" =~ ^"amzn".* ]]; then
docker exec "$build_container" bash -c "yum -y update" docker exec "$build_container" bash -c "yum -y update"
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm " package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
fi fi
if [[ "$os" =~ ^"debian".* ]]; then if [[ "$os" =~ ^"debian".* ]]; then
docker exec "$build_container" bash -c "apt update" docker exec "$build_container" bash -c "apt --allow-releaseinfo-change -y update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake " package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi fi
if [[ "$os" =~ ^"ubuntu".* ]]; then if [[ "$os" =~ ^"ubuntu".* ]]; then
@@ -57,6 +65,7 @@ make_package () {
git fetch origin master:master git fetch origin master:master
fi fi
docker exec "$build_container" mkdir -p /memgraph docker exec "$build_container" mkdir -p /memgraph
# TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI.
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/" docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
container_build_dir="/memgraph/build" container_build_dir="/memgraph/build"
@@ -67,14 +76,19 @@ make_package () {
# environment/os/{os}.sh does not come within the toolchain package. When # environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the # migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here. # TOOLCHAIN_RUN_DEPS installation from here.
# TODO(gitbuda): On the other side, having this here allows updating deps
# wihout reruning the build containers.
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..." echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS" docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS" docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..." echo "Building targeted package..."
# Fix issue with git marking directory as not safe
docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'"
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init" docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*" docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
if [[ "$os" == "debian-11-arm" ]]; then # TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX
if [[ "$os" =~ "-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .." docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else else
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .." docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
@@ -99,8 +113,13 @@ make_package () {
case "$1" in case "$1" in
init) init)
cd "$SCRIPT_DIR" cd "$SCRIPT_DIR"
docker-compose build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" if ! which "docker-compose" >/dev/null; then
docker-compose up -d docker_compose_cmd="docker compose"
else
docker_compose_cmd="docker-compose"
fi
$docker_compose_cmd build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
$docker_compose_cmd up -d
;; ;;
docker) docker)
@@ -140,6 +159,19 @@ case "$1" in
fi fi
;; ;;
build)
shift 1
if [[ "$#" -ne 2 ]]; then
print_help
fi
# in the vX format, e.g. v5
toolchain_version="$1"
# a name of the os folder, e.g. ubuntu-22.04-arm
os="$2"
cd "$SCRIPT_DIR/$os"
docker build -f Dockerfile --build-arg TOOLCHAIN_VERSION="toolchain-$toolchain_version" -t "memgraph/memgraph-builder:${toolchain_version}_$os" .
;;
test) test)
echo "TODO(gitbuda): Test all packages on mgtest containers." echo "TODO(gitbuda): Test all packages on mgtest containers."
;; ;;

View File

@@ -12,6 +12,7 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,6 +12,7 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,18 @@
FROM ubuntu:22.04
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,6 +12,7 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \ RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \ -O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt && tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz
ENTRYPOINT ["sleep", "infinity"] ENTRYPOINT ["sleep", "infinity"]

View File

@@ -30,7 +30,7 @@ BuildRequires: systemd
# This is needed to prevent Python compilation errors when building the RPM # This is needed to prevent Python compilation errors when building the RPM
# package # package
# https://github.com/scylladb/scylla/issues/2235 # https://github.com/scylladb/scylla/issues/2235
%if 0%{?rhel} < 8 %if 0%{?rhel} && 0%{?rhel} < 8
%global __os_install_post \ %global __os_install_post \
/usr/lib/rpm/redhat/brp-compress \ /usr/lib/rpm/redhat/brp-compress \
%{!?__debug_package:\ %{!?__debug_package:\
@@ -40,7 +40,9 @@ BuildRequires: systemd
/usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \ /usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
%{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \ %{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \
%{nil} %{nil}
%else %endif
%if 0%{?fedora} && 0%{?fedora} < 35
%global __os_install_post \ %global __os_install_post \
/usr/lib/rpm/brp-compress \ /usr/lib/rpm/brp-compress \
%{!?__debug_package:\ %{!?__debug_package:\

4
release/rpm/rpmlintrc Normal file
View File

@@ -0,0 +1,4 @@
# from https://github.com/google/earthenterprise/blob/master/earth_enterprise/rpmlintrc
# We are not packaging log dir
addFilter("E: logrotate-log-dir-not-packaged")

View File

@@ -1,7 +1,7 @@
# CMake configuration for the main memgraph library and executable # CMake configuration for the main memgraph library and executable
# add memgraph sub libraries, ordered by dependency # add memgraph sub libraries, ordered by dependency
add_subdirectory(lisp) add_subdirectory(csv)
add_subdirectory(utils) add_subdirectory(utils)
add_subdirectory(requests) add_subdirectory(requests)
add_subdirectory(io) add_subdirectory(io)
@@ -12,13 +12,12 @@ add_subdirectory(memory)
add_subdirectory(storage/v2) add_subdirectory(storage/v2)
add_subdirectory(integrations) add_subdirectory(integrations)
add_subdirectory(query) add_subdirectory(query)
add_subdirectory(glue)
add_subdirectory(slk) add_subdirectory(slk)
add_subdirectory(rpc) add_subdirectory(rpc)
add_subdirectory(license)
add_subdirectory(auth) add_subdirectory(auth)
add_subdirectory(audit)
if (MG_ENTERPRISE)
add_subdirectory(audit)
endif()
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -31,80 +30,93 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
# Memgraph Single Node v2 Executable # Memgraph Single Node v2 Executable
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
set(mg_single_node_v2_sources set(mg_single_node_v2_sources
glue/communication.cpp memgraph.cpp
memgraph.cpp
glue/auth.cpp
) )
set(mg_single_node_v2_libs stdc++fs Threads::Threads set(mg_single_node_v2_libs stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings) mg-telemetry mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-glue mg-audit)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
endif()
# memgraph main executable # memgraph main executable
add_executable(memgraph ${mg_single_node_v2_sources}) add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include) target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph ${mg_single_node_v2_libs}) target_link_libraries(memgraph ${mg_single_node_v2_libs})
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which # NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the # should be dynamically exported, so that `dlopen` can correctly link the
# symbols in custom procedure module libraries. # symbols in custom procedure module libraries.
target_link_libraries(memgraph "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms") target_link_libraries(memgraph "-Wl,--dynamic-list=${CMAKE_SOURCE_DIR}/include/mg_procedure.syms")
set_target_properties(memgraph PROPERTIES set_target_properties(memgraph PROPERTIES
# Set the executable output name to include version information.
OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}" # Set the executable output name to include version information.
# Output the executable in main binary dir. OUTPUT_NAME "memgraph-${MEMGRAPH_VERSION}_${CMAKE_BUILD_TYPE}"
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Output the executable in main binary dir.
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
# Create symlink to the built executable. # Create symlink to the built executable.
add_custom_command(TARGET memgraph POST_BUILD add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph COMMAND ${CMAKE_COMMAND} -E create_symlink $<TARGET_FILE:memgraph> ${CMAKE_BINARY_DIR}/memgraph
BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph BYPRODUCTS ${CMAKE_BINARY_DIR}/memgraph
COMMENT "Creating symlink to memgraph executable") COMMENT "Creating symlink to memgraph executable")
# Emulate the installed python_support, by creating a symlink # Emulate the installed python_support, by creating a symlink
add_custom_command(TARGET memgraph POST_BUILD add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_SOURCE_DIR}/include ${CMAKE_BINARY_DIR}/python_support
BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support BYPRODUCTS ${CMAKE_BINARY_DIR}/python_support
COMMENT "Creating symlink for python_support") COMMENT "Creating symlink for python_support")
# Strip the executable in release build. # Strip the executable in release build.
if (lower_build_type STREQUAL "release") if(lower_build_type STREQUAL "release")
add_custom_command(TARGET memgraph POST_BUILD add_custom_command(TARGET memgraph POST_BUILD
COMMAND strip -s $<TARGET_FILE:memgraph> COMMAND strip -s $<TARGET_FILE:memgraph>
COMMENT "Stripping symbols and sections from memgraph") COMMENT "Stripping symbols and sections from memgraph")
endif() endif()
# Generate the configuration file. # Generate the configuration file.
add_custom_command(TARGET memgraph POST_BUILD add_custom_command(TARGET memgraph POST_BUILD
COMMAND ${CMAKE_SOURCE_DIR}/config/generate.py COMMAND ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_BINARY_DIR}/memgraph ${CMAKE_BINARY_DIR}/memgraph
${CMAKE_BINARY_DIR}/config/memgraph.conf ${CMAKE_BINARY_DIR}/config/memgraph.conf
DEPENDS ${CMAKE_SOURCE_DIR}/config/generate.py DEPENDS ${CMAKE_SOURCE_DIR}/config/generate.py
${CMAKE_SOURCE_DIR}/config/flags.yaml ${CMAKE_SOURCE_DIR}/config/flags.yaml
BYPRODUCTS ${CMAKE_BINARY_DIR}/config/memgraph.conf BYPRODUCTS ${CMAKE_BINARY_DIR}/config/memgraph.conf
COMMENT "Generating memgraph configuration file") COMMENT "Generating memgraph configuration file")
# Everything here is under "memgraph" install component. # Everything here is under "memgraph" install component.
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph") set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "memgraph")
# TODO: Default directory permissions to 755 # TODO: Default directory permissions to 755
# NOTE: This is added in CMake 3.11, so enable it then # NOTE: This is added in CMake 3.11, so enable it then
#set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS # set(CMAKE_INSTALL_DEFAULT_DIRECTORY_PERMISSIONS
# OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ) # OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ WORLD_READ)
# Install and rename executable to just 'memgraph' Since we have to rename, # Install and rename executable to just 'memgraph' Since we have to rename,
# we cannot use the recommended `install(TARGETS ...)`. # we cannot use the recommended `install(TARGETS ...)`.
install(PROGRAMS $<TARGET_FILE:memgraph> install(PROGRAMS $<TARGET_FILE:memgraph>
DESTINATION lib/memgraph RENAME memgraph) DESTINATION lib/memgraph RENAME memgraph)
# Install Python source for supporting our embedded Python. # Install Python source for supporting our embedded Python.
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.py install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.py
DESTINATION lib/memgraph/python_support) DESTINATION lib/memgraph/python_support)
# Install the include file for writing custom procedures. install(FILES ${CMAKE_SOURCE_DIR}/include/mgp_mock.py
DESTINATION lib/memgraph/python_support)
install(FILES ${CMAKE_SOURCE_DIR}/include/_mgp_mock.py
DESTINATION lib/memgraph/python_support)
# Install the includes file for writing custom procedures in C and C++>
install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h install(FILES ${CMAKE_SOURCE_DIR}/include/mg_procedure.h
DESTINATION include/memgraph) DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/_mgp.hpp
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/mg_exceptions.hpp
DESTINATION include/memgraph)
install(FILES ${CMAKE_SOURCE_DIR}/include/mgp.hpp
DESTINATION include/memgraph)
# Install the config file (must use absolute path). # Install the config file (must use absolute path).
install(FILES ${CMAKE_BINARY_DIR}/config/memgraph.conf install(FILES ${CMAKE_BINARY_DIR}/config/memgraph.conf
DESTINATION /etc/memgraph RENAME memgraph.conf) DESTINATION /etc/memgraph RENAME memgraph.conf)
# Install logrotate configuration (must use absolute path). # Install logrotate configuration (must use absolute path).
install(FILES ${CMAKE_SOURCE_DIR}/release/logrotate.conf install(FILES ${CMAKE_SOURCE_DIR}/release/logrotate.conf
DESTINATION /etc/logrotate.d RENAME memgraph) DESTINATION /etc/logrotate.d RENAME memgraph)
@@ -120,15 +132,14 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
# Memgraph CSV Import Tool Executable # Memgraph CSV Import Tool Executable
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
add_executable(mg_import_csv mg_import_csv.cpp) add_executable(mg_import_csv mg_import_csv.cpp)
target_link_libraries(mg_import_csv mg-storage-v2) target_link_libraries(mg_import_csv mg-storage-v2)
# Strip the executable in release build. # Strip the executable in release build.
if (lower_build_type STREQUAL "release") if(lower_build_type STREQUAL "release")
add_custom_command(TARGET mg_import_csv POST_BUILD add_custom_command(TARGET mg_import_csv POST_BUILD
COMMAND strip -s mg_import_csv COMMAND strip -s mg_import_csv
COMMENT "Stripping symbols and sections from mg_import_csv") COMMENT "Stripping symbols and sections from mg_import_csv")
endif() endif()
install(TARGETS mg_import_csv RUNTIME DESTINATION bin) install(TARGETS mg_import_csv RUNTIME DESTINATION bin)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -116,12 +116,12 @@ Log::~Log() {
} }
void Log::Record(const std::string &address, const std::string &username, const std::string &query, void Log::Record(const std::string &address, const std::string &username, const std::string &query,
const storage::PropertyValue &params) { const storage::PropertyValue &params, const std::string &db) {
if (!started_.load(std::memory_order_relaxed)) return; if (!started_.load(std::memory_order_relaxed)) return;
auto timestamp = auto timestamp =
std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()) std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch())
.count(); .count();
buffer_->emplace(Item{timestamp, address, username, query, params}); buffer_->emplace(Item{timestamp, address, username, query, params, db});
} }
void Log::ReopenLog() { void Log::ReopenLog() {
@@ -136,8 +136,8 @@ void Log::Flush() {
for (uint64_t i = 0; i < buffer_size_; ++i) { for (uint64_t i = 0; i < buffer_size_; ++i) {
auto item = buffer_->pop(); auto item = buffer_->pop();
if (!item) break; if (!item) break;
log_.Write(fmt::format("{}.{:06d},{},{},{},{}\n", item->timestamp / 1000000, item->timestamp % 1000000, log_.Write(fmt::format("{}.{:06d},{},{},{},{},{}\n", item->timestamp / 1000000, item->timestamp % 1000000,
item->address, item->username, utils::Escape(item->query), item->address, item->username, item->db, utils::Escape(item->query),
utils::Escape(PropertyValueToJson(item->params).dump()))); utils::Escape(PropertyValueToJson(item->params).dump())));
} }
log_.Sync(); log_.Sync();

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -32,6 +32,7 @@ class Log {
std::string username; std::string username;
std::string query; std::string query;
storage::PropertyValue params; storage::PropertyValue params;
std::string db;
}; };
public: public:
@@ -51,7 +52,7 @@ class Log {
/// Adds an entry to the audit log. Thread-safe. /// Adds an entry to the audit log. Thread-safe.
void Record(const std::string &address, const std::string &username, const std::string &query, void Record(const std::string &address, const std::string &username, const std::string &query,
const storage::PropertyValue &params); const storage::PropertyValue &params, const std::string &db);
/// Reopens the log file. Used for log file rotation. Thread-safe. /// Reopens the log file. Used for log file rotation. Thread-safe.
void ReopenLog(); void ReopenLog();

View File

@@ -16,6 +16,10 @@ target_link_libraries(mg-auth mg-utils mg-kvstore mg-license )
target_link_libraries(mg-auth ${Seccomp_LIBRARIES}) target_link_libraries(mg-auth ${Seccomp_LIBRARIES})
target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS}) target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS})
find_package(OpenSSL REQUIRED)
target_link_libraries(mg-auth ${OPENSSL_LIBRARIES})
target_include_directories(mg-auth SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
# Install reference auth modules and their configuration files. # Install reference auth modules and their configuration files.
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/reference_modules/example.py install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/reference_modules/example.py
DESTINATION lib/memgraph/auth_module) DESTINATION lib/memgraph/auth_module)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -16,8 +16,8 @@
#include <fmt/format.h> #include <fmt/format.h>
#include "auth/exceptions.hpp" #include "auth/exceptions.hpp"
#include "license/license.hpp"
#include "utils/flag_validation.hpp" #include "utils/flag_validation.hpp"
#include "utils/license.hpp"
#include "utils/logging.hpp" #include "utils/logging.hpp"
#include "utils/message.hpp" #include "utils/message.hpp"
#include "utils/settings.hpp" #include "utils/settings.hpp"
@@ -68,10 +68,9 @@ Auth::Auth(const std::string &storage_directory) : storage_(storage_directory),
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) { std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
if (module_.IsUsed()) { if (module_.IsUsed()) {
const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings); const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
if (license_check_result.HasError()) { if (license_check_result.HasError()) {
spdlog::warn( spdlog::warn(license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
return std::nullopt; return std::nullopt;
} }
@@ -226,7 +225,7 @@ std::vector<auth::User> Auth::AllUsers() const {
if (username != utils::ToLowerCase(username)) continue; if (username != utils::ToLowerCase(username)) continue;
auto user = GetUser(username); auto user = GetUser(username);
if (user) { if (user) {
ret.push_back(*user); ret.push_back(std::move(*user));
} }
} }
return ret; return ret;
@@ -306,7 +305,7 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
if (it->second == rolename) { if (it->second == rolename) {
auto user = GetUser(username); auto user = GetUser(username);
if (user) { if (user) {
ret.push_back(*user); ret.push_back(std::move(*user));
} else { } else {
throw AuthException("Couldn't load user '{}'!", username); throw AuthException("Couldn't load user '{}'!", username);
} }
@@ -315,4 +314,57 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
return ret; return ret;
} }
#ifdef MG_ENTERPRISE
bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name) {
auto user = GetUser(name);
if (user) {
if (db == kAllDatabases) {
user->db_access().GrantAll();
} else {
user->db_access().Add(db);
}
SaveUser(*user);
return true;
}
return false;
}
bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name) {
auto user = GetUser(name);
if (user) {
if (db == kAllDatabases) {
user->db_access().DenyAll();
} else {
user->db_access().Remove(db);
}
SaveUser(*user);
return true;
}
return false;
}
void Auth::DeleteDatabase(const std::string &db) {
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
auto username = it->first.substr(kUserPrefix.size());
auto user = GetUser(username);
if (user) {
user->db_access().Delete(db);
SaveUser(*user);
}
}
}
bool Auth::SetMainDatabase(const std::string &db, const std::string &name) {
auto user = GetUser(name);
if (user) {
if (!user->db_access().SetDefault(db)) {
throw AuthException("Couldn't set default database '{}' for user '{}'!", db, name);
}
SaveUser(*user);
return true;
}
return false;
}
#endif
} // namespace memgraph::auth } // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -19,9 +19,12 @@
#include "utils/settings.hpp" #include "utils/settings.hpp"
namespace memgraph::auth { namespace memgraph::auth {
static const constexpr char *const kAllDatabases = "*";
/** /**
* This class serves as the main Authentication/Authorization storage. * This class serves as the main Authentication/Authorization storage.
* It provides functions for managing Users, Roles and Permissions. * It provides functions for managing Users, Roles, Permissions and FineGrainedAccessPermissions.
* NOTE: The non-const functions in this class aren't thread safe. * NOTE: The non-const functions in this class aren't thread safe.
* TODO (mferencevic): Disable user/role modification functions when they are * TODO (mferencevic): Disable user/role modification functions when they are
* being managed by the auth module. * being managed by the auth module.
@@ -155,6 +158,46 @@ class Auth final {
*/ */
std::vector<User> AllUsersForRole(const std::string &rolename) const; std::vector<User> AllUsersForRole(const std::string &rolename) const;
#ifdef MG_ENTERPRISE
/**
* @brief Revoke access to individual database for a user.
*
* @param db name of the database to revoke
* @param name user's username
* @return true on success
* @throw AuthException if unable to find or update the user
*/
bool RevokeDatabaseFromUser(const std::string &db, const std::string &name);
/**
* @brief Grant access to individual database for a user.
*
* @param db name of the database to revoke
* @param name user's username
* @return true on success
* @throw AuthException if unable to find or update the user
*/
bool GrantDatabaseToUser(const std::string &db, const std::string &name);
/**
* @brief Delete a database from all users.
*
* @param db name of the database to delete
* @throw AuthException if unable to read data
*/
void DeleteDatabase(const std::string &db);
/**
* @brief Set main database for an individual user.
*
* @param db name of the database to revoke
* @param name user's username
* @return true on success
* @throw AuthException if unable to find or update the user
*/
bool SetMainDatabase(const std::string &db, const std::string &name);
#endif
private: private:
// Even though the `kvstore::KVStore` class is guaranteed to be thread-safe, // Even though the `kvstore::KVStore` class is guaranteed to be thread-safe,
// Auth is not thread-safe because modifying users and roles might require // Auth is not thread-safe because modifying users and roles might require

View File

@@ -1,19 +1,64 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
// this file except in compliance with the License. You may obtain a copy of the License at https://memgraph.com/legal. // this file except in compliance with the License. You may obtain a copy of the License at https://memgraph.com/legal.
// //
// //
#include "auth/crypto.hpp" #include "auth/crypto.hpp"
#include <iomanip>
#include <sstream>
#include <gflags/gflags.h>
#include <libbcrypt/bcrypt.h> #include <libbcrypt/bcrypt.h>
#include <openssl/evp.h>
#include <openssl/opensslv.h>
#include <openssl/sha.h>
#include "auth/exceptions.hpp" #include "auth/exceptions.hpp"
#include "utils/enum.hpp"
#include "utils/flag_validation.hpp"
namespace {
using namespace std::literals;
inline constexpr std::array password_encryption_mappings{
std::pair{"bcrypt"sv, memgraph::auth::PasswordEncryptionAlgorithm::BCRYPT},
std::pair{"sha256"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256},
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256_MULTIPLE}};
inline constexpr uint64_t ONE_SHA_ITERATION = 1;
inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
} // namespace
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,misc-unused-parameters)
DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
"The password encryption algorithm used for authentication.", {
if (const auto result =
memgraph::utils::IsValidEnumValueString(value, password_encryption_mappings);
result.HasError()) {
const auto error = result.GetError();
switch (error) {
case memgraph::utils::ValidationError::EmptyValue: {
std::cout << "Password encryption algorithm cannot be empty." << std::endl;
break;
}
case memgraph::utils::ValidationError::InvalidValue: {
std::cout << "Invalid value for password encryption algorithm. Allowed values: "
<< memgraph::utils::GetAllowedEnumValuesString(password_encryption_mappings)
<< std::endl;
break;
}
}
return false;
}
return true;
});
namespace memgraph::auth { namespace memgraph::auth {
const std::string EncryptPassword(const std::string &password) { namespace BCrypt {
std::string EncryptPassword(const std::string &password) {
char salt[BCRYPT_HASHSIZE]; char salt[BCRYPT_HASHSIZE];
char hash[BCRYPT_HASHSIZE]; char hash[BCRYPT_HASHSIZE];
@@ -28,7 +73,7 @@ const std::string EncryptPassword(const std::string &password) {
throw AuthException("Couldn't hash password!"); throw AuthException("Couldn't hash password!");
} }
return std::string(hash); return {hash};
} }
bool VerifyPassword(const std::string &password, const std::string &hash) { bool VerifyPassword(const std::string &password, const std::string &hash) {
@@ -38,5 +83,102 @@ bool VerifyPassword(const std::string &password, const std::string &hash) {
} }
return ret == 0; return ret == 0;
} }
} // namespace BCrypt
namespace SHA {
#if OPENSSL_VERSION_MAJOR >= 3
std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t number_of_iterations) {
unsigned char hash[SHA256_DIGEST_LENGTH];
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
EVP_MD *md = EVP_MD_fetch(nullptr, "SHA2-256", nullptr);
EVP_DigestInit_ex(ctx, md, nullptr);
for (auto i = 0; i < number_of_iterations; i++) {
EVP_DigestUpdate(ctx, password.c_str(), password.size());
}
EVP_DigestFinal_ex(ctx, hash, nullptr);
EVP_MD_free(md);
EVP_MD_CTX_free(ctx);
std::stringstream result_stream;
for (auto hash_char : hash) {
result_stream << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
}
return result_stream.str();
}
#else
std::string EncryptPasswordOpenSSL1_1(const std::string &password, const uint64_t number_of_iterations) {
unsigned char hash[SHA256_DIGEST_LENGTH];
SHA256_CTX sha256;
SHA256_Init(&sha256);
for (auto i = 0; i < number_of_iterations; i++) {
SHA256_Update(&sha256, password.c_str(), password.size());
}
SHA256_Final(hash, &sha256);
std::stringstream ss;
for (auto hash_char : hash) {
ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
}
return ss.str();
}
#endif
std::string EncryptPassword(const std::string &password, const uint64_t number_of_iterations) {
#if OPENSSL_VERSION_MAJOR >= 3
return EncryptPasswordOpenSSL3(password, number_of_iterations);
#else
return EncryptPasswordOpenSSL1_1(password, number_of_iterations);
#endif
}
bool VerifyPassword(const std::string &password, const std::string &hash, const uint64_t number_of_iterations) {
auto password_hash = EncryptPassword(password, number_of_iterations);
return password_hash == hash;
}
} // namespace SHA
bool VerifyPassword(const std::string &password, const std::string &hash) {
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
FLAGS_password_encryption_algorithm, password_encryption_mappings);
if (!password_encryption_algorithm.has_value()) {
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
switch (password_encryption_algorithm.value()) {
case PasswordEncryptionAlgorithm::BCRYPT:
return BCrypt::VerifyPassword(password, hash);
case PasswordEncryptionAlgorithm::SHA256:
return SHA::VerifyPassword(password, hash, ONE_SHA_ITERATION);
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
return SHA::VerifyPassword(password, hash, MULTIPLE_SHA_ITERATIONS);
}
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
std::string EncryptPassword(const std::string &password) {
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
FLAGS_password_encryption_algorithm, password_encryption_mappings);
if (!password_encryption_algorithm.has_value()) {
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
}
switch (password_encryption_algorithm.value()) {
case PasswordEncryptionAlgorithm::BCRYPT:
return BCrypt::EncryptPassword(password);
case PasswordEncryptionAlgorithm::SHA256:
return SHA::EncryptPassword(password, ONE_SHA_ITERATION);
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
return SHA::EncryptPassword(password, MULTIPLE_SHA_ITERATIONS);
}
}
} // namespace memgraph::auth } // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -11,10 +11,11 @@
#include <string> #include <string>
namespace memgraph::auth { namespace memgraph::auth {
enum class PasswordEncryptionAlgorithm : uint8_t { BCRYPT, SHA256, SHA256_MULTIPLE };
/// @throw AuthException if unable to encrypt the password. /// @throw AuthException if unable to encrypt the password.
const std::string EncryptPassword(const std::string &password); std::string EncryptPassword(const std::string &password);
/// @throw AuthException if unable to verify the password. /// @throw AuthException if unable to verify the password.
bool VerifyPassword(const std::string &password, const std::string &hash); bool VerifyPassword(const std::string &password, const std::string &hash);
} // namespace memgraph::auth } // namespace memgraph::auth

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -8,14 +8,19 @@
#include "auth/models.hpp" #include "auth/models.hpp"
#include <cstdint>
#include <regex> #include <regex>
#include <gflags/gflags.h> #include <gflags/gflags.h>
#include "auth/crypto.hpp" #include "auth/crypto.hpp"
#include "auth/exceptions.hpp" #include "auth/exceptions.hpp"
#include "dbms/constants.hpp"
#include "license/license.hpp"
#include "query/constants.hpp"
#include "spdlog/spdlog.h"
#include "utils/cast.hpp" #include "utils/cast.hpp"
#include "utils/license.hpp" #include "utils/logging.hpp"
#include "utils/settings.hpp" #include "utils/settings.hpp"
#include "utils/string.hpp" #include "utils/string.hpp"
@@ -30,14 +35,34 @@ DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
namespace memgraph::auth { namespace memgraph::auth {
namespace { namespace {
// Constant list of all available permissions. // Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = { const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
Permission::MATCH, Permission::CREATE, Permission::MERGE, Permission::DELETE, Permission::CREATE,
Permission::SET, Permission::REMOVE, Permission::INDEX, Permission::STATS, Permission::MERGE,
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION, Permission::DELETE,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER, Permission::SET,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE, Permission::REMOVE,
Permission::WEBSOCKET}; Permission::INDEX,
Permission::STATS,
Permission::CONSTRAINT,
Permission::DUMP,
Permission::AUTH,
Permission::REPLICATION,
Permission::DURABILITY,
Permission::READ_FILE,
Permission::FREE_MEMORY,
Permission::TRIGGER,
Permission::CONFIG,
Permission::STREAM,
Permission::MODULE_READ,
Permission::MODULE_WRITE,
Permission::WEBSOCKET,
Permission::TRANSACTION_MANAGEMENT,
Permission::STORAGE_MODE,
Permission::MULTI_DATABASE_EDIT,
Permission::MULTI_DATABASE_USE};
} // namespace } // namespace
std::string PermissionToString(Permission permission) { std::string PermissionToString(Permission permission) {
@@ -84,8 +109,14 @@ std::string PermissionToString(Permission permission) {
return "MODULE_WRITE"; return "MODULE_WRITE";
case Permission::WEBSOCKET: case Permission::WEBSOCKET:
return "WEBSOCKET"; return "WEBSOCKET";
case Permission::LABELS: case Permission::TRANSACTION_MANAGEMENT:
return "LABELS"; return "TRANSACTION_MANAGEMENT";
case Permission::STORAGE_MODE:
return "STORAGE_MODE";
case Permission::MULTI_DATABASE_EDIT:
return "MULTI_DATABASE_EDIT";
case Permission::MULTI_DATABASE_USE:
return "MULTI_DATABASE_USE";
} }
} }
@@ -100,6 +131,55 @@ std::string PermissionLevelToString(PermissionLevel level) {
} }
} }
#ifdef MG_ENTERPRISE
FineGrainedPermission PermissionToFineGrainedPermission(const uint64_t permission) {
if (permission & FineGrainedPermission::CREATE_DELETE) {
return FineGrainedPermission::CREATE_DELETE;
}
if (permission & FineGrainedPermission::UPDATE) {
return FineGrainedPermission::UPDATE;
}
if (permission & FineGrainedPermission::READ) {
return FineGrainedPermission::READ;
}
return FineGrainedPermission::NOTHING;
}
std::string FineGrainedPermissionToString(const FineGrainedPermission level) {
switch (level) {
case FineGrainedPermission::CREATE_DELETE:
return "CREATE_DELETE";
case FineGrainedPermission::UPDATE:
return "UPDATE";
case FineGrainedPermission::READ:
return "READ";
case FineGrainedPermission::NOTHING:
return "NOTHING";
}
}
FineGrainedAccessPermissions Merge(const FineGrainedAccessPermissions &first,
const FineGrainedAccessPermissions &second) {
std::unordered_map<std::string, uint64_t> permissions{first.GetPermissions()};
std::optional<uint64_t> global_permission;
if (second.GetGlobalPermission().has_value()) {
global_permission = *second.GetGlobalPermission();
} else if (first.GetGlobalPermission().has_value()) {
global_permission = *first.GetGlobalPermission();
}
for (const auto &[label_name, permission] : second.GetPermissions()) {
permissions[label_name] = permission;
}
return FineGrainedAccessPermissions(permissions, global_permission);
}
#endif
Permissions::Permissions(uint64_t grants, uint64_t denies) { Permissions::Permissions(uint64_t grants, uint64_t denies) {
// The deny bitmask has higher priority than the grant bitmask. // The deny bitmask has higher priority than the grant bitmask.
denies_ = denies; denies_ = denies;
@@ -173,7 +253,7 @@ Permissions Permissions::Deserialize(const nlohmann::json &data) {
if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) { if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) {
throw AuthException("Couldn't load permissions data!"); throw AuthException("Couldn't load permissions data!");
} }
return {data["grants"], data["denies"]}; return Permissions{data["grants"], data["denies"]};
} }
uint64_t Permissions::grants() const { return grants_; } uint64_t Permissions::grants() const { return grants_; }
@@ -185,107 +265,191 @@ bool operator==(const Permissions &first, const Permissions &second) {
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); } bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
LabelPermissions::LabelPermissions(const std::unordered_set<std::string> &grants, #ifdef MG_ENTERPRISE
const std::unordered_set<std::string> &denies) FineGrainedAccessPermissions::FineGrainedAccessPermissions(const std::unordered_map<std::string, uint64_t> &permissions,
: grants_(grants), denies_(denies) {} const std::optional<uint64_t> &global_permission)
: permissions_(permissions), global_permission_(global_permission) {}
PermissionLevel LabelPermissions::Has(const std::string &permission) const { PermissionLevel FineGrainedAccessPermissions::Has(const std::string &permission,
if (denies_.find(permission) != denies_.end()) { const FineGrainedPermission fine_grained_permission) const {
return PermissionLevel::DENY; if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
}
if (grants_.find(permission) != denies_.end()) {
return PermissionLevel::GRANT; return PermissionLevel::GRANT;
} }
const auto concrete_permission = std::invoke([&]() -> uint64_t {
if (permissions_.contains(permission)) {
return permissions_.at(permission);
}
return PermissionLevel::NEUTRAL; if (global_permission_.has_value()) {
return global_permission_.value();
}
return 0;
});
const auto temp_permission = concrete_permission & fine_grained_permission;
return temp_permission > 0 ? PermissionLevel::GRANT : PermissionLevel::DENY;
} }
void LabelPermissions::Grant(const std::string &permission) { void FineGrainedAccessPermissions::Grant(const std::string &permission,
auto deniedPermissionIter = denies_.find(permission); const FineGrainedPermission fine_grained_permission) {
if (permission == query::kAsterisk) {
if (deniedPermissionIter != denies_.end()) { global_permission_ = CalculateGrant(fine_grained_permission);
denies_.erase(deniedPermissionIter); } else {
} permissions_[permission] = CalculateGrant(fine_grained_permission);
if (grants_.find(permission) == grants_.end()) {
grants_.insert(permission);
} }
} }
void LabelPermissions::Revoke(const std::string &permission) { void FineGrainedAccessPermissions::Revoke(const std::string &permission) {
auto deniedPermissionIter = denies_.find(permission); if (permission == query::kAsterisk) {
auto grantedPermissionIter = grants_.find(permission); permissions_.clear();
global_permission_ = std::nullopt;
if (deniedPermissionIter != denies_.end()) { } else {
denies_.erase(deniedPermissionIter); permissions_.erase(permission);
}
if (grantedPermissionIter != grants_.end()) {
grants_.erase(grantedPermissionIter);
} }
} }
void LabelPermissions::Deny(const std::string &permission) { nlohmann::json FineGrainedAccessPermissions::Serialize() const {
auto grantedPermissionIter = grants_.find(permission); if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
if (grantedPermissionIter != grants_.end()) {
grants_.erase(grantedPermissionIter);
} }
if (denies_.find(permission) == denies_.end()) {
denies_.insert(permission);
}
}
std::unordered_set<std::string> LabelPermissions::GetGrants() const { return grants_; }
std::unordered_set<std::string> LabelPermissions::GetDenies() const { return denies_; }
nlohmann::json LabelPermissions::Serialize() const {
nlohmann::json data = nlohmann::json::object(); nlohmann::json data = nlohmann::json::object();
data["grants"] = grants_; data["permissions"] = permissions_;
data["denies"] = denies_; data["global_permission"] = global_permission_.has_value() ? global_permission_.value() : -1;
return data; return data;
} }
LabelPermissions LabelPermissions::Deserialize(const nlohmann::json &data) { FineGrainedAccessPermissions FineGrainedAccessPermissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) { if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!"); throw AuthException("Couldn't load permissions data!");
} }
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
std::optional<uint64_t> global_permission;
return {LabelPermissions(data["grants"], data["denies"])}; if (data["global_permission"].empty() || data["global_permission"] == -1) {
global_permission = std::nullopt;
} else {
global_permission = data["global_permission"];
}
return FineGrainedAccessPermissions(data["permissions"], global_permission);
} }
std::unordered_set<std::string> LabelPermissions::grants() const { return grants_; } const std::unordered_map<std::string, uint64_t> &FineGrainedAccessPermissions::GetPermissions() const {
std::unordered_set<std::string> LabelPermissions::denies() const { return denies_; } return permissions_;
}
const std::optional<uint64_t> &FineGrainedAccessPermissions::GetGlobalPermission() const { return global_permission_; };
bool operator==(const LabelPermissions &first, const LabelPermissions &second) { uint64_t FineGrainedAccessPermissions::CalculateGrant(FineGrainedPermission fine_grained_permission) {
return first.grants() == second.grants() && first.denies() == second.denies(); uint64_t shift{1};
uint64_t result{0};
auto uint_fine_grained_permission = static_cast<uint64_t>(fine_grained_permission);
while (uint_fine_grained_permission > 0) {
result |= uint_fine_grained_permission;
uint_fine_grained_permission >>= shift;
}
return result;
} }
bool operator!=(const LabelPermissions &first, const LabelPermissions &second) { return !(first == second); } bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return first.GetPermissions() == second.GetPermissions() &&
first.GetGlobalPermission() == second.GetGlobalPermission();
}
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return !(first == second);
}
FineGrainedAccessHandler::FineGrainedAccessHandler(FineGrainedAccessPermissions labelPermissions,
FineGrainedAccessPermissions edgeTypePermissions)
: label_permissions_(std::move(labelPermissions)), edge_type_permissions_(std::move(edgeTypePermissions)) {}
const FineGrainedAccessPermissions &FineGrainedAccessHandler::label_permissions() const { return label_permissions_; }
FineGrainedAccessPermissions &FineGrainedAccessHandler::label_permissions() { return label_permissions_; }
const FineGrainedAccessPermissions &FineGrainedAccessHandler::edge_type_permissions() const {
return edge_type_permissions_;
}
FineGrainedAccessPermissions &FineGrainedAccessHandler::edge_type_permissions() { return edge_type_permissions_; }
nlohmann::json FineGrainedAccessHandler::Serialize() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
nlohmann::json data = nlohmann::json::object();
data["label_permissions"] = label_permissions_.Serialize();
data["edge_type_permissions"] = edge_type_permissions_.Serialize();
return data;
}
FineGrainedAccessHandler FineGrainedAccessHandler::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load role data!");
}
if (!data["label_permissions"].is_object() || !data["edge_type_permissions"].is_object()) {
throw AuthException("Couldn't load label_permissions or edge_type_permissions data!");
}
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessHandler{};
}
auto label_permissions = FineGrainedAccessPermissions::Deserialize(data["label_permissions"]);
auto edge_type_permissions = FineGrainedAccessPermissions::Deserialize(data["edge_type_permissions"]);
return FineGrainedAccessHandler(std::move(label_permissions), std::move(edge_type_permissions));
}
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second) {
return first.label_permissions_ == second.label_permissions_ &&
first.edge_type_permissions_ == second.edge_type_permissions_;
}
bool operator!=(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second) {
return !(first == second);
}
#endif
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {} Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename, const Permissions &permissions) Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {} : rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
Role::Role(const std::string &rolename, const Permissions &permissions, const LabelPermissions &labelPermissions) Role::Role(const std::string &rolename, const Permissions &permissions,
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions), labelPermissions_(labelPermissions) {} FineGrainedAccessHandler fine_grained_access_handler)
: rolename_(utils::ToLowerCase(rolename)),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
#endif
const std::string &Role::rolename() const { return rolename_; } const std::string &Role::rolename() const { return rolename_; }
const Permissions &Role::permissions() const { return permissions_; } const Permissions &Role::permissions() const { return permissions_; }
Permissions &Role::permissions() { return permissions_; } Permissions &Role::permissions() { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &Role::fine_grained_access_handler() const { return fine_grained_access_handler_; }
FineGrainedAccessHandler &Role::fine_grained_access_handler() { return fine_grained_access_handler_; }
LabelPermissions &Role::labelPermissions() { return labelPermissions_; } const FineGrainedAccessPermissions &Role::GetFineGrainedAccessLabelPermissions() const {
return fine_grained_access_handler_.label_permissions();
}
const FineGrainedAccessPermissions &Role::GetFineGrainedAccessEdgeTypePermissions() const {
return fine_grained_access_handler_.edge_type_permissions();
}
#endif
nlohmann::json Role::Serialize() const { nlohmann::json Role::Serialize() const {
nlohmann::json data = nlohmann::json::object(); nlohmann::json data = nlohmann::json::object();
data["rolename"] = rolename_; data["rolename"] = rolename_;
data["permissions"] = permissions_.Serialize(); data["permissions"] = permissions_.Serialize();
data["labelPermissions"] = labelPermissions_.Serialize(); #ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
} else {
data["fine_grained_access_handler"] = {};
}
#endif
return data; return data;
} }
@@ -297,26 +461,120 @@ Role Role::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load role data!"); throw AuthException("Couldn't load role data!");
} }
auto permissions = Permissions::Deserialize(data["permissions"]); auto permissions = Permissions::Deserialize(data["permissions"]);
auto labelPermissions = LabelPermissions::Deserialize(data["labelPermissions"]); #ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {data["rolename"], permissions, labelPermissions}; FineGrainedAccessHandler fine_grained_access_handler;
// We can have an empty fine_grained if the user was created without a valid license
if (data["fine_grained_access_handler"].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
}
return {data["rolename"], permissions, std::move(fine_grained_access_handler)};
}
#endif
return {data["rolename"], permissions};
} }
bool operator==(const Role &first, const Role &second) { bool operator==(const Role &first, const Role &second) {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_ &&
first.fine_grained_access_handler_ == second.fine_grained_access_handler_;
}
#endif
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_; return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_;
} }
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {} #ifdef MG_ENTERPRISE
void Databases::Add(const std::string &db) {
if (allow_all_) {
grants_dbs_.clear();
allow_all_ = false;
}
grants_dbs_.emplace(db);
denies_dbs_.erase(db);
}
void Databases::Remove(const std::string &db) {
denies_dbs_.emplace(db);
grants_dbs_.erase(db);
}
void Databases::Delete(const std::string &db) {
denies_dbs_.erase(db);
if (!allow_all_) {
grants_dbs_.erase(db);
}
// Reset if default deleted
if (default_db_ == db) {
default_db_ = "";
}
}
void Databases::GrantAll() {
allow_all_ = true;
grants_dbs_.clear();
denies_dbs_.clear();
}
void Databases::DenyAll() {
allow_all_ = false;
grants_dbs_.clear();
denies_dbs_.clear();
}
bool Databases::SetDefault(const std::string &db) {
if (!Contains(db)) return false;
default_db_ = db;
return true;
}
[[nodiscard]] bool Databases::Contains(const std::string &db) const {
return !denies_dbs_.contains(db) && (allow_all_ || grants_dbs_.contains(db));
}
const std::string &Databases::GetDefault() const {
if (!Contains(default_db_)) {
throw AuthException("No access to the set default database \"{}\".", default_db_);
}
return default_db_;
}
nlohmann::json Databases::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["grants"] = grants_dbs_;
data["denies"] = denies_dbs_;
data["allow_all"] = allow_all_;
data["default"] = default_db_;
return data;
}
Databases Databases::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load database data!");
}
if (!data["grants"].is_structured() || !data["denies"].is_structured() || !data["allow_all"].is_boolean() ||
!data["default"].is_string()) {
throw AuthException("Couldn't load database data!");
}
return {data["allow_all"], data["grants"], data["denies"], data["default"]};
}
#endif
User::User() {}
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions) User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {} : username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions, User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
const LabelPermissions &labelPermissions) FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
: username_(utils::ToLowerCase(username)), : username_(utils::ToLowerCase(username)),
password_hash_(password_hash), password_hash_(password_hash),
permissions_(permissions), permissions_(permissions),
labelPermissions_(labelPermissions) {} fine_grained_access_handler_(std::move(fine_grained_access_handler)),
database_access_(db_access) {}
#endif
bool User::CheckPassword(const std::string &password) { bool User::CheckPassword(const std::string &password) {
if (password_hash_.empty()) return true; if (password_hash_.empty()) return true;
@@ -333,13 +591,13 @@ void User::UpdatePassword(const std::optional<std::string> &password) {
} }
if (FLAGS_auth_password_strength_regex != default_password_regex) { if (FLAGS_auth_password_strength_regex != default_password_regex) {
if (const auto license_check_result = utils::license::global_license_checker.IsValidLicense(utils::global_settings); if (const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
license_check_result.HasError()) { license_check_result.HasError()) {
throw AuthException( throw AuthException(
"Custom password regex is a Memgraph Enterprise feature. Please set the config " "Custom password regex is a Memgraph Enterprise feature. Please set the config "
"(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}", "(\"--auth-password-strength-regex\") to its default value (\"{}\") or remove the flag.\n{}",
default_password_regex, default_password_regex,
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex")); license::LicenseCheckErrorToString(license_check_result.GetError(), "password regex"));
} }
} }
std::regex re(FLAGS_auth_password_strength_regex); std::regex re(FLAGS_auth_password_strength_regex);
@@ -359,19 +617,47 @@ void User::ClearRole() { role_ = std::nullopt; }
Permissions User::GetPermissions() const { Permissions User::GetPermissions() const {
if (role_) { if (role_) {
return Permissions(permissions_.grants() | role_->permissions().grants(), return Permissions{permissions_.grants() | role_->permissions().grants(),
permissions_.denies() | role_->permissions().denies()); permissions_.denies() | role_->permissions().denies()};
} }
return permissions_; return permissions_;
} }
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions User::GetFineGrainedAccessLabelPermissions() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
if (role_) {
return Merge(role()->fine_grained_access_handler().label_permissions(),
fine_grained_access_handler_.label_permissions());
}
return fine_grained_access_handler_.label_permissions();
}
FineGrainedAccessPermissions User::GetFineGrainedAccessEdgeTypePermissions() const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return FineGrainedAccessPermissions{};
}
if (role_) {
return Merge(role()->fine_grained_access_handler().edge_type_permissions(),
fine_grained_access_handler_.edge_type_permissions());
}
return fine_grained_access_handler_.edge_type_permissions();
}
#endif
const std::string &User::username() const { return username_; } const std::string &User::username() const { return username_; }
const Permissions &User::permissions() const { return permissions_; } const Permissions &User::permissions() const { return permissions_; }
Permissions &User::permissions() { return permissions_; } Permissions &User::permissions() { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &User::fine_grained_access_handler() const { return fine_grained_access_handler_; }
LabelPermissions &User::labelPermissions() { return labelPermissions_; } FineGrainedAccessHandler &User::fine_grained_access_handler() { return fine_grained_access_handler_; }
#endif
const Role *User::role() const { const Role *User::role() const {
if (role_.has_value()) { if (role_.has_value()) {
return &role_.value(); return &role_.value();
@@ -384,7 +670,15 @@ nlohmann::json User::Serialize() const {
data["username"] = username_; data["username"] = username_;
data["password_hash"] = password_hash_; data["password_hash"] = password_hash_;
data["permissions"] = permissions_.Serialize(); data["permissions"] = permissions_.Serialize();
data["labelPermissions"] = labelPermissions_.Serialize(); #ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data["fine_grained_access_handler"] = fine_grained_access_handler_.Serialize();
data["databases"] = database_access_.Serialize();
} else {
data["fine_grained_access_handler"] = {};
data["databases"] = {};
}
#endif
// The role shouldn't be serialized here, it is stored as a foreign key. // The role shouldn't be serialized here, it is stored as a foreign key.
return data; return data;
} }
@@ -397,12 +691,36 @@ User User::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load user data!"); throw AuthException("Couldn't load user data!");
} }
auto permissions = Permissions::Deserialize(data["permissions"]); auto permissions = Permissions::Deserialize(data["permissions"]);
auto labelPermissions = LabelPermissions::Deserialize(data["labelPermissions"]); #ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {data["username"], data["password_hash"], permissions, labelPermissions}; Databases db_access;
if (data["databases"].is_structured()) {
db_access = Databases::Deserialize(data["databases"]);
} else {
// Back-compatibility
spdlog::warn("User without specified database access. Given access to the default database.");
db_access.Add(dbms::kDefaultDB);
db_access.SetDefault(dbms::kDefaultDB);
}
FineGrainedAccessHandler fine_grained_access_handler;
// We can have an empty fine_grained if the user was created without a valid license
if (data["fine_grained_access_handler"].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data["fine_grained_access_handler"]);
}
return {data["username"], data["password_hash"], permissions, std::move(fine_grained_access_handler), db_access};
}
#endif
return {data["username"], data["password_hash"], permissions};
} }
bool operator==(const User &first, const User &second) { bool operator==(const User &first, const User &second) {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_ &&
first.fine_grained_access_handler_ == second.fine_grained_access_handler_;
}
#endif
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ && return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_; first.permissions_ == second.permissions_ && first.role_ == second.role_;
} }

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise // Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use // License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -9,10 +9,13 @@
#pragma once #pragma once
#include <optional> #include <optional>
#include <set>
#include <string> #include <string>
#include <unordered_map>
#include <json/json.hpp> #include <json/json.hpp>
#include <unordered_set> #include "dbms/constants.hpp"
#include "utils/logging.hpp"
namespace memgraph::auth { namespace memgraph::auth {
// These permissions must have values that are applicable for usage in a // These permissions must have values that are applicable for usage in a
@@ -40,26 +43,68 @@ enum class Permission : uint64_t {
MODULE_READ = 1U << 18U, MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U, MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U, WEBSOCKET = 1U << 20U,
LABELS = 1U << 21U TRANSACTION_MANAGEMENT = 1U << 21U,
STORAGE_MODE = 1U << 22U,
MULTI_DATABASE_EDIT = 1U << 23U,
MULTI_DATABASE_USE = 1U << 24U,
}; };
// clang-format on // clang-format on
#ifdef MG_ENTERPRISE
// clang-format off
enum class FineGrainedPermission : uint64_t {
NOTHING = 0,
READ = 1,
UPDATE = 1U << 1U,
CREATE_DELETE = 1U << 2U
};
// clang-format on
constexpr inline uint64_t operator|(FineGrainedPermission lhs, FineGrainedPermission rhs) {
return static_cast<uint64_t>(lhs) | static_cast<uint64_t>(rhs);
}
constexpr inline uint64_t operator|(uint64_t lhs, FineGrainedPermission rhs) {
return lhs | static_cast<uint64_t>(rhs);
}
constexpr inline uint64_t operator&(uint64_t lhs, FineGrainedPermission rhs) {
return (lhs & static_cast<uint64_t>(rhs)) != 0;
}
constexpr uint64_t kLabelPermissionAll = memgraph::auth::FineGrainedPermission::CREATE_DELETE |
memgraph::auth::FineGrainedPermission::UPDATE |
memgraph::auth::FineGrainedPermission::READ;
constexpr uint64_t kLabelPermissionMax = static_cast<uint64_t>(memgraph::auth::FineGrainedPermission::CREATE_DELETE);
constexpr uint64_t kLabelPermissionMin = static_cast<uint64_t>(memgraph::auth::FineGrainedPermission::READ);
#endif
// Function that converts a permission to its string representation. // Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission); std::string PermissionToString(Permission permission);
// Class that indicates what permission level the user/role has. // Class that indicates what permission level the user/role has.
enum class PermissionLevel { enum class PermissionLevel : uint8_t { GRANT, NEUTRAL, DENY };
GRANT,
NEUTRAL,
DENY,
};
// Function that converts a permission level to its string representation. // Function that converts a permission level to its string representation.
std::string PermissionLevelToString(PermissionLevel level); std::string PermissionLevelToString(PermissionLevel level);
#ifdef MG_ENTERPRISE
// Function that converts a label permission level to its string representation.
std::string FineGrainedPermissionToString(FineGrainedPermission level);
// Constructs a label permission from a permission
FineGrainedPermission PermissionToFineGrainedPermission(uint64_t permission);
#endif
class Permissions final { class Permissions final {
public: public:
Permissions(uint64_t grants = 0, uint64_t denies = 0); explicit Permissions(uint64_t grants = 0, uint64_t denies = 0);
Permissions(const Permissions &) = default;
Permissions &operator=(const Permissions &) = default;
Permissions(Permissions &&) noexcept = default;
Permissions &operator=(Permissions &&) noexcept = default;
~Permissions() = default;
PermissionLevel Has(Permission permission) const; PermissionLevel Has(Permission permission) const;
@@ -90,52 +135,96 @@ bool operator==(const Permissions &first, const Permissions &second);
bool operator!=(const Permissions &first, const Permissions &second); bool operator!=(const Permissions &first, const Permissions &second);
class LabelPermissions final { #ifdef MG_ENTERPRISE
class FineGrainedAccessPermissions final {
public: public:
LabelPermissions(const std::unordered_set<std::string> &grants = {}, explicit FineGrainedAccessPermissions(const std::unordered_map<std::string, uint64_t> &permissions = {},
const std::unordered_set<std::string> &denies = {}); const std::optional<uint64_t> &global_permission = std::nullopt);
FineGrainedAccessPermissions(const FineGrainedAccessPermissions &) = default;
FineGrainedAccessPermissions &operator=(const FineGrainedAccessPermissions &) = default;
FineGrainedAccessPermissions(FineGrainedAccessPermissions &&) = default;
FineGrainedAccessPermissions &operator=(FineGrainedAccessPermissions &&) = default;
~FineGrainedAccessPermissions() = default;
PermissionLevel Has(const std::string &permission, FineGrainedPermission fine_grained_permission) const;
PermissionLevel Has(const std::string &permission) const; void Grant(const std::string &permission, FineGrainedPermission fine_grained_permission);
void Grant(const std::string &permission);
void Revoke(const std::string &permission); void Revoke(const std::string &permission);
void Deny(const std::string &permission);
std::unordered_set<std::string> GetGrants() const;
std::unordered_set<std::string> GetDenies() const;
nlohmann::json Serialize() const; nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize. /// @throw AuthException if unable to deserialize.
static LabelPermissions Deserialize(const nlohmann::json &data); static FineGrainedAccessPermissions Deserialize(const nlohmann::json &data);
std::unordered_set<std::string> grants() const; const std::unordered_map<std::string, uint64_t> &GetPermissions() const;
std::unordered_set<std::string> denies() const; const std::optional<uint64_t> &GetGlobalPermission() const;
private: private:
std::unordered_set<std::string> grants_{}; std::unordered_map<std::string, uint64_t> permissions_{};
std::unordered_set<std::string> denies_{}; std::optional<uint64_t> global_permission_;
static uint64_t CalculateGrant(FineGrainedPermission fine_grained_permission);
}; };
bool operator==(const LabelPermissions &first, const LabelPermissions &second); bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
class FineGrainedAccessHandler final {
public:
explicit FineGrainedAccessHandler(FineGrainedAccessPermissions labelPermissions = FineGrainedAccessPermissions(),
FineGrainedAccessPermissions edgeTypePermissions = FineGrainedAccessPermissions());
FineGrainedAccessHandler(const FineGrainedAccessHandler &) = default;
FineGrainedAccessHandler &operator=(const FineGrainedAccessHandler &) = default;
FineGrainedAccessHandler(FineGrainedAccessHandler &&) noexcept = default;
FineGrainedAccessHandler &operator=(FineGrainedAccessHandler &&) noexcept = default;
~FineGrainedAccessHandler() = default;
const FineGrainedAccessPermissions &label_permissions() const;
FineGrainedAccessPermissions &label_permissions();
const FineGrainedAccessPermissions &edge_type_permissions() const;
FineGrainedAccessPermissions &edge_type_permissions();
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static FineGrainedAccessHandler Deserialize(const nlohmann::json &data);
friend bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
private:
FineGrainedAccessPermissions label_permissions_;
FineGrainedAccessPermissions edge_type_permissions_;
};
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
#endif
bool operator!=(const LabelPermissions &first, const LabelPermissions &second);
class Role final { class Role final {
public: public:
Role(const std::string &rolename); explicit Role(const std::string &rolename);
Role(const std::string &rolename, const Permissions &permissions); Role(const std::string &rolename, const Permissions &permissions);
#ifdef MG_ENTERPRISE
Role(const std::string &rolename, const Permissions &permissions, const LabelPermissions &labelPermissions); Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler);
#endif
Role(const Role &) = default;
Role &operator=(const Role &) = default;
Role(Role &&) noexcept = default;
Role &operator=(Role &&) noexcept = default;
~Role() = default;
const std::string &rolename() const; const std::string &rolename() const;
const Permissions &permissions() const; const Permissions &permissions() const;
Permissions &permissions(); Permissions &permissions();
#ifdef MG_ENTERPRISE
LabelPermissions &labelPermissions(); const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
const FineGrainedAccessPermissions &GetFineGrainedAccessLabelPermissions() const;
const FineGrainedAccessPermissions &GetFineGrainedAccessEdgeTypePermissions() const;
#endif
nlohmann::json Serialize() const; nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize. /// @throw AuthException if unable to deserialize.
@@ -146,20 +235,108 @@ class Role final {
private: private:
std::string rolename_; std::string rolename_;
Permissions permissions_; Permissions permissions_;
LabelPermissions labelPermissions_; #ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
#endif
}; };
bool operator==(const Role &first, const Role &second); bool operator==(const Role &first, const Role &second);
#ifdef MG_ENTERPRISE
class Databases final {
public:
Databases() : grants_dbs_({dbms::kDefaultDB}), allow_all_(false), default_db_(dbms::kDefaultDB) {}
Databases(const Databases &) = default;
Databases &operator=(const Databases &) = default;
Databases(Databases &&) noexcept = default;
Databases &operator=(Databases &&) noexcept = default;
~Databases() = default;
/**
* @brief Add database to the list of granted access. @note allow_all_ will be false after execution
*
* @param db name of the database to grant access to
*/
void Add(const std::string &db);
/**
* @brief Remove database to the list of granted access.
* @note if allow_all_ is set, the flag will remain set and the
* database will be added to the set of denied databases.
*
* @param db name of the database to grant access to
*/
void Remove(const std::string &db);
/**
* @brief Called when database is dropped. Removes it from granted (if allow_all is false) and denied set.
* @note allow_all_ is not changed
*
* @param db name of the database to grant access to
*/
void Delete(const std::string &db);
/**
* @brief Set allow_all_ to true and clears grants and denied sets.
*/
void GrantAll();
/**
* @brief Set allow_all_ to false and clears grants and denied sets.
*/
void DenyAll();
/**
* @brief Set the default database.
*/
bool SetDefault(const std::string &db);
/**
* @brief Checks if access is grated to the database.
*
* @param db name of the database
* @return true if allow_all and not denied or granted
*/
bool Contains(const std::string &db) const;
bool GetAllowAll() const { return allow_all_; }
const std::set<std::string> &GetGrants() const { return grants_dbs_; }
const std::set<std::string> &GetDenies() const { return denies_dbs_; }
const std::string &GetDefault() const;
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static Databases Deserialize(const nlohmann::json &data);
private:
Databases(bool allow_all, std::set<std::string> grant, std::set<std::string> deny,
const std::string &default_db = dbms::kDefaultDB)
: grants_dbs_(grant), denies_dbs_(deny), allow_all_(allow_all), default_db_(default_db) {}
std::set<std::string> grants_dbs_; //!< set of databases with granted access
std::set<std::string> denies_dbs_; //!< set of databases with denied access
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
std::string default_db_; //!< user's default database
};
#endif
// TODO (mferencevic): Implement password expiry. // TODO (mferencevic): Implement password expiry.
class User final { class User final {
public: public:
User(const std::string &username); User();
explicit User(const std::string &username);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions); User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
#ifdef MG_ENTERPRISE
User(const std::string &username, const std::string &password_hash, const Permissions &permissions, User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
const LabelPermissions &labelPermissions); FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
#endif
User(const User &) = default;
User &operator=(const User &) = default;
User(User &&) noexcept = default;
User &operator=(User &&) noexcept = default;
~User() = default;
/// @throw AuthException if unable to verify the password. /// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password); bool CheckPassword(const std::string &password);
@@ -173,6 +350,12 @@ class User final {
Permissions GetPermissions() const; Permissions GetPermissions() const;
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions GetFineGrainedAccessLabelPermissions() const;
FineGrainedAccessPermissions GetFineGrainedAccessEdgeTypePermissions() const;
const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
#endif
const std::string &username() const; const std::string &username() const;
const Permissions &permissions() const; const Permissions &permissions() const;
@@ -180,7 +363,10 @@ class User final {
const Role *role() const; const Role *role() const;
LabelPermissions &labelPermissions(); #ifdef MG_ENTERPRISE
Databases &db_access() { return database_access_; }
const Databases &db_access() const { return database_access_; }
#endif
nlohmann::json Serialize() const; nlohmann::json Serialize() const;
@@ -193,10 +379,17 @@ class User final {
std::string username_; std::string username_;
std::string password_hash_; std::string password_hash_;
Permissions permissions_; Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
Databases database_access_;
#endif
std::optional<Role> role_; std::optional<Role> role_;
LabelPermissions labelPermissions_;
}; };
bool operator==(const User &first, const User &second); bool operator==(const User &first, const User &second);
#ifdef MG_ENTERPRISE
FineGrainedAccessPermissions Merge(const FineGrainedAccessPermissions &first,
const FineGrainedAccessPermissions &second);
#endif
} // namespace memgraph::auth } // namespace memgraph::auth

Some files were not shown because too many files have changed in this diff Show More