Compare commits

..

155 Commits

Author SHA1 Message Date
Andi Skrgat
b5c6d209b4 Merge branch 'mg-larger-than-memory' of github.com:memgraph/memgraph into mg-larger-than-memory 2023-05-19 14:46:45 +02:00
Andi Skrgat
7657e671fe Disable storage_rocks test 2023-05-19 14:46:40 +02:00
Aidar Samerkhanov
0751d77196 Make type parameterized tests for querym plan accumulate aggregate. 2023-05-17 21:06:13 +00:00
Andi Skrgat
533920bde4 ASAN partial fix for storage_rocks 2023-05-17 15:35:46 +02:00
Andi Skrgat
a015df3b93 Fixed imports and edge_properties in disk storage 2023-05-17 13:25:38 +02:00
Andi Skrgat
c8d7bb743d Imports cleared in mg-storage 2023-05-17 12:20:25 +02:00
Andi Skrgat
edf08c9b6b Removed disk storage typed test 2023-05-17 11:47:27 +02:00
Andi Skrgat
6df79724ca (un)lock path, get info and transaction id deduplication 2023-05-17 11:44:00 +02:00
Andi Skrgat
6f652fff42 NameIDMapper refactored 2023-05-17 10:59:13 +02:00
Andi Skrgat
c53449f99d Merge branch 'mg-larger-than-memory' of github.com:memgraph/memgraph into mg-larger-than-memory 2023-05-17 09:59:39 +02:00
Andi Skrgat
3b4f949a52 Storage fields refactoring 2023-05-17 09:59:35 +02:00
Aidar Samerkhanov
c768c23610 Add typed parameterization to query expression evaluator tests 2023-05-16 19:10:50 +00:00
Andi Skrgat
7cb5a2e3b5 Merge branch 'mg-larger-than-memory' of github.com:memgraph/memgraph into mg-larger-than-memory 2023-05-16 20:11:35 +02:00
Andi Skrgat
331d7b9291 Variables refactoring 2023-05-16 20:11:29 +02:00
Aidar Samerkhanov
f32b4c4dc9 Add type-parameterized test to query dump unit test 2023-05-16 15:15:36 +00:00
Aidar Samerkhanov
0b08a76999 Fix log message in case of null delta in FindVertex 2023-05-16 14:09:33 +00:00
Andi Skrgat
8dd2b8b110 (de)serialization rocksdb unit tests 2023-05-16 15:17:48 +02:00
Andi Skrgat
82a644ebf2 Merge branch 'mg-larger-than-memory' of github.com:memgraph/memgraph into mg-larger-than-memory 2023-05-16 11:25:58 +02:00
Andi Skrgat
6d51522328 Deserializing vertices fix 2023-05-16 11:25:52 +02:00
Aidar Samerkhanov
aee9e5780d Make parameterized storage v2 unit tests for in-memory and on-disk types of storages 2023-05-16 09:15:45 +00:00
Aidar Samerkhanov
9d1733df07 Fix Get/SetStorageMode for DiskStorage. 2023-05-16 09:15:20 +00:00
Andi Skrgat
d245b7931d Disk storage compliance 2023-05-16 10:53:16 +02:00
Andi Skrgat
49a7e29445 Clang-tidy fixes 2023-05-16 09:45:08 +02:00
Andi Skrgat
1ce79213e6 e2e config check 2023-05-15 16:39:41 +02:00
Andi Skrgat
41588024fc Unit tests passing 2023-05-15 16:06:29 +02:00
Andi Skrgat
4b063fc457 Bolt encoder 2023-05-15 15:50:21 +02:00
Andi Skrgat
c88fba3322 Storage unit tests 2023-05-15 15:47:32 +02:00
Aidar Samerkhanov
66925c2036 Fix query procedures mgp graph unit test 2023-05-15 11:43:46 +00:00
Aidar Samerkhanov
3c49bf2ab0 Fix query procedure py module unit test 2023-05-15 11:21:08 +00:00
Andi Skrgat
c4f1199b80 Create set remove delete 2023-05-15 12:48:19 +02:00
Andi Skrgat
0f69e0aaa1 Fixes 2 2023-05-15 12:44:09 +02:00
Andi Skrgat
d21e4f13f7 Virtual storage fixes 1 2023-05-15 12:05:27 +02:00
Andi Skrgat
dd3f29fba8 Merge branch 'disk-storage-v1' into mg-larger-than-memory 2023-05-12 17:36:38 +02:00
Andi
976c36018a Merge branch 'master' into add-on-disk-transactional-storage 2023-05-12 17:21:15 +02:00
Aidar Samerkhanov
fa0c2bc97f Merge branch 'master' into add-on-disk-transactional-storage 2023-05-09 15:24:35 +00:00
Andi Skrgat
153222a2a7 Merge remote-tracking branch 'origin/accessors-interface' into disk-storage-v1 2023-05-05 14:29:23 +02:00
Andi Skrgat
cbe23f7efa Properties deserialization 2023-05-05 14:28:59 +02:00
Aidar Samerkhanov
9e710aa00b Replace implicit move by copy in MakeEdgeAccessor 2023-05-03 21:13:18 +00:00
Aidar Samerkhanov
59ed253b82 Fix SubgraphVertexAccessor in and out eges in e2e 2023-05-03 15:49:16 +00:00
Andi Skrgat
95c119999f Indexes e2e 2023-05-03 15:28:55 +02:00
Andi Skrgat
bd0801dc1f Create index and rocksb objects reorganization 2023-05-03 13:14:04 +02:00
Andi Skrgat
04261f7cce Disk vertices iterable 2023-05-02 18:37:38 +02:00
Andi Skrgat
7bf38442aa Non-abstract indices 2023-05-02 15:22:50 +02:00
Aidar Samerkhanov
dfdba859ff Fix query engine build 2023-04-28 15:02:14 +00:00
Aidar Samerkhanov
01428b49e1 Fix random_graph in property based testing 2023-04-28 12:46:32 +00:00
Aidar Samerkhanov
b98c9b94ca Fix build of full memgraph 2023-04-27 13:34:10 +00:00
Andi Skrgat
43fd6906b6 Expand from vertex 2023-04-27 13:12:36 +02:00
Aidar Samerkhanov
54e6aef954 Fixed build of storage v2 gc and random_graph test 2023-04-26 19:24:41 +00:00
Aidar Samerkhanov
20f16eafa3 Fix mg_import_csv build 2023-04-26 12:51:11 +00:00
Aidar Samerkhanov
a69829d01b Fix build of concurrent storage indices tests 2023-04-26 12:12:13 +00:00
Andi Skrgat
697bed4348 Merge remote-tracking branch 'origin/accessors-interface' into disk-storage-v1 2023-04-26 13:55:06 +02:00
Aidar Samerkhanov
01e0dcdfb4 Fix build of concurrent storage unique constraints test 2023-04-26 11:47:05 +00:00
Aidar Samerkhanov
8bd561694c Fix build of auth checker tests 2023-04-25 22:54:47 +00:00
Aidar Samerkhanov
de40594f53 Fix build of bfs tests 2023-04-25 22:52:49 +00:00
Aidar Samerkhanov
d400bf9355 Fix build and run of bolt encoder tests 2023-04-25 22:44:42 +00:00
Aidar Samerkhanov
bf3a57c8ca Fix build and run cpp api tests 2023-04-25 22:33:11 +00:00
Aidar Samerkhanov
cb16ba9e1e Fix build of interpreter tests 2023-04-25 21:55:25 +00:00
Aidar Samerkhanov
28dbf73569 Fix build of plan pretty print tests 2023-04-25 21:48:41 +00:00
Aidar Samerkhanov
f8df97cf4e Fix build for typed_value test 2023-04-25 21:35:43 +00:00
Aidar Samerkhanov
f8b8181e81 Fix build of transaction queue multiple test 2023-04-25 21:29:16 +00:00
Aidar Samerkhanov
f734285321 Fix build of transaction queue test 2023-04-25 21:27:56 +00:00
Aidar Samerkhanov
1d3ba624da Fix build and run of storage v2 replication tests 2023-04-25 21:05:27 +00:00
Aidar Samerkhanov
045d8b8182 Fix build of storage v2 isolation level tests 2023-04-25 14:10:42 +00:00
Aidar Samerkhanov
626e9b2b6d Fix build of storage v2 indices tests 2023-04-25 14:04:26 +00:00
Aidar Samerkhanov
9cb27e5f26 Fix build of storage v2 gc tests 2023-04-25 13:38:34 +00:00
Andi Skrgat
e45decfae8 Detach delete 2023-04-25 15:17:38 +02:00
Andi Skrgat
6a5ec391d9 Delete vertex and edges 2023-04-25 15:07:01 +02:00
Andi Skrgat
243e716812 Creating objects compatible with in-memory 2023-04-25 12:55:01 +02:00
Andi Skrgat
7ed38621ce Scan edges from the disk 2023-04-25 11:57:19 +02:00
Aidar Samerkhanov
a0b9309c56 Fix build and run tests of storage v2 edge 2023-04-25 09:19:05 +00:00
Andi Skrgat
5f1b05337f KV store refactor 2023-04-25 09:04:19 +02:00
Aidar Samerkhanov
253a845c69 Fix build and run of storage v2 durability tests 2023-04-24 20:27:22 +00:00
Andi Skrgat
7a4d910282 Merge remote-tracking branch 'origin/accessors-interface' into disk-storage-v1 2023-04-24 14:03:17 +02:00
Andi Skrgat
d6b9309618 Deserialize edge and edge init properties 2023-04-24 14:01:09 +02:00
Andi Skrgat
0505f29b52 Delete vertex support 2023-04-24 13:21:37 +02:00
Andi Skrgat
e4225c4ac1 Commit ts inside rocksdb 2023-04-24 11:38:36 +02:00
Aidar Samerkhanov
ab408c0795 Fix build storage v2 contraints tests 2023-04-24 07:56:14 +00:00
Aidar Samerkhanov
04467e03a0 Fix build of storage v2 tests 2023-04-24 07:27:08 +00:00
Andi Skrgat
c960d253dd RocksDB commit timestamp 2023-04-24 07:45:03 +02:00
Aidar Samerkhanov
38062e6506 Fix build of query trigger tests 2023-04-23 23:13:09 +00:00
Aidar Samerkhanov
c84a9279c1 Fix build query variable start planner tests 2023-04-23 23:06:31 +00:00
Aidar Samerkhanov
219cd4c765 Fix build of query streams tests 2023-04-23 22:57:46 +00:00
Aidar Samerkhanov
f6841e9f08 Fix build of query semantic tests 2023-04-23 22:55:51 +00:00
Aidar Samerkhanov
8901bfd10a Fix build query procedures mgp graph 2023-04-23 22:53:37 +00:00
Aidar Samerkhanov
e3eea7fb25 Fix run and build query procedure py module tests 2023-04-23 22:36:15 +00:00
Aidar Samerkhanov
b1f40aff7e Fix build query procedure mgp type tests 2023-04-23 21:45:05 +00:00
Aidar Samerkhanov
4b9fda74f6 Fix build query pretty print tests 2023-04-23 21:42:38 +00:00
Aidar Samerkhanov
822a711762 Fix build query plan v2 create set remove delete 2023-04-23 21:39:58 +00:00
Aidar Samerkhanov
cb232e71c6 Fix build query plan read write typecheck 2023-04-23 21:35:55 +00:00
Aidar Samerkhanov
6547c161e2 Fix build query plan match filter return tests 2023-04-23 21:08:57 +00:00
Aidar Samerkhanov
ba06b1137b Fix build plan edge cases tests 2023-04-23 20:55:17 +00:00
Aidar Samerkhanov
d7dc8e1897 Fix tests in query plan create set remove delete 2023-04-23 20:51:09 +00:00
Aidar Samerkhanov
aa56e2eb7c Fix build in query plan create set remove delete tests 2023-04-22 20:29:56 +00:00
Aidar Samerkhanov
268f70bbed Fix buid query plan bag semantics 2023-04-22 20:17:51 +00:00
Aidar Samerkhanov
4ed8220cf8 Fix build query_plan_accumulate_aggregate 2023-04-22 20:15:16 +00:00
Aidar Samerkhanov
0803749269 Fix query expression evaluation tests 2023-04-22 20:06:11 +00:00
Andi Skrgat
f3c4b4cea1 Delete vertex working from the cache 2023-04-21 11:45:26 +02:00
Aidar Samerkhanov
6408bd1760 Query dump tests fixes 2023-04-21 08:08:10 +00:00
Andi Skrgat
30328832f4 Storage mode config 2023-04-21 08:55:57 +02:00
Andi Skrgat
6ffbade6d8 MATCH (n) RETURN n cached and from disk 2023-04-20 14:26:14 +02:00
Andi Skrgat
d37842e523 Create vertex works 2023-04-20 11:17:33 +02:00
Andi Skrgat
ead0f9db22 Garbage collection solved 2023-04-20 10:25:15 +02:00
Andi Skrgat
9e2dc55750 Approx vertex count, sessions correctly closed 2023-04-20 08:25:33 +02:00
Andi Skrgat
245f2a34e1 Disk-based Memgraph accepting connections 2023-04-19 15:28:50 +02:00
Andi Skrgat
e51f82db1d Memgraph with disk compiled 2023-04-19 14:18:21 +02:00
Andi Skrgat
225922206c Fixed disk storage build with comparators 2023-04-19 13:33:30 +02:00
Andi Skrgat
26f56352e4 In-memory compile 2023-04-19 13:08:33 +02:00
Andi Skrgat
39c14746ad Merge remote-tracking branch 'origin/accessors-interface' into disk-storage-v1 2023-04-19 12:56:38 +02:00
Andi Skrgat
955d2f1322 Create vertex, create edge, find vertex 2023-04-19 12:54:14 +02:00
Aidar Samerkhanov
82722c6fed Fix memgraph binary build 2023-04-19 10:31:43 +00:00
Andi Skrgat
69778d6ca9 Flush cache at commit time 2023-04-19 10:42:15 +02:00
Aidar Samerkhanov
9caf1fa099 Fix unique_ptr ownership.
Query engine accessors must be initialized through unique-ptr instead of raw pointer.
2023-04-18 19:56:09 +00:00
Andi Skrgat
c4b8d7d568 Cache modification_ts 2023-04-18 13:40:34 +02:00
Andi Skrgat
c2ebd511ab DiskVertex and DiskEdge 2023-04-18 13:34:59 +02:00
Andi Skrgat
9ce31bbab1 (de)serialization of commit timestamp 2023-04-18 12:35:57 +02:00
Andi Skrgat
797fb78910 Cache gid on accessor level 2023-04-18 08:49:18 +02:00
Andi Skrgat
a1bf90a687 Naming and memgraph build partial fix 2023-04-18 08:27:02 +02:00
Andi Skrgat
3a96db481e Mg-glue build fix and merge with accessors 2023-04-18 08:11:50 +02:00
Andi Skrgat
6d959d07d3 Merge remote-tracking branch 'origin/accessors-interface' into disk-storage-v1 2023-04-18 07:35:05 +02:00
Aidar Samerkhanov
3a7fa763f7 Fix build of query_cost_estimator test 2023-04-17 21:29:44 +00:00
Aidar Samerkhanov
188162579c Fix mg-query build 2023-04-17 14:56:30 +00:00
Aidar Samerkhanov
5e226d2a72 Start building trigger and procedures. 2023-04-17 12:58:39 +00:00
Andi Skrgat
9b3bdb5c84 Merge remote-tracking branch 'origin/accessors-interface' into disk-storage-v1 2023-04-17 13:40:53 +02:00
Andi Skrgat
177d9dabf5 All vertices from the disk 2023-04-17 13:39:52 +02:00
Andi Skrgat
c819012a52 Transaction level cache 2023-04-17 12:21:11 +02:00
Aidar Samerkhanov
b5646489ba Make query Accessors owners of pointers to storage accessors.
Add Copy() function to VertexAccessor and EdgeAccessor.
2023-04-17 07:10:36 +00:00
Andi Skrgat
53623da2b7 rocksdb functions on disk storage 2023-04-14 15:49:10 +02:00
Andi Skrgat
148f6e1498 Disk partial merge 2023-04-14 14:17:19 +02:00
Andi Skrgat
bc6f8e31fc Merge remote-tracking branch 'origin/accessors-interface' into HEAD 2023-04-14 14:15:34 +02:00
Andi Skrgat
b34044459e Rocks to v2 storage 2023-04-14 10:54:18 +02:00
Andi Skrgat
a22b011736 Compaction filter poc 2023-04-14 10:09:38 +02:00
Aidar Samerkhanov
ed76943dff Fix build of mg-query 2023-04-13 22:47:09 +00:00
Aidar Samerkhanov
e0da73780a Fix db_accessor build in query engine 2023-04-13 19:56:09 +00:00
Aidar Samerkhanov
18bf1c237b Fix some compilation errors in query engine module 2023-04-12 22:16:54 +00:00
Aidar Samerkhanov
faa3e542b6 Fix storage unit tests 2023-04-12 21:50:26 +00:00
Aidar Samerkhanov
2250cadd28 Fix build for storage tests 2023-04-12 21:38:52 +00:00
Aidar Samerkhanov
3c0ae50a15 Fix build of storage library. 2023-04-12 20:53:48 +00:00
Andi Skrgat
876359f4b6 Code edits 2023-04-12 13:43:47 +02:00
Aidar Samerkhanov
c64b607478 Add abstract classes for Accessor, VertexAccessor and EdgeAccessor. 2023-04-11 12:34:09 +00:00
Andi Skrgat
c441b7de0b Solved memory leaks 2023-04-07 15:10:59 +02:00
Andi Skrgat
b6beffa9e2 Delete api 2023-04-07 09:56:00 +02:00
Andi Skrgat
1bed62da4a Code cleaning 2023-04-06 16:28:29 +02:00
Andi Skrgat
2019f5cc72 in and out edges 2023-04-06 11:31:50 +02:00
Andi Skrgat
f070329e77 Edge serializaiton 2023-04-05 15:29:44 +02:00
Andi Skrgat
b8d956294b Buffer serialization of properties 2023-04-05 12:04:05 +02:00
Andi Skrgat
72371cc8ee Dummy properties serialization 2023-04-05 09:33:47 +02:00
Andi Skrgat
e193d4036e Multiple labels serialization test cases 2023-04-04 11:12:58 +02:00
Andi Skrgat
ceb3e67f8e Merge branch 'add-on-disk-transactional-storage' into disk-storage-poc 2023-04-03 11:13:25 +02:00
Andi Skrgat
da8aa68c86 Merge branch 'master' into add-on-disk-transactional-storage 2023-04-03 11:12:46 +02:00
Andi Skrgat
cc765f0ebd Merge branch 'master' of github.com:memgraph/memgraph 2023-04-03 11:09:57 +02:00
Andi Skrgat
f7c1923ab2 ScanAll 2023-04-03 11:08:47 +02:00
Josipmrden
7fe17ba9ef [E216 < T1245] Add subqueries (#794) (#851)
Add subqueries

Co-authored-by: Bruno Sačarić <bruno.sacaric@gmail.com>
2023-03-31 15:24:02 +02:00
Marko Budiselic
3f209e3863 Merge branch 'master' into add-on-disk-transactional-storage 2023-03-31 13:04:25 +00:00
Andi Skrgat
7503b304f2 POC started 2023-03-31 15:00:08 +02:00
Marko Budiselic
81a329ce56 Add ON_DISK_TRANSACTIONAL storage 2023-03-29 20:46:20 +00:00
383 changed files with 10677 additions and 28492 deletions

View File

@@ -1,7 +1,6 @@
---
BasedOnStyle: Google
---
Language: Cpp
BasedOnStyle: Google
Standard: "c++20"
UseTab: Never
DerivePointerAlignment: false

View File

@@ -10,5 +10,5 @@
To keep docs changelog up to date, one more thing to do:
- [ ] Write a release note here, including added/changed clauses
- [ ] Write a release note here
- [ ] Tag someone from docs team in the comments

View File

@@ -196,7 +196,22 @@ jobs:
- name: Run integration tests
run: |
tests/integration/run.sh
cd tests/integration
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format
run: |
@@ -266,11 +281,12 @@ jobs:
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
./run.sh
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |
@@ -292,6 +308,7 @@ jobs:
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build
# create mgconsole
@@ -338,8 +355,10 @@ jobs:
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build only memgraph release binarie.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
@@ -348,7 +367,7 @@ jobs:
- name: Run Jepsen tests
run: |
cd tests/jepsen
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3

View File

@@ -6,7 +6,7 @@ on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'vX.Y.Z'"
required: false
jobs:
@@ -253,4 +253,4 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/"
DEST_DIR: "memgraph/${{ github.event.inputs.memgraph_version }}/"

View File

@@ -146,7 +146,22 @@ jobs:
- name: Run integration tests
run: |
tests/integration/run.sh
cd tests/integration
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format
run: |
@@ -265,11 +280,12 @@ jobs:
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
./run.sh
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |

View File

@@ -146,7 +146,22 @@ jobs:
- name: Run integration tests
run: |
tests/integration/run.sh
cd tests/integration
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format
run: |
@@ -264,11 +279,12 @@ jobs:
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
./run.sh
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |
@@ -318,8 +334,10 @@ jobs:
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build only memgraph release binary.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
@@ -328,7 +346,7 @@ jobs:
- name: Run Jepsen tests
run: |
cd tests/jepsen
./run.sh test-all-individually --binary ../../build/memgraph --ignore-run-stdout-logs --ignore-run-stderr-logs
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3

View File

@@ -146,7 +146,22 @@ jobs:
- name: Run integration tests
run: |
tests/integration/run.sh
cd tests/integration
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format
run: |
@@ -264,11 +279,12 @@ jobs:
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
./run.sh
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run stress test (plain)
run: |

View File

@@ -1,32 +0,0 @@
name: Upload Package All artifacts to S3
on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. Format: 'X.Y.Z'"
required: true
run_number:
description: "# of the package_all workflow run to upload artifacts from. Format: '#XYZ'"
required: true
jobs:
upload-to-s3:
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: dawidd6/action-download-artifact@v2
with:
workflow: package_all.yaml
workflow_conclusion: success
run_number: "${{ github.event.inputs.run_number }}"
path: build/output/release
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: "download.memgraph.com"
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/"

View File

@@ -3,7 +3,6 @@ repos:
rev: v4.4.0
hooks:
- id: check-yaml
args: [--allow-multiple-documents]
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black

View File

@@ -1,7 +1,6 @@
# MemGraph CMake configuration
cmake_minimum_required(VERSION 3.12)
cmake_policy(SET CMP0076 NEW)
cmake_minimum_required(VERSION 3.8)
# !! IMPORTANT !! run ./project_root/init.sh before cmake command
# to download dependencies
@@ -19,12 +18,10 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM TRUE)
# during the code coverage process
find_program(CCACHE_FOUND ccache)
option(USE_CCACHE "ccache:" ON)
message(STATUS "CCache: ${USE_CCACHE}")
if(CCACHE_FOUND AND USE_CCACHE)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
message(STATUS "CCache: Used")
else ()
message(STATUS "CCache: Not used")
endif(CCACHE_FOUND AND USE_CCACHE)
# choose a compiler
@@ -40,14 +37,7 @@ endif()
# -----------------------------------------------------------------------------
project(memgraph LANGUAGES C CXX)
#TODO: upgrade to cmake 3.24 + CheckIPOSupported
#cmake_policy(SET CMP0138 NEW)
#include(CheckIPOSupported)
#check_ipo_supported()
#set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_Release TRUE)
#set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RelWithDebInfo TRUE)
project(memgraph)
# Install licenses.
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
@@ -153,9 +143,7 @@ endif()
# files used can be seen here:
# https://git-scm.com/book/en/v2/Git-Internals-Git-References
set(git_directory "${CMAKE_SOURCE_DIR}/.git")
# Check for directory because if the repo is cloned as a git submodule, .git is
# a file and below code doesn't work.
if (IS_DIRECTORY "${git_directory}")
if (EXISTS "${git_directory}")
set_property(DIRECTORY APPEND PROPERTY
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
file(STRINGS "${git_directory}/HEAD" git_head_data)
@@ -170,7 +158,7 @@ endif()
# setup CMake module path, defines path for include() and find_package()
# https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake)
# custom function definitions
include(functions)
# -----------------------------------------------------------------------------
@@ -243,8 +231,6 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
add_definitions( -DCMAKE_BUILD_TYPE_NAME="${CMAKE_BUILD_TYPE}")
if (NOT MG_ARCH)
set(MG_ARCH_DESCR "Host architecture to build Memgraph on. Supported values are x86_64, ARM64.")
if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")

View File

@@ -23,14 +23,6 @@ def wrap_text(s, initial_indent="# "):
def extract_flags(binary_path):
ret = {}
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
# If something is printed out before the help output, it will break the the
# XML parsing -> filter out if something is not XML line because something
# can be logged before gflags output (e.g. during the global objects init).
# This gets called during memgraph build phase to generate default config
# file later installed under /etc/memgraph/memgraph.conf
# NOTE: Don't use \n in the gflags description strings.
# NOTE: Check here if gflags version changes because of the XML format.
data = "\n".join([line for line in data.split("\n") if line.startswith("<")])
root = ET.fromstring(data)
for child in root:
if child.tag == "usage" and child.text.lower().count("warning"):

View File

@@ -202,29 +202,3 @@ for row in csv.reader(stream, delimiter=',', doublequote=True,
For more information about the meaning of the above values, see:
https://docs.python.org/3/library/csv.html#csv.Dialect
## Errors
1. [Skipping duplicate node with ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-1)
2. [Skipping bad relationship with START_ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-2)
3. [Skipping bad relationship with END_ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-3)
## Skipping duplicate node with ID {} {#error-1}
Duplicate nodes are nodes that have an ID that is the same as another node that
was already imported. You can instruct the importer to ignore all duplicate
nodes (instead of raising an error) by using the `--skip-duplicate-nodes` flag.
## Skipping bad relationship with START_ID {} {#error-2}
A node with the id `START_ID` doesn't exist. You can instruct the importer to
ignore all bad relationships (instead of raising an error) that refer to nodes
that don't exist in the node files by using the `--skip-bad-relationships` flag.
## Skipping bad relationship with END_ID {} {#error-3}
A node with the id `END_ID` doesn't exist. You can instruct the importer to
ignore all bad relationships (instead of raising an error) that refer to nodes
that don't exist in the node files by using the `--skip-bad-relationships` flag.

View File

@@ -1,10 +0,0 @@
# Memgraph Operating Environments
## os
Under the `os` directory, you can find scripts to install all required system
dependencies on operating systems where Memgraph natively builds. The testing
script helps to see how to install all packages (in the case of a new package),
or make any adjustments in the overall system setup. Also, the testing script
helps check if Memgraph runs on a freshly installed operating system (with no
packages installed).

View File

@@ -1,6 +1,3 @@
*.deb
*.deb.*
*.rpm
*.rpm.*
*.tar.gz
*.tar.gz.*

View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake # build system
make # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -57,18 +57,9 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang custom-golang1.18.9 zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.3 # for driver tests
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -82,18 +73,6 @@ check() {
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""}
LD_LIBRARY_PATH=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
@@ -124,27 +103,8 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == jdk-17 ]; then
if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then
wget --no-check-certificate -c --header "Cookie: oraclelicense=accept-securebackup-cookie" https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
rpm -Uvh jdk-17_linux-x64_bin.rpm
# NOTE: Set Java 11 as default.
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm

View File

@@ -39,7 +39,7 @@ TOOLCHAIN_RUN_DEPS=(
)
MEMGRAPH_BUILD_DEPS=(
make cmake pkgconfig # build system
make pkgconfig # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -56,19 +56,9 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # for custom Lisp C++ preprocessing
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 golang custom-golang1.18.9 # for driver tests
nodejs zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.3 # for driver tests
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -78,18 +68,6 @@ list() {
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == git ]; then
if ! which "git" >/dev/null; then
missing="git $missing"
@@ -132,25 +110,7 @@ install() {
yum update -y
yum install -y wget python3 python3-pip
yum install -y git
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == jdk-17 ]; then
if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then
wget https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
rpm -ivh jdk-17_linux-x64_bin.rpm
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm

View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkgconf-pkg-config # build system
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -56,20 +56,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang custom-golang1.18.9 # for driver tests
zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.3 # for driver tests
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -79,18 +69,6 @@ list() {
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
@@ -125,16 +103,7 @@ install() {
fi
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
# Since there is no support for libipt-devel for CentOS 9 we install
# Fedoras version of same libs, they are the same version but released
# for different OS

View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,19 +53,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless oracle-java17-installer custom-maven3.9.3 # for driver tests
dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -73,28 +64,7 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
@@ -105,15 +75,8 @@ deb http://deb.debian.org/debian/ buster-updates main contrib non-free
deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free
deb http://security.debian.org/debian-security buster/updates main contrib non-free
deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
EOF
apt --allow-releaseinfo-change update
cat >/etc/apt/sources.list.d/java.list << EOF
deb http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main
deb-src http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main
EOF
cd "$DIR"
apt install -y gnupg
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EA8CACC073C3DB2A
apt --allow-releaseinfo-change update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
@@ -122,26 +85,8 @@ EOF
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == oracle-java17-installer ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
echo oracle-java17-installer shared/accepted-oracle-license-v1-3 select true | /usr/bin/debconf-set-selections
echo oracle-java17-installer shared/accepted-oracle-license-v1-3 seen true | /usr/bin/debconf-set-selections
apt install -y "$pkg"
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

View File

@@ -54,19 +54,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.3 # for driver tests
golang custom-golang1.18.9 nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -74,28 +65,7 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
@@ -119,25 +89,7 @@ EOF
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-arm64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-arm64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -54,19 +54,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.3 # for driver tests
dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -74,28 +65,7 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
@@ -119,25 +89,7 @@ EOF
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkgconf-pkg-config # build system
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -52,21 +52,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
java-11-openjdk-devel java-17-openjdk-devel custom-maven3.9.3 # for driver tests
which zip unzip
nodejs golang custom-golang1.18.9 # for driver tests
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -75,25 +64,11 @@ list() {
check() {
local missing=""
if [ -v LD_LIBRARY_PATH ]; then
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
fi
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
@@ -102,10 +77,7 @@ check() {
echo "MISSING PACKAGES: $missing"
exit 1
fi
if [ -v OLD_LD_LIBRARY_PATH ]; then
echo "Restoring LD_LIBRARY_PATH..."
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
fi
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
}
install() {
@@ -122,25 +94,7 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == java-17-openjdk-devel ]; then
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
dnf install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
dnf install -y "$pkg"
done
}

View File

@@ -1,145 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
IFS=' '
# NOTE: docker_image_name could be local image build based on release/package images.
# NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space.
# "docker_container_type script_name docker_image_name"
OPERATING_SYSTEMS=(
"mgrun amzn-2 amazonlinux:2"
"mgrun centos-7 centos:7"
"mgrun centos-9 dokken/centos-stream-9"
"mgrun debian-10 debian:10"
"mgrun debian-11 debian:11"
"mgrun fedora-36 fedora:36"
"mgrun ubuntu-18.04 ubuntu:18.04"
"mgrun ubuntu-20.04 ubuntu:20.04"
"mgrun ubuntu-22.04 ubuntu:22.04"
# "mgbuild centos-7 package-mgbuild_centos-7"
)
if [ ! "$(docker info)" ]; then
echo "ERROR: Docker is required"
exit 1
fi
print_help () {
echo -e "$0 all\t\t\t\t => start + init all containers in the background"
echo -e "$0 check\t\t\t\t => check all containers"
echo -e "$0 delete\t\t\t\t => stop + remove all containers"
echo -e "$0 copy src_container dst_container => copy build package from src to dst container"
exit 1
}
# NOTE: This is an idempotent operation!
# TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function.
docker_run () {
cnt_name="$1"
cnt_image="$2"
if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
echo "Cleanup of the old exited container..."
docker rm $cnt_name
fi
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
fi
echo "The $cnt_image container is active under $cnt_name name!"
}
docker_exec () {
cnt_name="$1"
cnt_cmd="$2"
docker exec -it "$cnt_name" bash -c "$cnt_cmd"
}
docker_stop_and_rm () {
cnt_name="$1"
if [ "$(docker ps -q -f name=$cnt_name)" ]; then
docker stop "$1"
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
docker rm "$1"
fi
fi
}
# TODO(gitbuda): Make the call to `install NEW_DEPS` configurable, the question what else is useful?
start_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_run "$docker_name" "$docker_image"
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""
done
}
check_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh check NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""
done
}
delete_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
docker_stop_and_rm "$docker_name"
echo "~~~~ $docker_image as $docker_name DELETED"
done
}
# TODO(gitbuda): Copy file between containers is a useful util, also delete, + consider copying of a whole folder.
# TODO(gitbuda): Add args: src_cnt dst_cnt abs_path; both file and recursive folder, always delete + copy.
copy_build_package () {
src_container="$1"
dst_container="$2"
src="$src_container:/memgraph/build/output"
tmp_dst="$SCRIPT_DIR/../../build"
mkdir -p "$tmp_dst"
rm -rf "$tmp_dst/output"
dst="$dst_container:/"
docker cp "$src" "$tmp_dst"
docker cp "$tmp_dst/output" "$dst"
}
if [ "$#" -eq 0 ]; then
print_help
else
case $1 in
all)
start_all
;;
check)
check_all
;;
delete)
delete_all
;;
copy) # src_container dst_container
if [ "$#" -ne 3 ]; then
print_help
fi
copy_build_package "$2" "$3"
;;
*)
print_help
;;
esac
fi

View File

@@ -6,7 +6,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "todo-os-name"
check_architecture "todo-arch-name"
TOOLCHAIN_BUILD_DEPS=(
pkg
@@ -20,16 +19,6 @@ MEMGRAPH_BUILD_DEPS=(
pkg
)
MEMGRAPH_RUN_DEPS=(
pkg
)
# NEW_DEPS is useful when you won't to test the installation of a new package.
# During the test you can put here packages like wget curl tar gzip
NEW_DEPS=(
pkg
)
list() {
echo "$1"
}

View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
make pkg-config # build system
curl wget # downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # memgraph console
@@ -53,19 +53,9 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # driver tests
custom-golang1.18.9 # for driver tests
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -73,53 +63,11 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
apt update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
apt install -y $1
}
deps=$2"[*]"

View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,19 +53,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -73,35 +64,12 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
cd "$DIR"
export DEBIAN_FRONTEND=noninteractive
apt update -y
apt install -y wget
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -109,16 +77,8 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -128,15 +88,6 @@ install() {
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,19 +53,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-6.0 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -73,28 +64,7 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
@@ -108,16 +78,7 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -127,15 +88,6 @@ install() {
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-arm64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-arm64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,19 +53,10 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm # for driver tests
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-6.0 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -73,34 +64,12 @@ list() {
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
check_all_dpkg "$1"
}
install() {
cd "$DIR"
apt update -y
apt install -y wget
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -108,16 +77,8 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -127,15 +88,6 @@ install() {
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

View File

@@ -1,15 +1,8 @@
#!/bin/bash
function operating_system() {
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
grep -E '^(VERSION_)?ID=' /etc/os-release | \
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
elif [[ "$OSTYPE" == "darwin"* ]]; then
echo "$(sw_vers -productName)-$(sw_vers -productVersion | cut -d '.' -f 1)"
else
echo "operating_system called on an unknown OS"
exit 1
fi
grep -E '^(VERSION_)?ID=' /etc/os-release | \
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
}
function check_operating_system() {
@@ -83,28 +76,3 @@ function install_all_apt() {
apt install -y "$pkg"
done
}
function install_custom_golang() {
# NOTE: The official https://go.dev/doc/manage-install doesn't seem to be working.
GOVERSION="$1"
GOINSTALLDIR="/opt/go$GOVERSION"
GOROOT="$GOINSTALLDIR/go" # GOPATH=$HOME/go
if [ ! -f "$GOROOT/bin/go" ]; then
curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
mkdir -p "$GOINSTALLDIR"
tar -C "$GOINSTALLDIR" -xzf go$GOVERSION.linux-amd64.tar.gz
fi
echo "go $GOVERSION installed under $GOROOT"
}
function install_custom_maven() {
MVNVERSION="$1"
MVNINSTALLDIR="/opt/apache-maven-$MVNVERSION"
MVNURL="https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/maven/apache-maven-$MVNVERSION-bin.tar.gz"
if [ ! -f "$MVNINSTALLDIR/bin/mvn" ]; then
echo "Downloading maven from $MVNURL"
curl -LO "$MVNURL"
tar -C "/opt" -xzf "apache-maven-$MVNVERSION-bin.tar.gz"
fi
echo "maven $MVNVERSION installed under $MVNINSTALLDIR"
}

View File

@@ -1,26 +0,0 @@
#!/bin/bash -e
COLOR_ORANGE="\e[38;5;208m"
COLOR_GREEN="\e[38;5;35m"
COLOR_RED="\e[0;31m"
COLOR_NULL="\e[0m"
print_help() {
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 memgraph_logs_file_path cypherl_output_path"
exit 1
}
if [ "$#" -ne 2 ]; then
print_help
fi
INPUT="$1"
OUTPUT="$2"
if [ ! -f "$INPUT" ]; then
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} memgraph_logs_file_path is not a file!"
print_help
fi
awk -v RS="Run] '" 'NR>1 { print $0 }' < "$INPUT" | sed -e "/^\[/d;" -e "s/'\([^']*\)$/;/g" > "$OUTPUT"
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl file under $OUTPUT"
echo ""
echo "Import can be done by executing => \`cat $OUTPUT | mgconsole\`"

View File

@@ -1,35 +0,0 @@
#!/bin/bash -e
COLOR_ORANGE="\e[38;5;208m"
COLOR_GREEN="\e[38;5;35m"
COLOR_RED="\e[0;31m"
COLOR_NULL="\e[0m"
print_help() {
echo -e "${COLOR_ORANGE}HOW TO RUN:${COLOR_NULL} $0 input_file_path output_file_path"
exit 1
}
if [ "$#" -ne 2 ]; then
print_help
fi
INPUT="$1"
OUTPUT="$2"
if [ ! -f "$INPUT" ]; then
echo -e "${COLOR_RED}ERROR:${COLOR_NULL} input_file_path is not a file!"
print_help
fi
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} BEGIN and COMMIT are required because variables share the same name (e.g. row)"
echo -e "${COLOR_ORANGE}NOTE:${COLOR_NULL} CONSTRAINTS are just skipped -> ${COLOR_RED}please create consraints manually if needed${COLOR_NULL}"
sed -e 's/^:begin/BEGIN/g; s/^BEGIN$/BEGIN;/g;' \
-e 's/^:commit/COMMIT/g; s/^COMMIT$/COMMIT;/g;' \
-e '/^CALL/d; /^SCHEMA AWAIT/d;' \
-e 's/CREATE RANGE INDEX FOR (n:/CREATE INDEX ON :/g;' \
-e 's/) ON (n./(/g;' \
-e '/^CREATE CONSTRAINT/d; /^DROP CONSTRAINT/d;' "$INPUT" > "$OUTPUT"
echo ""
echo -e "${COLOR_GREEN}DONE!${COLOR_NULL} Please find Memgraph compatible cypherl|.cypher file under $OUTPUT"
echo ""
echo "Please import data by executing => \`cat $OUTPUT | mgconsole\`"

View File

@@ -135,13 +135,6 @@ inline int64_t value_get_int(mgp_value *val) { return MgInvoke<int64_t>(mgp_valu
inline double value_get_double(mgp_value *val) { return MgInvoke<double>(mgp_value_get_double, val); }
inline double value_get_numeric(mgp_value *val) {
if (MgInvoke<int>(mgp_value_is_int, val)) {
return static_cast<double>(value_get_int(val));
}
return value_get_double(val);
}
inline const char *value_get_string(mgp_value *val) { return MgInvoke<const char *>(mgp_value_get_string, val); }
inline mgp_list *value_get_list(mgp_value *val) { return MgInvoke<mgp_list *>(mgp_value_get_list, val); }
@@ -178,8 +171,6 @@ inline bool value_is_int(mgp_value *val) { return MgInvoke<int>(mgp_value_is_int
inline bool value_is_double(mgp_value *val) { return MgInvoke<int>(mgp_value_is_double, val); }
inline bool value_is_numeric(mgp_value *val) { return value_is_int(val) || value_is_double(val); }
inline bool value_is_string(mgp_value *val) { return MgInvoke<int>(mgp_value_is_string, val); }
inline bool value_is_list(mgp_value *val) { return MgInvoke<int>(mgp_value_is_list, val); }
@@ -686,16 +677,6 @@ inline mgp_proc *module_add_write_procedure(mgp_module *module, const char *name
return MgInvoke<mgp_proc *>(mgp_module_add_write_procedure, module, name, cb);
}
inline mgp_proc *module_add_batch_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup) {
return MgInvoke<mgp_proc *>(mgp_module_add_batch_read_procedure, module, name, cb, initializer, cleanup);
}
inline mgp_proc *module_add_batch_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup) {
return MgInvoke<mgp_proc *>(mgp_module_add_batch_write_procedure, module, name, cb, initializer, cleanup);
}
inline void proc_add_arg(mgp_proc *proc, const char *name, mgp_type *type) {
MgInvokeVoid(mgp_proc_add_arg, proc, name, type);
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -1318,13 +1318,6 @@ MGP_ENUM_CLASS mgp_log_level{
/// to allocate global resources.
typedef void (*mgp_proc_cb)(struct mgp_list *, struct mgp_graph *, struct mgp_result *, struct mgp_memory *);
/// Cleanup for a query module read procedure. Can't be invoked through OpenCypher. Cleans batched stream.
typedef void (*mgp_proc_cleanup)();
/// Initializer for a query module batched read procedure. Can't be invoked through OpenCypher. Initializes batched
/// stream.
typedef void (*mgp_proc_initializer)(struct mgp_list *, struct mgp_graph *, struct mgp_memory *);
/// Register a read-only procedure to a module.
///
/// The `name` must be a sequence of digits, underscores, lowercase and
@@ -1349,30 +1342,6 @@ enum mgp_error mgp_module_add_read_procedure(struct mgp_module *module, const ch
enum mgp_error mgp_module_add_write_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb,
struct mgp_proc **result);
/// Register a readable batched procedure to a module.
///
/// The `name` must be a valid identifier, following the same rules as the
/// procedure`name` in mgp_module_add_read_procedure.
///
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if unable to allocate memory for mgp_proc.
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid procedure name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a procedure with the same name was already registered.
enum mgp_error mgp_module_add_batch_read_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup,
struct mgp_proc **result);
/// Register a writeable batched procedure to a module.
///
/// The `name` must be a valid identifier, following the same rules as the
/// procedure`name` in mgp_module_add_read_procedure.
///
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if unable to allocate memory for mgp_proc.
/// Return mgp_error::MGP_ERROR_INVALID_ARGUMENT if `name` is not a valid procedure name.
/// RETURN mgp_error::MGP_ERROR_LOGIC_ERROR if a procedure with the same name was already registered.
enum mgp_error mgp_module_add_batch_write_procedure(struct mgp_module *module, const char *name, mgp_proc_cb cb,
mgp_proc_initializer initializer, mgp_proc_cleanup cleanup,
struct mgp_proc **result);
/// Add a required argument to a procedure.
///
/// The order of adding arguments will correspond to the order the procedure

View File

@@ -11,17 +11,14 @@
#pragma once
#include <cstdlib>
#include <cstring>
#include <functional>
#include <map>
#include <set>
#include <string>
#include <string_view>
#include <vector>
#include "_mgp.hpp"
#include "mg_exceptions.hpp"
#include "mg_procedure.h"
namespace mgp {
@@ -949,7 +946,6 @@ class Duration {
/* #region Value */
enum class Type : uint8_t {
Null,
Any,
Bool,
Int,
Double,
@@ -1183,8 +1179,6 @@ class Record {
void Insert(const char *field_name, const LocalDateTime &local_date_time);
/// @brief Inserts a @ref Duration value under field `field_name`.
void Insert(const char *field_name, const Duration &duration);
/// @brief Inserts a @ref Value value under field `field_name`, and then call appropriate insert.
void Insert(const char *field_name, const Value &value);
private:
mgp_result_record *record_;
@@ -1324,20 +1318,6 @@ inline void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureT
std::vector<Parameter> parameters, std::vector<Return> returns, mgp_module *module,
mgp_memory *memory);
/// @brief Adds a batch procedure to the query module.
/// @param callback - procedure callback
/// @param initializer - procedure initializer
/// @param cleanup - procedure cleanup
/// @param name - procedure name
/// @param proc_type - procedure type (read/write)
/// @param parameters - procedure parameters
/// @param returns - procedure return values
/// @param module - the query module that the procedure is added to
/// @param memory - access to memory
inline void AddBatchProcedure(mgp_proc_cb callback, mgp_proc_initializer initializer, mgp_proc_cleanup cleanup,
std::string_view name, ProcedureType proc_type, std::vector<Parameter> parameters,
std::vector<Return> returns, mgp_module *module, mgp_memory *memory);
/// @brief Adds a function to the query module.
/// @param callback - function callback
/// @param name - function name
@@ -1350,67 +1330,6 @@ inline void AddFunction(mgp_func_cb callback, std::string_view name, std::vector
/* #endregion */
namespace util {
inline uint64_t Fnv(const std::string_view s) {
// fnv1a is recommended so use it as the default implementation.
uint64_t hash = 14695981039346656037UL;
for (const auto &ch : s) {
hash = (hash ^ (uint64_t)ch) * 1099511628211UL;
}
return hash;
}
/**
* Does FNV-like hashing on a collection. Not truly FNV
* because it operates on 8-bit elements, while this
* implementation uses size_t elements (collection item
* hash).
*
* https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
*
*
* @tparam TIterable A collection type that has begin() and end().
* @tparam TElement Type of element in the collection.
* @tparam THash Hash type (has operator() that accepts a 'const TEelement &'
* and returns size_t. Defaults to std::hash<TElement>.
* @param iterable A collection of elements.
* @param element_hash Function for hashing a single element.
* @return The hash of the whole collection.
*/
template <typename TIterable, typename TElement, typename THash = std::hash<TElement>>
struct FnvCollection {
size_t operator()(const TIterable &iterable) const {
uint64_t hash = 14695981039346656037u;
THash element_hash;
for (const TElement &element : iterable) {
hash *= fnv_prime;
hash ^= element_hash(element);
}
return hash;
}
private:
static const uint64_t fnv_prime = 1099511628211u;
};
/**
* Like FNV hashing for a collection, just specialized for two elements to avoid
* iteration overhead.
*/
template <typename TA, typename TB, typename TAHash = std::hash<TA>, typename TBHash = std::hash<TB>>
struct HashCombine {
size_t operator()(const TA &a, const TB &b) const {
static constexpr size_t fnv_prime = 1099511628211UL;
static constexpr size_t fnv_offset = 14695981039346656037UL;
size_t ret = fnv_offset;
ret ^= TAHash()(a);
ret *= fnv_prime;
ret ^= TBHash()(b);
return ret;
}
};
// uint to int conversion in C++ is a bit tricky. Take a look here
// https://stackoverflow.com/questions/14623266/why-cant-i-reinterpret-cast-uint-to-int
// for more details.
@@ -1533,10 +1452,6 @@ inline bool ValuesEqual(mgp_value *value1, mgp_value *value2) {
if (value1 == value2) {
return true;
}
// Make int and double comparable, (ex. this is true -> 1.0 == 1)
if (mgp::value_is_numeric(value1) && mgp::value_is_numeric(value2)) {
return mgp::value_get_numeric(value1) == mgp::value_get_numeric(value2);
}
if (mgp::value_get_type(value1) != mgp::value_get_type(value2)) {
return false;
}
@@ -1576,8 +1491,6 @@ inline bool ValuesEqual(mgp_value *value1, mgp_value *value2) {
/// @brief Converts C++ API types to their MGP API equivalents.
inline mgp_type *ToMGPType(Type type) {
switch (type) {
case Type::Any:
return mgp::type_any();
case Type::Bool:
return mgp::type_bool();
case Type::Int:
@@ -2288,14 +2201,7 @@ inline bool Map::Empty() const { return Size() == 0; }
inline const Value Map::operator[](std::string_view key) const { return Value(mgp::map_at(ptr_, key.data())); }
inline const Value Map::At(std::string_view key) const {
auto *ptr = mgp::map_at(ptr_, key.data());
if (ptr) {
return Value(ptr);
}
return Value();
}
inline const Value Map::At(std::string_view key) const { return Value(mgp::map_at(ptr_, key.data())); }
inline Map::Iterator::Iterator(mgp_map_items_iterator *map_items_iterator) : map_items_iterator_(map_items_iterator) {
if (map_items_iterator_ == nullptr) return;
@@ -2606,7 +2512,6 @@ inline void Path::Expand(const Relationship &relationship) { mgp::path_expand(pt
inline bool Path::operator==(const Path &other) const { return util::PathsEqual(ptr_, other.ptr_); }
inline bool Path::operator!=(const Path &other) const { return !(*this == other); }
/* #endregion */
/* #region Temporal types (Date, LocalTime, LocalDateTime, Duration) */
@@ -3256,42 +3161,6 @@ inline bool Value::IsDuration() const { return mgp::value_is_duration(ptr_); }
inline bool Value::operator==(const Value &other) const { return util::ValuesEqual(ptr_, other.ptr_); }
inline bool Value::operator!=(const Value &other) const { return !(*this == other); }
inline std::ostream &operator<<(std::ostream &os, const mgp::Type &type) {
switch (type) {
case mgp::Type::Null:
return os << "null";
case mgp::Type::Bool:
return os << "bool";
case mgp::Type::Int:
return os << "int";
case mgp::Type::Double:
return os << "double";
case mgp::Type::String:
return os << "string";
case mgp::Type::List:
return os << "list";
case mgp::Type::Map:
return os << "map";
case mgp::Type::Node:
return os << "vertex";
case mgp::Type::Relationship:
return os << "edge";
case mgp::Type::Path:
return os << "path";
case mgp::Type::Date:
return os << "date";
case mgp::Type::LocalTime:
return os << "local_time";
case mgp::Type::LocalDateTime:
return os << "local_date_time";
case mgp::Type::Duration:
return os << "duration";
default:
throw ValueException("Unknown type");
}
}
/* #endregion */
/* #region Record */
@@ -3383,40 +3252,6 @@ inline void Record::Insert(const char *field_name, const Duration &duration) {
mgp::value_destroy(mgp_val);
}
inline void Record::Insert(const char *field_name, const Value &value) {
switch (value.Type()) {
case Type::Bool:
return Insert(field_name, value.ValueBool());
case Type::Int:
return Insert(field_name, value.ValueInt());
case Type::Double:
return Insert(field_name, value.ValueDouble());
case Type::String:
return Insert(field_name, value.ValueString());
case Type::List:
return Insert(field_name, value.ValueList());
case Type::Map:
return Insert(field_name, value.ValueMap());
case Type::Node:
return Insert(field_name, value.ValueNode());
case Type::Relationship:
return Insert(field_name, value.ValueRelationship());
case Type::Path:
return Insert(field_name, value.ValuePath());
case Type::Date:
return Insert(field_name, value.ValueDate());
case Type::LocalTime:
return Insert(field_name, value.ValueLocalTime());
case Type::LocalDateTime:
return Insert(field_name, value.ValueLocalDateTime());
case Type::Duration:
return Insert(field_name, value.ValueDuration());
default:
throw ValueException("No Record.Insert for this datatype");
}
}
// RecordFactory:
inline RecordFactory::RecordFactory(mgp_result *result) : result_(result) {}
@@ -3592,12 +3427,14 @@ inline mgp_type *Return::GetMGPType() const {
return util::ToMGPType(type_);
}
// do not enter
namespace detail {
inline void AddParamsReturnsToProc(mgp_proc *proc, std::vector<Parameter> &parameters,
const std::vector<Return> &returns) {
void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureType proc_type,
std::vector<Parameter> parameters, std::vector<Return> returns, mgp_module *module,
mgp_memory *memory) {
auto proc = (proc_type == ProcedureType::Read) ? mgp::module_add_read_procedure(module, name.data(), callback)
: mgp::module_add_write_procedure(module, name.data(), callback);
for (const auto &parameter : parameters) {
const auto *parameter_name = parameter.name.data();
auto parameter_name = parameter.name.data();
if (!parameter.optional) {
mgp::proc_add_arg(proc, parameter_name, parameter.GetMGPType());
} else {
@@ -3606,35 +3443,18 @@ inline void AddParamsReturnsToProc(mgp_proc *proc, std::vector<Parameter> &param
}
for (const auto return_ : returns) {
const auto *return_name = return_.name.data();
auto return_name = return_.name.data();
mgp::proc_add_result(proc, return_name, return_.GetMGPType());
}
}
} // namespace detail
void AddProcedure(mgp_proc_cb callback, std::string_view name, ProcedureType proc_type,
std::vector<Parameter> parameters, std::vector<Return> returns, mgp_module *module,
mgp_memory *memory) {
auto *proc = (proc_type == ProcedureType::Read) ? mgp::module_add_read_procedure(module, name.data(), callback)
: mgp::module_add_write_procedure(module, name.data(), callback);
detail::AddParamsReturnsToProc(proc, parameters, returns);
}
void AddBatchProcedure(mgp_proc_cb callback, mgp_proc_initializer initializer, mgp_proc_cleanup cleanup,
std::string_view name, ProcedureType proc_type, std::vector<Parameter> parameters,
std::vector<Return> returns, mgp_module *module, mgp_memory *memory) {
auto *proc = (proc_type == ProcedureType::Read)
? mgp::module_add_batch_read_procedure(module, name.data(), callback, initializer, cleanup)
: mgp::module_add_batch_write_procedure(module, name.data(), callback, initializer, cleanup);
detail::AddParamsReturnsToProc(proc, parameters, returns);
}
void AddFunction(mgp_func_cb callback, std::string_view name, std::vector<Parameter> parameters, mgp_module *module,
mgp_memory *memory) {
auto *func = mgp::module_add_function(module, name.data(), callback);
auto func = mgp::module_add_function(module, name.data(), callback);
for (const auto &parameter : parameters) {
const auto *parameter_name = parameter.name.data();
auto parameter_name = parameter.name.data();
if (!parameter.optional) {
mgp::func_add_arg(func, parameter_name, parameter.GetMGPType());
@@ -3664,28 +3484,6 @@ struct hash<mgp::Relationship> {
size_t operator()(const mgp::Relationship &x) const { return hash<int64_t>()(x.Id().AsInt()); };
};
template <>
struct hash<mgp::Path> {
size_t operator()(const mgp::Path &x) const {
// https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function
// See mgp::util::FnvCollection
constexpr const uint64_t fnv_prime = 1099511628211U;
uint64_t hash = 14695981039346656037U;
auto multiply_and_xor = [](uint64_t &hash, size_t element_hash) {
hash *= fnv_prime;
hash ^= element_hash;
};
for (size_t i = 0; i < x.Length() - 1; ++i) {
multiply_and_xor(hash, std::hash<mgp::Node>{}(x.GetNodeAt(i)));
multiply_and_xor(hash, std::hash<mgp::Relationship>{}(x.GetRelationshipAt(i)));
}
multiply_and_xor(hash, std::hash<mgp::Node>{}(x.GetNodeAt(x.Length())));
return hash;
}
};
template <>
struct hash<mgp::Date> {
size_t operator()(const mgp::Date &x) const { return hash<int64_t>()(x.Timestamp()); };
@@ -3710,59 +3508,4 @@ template <>
struct hash<mgp::MapItem> {
size_t operator()(const mgp::MapItem &x) const { return hash<std::string_view>()(x.key); };
};
template <>
struct hash<mgp::Map> {
size_t operator()(const mgp::Map &x) const {
return mgp::util::FnvCollection<mgp::Map, mgp::MapItem, std::hash<mgp::MapItem>>{}(x);
}
};
template <>
struct hash<mgp::Value> {
size_t operator()(const mgp::Value &x) const {
switch (x.Type()) {
case mgp::Type::Null:
return 31;
case mgp::Type::Any:
throw mg_exception::InvalidArgumentException();
case mgp::Type::Bool:
return std::hash<bool>{}(x.ValueBool());
case mgp::Type::Int:
// we cast int to double for hashing purposes
// to be consistent with equality (2.0 == 2) == true
return std::hash<double>{}((double)x.ValueInt());
case mgp::Type::Double:
return std::hash<double>{}(x.ValueDouble());
case mgp::Type::String:
return std::hash<std::string_view>{}(x.ValueString());
case mgp::Type::List:
return mgp::util::FnvCollection<mgp::List, mgp::Value, std::hash<mgp::Value>>{}(x.ValueList());
case mgp::Type::Map:
return std::hash<mgp::Map>{}(x.ValueMap());
case mgp::Type::Node:
return std::hash<mgp::Node>{}(x.ValueNode());
case mgp::Type::Relationship:
return std::hash<mgp::Relationship>{}(x.ValueRelationship());
case mgp::Type::Path:
return std::hash<mgp::Path>{}(x.ValuePath());
case mgp::Type::Date:
return std::hash<mgp::Date>{}(x.ValueDate());
case mgp::Type::LocalTime:
return std::hash<mgp::LocalTime>{}(x.ValueLocalTime());
case mgp::Type::LocalDateTime:
return std::hash<mgp::LocalDateTime>{}(x.ValueLocalDateTime());
case mgp::Type::Duration:
return std::hash<mgp::Duration>{}(x.ValueDuration());
}
throw mg_exception::InvalidArgumentException();
}
};
template <>
struct hash<mgp::List> {
size_t operator()(const mgp::List &x) {
return mgp::util::FnvCollection<mgp::List, mgp::Value, std::hash<mgp::Value>>{}(x);
}
};
} // namespace std

View File

@@ -1402,13 +1402,6 @@ class UnsupportedTypingError(Exception):
super().__init__("Unsupported typing annotation '{}'".format(type_))
class UnequalTypesError(Exception):
"""Signals a typing annotation is not equal between types"""
def __init__(self, type1_: typing.Any, type2_: typing.Any):
super().__init__(f"Unequal typing annotation '{type1_}' and '{type2_}'")
def _typing_to_cypher_type(type_):
"""Convert typing annotation to a _mgp.CypherType instance."""
simple_types = {
@@ -1521,72 +1514,6 @@ def _typing_to_cypher_type(type_):
return parse_typing(str(type_))
def _is_typing_same(type1_, type2_):
"""Convert typing annotation to a _mgp.CypherType instance."""
simple_types = {
typing.Any: 1,
object: 2,
list: 3,
Any: 4,
bool: 5,
str: 6,
int: 7,
float: 8,
Number: 9,
Map: 10,
Vertex: 11,
Edge: 12,
Path: 13,
Date: 14,
LocalTime: 15,
LocalDateTime: 16,
Duration: 17,
}
try:
return simple_types[type1_] == simple_types[type2_]
except KeyError:
pass
if sys.version_info < (3, 8):
# skip type checks
return True
complex_type1 = typing.get_origin(type1_)
type_args1 = typing.get_args(type2_)
complex_type2 = typing.get_origin(type1_)
type_args2 = typing.get_args(type2_)
if complex_type2 != complex_type1:
raise UnequalTypesError(type1_, type2_)
if complex_type1 == typing.Union:
contains_none_arg1 = type(None) in type_args1
contains_none_arg2 = type(None) in type_args2
if contains_none_arg1 != contains_none_arg2:
raise UnequalTypesError(type1_, type2_)
if contains_none_arg1:
types1 = tuple(t for t in type_args1 if t is not type(None)) # noqa E721
types2 = tuple(t for t in type_args2 if t is not type(None)) # noqa E721
if len(types1) != len(types2):
raise UnequalTypesError(types1, types2)
if len(types1) == 1:
(type_arg1,) = types1
(type_arg2,) = types2
else:
type_arg1 = typing.Union.__getitem__(types1)
type_arg2 = typing.Union.__getitem__(types2)
return _is_typing_same(type_arg1, type_arg2)
elif complex_type1 == list:
(type_arg1,) = type_args1
(type_arg2,) = type_args2
return _is_typing_same(type_arg1, type_arg2)
# skip type checks
return True
# Procedure registration
@@ -1746,92 +1673,6 @@ def write_proc(func: typing.Callable[..., Record]):
return _register_proc(func, True)
def _register_batch_proc(
func: typing.Callable[..., Record], initializer: typing.Callable, cleanup: typing.Callable, is_write: bool
):
raise_if_does_not_meet_requirements(func)
register_func = _mgp.Module.add_batch_write_procedure if is_write else _mgp.Module.add_batch_read_procedure
func_sig = inspect.signature(func)
func_params = tuple(func_sig.parameters.values())
initializer_sig = inspect.signature(initializer)
initializer_params = tuple(initializer_sig.parameters.values())
assert (
func_params and initializer_params or not func_params and not initializer_params
), "Both function params and initializer params must exist or not exist"
assert len(func_params) == len(initializer_params), "Number of params must be same"
assert initializer_sig.return_annotation is initializer_sig.empty, "Initializer can't return anything"
if func_params and func_params[0].annotation is ProcCtx:
assert (
initializer_params and initializer_params[0].annotation is ProcCtx
), "Initializer must have mgp.ProcCtx as first parameter"
@wraps(func)
def wrapper_func(graph, args):
return func(ProcCtx(graph), *args)
@wraps(initializer)
def wrapper_initializer(graph, args):
return initializer(ProcCtx(graph), *args)
func_params = func_params[1:]
initializer_params = initializer_params[1:]
mgp_proc = register_func(_mgp._MODULE, wrapper_func, wrapper_initializer, cleanup)
else:
@wraps(func)
def wrapper_func(graph, args):
return func(*args)
@wraps(initializer)
def wrapper_initializer(graph, args):
return initializer(*args)
mgp_proc = register_func(_mgp._MODULE, wrapper_func, wrapper_initializer, cleanup)
for func_param, initializer_param in zip(func_params, initializer_params):
func_param_name = func_param.name
func_param_type_ = func_param.annotation
if func_param_type_ is func_param.empty:
func_param_type_ = object
initializer_param_type_ = initializer_param.annotation
if initializer_param.annotation is initializer_param.empty:
initializer_param_type_ = object
assert _is_typing_same(
func_param_type_, initializer_param_type_
), "Types of initializer and function must be same"
func_cypher_type = _typing_to_cypher_type(func_param_type_)
if func_param.default is func_param.empty:
mgp_proc.add_arg(func_param_name, func_cypher_type)
else:
mgp_proc.add_opt_arg(func_param_name, func_cypher_type, func_param.default)
if func_sig.return_annotation is not func_sig.empty:
record = func_sig.return_annotation
if not isinstance(record, Record):
raise TypeError("Expected '{}' to return 'mgp.Record', got '{}'".format(func.__name__, type(record)))
for name, type_ in record.fields.items():
if isinstance(type_, Deprecated):
cypher_type = _typing_to_cypher_type(type_.field_type)
mgp_proc.add_deprecated_result(name, cypher_type)
else:
mgp_proc.add_result(name, _typing_to_cypher_type(type_))
return func
def add_batch_write_proc(func: typing.Callable[..., Record], initializer: typing.Callable, cleanup: typing.Callable):
return _register_batch_proc(func, initializer, cleanup, True)
def add_batch_read_proc(func: typing.Callable[..., Record], initializer: typing.Callable, cleanup: typing.Callable):
return _register_batch_proc(func, initializer, cleanup, False)
class InvalidMessageError(Exception):
"""
Signals using a message instance outside of the registered transformation.

18
init
View File

@@ -77,9 +77,7 @@ fi
# Fix for centos 7 during release
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then
if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then
python3 -m pip uninstall -y virtualenv
fi
python3 -m pip uninstall -y virtualenv
python3 -m pip install virtualenv
fi
@@ -102,15 +100,11 @@ setup_virtualenv tests/integration/ldap
echo "Done installing dependencies for Memgraph"
echo "Linking git hooks OR skip if .git folder is not there"
if [ -d "$DIR/.git" ]; then
for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook"
echo "Added $hook hook"
done;
else
echo "WARNING: .git folder not present, skip adding hooks"
fi
echo "Linking git hooks"
for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook"
echo "Added $hook hook"
done;
# Install precommit hook except on old operating systems because we don't
# develop on them -> pre-commit hook not required -> we can use latest

1
libs/.gitignore vendored
View File

@@ -6,4 +6,3 @@
!__main.cpp
!pulsar.patch
!antlr4.10.1.patch
!rocksdb8.1.1.patch

View File

@@ -12,7 +12,6 @@ find_package(Boost 1.78 REQUIRED)
find_package(BZip2 1.0.6 REQUIRED)
find_package(Threads REQUIRED)
set(GFLAGS_NOTHREADS OFF)
# NOTE: config/generate.py depends on the gflags help XML format.
find_package(gflags REQUIRED)
find_package(fmt 8.0.1)
find_package(Jemalloc REQUIRED)
@@ -29,7 +28,6 @@ function(import_header_library name include_dir)
set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
"Path to ${name} include directory" FORCE)
mark_as_advanced(${_upper_name}_INCLUDE_DIR)
add_library(lib::${name} ALIAS ${name})
endfunction(import_header_library)
function(import_library name type location include_dir)
@@ -258,6 +256,3 @@ import_external_library(librdtsc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include
CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS}
BUILD_COMMAND $(MAKE) rdtsc)
# setup ctre
import_header_library(ctre ${CMAKE_CURRENT_SOURCE_DIR})

View File

@@ -1,13 +0,0 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 598c728..816c705 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1242,7 +1242,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(ROCKSDB_BUILD_SHARED)
install(
- TARGETS ${ROCKSDB_SHARED_LIB}
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
EXPORT RocksDBTargets
COMPONENT runtime
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"

View File

@@ -71,8 +71,8 @@ file_get_try_double () {
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
filename="$(basename "$secondary_url")"
# Redirect primary/cache to /dev/null to make it less confusing for a new contributor because only CI has access to the cache.
wget -nv "$primary_url" -O "$filename" >/dev/null 2>&1 || wget -nv "$secondary_url" -O "$filename" || exit 1
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
echo ""
}
repo_clone_try_double () {
@@ -86,8 +86,8 @@ repo_clone_try_double () {
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
# Redirect primary/cache to /dev/null to make it less confusing for a new contributor because only CI has access to the cache.
clone "$primary_url" "$folder_name" "$ref" "$shallow" >/dev/null 2>&1 || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1
clone "$primary_url" "$folder_name" "$ref" "$shallow" || clone "$secondary_url" "$folder_name" "$ref" "$shallow" || exit 1
echo ""
}
# List all dependencies.
@@ -122,7 +122,6 @@ declare -A primary_urls=(
["protobuf"]="http://$local_cache_host/git/protobuf.git"
["pulsar"]="http://$local_cache_host/git/pulsar.git"
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
)
# The goal of secondary urls is to have links to the "source of truth" of
@@ -148,7 +147,6 @@ declare -A secondary_urls=(
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
["pulsar"]="https://github.com/apache/pulsar.git"
["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
["ctre"]="https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
)
# antlr
@@ -194,10 +192,10 @@ cd json
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
rocksdb_tag="v8.1.1" # (2023-04-21)
rocksdb_tag="v6.14.6" # (2020-10-14)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
pushd rocksdb
git apply ../rocksdb8.1.1.patch
git apply ../rocksdb.patch
popd
# mgclient
@@ -240,9 +238,3 @@ repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}"
pushd librdtsc
git apply ../librdtsc.patch
popd
#ctre
mkdir -p ctre
cd ctre
file_get_try_double "${primary_urls[ctre]}" "${secondary_urls[ctre]}"
cd ..

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2027-20-07
CHANGE DATE: 2027-05-04
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -48,7 +48,7 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6")
# Setting arhitecture extension for rpm packages
set(MG_ARCH_EXTENSION_RPM "noarch")
@@ -69,12 +69,6 @@ set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/rpm/memgraph.spec.in")
set(CPACK_RPM_PACKAGE_LICENSE "Memgraph License")
# CPACK deduces dependency to libstdc++ which:
# * can't be easily installed on Centos 7 (the one from the toolchain,
# only required to avoid printing issue within query modules)
# * it causes issues with glibcxx 2.4
# `if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")`
set(CPACK_RPM_PACKAGE_AUTOREQ " no")
# Description formatting is important, no line must be greater than 80 characters.
set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
@@ -83,7 +77,13 @@ the next generation of applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, logrotate")
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc++ >= 3.4.29, logrotate")
# If amzn-2
if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")
# It causes issues with glibcxx 2.4
set(CPACK_RPM_PACKAGE_AUTOREQ " no")
endif()
# All variables must be set before including.
include(CPack)

View File

@@ -55,8 +55,7 @@ image_name="memgraph:${version}"
image_package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
# NOTE: --pull is here to always pull that latest base image because of security patches.
docker build --pull -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
--build-arg BINARY_NAME=${package_name} \
--build-arg EXTENSION=${extension} \
--build-arg TARGETARCH="" .

View File

@@ -9,7 +9,6 @@ RUN yum -y update \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-amzn-2-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -9,7 +9,6 @@ RUN yum -y update \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -9,7 +9,6 @@ RUN yum -y update \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -10,7 +10,6 @@ RUN yum -y update \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-fedora-36-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -17,8 +17,7 @@ ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () {
# TODO(gitbuda): Update the release/package/run.sh help
echo "$0 init|package|docker|test {os} [--for-docker|--for-platform]"
echo "$0 init|package {os} [--for-docker|--for-platform]|docker|test"
echo ""
echo " OSs: ${SUPPORTED_OS[*]}"
exit 1
@@ -36,7 +35,7 @@ make_package () {
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
docker exec "$build_container" bash -c "apt --allow-releaseinfo-change -y update"
docker exec "$build_container" bash -c "apt update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
@@ -65,7 +64,6 @@ make_package () {
git fetch origin master:master
fi
docker exec "$build_container" mkdir -p /memgraph
# TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI.
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
container_build_dir="/memgraph/build"
@@ -76,8 +74,6 @@ make_package () {
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
# TODO(gitbuda): On the other side, having this here allows updating deps
# wihout reruning the build containers.
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
@@ -87,7 +83,6 @@ make_package () {
docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'"
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
# TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX
if [[ "$os" =~ "-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else
@@ -113,13 +108,8 @@ make_package () {
case "$1" in
init)
cd "$SCRIPT_DIR"
if ! which "docker-compose" >/dev/null; then
docker_compose_cmd="docker compose"
else
docker_compose_cmd="docker-compose"
fi
$docker_compose_cmd build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
$docker_compose_cmd up -d
docker-compose build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
docker-compose up -d
;;
docker)
@@ -159,19 +149,6 @@ case "$1" in
fi
;;
build)
shift 1
if [[ "$#" -ne 2 ]]; then
print_help
fi
# in the vX format, e.g. v5
toolchain_version="$1"
# a name of the os folder, e.g. ubuntu-22.04-arm
os="$2"
cd "$SCRIPT_DIR/$os"
docker build -f Dockerfile --build-arg TOOLCHAIN_VERSION="toolchain-$toolchain_version" -t "memgraph/memgraph-builder:${toolchain_version}_$os" .
;;
test)
echo "TODO(gitbuda): Test all packages on mgtest containers."
;;

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-arm64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -12,7 +12,6 @@ RUN apt update && apt install -y \
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt \
&& rm ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,7 +1,6 @@
# CMake configuration for the main memgraph library and executable
# add memgraph sub libraries, ordered by dependency
add_subdirectory(csv)
add_subdirectory(utils)
add_subdirectory(requests)
add_subdirectory(io)
@@ -17,7 +16,10 @@ add_subdirectory(slk)
add_subdirectory(rpc)
add_subdirectory(license)
add_subdirectory(auth)
add_subdirectory(audit)
if(MG_ENTERPRISE)
add_subdirectory(audit)
endif()
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -34,7 +36,12 @@ set(mg_single_node_v2_sources
)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
mg-telemetry mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-glue mg-audit)
mg-telemetry mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-glue)
if(MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
endif()
# memgraph main executable
add_executable(memgraph ${mg_single_node_v2_sources})

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,8 +20,6 @@ inline constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
enum class Signature : uint8_t {
Noop = 0x00,
Init = 0x01,
LogOn = 0x6A,
LogOff = 0x6B,
AckFailure = 0x0E, // only v1
Reset = 0x0F,
Goodbye = 0x02,

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -28,7 +28,7 @@ inline constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
*/
inline constexpr size_t kHandshakeSize = 20;
inline constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403, 0x0502};
inline constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
inline constexpr int kPullAll = -1;
inline constexpr int kPullLast = -1;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,14 +33,7 @@ namespace memgraph::communication::bolt {
template <typename Buffer>
class Decoder {
public:
explicit Decoder(Buffer &buffer) : buffer_(buffer), major_v_(0) {}
/**
* Lets the user update the version.
* This is all single thread for now. TODO: Update if ever multithreaded.
* @param major_v the major version of the Bolt protocol used.
*/
void UpdateVersion(int major_v) { major_v_ = major_v; }
explicit Decoder(Buffer &buffer) : buffer_(buffer) {}
/**
* Reads a Value from the available data in the buffer.
@@ -215,10 +208,6 @@ class Decoder {
protected:
Buffer &buffer_;
int major_v_; //!< Major version of the underlying Bolt protocol
// TODO: when refactoring
// Ideally the major_v would be a compile time constant. If the higher level (Bolt driver) ends up being separate
// classes, this could be just a template and each version of the driver would use the appropriate decoder.
private:
bool ReadNull(const Marker &marker, Value *data) {
@@ -381,7 +370,11 @@ class Decoder {
}
ret.emplace(std::move(dv_key.ValueString()), std::move(dv_val));
}
return ret.size() == size;
if (ret.size() != size) {
return false;
}
return true;
}
bool ReadVertex(Value *data) {
@@ -414,14 +407,6 @@ class Decoder {
}
vertex.properties = std::move(dv.ValueMap());
if (major_v_ > 4) {
// element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
vertex.element_id = std::move(dv.ValueString());
}
return true;
}
@@ -460,23 +445,6 @@ class Decoder {
}
edge.properties = std::move(dv.ValueMap());
if (major_v_ > 4) {
// element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.element_id = std::move(dv.ValueString());
// from_element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.from_element_id = std::move(dv.ValueString());
// to_element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.to_element_id = std::move(dv.ValueString());
}
return true;
}
@@ -503,14 +471,6 @@ class Decoder {
}
edge.properties = std::move(dv.ValueMap());
if (major_v_ > 4) {
// element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.element_id = std::move(dv.ValueString());
}
return true;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -36,14 +36,7 @@ namespace memgraph::communication::bolt {
template <typename Buffer>
class BaseEncoder {
public:
explicit BaseEncoder(Buffer &buffer) : buffer_(buffer), major_v_(0) {}
/**
* Lets the user update the version.
* This is all single thread for now. TODO: Update if ever multithreaded.
* @param major_v the major version of the Bolt protocol used.
*/
void UpdateVersion(int major_v) { major_v_ = major_v; }
explicit BaseEncoder(Buffer &buffer) : buffer_(buffer) {}
void WriteRAW(const uint8_t *data, uint64_t len) { buffer_.Write(data, len); }
@@ -123,8 +116,7 @@ class BaseEncoder {
}
void WriteVertex(const Vertex &vertex) {
int struct_n = 3 + 1 * int(major_v_ > 4); // element_id introduced from v5
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + struct_n);
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
WriteRAW(utils::UnderlyingCast(Signature::Node));
WriteInt(vertex.id.AsInt());
@@ -140,16 +132,10 @@ class BaseEncoder {
WriteString(prop.first);
WriteValue(prop.second);
}
if (major_v_ > 4) {
// element_id introduced in v5.0
WriteString(vertex.element_id);
}
}
void WriteEdge(const Edge &edge, bool unbound = false) {
int struct_n = (unbound ? 3 + 1 * int(major_v_ > 4) : 5 + 3 * int(major_v_ > 4)); // element_id introduced from v5
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + struct_n);
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + (unbound ? 3 : 5));
WriteRAW(utils::UnderlyingCast(unbound ? Signature::UnboundRelationship : Signature::Relationship));
WriteInt(edge.id.AsInt());
@@ -166,22 +152,10 @@ class BaseEncoder {
WriteString(prop.first);
WriteValue(prop.second);
}
if (major_v_ > 4) {
// element_id introduced in v5.0
WriteString(edge.element_id);
if (!unbound) {
// from_element_id introduced in v5.0
WriteString(edge.from_element_id);
// to_element_id introduced in v5.0
WriteString(edge.to_element_id);
}
}
}
void WriteEdge(const UnboundedEdge &edge) {
const int struct_n = 3 + 1 * int(major_v_ > 4); // element_id introduced from v5
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + struct_n);
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
WriteRAW(utils::UnderlyingCast(Signature::UnboundRelationship));
WriteInt(edge.id.AsInt());
@@ -194,11 +168,6 @@ class BaseEncoder {
WriteString(prop.first);
WriteValue(prop.second);
}
if (major_v_ > 4) {
// element_id introduced in v5.0
WriteString(edge.element_id);
}
}
void WritePath(const Path &path) {
@@ -295,7 +264,6 @@ class BaseEncoder {
protected:
Buffer &buffer_;
int major_v_; //!< Major version of the underlying Bolt protocol (TODO: Think about reimplementing the versioning)
private:
template <class T>

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -41,8 +41,6 @@ class ClientEncoder : private BaseEncoder<Buffer> {
public:
ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
using BaseEncoder<Buffer>::UpdateVersion;
/**
* Writes a Init message.
*

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -34,8 +34,6 @@ class Encoder : private BaseEncoder<Buffer> {
public:
Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
using BaseEncoder<Buffer>::UpdateVersion;
/**
* Sends a Record message.
*

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -63,8 +63,7 @@ class Session {
* if an explicit transaction was started.
*/
virtual std::pair<std::vector<std::string>, std::optional<int>> Interpret(
const std::string &query, const std::map<std::string, Value> &params,
const std::map<std::string, memgraph::communication::bolt::Value> &metadata) = 0;
const std::string &query, const std::map<std::string, Value> &params) = 0;
/**
* Put results of the processed query in the `encoder`.
@@ -86,7 +85,7 @@ class Session {
*/
virtual std::map<std::string, Value> Discard(std::optional<int> n, std::optional<int> qid) = 0;
virtual void BeginTransaction(const std::map<std::string, memgraph::communication::bolt::Value> &) = 0;
virtual void BeginTransaction() = 0;
virtual void CommitTransaction() = 0;
virtual void RollbackTransaction() = 0;
@@ -121,9 +120,6 @@ class Session {
return;
}
handshake_done_ = true;
// Update the decoder's Bolt version (v5 has changed the undelying structure)
decoder_.UpdateVersion(version_.major);
encoder_.UpdateVersion(version_.major);
}
ChunkState chunk_state;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,15 +22,10 @@
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/states/handlers.hpp"
#include "communication/bolt/v1/value.hpp"
#include "utils/event_counter.hpp"
#include "utils/likely.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace memgraph::metrics {
extern const Event BoltMessages;
} // namespace memgraph::metrics
namespace memgraph::communication::bolt {
template <typename TSession>
@@ -91,37 +86,6 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
}
}
template <typename TSession>
State RunHandlerV5(Signature signature, TSession &session, State state, Marker marker) {
switch (signature) {
case Signature::Run:
return HandleRunV5<TSession>(session, state, marker);
case Signature::Pull:
return HandlePullV5<TSession>(session, state, marker);
case Signature::Discard:
return HandleDiscardV5<TSession>(session, state, marker);
case Signature::Reset:
return HandleReset<TSession>(session, marker);
case Signature::Begin:
return HandleBegin<TSession>(session, state, marker);
case Signature::Commit:
return HandleCommit<TSession>(session, state, marker);
case Signature::Goodbye:
return HandleGoodbye<TSession>();
case Signature::Rollback:
return HandleRollback<TSession>(session, state, marker);
case Signature::Noop:
return HandleNoop<TSession>(state);
case Signature::Route:
return HandleRoute<TSession>(session, marker);
case Signature::LogOff:
return HandleLogOff<TSession>();
default:
spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
return State::Close;
}
}
/**
* Executor state run function
* This function executes an initialized Bolt session.
@@ -139,10 +103,8 @@ State StateExecutingRun(TSession &session, State state) {
switch (session.version_.major) {
case 1:
memgraph::metrics::IncrementCounter(memgraph::metrics::BoltMessages);
return RunHandlerV1(signature, session, state, marker);
case 4: {
memgraph::metrics::IncrementCounter(memgraph::metrics::BoltMessages);
if (session.version_.minor >= 3) {
return RunHandlerV4<TSession, 3>(signature, session, state, marker);
}
@@ -151,8 +113,6 @@ State StateExecutingRun(TSession &session, State state) {
}
return RunHandlerV4<TSession>(signature, session, state, marker);
}
case 5:
return RunHandlerV5<TSession>(signature, session, state, marker);
default:
spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
return State::Close;

View File

@@ -12,7 +12,6 @@
#pragma once
#include <map>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
@@ -23,7 +22,6 @@
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
@@ -73,23 +71,6 @@ inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::ex
"should be in database logs."};
}
namespace helpers {
/** Extracts metadata from the extras field.
* NOTE: In order to avoid a copy, the metadata in moved.
* TODO: Update if extra field is used for anything else.
*/
inline std::map<std::string, Value> ConsumeMetadata(Value &extra) {
std::map<std::string, Value> md;
auto &md_tv = extra.ValueMap()["tx_metadata"];
if (md_tv.IsMap()) {
md = std::move(md_tv.ValueMap());
}
return md;
}
} // namespace helpers
namespace details {
template <bool is_pull, typename TSession>
@@ -228,7 +209,7 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap(), {});
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
@@ -269,7 +250,6 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
// Even though this part seems unnecessary it is needed to move the buffer
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra field!");
return State::Close;
}
if (state != State::Idle) {
@@ -286,8 +266,7 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
try {
// Interpret can throw.
const auto [header, qid] =
session.Interpret(query.ValueString(), params.ValueMap(), helpers::ConsumeMetadata(extra));
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
@@ -309,12 +288,6 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
}
}
template <typename TSession>
State HandleRunV5(TSession &session, const State state, const Marker marker) {
// Using V4 on purpose
return HandleRunV4<TSession>(session, state, marker);
}
template <typename TSession>
State HandlePullV1(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV1<true>(session, state, marker);
@@ -325,12 +298,6 @@ State HandlePullV4(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV4<true>(session, state, marker);
}
template <typename TSession>
State HandlePullV5(TSession &session, const State state, const Marker marker) {
// Using V4 on purpose
return HandlePullV4<TSession>(session, state, marker);
}
template <typename TSession>
State HandleDiscardV1(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV1<false>(session, state, marker);
@@ -341,12 +308,6 @@ State HandleDiscardV4(TSession &session, const State state, const Marker marker)
return details::HandlePullDiscardV4<false>(session, state, marker);
}
template <typename TSession>
State HandleDiscardV5(TSession &session, const State state, const Marker marker) {
// Using V4 on purpose
return HandleDiscardV4<TSession>(session, state, marker);
}
template <typename TSession>
State HandleReset(TSession &session, const Marker marker) {
// IMPORTANT: This implementation of the Bolt RESET command isn't fully
@@ -399,7 +360,7 @@ State HandleBegin(TSession &session, const State state, const Marker marker) {
}
try {
session.BeginTransaction(helpers::ConsumeMetadata(extra));
session.BeginTransaction();
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
@@ -504,10 +465,4 @@ State HandleRoute(TSession &session, const Marker marker) {
}
return State::Error;
}
template <typename TSession>
State HandleLogOff() {
// Not arguments sent, the user just needs to reauthenticate
return State::Init;
}
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -27,25 +27,17 @@ namespace details {
template <typename TSession>
std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
// Get authentication data.
// From neo4j driver v4.4, fields that have a default value are not sent.
// In order to have back-compatibility, the missing fields will be added.
auto &data = metadata.ValueMap();
if (data.empty()) { // Special case auth=None
spdlog::warn("The client didn't supply the authentication scheme! Trying with \"none\"...");
data["scheme"] = "none";
if (!data.count("scheme")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
}
std::string username;
std::string password;
if (data["scheme"].ValueString() == "basic") {
if (!data.count("principal")) { // Special case principal = ""
spdlog::warn("The client didn't supply the principal field! Trying with \"\"...");
data["principal"] = "";
}
if (!data.count("credentials")) { // Special case credentials = ""
spdlog::warn("The client didn't supply the credentials field! Trying with \"\"...");
data["credentials"] = "";
if (!data.count("principal") || !data.count("credentials")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
}
username = data["principal"].ValueString();
password = data["credentials"].ValueString();
@@ -114,30 +106,6 @@ std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
return std::nullopt;
}
auto &data = metadata.ValueMap();
if (!data.count("user_agent")) {
spdlog::warn("The client didn't supply the user agent!");
return std::nullopt;
}
spdlog::info("Client connected '{}'", data.at("user_agent").ValueString());
return metadata;
}
template <typename TSession>
std::optional<Value> GetInitDataV5(TSession &session, const Marker marker) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return std::nullopt;
}
Value metadata;
if (!session.decoder_.ReadValue(&metadata, Value::Type::Map)) {
spdlog::trace("Couldn't read metadata!");
return std::nullopt;
}
const auto &data = metadata.ValueMap();
if (!data.count("user_agent")) {
spdlog::warn("The client didn't supply the user agent!");
@@ -149,22 +117,6 @@ std::optional<Value> GetInitDataV5(TSession &session, const Marker marker) {
return metadata;
}
template <typename TSession>
std::optional<Value> GetAuthDataV5(TSession &session, const Marker marker) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return std::nullopt;
}
Value metadata;
if (!session.decoder_.ReadValue(&metadata, Value::Type::Map)) {
spdlog::trace("Couldn't read metadata!");
return std::nullopt;
}
return metadata;
}
template <typename TSession>
State SendSuccessMessage(TSession &session) {
// Neo4j's Java driver 4.1.1+ requires connection_id.
@@ -228,57 +180,6 @@ State StateInitRunV4(TSession &session, Marker marker, Signature signature) {
return SendSuccessMessage(session);
}
template <typename TSession>
State StateInitRunV5(TSession &session, Marker marker, Signature signature) {
if (signature == Signature::Noop) [[unlikely]] {
SPDLOG_DEBUG("Received NOOP message");
return State::Init;
}
if (signature == Signature::Init) {
auto maybeMetadata = GetInitDataV5(session, marker);
if (!maybeMetadata) {
return State::Close;
}
if (SendSuccessMessage(session) == State::Close) {
return State::Close;
}
// Stay in Init
return State::Init;
} else if (signature == Signature::LogOn) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace(
"The client sent malformed data, but we are continuing "
"because the official Neo4j Java driver sends malformed "
"data. D'oh!");
return State::Close;
}
auto maybeMetadata = GetAuthDataV5(session, marker);
if (!maybeMetadata) {
return State::Close;
}
auto result = AuthenticateUser(session, *maybeMetadata);
if (result) {
spdlog::trace("Failed to authenticate, closing connection...");
return State::Close;
}
if (SendSuccessMessage(session) == State::Close) {
return State::Close;
}
return State::Idle;
} else [[unlikely]] {
spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
return State::Close;
}
}
} // namespace details
/**
@@ -307,9 +208,6 @@ State StateInitRun(TSession &session) {
}
return details::StateInitRunV4<TSession>(session, marker, signature);
}
case 5: {
return details::StateInitRunV5<TSession>(session, marker, signature);
}
}
spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
return State::Close;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -342,9 +342,6 @@ std::ostream &operator<<(std::ostream &os, const Vertex &vertex) {
[&](auto &stream, const auto &pair) { stream << pair.first << ": " << pair.second; });
os << "}";
}
if (!vertex.element_id.empty()) {
os << " element_id: " << vertex.element_id;
}
return os << ")";
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -57,7 +57,6 @@ struct Vertex {
Id id;
std::vector<std::string> labels;
std::map<std::string, Value> properties;
std::string element_id;
};
/**
@@ -70,9 +69,6 @@ struct Edge {
Id to;
std::string type;
std::map<std::string, Value> properties;
std::string element_id;
std::string from_element_id;
std::string to_element_id;
};
/**
@@ -83,7 +79,6 @@ struct UnboundedEdge {
Id id;
std::string type;
std::map<std::string, Value> properties;
std::string element_id;
};
/**

View File

@@ -1,108 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <list>
#include <memory>
#include <spdlog/spdlog.h>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core.hpp>
#include "communication/context.hpp"
#include "communication/http/session.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
namespace memgraph::communication::http {
template <class TRequestHandler, typename TSessionData>
class Listener final : public std::enable_shared_from_this<Listener<TRequestHandler, TSessionData>> {
using tcp = boost::asio::ip::tcp;
using SessionHandler = Session<TRequestHandler, TSessionData>;
using std::enable_shared_from_this<Listener<TRequestHandler, TSessionData>>::shared_from_this;
public:
Listener(const Listener &) = delete;
Listener(Listener &&) = delete;
Listener &operator=(const Listener &) = delete;
Listener &operator=(Listener &&) = delete;
~Listener() {}
template <typename... Args>
static std::shared_ptr<Listener> Create(Args &&...args) {
return std::shared_ptr<Listener>{new Listener(std::forward<Args>(args)...)};
}
// Start accepting incoming connections
void Run() { DoAccept(); }
tcp::endpoint GetEndpoint() const { return acceptor_.local_endpoint(); }
private:
Listener(boost::asio::io_context &ioc, TSessionData *data, ServerContext *context, tcp::endpoint endpoint)
: ioc_(ioc), data_(data), context_(context), acceptor_(ioc) {
boost::beast::error_code ec;
// Open the acceptor
acceptor_.open(endpoint.protocol(), ec);
if (ec) {
LogError(ec, "open");
return;
}
// Allow address reuse
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
if (ec) {
LogError(ec, "set_option");
return;
}
// Bind to the server address
acceptor_.bind(endpoint, ec);
if (ec) {
LogError(ec, "bind");
return;
}
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
if (ec) {
LogError(ec, "listen");
return;
}
spdlog::info("HTTP server is listening on {}:{}", endpoint.address(), endpoint.port());
}
void DoAccept() {
acceptor_.async_accept(ioc_, [shared_this = shared_from_this()](auto ec, auto socket) {
shared_this->OnAccept(ec, std::move(socket));
});
}
void OnAccept(boost::beast::error_code ec, tcp::socket socket) {
if (ec) {
return LogError(ec, "accept");
}
SessionHandler::Create(std::move(socket), data_, *context_)->Run();
DoAccept();
}
boost::asio::io_context &ioc_;
TSessionData *data_;
ServerContext *context_;
tcp::acceptor acceptor_;
};
} // namespace memgraph::communication::http

View File

@@ -1,65 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <thread>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include "communication/http/listener.hpp"
#include "io/network/endpoint.hpp"
namespace memgraph::communication::http {
template <class TRequestHandler, typename TSessionData>
class Server final {
using tcp = boost::asio::ip::tcp;
public:
explicit Server(io::network::Endpoint endpoint, TSessionData *data, ServerContext *context)
: listener_{Listener<TRequestHandler, TSessionData>::Create(
ioc_, data, context, tcp::endpoint{boost::asio::ip::make_address(endpoint.address), endpoint.port})} {}
Server(const Server &) = delete;
Server(Server &&) = delete;
Server &operator=(const Server &) = delete;
Server &operator=(Server &&) = delete;
~Server() {
MG_ASSERT(!background_thread_ || (ioc_.stopped() && !background_thread_->joinable()),
"Server wasn't shutdown properly");
}
void Start() {
MG_ASSERT(!background_thread_, "The server was already started!");
listener_->Run();
background_thread_.emplace([this] { ioc_.run(); });
}
void Shutdown() { ioc_.stop(); }
void AwaitShutdown() {
if (background_thread_ && background_thread_->joinable()) {
background_thread_->join();
}
}
bool IsRunning() const { return background_thread_ && !ioc_.stopped(); }
tcp::endpoint GetEndpoint() const { return listener_->GetEndpoint(); }
private:
boost::asio::io_context ioc_;
std::shared_ptr<Listener<TRequestHandler, TSessionData>> listener_;
std::optional<std::thread> background_thread_;
};
} // namespace memgraph::communication::http

View File

@@ -1,193 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <deque>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <variant>
#include <spdlog/spdlog.h>
#include <boost/asio/bind_executor.hpp>
#include <boost/asio/dispatch.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core/buffers_to_string.hpp>
#include <boost/beast/core/stream_traits.hpp>
#include <boost/beast/core/tcp_stream.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/version.hpp>
#include <json/json.hpp>
#include "communication/context.hpp"
#include "utils/logging.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::communication::http {
inline constexpr uint16_t kSSLExpirySeconds = 30;
inline void LogError(boost::beast::error_code ec, const std::string_view what) {
spdlog::warn("HTTP session failed on {}: {}", what, ec.message());
}
template <class TRequestHandler, typename TSessionData>
class Session : public std::enable_shared_from_this<Session<TRequestHandler, TSessionData>> {
using tcp = boost::asio::ip::tcp;
using std::enable_shared_from_this<Session<TRequestHandler, TSessionData>>::shared_from_this;
public:
template <typename... Args>
static std::shared_ptr<Session> Create(Args &&...args) {
return std::shared_ptr<Session>{new Session{std::forward<Args>(args)...}};
}
void Run() {
if (auto *ssl = std::get_if<SSLSocket>(&stream_); ssl != nullptr) {
try {
boost::beast::get_lowest_layer(*ssl).expires_after(std::chrono::seconds(kSSLExpirySeconds));
ssl->handshake(boost::asio::ssl::stream_base::server);
} catch (const boost::system::system_error &e) {
spdlog::warn("Failed on SSL handshake: {}", e.what());
return;
}
}
// run on the strand
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
}
private:
using PlainSocket = boost::beast::tcp_stream;
using SSLSocket = boost::beast::ssl_stream<boost::beast::tcp_stream>;
explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &context)
: stream_(CreateSocket(std::move(socket), context)),
handler_(data),
strand_{boost::asio::make_strand(GetExecutor())} {}
std::variant<PlainSocket, SSLSocket> CreateSocket(tcp::socket &&socket, ServerContext &context) {
if (context.use_ssl()) {
ssl_context_.emplace(context.context_clone());
return Session::SSLSocket{std::move(socket), *ssl_context_};
}
return Session::PlainSocket{std::move(socket)};
}
void OnWrite(boost::beast::error_code ec, size_t bytes_transferred) {
boost::ignore_unused(bytes_transferred);
if (ec) {
close_ = true;
return LogError(ec, "write");
}
if (close_) {
DoClose();
return;
}
res_ = nullptr;
DoRead();
}
void DoRead() {
req_ = {};
ExecuteForStream([this](auto &&stream) {
boost::beast::get_lowest_layer(stream).expires_after(std::chrono::seconds(kSSLExpirySeconds));
boost::beast::http::async_read(
stream, buffer_, req_,
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnRead, shared_from_this())));
});
}
void OnRead(boost::beast::error_code ec, size_t bytes_transferred) {
boost::ignore_unused(bytes_transferred);
if (ec == boost::beast::http::error::end_of_stream) {
DoClose();
return;
}
if (ec) {
return LogError(ec, "read");
}
auto async_write = [this](boost::beast::http::response<boost::beast::http::string_body> msg) {
ExecuteForStream([this, &msg](auto &&stream) {
// The lifetime of the message has to extend
// for the duration of the async operation so
// we use a shared_ptr to manage it.
auto sp = std::make_shared<boost::beast::http::response<boost::beast::http::string_body>>(std::move(msg));
// Store a type-erased version of the shared
// pointer in the class to keep it alive.
res_ = sp;
// Write the response
boost::beast::http::async_write(
stream, *sp, boost::asio::bind_executor(strand_, std::bind_front(&Session::OnWrite, shared_from_this())));
});
};
// handle request
handler_.HandleRequest(std::move(req_), async_write);
}
void DoClose() {
std::visit(utils::Overloaded{[this](SSLSocket &stream) {
boost::beast::get_lowest_layer(stream).expires_after(std::chrono::seconds(30));
// Perform the SSL shutdown
stream.async_shutdown(
boost::beast::bind_front_handler(&Session::OnClose, shared_from_this()));
},
[](PlainSocket &stream) {
boost::beast::error_code ec;
stream.socket().shutdown(tcp::socket::shutdown_send, ec);
}},
stream_);
}
void OnClose(boost::beast::error_code ec) {
if (ec) {
LogError(ec, "close");
}
// At this point the connection is closed gracefully
}
auto GetExecutor() {
return std::visit(utils::Overloaded{[](auto &&stream) { return stream.get_executor(); }}, stream_);
}
template <typename F>
decltype(auto) ExecuteForStream(F &&fn) {
return std::visit(utils::Overloaded{std::forward<F>(fn)}, stream_);
}
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ssl_context_;
std::variant<PlainSocket, SSLSocket> stream_;
boost::beast::flat_buffer buffer_;
TRequestHandler handler_;
boost::beast::http::request<boost::beast::http::string_body> req_;
std::shared_ptr<void> res_;
boost::asio::strand<boost::beast::tcp_stream::executor_type> strand_;
bool close_{false};
};
} // namespace memgraph::communication::http

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,7 +30,6 @@
#include "communication/context.hpp"
#include "communication/v2/pool.hpp"
#include "communication/v2/session.hpp"
#include "utils/message.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"

View File

@@ -44,18 +44,9 @@
#include "communication/buffer.hpp"
#include "communication/context.hpp"
#include "communication/exceptions.hpp"
#include "utils/event_counter.hpp"
#include "utils/logging.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::metrics {
extern const Event ActiveSessions;
extern const Event ActiveTCPSessions;
extern const Event ActiveSSLSessions;
extern const Event ActiveWebSocketSessions;
} // namespace memgraph::metrics
namespace memgraph::communication::v2 {
/**
@@ -109,8 +100,6 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
// Start the asynchronous accept operation
template <class Body, class Allocator>
void DoAccept(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> req) {
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveWebSocketSessions);
execution_active_ = true;
// Set suggested timeout settings for the websocket
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
@@ -225,10 +214,6 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
if (!IsConnected()) {
return;
}
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveSessions);
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveWebSocketSessions);
if (ec) {
return OnError(ec, "close");
}
@@ -275,19 +260,12 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
if (execution_active_) {
return false;
}
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveSessions);
execution_active_ = true;
timeout_timer_.async_wait(boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
if (std::holds_alternative<SSLSocket>(socket_)) {
utils::OnScopeExit increment_counter(
[] { memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveSSLSessions); });
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoHandshake(); });
} else {
utils::OnScopeExit increment_counter(
[] { memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveTCPSessions); });
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
}
return true;
@@ -473,14 +451,6 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
}
void OnClose(const boost::system::error_code &ec) {
if (ssl_context_.has_value()) {
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveSSLSessions);
} else {
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTCPSessions);
}
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveSessions);
if (ec) {
return OnError(ec);
}
@@ -496,7 +466,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
// The deadline has passed. Stop the session. The other actors will
// terminate as soon as possible.
spdlog::info("Shutting down session after {} seconds of inactivity", timeout_seconds_.count());
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
DoShutdown();
} else {
// Put the actor back to sleep.

View File

@@ -1,18 +0,0 @@
add_library(mg-csv STATIC)
add_library(mg::csv ALIAS mg-csv)
target_sources(mg-csv
PUBLIC
include/csv/parsing.hpp
PRIVATE
parsing.cpp
)
target_include_directories(mg-csv PUBLIC include)
find_package(Boost REQUIRED COMPONENTS iostreams)
target_link_libraries(mg-csv
PUBLIC mg-utils
PRIVATE lib::ctre mg-requests Boost::iostreams
)
add_subdirectory(fuzz)

View File

@@ -1,5 +0,0 @@
add_executable(fuzz_csv EXCLUDE_FROM_ALL)
target_sources(fuzz_csv PRIVATE fuzz_reader.cpp)
target_link_libraries(fuzz_csv PRIVATE mg::csv)
target_compile_options(fuzz_csv PRIVATE -fsanitize=fuzzer)
target_link_libraries(fuzz_csv PRIVATE -fsanitize=fuzzer)

View File

@@ -1,78 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <cstdint>
#include <optional>
#include <sstream>
#include "csv/parsing.hpp"
#include "utils/string.hpp"
namespace mg = memgraph;
namespace csv = mg::csv;
using pmr_str = mg::utils::pmr::string;
extern "C" int LLVMFuzzerTestOneInput(std::uint8_t const *data, std::size_t size) {
// need to parse a config
if (size < 4) return 0;
auto const with_header = bool(data[0]);
auto const ignore_bad = bool(data[1]);
auto const delim_size = data[2];
auto const quote_size = data[3];
// 0 will be nullopt, everything else will be one smaller
// 0 , 1, 2, 3, 4
// nullopt, 0, 1, 2, 3
auto const delim_real_size = delim_size == 0 ? 0 : delim_size - 1;
auto const quote_real_size = quote_size == 0 ? 0 : quote_size - 1;
// not worth testing if too large
if (delim_real_size > 3 || quote_real_size > 3) return 0;
// is there enough space for them to exist
if (size < 4 + delim_real_size + quote_real_size) return 0;
auto const delim_start = 4;
auto const delim_end = delim_start + delim_real_size;
auto const quote_start = delim_end;
auto const quote_end = quote_start + quote_real_size;
auto *mem = mg::utils::NewDeleteResource();
auto delim = delim_size == 0 ? std::optional<pmr_str>{} : pmr_str(&data[delim_start], &data[delim_end], mem);
auto quote = quote_size == 0 ? std::optional<pmr_str>{} : pmr_str(&data[quote_start], &data[quote_end], mem);
auto const remaining = static_cast<int64_t>(size) - quote_end;
if (remaining < 0) __builtin_trap(); // if this hits, above parsing is wrong
// #############################################################################################################
// build Config
auto cfg = csv::Reader::Config{with_header, ignore_bad, std::move(delim), std::move(quote)};
// build CSV source
auto ss = std::stringstream{};
ss.write(reinterpret_cast<char const *>(&data[quote_end]), static_cast<std::streamsize>(remaining));
auto source = csv::StreamCsvSource{std::move(ss)};
// #############################################################################################################
try {
auto reader = memgraph::csv::Reader(std::move(source), std::move(cfg));
auto const header = reader.GetHeader();
asm volatile("" : : "g"(header) : "memory");
while (true) {
auto row = reader.GetNextRow(mem);
if (!row) break;
asm volatile("" : : "g"(row) : "memory");
}
} catch (csv::CsvReadException const &) {
// CsvReadException is ok
}
return 0;
}

View File

@@ -150,9 +150,7 @@ storage::Result<communication::bolt::Vertex> ToBoltVertex(const storage::VertexA
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
// Introduced in Bolt v5 (for now just send the ID)
const auto element_id = std::to_string(id.AsInt());
return communication::bolt::Vertex{id, labels, properties, element_id};
return communication::bolt::Vertex{id, labels, properties};
}
storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccessor &edge, const storage::Storage &db,
@@ -167,11 +165,7 @@ storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccesso
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
// Introduced in Bolt v5 (for now just send the ID)
const auto element_id = std::to_string(id.AsInt());
const auto from_element_id = std::to_string(from.AsInt());
const auto to_element_id = std::to_string(to.AsInt());
return communication::bolt::Edge{id, from, to, type, properties, element_id, from_element_id, to_element_id};
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, const storage::Storage &db,

View File

@@ -1,4 +0,0 @@
set(mg_http_handlers_sources)
add_library(mg-http-handlers STATIC ${mg_http_handlers_sources})
target_link_libraries(mg-http-handlers mg-query mg-storage-v2)

View File

@@ -1,211 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <tuple>
#include <vector>
#include <spdlog/spdlog.h>
#include <boost/beast/http.hpp>
#include <boost/beast/version.hpp>
#include <json/json.hpp>
#include <utils/event_counter.hpp>
#include <utils/event_gauge.hpp>
#include "storage/v2/storage.hpp"
#include "utils/event_gauge.hpp"
#include "utils/event_histogram.hpp"
namespace memgraph::http {
struct MetricsResponse {
uint64_t vertex_count;
uint64_t edge_count;
double average_degree;
uint64_t memory_usage;
uint64_t disk_usage;
// Storage of all the counter values throughout the system
// e.g. number of active transactions
std::vector<std::tuple<std::string, std::string, uint64_t>> event_counters{};
// Storage of all the current values throughout the system
std::vector<std::tuple<std::string, std::string, uint64_t>> event_gauges{};
// Storage of all the percentile values across the histograms in the system
// e.g. query latency percentiles, snapshot recovery duration percentiles, etc.
std::vector<std::tuple<std::string, std::string, uint64_t>> event_histograms{};
};
template <typename TSessionData>
class MetricsService {
public:
explicit MetricsService(TSessionData *data) : db_(data->interpreter_context->db.get()) {}
nlohmann::json GetMetricsJSON() {
auto response = GetMetrics();
return AsJson(response);
}
private:
const storage::Storage *db_;
MetricsResponse GetMetrics() {
auto info = db_->GetInfo();
return MetricsResponse{.vertex_count = info.vertex_count,
.edge_count = info.edge_count,
.average_degree = info.average_degree,
.memory_usage = info.memory_usage,
.disk_usage = info.disk_usage,
.event_counters = GetEventCounters(),
.event_gauges = GetEventGauges(),
.event_histograms = GetEventHistograms()};
}
nlohmann::json AsJson(MetricsResponse response) {
auto metrics_response = nlohmann::json();
const auto *general_type = "General";
metrics_response[general_type]["vertex_count"] = response.vertex_count;
metrics_response[general_type]["edge_count"] = response.edge_count;
metrics_response[general_type]["average_degree"] = response.average_degree;
metrics_response[general_type]["memory_usage"] = response.memory_usage;
metrics_response[general_type]["disk_usage"] = response.disk_usage;
for (const auto &[name, type, value] : response.event_counters) {
metrics_response[type][name] = value;
}
for (const auto &[name, type, value] : response.event_gauges) {
metrics_response[type][name] = value;
}
for (const auto &[name, type, value] : response.event_histograms) {
metrics_response[type][name] = value;
}
return metrics_response;
}
auto GetEventCounters() {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
std::vector<std::tuple<std::string, std::string, uint64_t>> event_counters{};
for (auto i = 0; i < memgraph::metrics::CounterEnd(); i++) {
event_counters.emplace_back(memgraph::metrics::GetCounterName(i), memgraph::metrics::GetCounterType(i),
memgraph::metrics::global_counters[i].load(std::memory_order_acquire));
}
return event_counters;
}
auto GetEventGauges() {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
std::vector<std::tuple<std::string, std::string, uint64_t>> event_gauges{};
for (auto i = 0; i < memgraph::metrics::GaugeEnd(); i++) {
event_gauges.emplace_back(memgraph::metrics::GetGaugeName(i), memgraph::metrics::GetGaugeType(i),
memgraph::metrics::global_gauges[i].load(std::memory_order_acquire));
}
return event_gauges;
}
auto GetEventHistograms() {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
std::vector<std::tuple<std::string, std::string, uint64_t>> event_histograms{};
for (auto i = 0; i < memgraph::metrics::HistogramEnd(); i++) {
const auto *name = memgraph::metrics::GetHistogramName(i);
auto &histogram = memgraph::metrics::global_histograms[i];
for (auto &[percentile, value] : histogram.YieldPercentiles()) {
auto metric_name = std::string(name) + "_" + std::to_string(percentile) + "p";
event_histograms.emplace_back(metric_name, memgraph::metrics::GetHistogramType(i), value);
}
}
return event_histograms;
}
};
template <typename TSessionData>
class MetricsRequestHandler final {
public:
explicit MetricsRequestHandler(TSessionData *data) : service_(data) {
spdlog::info("Basic request handler started!");
}
MetricsRequestHandler(const MetricsRequestHandler &) = delete;
MetricsRequestHandler(MetricsRequestHandler &&) = delete;
MetricsRequestHandler &operator=(const MetricsRequestHandler &) = delete;
MetricsRequestHandler &operator=(MetricsRequestHandler &&) = delete;
~MetricsRequestHandler() = default;
template <class Body, class Allocator>
// NOLINTNEXTLINE(misc-unused-parameters)
void HandleRequest(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> &&req,
std::function<void(boost::beast::http::response<boost::beast::http::string_body>)> &&send) {
auto response_json = nlohmann::json();
// Returns a bad request response
auto const bad_request = [&req, &response_json](const auto why) {
response_json["error"] = std::string(why);
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
boost::beast::http::response<boost::beast::http::string_body> res{boost::beast::http::status::bad_request,
req.version()};
res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING);
res.set(boost::beast::http::field::content_type, "application/json");
res.keep_alive(req.keep_alive());
res.body() = response_json.dump();
res.prepare_payload();
return res;
};
// Make sure we can handle the method
if (req.method() != boost::beast::http::verb::get) {
return send(bad_request("Unknown HTTP-method"));
}
// Request path must be absolute and not contain "..".
if (req.target().empty() || req.target()[0] != '/' || req.target().find("..") != boost::beast::string_view::npos) {
return send(bad_request("Illegal request-target"));
}
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
boost::beast::http::string_body::value_type body;
auto service_response = service_.GetMetricsJSON();
body.append(service_response.dump());
// Cache the size since we need it after the move
const auto size = body.size();
// Respond to GET request
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
boost::beast::http::response<boost::beast::http::string_body> res{
std::piecewise_construct, std::make_tuple(std::move(body)),
std::make_tuple(boost::beast::http::status::ok, req.version())};
res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING);
res.set(boost::beast::http::field::content_type, "application/json");
res.content_length(size);
res.keep_alive(req.keep_alive());
return send(std::move(res));
}
private:
MetricsService<TSessionData> service_;
};
} // namespace memgraph::http

View File

@@ -34,16 +34,13 @@
#include <spdlog/sinks/dist_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include "audit/log.hpp"
#include "auth/models.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/http/server.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "glue/auth_checker.hpp"
#include "glue/auth_handler.hpp"
#include "helpers.hpp"
#include "http_handlers/metrics.hpp"
#include "license/license.hpp"
#include "license/license_sender.hpp"
#include "py/py.hpp"
@@ -102,6 +99,10 @@
#include "auth/auth.hpp"
#include "glue/auth.hpp"
#ifdef MG_ENTERPRISE
#include "audit/log.hpp"
#endif
constexpr const char *kMgUser = "MEMGRAPH_USER";
constexpr const char *kMgPassword = "MEMGRAPH_PASSWORD";
constexpr const char *kMgPassfile = "MEMGRAPH_PASSFILE";
@@ -115,9 +116,6 @@ DEFINE_string(bolt_address, "0.0.0.0", "IP address on which the Bolt server shou
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(monitoring_address, "0.0.0.0",
"IP address on which the websocket server for Memgraph monitoring should listen.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(metrics_address, "0.0.0.0",
"IP address on which the Memgraph server for exposing metrics should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687, "Port on which the Bolt server should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
@@ -125,9 +123,6 @@ DEFINE_VALIDATED_int32(monitoring_port, 7444,
"Port on which the websocket server for Memgraph monitoring should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(metrics_port, 9091, "Port on which the Memgraph server for exposing metrics should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(bolt_num_workers, std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers used by the Bolt server. By default, this will be the "
"number of processing units available on the machine.",
@@ -261,8 +256,6 @@ DEFINE_double(query_execution_timeout_sec, 600,
DEFINE_uint64(replication_replica_check_frequency_sec, 1,
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
"The MAIN instance allocates a new thread for each REPLICA.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(replication_restore_state_on_startup, false, "Restore replication state on startup, e.g. recover replica");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(
@@ -448,19 +441,21 @@ struct SessionData {
// supplied.
#if MG_ENTERPRISE
SessionData(memgraph::query::InterpreterContext *interpreter_context,
SessionData(memgraph::storage::Storage *db, memgraph::query::InterpreterContext *interpreter_context,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
memgraph::audit::Log *audit_log)
: interpreter_context(interpreter_context), auth(auth), audit_log(audit_log) {}
: db(db), interpreter_context(interpreter_context), auth(auth), audit_log(audit_log) {}
memgraph::storage::Storage *db;
memgraph::query::InterpreterContext *interpreter_context;
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth;
memgraph::audit::Log *audit_log;
#else
SessionData(memgraph::query::InterpreterContext *interpreter_context,
SessionData(memgraph::storage::Storage *db, memgraph::query::InterpreterContext *interpreter_context,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth)
: interpreter_context(interpreter_context), auth(auth) {}
: db(db), interpreter_context(interpreter_context), auth(auth) {}
memgraph::storage::Storage *db;
memgraph::query::InterpreterContext *interpreter_context;
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth;
@@ -473,36 +468,33 @@ struct SessionData {
DEFINE_string(auth_user_or_role_name_regex, memgraph::glue::kDefaultUserRoleRegex.data(),
"Set to the regular expression that each user or role name must fulfill.");
void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, std::string cypherl_file_path,
memgraph::audit::Log *audit_log = nullptr) {
void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, std::string cypherl_file_path
#ifdef MG_ENTERPRISE
,
memgraph::audit::Log *audit_log
#endif
) {
memgraph::query::Interpreter interpreter(&ctx);
std::ifstream file(cypherl_file_path);
if (file.is_open()) {
std::string line;
while (std::getline(file, line)) {
if (!line.empty()) {
auto results = interpreter.Prepare(line, {}, {});
memgraph::query::DiscardValueResultStream stream;
interpreter.Pull(&stream, {}, results.qid);
if (!file.is_open()) {
spdlog::trace("Could not find init file {}", cypherl_file_path);
return;
}
std::string line;
while (std::getline(file, line)) {
if (!line.empty()) {
auto results = interpreter.Prepare(line, {}, {});
memgraph::query::DiscardValueResultStream stream;
interpreter.Pull(&stream, {}, results.qid);
if (audit_log) {
audit_log->Record("", "", line, {});
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
audit_log->Record("", "", line, {});
}
#endif
}
}
file.close();
}
file.close();
}
namespace memgraph::metrics {
extern const Event ActiveBoltSessions;
} // namespace memgraph::metrics
class BoltSession final : public memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream> {
public:
@@ -511,6 +503,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
memgraph::communication::v2::OutputStream *output_stream)
: memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream>(input_stream, output_stream),
db_(data->db),
interpreter_context_(data->interpreter_context),
interpreter_(data->interpreter_context),
auth_(data->auth),
@@ -519,41 +512,26 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
#endif
endpoint_(endpoint),
run_id_(data->run_id) {
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions);
interpreter_context_->interpreters.WithLock([this](auto &interpreters) { interpreters.insert(&interpreter_); });
}
~BoltSession() override {
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveBoltSessions);
interpreter_context_->interpreters.WithLock([this](auto &interpreters) { interpreters.erase(&interpreter_); });
}
using memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream>::TEncoder;
void BeginTransaction(const std::map<std::string, memgraph::communication::bolt::Value> &metadata) override {
std::map<std::string, memgraph::storage::PropertyValue> metadata_pv;
for (const auto &[key, bolt_value] : metadata) {
metadata_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_value));
}
interpreter_.BeginTransaction(metadata_pv);
}
void BeginTransaction() override { interpreter_.BeginTransaction(); }
void CommitTransaction() override { interpreter_.CommitTransaction(); }
void RollbackTransaction() override { interpreter_.RollbackTransaction(); }
std::pair<std::vector<std::string>, std::optional<int>> Interpret(
const std::string &query, const std::map<std::string, memgraph::communication::bolt::Value> &params,
const std::map<std::string, memgraph::communication::bolt::Value> &metadata) override {
const std::string &query, const std::map<std::string, memgraph::communication::bolt::Value> &params) override {
std::map<std::string, memgraph::storage::PropertyValue> params_pv;
std::map<std::string, memgraph::storage::PropertyValue> metadata_pv;
for (const auto &[key, bolt_param] : params) {
params_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_param));
}
for (const auto &[key, bolt_md] : metadata) {
metadata_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_md));
}
for (const auto &kv : params) params_pv.emplace(kv.first, memgraph::glue::ToPropertyValue(kv.second));
const std::string *username{nullptr};
if (user_) {
username = &user_->username();
@@ -565,7 +543,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
}
#endif
try {
auto result = interpreter_.Prepare(query, params_pv, username, metadata_pv);
auto result = interpreter_.Prepare(query, params_pv, username);
if (user_ && !memgraph::glue::AuthChecker::IsUserAuthorized(*user_, result.privileges)) {
interpreter_.Abort();
throw memgraph::communication::bolt::ClientError(
@@ -585,7 +563,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
std::map<std::string, memgraph::communication::bolt::Value> Pull(TEncoder *encoder, std::optional<int> n,
std::optional<int> qid) override {
TypedValueResultStream stream(encoder, interpreter_context_->db.get());
TypedValueResultStream stream(encoder, db_);
return PullResults(stream, n, qid);
}
@@ -619,8 +597,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
const auto &summary = interpreter_.Pull(&stream, n, qid);
std::map<std::string, memgraph::communication::bolt::Value> decoded_summary;
for (const auto &kv : summary) {
auto maybe_value =
memgraph::glue::ToBoltValue(kv.second, *interpreter_context_->db, memgraph::storage::View::NEW);
auto maybe_value = memgraph::glue::ToBoltValue(kv.second, *db_, memgraph::storage::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:
@@ -684,6 +661,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
};
// NOTE: Needed only for ToBoltValue conversions
const memgraph::storage::Storage *db_;
memgraph::query::InterpreterContext *interpreter_context_;
memgraph::query::Interpreter interpreter_;
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
@@ -697,8 +675,6 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
};
using ServerT = memgraph::communication::v2::Server<BoltSession, SessionData>;
using MonitoringServerT =
memgraph::communication::http::Server<memgraph::http::MetricsRequestHandler<SessionData>, SessionData>;
using memgraph::communication::ServerContext;
// Needed to correctly handle memgraph destruction from a signal handler.
@@ -882,6 +858,9 @@ int main(int argc, char **argv) {
// End enterprise features initialization
#endif
// auto type = memgraph::storage::Config::StorageMode::Type::IN_MEMORY;
auto type = memgraph::storage::Config::StorageMode::Type::PERSISTENT;
// Main storage and execution engines initialization
memgraph::storage::Config db_config{
.gc = {.type = memgraph::storage::Config::Gc::Type::PERIODIC,
@@ -893,19 +872,9 @@ int main(int argc, char **argv) {
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit,
.restore_replication_state_on_startup = FLAGS_replication_restore_state_on_startup,
.items_per_batch = FLAGS_storage_items_per_batch,
.recovery_thread_count = FLAGS_storage_recovery_thread_count,
.allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery},
.restore_replicas_on_startup = true},
.transaction = {.isolation_level = ParseIsolationLevel()},
.disk = {.main_storage_directory = FLAGS_data_directory + "/rocksdb_main_storage",
.label_index_directory = FLAGS_data_directory + "/rocksdb_label_index",
.label_property_index_directory = FLAGS_data_directory + "/rocksdb_label_property_index",
.unique_constraints_directory = FLAGS_data_directory + "/rocksdb_unique_constraints",
.name_id_mapper_directory = FLAGS_data_directory + "/rocksdb_name_id_mapper",
.id_name_mapper_directory = FLAGS_data_directory + "/rocksdb_id_name_mapper",
.durability_directory = FLAGS_data_directory + "/rocksdb_durability",
.wal_directory = FLAGS_data_directory + "/rocksdb_wal"}};
.storage_mode = {.type = type}};
if (FLAGS_storage_snapshot_interval_sec == 0) {
if (FLAGS_storage_wal_enabled) {
LOG_FATAL(
@@ -924,9 +893,11 @@ int main(int argc, char **argv) {
}
db_config.durability.snapshot_interval = std::chrono::seconds(FLAGS_storage_snapshot_interval_sec);
}
// auto db = std::unique_ptr<memgraph::storage::Storage>(new memgraph::storage::InMemoryStorage(db_config));
auto db = std::unique_ptr<memgraph::storage::Storage>(new memgraph::storage::DiskStorage(db_config));
memgraph::query::InterpreterContext interpreter_context{
db_config,
db.get(),
{.query = {.allow_load_csv = FLAGS_allow_load_csv},
.execution_timeout_sec = FLAGS_query_execution_timeout_sec,
.replication_replica_check_frequency = std::chrono::seconds(FLAGS_replication_replica_check_frequency_sec),
@@ -936,9 +907,9 @@ int main(int argc, char **argv) {
.stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)},
FLAGS_data_directory};
#ifdef MG_ENTERPRISE
SessionData session_data{&interpreter_context, &auth, &audit_log};
SessionData session_data{db.get(), &interpreter_context, &auth, &audit_log};
#else
SessionData session_data{&interpreter_context, &auth};
SessionData session_data{db.get(), &interpreter_context, &auth};
#endif
memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories, FLAGS_data_directory);
@@ -950,12 +921,10 @@ int main(int argc, char **argv) {
interpreter_context.auth_checker = &auth_checker;
if (!FLAGS_init_file.empty()) {
spdlog::info("Running init file...");
spdlog::info("Running init file.");
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
InitFromCypherlFile(interpreter_context, FLAGS_init_file, &audit_log);
} else {
InitFromCypherlFile(interpreter_context, FLAGS_init_file);
}
#else
InitFromCypherlFile(interpreter_context, FLAGS_init_file);
@@ -1011,15 +980,14 @@ int main(int argc, char **argv) {
std::optional<memgraph::telemetry::Telemetry> telemetry;
if (FLAGS_telemetry_enabled) {
telemetry.emplace(telemetry_server, data_directory / "telemetry", run_id, machine_id, std::chrono::minutes(10));
telemetry->AddCollector("storage", [db_ = interpreter_context.db.get()]() -> nlohmann::json {
auto info = db_->GetInfo();
telemetry->AddCollector("storage", [&db]() -> nlohmann::json {
auto info = db->GetInfo();
return {{"vertices", info.vertex_count}, {"edges", info.edge_count}};
});
telemetry->AddCollector("event_counters", []() -> nlohmann::json {
nlohmann::json ret;
for (size_t i = 0; i < memgraph::metrics::CounterEnd(); ++i) {
ret[memgraph::metrics::GetCounterName(i)] =
memgraph::metrics::global_counters[i].load(std::memory_order_relaxed);
for (size_t i = 0; i < EventCounter::End(); ++i) {
ret[EventCounter::GetName(i)] = EventCounter::global_counters[i].load(std::memory_order_relaxed);
}
return ret;
});
@@ -1035,43 +1003,6 @@ int main(int argc, char **argv) {
{FLAGS_monitoring_address, static_cast<uint16_t>(FLAGS_monitoring_port)}, &context, websocket_auth};
AddLoggerSink(websocket_server.GetLoggingSink());
MonitoringServerT metrics_server{
{FLAGS_metrics_address, static_cast<uint16_t>(FLAGS_metrics_port)}, &session_data, &context};
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
// Handler for regular termination signals
auto shutdown = [&metrics_server, &websocket_server, &server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
// a race condition when a transaction is accepted during server shutdown.
server.Shutdown();
// After the server is notified to stop accepting and processing
// connections we tell the execution engine to stop processing all pending
// queries.
memgraph::query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
metrics_server.Shutdown();
};
InitSignalHandlers(shutdown);
} else {
// Handler for regular termination signals
auto shutdown = [&websocket_server, &server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
// a race condition when a transaction is accepted during server shutdown.
server.Shutdown();
// After the server is notified to stop accepting and processing
// connections we tell the execution engine to stop processing all pending
// queries.
memgraph::query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
};
InitSignalHandlers(shutdown);
}
#else
// Handler for regular termination signals
auto shutdown = [&websocket_server, &server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
@@ -1081,29 +1012,19 @@ int main(int argc, char **argv) {
// connections we tell the execution engine to stop processing all pending
// queries.
memgraph::query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
};
InitSignalHandlers(shutdown);
#endif
MG_ASSERT(server.Start(), "Couldn't start the Bolt server!");
websocket_server.Start();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
metrics_server.Start();
}
#endif
if (!FLAGS_init_data_file.empty()) {
spdlog::info("Running init data file.");
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
InitFromCypherlFile(interpreter_context, FLAGS_init_data_file, &audit_log);
} else {
InitFromCypherlFile(interpreter_context, FLAGS_init_data_file);
}
#else
InitFromCypherlFile(interpreter_context, FLAGS_init_data_file);
@@ -1112,11 +1033,6 @@ int main(int argc, char **argv) {
server.AwaitShutdown();
websocket_server.AwaitShutdown();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
metrics_server.AwaitShutdown();
}
#endif
memgraph::query::procedure::gModuleRegistry.UnloadAllModules();

View File

@@ -437,7 +437,7 @@ void ProcessNodeRow(memgraph::storage::Storage *store, const std::vector<std::st
if (it != node_id_map->end()) {
if (FLAGS_skip_duplicate_nodes) {
spdlog::warn(memgraph::utils::MessageWithLink("Skipping duplicate node with ID '{}'.", node_id,
"https://memgr.ph/csv-import-tool"));
"https://memgr.ph/csv"));
return;
} else {
throw LoadException("Node with ID '{}' already exists", node_id);
@@ -530,7 +530,7 @@ void ProcessRelationshipsRow(memgraph::storage::Storage *store, const std::vecto
if (it == node_id_map.end()) {
if (FLAGS_skip_bad_relationships) {
spdlog::warn(memgraph::utils::MessageWithLink("Skipping bad relationship with START_ID '{}'.", node_id,
"https://memgr.ph/csv-import-tool"));
"https://memgr.ph/csv"));
return;
} else {
throw LoadException("Node with ID '{}' does not exist", node_id);
@@ -548,7 +548,7 @@ void ProcessRelationshipsRow(memgraph::storage::Storage *store, const std::vecto
if (it == node_id_map.end()) {
if (FLAGS_skip_bad_relationships) {
spdlog::warn(memgraph::utils::MessageWithLink("Skipping bad relationship with END_ID '{}'.", node_id,
"https://memgr.ph/csv-import-tool"));
"https://memgr.ph/csv"));
return;
} else {
throw LoadException("Node with ID '{}' does not exist", node_id);

View File

@@ -37,9 +37,12 @@ set(mg_query_sources
graph.cpp
db_accessor.cpp)
find_package(Boost REQUIRED)
add_library(mg-query STATIC ${mg_query_sources})
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query PUBLIC dl cppitertools Python3::Python mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory mg::csv)
target_link_libraries(mg-query dl cppitertools Boost::headers)
target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()
@@ -48,6 +51,7 @@ if("${MG_PYTHON_VERSION}" STREQUAL "")
else()
find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
endif()
target_link_libraries(mg-query Python3::Python)
# Generate Antlr openCypher parser
@@ -90,4 +94,4 @@ add_library(antlr_opencypher_parser_lib STATIC ${antlr_opencypher_generated_src}
add_dependencies(antlr_opencypher_parser_lib generate_opencypher_parser)
target_link_libraries(antlr_opencypher_parser_lib antlr4)
target_link_libraries(mg-query PUBLIC antlr_opencypher_parser_lib)
target_link_libraries(mg-query antlr_opencypher_parser_lib)

View File

@@ -14,6 +14,8 @@
#include <string>
namespace memgraph::query {
inline constexpr uint16_t kDefaultReplicationPort = 10000;
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
inline const std::string kAsterisk = "*";
inline constexpr uint16_t kComputeStatisticsNumResults = 7;
inline constexpr uint16_t kDeleteStatisticsNumResults = 6;
} // namespace memgraph::query

View File

@@ -22,8 +22,6 @@
#include "query/trigger.hpp"
#include "utils/async_timer.hpp"
#include "query/frame_change.hpp"
namespace memgraph::query {
enum class TransactionStatus {
@@ -84,7 +82,6 @@ struct ExecutionContext {
plan::ProfilingStats *stats_root{nullptr};
ExecutionStats execution_stats;
TriggerContextCollector *trigger_context_collector{nullptr};
FrameChangeCollector *frame_change_collector{nullptr};
utils::AsyncTimer timer;
#ifdef MG_ENTERPRISE
std::unique_ptr<FineGrainedAuthChecker> auth_checker{nullptr};

View File

@@ -21,7 +21,6 @@
#include "storage/v2/id_types.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/result.hpp"
#include "storage/v2/storage_mode.hpp"
#include "utils/pmr/unordered_set.hpp"
#include "utils/variant_helpers.hpp"
@@ -351,7 +350,7 @@ class DbAccessor final {
void PrefetchOutEdges(const VertexAccessor &vertex) const { accessor_->PrefetchOutEdges(vertex.impl_); }
void PrefetchInEdges(const VertexAccessor &vertex) const { accessor_->PrefetchInEdges(vertex.impl_); }
void PrefetchInEdges(const VertexAccessor &vertex) const { accessor_->PrefetchOutEdges(vertex.impl_); }
storage::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::EdgeTypeId &edge_type) {
@@ -378,8 +377,6 @@ class DbAccessor final {
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
accessor_->PrefetchOutEdges(vertex_accessor->impl_);
accessor_->PrefetchInEdges(vertex_accessor->impl_);
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
@@ -432,44 +429,28 @@ class DbAccessor final {
void Abort() { accessor_->Abort(); }
storage::StorageMode GetStorageMode() const { return accessor_->GetCreationStorageMode(); }
bool LabelIndexExists(storage::LabelId label) const { return accessor_->LabelIndexExists(label); }
bool LabelPropertyIndexExists(storage::LabelId label, storage::PropertyId prop) const {
return accessor_->LabelPropertyIndexExists(label, prop);
}
std::optional<storage::LabelIndexStats> GetIndexStats(const storage::LabelId &label) const {
return accessor_->GetIndexStats(label);
}
std::optional<storage::LabelPropertyIndexStats> GetIndexStats(const storage::LabelId &label,
const storage::PropertyId &property) const {
std::optional<storage::IndexStats> GetIndexStats(const storage::LabelId &label,
const storage::PropertyId &property) const {
return accessor_->GetIndexStats(label, property);
}
std::vector<std::pair<storage::LabelId, storage::PropertyId>> ClearLabelPropertyIndexStats() {
return accessor_->ClearLabelPropertyIndexStats();
std::vector<std::pair<storage::LabelId, storage::PropertyId>> ClearIndexStats() {
return accessor_->ClearIndexStats();
}
std::vector<storage::LabelId> ClearLabelIndexStats() { return accessor_->ClearLabelIndexStats(); }
std::vector<std::pair<storage::LabelId, storage::PropertyId>> DeleteLabelPropertyIndexStats(
std::vector<std::pair<storage::LabelId, storage::PropertyId>> DeleteIndexStatsForLabels(
const std::span<std::string> labels) {
return accessor_->DeleteLabelPropertyIndexStats(labels);
}
std::vector<storage::LabelId> DeleteLabelIndexStats(const std::span<std::string> labels) {
return accessor_->DeleteLabelIndexStats(labels);
}
void SetIndexStats(const storage::LabelId &label, const storage::LabelIndexStats &stats) {
accessor_->SetIndexStats(label, stats);
return accessor_->DeleteIndexStatsForLabels(labels);
}
void SetIndexStats(const storage::LabelId &label, const storage::PropertyId &property,
const storage::LabelPropertyIndexStats &stats) {
const storage::IndexStats &stats) {
accessor_->SetIndexStats(label, property, stats);
}
@@ -518,10 +499,6 @@ class SubgraphDbAccessor final {
const std::string &EdgeTypeToName(storage::EdgeTypeId type) const;
void PrefetchOutEdges(const SubgraphVertexAccessor &vertex) const { db_accessor_.PrefetchOutEdges(vertex.impl_); }
void PrefetchInEdges(const SubgraphVertexAccessor &vertex) const { db_accessor_.PrefetchInEdges(vertex.impl_); }
storage::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge);
storage::Result<EdgeAccessor> InsertEdge(SubgraphVertexAccessor *from, SubgraphVertexAccessor *to,

View File

@@ -175,34 +175,18 @@ class ReplicationModificationInMulticommandTxException : public QueryException {
: QueryException("Replication clause not allowed in multicommand transactions.") {}
};
class ReplicationDisabledOnDiskStorage : public QueryException {
public:
ReplicationDisabledOnDiskStorage() : QueryException("Replication is not supported while in on-disk storage mode.") {}
};
class LockPathModificationInMulticommandTxException : public QueryException {
public:
LockPathModificationInMulticommandTxException()
: QueryException("Lock path query not allowed in multicommand transactions.") {}
};
class LockPathDisabledOnDiskStorage : public QueryException {
public:
LockPathDisabledOnDiskStorage()
: QueryException("Lock path disabled on disk storage since all data is already persisted. ") {}
};
class FreeMemoryModificationInMulticommandTxException : public QueryException {
public:
FreeMemoryModificationInMulticommandTxException()
: QueryException("Free memory query not allowed in multicommand transactions.") {}
};
class FreeMemoryDisabledOnDiskStorage : public QueryException {
public:
FreeMemoryDisabledOnDiskStorage() : QueryException("Free memory does nothing when using disk storage. ") {}
};
class ShowConfigModificationInMulticommandTxException : public QueryException {
public:
ShowConfigModificationInMulticommandTxException()
@@ -248,11 +232,6 @@ class CreateSnapshotInMulticommandTxException final : public QueryException {
: QueryException("Snapshot cannot be created in multicommand transactions.") {}
};
class CreateSnapshotDisabledOnDiskStorage final : public QueryException {
public:
CreateSnapshotDisabledOnDiskStorage() : QueryException("In the on-disk storage mode data is already persistent.") {}
};
class SettingConfigInMulticommandTxException final : public QueryException {
public:
SettingConfigInMulticommandTxException()
@@ -275,7 +254,7 @@ class ReplicationException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
explicit ReplicationException(const std::string &message)
: utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICAS' query.",
: utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICA' query.",
message) {}
};
@@ -285,14 +264,4 @@ class TransactionQueueInMulticommandTxException : public QueryException {
: QueryException("Transaction queue queries not allowed in multicommand transactions.") {}
};
class IndexPersistenceException : public QueryException {
public:
IndexPersistenceException() : QueryException("Persisting index on disk failed.") {}
};
class ConstraintsPersistenceException : public QueryException {
public:
ConstraintsPersistenceException() : QueryException("Persisting constraints on disk failed.") {}
};
} // namespace memgraph::query

View File

@@ -1,122 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/typed_value.hpp"
#include "utils/memory.hpp"
#include "utils/pmr/unordered_map.hpp"
#include "utils/pmr/vector.hpp"
namespace memgraph::query {
// Key is hash output, value is vector of unique elements
using CachedType = utils::pmr::unordered_map<size_t, std::vector<TypedValue>>;
struct CachedValue {
// Cached value, this can be probably templateized
CachedType cache_;
explicit CachedValue(utils::MemoryResource *mem) : cache_(mem) {}
CachedValue(CachedType &&cache, memgraph::utils::MemoryResource *memory) : cache_(std::move(cache), memory) {}
CachedValue(const CachedValue &other, memgraph::utils::MemoryResource *memory) : cache_(other.cache_, memory) {}
CachedValue(CachedValue &&other, memgraph::utils::MemoryResource *memory) : cache_(std::move(other.cache_), memory) {}
CachedValue(CachedValue &&other) noexcept = delete;
/// Copy construction without memgraph::utils::MemoryResource is not allowed.
CachedValue(const CachedValue &) = delete;
CachedValue &operator=(const CachedValue &) = delete;
CachedValue &operator=(CachedValue &&) = delete;
~CachedValue() = default;
memgraph::utils::MemoryResource *GetMemoryResource() const noexcept {
return cache_.get_allocator().GetMemoryResource();
}
// Func to check if cache_ contains value
bool CacheValue(const TypedValue &value) {
if (!value.IsList()) {
return false;
}
const auto &list = value.ValueList();
TypedValue::Hash hash{};
for (const TypedValue &element : list) {
const auto key = hash(element);
auto &vector_values = cache_[key];
if (!IsValueInVec(vector_values, element)) {
vector_values.push_back(element);
}
}
return true;
}
// Func to cache_value inside cache_
bool ContainsValue(const TypedValue &value) const {
TypedValue::Hash hash{};
const auto key = hash(value);
if (cache_.contains(key)) {
return IsValueInVec(cache_.at(key), value);
}
return false;
}
private:
bool IsValueInVec(const std::vector<TypedValue> &vec_values, const TypedValue &value) const {
return std::any_of(vec_values.begin(), vec_values.end(), [&value](auto &vec_value) {
const auto is_value_equal = vec_value == value;
if (is_value_equal.IsNull()) return false;
return is_value_equal.ValueBool();
});
}
};
// Class tracks keys for which user can cache values which help with faster search or faster retrieval
// in the future.
class FrameChangeCollector {
public:
explicit FrameChangeCollector(utils::MemoryResource *mem) : tracked_values_(mem){};
// Add tracking key to cache later value
CachedValue &AddTrackingKey(const std::string &key) {
const auto &[it, _] = tracked_values_.emplace(key, tracked_values_.get_allocator().GetMemoryResource());
return it->second;
}
// Is key tracked
bool IsKeyTracked(const std::string &key) const { return tracked_values_.contains(key); }
// Is value for given key cached
bool IsKeyValueCached(const std::string &key) const {
return tracked_values_.contains(key) && !tracked_values_.at(key).cache_.empty();
}
// Reset value for tracking key
bool ResetTrackingValue(const std::string &key) {
if (tracked_values_.contains(key)) {
tracked_values_.erase(key);
AddTrackingKey(key);
}
return true;
}
// Get value cached for tracking key, throws if key is not in tracked
CachedValue &GetCachedValue(const std::string &key) { return tracked_values_.at(key); }
// Checks for keys tracked
bool IsTrackingValues() const { return !tracked_values_.empty(); }
private:
// Key is output of utils::GetFrameChangeId, value is utils::pmr::unordered_map
memgraph::utils::pmr::unordered_map<std::string, CachedValue> tracked_values_;
};
} // namespace memgraph::query

View File

@@ -114,18 +114,12 @@ constexpr utils::TypeInfo query::ListLiteral::kType{utils::TypeId::AST_LIST_LITE
constexpr utils::TypeInfo query::MapLiteral::kType{utils::TypeId::AST_MAP_LITERAL, "MapLiteral",
&query::BaseLiteral::kType};
constexpr utils::TypeInfo query::MapProjectionLiteral::kType{utils::TypeId::AST_MAP_PROJECTION_LITERAL,
"MapProjectionLiteral", &query::BaseLiteral::kType};
constexpr utils::TypeInfo query::Identifier::kType{utils::TypeId::AST_IDENTIFIER, "Identifier",
&query::Expression::kType};
constexpr utils::TypeInfo query::PropertyLookup::kType{utils::TypeId::AST_PROPERTY_LOOKUP, "PropertyLookup",
&query::Expression::kType};
constexpr utils::TypeInfo query::AllPropertiesLookup::kType{utils::TypeId::AST_ALL_PROPERTIES_LOOKUP,
"AllPropertiesLookup", &query::Expression::kType};
constexpr utils::TypeInfo query::LabelsTest::kType{utils::TypeId::AST_LABELS_TEST, "LabelsTest",
&query::Expression::kType};

View File

@@ -1063,9 +1063,8 @@ class MapLiteral : public memgraph::query::BaseLiteral {
DEFVISITABLE(ExpressionVisitor<void>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
for (auto pair : elements_) {
for (auto pair : elements_)
if (!pair.second->Accept(visitor)) break;
}
}
return visitor.PostVisit(*this);
}
@@ -1088,60 +1087,6 @@ class MapLiteral : public memgraph::query::BaseLiteral {
friend class AstStorage;
};
struct MapProjectionData {
Expression *map_variable;
std::unordered_map<PropertyIx, Expression *> elements;
};
class MapProjectionLiteral : public memgraph::query::BaseLiteral {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
MapProjectionLiteral() = default;
DEFVISITABLE(ExpressionVisitor<TypedValue>);
DEFVISITABLE(ExpressionVisitor<TypedValue *>);
DEFVISITABLE(ExpressionVisitor<void>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
for (auto pair : elements_) {
if (!pair.second) continue;
if (!pair.second->Accept(visitor)) break;
}
}
return visitor.PostVisit(*this);
}
Expression *map_variable_;
std::unordered_map<PropertyIx, Expression *> elements_;
MapProjectionLiteral *Clone(AstStorage *storage) const override {
MapProjectionLiteral *object = storage->Create<MapProjectionLiteral>();
object->map_variable_ = map_variable_;
for (const auto &entry : elements_) {
auto key = storage->GetPropertyIx(entry.first.name);
if (!entry.second) {
object->elements_[key] = nullptr;
continue;
}
object->elements_[key] = entry.second->Clone(storage);
}
return object;
}
protected:
explicit MapProjectionLiteral(Expression *map_variable, std::unordered_map<PropertyIx, Expression *> &&elements)
: map_variable_(map_variable), elements_(std::move(elements)) {}
private:
friend class AstStorage;
};
class Identifier : public memgraph::query::Expression {
public:
static const utils::TypeInfo kType;
@@ -1213,38 +1158,6 @@ class PropertyLookup : public memgraph::query::Expression {
friend class AstStorage;
};
class AllPropertiesLookup : public memgraph::query::Expression {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
AllPropertiesLookup() = default;
DEFVISITABLE(ExpressionVisitor<TypedValue>);
DEFVISITABLE(ExpressionVisitor<TypedValue *>);
DEFVISITABLE(ExpressionVisitor<void>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
expression_->Accept(visitor);
}
return visitor.PostVisit(*this);
}
memgraph::query::Expression *expression_{nullptr};
AllPropertiesLookup *Clone(AstStorage *storage) const override {
AllPropertiesLookup *object = storage->Create<AllPropertiesLookup>();
object->expression_ = expression_ ? expression_->Clone(storage) : nullptr;
return object;
}
protected:
explicit AllPropertiesLookup(Expression *expression) : expression_(expression) {}
private:
friend class AstStorage;
};
class LabelsTest : public memgraph::query::Expression {
public:
static const utils::TypeInfo kType;
@@ -2873,7 +2786,7 @@ class InfoQuery : public memgraph::query::Query {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class InfoType { STORAGE, INDEX, CONSTRAINT, BUILD };
enum class InfoType { STORAGE, INDEX, CONSTRAINT };
DEFVISITABLE(QueryVisitor<void>);
@@ -2985,7 +2898,7 @@ class LockPathQuery : public memgraph::query::Query {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class Action { LOCK_PATH, UNLOCK_PATH, STATUS };
enum class Action { LOCK_PATH, UNLOCK_PATH };
LockPathQuery() = default;
@@ -3022,7 +2935,6 @@ class LoadCsv : public memgraph::query::Clause {
bool ignore_bad_;
memgraph::query::Expression *delimiter_{nullptr};
memgraph::query::Expression *quote_{nullptr};
memgraph::query::Expression *nullif_{nullptr};
memgraph::query::Identifier *row_var_{nullptr};
LoadCsv *Clone(AstStorage *storage) const override {
@@ -3032,20 +2944,18 @@ class LoadCsv : public memgraph::query::Clause {
object->ignore_bad_ = ignore_bad_;
object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr;
object->quote_ = quote_ ? quote_->Clone(storage) : nullptr;
object->nullif_ = nullif_;
object->row_var_ = row_var_ ? row_var_->Clone(storage) : nullptr;
return object;
}
protected:
explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, Expression *delimiter, Expression *quote,
Expression *nullif, Identifier *row_var)
Identifier *row_var)
: file_(file),
with_header_(with_header),
ignore_bad_(ignore_bad),
delimiter_(delimiter),
quote_(quote),
nullif_(nullif),
row_var_(row_var) {
DMG_ASSERT(row_var, "LoadCsv cannot take nullptr for identifier");
}
@@ -3143,7 +3053,7 @@ class StorageModeQuery : public memgraph::query::Query {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class StorageMode { IN_MEMORY_TRANSACTIONAL, IN_MEMORY_ANALYTICAL, ON_DISK_TRANSACTIONAL };
enum class StorageMode { IN_MEMORY_TRANSACTIONAL, IN_MEMORY_ANALYTICAL };
StorageModeQuery() = default;

View File

@@ -22,7 +22,6 @@ class CypherUnion;
class NamedExpression;
class Identifier;
class PropertyLookup;
class AllPropertiesLookup;
class LabelsTest;
class Aggregation;
class Function;
@@ -45,7 +44,6 @@ class EdgeAtom;
class PrimitiveLiteral;
class ListLiteral;
class MapLiteral;
class MapProjectionLiteral;
class OrOperator;
class XorOperator;
class AndOperator;
@@ -108,10 +106,9 @@ using TreeCompositeVisitor = utils::CompositeVisitor<
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
MapProjectionLiteral, PropertyLookup, AllPropertiesLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce,
Extract, All, Single, Any, None, CallProcedure, Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete,
Where, SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv,
Foreach, Exists, CallSubquery, CypherQuery>;
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach, Exists, CallSubquery, CypherQuery>;
using TreeLeafVisitor = utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
@@ -125,14 +122,13 @@ class HierarchicalTreeVisitor : public TreeCompositeVisitor, public TreeLeafVisi
template <class TResult>
class ExpressionVisitor
: public utils::Visitor<TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator,
AdditionOperator, SubtractionOperator, MultiplicationOperator, DivisionOperator,
ModOperator, NotEqualOperator, EqualOperator, LessOperator, GreaterOperator,
LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator,
ListLiteral, MapLiteral, MapProjectionLiteral, PropertyLookup, AllPropertiesLookup,
LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None,
ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch, Exists> {};
: public utils::Visitor<
TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral,
MapLiteral, PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any,
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch, Exists> {};
template <class TResult>
class QueryVisitor

View File

@@ -124,9 +124,6 @@ antlrcpp::Any CypherMainVisitor::visitInfoQuery(MemgraphCypher::InfoQueryContext
} else if (ctx->constraintInfo()) {
info_query->info_type_ = InfoQuery::InfoType::CONSTRAINT;
return info_query;
} else if (ctx->buildInfo()) {
info_query->info_type_ = InfoQuery::InfoType::BUILD;
return info_query;
} else {
throw utils::NotYetImplemented("Info query: '{}'", ctx->getText());
}
@@ -328,9 +325,7 @@ antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasC
antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) {
auto *lock_query = storage_->Create<LockPathQuery>();
if (ctx->STATUS()) {
lock_query->action_ = LockPathQuery::Action::STATUS;
} else if (ctx->LOCK()) {
if (ctx->LOCK()) {
lock_query->action_ = LockPathQuery::Action::LOCK_PATH;
} else if (ctx->UNLOCK()) {
lock_query->action_ = LockPathQuery::Action::UNLOCK_PATH;
@@ -362,11 +357,6 @@ antlrcpp::Any CypherMainVisitor::visitLoadCsv(MemgraphCypher::LoadCsvContext *ct
// handle skip bad row option
load_csv->ignore_bad_ = ctx->IGNORE() && ctx->BAD();
// handle character sequence which will correspond to nulls
if (ctx->NULLIF()) {
load_csv->nullif_ = std::any_cast<Expression *>(ctx->nullif()->accept(this));
}
// handle delimiter
if (ctx->DELIMITER()) {
if (ctx->delimiter()->literal()->StringLiteral()) {
@@ -504,10 +494,7 @@ antlrcpp::Any CypherMainVisitor::visitStorageModeQuery(MemgraphCypher::StorageMo
if (mode->IN_MEMORY_ANALYTICAL()) {
return StorageModeQuery::StorageMode::IN_MEMORY_ANALYTICAL;
}
if (mode->IN_MEMORY_TRANSACTIONAL()) {
return StorageModeQuery::StorageMode::IN_MEMORY_TRANSACTIONAL;
}
return StorageModeQuery::StorageMode::ON_DISK_TRANSACTIONAL;
return StorageModeQuery::StorageMode::IN_MEMORY_TRANSACTIONAL;
});
query_ = storage_mode_query;
@@ -1709,38 +1696,6 @@ antlrcpp::Any CypherMainVisitor::visitMapLiteral(MemgraphCypher::MapLiteralConte
return map;
}
antlrcpp::Any CypherMainVisitor::visitMapProjectionLiteral(MemgraphCypher::MapProjectionLiteralContext *ctx) {
MapProjectionData map_projection_data;
map_projection_data.map_variable =
storage_->Create<Identifier>(std::any_cast<std::string>(ctx->variable()->accept(this)));
for (auto *map_el : ctx->mapElement()) {
if (map_el->propertyLookup()) {
auto key = std::any_cast<PropertyIx>(map_el->propertyLookup()->propertyKeyName()->accept(this));
auto property = std::any_cast<PropertyIx>(map_el->propertyLookup()->accept(this));
auto *property_lookup = storage_->Create<PropertyLookup>(map_projection_data.map_variable, property);
map_projection_data.elements.insert_or_assign(key, property_lookup);
}
if (map_el->allPropertiesLookup()) {
auto key = AddProperty("*");
auto *all_properties_lookup = storage_->Create<AllPropertiesLookup>(map_projection_data.map_variable);
map_projection_data.elements.insert_or_assign(key, all_properties_lookup);
}
if (map_el->variable()) {
auto key = AddProperty(std::any_cast<std::string>(map_el->variable()->accept(this)));
auto *variable = storage_->Create<Identifier>(std::any_cast<std::string>(map_el->variable()->accept(this)));
map_projection_data.elements.insert_or_assign(key, variable);
}
if (map_el->propertyKeyValuePair()) {
auto key = std::any_cast<PropertyIx>(map_el->propertyKeyValuePair()->propertyKeyName()->accept(this));
auto *value = std::any_cast<Expression *>(map_el->propertyKeyValuePair()->expression()->accept(this));
map_projection_data.elements.insert_or_assign(key, value);
}
}
return map_projection_data;
}
antlrcpp::Any CypherMainVisitor::visitListLiteral(MemgraphCypher::ListLiteralContext *ctx) {
std::vector<Expression *> expressions;
for (auto *expr_ctx : ctx->expression()) {
@@ -2321,10 +2276,6 @@ antlrcpp::Any CypherMainVisitor::visitLiteral(MemgraphCypher::LiteralContext *ct
} else if (ctx->listLiteral()) {
return static_cast<Expression *>(
storage_->Create<ListLiteral>(std::any_cast<std::vector<Expression *>>(ctx->listLiteral()->accept(this))));
} else if (ctx->mapProjectionLiteral()) {
auto map_projection_data = std::any_cast<MapProjectionData>(ctx->mapProjectionLiteral()->accept(this));
return static_cast<Expression *>(storage_->Create<MapProjectionLiteral>(map_projection_data.map_variable,
std::move(map_projection_data.elements)));
} else {
return static_cast<Expression *>(storage_->Create<MapLiteral>(
std::any_cast<std::unordered_map<PropertyIx, Expression *>>(ctx->mapLiteral()->accept(this))));

View File

@@ -15,6 +15,8 @@
#include <unordered_set>
#include <utility>
#include <antlr4-runtime.h>
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/opencypher/generated/MemgraphCypherBaseVisitor.h"
#include "utils/exceptions.hpp"
@@ -606,11 +608,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitMapLiteral(MemgraphCypher::MapLiteralContext *ctx) override;
/**
* @return MapProjectionData
*/
antlrcpp::Any visitMapProjectionLiteral(MemgraphCypher::MapProjectionLiteralContext *ctx) override;
/**
* @return vector<Expression*>
*/

View File

@@ -54,7 +54,6 @@ class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
void Visit(IfOperator &op) override;
void Visit(ListLiteral &op) override;
void Visit(MapLiteral &op) override;
void Visit(MapProjectionLiteral &op) override;
void Visit(LabelsTest &op) override;
void Visit(Aggregation &op) override;
void Visit(Function &op) override;
@@ -69,7 +68,6 @@ class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
void Visit(Identifier &op) override;
void Visit(PrimitiveLiteral &op) override;
void Visit(PropertyLookup &op) override;
void Visit(AllPropertiesLookup &op) override;
void Visit(ParameterLookup &op) override;
void Visit(NamedExpression &op) override;
void Visit(RegexMatch &op) override;
@@ -91,8 +89,6 @@ void PrintObject(std::ostream *out, Aggregation::Op op);
void PrintObject(std::ostream *out, Expression *expr);
void PrintObject(std::ostream *out, AllPropertiesLookup *apl);
void PrintObject(std::ostream *out, Identifier *expr);
void PrintObject(std::ostream *out, const storage::PropertyValue &value);
@@ -126,15 +122,6 @@ void PrintObject(std::ostream *out, Expression *expr) {
}
}
void PrintObject(std::ostream *out, AllPropertiesLookup *apl) {
if (apl) {
ExpressionPrettyPrinter printer{out};
*out << ".*";
} else {
*out << "<null>";
}
}
void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
void PrintObject(std::ostream *out, const storage::PropertyValue &value) {
@@ -262,17 +249,6 @@ void ExpressionPrettyPrinter::Visit(MapLiteral &op) {
PrintObject(out_, map);
}
void ExpressionPrettyPrinter::Visit(MapProjectionLiteral &op) {
std::map<std::string, Expression *> map_projection_elements;
for (const auto &kv : op.elements_) {
map_projection_elements[kv.first.name] = kv.second;
}
PrintObject(out_, op.map_variable_);
PrintObject(out_, map_projection_elements);
}
void ExpressionPrettyPrinter::Visit(AllPropertiesLookup &op) { PrintObject(out_, &op); }
void ExpressionPrettyPrinter::Visit(LabelsTest &op) { PrintOperator(out_, "LabelsTest", op.expression_); }
void ExpressionPrettyPrinter::Visit(Aggregation &op) { PrintOperator(out_, "Aggregation", op.op_); }

View File

@@ -46,9 +46,7 @@ indexInfo : INDEX INFO ;
constraintInfo : CONSTRAINT INFO ;
buildInfo : BUILD INFO ;
infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo | buildInfo) ;
infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo ) ;
explainQuery : EXPLAIN cypherQuery ;
@@ -250,7 +248,6 @@ literal : numberLiteral
| booleanLiteral
| CYPHERNULL
| mapLiteral
| mapProjectionLiteral
| listLiteral
;
@@ -293,8 +290,6 @@ patternComprehension : '[' ( variable '=' )? relationshipsPattern ( WHERE expres
propertyLookup : '.' ( propertyKeyName ) ;
allPropertiesLookup : '.' '*' ;
caseExpression : ( ( CASE ( caseAlternatives )+ ) | ( CASE test=expression ( caseAlternatives )+ ) ) ( ELSE else_expression=expression )? END ;
caseAlternatives : WHEN when_expression=expression THEN then_expression=expression ;
@@ -307,22 +302,12 @@ numberLiteral : doubleLiteral
mapLiteral : '{' ( propertyKeyName ':' expression ( ',' propertyKeyName ':' expression )* )? '}' ;
mapProjectionLiteral : variable '{' ( mapElement ( ',' mapElement )* )? '}' ;
mapElement : propertyLookup
| allPropertiesLookup
| variable
| propertyKeyValuePair
;
parameter : '$' ( symbolicName | DecimalLiteral ) ;
propertyExpression : atom ( propertyLookup )+ ;
propertyKeyName : symbolicName ;
propertyKeyValuePair : propertyKeyName ':' expression ;
integerLiteral : DecimalLiteral
| OctalLiteral
| HexadecimalLiteral

View File

@@ -31,7 +31,6 @@ memgraphCypherKeyword : cypherKeyword
| BATCH_SIZE
| BEFORE
| BOOTSTRAP_SERVERS
| BUILD
| CHECK
| CLEAR
| COMMIT
@@ -59,7 +58,6 @@ memgraphCypherKeyword : cypherKeyword
| GRANT
| HEADER
| IDENTIFIED
| NULLIF
| ISOLATION
| IN_MEMORY_ANALYTICAL
| IN_MEMORY_TRANSACTIONAL
@@ -92,7 +90,6 @@ memgraphCypherKeyword : cypherKeyword
| SNAPSHOT
| START
| STATS
| STATUS
| STORAGE
| STREAM
| STREAMS
@@ -225,7 +222,6 @@ loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
( IGNORE BAD ) ?
( DELIMITER delimiter ) ?
( QUOTE quote ) ?
( NULLIF nullif ) ?
AS rowVar ;
csvFile : literal ;
@@ -234,8 +230,6 @@ delimiter : literal ;
quote : literal ;
nullif : literal ;
rowVar : variable ;
userOrRoleName : symbolicName ;
@@ -340,7 +334,7 @@ dropReplica : DROP REPLICA replicaName ;
showReplicas : SHOW REPLICAS ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY | DATA DIRECTORY LOCK STATUS;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY ;
freeMemoryQuery : FREE MEMORY ;
@@ -365,7 +359,7 @@ isolationLevelScope : GLOBAL | SESSION | NEXT ;
isolationLevelQuery : SET isolationLevelScope TRANSACTION ISOLATION LEVEL isolationLevel ;
storageMode : IN_MEMORY_ANALYTICAL | IN_MEMORY_TRANSACTIONAL | ON_DISK_TRANSACTIONAL ;
storageMode : IN_MEMORY_ANALYTICAL | IN_MEMORY_TRANSACTIONAL ;
storageModeQuery : STORAGE MODE storageMode ;

View File

@@ -35,7 +35,6 @@ BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ;
BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ;
BATCH_SIZE : B A T C H UNDERSCORE S I Z E ;
BEFORE : B E F O R E ;
BUILD : B U I L D ;
BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ;
CALL : C A L L ;
CHECK : C H E C K ;
@@ -85,8 +84,6 @@ MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ;
NEXT : N E X T ;
NO : N O ;
NOTHING : N O T H I N G ;
ON_DISK_TRANSACTIONAL : O N UNDERSCORE D I S K UNDERSCORE T R A N S A C T I O N A L ;
NULLIF : N U L L I F ;
PASSWORD : P A S S W O R D ;
PORT : P O R T ;
PRIVILEGES : P R I V I L E G E S ;
@@ -109,7 +106,6 @@ SNAPSHOT : S N A P S H O T ;
START : S T A R T ;
STATISTICS : S T A T I S T I C S ;
STATS : S T A T S ;
STATUS : S T A T U S ;
STOP : S T O P ;
STORAGE : S T O R A G E;
STORAGE_MODE : S T O R A G E UNDERSCORE MODE;

View File

@@ -43,7 +43,6 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
AddPrivilege(AuthQuery::Privilege::INDEX);
break;
case InfoQuery::InfoType::STORAGE:
case InfoQuery::InfoType::BUILD:
AddPrivilege(AuthQuery::Privilege::STATS);
break;
case InfoQuery::InfoType::CONSTRAINT:

View File

@@ -157,34 +157,25 @@ void SymbolGenerator::VisitReturnBody(ReturnBody &body, Where *where) {
// Query
bool SymbolGenerator::PreVisit(SingleQuery &) {
auto &scope = scopes_.back();
scope.prev_return_names = scope.curr_return_names;
scope.curr_return_names.clear();
prev_return_names_ = curr_return_names_;
curr_return_names_.clear();
return true;
}
// Union
bool SymbolGenerator::PreVisit(CypherUnion &) {
auto next_scope = Scope();
next_scope.curr_return_names = scopes_.back().curr_return_names;
scopes_.pop_back();
scopes_.push_back(next_scope);
scopes_.back() = Scope();
return true;
}
bool SymbolGenerator::PostVisit(CypherUnion &cypher_union) {
auto &scope = scopes_.back();
if (scope.prev_return_names != scope.curr_return_names) {
if (prev_return_names_ != curr_return_names_) {
throw SemanticException("All subqueries in an UNION must have the same column names.");
}
// create new symbols for the result of the union
for (const auto &name : scope.curr_return_names) {
for (const auto &name : curr_return_names_) {
auto symbol = CreateSymbol(name, false);
cypher_union.union_symbols_.push_back(symbol);
}
@@ -268,9 +259,7 @@ bool SymbolGenerator::PreVisit(Return &ret) {
}
bool SymbolGenerator::PostVisit(Return &) {
auto &scope = scopes_.back();
for (const auto &name_symbol : scope.symbols) scope.curr_return_names.insert(name_symbol.first);
for (const auto &name_symbol : scopes_.back().symbols) curr_return_names_.insert(name_symbol.first);
return true;
}

View File

@@ -140,8 +140,6 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
std::vector<Identifier *> identifiers_in_match;
// Number of nested IfOperators.
int num_if_operators{0};
std::unordered_set<std::string> prev_return_names{};
std::unordered_set<std::string> curr_return_names{};
};
static std::optional<Symbol> FindSymbolInScope(const std::string &name, const Scope &scope, Symbol::Type type);
@@ -173,6 +171,8 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
// is mapped by its name.
std::unordered_map<std::string, Identifier *> predefined_identifiers_;
std::vector<Scope> scopes_;
std::unordered_set<std::string> prev_return_names_;
std::unordered_set<std::string> curr_return_names_;
};
inline SymbolTable MakeSymbolTable(CypherQuery *query, const std::vector<Identifier *> &predefined_identifiers = {}) {

View File

@@ -145,7 +145,6 @@ const trie::Trie kKeywords = {"union",
"drop",
"show",
"stats",
"status",
"unique",
"explain",
"profile",
@@ -212,12 +211,7 @@ const trie::Trie kKeywords = {"union",
"edge_types",
"off",
"in_memory_transactional",
"in_memory_analytical",
"data",
"directory",
"lock",
"unlock"
"build"};
"in_memory_analytical"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

View File

@@ -13,12 +13,10 @@
#pragma once
#include <algorithm>
#include <cstddef>
#include <limits>
#include <map>
#include <optional>
#include <regex>
#include <string>
#include <vector>
#include "query/common.hpp"
@@ -29,11 +27,7 @@
#include "query/frontend/semantic/symbol_table.hpp"
#include "query/interpret/frame.hpp"
#include "query/typed_value.hpp"
#include "spdlog/spdlog.h"
#include "utils/exceptions.hpp"
#include "utils/frame_change_id.hpp"
#include "utils/logging.hpp"
#include "utils/pmr/unordered_map.hpp"
namespace memgraph::query {
@@ -79,13 +73,11 @@ class ReferenceExpressionEvaluator : public ExpressionVisitor<TypedValue *> {
UNSUCCESSFUL_VISIT(ListSlicingOperator);
UNSUCCESSFUL_VISIT(IsNullOperator);
UNSUCCESSFUL_VISIT(PropertyLookup);
UNSUCCESSFUL_VISIT(AllPropertiesLookup);
UNSUCCESSFUL_VISIT(LabelsTest);
UNSUCCESSFUL_VISIT(PrimitiveLiteral);
UNSUCCESSFUL_VISIT(ListLiteral);
UNSUCCESSFUL_VISIT(MapLiteral);
UNSUCCESSFUL_VISIT(MapProjectionLiteral);
UNSUCCESSFUL_VISIT(Aggregation);
UNSUCCESSFUL_VISIT(Coalesce);
UNSUCCESSFUL_VISIT(Function);
@@ -108,13 +100,8 @@ class ReferenceExpressionEvaluator : public ExpressionVisitor<TypedValue *> {
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
public:
ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
storage::View view, FrameChangeCollector *frame_change_collector = nullptr)
: frame_(frame),
symbol_table_(&symbol_table),
ctx_(&ctx),
dba_(dba),
view_(view),
frame_change_collector_(frame_change_collector) {}
storage::View view)
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
using ExpressionVisitor<TypedValue>::Visit;
@@ -203,78 +190,25 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(InListOperator &in_list) override {
TypedValue *_list_ptr = nullptr;
TypedValue _list;
auto literal = in_list.expression1_->Accept(*this);
auto get_list_literal = [this, &in_list, &_list, &_list_ptr]() -> void {
ReferenceExpressionEvaluator reference_expression_evaluator{frame_, symbol_table_, ctx_};
_list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
if (nullptr == _list_ptr) {
_list = in_list.expression2_->Accept(*this);
_list_ptr = &_list;
}
};
auto do_list_literal_checks = [this, &literal, &_list_ptr]() -> std::optional<TypedValue> {
MG_ASSERT(_list_ptr, "List literal should have been defined");
if (_list_ptr->IsNull()) {
return TypedValue(ctx_->memory);
}
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list_ptr->type() != TypedValue::Type::List) {
throw QueryRuntimeException("IN expected a list, got {}.", _list_ptr->type());
}
const auto &list = _list_ptr->ValueList();
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
return {};
};
const auto cached_id = memgraph::utils::GetFrameChangeId(in_list);
const auto do_cache{frame_change_collector_ != nullptr && cached_id &&
frame_change_collector_->IsKeyTracked(*cached_id)};
if (do_cache) {
if (!frame_change_collector_->IsKeyValueCached(*cached_id)) {
// Check only first time if everything is okay, later when we use
// cache there is no need to check again as we did check first time
get_list_literal();
auto preoperational_checks = do_list_literal_checks();
if (preoperational_checks) {
return std::move(*preoperational_checks);
}
auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
cached_value.CacheValue(*_list_ptr);
spdlog::trace("Value cached {}", *cached_id);
}
const auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
if (cached_value.ContainsValue(literal)) {
return TypedValue(true, ctx_->memory);
}
// has null
if (cached_value.ContainsValue(TypedValue(ctx_->memory))) {
return TypedValue(ctx_->memory);
}
return TypedValue(false, ctx_->memory);
auto _list = in_list.expression2_->Accept(*this);
if (_list.IsNull()) {
return TypedValue(ctx_->memory);
}
// When caching is not an option, we need to evaluate list literal every time
// and do the checks
get_list_literal();
auto preoperational_checks = do_list_literal_checks();
if (preoperational_checks) {
return std::move(*preoperational_checks);
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list.type() != TypedValue::Type::List) {
throw QueryRuntimeException("IN expected a list, got {}.", _list.type());
}
const auto &list = _list.ValueList();
spdlog::trace("Not using cache on IN LIST operator");
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
auto has_null = false;
for (const auto &element : list) {
auto result = literal == element;
@@ -532,101 +466,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
throw QueryRuntimeException("Invalid property name {} for Graph", prop_name);
}
default:
throw QueryRuntimeException(
"Only nodes, edges, maps, temporal types and graphs have properties to be looked up.");
}
}
TypedValue Visit(AllPropertiesLookup &all_properties_lookup) override {
TypedValue::TMap result(ctx_->memory);
auto expression_result = all_properties_lookup.expression_->Accept(*this);
switch (expression_result.type()) {
case TypedValue::Type::Null:
return TypedValue(ctx_->memory);
case TypedValue::Type::Vertex: {
for (const auto properties = *expression_result.ValueVertex().Properties(view_);
const auto &[property_id, value] : properties) {
result.emplace(dba_->PropertyToName(property_id), value);
}
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Edge: {
for (const auto properties = *expression_result.ValueEdge().Properties(view_);
const auto &[property_id, value] : properties) {
result.emplace(dba_->PropertyToName(property_id), value);
}
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Map: {
for (auto &[name, value] : expression_result.ValueMap()) {
result.emplace(name, value);
}
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Duration: {
const auto &dur = expression_result.ValueDuration();
result.emplace("day", TypedValue(dur.Days(), ctx_->memory));
result.emplace("hour", TypedValue(dur.SubDaysAsHours(), ctx_->memory));
result.emplace("minute", TypedValue(dur.SubDaysAsMinutes(), ctx_->memory));
result.emplace("second", TypedValue(dur.SubDaysAsSeconds(), ctx_->memory));
result.emplace("millisecond", TypedValue(dur.SubDaysAsMilliseconds(), ctx_->memory));
result.emplace("microseconds", TypedValue(dur.SubDaysAsMicroseconds(), ctx_->memory));
result.emplace("nanoseconds", TypedValue(dur.SubDaysAsNanoseconds(), ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Date: {
const auto &date = expression_result.ValueDate();
result.emplace("year", TypedValue(date.year, ctx_->memory));
result.emplace("month", TypedValue(date.month, ctx_->memory));
result.emplace("day", TypedValue(date.day, ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::LocalTime: {
const auto &lt = expression_result.ValueLocalTime();
result.emplace("hour", TypedValue(lt.hour, ctx_->memory));
result.emplace("minute", TypedValue(lt.minute, ctx_->memory));
result.emplace("second", TypedValue(lt.second, ctx_->memory));
result.emplace("millisecond", TypedValue(lt.millisecond, ctx_->memory));
result.emplace("microsecond", TypedValue(lt.microsecond, ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::LocalDateTime: {
const auto &ldt = expression_result.ValueLocalDateTime();
const auto &date = ldt.date;
const auto &lt = ldt.local_time;
result.emplace("year", TypedValue(date.year, ctx_->memory));
result.emplace("month", TypedValue(date.month, ctx_->memory));
result.emplace("day", TypedValue(date.day, ctx_->memory));
result.emplace("hour", TypedValue(lt.hour, ctx_->memory));
result.emplace("minute", TypedValue(lt.minute, ctx_->memory));
result.emplace("second", TypedValue(lt.second, ctx_->memory));
result.emplace("millisecond", TypedValue(lt.millisecond, ctx_->memory));
result.emplace("microsecond", TypedValue(lt.microsecond, ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Graph: {
const auto &graph = expression_result.ValueGraph();
utils::pmr::vector<TypedValue> vertices(ctx_->memory);
vertices.reserve(graph.vertices().size());
for (const auto &v : graph.vertices()) {
vertices.emplace_back(TypedValue(v, ctx_->memory));
}
result.emplace("nodes", TypedValue(std::move(vertices), ctx_->memory));
utils::pmr::vector<TypedValue> edges(ctx_->memory);
edges.reserve(graph.edges().size());
for (const auto &e : graph.edges()) {
edges.emplace_back(TypedValue(e, ctx_->memory));
}
result.emplace("edges", TypedValue(std::move(edges), ctx_->memory));
return TypedValue(result, ctx_->memory);
}
default:
throw QueryRuntimeException(
"Only nodes, edges, maps, temporal types and graphs have properties to be looked up.");
throw QueryRuntimeException("Only nodes, edges, maps and temporal types have properties to be looked-up.");
}
}
@@ -691,30 +531,6 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(MapProjectionLiteral &literal) override {
constexpr std::string_view kAllPropertiesSelector{"*"};
TypedValue::TMap result(ctx_->memory);
TypedValue::TMap all_properties_lookup(ctx_->memory);
for (const auto &[property_key, property_value] : literal.elements_) {
if (property_key.name == kAllPropertiesSelector.data()) {
auto maybe_all_properties_lookup = property_value->Accept(*this);
if (maybe_all_properties_lookup.type() != TypedValue::Type::Map) {
throw QueryRuntimeException("Expected a map from AllPropertiesLookup, got {}.",
maybe_all_properties_lookup.type());
}
all_properties_lookup = std::move(maybe_all_properties_lookup.ValueMap());
continue;
}
result.emplace(property_key.name, property_value->Accept(*this));
}
if (!all_properties_lookup.empty()) result.merge(all_properties_lookup);
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(Aggregation &aggregation) override {
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
}
@@ -1036,8 +852,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
DbAccessor *dba_;
// which switching approach should be used when evaluating
storage::View view_;
FrameChangeCollector *frame_change_collector_;
}; // namespace memgraph::query
};
/// A helper function for evaluating an expression that's an int.
///

Some files were not shown because too many files have changed in this diff Show More