Compare commits

..

80 Commits

Author SHA1 Message Date
imilinovic
3bf6cbf9b9 Merge branch 'master' of github.com:memgraph/memgraph into gc-proactive-deletion 2023-11-14 11:47:41 +01:00
Marko Barišić
9cc060c4b0 Fix error in upload-to-s3 job (#1504) 2023-11-13 13:01:01 +01:00
Marko Barišić
e671a0737e Fix package specific workflow file (#1503) 2023-11-13 12:54:19 +01:00
Marko Barišić
11be3972c4 Add workflow for packaging memgraph for specific target OS (#1502) 2023-11-13 12:20:49 +01:00
DavIvek
fdab42a023 Use static linking on c++ query modules for glibcxx (#1490) 2023-11-13 12:08:48 +01:00
Andi
e5b2c19ea2 Empty Collect() returns nothing (#1482) 2023-11-13 11:45:09 +01:00
Josipmrden
e907817854 Fix for in list segmentation fault (#1494) 2023-11-13 05:17:10 +01:00
Josipmrden
0756cd6898 Add fix indexed join crash (#1478) 2023-11-12 22:12:25 -05:00
Josipmrden
38ad5e2146 Fix parallel index loading (#1479) 2023-11-12 23:51:00 +01:00
Josipmrden
3c413a7e50 Fix hash join expression matching (#1496) 2023-11-12 14:45:02 -05:00
Antonio Filipovic
17915578f8 Fix race condition and arena tracking bug (#1468) 2023-11-09 18:56:36 +01:00
Marko Barišić
4e9a036881 Fix v2.12 release pipeline (#1445) 2023-11-08 07:09:02 -05:00
DavIvek
c8fe9ee7d9 Fix accessing a variable bound to a list within BFS function (#1380) 2023-11-07 20:34:50 +01:00
Ante Javor
e4afddf518 Fix compare results in mgbench (#1319) 2023-11-07 17:04:37 +01:00
Antonio Filipovic
4d5ea03dfa Use extent hooks for memory procedure limit (#1443) 2023-11-07 16:04:29 +01:00
DavIvek
ece4b0dba8 Fix cached plan not getting invalidated (#1348) 2023-11-07 13:34:03 +01:00
Andi
66487a6dce Durability improvements (#1385) 2023-11-07 11:37:54 +01:00
Andi
f4b97fc03d Fix missing statistics for SetPropertiesCursor (#1460) 2023-11-07 09:11:20 +01:00
Antonio Filipovic
58648d1a70 Revert license sender info (#1461) 2023-11-06 16:57:09 +01:00
Antonio Filipovic
1ab7f6ac78 Add notification for user on max map count (#1408) 2023-11-06 15:44:26 +01:00
andrejtonev
dbc6054689 Replication refactor (part 5) (#1378) 2023-11-06 11:50:49 +00:00
Aidar Samerkhanov
16b8c7b27c Fix Kafka flaky unit test (#1409) 2023-11-05 20:51:56 +01:00
Antonio Filipovic
48631d1e37 Rename memory usage and memory allocated (#1426) 2023-11-03 14:40:45 +01:00
Antonio Filipovic
93e6d058d2 Remove all_shortest paths unnecessary logs (#1425) 2023-11-03 12:46:06 +01:00
Andi
3e9f25b8e4 Support creating date and localtime from localdatetime (#1381) 2023-11-03 10:54:01 +01:00
Gareth Lloyd
cab59feb50 More proactive deletion 2023-11-02 21:57:34 +00:00
Andi
c94201621a Support deleting paths (#1383) 2023-11-02 14:07:48 +01:00
Andi
fdbc390d53 Throw when reduce inside exists (#1392) 2023-11-02 12:18:15 +01:00
Andi
5e6c5618f5 Make FrameChangeCollector owning the memory resource (#1398) 2023-11-02 09:54:39 +01:00
Andi
4aacd45640 Throw when exists() combined with CASE (#1382) 2023-11-02 08:25:34 +01:00
Gareth Andrew Lloyd
157b36162b Speedup socket unit test (#1444)
Was testing a setup that wasn't used in production, it would
unnecessarily thrash small buffers.
2023-11-01 17:24:24 +00:00
Marko Barišić
af4fdef029 Change license date before release (#1433) 2023-10-30 13:01:14 +01:00
Josipmrden
5b9802bd7b Extend property cache to the expression evaluator (#1432)
* Add support for property cache in the produce
* Fix the previous implementation in the map literal
2023-10-28 20:32:58 -07:00
Ante Pušić
b1c3168308 Fix PROFILE infinite loop (#1431) 2023-10-28 15:34:52 +02:00
Andi
011caf3bf1 Fix mgbench daily upload (#1410) 2023-10-28 12:52:14 +02:00
Andi
b1bd977f7b Fix GQL release worklflows (#1424) 2023-10-28 12:50:16 +02:00
gvolfing
c296dc67ce Add index count to index info (#1229) 2023-10-27 18:13:05 +02:00
Ante Pušić
989bb97514 Extend Cypher queries with the index hinting feature (#1345) 2023-10-27 14:26:19 +02:00
Marko Barišić
a94588bde3 Revert CMake default build type error (#1420) 2023-10-27 08:58:37 +02:00
Marko Barišić
e9f3a5fd1b Fix default value in release workflows (#1418) 2023-10-26 16:17:16 +02:00
Marko Budiselić
80e1fba8f5 Update bug_report.md template (#1400) 2023-10-26 13:09:16 +02:00
Ante Pušić
3158a16ffd Add filtering details to EXPLAIN and PROFILE (#1265) 2023-10-25 21:36:20 +02:00
Matija Pintarić
411f8c9d56 Move essential query modules from MAGE to Memgraph (#1384)
* schema.cpp
* mgps.py
* convert.py
2023-10-25 18:27:44 +02:00
Antonio Filipovic
a84f570c6d Use extent hooks for per query memory limit (#1340) 2023-10-25 16:01:59 +02:00
Josipmrden
3d4d841753 Add constraint verification update only on necessary actions (#1341) 2023-10-25 16:01:02 +02:00
Antonio Filipovic
2426d7980d Add OOM enabler in operator tree (#1379) 2023-10-25 12:16:11 +02:00
Josipmrden
7ef10dd82a Fix gql behave dropping connection on Memgraph (#1399) 2023-10-25 10:59:02 +02:00
Gareth Andrew Lloyd
5b91f85161 Improve storage GC (#1387) 2023-10-24 23:41:21 +02:00
Antonio Filipovic
0d9bd5554c Fix potential bug on memory pool (#1299) 2023-10-24 22:31:36 +02:00
Josipmrden
e617ff9b59 Provide textual information for inefficient plans with notifications (#1343) 2023-10-24 22:20:05 +02:00
Josipmrden
be16ca7362 Add cartesian and hash join operators (#1193) 2023-10-24 21:54:42 +02:00
Josipmrden
fdf63436ab Add cartesian and hash join mgbench (#1393) 2023-10-24 19:44:11 +02:00
Josipmrden
4e8148f7d9 Add retry logic possible when conflicting transactions (#1361) 2023-10-24 19:43:23 +02:00
imilinovic
1f118e7521 Add renaming of edge types (#1364) 2023-10-24 17:12:09 +02:00
Marko Damjanić
9803f47828 Update contributing part in the README (#1224) 2023-10-24 14:38:06 +02:00
DavIvek
98680b04c9 Add DNS support for cluster replica address (#1323) 2023-10-24 13:11:36 +02:00
Josipmrden
1d45016217 Add values and keys function to map (#1246) 2023-10-24 06:19:20 +02:00
Matija Pintarić
97ed912ab6 Implement map key exists in mgp (#1336) 2023-10-23 15:29:41 +02:00
gvolfing
aec4c3dd2b Fix bug in alias mappings (#1252) 2023-10-23 13:07:46 +02:00
Antonio Filipovic
7f7f3adfcb Implement jemalloc extent hooks memory tracker (#1250)
Should improve/fix memory usage exceeds --memory-limit issues
2023-10-23 12:48:26 +02:00
andrejtonev
26e31ca06f Fix SHOW CONFIG to show the run-time flag status (#1278) 2023-10-23 10:18:07 +02:00
DavIvek
3ff2c72db9 Fix crash caused by deleting non-existing edge in DETACH DELETE (#1355) 2023-10-23 08:36:28 +02:00
Andi
af56ab6ea8 Forbid changing isolation level for disk and analytical (#1367)
Co-authored-by: Marko Budiselić <marko.budiselic@memgraph.com>
2023-10-23 06:02:56 +02:00
Marko Budiselić
945388fba6 Revert to Release under Diff Release workflow (#1390) 2023-10-22 22:31:06 +02:00
Marko Budiselić
b6693a7df0 Improve systemd config (#1288)
* Add a comment on how to restart automatically
* Add comment on how to deal with out-of-memory (OOM)
2023-10-22 18:44:24 +02:00
Aidar Samerkhanov
667e7f670e Parametrize CI workflows build type (#1324)
The Release is the default, it's also possible to run package_all with
RelWithDebInfo
2023-10-17 00:04:08 +02:00
andrejtonev
a100b900c5 Fix DB configuration error under expansion gbench (#1368) 2023-10-16 23:12:49 +02:00
Gareth Andrew Lloyd
64bf75117b Fix subtle replication bug (#1370)
When going from REPLICA to MAIN, need to ensure current WAL files are
finalised.
2023-10-16 22:56:56 +02:00
Marko Budiselić
9524a51576 Add v5 toolchain (#608)
* clang 17.0.2
* gcc 13.2
* upgrade libs
* tmp disable gpg check, tmp disable fblibs
2023-10-16 19:01:39 +02:00
Marko Budiselić
fd10d1c9f8 Fix CPU 100% usage by websocket error handling improvement (#1327) 2023-10-16 15:41:12 +02:00
andrejtonev
22d8ef75e0 Updated telemetry client-side (#1337) 2023-10-16 14:16:00 +02:00
Andi
7b0bafa21e Add human readable memory allocations in show storage info (#1366) 2023-10-16 11:35:44 +02:00
Andi
de9280b334 Refactor disk storage (#1347) 2023-10-16 09:11:07 +02:00
Kruno Golubic
766ac48261 Update banner in README (#1344) 2023-10-12 14:27:36 +02:00
Andi
06868c8be7 Run separate GQL suits for different storage modes (#1346) 2023-10-11 11:42:41 +02:00
Andi
1a3c5af797 Improve expansions on disk (#1335)
* Improve disk expansions
2023-10-11 10:18:50 +02:00
Gareth Andrew Lloyd
d278a33f31 Decouple pure replication state from storage [part 1] (#1325)
A major refactor to decouple replication state from storage.
ATM it is still owned by storage but a following part should fix that.
2023-10-10 11:44:19 +01:00
Aidar Samerkhanov
7fbf5857f2 Add GQL behave tests for on-disk storage (#1238) 2023-10-10 09:27:11 +03:00
DavIvek
0d51a20a02 Fix a crash caused by declaring a path with only one node in OPTIONAL MATCH clause (#1318) 2023-10-09 15:25:25 +02:00
DavIvek
3143c986de Fix crash caused by using exists() in a RETURN statement (#1303) 2023-10-09 11:31:49 +02:00
476 changed files with 26254 additions and 6545 deletions

View File

@@ -33,4 +33,4 @@ for file in $modified_files; do
fi fi
done; done;
return ${FAIL} exit ${FAIL}

View File

@@ -1,19 +1,18 @@
--- ---
name: Bug report name: Bug report
about: Create a report to help us improve about: Create a report to help us improve
title: "[BUG] " title: ""
labels: bug labels: bug
assignees: gitbuda, antonio2368 assignees: gitbuda
--- ---
**Memgraph version** **Memgraph version**
Which version did you use? Which version did you use?
**Environment** **Environment**
Some information about the environment you are using Memgraph on: operating Some information about the environment you are using Memgraph on: operating
system, how do you connect, with or without docker, which driver etc. system, architecture (ARM, x86), how do you connect, with or without docker,
which driver etc.
**Describe the bug** **Describe the bug**
A clear and concise description of what the bug is. A clear and concise description of what the bug is.
@@ -22,6 +21,7 @@ A clear and concise description of what the bug is.
Steps to reproduce the behavior: Steps to reproduce the behavior:
1. Run the following query '...' 1. Run the following query '...'
2. Click on '....' 2. Click on '....'
3. ... IDEALLY: link to the workload info (DATASET & QUERIES) ...
**Expected behavior** **Expected behavior**
A clear and concise description of what you expected to happen. A clear and concise description of what you expected to happen.
@@ -32,3 +32,11 @@ your problem.
**Additional context** **Additional context**
Add any other context about the problem here. Add any other context about the problem here.
**Verification Environment**
Once we fix it, what do you need to verify the fix?
Do you need:
* Plain memgraph package -> for which Linux?
* Plain memgraph Docker image?
* Which architecture do you use ARM | x86?
* Full Memgraph platform?

View File

@@ -59,7 +59,7 @@ jobs:
source ve3/bin/activate source ve3/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
./main.py --benchmark-name "macro_benchmark" \ ./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \ --benchmark-results "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"
@@ -73,6 +73,8 @@ jobs:
./benchmark.py vendor-native --num-workers-for-benchmark 1 --export-results benchmark_high_write_set_property.json high_write_set_property ./benchmark.py vendor-native --num-workers-for-benchmark 1 --export-results benchmark_high_write_set_property.json high_write_set_property
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results cartesian.json cartesian
- name: Upload mgbench results - name: Upload mgbench results
run: | run: |
cd tools/bench-graph-client cd tools/bench-graph-client
@@ -80,19 +82,25 @@ jobs:
source ve3/bin/activate source ve3/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \ ./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_pokec.json" \ --benchmark-results "../../tests/mgbench/benchmark_pokec.json" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "supernode" \ ./main.py --benchmark-name "supernode" \
--benchmark-results-path "../../tests/mgbench/benchmark_supernode.json" \ --benchmark-results "../../tests/mgbench/benchmark_supernode.json" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "high_write_set_property" \ ./main.py --benchmark-name "high_write_set_property" \
--benchmark-results-path "../../tests/mgbench/benchmark_high_write_set_property.json" \ --benchmark-results "../../tests/mgbench/benchmark_high_write_set_property.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "cartesian" \
--benchmark-results "../../tests/mgbench/cartesian.json" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -43,7 +43,7 @@ jobs:
# Build community binaries. # Build community binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMG_ENTERPRISE=OFF ..
make -j$THREADS make -j$THREADS
- name: Run unit tests - name: Run unit tests
@@ -244,12 +244,14 @@ jobs:
# Build release binaries. # Build release binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$THREADS make -j$THREADS
- name: Run GQL Behave tests - name: Run GQL Behave tests
run: | run: |
cd tests/gql_behave cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status
@@ -349,7 +351,7 @@ jobs:
./init ./init
# Build only memgraph release binarie. # Build only memgraph release binarie.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
make -j$THREADS memgraph make -j$THREADS memgraph
- name: Run Jepsen tests - name: Run Jepsen tests

View File

@@ -8,6 +8,13 @@ on:
memgraph_version: memgraph_version:
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'" description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
required: false required: false
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
jobs: jobs:
centos-7: centos-7:
@@ -20,7 +27,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package centos-7 ./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -37,7 +44,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package centos-9 ./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -54,7 +61,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package debian-10 ./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -71,7 +78,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package debian-11 ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -89,7 +96,7 @@ jobs:
- name: "Build package" - name: "Build package"
run: | run: |
cd release/package cd release/package
./run.sh package debian-11 --for-docker ./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker
./run.sh docker ./run.sh docker
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
@@ -107,7 +114,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package ubuntu-18.04 ./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -124,7 +131,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package ubuntu-20.04 ./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -141,7 +148,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package ubuntu-22.04 ./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -158,7 +165,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package debian-11 --for-platform ./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -175,7 +182,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package fedora-36 ./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -192,7 +199,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package amzn-2 ./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -209,7 +216,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package debian-11-arm ./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:
@@ -226,7 +233,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package" - name: "Build package"
run: | run: |
./release/package/run.sh package ubuntu-22.04-arm ./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }}
- name: "Upload package" - name: "Upload package"
uses: actions/upload-artifact@v3 uses: actions/upload-artifact@v3
with: with:

294
.github/workflows/package_specific.yaml vendored Normal file
View File

@@ -0,0 +1,294 @@
name: Package Specific
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. Leave this field empty if you don't want to upload binaries to S3. Format: 'X.Y.Z'"
required: false
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
target_os:
type: choice
description: "Target OS for which memgraph will be packaged. Default is Ubuntu 22.04"
default: 'ubuntu-22_04'
options:
- amzn-2
- centos-7
- centos-9
- debian-10
- debian-11
- debian-11-arm
- debian-11-platform
- docker
- fedora-36
- ubuntu-18_04
- ubuntu-20_04
- ubuntu-22_04
- ubuntu-22_04-arm
jobs:
amzn-2:
if: ${{ github.event.inputs.target_os == 'amzn-2' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: amzn-2
path: build/output/amzn-2/memgraph*.rpm
centos-7:
if: ${{ github.event.inputs.target_os == 'centos-7' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
centos-9:
if: ${{ github.event.inputs.target_os == 'centos-9' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
debian-10:
if: ${{ github.event.inputs.target_os == 'debian-10' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
debian-11:
if: ${{ github.event.inputs.target_os == 'debian-11' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
debian-11-arm:
if: ${{ github.event.inputs.target_os == 'debian-11-arm' }}
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 120
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11-aarch64
path: build/output/debian-11-arm/memgraph*.deb
debian-11-platform:
if: ${{ github.event.inputs.target_os == 'debian-11-platform' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
docker:
if: ${{ github.event.inputs.target_os == 'docker' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: docker
path: build/output/docker/memgraph*.tar.gz
fedora-36:
if: ${{ github.event.inputs.target_os == 'fedora-36' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: fedora-36
path: build/output/fedora-36/memgraph*.rpm
ubuntu-18_04:
if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-18.04
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-20_04:
if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-20.04
path: build/output/ubuntu-20.04/memgraph*.deb
ubuntu-22_04:
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04
path: build/output/ubuntu-22.04/memgraph*.deb
ubuntu-22_04-arm:
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' }}
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 120
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04-aarch64
path: build/output/ubuntu-22.04-arm/memgraph*.deb
upload-to-s3:
# only run upload if we specified version. Allows for runs without upload
if: "${{ github.event.inputs.memgraph_version != '' }}"
needs: [amzn-2, centos-7, centos-9, debian-10, debian-11, debian-11-arm, debian-11-platform, docker, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04, ubuntu-22_04-arm]
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v3
with:
# name: # if name input parameter is not provided, all artifacts are downloaded
# and put in directories named after each one.
path: build/output/release
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: "download.memgraph.com"
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph/v${{ github.event.inputs.memgraph_version }}/"

View File

@@ -30,7 +30,7 @@ jobs:
# Build only memgraph release binaries. # Build only memgraph release binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=Release .. cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS make -j$THREADS
- name: Get branch name (merge) - name: Get branch name (merge)
@@ -52,6 +52,8 @@ jobs:
./benchmark.py vendor-native --num-workers-for-benchmark 1 --export-results benchmark_high_write_set_property.json high_write_set_property ./benchmark.py vendor-native --num-workers-for-benchmark 1 --export-results benchmark_high_write_set_property.json high_write_set_property
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_cartesian.json cartesian
- name: Upload benchmark results - name: Upload benchmark results
run: | run: |
cd tools/bench-graph-client cd tools/bench-graph-client
@@ -59,19 +61,25 @@ jobs:
source ve3/bin/activate source ve3/bin/activate
pip install -r requirements.txt pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \ ./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \ --benchmark-results "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "supernode" \ ./main.py --benchmark-name "supernode" \
--benchmark-results-path "../../tests/mgbench/benchmark_supernode.json" \ --benchmark-results "../../tests/mgbench/benchmark_supernode.json" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "high_write_set_property" \ ./main.py --benchmark-name "high_write_set_property" \
--benchmark-results-path "../../tests/mgbench/benchmark_high_write_set_property.json" \ --benchmark-results "../../tests/mgbench/benchmark_high_write_set_property.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"
./main.py --benchmark-name "cartesian" \
--benchmark-results "../../tests/mgbench/cartesian.json" \
--github-run-id "${{ github.run_id }}" \ --github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \ --github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}" --head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -2,6 +2,15 @@ name: Release CentOS 8
on: on:
workflow_dispatch: workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule: schedule:
- cron: "0 22 * * *" - cron: "0 22 * * *"
@@ -31,9 +40,13 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build community binaries. # Build community binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
make -j$THREADS make -j$THREADS
- name: Run unit tests - name: Run unit tests
@@ -188,9 +201,13 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build release binaries. # Build release binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS make -j$THREADS
- name: Create enterprise RPM package - name: Create enterprise RPM package
@@ -243,7 +260,9 @@ jobs:
- name: Run GQL Behave tests - name: Run GQL Behave tests
run: | run: |
cd tests/gql_behave cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status

View File

@@ -2,6 +2,15 @@ name: Release Debian 10
on: on:
workflow_dispatch: workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule: schedule:
- cron: "0 22 * * *" - cron: "0 22 * * *"
@@ -31,9 +40,13 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build community binaries. # Build community binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
make -j$THREADS make -j$THREADS
- name: Run unit tests - name: Run unit tests
@@ -188,9 +201,13 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build release binaries. # Build release binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS make -j$THREADS
- name: Create enterprise DEB package - name: Create enterprise DEB package
@@ -242,7 +259,9 @@ jobs:
- name: Run GQL Behave tests - name: Run GQL Behave tests
run: | run: |
cd tests/gql_behave cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status
@@ -320,9 +339,14 @@ jobs:
source /opt/toolchain-v4/activate source /opt/toolchain-v4/activate
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build only memgraph release binary. # Build only memgraph release binary.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS memgraph make -j$THREADS memgraph
- name: Run Jepsen tests - name: Run Jepsen tests

View File

@@ -2,6 +2,15 @@ name: Release Ubuntu 20.04
on: on:
workflow_dispatch: workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule: schedule:
- cron: "0 22 * * *" - cron: "0 22 * * *"
@@ -31,9 +40,13 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build community binaries. # Build community binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DMG_ENTERPRISE=OFF ..
make -j$THREADS make -j$THREADS
- name: Run unit tests - name: Run unit tests
@@ -188,9 +201,13 @@ jobs:
# Initialize dependencies. # Initialize dependencies.
./init ./init
# Set default build_type to Release
INPUT_BUILD_TYPE=${{ github.event.inputs.build_type }}
BUILD_TYPE=${INPUT_BUILD_TYPE:-"Release"}
# Build release binaries. # Build release binaries.
cd build cd build
cmake -DCMAKE_BUILD_TYPE=release .. cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE ..
make -j$THREADS make -j$THREADS
- name: Create enterprise DEB package - name: Create enterprise DEB package
@@ -242,7 +259,9 @@ jobs:
- name: Run GQL Behave tests - name: Run GQL Behave tests
run: | run: |
cd tests/gql_behave cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
./continuous_integration ./continuous_integration
- name: Save quality assurance status - name: Save quality assurance status

2
.gitignore vendored
View File

@@ -60,3 +60,5 @@ src/storage/distributed/rpc/concurrent_id_mapper_rpc_messages.hpp
src/transactions/distributed/engine_rpc_messages.hpp src/transactions/distributed/engine_rpc_messages.hpp
/tests/manual/js/transaction_timeout/package-lock.json /tests/manual/js/transaction_timeout/package-lock.json
/tests/manual/js/transaction_timeout/node_modules/ /tests/manual/js/transaction_timeout/node_modules/
.vscode/
src/query/frontend/opencypher/grammar/.antlr/*

View File

@@ -42,10 +42,6 @@ endif()
project(memgraph LANGUAGES C CXX) project(memgraph LANGUAGES C CXX)
# NOTE: once in a while this needs to be toggled to check headers are
# correct and PCH isn't masking any include issues
set(CMAKE_DISABLE_PRECOMPILE_HEADERS OFF)
#TODO: upgrade to cmake 3.24 + CheckIPOSupported #TODO: upgrade to cmake 3.24 + CheckIPOSupported
#cmake_policy(SET CMP0138 NEW) #cmake_policy(SET CMP0138 NEW)
#include(CheckIPOSupported) #include(CheckIPOSupported)
@@ -239,7 +235,6 @@ else()
endif() endif()
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# default build type is debug # default build type is debug
if (NOT CMAKE_BUILD_TYPE) if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Debug") set(CMAKE_BUILD_TYPE "Debug")
@@ -297,7 +292,7 @@ if (MG_ENTERPRISE)
add_definitions(-DMG_ENTERPRISE) add_definitions(-DMG_ENTERPRISE)
endif() endif()
set(ENABLE_JEMALLOC ON) option(ENABLE_JEMALLOC "Use jemalloc" ON)
if (ASAN) if (ASAN)
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN") message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")

View File

@@ -1,5 +1,5 @@
<p align="center"> <p align="center">
<img width="400px" src="https://uploads-ssl.webflow.com/5e7ceb09657a69bdab054b3a/5e7ceb09657a6937ab054bba_Black_Original%20_Logo.png"> <img src="https://public-assets.memgraph.com/github-readme-images/github-memgraph-repo-banner.png">
</p> </p>
--- ---
@@ -141,29 +141,20 @@ guide](https://memgraph.com/docs/memgraph/reference-guide/configuration).
## :trophy: Contributing ## :trophy: Contributing
The main purpose of this repository is to continue evolving Memgraph, making it Welcome to the heart of Memgraph development! We're on a mission to supercharge Memgraph, making it faster, more user-friendly, and even more powerful. We owe a big thanks to our fantastic community of contributors who help us fix bugs and bring incredible improvements to life. If you're passionate about databases and open source, here's your chance to make a difference!
faster and easier to use. Development of Memgraph happens in the open on GitHub,
and we are grateful to the community for contributing bug fixes and ### Explore Memgraph Internals
improvements. Read below to learn how you can take part in improving Memgraph.
Interested in the nuts and bolts of Memgraph? Our [internals documentation](https://memgraph.notion.site/Memgraph-Internals-12b69132d67a417898972927d6870bd2) is where you can uncover the inner workings of Memgraph's architecture, learn how to build the project from scratch, and discover the secrets of effective contributions. Dive deep into the database!
### Dive into the Contributing Guide
Ready to jump into the action? Explore our [contributing guide](CONTRIBUTING.md) to get the inside scoop on how we develop Memgraph. It's your roadmap for suggesting bug fixes and enhancements. Contribute your skills and ideas!
### Code of Conduct ### Code of Conduct
Memgraph has adopted a Code of Conduct that we expect project participants to Our commitment to a respectful and professional community is unwavering. Every participant in Memgraph is expected to adhere to a stringent Code of Conduct. Please carefully review [the complete text](CODE_OF_CONDUCT.md) to gain a comprehensive understanding of the behaviors that are both expected and explicitly prohibited.
adhere to. Please read [the full text](CODE_OF_CONDUCT.md) so that you can
understand what actions will and will not be tolerated.
### Contributing Guide We maintain a zero-tolerance policy towards any violations. Our shared commitment to this Code of Conduct ensures that Memgraph remains a place where integrity and excellence are paramount.
Read our [contributing guide](CONTRIBUTING.md) to learn about our development
process and how to propose bug fixes and improvements.
### Internals
Read our
[internal](https://memgraph.notion.site/Memgraph-Internals-12b69132d67a417898972927d6870bd2)
docs to learn more about Memgraph's architecture, how to build the project from
source and how to start contributing. All information related to the database,
can be found in the aforementioned docs.
### :scroll: License ### :scroll: License

View File

@@ -1,8 +1,8 @@
{ {
"dbms.components": "mgps.components", "dbms.components": "mgps.components",
"apoc.util.validate": "mgps.validate", "apoc.util.validate": "mgps.validate",
"db.schema.nodeTypeProperties":"schema.node_type_properties", "db.schema.nodeTypeProperties": "schema.NodeTypeOroperties",
"db.schema.relTypeProperties":"schema.rel_type_properties", "db.schema.relTypeProperties": "schema.RelTypeProperties",
"apoc.coll.contains": "collections.contains", "apoc.coll.contains": "collections.contains",
"apoc.coll.partition": "collections.partition", "apoc.coll.partition": "collections.partition",
"apoc.coll.toSet": "collections.to_set", "apoc.coll.toSet": "collections.to_set",
@@ -23,5 +23,4 @@
"apoc.refactor.cloneSubgraph": "refactor.clone_subgraph", "apoc.refactor.cloneSubgraph": "refactor.clone_subgraph",
"apoc.refactor.cloneSubgraphFromPath": "refactor.clone_subgraph_from_path", "apoc.refactor.cloneSubgraphFromPath": "refactor.clone_subgraph_from_path",
"apoc.label.exists": "label.exists" "apoc.label.exists": "label.exists"
} }

View File

@@ -1,2 +0,0 @@
archives
build

View File

@@ -1,5 +1,10 @@
# Memgraph Operating Environments # Memgraph Operating Environments
## Issues related to build toolchain
* GCC 11.2 (toolchain-v4) doesn't compile on Fedora 38, multiple definitions of enum issue
* spdlog 1.10/11 doesn't work with fmt 10.0.0
## os ## os
Under the `os` directory, you can find scripts to install all required system Under the `os` directory, you can find scripts to install all required system

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
curl libcurl-devel # for cmake curl libcurl-devel # for cmake
readline-devel # for cmake and llvm readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig libedit-devel pcre-devel pcre2-devel automake bison # for swig
file file
openssl-devel openssl-devel
gmp-devel gmp-devel
@@ -158,9 +158,9 @@ install() {
continue continue
fi fi
if [ "$pkg" == nodejs ]; then if [ "$pkg" == nodejs ]; then
curl -sL https://rpm.nodesource.com/setup_16.x | bash -
if ! yum list installed nodejs >/dev/null 2>/dev/null; then if ! yum list installed nodejs >/dev/null 2>/dev/null; then
yum install -y nodejs yum install https://rpm.nodesource.com/pub_16.x/nodistro/repo/nodesource-release-nodistro-1.noarch.rpm -y
yum install nodejs -y --setopt=nodesource-nodejs.module_hotfixes=1
fi fi
continue continue
fi fi
@@ -172,13 +172,6 @@ install() {
fi fi
continue continue
fi fi
if [ "$pkg" == nodejs ]; then
curl -sL https://rpm.nodesource.com/setup_16.x | bash -
if ! yum list installed nodejs >/dev/null 2>/dev/null; then
yum install -y nodejs
fi
continue
fi
if [ "$pkg" == java-11-openjdk ]; then if [ "$pkg" == java-11-openjdk ]; then
amazon-linux-extras install -y java-openjdk11 amazon-linux-extras install -y java-openjdk11
continue continue

View File

@@ -20,7 +20,7 @@ TOOLCHAIN_BUILD_DEPS=(
curl # snappy curl # snappy
readline-devel # cmake and llvm readline-devel # cmake and llvm
libffi-devel libxml2-devel perl-Digest-MD5 # llvm libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel automake bison # swig libedit-devel pcre-devel pcre2-devel automake bison # swig
file file
openssl-devel openssl-devel
gmp-devel gmp-devel

View File

@@ -17,7 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
readline-devel # for cmake and llvm readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig libedit-devel pcre-devel pcre2-devel automake bison # for swig
file file
openssl-devel openssl-devel
gmp-devel gmp-devel

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb libgmp-dev # for gdb
gperf # for proxygen gperf # for proxygen
git # for fbthrift git # for fbthrift
libedit-dev libpcre3-dev automake bison # for swig libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
) )
TOOLCHAIN_RUN_DEPS=( TOOLCHAIN_RUN_DEPS=(

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
curl # snappy curl # snappy
file # for libunwind file # for libunwind
libssl-dev # for libevent libssl-dev # for libevent

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
curl # snappy curl # snappy
file # for libunwind file # for libunwind
libssl-dev # for libevent libssl-dev # for libevent

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
curl libcurl-devel # for cmake curl libcurl-devel # for cmake
readline-devel # for cmake and llvm readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig libedit-devel pcre-devel pcre2-devel automake bison # for swig
file file
openssl-devel openssl-devel
gmp-devel gmp-devel

108
environment/os/fedora-38.sh Executable file
View File

@@ -0,0 +1,108 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "fedora-38"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
perl # for openssl
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
if [ -v LD_LIBRARY_PATH ]; then
# On Fedora 38 yum/dnf and python11 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
fi
local missing=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
if [ -v OLD_LD_LIBRARY_PATH ]; then
echo "Restoring LD_LIBRARY_PATH..."
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -25,7 +25,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb libgmp-dev # for gdb
gperf # for proxygen gperf # for proxygen
libssl-dev libssl-dev
libedit-dev libpcre3-dev automake bison # swig libedit-dev libpcre2-dev libpcre3-dev automake bison # swig
) )
TOOLCHAIN_RUN_DEPS=( TOOLCHAIN_RUN_DEPS=(

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb libgmp-dev # for gdb
gperf # for proxygen gperf # for proxygen
libssl-dev libssl-dev
libedit-dev libpcre3-dev automake bison # for swig libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
) )
TOOLCHAIN_RUN_DEPS=( TOOLCHAIN_RUN_DEPS=(

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb libgmp-dev # for gdb
gperf # for proxygen gperf # for proxygen
libssl-dev libssl-dev
libedit-dev libpcre3-dev automake bison # for swig libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
) )
TOOLCHAIN_RUN_DEPS=( TOOLCHAIN_RUN_DEPS=(

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb libgmp-dev # for gdb
gperf # for proxygen gperf # for proxygen
libssl-dev libssl-dev
libedit-dev libpcre3-dev automake bison # for swig libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
) )
TOOLCHAIN_RUN_DEPS=( TOOLCHAIN_RUN_DEPS=(

View File

@@ -1 +1,4 @@
archives
build
output
*.tar.gz *.tar.gz

View File

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo ) CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR" cd "$DIR"
source "$DIR/../util.sh" source "$DIR/../../util.sh"
DISTRO="$(operating_system)" DISTRO="$(operating_system)"
# toolchain version # toolchain version
@@ -30,10 +30,10 @@ LLVM_VERSION=11.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Check for the dependencies. # Check for the dependencies.
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)" echo "ALL BUILD PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS $DIR/../../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)" echo "ALL RUN PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS $DIR/../../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
# check installation directory # check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -442,7 +442,7 @@ In order to be able to run all of these tools you should install the following
packages: packages:
\`\`\` \`\`\`
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS) $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
\`\`\` \`\`\`
## Usage ## Usage

View File

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo ) CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR" cd "$DIR"
source "$DIR/../util.sh" source "$DIR/../../util.sh"
DISTRO="$(operating_system)" DISTRO="$(operating_system)"
# toolchain version # toolchain version
@@ -31,10 +31,10 @@ LLVM_VERSION_LONG=12.0.1-rc4
SWIG_VERSION=4.0.2 # used only for LLVM compilation SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Check for the dependencies. # Check for the dependencies.
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)" echo "ALL BUILD PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS $DIR/../../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)" echo "ALL RUN PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS $DIR/../../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
# check installation directory # check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -452,7 +452,7 @@ In order to be able to run all of these tools you should install the following
packages: packages:
\`\`\` \`\`\`
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS) $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
\`\`\` \`\`\`
## Usage ## Usage

View File

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo ) CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR" cd "$DIR"
source "$DIR/../util.sh" source "$DIR/../../util.sh"
DISTRO="$(operating_system)" DISTRO="$(operating_system)"
function log_tool_name () { function log_tool_name () {
@@ -51,17 +51,13 @@ CPPCHECK_VERSION=2.6
LLVM_VERSION=13.0.0 LLVM_VERSION=13.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Set the right env script # Set the right operating system setup script.
ENV_SCRIPT="$DIR/../os/$DISTRO.sh" ENV_SCRIPT="$DIR/../../os/$DISTRO.sh"
if [[ "$for_arm" = true ]]; then if [[ "$for_arm" = true ]]; then
ENV_SCRIPT="$DIR/../os/$DISTRO-arm.sh" ENV_SCRIPT="$DIR/../../os/$DISTRO-arm.sh"
fi fi
# Check for the toolchain build dependencies.
echo "ALL BUILD PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_BUILD_DEPS)" echo "ALL BUILD PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_BUILD_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_BUILD_DEPS ${ENV_SCRIPT} check TOOLCHAIN_BUILD_DEPS
# Check for the toolchain run dependencies.
echo "ALL RUN PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_RUN_DEPS)" echo "ALL RUN PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_RUN_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_RUN_DEPS ${ENV_SCRIPT} check TOOLCHAIN_RUN_DEPS
@@ -658,7 +654,7 @@ In order to be able to run all of these tools you should install the following
packages: packages:
\`\`\` \`\`\`
$($DIR/../os/$ENV_SCRIPT.sh list TOOLCHAIN_RUN_DEPS) $($DIR/../../os/$ENV_SCRIPT.sh list TOOLCHAIN_RUN_DEPS)
\`\`\` \`\`\`
## Usage ## Usage

View File

@@ -4,7 +4,7 @@ diff -ur a/CMakeLists.txt b/CMakeLists.txt
@@ -52,9 +52,9 @@ @@ -52,9 +52,9 @@
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0) add_definitions(-D_HAS_EXCEPTIONS=0)
- # Disable RTTI. - # Disable RTTI.
- string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
@@ -17,7 +17,7 @@ diff -ur a/CMakeLists.txt b/CMakeLists.txt
@@ -77,9 +77,9 @@ @@ -77,9 +77,9 @@
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
- # Disable RTTI. - # Disable RTTI.
- string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
@@ -25,5 +25,5 @@ diff -ur a/CMakeLists.txt b/CMakeLists.txt
+ # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti") + # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make # BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make

1280
environment/toolchain/v5/build.sh Executable file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,42 @@
#!/bin/bash -ex
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PREFIX=/opt/toolchain-v5
# NOTE: Often times when versions in the build script are changes, something
# doesn't work. To avoid rebuild of the whole toolchain but rebuild specific
# lib from 0, just comment specific line under this cript and run it. Don't
# forget to comment back to avoid unnecessary deletes next time your run this
# cript.
# rm -rf "$DIR/build"
# rm -rf "$DIR/output"
# rm -rf "$PREFIX/bin/gcc"
# rm -rf "$PREFIX/bin/ld.gold"
# rm -rf "$PREFIX/bin/gdb"
# rm -rf "$PREFIX/bin/cmake"
# rm -rf "$PREFIX/bin/clang"
# rm -rf "$PREFIX/include/bzlib.h"
# rm -rf "$PREFIX/include/fmt"
# rm -rf "$PREFIX/include/lz4.h"
# rm -rf "$PREFIX/include/lzma.h"
# rm -rf "$PREFIX/include/zlib.h"
# rm -rf "$PREFIX/include/zstd.h"
# rm -rf "$PREFIX/include/jemalloc"
# rm -rf "$PREFIX/include/boost"
# rm -rf "$PREFIX/include/double-conversion"
# rm -rf "$PREFIX/include/gflags"
# rm -rf "$PREFIX/include/libunwind.h"
# rm -rf "$PREFIX/include/glog"
# rm -rf "$PREFIX/include/event2"
# rm -rf "$PREFIX/include/sodium.h"
# rm -rf "$PREFIX/include/libaio.h"
# rm -rf "$PREFIX/include/FlexLexer.h"
# rm -rf "$PREFIX/include/snappy.h"
# rm -rf "$PREFIX/include/fizz"
# rm -rf "$PREFIX/include/folly"
# rm -rf "$PREFIX/include/proxygen"
# rm -rf "$PREFIX/include/wangle"
# rm -rf "$PREFIX/include/thrift"
# rm -rf "$PREFIX"

View File

@@ -0,0 +1,41 @@
diff -ur a/folly/CMakeLists.txt b/folly/CMakeLists.txt
--- a/folly/CMakeLists.txt 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/CMakeLists.txt 2022-02-03 15:19:41.349693134 +0100
@@ -28,7 +28,6 @@
)
add_subdirectory(experimental/exception_tracer)
-add_subdirectory(logging/example)
if (PYTHON_EXTENSIONS)
# Create tree of symbolic links in structure required for successful
diff -ur a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp
--- a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2022-02-03 15:19:11.003368891 +0100
@@ -96,6 +96,7 @@
#define __builtin_unreachable()
#endif
+#if 0
namespace __cxxabiv1 {
void __cxa_throw(
@@ -154,5 +155,5 @@
}
} // namespace std
-
+#endif
#endif // defined(__GLIBCXX__)
diff -ur a/folly/Portability.h b/folly/Portability.h
--- a/folly/Portability.h 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/Portability.h 2022-02-03 15:19:11.003368891 +0100
@@ -566,7 +566,7 @@
#define FOLLY_HAS_COROUTINES 0
#elif (__cpp_coroutines >= 201703L || __cpp_impl_coroutine >= 201902L) && \
(__has_include(<coroutine>) || __has_include(<experimental/coroutine>))
-#define FOLLY_HAS_COROUTINES 1
+#define FOLLY_HAS_COROUTINES 0
// This is mainly to workaround bugs triggered by LTO, when stack allocated
// variables in await_suspend end up on a coroutine frame.
#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE

View File

@@ -0,0 +1,26 @@
diff --git a/folly/CMakeLists.txt b/folly/CMakeLists.txt
index e0e16df..471131e 100644
--- a/folly/CMakeLists.txt
+++ b/folly/CMakeLists.txt
@@ -28,7 +28,7 @@ install(
)
add_subdirectory(experimental/exception_tracer)
-add_subdirectory(logging/example)
+# add_subdirectory(logging/example)
if (PYTHON_EXTENSIONS)
# Create tree of symbolic links in structure required for successful
diff --git a/folly/Portability.h b/folly/Portability.h
index 365ef1b..42d24b8 100644
--- a/folly/Portability.h
+++ b/folly/Portability.h
@@ -560,7 +560,7 @@ constexpr auto kCpplibVer = 0;
(defined(__cpp_coroutines) && __cpp_coroutines >= 201703L) || \
(defined(__cpp_impl_coroutine) && __cpp_impl_coroutine >= 201902L)) && \
(__has_include(<coroutine>) || __has_include(<experimental/coroutine>))
-#define FOLLY_HAS_COROUTINES 1
+#define FOLLY_HAS_COROUTINES 0
// This is mainly to workaround bugs triggered by LTO, when stack allocated
// variables in await_suspend end up on a coroutine frame.
#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE

View File

@@ -0,0 +1,29 @@
diff -ur a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2021-05-05 00:53:34.000000000 +0200
+++ b/CMakeLists.txt 2022-01-27 17:18:34.758302398 +0100
@@ -52,9 +52,9 @@
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0)
- # Disable RTTI.
- string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Use -Wall for clang and gcc.
if(NOT CMAKE_CXX_FLAGS MATCHES "-Wall")
@@ -77,9 +77,9 @@
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
- # Disable RTTI.
- string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make

View File

@@ -0,0 +1,75 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBEzEOZIBEACxg/IuXERlDB48JBWmF4NxNUuuup1IhJAJyFGFSKh3OGAO2Ard
sNuRLjANsFXA7m7P5eTFcG+BoHHuAVYmKnI3PPZtHVLnUt4pGItPczQZ2BE1WpcI
ayjGTBJeKItX3Npqg9D/odO9WWS1i3FQPVdrLn0YH37/BA66jeMQCRo7g7GLpaNf
IrvYGsqTbxCwsmA37rpE7oyU4Yrf74HT091WBsRIoq/MelhbxTDMR8eu/dUGZQVc
Kj3lN55RepwWwUUKyqarY0zMt4HkFJ7v7yRL+Cvzy92Ouv4Wf2FlhNtEs5LE4Tax
W0PO5AEmUoKjX87SezQK0f652018b4u6Ex52cY7p+n5TII/UyoowH6+tY8UHo9yb
fStrqgNE/mY2bhA6+AwCaOUGsFzVVPTbjtxL3HacUP/jlA1h78V8VTvTs5d55iG7
jSqR9o05wje8rwNiXXK0xtiJahyNzL97Kn/DgPSqPIi45G+8nxWSPFM5eunBKRl9
vAnsvwrdPRsR6YR3uMHTuVhQX9/CY891MHkaZJ6wydWtKt3yQwJLYqwo5d4DwnUX
CduUwSKv+6RmtWI5ZmTQYOcBRcZyGKml9X9Q8iSbm6cnpFXmLrNQwCJN+D3SiYGc
MtbltZo0ysPMa6Xj5xFaYqWk/BI4iLb2Gs+ByGo/+a0Eq4XYBMOpitNniQARAQAB
tCdMYXNzZSBDb2xsaW4gPGxhc3NlLmNvbGxpbkB0dWthYW5pLm9yZz6JAlEEEwEK
ADsCGwMCHgECF4AECwkIBwMVCggFFgIDAQAWIQQ2kMJAzlG0Zw0wrRw47nV9aRhG
IAUCYEt9dQUJFxeR4wAKCRA47nV9aRhGIBNDEACxD6vJ+enZwe3IgkJh5JtLsC9b
MWCQRlPW1EVMsg96Cb5Rtron1eN1pp1TlzENJu1/C7C/VEsr9WwOPg26Men7fNf/
O21QM9IBWd/uB0Pu333WqKh92ESS5x9ST9DrG39nVGSPkQQBMuia72VrA+crPnwT
/h/u1IN6/sff5VDIU24rUiqW2Npy733dANruj7Ny0scRXVPltnVdhqwPHt6qNjC1
t+/cCnwHgW1BR1RYXBPpB42z/m29dL9rPrG0YPGWs2Bc+EATUICfEE6eIvwfciue
IJTjKT9Y9DrogJC2AYFhjC7N04OKdCB2hFs4BjexJwr4X0GJO7LhFl03c951AsIE
GHwrucRPB5bo2vmvQ8IvZn7CmtdUJzXv9JlyU6p+MIK1pz7TK6GgSOSffQIXZn6e
nUPtm9mEwuncOfmW8/ODYPs1gCWYgyiFJx8h7eEu+M4MxHSFBs7MwXf/Ae2fSp+M
P/p198qB8fC5oVBnF95qb0Qi0uc1D+Gb+gpBF+ymMb+s/VBOR3QWiym7AzBrJ62g
UnbC9jMLGnSRI+7p7raUfMTgXr5/oQoBw7ExJVltSSRrim2YH/t4CV47mO6dR9J3
1RtsTFIRNhz+07XPsETcuCV/dgqeC8fOFLt9MY17Sufhb1DcGy4urZBOIhXcpTV7
vHVj5IYH5nYOT49NRYkCOAQTAQIAIgUCTMQ5kgIbAwYLCQgHAwIGFQgCCQoLBBYC
AwECHgECF4AACgkQOO51fWkYRiAg4A/7BXKwoRaXrMbMPOW7vuVF7c2IKB2Yqzn1
vLBCwuEHkqY237lDcXY4/5LR+1gcZ3Duw1n/BRSm0FBdvyX/JTWiWNSDUkKAO/0l
T2Tg44YLrDT3bzwu8dbU9xQt6kH+SCOHvv5Oe4k79l5mro6fF3H1M0bN63x/YoFY
ojy09D7/JptY82oR4f/VdKnfZLJcCViCb0wp8SD2NkDAudKg+K+7PD8HlTWklQQg
TZdRXxVZKIJeU42aJDqnRbAhJd64YHyClhqut9F5LUmiP5qfLfNhkKDhNOwk2Blr
BGBJkSd7wPyzcX4Mun/L6YspHjbeVMt9TD7HQlo+OOd2OjAHCx6pqwkXnzeLPEaE
cPdQ1SHgrBViAxX3DNPubLP0Knw8XwFu96EuhHZgexE1W7bB4LFsJyXAc5k1PqPD
CLsAauxmvI2OfI7opG/8wyxDvNgoPjG8fZNAgY0REqPC0JnTXChH31IxUmhNotH8
tD3DDTZOHw05n5MwwUrEE9xiETVDfFQcMLfxZ9KLz+BC2g1t5LYublRgnCMNJzFg
sNUMM02CphABzl/LCLnumr0eyQQ/weV4twEhLwSDmqLYHL0EdYW0Y3CnnU9vmYxQ
cXKbstS71sEJJYBBmSBbf9GxkOY8BRNtwVwY0kPgxv1WqdVBiAFvfB+pyAsrax9B
3UeB7ZSwRD6JAhwEEAEKAAYFAlS25GwACgkQlbYYGy0z6ew92Q//ZA9/6piQtoW4
PwP/1DtWGyKU8hwR+9FG669iPk/dAG+yoEJtFMOUpg/FUFmCX8Bc4oEHsCVyLxKt
DcCVUIRcYNSFi5hTZaBEbwsOlDT37gtlfIIu34hhHRccKaLnN/N9gNMNw8wGh9xg
Q/KtxZwcbk/bZIlDkKTJkFBRAekdEGAFDWb/AZOy+LQxS8ZAh1eWkfV0i8opmK9k
gPXtLE0WSsqtYyGs58z+BFE9NH3tEUwK6jSvtuLwQl4UrICNbKthcpb8WwH6UXzb
q3QNSYVOpf/cqRdBJA6bvb/ku/xyKVL08lGmxD9v1b137R7mafDAFPTsvH2Mt/0V
YuhtWav3r1Bl9QksDxt2DTS8wiWDUBetGqOVdcw7vBrXPEWDNBmxeJXsiJ7zJlR+
9wrJOm6RV2+l1IPxu96EaPS+kTNBijKrhxb67bww8BTEWTd0wcdJmgWRkM8SIstp
IKqd0L2TFYph2/NtrBhRg+DIEPJPpSTGsUMcCEXCZPQ+cIdlQKsWpk0tZ62DlvEl
r7E+wgUSQolRfx5KrpZifiS2zQlhzdXv28CJhsVbLyw5fUAWUKIH/dCo5NKsNLk2
Lc5DH9VWnFgxAAtW290FqeK/4ulMq7Vs1dQSwyHM2Ni3QqqeaiOrh8gbSY5CMLFN
Y3HYRwuTYPa3AobsozCzBj0Zdf/6AFe5Ag0ETMQ5kgEQAL/FwKdjxgPxtSpgq1SM
zgZtTTyLqhgGD3NZfadHWHYRIL38NDV3JeTA79Y2zj2dj7KQPDT+0aqeizTV2E3j
P3iCQ53VOT4consBaQAgKexpptnS+T1DobtICFJ0GGzf0HRj6KO2zSOuOitWPWlU
wbvX7M0LLI2+hqlx0jTPqbJFZ/Za6KTtbS6xdCPVUpUqYZQpokEZcwQmUp8Q+lGo
JD2sNYCZyap63X/aAOgCGr2RXYddOH5e8vGzGW+mwtCv+WQ9Ay35mGqI5MqkbZd1
Qbuv2b1647E/QEEucfRHVbJVKGGPpFMUJtcItyyIt5jo+r9CCL4Cs47dF/9/RNwu
NvpvHXUyqMBQdWNZRMx4k/NGD/WviPi9m6mIMui6rOQsSOaqYdcUX4Nq2Orr3Oaz
2JPQdUfeI23iot1vK8hxvUCQTV3HfJghizN6spVl0yQOKBiE8miJRgrjHilH3hTb
xoo42xDkNAq+CQo3QAm1ibDxKCDq0RcWPjcCRAN/Q5MmpcodpdKkzV0yGIS4g7s5
frVrgV/kox2r4/Yxsr8K909+4H82AjTKGX/BmsQFCTAqBk6p7I0zxjIqJ/w33TZB
Q0Pn4r3WIlUPafzY6a9/LAvN1fHRxf9SpCByJsszD03Qu5f5TB8gthsdnVmTo7jj
iordEKMtw2aEMLzdWWTQ/TNVABEBAAGJAjwEGAEKACYCGwwWIQQ2kMJAzlG0Zw0w
rRw47nV9aRhGIAUCYEt9YAUJFxeRzgAKCRA47nV9aRhGIMLtD/9HuKM4pngImcuz
YwzQmdv4j26YYyh4jVsKEmVWTiRcehEgUIlrWkCu3qzd5NK+RetS7kJ8MPnzEUfj
YbpdC6yrF6n1mSrZZ4VJMkV2ev37bIgXM+Wp1mCAGbjNxQnjn9RabT/gjIqmGuRn
AP7RsSeOSuO/gO9h2Pteciz23ussTilB+8cTooQEQQZe6Kv/zukvL+ccSehLHsZ7
qVfRUAmtt8nFkXXE+s8jfLfhqstaI2/RJu5witaPcXM8Mnz2E95aASAbZy0eQot9
0Pvf07n9yuC3tueTvzvlXx3h5U3yT44tIOmzANIQjay1TGdm+RBJ2ZYyhyLawlZ2
NVUXXSp4QZZXPA0UWbF+pb7Q9cdKDNFVuvGBljuea0Yd0T2o+ibDq43HziX9ll+l
SXk9mqvW1UcDOaxWrSsm1Gc1O9g3wqH5xHAhtY8GPh/7VgAawskPkmnlkMW6pYPy
zibbeISJL1gd1jIT63y6aoVrtNoo+wYJm280ROflh4+5QOo6QJ+jm70fkXSG/qJ5
a8/qCPTHkJc/rpkL6/TDQAJURi9RhDAC0gb40HtusbN1LZEA+i0cWTmYXap+DB4Y
R4pApilpaG87M+VUokR4xpnx7vTb2MPa7Mdenvi9FEGnKXadmT8038vlfzz5GGUT
MlVin9BQPTpdA+PpRiJvKJgVDeAFOg==
=asTC
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -265,6 +265,11 @@ inline mgp_edge *graph_edge_set_to(struct mgp_graph *graph, struct mgp_edge *e,
return MgInvoke<mgp_edge *>(mgp_graph_edge_set_to, graph, e, new_to, memory); return MgInvoke<mgp_edge *>(mgp_graph_edge_set_to, graph, e, new_to, memory);
} }
inline mgp_edge *graph_edge_change_type(struct mgp_graph *graph, struct mgp_edge *e, struct mgp_edge_type new_type,
mgp_memory *memory) {
return MgInvoke<mgp_edge *>(mgp_graph_edge_change_type, graph, e, new_type, memory);
}
inline void graph_delete_edge(mgp_graph *graph, mgp_edge *edge) { MgInvokeVoid(mgp_graph_delete_edge, graph, edge); } inline void graph_delete_edge(mgp_graph *graph, mgp_edge *edge) { MgInvokeVoid(mgp_graph_delete_edge, graph, edge); }
inline mgp_vertex *graph_get_vertex_by_id(mgp_graph *g, mgp_vertex_id id, mgp_memory *memory) { inline mgp_vertex *graph_get_vertex_by_id(mgp_graph *g, mgp_vertex_id id, mgp_memory *memory) {
@@ -355,6 +360,8 @@ inline size_t map_size(mgp_map *map) { return MgInvoke<size_t>(mgp_map_size, map
inline mgp_value *map_at(mgp_map *map, const char *key) { return MgInvoke<mgp_value *>(mgp_map_at, map, key); } inline mgp_value *map_at(mgp_map *map, const char *key) { return MgInvoke<mgp_value *>(mgp_map_at, map, key); }
inline bool key_exists(mgp_map *map, const char *key) { return MgInvoke<int>(mgp_key_exists, map, key); }
inline const char *map_item_key(mgp_map_item *item) { return MgInvoke<const char *>(mgp_map_item_key, item); } inline const char *map_item_key(mgp_map_item *item) { return MgInvoke<const char *>(mgp_map_item_key, item); }
inline mgp_value *map_item_value(mgp_map_item *item) { return MgInvoke<mgp_value *>(mgp_map_item_value, item); } inline mgp_value *map_item_value(mgp_map_item *item) { return MgInvoke<mgp_value *>(mgp_map_item_value, item); }

View File

@@ -115,19 +115,6 @@ void mgp_global_free(void *p);
/// State of the graph database. /// State of the graph database.
struct mgp_graph; struct mgp_graph;
/// Allocations are tracked only for master thread. If new threads are spawned
/// inside procedure, by calling following function with thread id
/// you can start tracking allocations for that thread too. This
/// is important if you need query memory limit to work
/// for given procedure or per procedure memory limit.
enum mgp_error mgp_track_thread_allocations(struct mgp_graph *graph, const char *thread_id);
/// Once allocations are tracked for custom thread, you need to stop tracking allocations
/// for given thread, before thread finishes with execution, or is detached.
/// Otherwise it might result in slowdown of system due to unnecessary tracking of
/// allocations.
enum mgp_error mgp_untrack_thread_allocations(struct mgp_graph *graph, const char *thread_id);
/// Allocations are tracked only for master thread. If new threads are spawned /// Allocations are tracked only for master thread. If new threads are spawned
/// inside procedure, by calling following function /// inside procedure, by calling following function
/// you can start tracking allocations for current thread too. This /// you can start tracking allocations for current thread too. This
@@ -511,6 +498,9 @@ enum mgp_error mgp_map_size(struct mgp_map *map, size_t *result);
/// Result is NULL if no mapping exists. /// Result is NULL if no mapping exists.
enum mgp_error mgp_map_at(struct mgp_map *map, const char *key, struct mgp_value **result); enum mgp_error mgp_map_at(struct mgp_map *map, const char *key, struct mgp_value **result);
/// Returns true if key in map.
enum mgp_error mgp_key_exists(struct mgp_map *map, const char *key, int *result);
/// An item in the mgp_map. /// An item in the mgp_map.
struct mgp_map_item; struct mgp_map_item;
@@ -934,6 +924,13 @@ enum mgp_error mgp_graph_edge_set_from(struct mgp_graph *graph, struct mgp_edge
enum mgp_error mgp_graph_edge_set_to(struct mgp_graph *graph, struct mgp_edge *e, struct mgp_vertex *new_to, enum mgp_error mgp_graph_edge_set_to(struct mgp_graph *graph, struct mgp_edge *e, struct mgp_vertex *new_to,
struct mgp_memory *memory, struct mgp_edge **result); struct mgp_memory *memory, struct mgp_edge **result);
/// Change edge type
/// Return mgp_error::MGP_ERROR_IMMUTABLE_OBJECT if `graph` is immutable.
/// Return mgp_error::MGP_ERROR_SERIALIZATION_ERROR if `edge`, its source or destination vertex has been modified by
/// another transaction.
enum mgp_error mgp_graph_edge_change_type(struct mgp_graph *graph, struct mgp_edge *e, struct mgp_edge_type new_type,
struct mgp_memory *memory, struct mgp_edge **result);
/// Delete an edge from the graph. /// Delete an edge from the graph.
/// Return mgp_error::MGP_ERROR_IMMUTABLE_OBJECT if `graph` is immutable. /// Return mgp_error::MGP_ERROR_IMMUTABLE_OBJECT if `graph` is immutable.
/// Return mgp_error::MGP_ERROR_SERIALIZATION_ERROR if `edge`, its source or destination vertex has been modified by /// Return mgp_error::MGP_ERROR_SERIALIZATION_ERROR if `edge`, its source or destination vertex has been modified by

View File

@@ -260,6 +260,8 @@ class Graph {
void SetFrom(Relationship &relationship, const Node &new_from); void SetFrom(Relationship &relationship, const Node &new_from);
/// @brief Changes a relationship to node. /// @brief Changes a relationship to node.
void SetTo(Relationship &relationship, const Node &new_to); void SetTo(Relationship &relationship, const Node &new_to);
/// @brief Changes the relationship type.
void ChangeType(Relationship &relationship, std::string_view new_type);
/// @brief Deletes a relationship from the graph. /// @brief Deletes a relationship from the graph.
void DeleteRelationship(const Relationship &relationship); void DeleteRelationship(const Relationship &relationship);
@@ -630,6 +632,9 @@ class Map {
/// @brief Returns the value at the given `key`. /// @brief Returns the value at the given `key`.
Value const At(std::string_view key) const; Value const At(std::string_view key) const;
/// @brief Returns true if the given `key` exists.
bool KeyExists(std::string_view key) const;
class Iterator { class Iterator {
public: public:
friend class Map; friend class Map;
@@ -2027,6 +2032,13 @@ inline void Graph::SetTo(Relationship &relationship, const Node &new_to) {
mgp::edge_destroy(edge); mgp::edge_destroy(edge);
} }
inline void Graph::ChangeType(Relationship &relationship, std::string_view new_type) {
mgp_edge *edge = mgp::MemHandlerCallback(mgp::graph_edge_change_type, graph_, relationship.ptr_,
mgp_edge_type{.name = new_type.data()});
relationship = Relationship(edge);
mgp::edge_destroy(edge);
}
inline void Graph::DeleteRelationship(const Relationship &relationship) { inline void Graph::DeleteRelationship(const Relationship &relationship) {
mgp::graph_delete_edge(graph_, relationship.ptr_); mgp::graph_delete_edge(graph_, relationship.ptr_);
} }
@@ -2573,6 +2585,8 @@ inline const Value Map::At(std::string_view key) const {
return Value(); return Value();
} }
inline bool Map::KeyExists(std::string_view key) const { return mgp::key_exists(ptr_, key.data()); }
inline Map::Iterator::Iterator(mgp_map_items_iterator *map_items_iterator) : map_items_iterator_(map_items_iterator) { inline Map::Iterator::Iterator(mgp_map_items_iterator *map_items_iterator) : map_items_iterator_(map_items_iterator) {
if (map_items_iterator_ == nullptr) return; if (map_items_iterator_ == nullptr) return;
if (mgp::map_items_iterator_get(map_items_iterator_) == nullptr) { if (mgp::map_items_iterator_get(map_items_iterator_) == nullptr) {

View File

@@ -4,7 +4,8 @@ include(GNUInstallDirs)
include(ProcessorCount) include(ProcessorCount)
ProcessorCount(NPROC) ProcessorCount(NPROC)
if (NPROC EQUAL 0)
if(NPROC EQUAL 0)
set(NPROC 1) set(NPROC 1)
endif() endif()
@@ -12,6 +13,7 @@ find_package(Boost 1.78 REQUIRED)
find_package(BZip2 1.0.6 REQUIRED) find_package(BZip2 1.0.6 REQUIRED)
find_package(Threads REQUIRED) find_package(Threads REQUIRED)
set(GFLAGS_NOTHREADS OFF) set(GFLAGS_NOTHREADS OFF)
# NOTE: config/generate.py depends on the gflags help XML format. # NOTE: config/generate.py depends on the gflags help XML format.
find_package(gflags REQUIRED) find_package(gflags REQUIRED)
find_package(fmt 8.0.1) find_package(fmt 8.0.1)
@@ -23,24 +25,27 @@ set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
function(import_header_library name include_dir) function(import_header_library name include_dir)
add_library(${name} INTERFACE IMPORTED GLOBAL) add_library(${name} INTERFACE IMPORTED GLOBAL)
set_property(TARGET ${name} PROPERTY set_property(TARGET ${name} PROPERTY
INTERFACE_INCLUDE_DIRECTORIES ${include_dir}) INTERFACE_INCLUDE_DIRECTORIES ${include_dir})
string(TOUPPER ${name} _upper_name) string(TOUPPER ${name} _upper_name)
set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
"Path to ${name} include directory" FORCE) "Path to ${name} include directory" FORCE)
mark_as_advanced(${_upper_name}_INCLUDE_DIR) mark_as_advanced(${_upper_name}_INCLUDE_DIR)
add_library(lib::${name} ALIAS ${name}) add_library(lib::${name} ALIAS ${name})
endfunction(import_header_library) endfunction(import_header_library)
function(import_library name type location include_dir) function(import_library name type location include_dir)
add_library(${name} ${type} IMPORTED GLOBAL) add_library(${name} ${type} IMPORTED GLOBAL)
if (${ARGN})
if(${ARGN})
# Optional argument is the name of the external project that we need to # Optional argument is the name of the external project that we need to
# depend on. # depend on.
add_dependencies(${name} ${ARGN0}) add_dependencies(${name} ${ARGN0})
else() else()
add_dependencies(${name} ${name}-proj) add_dependencies(${name} ${name}-proj)
endif() endif()
set_property(TARGET ${name} PROPERTY IMPORTED_LOCATION ${location}) set_property(TARGET ${name} PROPERTY IMPORTED_LOCATION ${location})
# We need to create the include directory first in order to be able to add it # We need to create the include directory first in order to be able to add it
# as an include directory. The header files in the include directory will be # as an include directory. The header files in the include directory will be
# generated later during the build process. # generated later during the build process.
@@ -60,29 +65,34 @@ function(add_external_project name)
set(options NO_C_COMPILER) set(options NO_C_COMPILER)
set(one_value_kwargs SOURCE_DIR BUILD_IN_SOURCE) set(one_value_kwargs SOURCE_DIR BUILD_IN_SOURCE)
set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND set(multi_value_kwargs CMAKE_ARGS DEPENDS INSTALL_COMMAND BUILD_COMMAND
CONFIGURE_COMMAND) CONFIGURE_COMMAND)
cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN}) cmake_parse_arguments(KW "${options}" "${one_value_kwargs}" "${multi_value_kwargs}" ${ARGN})
set(source_dir ${CMAKE_CURRENT_SOURCE_DIR}/${name}) set(source_dir ${CMAKE_CURRENT_SOURCE_DIR}/${name})
if (KW_SOURCE_DIR)
if(KW_SOURCE_DIR)
set(source_dir ${KW_SOURCE_DIR}) set(source_dir ${KW_SOURCE_DIR})
endif() endif()
set(build_in_source 0) set(build_in_source 0)
if (KW_BUILD_IN_SOURCE)
if(KW_BUILD_IN_SOURCE)
set(build_in_source ${KW_BUILD_IN_SOURCE}) set(build_in_source ${KW_BUILD_IN_SOURCE})
endif() endif()
if (NOT KW_NO_C_COMPILER)
if(NOT KW_NO_C_COMPILER)
set(KW_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} ${KW_CMAKE_ARGS}) set(KW_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} ${KW_CMAKE_ARGS})
endif() endif()
ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS} ExternalProject_Add(${name}-proj DEPENDS ${KW_DEPENDS}
PREFIX ${source_dir} SOURCE_DIR ${source_dir} PREFIX ${source_dir} SOURCE_DIR ${source_dir}
BUILD_IN_SOURCE ${build_in_source} BUILD_IN_SOURCE ${build_in_source}
CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND} CONFIGURE_COMMAND ${KW_CONFIGURE_COMMAND}
CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_INSTALL_PREFIX=${source_dir} -DCMAKE_INSTALL_PREFIX=${source_dir}
${KW_CMAKE_ARGS} ${KW_CMAKE_ARGS}
INSTALL_COMMAND ${KW_INSTALL_COMMAND} INSTALL_COMMAND ${KW_INSTALL_COMMAND}
BUILD_COMMAND ${KW_BUILD_COMMAND}) BUILD_COMMAND ${KW_BUILD_COMMAND})
endfunction(add_external_project) endfunction(add_external_project)
# Calls `add_external_project`, sets NAME_LIBRARY, NAME_INCLUDE_DIR variables # Calls `add_external_project`, sets NAME_LIBRARY, NAME_INCLUDE_DIR variables
@@ -91,9 +101,9 @@ macro(import_external_library name type library_location include_dir)
add_external_project(${name} ${ARGN}) add_external_project(${name} ${ARGN})
string(TOUPPER ${name} _upper_name) string(TOUPPER ${name} _upper_name)
set(${_upper_name}_LIBRARY ${library_location} CACHE FILEPATH set(${_upper_name}_LIBRARY ${library_location} CACHE FILEPATH
"Path to ${name} library" FORCE) "Path to ${name} library" FORCE)
set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
"Path to ${name} include directory" FORCE) "Path to ${name} include directory" FORCE)
mark_as_advanced(${_upper_name}_LIBRARY ${_upper_name}_INCLUDE_DIR) mark_as_advanced(${_upper_name}_LIBRARY ${_upper_name}_INCLUDE_DIR)
import_library(${name} ${type} ${${_upper_name}_LIBRARY} ${${_upper_name}_INCLUDE_DIR}) import_library(${name} ${type} ${${_upper_name}_LIBRARY} ${${_upper_name}_INCLUDE_DIR})
endmacro(import_external_library) endmacro(import_external_library)
@@ -115,10 +125,10 @@ import_external_library(antlr4 STATIC
${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/include/antlr4-runtime ${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/include/antlr4-runtime
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp
CMAKE_ARGS # http://stackoverflow.com/questions/37096062/get-a-basic-c-program-to-compile-using-clang-on-ubuntu-16/38385967#38385967 CMAKE_ARGS # http://stackoverflow.com/questions/37096062/get-a-basic-c-program-to-compile-using-clang-on-ubuntu-16/38385967#38385967
-DWITH_LIBCXX=OFF # because of debian bug -DWITH_LIBCXX=OFF # because of debian bug
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
-DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD=20
-DANTLR_BUILD_CPP_TESTS=OFF -DANTLR_BUILD_CPP_TESTS=OFF
BUILD_COMMAND $(MAKE) antlr4_static BUILD_COMMAND $(MAKE) antlr4_static
INSTALL_COMMAND $(MAKE) install) INSTALL_COMMAND $(MAKE) install)
@@ -126,6 +136,7 @@ import_external_library(antlr4 STATIC
import_external_library(benchmark STATIC import_external_library(benchmark STATIC
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/${CMAKE_INSTALL_LIBDIR}/libbenchmark.a ${CMAKE_CURRENT_SOURCE_DIR}/benchmark/${CMAKE_INSTALL_LIBDIR}/libbenchmark.a
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/include ${CMAKE_CURRENT_SOURCE_DIR}/benchmark/include
# Skip testing. The tests don't compile with Clang 8. # Skip testing. The tests don't compile with Clang 8.
CMAKE_ARGS -DBENCHMARK_ENABLE_TESTING=OFF) CMAKE_ARGS -DBENCHMARK_ENABLE_TESTING=OFF)
@@ -141,15 +152,15 @@ add_subdirectory(rapidcheck EXCLUDE_FROM_ALL)
# setup google test # setup google test
add_external_project(gtest SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest) add_external_project(gtest SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest)
set(GTEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest/include set(GTEST_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/googletest/include
CACHE PATH "Path to gtest and gmock include directory" FORCE) CACHE PATH "Path to gtest and gmock include directory" FORCE)
set(GMOCK_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgmock.a set(GMOCK_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgmock.a
CACHE FILEPATH "Path to gmock library" FORCE) CACHE FILEPATH "Path to gmock library" FORCE)
set(GMOCK_MAIN_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgmock_main.a set(GMOCK_MAIN_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgmock_main.a
CACHE FILEPATH "Path to gmock_main library" FORCE) CACHE FILEPATH "Path to gmock_main library" FORCE)
set(GTEST_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgtest.a set(GTEST_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgtest.a
CACHE FILEPATH "Path to gtest library" FORCE) CACHE FILEPATH "Path to gtest library" FORCE)
set(GTEST_MAIN_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgtest_main.a set(GTEST_MAIN_LIBRARY ${CMAKE_CURRENT_SOURCE_DIR}/googletest/lib/libgtest_main.a
CACHE FILEPATH "Path to gtest_main library" FORCE) CACHE FILEPATH "Path to gtest_main library" FORCE)
mark_as_advanced(GTEST_INCLUDE_DIR GMOCK_LIBRARY GMOCK_MAIN_LIBRARY GTEST_LIBRARY GTEST_MAIN_LIBRARY) mark_as_advanced(GTEST_INCLUDE_DIR GMOCK_LIBRARY GMOCK_MAIN_LIBRARY GTEST_LIBRARY GTEST_MAIN_LIBRARY)
import_library(gtest STATIC ${GTEST_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj) import_library(gtest STATIC ${GTEST_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj)
import_library(gtest_main STATIC ${GTEST_MAIN_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj) import_library(gtest_main STATIC ${GTEST_MAIN_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj)
@@ -167,10 +178,10 @@ import_external_library(rocksdb STATIC
${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/lib/librocksdb.a ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/lib/librocksdb.a
${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include
CMAKE_ARGS -DUSE_RTTI=ON CMAKE_ARGS -DUSE_RTTI=ON
-DWITH_TESTS=OFF -DWITH_TESTS=OFF
-DGFLAGS_NOTHREADS=OFF -DGFLAGS_NOTHREADS=OFF
-DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
BUILD_COMMAND $(MAKE) rocksdb) BUILD_COMMAND $(MAKE) rocksdb)
# Setup libbcrypt # Setup libbcrypt
@@ -179,8 +190,8 @@ import_external_library(libbcrypt STATIC
${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt
CONFIGURE_COMMAND sed s/-Wcast-align// -i ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt/crypt_blowfish/Makefile CONFIGURE_COMMAND sed s/-Wcast-align// -i ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt/crypt_blowfish/Makefile
BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/libbcrypt
CC=${CMAKE_C_COMPILER} CC=${CMAKE_C_COMPILER}
CXX=${CMAKE_CXX_COMPILER} CXX=${CMAKE_CXX_COMPILER}
INSTALL_COMMAND true) INSTALL_COMMAND true)
# Setup mgclient # Setup mgclient
@@ -188,16 +199,16 @@ import_external_library(mgclient STATIC
${CMAKE_CURRENT_SOURCE_DIR}/mgclient/lib/libmgclient.a ${CMAKE_CURRENT_SOURCE_DIR}/mgclient/lib/libmgclient.a
${CMAKE_CURRENT_SOURCE_DIR}/mgclient/include ${CMAKE_CURRENT_SOURCE_DIR}/mgclient/include
CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DBUILD_TESTING=OFF -DBUILD_TESTING=OFF
-DBUILD_CPP_BINDINGS=ON) -DBUILD_CPP_BINDINGS=ON)
find_package(OpenSSL REQUIRED) find_package(OpenSSL REQUIRED)
target_link_libraries(mgclient INTERFACE ${OPENSSL_LIBRARIES}) target_link_libraries(mgclient INTERFACE ${OPENSSL_LIBRARIES})
add_external_project(mgconsole add_external_project(mgconsole
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/mgconsole SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/mgconsole
CMAKE_ARGS CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR} -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_BINARY_DIR}
BUILD_COMMAND $(MAKE) mgconsole) BUILD_COMMAND $(MAKE) mgconsole)
add_custom_target(mgconsole DEPENDS mgconsole-proj) add_custom_target(mgconsole DEPENDS mgconsole-proj)
@@ -214,14 +225,15 @@ import_external_library(librdkafka STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka.a ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka.a
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/include ${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/include
CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON
-DRDKAFKA_BUILD_EXAMPLES=OFF -DRDKAFKA_BUILD_EXAMPLES=OFF
-DRDKAFKA_BUILD_TESTS=OFF -DRDKAFKA_BUILD_TESTS=OFF
-DWITH_ZSTD=OFF -DWITH_ZSTD=OFF
-DENABLE_LZ4_EXT=OFF -DENABLE_LZ4_EXT=OFF
-DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_LIBDIR=lib
-DWITH_SSL=ON -DWITH_SSL=ON
# If we want SASL, we need to install it on build machines
-DWITH_SASL=OFF) # If we want SASL, we need to install it on build machines
-DWITH_SASL=OFF)
target_link_libraries(librdkafka INTERFACE ${OPENSSL_LIBRARIES} ZLIB::ZLIB) target_link_libraries(librdkafka INTERFACE ${OPENSSL_LIBRARIES} ZLIB::ZLIB)
import_library(librdkafka++ STATIC import_library(librdkafka++ STATIC
@@ -242,24 +254,24 @@ import_external_library(pulsar STATIC
${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install/include ${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install/include
BUILD_IN_SOURCE 1 BUILD_IN_SOURCE 1
CONFIGURE_COMMAND cmake pulsar-client-cpp CONFIGURE_COMMAND cmake pulsar-client-cpp
-DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DBUILD_DYNAMIC_LIB=OFF -DBUILD_DYNAMIC_LIB=OFF
-DBUILD_STATIC_LIB=ON -DBUILD_STATIC_LIB=ON
-DBUILD_TESTS=OFF -DBUILD_TESTS=OFF
-DLINK_STATIC=ON -DLINK_STATIC=ON
-DPROTOC_PATH=${PROTOBUF_ROOT}/bin/protoc -DPROTOC_PATH=${PROTOBUF_ROOT}/bin/protoc
-DBOOST_ROOT=${BOOST_ROOT} -DBOOST_ROOT=${BOOST_ROOT}
-DCMAKE_PREFIX_PATH=${PROTOBUF_ROOT} -DCMAKE_PREFIX_PATH=${PROTOBUF_ROOT}
-DProtobuf_INCLUDE_DIRS=${PROTOBUF_ROOT}/include -DProtobuf_INCLUDE_DIRS=${PROTOBUF_ROOT}/include
-DBUILD_PYTHON_WRAPPER=OFF -DBUILD_PYTHON_WRAPPER=OFF
-DBUILD_PERF_TOOLS=OFF -DBUILD_PERF_TOOLS=OFF
-DUSE_LOG4CXX=OFF -DUSE_LOG4CXX=OFF
BUILD_COMMAND $(MAKE) pulsarStaticWithDeps) BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
add_dependencies(pulsar-proj protobuf) add_dependencies(pulsar-proj protobuf)
if (${MG_ARCH} STREQUAL "ARM64") if(${MG_ARCH} STREQUAL "ARM64")
set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON) set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON)
endif() endif()
@@ -280,3 +292,5 @@ add_subdirectory(absl EXCLUDE_FROM_ALL)
set_path_external_library(jemalloc STATIC set_path_external_library(jemalloc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/lib/libjemalloc.a ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/lib/libjemalloc.a
${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/include/) ${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/include/)
import_header_library(rangev3 ${CMAKE_CURRENT_SOURCE_DIR}/rangev3/include)

View File

@@ -125,6 +125,7 @@ declare -A primary_urls=(
["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp" ["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
["absl"]="https://$local_cache_host/git/abseil-cpp.git" ["absl"]="https://$local_cache_host/git/abseil-cpp.git"
["jemalloc"]="https://$local_cache_host/git/jemalloc.git" ["jemalloc"]="https://$local_cache_host/git/jemalloc.git"
["range-v3"]="https://$local_cache_host/git/ericniebler/range-v3.git"
) )
# The goal of secondary urls is to have links to the "source of truth" of # The goal of secondary urls is to have links to the "source of truth" of
@@ -153,6 +154,7 @@ declare -A secondary_urls=(
["ctre"]="https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp" ["ctre"]="https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
["absl"]="https://github.com/abseil/abseil-cpp.git" ["absl"]="https://github.com/abseil/abseil-cpp.git"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git" ["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["range-v3"]="https://github.com/ericniebler/range-v3.git"
) )
# antlr # antlr
@@ -255,7 +257,6 @@ cd ..
absl_ref="20230125.3" absl_ref="20230125.3"
repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" "$absl_ref" repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" "$absl_ref"
# jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756 # jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756
JEMALLOC_COMMIT_VERSION="5.2.1" JEMALLOC_COMMIT_VERSION="5.2.1"
repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION" repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
@@ -272,3 +273,7 @@ MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_m
make -j$CPUS install make -j$CPUS install
popd popd
#range-v3 release-0.12.0
range_v3_ref="release-0.12.0"
repo_clone_try_double "${primary_urls[range-v3]}" "${secondary_urls[range-v3]}" "rangev3" "$range_v3_ref"

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution 3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to which competes (or might reasonably be expected to
compete) with the Licensed Work. compete) with the Licensed Work.
CHANGE DATE: 2027-13-09 CHANGE DATE: 2027-30-10
CHANGE LICENSE: Apache License, Version 2.0 CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal. For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -13,6 +13,7 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
add_library(example_c SHARED example.c) add_library(example_c SHARED example.c)
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include) target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_c PRIVATE -Wall) target_compile_options(example_c PRIVATE -Wall)
target_link_libraries(example_c PRIVATE -static-libgcc -static-libstdc++)
# Strip C example in release build. # Strip C example in release build.
if (lower_build_type STREQUAL "release") if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_c POST_BUILD add_custom_command(TARGET example_c POST_BUILD
@@ -28,6 +29,7 @@ install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
add_library(example_cpp SHARED example.cpp) add_library(example_cpp SHARED example.cpp)
target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include) target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_cpp PRIVATE -Wall) target_compile_options(example_cpp PRIVATE -Wall)
target_link_libraries(example_cpp PRIVATE -static-libgcc -static-libstdc++)
# Strip C++ example in release build. # Strip C++ example in release build.
if (lower_build_type STREQUAL "release") if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_cpp POST_BUILD add_custom_command(TARGET example_cpp POST_BUILD
@@ -40,9 +42,27 @@ install(PROGRAMS $<TARGET_FILE:example_cpp>
# Also install the source of the example, so user can read it. # Also install the source of the example, so user can read it.
install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src) install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
add_library(schema SHARED schema.cpp)
target_include_directories(schema PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(schema PRIVATE -Wall)
target_link_libraries(schema PRIVATE -static-libgcc -static-libstdc++)
# Strip C++ example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET schema POST_BUILD
COMMAND strip -s $<TARGET_FILE:schema>
COMMENT "Stripping symbols and sections from the C++ schema module")
endif()
install(PROGRAMS $<TARGET_FILE:schema>
DESTINATION lib/memgraph/query_modules
RENAME schema.so)
# Also install the source of the example, so user can read it.
install(FILES schema.cpp DESTINATION lib/memgraph/query_modules/src)
# Install the Python example and modules # Install the Python example and modules
install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py) install(FILES example.py DESTINATION lib/memgraph/query_modules RENAME py_example.py)
install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules) install(FILES graph_analyzer.py DESTINATION lib/memgraph/query_modules)
install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules) install(FILES mgp_networkx.py DESTINATION lib/memgraph/query_modules)
install(FILES nxalg.py DESTINATION lib/memgraph/query_modules) install(FILES nxalg.py DESTINATION lib/memgraph/query_modules)
install(FILES wcc.py DESTINATION lib/memgraph/query_modules) install(FILES wcc.py DESTINATION lib/memgraph/query_modules)
install(FILES mgps.py DESTINATION lib/memgraph/query_modules)
install(FILES convert.py DESTINATION lib/memgraph/query_modules)

10
query_modules/convert.py Normal file
View File

@@ -0,0 +1,10 @@
from json import loads
import mgp
@mgp.function
def str2object(string: str) -> mgp.Any:
if string:
return loads(string)
return None

8
query_modules/mgps.py Normal file
View File

@@ -0,0 +1,8 @@
import mgp
@mgp.read_proc
def components(
context: mgp.ProcCtx,
) -> mgp.Record(versions=list, edition=str, name=str):
return mgp.Record(versions=["5.9.0"], edition="community", name="Memgraph")

187
query_modules/schema.cpp Normal file
View File

@@ -0,0 +1,187 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <mgp.hpp>
namespace Schema {
/*NodeTypeProperties and RelTypeProperties constants*/
constexpr std::string_view kReturnNodeType = "nodeType";
constexpr std::string_view kProcedureNodeType = "node_type_properties";
constexpr std::string_view kProcedureRelType = "rel_type_properties";
constexpr std::string_view kReturnLabels = "nodeLabels";
constexpr std::string_view kReturnRelType = "relType";
constexpr std::string_view kReturnPropertyName = "propertyName";
constexpr std::string_view kReturnPropertyType = "propertyTypes";
constexpr std::string_view kReturnMandatory = "mandatory";
std::string TypeOf(const mgp::Type &type);
template <typename T>
void ProcessPropertiesNode(mgp::Record &record, const std::string &type, const mgp::List &labels,
const std::string &propertyName, const T &propertyType, const bool &mandatory);
template <typename T>
void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, const std::string &propertyName,
const T &propertyType, const bool &mandatory);
void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
} // namespace Schema
/*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this
module, so I implemented a small function here*/
std::string Schema::TypeOf(const mgp::Type &type) {
switch (type) {
case mgp::Type::Null:
return "Null";
case mgp::Type::Bool:
return "Bool";
case mgp::Type::Int:
return "Int";
case mgp::Type::Double:
return "Double";
case mgp::Type::String:
return "String";
case mgp::Type::List:
return "List[Any]";
case mgp::Type::Map:
return "Map[Any]";
case mgp::Type::Node:
return "Vertex";
case mgp::Type::Relationship:
return "Edge";
case mgp::Type::Path:
return "Path";
case mgp::Type::Date:
return "Date";
case mgp::Type::LocalTime:
return "LocalTime";
case mgp::Type::LocalDateTime:
return "LocalDateTime";
case mgp::Type::Duration:
return "Duration";
default:
throw mgp::ValueException("Unsupported type");
}
}
template <typename T>
void Schema::ProcessPropertiesNode(mgp::Record &record, const std::string &type, const mgp::List &labels,
const std::string &propertyName, const T &propertyType, const bool &mandatory) {
record.Insert(std::string(kReturnNodeType).c_str(), type);
record.Insert(std::string(kReturnLabels).c_str(), labels);
record.Insert(std::string(kReturnPropertyName).c_str(), propertyName);
record.Insert(std::string(kReturnPropertyType).c_str(), propertyType);
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
}
template <typename T>
void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, const std::string &propertyName,
const T &propertyType, const bool &mandatory) {
record.Insert(std::string(kReturnRelType).c_str(), type);
record.Insert(std::string(kReturnPropertyName).c_str(), propertyName);
record.Insert(std::string(kReturnPropertyType).c_str(), propertyType);
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
}
void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
;
const auto record_factory = mgp::RecordFactory(result);
try {
const mgp::Graph graph = mgp::Graph(memgraph_graph);
for (auto node : graph.Nodes()) {
std::string type = "";
mgp::List labels = mgp::List();
for (auto label : node.Labels()) {
labels.AppendExtend(mgp::Value(label));
type += ":`" + std::string(label) + "`";
}
if (node.Properties().size() == 0) {
auto record = record_factory.NewRecord();
ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
continue;
}
for (auto &[key, prop] : node.Properties()) {
auto property_type = mgp::List();
auto record = record_factory.NewRecord();
property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
ProcessPropertiesNode<mgp::List>(record, type, labels, key, property_type, true);
}
}
} catch (const std::exception &e) {
record_factory.SetErrorMessage(e.what());
return;
}
}
void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
;
const auto record_factory = mgp::RecordFactory(result);
try {
const mgp::Graph graph = mgp::Graph(memgraph_graph);
for (auto rel : graph.Relationships()) {
std::string type = ":`" + std::string(rel.Type()) + "`";
if (rel.Properties().size() == 0) {
auto record = record_factory.NewRecord();
ProcessPropertiesRel<std::string>(record, type, "", "", false);
continue;
}
for (auto &[key, prop] : rel.Properties()) {
auto property_type = mgp::List();
auto record = record_factory.NewRecord();
property_type.AppendExtend(mgp::Value(TypeOf(prop.Type())));
ProcessPropertiesRel<mgp::List>(record, type, key, property_type, true);
}
}
} catch (const std::exception &e) {
record_factory.SetErrorMessage(e.what());
return;
}
}
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
try {
mgp::MemoryDispatcherGuard guard{memory};
;
AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read,
{},
{mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}),
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
module, memory);
AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read,
{},
{mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
module, memory);
} catch (const std::exception &e) {
return 1;
}
return 0;
}
extern "C" int mgp_shutdown_module() { return 0; }

View File

@@ -1,12 +1,27 @@
[Unit] [Unit]
Description=Memgraph: High performance, in-memory, transactional graph database Description=Memgraph: High performance, in-memory, transactional graph database
# Auto-restart config
#StartLimitIntervalSec=300
#StartLimitBurst=5
[Service] [Service]
User=memgraph User=memgraph
Group=memgraph Group=memgraph
ExecStart=/usr/lib/memgraph/memgraph ExecStart=/usr/lib/memgraph/memgraph
# Uncomment this if Memgraph needs more time to write the snapshot on exit. # Uncomment this if Memgraph needs more time to write the snapshot on exit.
#TimeoutStopSec=5min #TimeoutStopSec=5min
# Auto-restart config
#Restart=on-failure
#RestartSec=10s
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
# Checks processes' memory usage and force kills it when it is taking up enough
# memory to make the system unstable / about to crash, e.g. on Debian-based OS:
# apt install systemd-oomd
# systemctl enable systemd-oomd
# systemctl start systemd-oomd

View File

@@ -11,6 +11,12 @@ SUPPORTED_OS=(
amzn-2 amzn-2
) )
SUPPORTED_BUILD_TYPES=(
Debug
Release
RelWithDebInfo
)
PROJECT_ROOT="$SCRIPT_DIR/../.." PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v4" TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate" ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
@@ -18,14 +24,16 @@ HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () { print_help () {
# TODO(gitbuda): Update the release/package/run.sh help # TODO(gitbuda): Update the release/package/run.sh help
echo "$0 init|package|docker|test {os} [--for-docker|--for-platform]" echo "$0 init|package|docker|test {os} {build_type} [--for-docker|--for-platform]"
echo "" echo ""
echo " OSs: ${SUPPORTED_OS[*]}" echo " OSs: ${SUPPORTED_OS[*]}"
echo " Build types: ${SUPPORTED_BUILD_TYPES[*]}"
exit 1 exit 1
} }
make_package () { make_package () {
os="$1" os="$1"
build_type="$2"
build_container="mgbuild_$os" build_container="mgbuild_$os"
echo "Building Memgraph for $os on $build_container..." echo "Building Memgraph for $os on $build_container..."
@@ -44,10 +52,10 @@ make_package () {
package_command=" cpack -G DEB --config ../CPackConfig.cmake " package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi fi
telemetry_id_override_flag="" telemetry_id_override_flag=""
if [[ "$#" -gt 1 ]]; then if [[ "$#" -gt 2 ]]; then
if [[ "$2" == "--for-docker" ]]; then if [[ "$3" == "--for-docker" ]]; then
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER " telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER "
elif [[ "$2" == "--for-platform" ]]; then elif [[ "$3" == "--for-platform" ]]; then
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM" telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM"
else else
print_help print_help
@@ -64,6 +72,10 @@ make_package () {
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
git fetch origin master:master git fetch origin master:master
fi fi
# Ensure we have a clean build directory
docker exec "$build_container" rm -rf /memgraph
docker exec "$build_container" mkdir -p /memgraph docker exec "$build_container" mkdir -p /memgraph
# TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI. # TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI.
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/" docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
@@ -89,9 +101,9 @@ make_package () {
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*" docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
# TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX # TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX
if [[ "$os" =~ "-arm" ]]; then if [[ "$os" =~ "-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .." docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=$build_type -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else else
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .." docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=$build_type $telemetry_id_override_flag .."
fi fi
# ' is used instead of " because we need to run make within the allowed # ' is used instead of " because we need to run make within the allowed
# container resources. # container resources.
@@ -141,20 +153,34 @@ case "$1" in
package) package)
shift 1 shift 1
if [[ "$#" -lt 1 ]]; then if [[ "$#" -lt 2 ]]; then
print_help print_help
fi fi
os="$1" os="$1"
shift 1 build_type="$2"
shift 2
is_os_ok=false is_os_ok=false
for supported_os in "${SUPPORTED_OS[@]}"; do for supported_os in "${SUPPORTED_OS[@]}"; do
if [[ "$supported_os" == "${os}" ]]; then if [[ "$supported_os" == "${os}" ]]; then
is_os_ok=true is_os_ok=true
break
fi fi
done done
if [[ "$is_os_ok" == true ]]; then is_build_type_ok=false
make_package "$os" "$@" for supported_build_type in "${SUPPORTED_BUILD_TYPES[@]}"; do
if [[ "$supported_build_type" == "${build_type}" ]]; then
is_build_type_ok=true
break
fi
done
if [[ "$is_os_ok" == true && "$is_build_type_ok" == true ]]; then
make_package "$os" "$build_type" "$@"
else else
if [[ "$is_os_ok" == false ]]; then
echo "Unsupported OS: $os"
elif [[ "$is_build_type_ok" == false ]]; then
echo "Unsupported build type: $build_type"
fi
print_help print_help
fi fi
;; ;;

View File

@@ -21,6 +21,7 @@ add_subdirectory(audit)
add_subdirectory(dbms) add_subdirectory(dbms)
add_subdirectory(flags) add_subdirectory(flags)
add_subdirectory(distributed) add_subdirectory(distributed)
add_subdirectory(replication)
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type) string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -40,7 +41,7 @@ set(mg_single_node_v2_sources
add_executable(memgraph ${mg_single_node_v2_sources}) add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include) target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph stdc++fs Threads::Threads target_link_libraries(memgraph stdc++fs Threads::Threads
mg-telemetry mg-communication mg-memory mg-utils mg-license mg-settings mg-glue mg-flags) mg-telemetry mg-communication mg-communication-metrics mg-memory mg-utils mg-license mg-settings mg-glue mg-flags)
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which # NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the # should be dynamically exported, so that `dlopen` can correctly link the
@@ -142,7 +143,7 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
# Memgraph CSV Import Tool Executable # Memgraph CSV Import Tool Executable
# ---------------------------------------------------------------------------- # ----------------------------------------------------------------------------
add_executable(mg_import_csv mg_import_csv.cpp) add_executable(mg_import_csv mg_import_csv.cpp)
target_link_libraries(mg_import_csv mg-storage-v2) target_link_libraries(mg_import_csv mg-storage-v2 mg-dbms)
# Strip the executable in release build. # Strip the executable in release build.
if(lower_build_type STREQUAL "release") if(lower_build_type STREQUAL "release")

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,5 +21,6 @@ namespace memgraph::auth {
class AuthException : public utils::BasicException { class AuthException : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(AuthException)
}; };
} // namespace memgraph::auth } // namespace memgraph::auth

View File

@@ -16,10 +16,12 @@ set(communication_src_files
find_package(Boost REQUIRED) find_package(Boost REQUIRED)
add_library(mg-communication-metrics STATIC metrics.cpp)
target_link_libraries(mg-communication-metrics json)
add_library(mg-communication STATIC ${communication_src_files}) add_library(mg-communication STATIC ${communication_src_files})
target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io mg-auth fmt::fmt gflags) target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io mg-auth fmt::fmt gflags mg-communication-metrics mg-events)
find_package(OpenSSL REQUIRED) find_package(OpenSSL REQUIRED)
target_link_libraries(mg-communication ${OPENSSL_LIBRARIES}) target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})
target_include_directories(mg-communication SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) target_include_directories(mg-communication SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
target_precompile_headers(mg-communication INTERFACE http/server.hpp <boost/beast/websocket.hpp> bolt/v1/session.hpp)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -264,4 +264,5 @@ bool Client::ReadMessageData(Marker marker, Value &ret) {
} }
return false; return false;
} }
} // namespace memgraph::communication::bolt } // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -38,6 +38,7 @@ class FailureResponseException : public utils::BasicException {
: utils::BasicException{message}, code_{code} {} : utils::BasicException{message}, code_{code} {}
const std::string &code() const { return code_; } const std::string &code() const { return code_; }
SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException)
private: private:
std::string code_; std::string code_;
@@ -49,6 +50,7 @@ class FailureResponseException : public utils::BasicException {
class ClientQueryException : public FailureResponseException { class ClientQueryException : public FailureResponseException {
public: public:
using FailureResponseException::FailureResponseException; using FailureResponseException::FailureResponseException;
SPECIALIZE_GET_EXCEPTION_NAME(ClientQueryException)
}; };
/// This exception is thrown whenever a fatal error occurs during query /// This exception is thrown whenever a fatal error occurs during query
@@ -57,6 +59,7 @@ class ClientQueryException : public FailureResponseException {
class ClientFatalException : public utils::BasicException { class ClientFatalException : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(ClientFatalException)
}; };
// Internal exception used whenever a communication error occurs. You should // Internal exception used whenever a communication error occurs. You should
@@ -64,6 +67,7 @@ class ClientFatalException : public utils::BasicException {
class ServerCommunicationException : public ClientFatalException { class ServerCommunicationException : public ClientFatalException {
public: public:
ServerCommunicationException() : ClientFatalException("Couldn't communicate with the server!") {} ServerCommunicationException() : ClientFatalException("Couldn't communicate with the server!") {}
SPECIALIZE_GET_EXCEPTION_NAME(ServerCommunicationException)
}; };
// Internal exception used whenever a malformed data error occurs. You should // Internal exception used whenever a malformed data error occurs. You should
@@ -71,6 +75,7 @@ class ServerCommunicationException : public ClientFatalException {
class ServerMalformedDataException : public ClientFatalException { class ServerMalformedDataException : public ClientFatalException {
public: public:
ServerMalformedDataException() : ClientFatalException("The server sent malformed data!") {} ServerMalformedDataException() : ClientFatalException("The server sent malformed data!") {}
SPECIALIZE_GET_EXCEPTION_NAME(ServerMalformedDataException)
}; };
/// Structure that is used to return results from an executed query. /// Structure that is used to return results from an executed query.
@@ -155,4 +160,5 @@ class Client final {
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_}; ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_};
ClientEncoder encoder_{encoder_buffer_}; ClientEncoder encoder_{encoder_buffer_};
}; };
} // namespace memgraph::communication::bolt } // namespace memgraph::communication::bolt

View File

@@ -0,0 +1,55 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "communication/metrics.hpp"
namespace memgraph::communication::bolt {
template <typename TSession>
inline void RegisterNewSession(TSession &session, Value &metadata) {
auto &data = metadata.ValueMap();
session.metrics_ = bolt_metrics.Add(data.contains("user_agent") ? data["user_agent"].ValueString() : "unknown",
fmt::format("{}.{}", session.version_.major, session.version_.minor),
session.client_supported_bolt_versions_);
++session.metrics_.value()->sessions;
auto conn_type = (!data.contains("scheme") || data["scheme"].ValueString() == "none")
? BoltMetrics::ConnectionType::kAnonymous
: BoltMetrics::ConnectionType::kBasic;
++session.metrics_.value()->connection_types[(int)conn_type];
}
template <typename TSession>
inline void TouchNewSession(TSession &session, Value &metadata) {
auto &data = metadata.ValueMap();
session.metrics_ = bolt_metrics.Add(data.contains("user_agent") ? data["user_agent"].ValueString() : "unknown");
}
template <typename TSession>
inline void UpdateNewSession(TSession &session, Value &metadata) {
auto &data = metadata.ValueMap();
session.metrics_.value()->bolt_v = fmt::format("{}.{}", session.version_.major, session.version_.minor);
session.metrics_.value()->supported_bolt_v = session.client_supported_bolt_versions_;
++session.metrics_.value()->sessions;
auto conn_type = (!data.contains("scheme") || data["scheme"].ValueString() == "none")
? BoltMetrics::ConnectionType::kAnonymous
: BoltMetrics::ConnectionType::kBasic;
++session.metrics_.value()->connection_types[(int)conn_type];
}
template <typename TSession>
inline void IncrementQueryMetrics(TSession &session) {
++session.metrics_.value()->queries;
}
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,6 +30,7 @@ namespace memgraph::communication::bolt {
class ClientError : public utils::BasicException { class ClientError : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(ClientError)
}; };
/** /**
@@ -67,6 +68,7 @@ class VerboseError : public utils::BasicException {
code_(fmt::format("Memgraph.{}.{}.{}", ClassificationToString(classification), category, title)) {} code_(fmt::format("Memgraph.{}.{}.{}", ClassificationToString(classification), category, title)) {}
const std::string &code() const noexcept { return code_; } const std::string &code() const noexcept { return code_; }
SPECIALIZE_GET_EXCEPTION_NAME(VerboseError)
private: private:
std::string ClassificationToString(Classification classification) { std::string ClassificationToString(Classification classification) {

View File

@@ -27,6 +27,7 @@
#include "communication/bolt/v1/states/handshake.hpp" #include "communication/bolt/v1/states/handshake.hpp"
#include "communication/bolt/v1/states/init.hpp" #include "communication/bolt/v1/states/init.hpp"
#include "communication/bolt/v1/value.hpp" #include "communication/bolt/v1/value.hpp"
#include "communication/metrics.hpp"
#include "dbms/constants.hpp" #include "dbms/constants.hpp"
#include "dbms/global.hpp" #include "dbms/global.hpp"
#include "utils/exceptions.hpp" #include "utils/exceptions.hpp"
@@ -43,6 +44,7 @@ namespace memgraph::communication::bolt {
class SessionException : public utils::BasicException { class SessionException : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(SessionException)
}; };
/** /**
@@ -207,6 +209,8 @@ class Session {
}; };
Version version_; Version version_;
std::vector<std::string> client_supported_bolt_versions_;
std::optional<BoltMetrics::Metrics> metrics_;
virtual std::string GetCurrentDB() const = 0; virtual std::string GetCurrentDB() const = 0;
std::string UUID() const { return session_uuid_; } std::string UUID() const { return session_uuid_; }

View File

@@ -18,6 +18,7 @@
#include <string_view> #include <string_view>
#include <vector> #include <vector>
#include "communication/bolt/metrics.hpp"
#include "communication/bolt/v1/codes.hpp" #include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/constants.hpp" #include "communication/bolt/v1/constants.hpp"
#include "communication/bolt/v1/exceptions.hpp" #include "communication/bolt/v1/exceptions.hpp"
@@ -208,7 +209,14 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state"); DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
#if MG_ENTERPRISE
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString()); spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
#else
spdlog::debug("[Run] '{}'", query.ValueString());
#endif
// Increment number of queries in the metrics
IncrementQueryMetrics(session);
try { try {
// Interpret can throw. // Interpret can throw.
@@ -272,7 +280,14 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
return HandleFailure(session, e); return HandleFailure(session, e);
} }
#if MG_ENTERPRISE
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString()); spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
#else
spdlog::debug("[Run] '{}'", query.ValueString());
#endif
// Increment number of queries in the metrics
IncrementQueryMetrics(session);
try { try {
// Interpret can throw. // Interpret can throw.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,6 +24,31 @@
namespace memgraph::communication::bolt { namespace memgraph::communication::bolt {
inline std::vector<std::string> StringifySupportedVersions(uint8_t *data) {
std::vector<std::string> res;
uint8_t range_i = 1;
uint8_t minor_i = 2;
uint8_t major_i = 3;
uint8_t chunk_size = 4;
uint8_t n_chunks = 4;
auto stringify_version = [](uint8_t major, uint8_t minor) { return fmt::format("{}.{}", major, minor); };
for (uint8_t i = 0; i < n_chunks; ++i) {
const uint32_t full_version = *((uint32_t *)data);
if (full_version == 0) break;
if (data[1] != 0) { // Supports a range of versions
uint8_t range = data[range_i];
auto max_minor = data[minor_i];
for (uint8_t r = 0; r <= range; ++r) {
res.push_back(stringify_version(data[major_i], max_minor - r));
}
} else {
res.push_back(stringify_version(data[major_i], data[minor_i]));
}
data += chunk_size;
}
return res;
}
inline bool CopyProtocolInformationIfSupported(uint16_t version, uint8_t *protocol) { inline bool CopyProtocolInformationIfSupported(uint16_t version, uint8_t *protocol) {
const auto *supported_version = std::find(std::begin(kSupportedVersions), std::end(kSupportedVersions), version); const auto *supported_version = std::find(std::begin(kSupportedVersions), std::end(kSupportedVersions), version);
if (supported_version != std::end(kSupportedVersions)) { if (supported_version != std::end(kSupportedVersions)) {
@@ -71,6 +96,8 @@ State StateHandshakeRun(TSession &session) {
auto dataPosition = session.input_stream_.data() + sizeof(kPreamble); auto dataPosition = session.input_stream_.data() + sizeof(kPreamble);
uint8_t protocol[4] = {0x00}; uint8_t protocol[4] = {0x00};
session.client_supported_bolt_versions_ = std::move(StringifySupportedVersions(dataPosition));
for (int i = 0; i < 4 && !protocol[3]; ++i) { for (int i = 0; i < 4 && !protocol[3]; ++i) {
// If there is an offset defined (e.g. 0x00 0x03 0x03 0x04) the second byte // If there is an offset defined (e.g. 0x00 0x03 0x03 0x04) the second byte
// That would enable the client to pick between 4.0 and 4.3 versions // That would enable the client to pick between 4.0 and 4.3 versions

View File

@@ -11,6 +11,7 @@
#pragma once #pragma once
#include <fmt/core.h>
#include <fmt/format.h> #include <fmt/format.h>
#include <optional> #include <optional>
@@ -18,6 +19,7 @@
#include "communication/bolt/v1/state.hpp" #include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp" #include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp" #include "communication/exceptions.hpp"
#include "communication/metrics.hpp"
#include "spdlog/spdlog.h" #include "spdlog/spdlog.h"
#include "utils/likely.hpp" #include "utils/likely.hpp"
#include "utils/logging.hpp" #include "utils/logging.hpp"
@@ -201,6 +203,9 @@ State StateInitRunV1(TSession &session, const Marker marker, const Signature sig
return result.value(); return result.value();
} }
// Register session to metrics
RegisterNewSession(session, *maybeMetadata);
return SendSuccessMessage(session); return SendSuccessMessage(session);
} }
@@ -227,6 +232,9 @@ State StateInitRunV4(TSession &session, Marker marker, Signature signature) {
return result.value(); return result.value();
} }
// Register session to metrics
RegisterNewSession(session, *maybeMetadata);
return SendSuccessMessage(session); return SendSuccessMessage(session);
} }
@@ -247,6 +255,10 @@ State StateInitRunV5(TSession &session, Marker marker, Signature signature) {
if (SendSuccessMessage(session) == State::Close) { if (SendSuccessMessage(session) == State::Close) {
return State::Close; return State::Close;
} }
// Register session to metrics
TouchNewSession(session, *maybeMetadata);
// Stay in Init // Stay in Init
return State::Init; return State::Init;
} }
@@ -274,6 +286,10 @@ State StateInitRunV5(TSession &session, Marker marker, Signature signature) {
if (SendSuccessMessage(session) == State::Close) { if (SendSuccessMessage(session) == State::Close) {
return State::Close; return State::Close;
} }
// Register session to metrics
UpdateNewSession(session, *maybeMetadata);
return State::Idle; return State::Idle;
} }

View File

@@ -276,6 +276,7 @@ class ValueException : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
ValueException() : BasicException("Incompatible template param and type!") {} ValueException() : BasicException("Incompatible template param and type!") {}
SPECIALIZE_GET_EXCEPTION_NAME(ValueException)
}; };
/** /**

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,5 +21,6 @@ namespace memgraph::communication {
*/ */
class SessionClosedException : public utils::BasicException { class SessionClosedException : public utils::BasicException {
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(SessionClosedException)
}; };
} // namespace memgraph::communication } // namespace memgraph::communication

View File

@@ -0,0 +1,42 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "metrics.hpp"
namespace {
constexpr auto kName = "name";
constexpr auto kSupportedBoltVersions = "supported_bolt_versions";
constexpr auto kBoltVersion = "bolt_version";
constexpr auto kConnectionTypes = "connection_types";
constexpr auto kSessions = "sessions";
constexpr auto kQueries = "queries";
} // namespace
namespace memgraph::communication {
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
BoltMetrics bolt_metrics;
nlohmann::json BoltMetrics::Info::ToJson() const {
nlohmann::json res;
res[kName] = name;
res[kSupportedBoltVersions] = nlohmann::json::array();
for (const auto &sbv : supported_bolt_v) {
res[kSupportedBoltVersions].push_back(sbv);
}
res[kBoltVersion] = bolt_v;
res[kConnectionTypes] = {{ConnectionTypeStr((ConnectionType)0), connection_types[0].load()},
{ConnectionTypeStr((ConnectionType)1), connection_types[1].load()}};
res[kSessions] = sessions.load();
res[kQueries] = queries.load();
return res;
}
} // namespace memgraph::communication

View File

@@ -0,0 +1,116 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <map>
#include <mutex>
#include <shared_mutex>
#include <string>
#include <vector>
#include <json/json.hpp>
namespace memgraph::communication {
class BoltMetrics {
public:
enum class ConnectionType { kAnonymous = 0, kBasic, Count };
static constexpr std::array<std::string_view, (int)ConnectionType::Count> ct_to_str = {"anonymous", "basic"};
static std::string ConnectionTypeStr(ConnectionType type) { return std::string(ct_to_str[(int)type]); }
class Metrics;
struct Info {
explicit Info(std::string name) : name(std::move(name)) {}
Info(std::string name, std::string bolt_v, std::vector<std::string> supported_bolt_v)
: name(std::move(name)), bolt_v(std::move(bolt_v)), supported_bolt_v(std::move(supported_bolt_v)) {}
const std::string name; //!< Driver name
std::string bolt_v; //!< Bolt version used
std::vector<std::string> supported_bolt_v; //!< Bolt versions supported by the driver
std::atomic_int connection_types[(int)ConnectionType::Count]; //!< Authentication used when connecting
std::atomic_int sessions; //!< Number of sessions using the same driver
std::atomic_int queries; //!< Queries executed by the driver
nlohmann::json ToJson() const;
private:
friend class Metrics;
std::atomic_int concurrent_users;
};
class Metrics {
friend class BoltMetrics;
explicit Metrics(Info &info) : info_(&info) { ++info_->concurrent_users; }
public:
~Metrics() {
if (info_) --info_->concurrent_users;
}
Metrics(const Metrics &other) : info_(other.info_) { ++info_->concurrent_users; }
Metrics(Metrics &&other) noexcept : info_(other.info_) { other.info_ = nullptr; }
Metrics &operator=(const Metrics &other) {
if (this != &other) {
if (info_) --info_->concurrent_users;
info_ = other.info_;
if (info_) ++info_->concurrent_users;
}
return *this;
}
Metrics &operator=(Metrics &&other) noexcept {
if (this != &other) {
if (info_) --info_->concurrent_users;
info_ = other.info_;
other.info_ = nullptr; // Invalidate the source object
}
return *this;
}
Info *operator->() { return info_; }
Info *info_;
};
Metrics Add(std::string name) {
std::unique_lock<std::mutex> l(mtx);
auto key = name;
auto [it, _] = info.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)),
std::forward_as_tuple(std::move(name)));
return Metrics(it->second);
}
Metrics Add(std::string name, std::string bolt_v, std::vector<std::string> supported_bolt_v) {
std::unique_lock<std::mutex> l(mtx);
auto key = name;
auto [it, _] = info.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)),
std::forward_as_tuple(std::move(name), std::move(bolt_v), std::move(supported_bolt_v)));
return Metrics(it->second);
}
nlohmann::json ToJson() {
std::unique_lock<std::mutex> l(mtx);
auto res = nlohmann::json::array();
for (const auto &[_, client_info] : info) {
res.push_back(client_info.ToJson());
}
return res;
}
private:
mutable std::mutex mtx;
std::map<std::string, Info> info;
};
extern BoltMetrics bolt_metrics;
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -175,6 +175,11 @@ void Session::OnRead(const boost::beast::error_code ec, const size_t /*bytes_tra
return; return;
} }
if (ec) {
LogError(ec, "read");
return;
}
if (!IsAuthenticated()) { if (!IsAuthenticated()) {
auto response = nlohmann::json(); auto response = nlohmann::json();
auto auth_failed = [this, &response](const std::string &message) { auto auth_failed = [this, &response](const std::string &message) {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd. // Copyright 2023 Memgraph Ltd.
// //
// Use of this software is governed by the Business Source License // Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -35,6 +35,7 @@ namespace memgraph::csv {
class CsvReadException : public utils::BasicException { class CsvReadException : public utils::BasicException {
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(CsvReadException)
}; };
class FileCsvSource { class FileCsvSource {

View File

@@ -1,3 +1,3 @@
add_library(mg-dbms STATIC database.cpp) add_library(mg-dbms STATIC database.cpp replication_handler.cpp inmemory/replication_handlers.cpp)
target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query) target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query)

View File

@@ -10,6 +10,8 @@
// licenses/APL.txt. // licenses/APL.txt.
#include "dbms/database.hpp" #include "dbms/database.hpp"
#include "dbms/inmemory/storage_helper.hpp"
#include "dbms/replication_handler.hpp"
#include "flags/storage_mode.hpp" #include "flags/storage_mode.hpp"
#include "storage/v2/disk/storage.hpp" #include "storage/v2/disk/storage.hpp"
#include "storage/v2/inmemory/storage.hpp" #include "storage/v2/inmemory/storage.hpp"
@@ -19,14 +21,16 @@ template struct memgraph::utils::Gatekeeper<memgraph::dbms::Database>;
namespace memgraph::dbms { namespace memgraph::dbms {
Database::Database(const storage::Config &config) Database::Database(storage::Config config, const replication::ReplicationState &repl_state)
: trigger_store_(config.durability.storage_directory / "triggers"), : trigger_store_(config.durability.storage_directory / "triggers"),
streams_{config.durability.storage_directory / "streams"} { streams_{config.durability.storage_directory / "streams"},
plan_cache_{FLAGS_query_plan_cache_max_size},
repl_state_(&repl_state) {
if (config.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk || if (config.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk ||
utils::DirExists(config.disk.main_storage_directory)) { utils::DirExists(config.disk.main_storage_directory)) {
storage_ = std::make_unique<storage::DiskStorage>(config); storage_ = std::make_unique<storage::DiskStorage>(std::move(config));
} else { } else {
storage_ = std::make_unique<storage::InMemoryStorage>(config, config.storage_mode); storage_ = dbms::CreateInMemoryStorage(std::move(config), repl_state);
} }
} }

View File

@@ -24,9 +24,19 @@
#include "query/trigger.hpp" #include "query/trigger.hpp"
#include "storage/v2/storage.hpp" #include "storage/v2/storage.hpp"
#include "utils/gatekeeper.hpp" #include "utils/gatekeeper.hpp"
#include "utils/lru_cache.hpp"
#include "utils/synchronized.hpp"
namespace memgraph::dbms { namespace memgraph::dbms {
struct DatabaseInfo {
storage::StorageInfo storage_info;
uint64_t triggers;
uint64_t streams;
};
static inline nlohmann::json ToJson(const DatabaseInfo &info) { return ToJson(info.storage_info); }
/** /**
* @brief Class containing everything associated with a single Database * @brief Class containing everything associated with a single Database
* *
@@ -38,7 +48,7 @@ class Database {
* *
* @param config storage configuration * @param config storage configuration
*/ */
explicit Database(const storage::Config &config); explicit Database(storage::Config config, const replication::ReplicationState &repl_state);
/** /**
* @brief Returns the raw storage pointer. * @brief Returns the raw storage pointer.
@@ -48,6 +58,7 @@ class Database {
* @return storage::Storage* * @return storage::Storage*
*/ */
storage::Storage *storage() { return storage_.get(); } storage::Storage *storage() { return storage_.get(); }
storage::Storage const *storage() const { return storage_.get(); }
/** /**
* @brief Storage's Accessor * @brief Storage's Accessor
@@ -57,12 +68,12 @@ class Database {
*/ */
std::unique_ptr<storage::Storage::Accessor> Access( std::unique_ptr<storage::Storage::Accessor> Access(
std::optional<storage::IsolationLevel> override_isolation_level = {}) { std::optional<storage::IsolationLevel> override_isolation_level = {}) {
return storage_->Access(override_isolation_level); return storage_->Access(override_isolation_level, repl_state_->IsMain());
} }
std::unique_ptr<storage::Storage::Accessor> UniqueAccess( std::unique_ptr<storage::Storage::Accessor> UniqueAccess(
std::optional<storage::IsolationLevel> override_isolation_level = {}) { std::optional<storage::IsolationLevel> override_isolation_level = {}) {
return storage_->UniqueAccess(override_isolation_level); return storage_->UniqueAccess(override_isolation_level, repl_state_->IsMain());
} }
/** /**
@@ -89,9 +100,16 @@ class Database {
/** /**
* @brief Get the storage info * @brief Get the storage info
* *
* @return storage::StorageInfo * @param force_directory Use the configured directory, do not try to decipher the multi-db version
* @return DatabaseInfo
*/ */
storage::StorageInfo GetInfo() const { return storage_->GetInfo(); } DatabaseInfo GetInfo(bool force_directory = false) const {
DatabaseInfo info;
info.storage_info = storage_->GetInfo(force_directory);
info.triggers = trigger_store_.GetTriggerInfo().size();
info.streams = streams_.GetStreamInfo().size();
return info;
}
/** /**
* @brief Switch storage to OnDisk * @brief Switch storage to OnDisk
@@ -130,9 +148,9 @@ class Database {
/** /**
* @brief Returns the PlanCache vector raw pointer * @brief Returns the PlanCache vector raw pointer
* *
* @return utils::SkipList<query::PlanCacheEntry>* * @return utils::Synchronized<utils::LRUCache<uint64_t, std::shared_ptr<PlanWrapper>>, utils::RWSpinLock>
*/ */
utils::SkipList<query::PlanCacheEntry> *plan_cache() { return &plan_cache_; } query::PlanCacheLRU *plan_cache() { return &plan_cache_; }
private: private:
std::unique_ptr<storage::Storage> storage_; //!< Underlying storage std::unique_ptr<storage::Storage> storage_; //!< Underlying storage
@@ -141,7 +159,9 @@ class Database {
query::stream::Streams streams_; //!< Streams associated with the storage query::stream::Streams streams_; //!< Streams associated with the storage
// TODO: Move to a better place // TODO: Move to a better place
utils::SkipList<query::PlanCacheEntry> plan_cache_; //!< Plan cache associated with the storage query::PlanCacheLRU plan_cache_; //!< Plan cache associated with the storage
const replication::ReplicationState *repl_state_;
}; };
} // namespace memgraph::dbms } // namespace memgraph::dbms

View File

@@ -51,7 +51,8 @@ class DatabaseHandler : public Handler<Database> {
* @param config Storage configuration * @param config Storage configuration
* @return HandlerT::NewResult * @return HandlerT::NewResult
*/ */
HandlerT::NewResult New(std::string_view name, storage::Config config) { HandlerT::NewResult New(std::string_view name, storage::Config config,
const replication::ReplicationState &repl_state) {
// Control that no one is using the same data directory // Control that no one is using the same data directory
if (std::any_of(begin(), end(), [&](auto &elem) { if (std::any_of(begin(), end(), [&](auto &elem) {
auto db_acc = elem.second.access(); auto db_acc = elem.second.access();
@@ -62,7 +63,7 @@ class DatabaseHandler : public Handler<Database> {
return NewError::EXISTS; return NewError::EXISTS;
} }
config.name = name; // Set storage id via config config.name = name; // Set storage id via config
return HandlerT::New(std::piecewise_construct, name, config); return HandlerT::New(std::piecewise_construct, name, config, repl_state);
} }
/** /**
@@ -93,5 +94,4 @@ class DatabaseHandler : public Handler<Database> {
}; };
} // namespace memgraph::dbms } // namespace memgraph::dbms
#endif #endif

View File

@@ -25,13 +25,17 @@
#include "auth/auth.hpp" #include "auth/auth.hpp"
#include "constants.hpp" #include "constants.hpp"
#include "dbms/database.hpp"
#ifdef MG_ENTERPRISE
#include "dbms/database_handler.hpp" #include "dbms/database_handler.hpp"
#endif
#include "global.hpp" #include "global.hpp"
#include "query/config.hpp" #include "query/config.hpp"
#include "query/interpreter_context.hpp" #include "query/interpreter_context.hpp"
#include "spdlog/spdlog.h" #include "spdlog/spdlog.h"
#include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/durability.hpp"
#include "storage/v2/durability/paths.hpp" #include "storage/v2/durability/paths.hpp"
#include "storage/v2/isolation_level.hpp"
#include "utils/exceptions.hpp" #include "utils/exceptions.hpp"
#include "utils/file.hpp" #include "utils/file.hpp"
#include "utils/logging.hpp" #include "utils/logging.hpp"
@@ -42,9 +46,42 @@
namespace memgraph::dbms { namespace memgraph::dbms {
#ifdef MG_ENTERPRISE struct Statistics {
uint64_t num_vertex; //!< Sum of vertexes in every database
uint64_t num_edges; //!< Sum of edges in every database
uint64_t triggers; //!< Sum of triggers in every database
uint64_t streams; //!< Sum of streams in every database
uint64_t users; //!< Number of defined users
uint64_t num_databases; //!< Number of isolated databases
uint64_t indices; //!< Sum of indices in every database
uint64_t constraints; //!< Sum of constraints in every database
uint64_t storage_modes[3]; //!< Number of databases in each storage mode [IN_MEM_TX, IN_MEM_ANA, ON_DISK_TX]
uint64_t isolation_levels[3]; //!< Number of databases in each isolation level [SNAPSHOT, READ_COMM, READ_UNC]
uint64_t snapshot_enabled; //!< Number of databases with snapshots enabled
uint64_t wal_enabled; //!< Number of databases with WAL enabled
};
using DeleteResult = utils::BasicResult<DeleteError>; static inline nlohmann::json ToJson(const Statistics &stats) {
nlohmann::json res;
res["edges"] = stats.num_edges;
res["vertices"] = stats.num_vertex;
res["triggers"] = stats.triggers;
res["streams"] = stats.streams;
res["users"] = stats.users;
res["databases"] = stats.num_databases;
res["indices"] = stats.indices;
res["constraints"] = stats.constraints;
res["storage_modes"] = {{storage::StorageModeToString((storage::StorageMode)0), stats.storage_modes[0]},
{storage::StorageModeToString((storage::StorageMode)1), stats.storage_modes[1]},
{storage::StorageModeToString((storage::StorageMode)2), stats.storage_modes[2]}};
res["isolation_levels"] = {{storage::IsolationLevelToString((storage::IsolationLevel)0), stats.isolation_levels[0]},
{storage::IsolationLevelToString((storage::IsolationLevel)1), stats.isolation_levels[1]},
{storage::IsolationLevelToString((storage::IsolationLevel)2), stats.isolation_levels[2]}};
res["durability"] = {{"snapshot_enabled", stats.snapshot_enabled}, {"WAL_enabled", stats.wal_enabled}};
return res;
}
/** /**
* @brief Multi-database session contexts handler. * @brief Multi-database session contexts handler.
@@ -52,28 +89,29 @@ using DeleteResult = utils::BasicResult<DeleteError>;
class DbmsHandler { class DbmsHandler {
public: public:
using LockT = utils::RWLock; using LockT = utils::RWLock;
using NewResultT = utils::BasicResult<NewError, DatabaseAccess>; #ifdef MG_ENTERPRISE
struct Statistics { using NewResultT = utils::BasicResult<NewError, DatabaseAccess>;
uint64_t num_vertex; //!< Sum of vertexes in every database using DeleteResult = utils::BasicResult<DeleteError>;
uint64_t num_edges; //!< Sum of edges in every database
uint64_t num_databases; //! number of isolated databases
};
/** /**
* @brief Initialize the handler. * @brief Initialize the handler.
* *
* @param configs storage and interpreter configurations * @param configs storage configuration
* @param auth pointer to the global authenticator * @param auth pointer to the global authenticator
* @param recovery_on_startup restore databases (and its content) and authentication data * @param recovery_on_startup restore databases (and its content) and authentication data
* @param delete_on_drop when dropping delete any associated directories on disk * @param delete_on_drop when dropping delete any associated directories on disk
*/ */
DbmsHandler(storage::Config config, auto *auth, bool recovery_on_startup, bool delete_on_drop) DbmsHandler(storage::Config config, const replication::ReplicationState &repl_state, auto *auth,
: lock_{utils::RWLock::Priority::READ}, default_config_{std::move(config)}, delete_on_drop_(delete_on_drop) { bool recovery_on_startup, bool delete_on_drop)
: lock_{utils::RWLock::Priority::READ},
default_config_{std::move(config)},
repl_state_(repl_state),
delete_on_drop_(delete_on_drop) {
// TODO: Decouple storage config from dbms config // TODO: Decouple storage config from dbms config
// TODO: Save individual db configs inside the kvstore and restore from there // TODO: Save individual db configs inside the kvstore and restore from there
storage::UpdatePaths(*default_config_, default_config_->durability.storage_directory / "databases"); storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases");
const auto &db_dir = default_config_->durability.storage_directory; const auto &db_dir = default_config_.durability.storage_directory;
const auto durability_dir = db_dir / ".durability"; const auto durability_dir = db_dir / ".durability";
utils::EnsureDirOrDie(db_dir); utils::EnsureDirOrDie(db_dir);
utils::EnsureDirOrDie(durability_dir); utils::EnsureDirOrDie(durability_dir);
@@ -81,7 +119,6 @@ class DbmsHandler {
// Generate the default database // Generate the default database
MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB."); MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB.");
// Recover previous databases // Recover previous databases
if (recovery_on_startup) { if (recovery_on_startup) {
for (const auto &[name, _] : *durability_) { for (const auto &[name, _] : *durability_) {
@@ -99,7 +136,21 @@ class DbmsHandler {
} }
} }
} }
#else
/**
* @brief Initialize the handler. A single database is supported in community edition.
*
* @param configs storage configuration
*/
DbmsHandler(storage::Config config, const replication::ReplicationState &repl_state)
: db_gatekeeper_{[&] {
config.name = kDefaultDB;
return std::move(config);
}(),
repl_state} {}
#endif
#ifdef MG_ENTERPRISE
/** /**
* @brief Create a new Database associated with the "name" database * @brief Create a new Database associated with the "name" database
* *
@@ -118,11 +169,24 @@ class DbmsHandler {
* @return DatabaseAccess * @return DatabaseAccess
* @throw UnknownDatabaseException if database not found * @throw UnknownDatabaseException if database not found
*/ */
DatabaseAccess Get(std::string_view name) { DatabaseAccess Get(std::string_view name = kDefaultDB) {
std::shared_lock<LockT> rd(lock_); std::shared_lock<LockT> rd(lock_);
return Get_(name); return Get_(name);
} }
#else
/**
* @brief Get the context associated with the default database
*
* @return DatabaseAccess
*/
DatabaseAccess Get() {
auto acc = db_gatekeeper_.access();
MG_ASSERT(acc, "Failed to get default database!");
return *acc;
}
#endif
#ifdef MG_ENTERPRISE
/** /**
* @brief Delete database. * @brief Delete database.
* *
@@ -168,6 +232,7 @@ class DbmsHandler {
return {}; // Success return {}; // Success
} }
#endif
/** /**
* @brief Return all active databases. * @brief Return all active databases.
@@ -175,30 +240,72 @@ class DbmsHandler {
* @return std::vector<std::string> * @return std::vector<std::string>
*/ */
std::vector<std::string> All() const { std::vector<std::string> All() const {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_); std::shared_lock<LockT> rd(lock_);
return db_handler_.All(); return db_handler_.All();
#else
return {db_gatekeeper_.access()->get()->id()};
#endif
} }
/** /**
* @brief Return the number of vertex across all databases. * @brief Return the statistics all databases.
* *
* @return uint64_t * @return Statistics
*/ */
Statistics Info() { Statistics Stats() {
Statistics stats{};
// TODO: Handle overflow? // TODO: Handle overflow?
uint64_t nv = 0; #ifdef MG_ENTERPRISE
uint64_t ne = 0;
std::shared_lock<LockT> rd(lock_); std::shared_lock<LockT> rd(lock_);
const uint64_t ndb = std::distance(db_handler_.cbegin(), db_handler_.cend());
for (auto &[_, db_gk] : db_handler_) { for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc_opt = db_gk.access(); auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue; if (db_acc_opt) {
auto &db_acc = *db_acc_opt; auto &db_acc = *db_acc_opt;
const auto &info = db_acc->GetInfo(); const auto &info = db_acc->GetInfo();
nv += info.vertex_count; const auto &storage_info = info.storage_info;
ne += info.edge_count; stats.num_vertex += storage_info.vertex_count;
stats.num_edges += storage_info.edge_count;
stats.triggers += info.triggers;
stats.streams += info.streams;
++stats.num_databases;
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
++stats.storage_modes[(int)storage_info.storage_mode];
++stats.isolation_levels[(int)storage_info.isolation_level];
stats.snapshot_enabled += storage_info.durability_snapshot_enabled;
stats.wal_enabled += storage_info.durability_wal_enabled;
}
} }
return {nv, ne, ndb}; return stats;
}
/**
* @brief Return a vector with all database info.
*
* @return std::vector<DatabaseInfo>
*/
std::vector<DatabaseInfo> Info() {
std::vector<DatabaseInfo> res;
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
res.reserve(std::distance(db_handler_.cbegin(), db_handler_.cend()));
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc_opt = db_gk.access();
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
res.push_back(db_acc->GetInfo());
}
}
return res;
} }
/** /**
@@ -208,15 +315,21 @@ class DbmsHandler {
* @param ic global InterpreterContext * @param ic global InterpreterContext
*/ */
void RestoreTriggers(query::InterpreterContext *ic) { void RestoreTriggers(query::InterpreterContext *ic) {
#ifdef MG_ENTERPRISE
std::lock_guard<LockT> wr(lock_); std::lock_guard<LockT> wr(lock_);
for (auto &[_, db_gk] : db_handler_) { for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc_opt = db_gk.access(); auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue; if (db_acc_opt) {
auto &db_acc = *db_acc_opt; auto &db_acc = *db_acc_opt;
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id()); spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id());
auto storage_accessor = db_acc->Access(); auto storage_accessor = db_acc->Access();
auto dba = memgraph::query::DbAccessor{storage_accessor.get()}; auto dba = memgraph::query::DbAccessor{storage_accessor.get()};
db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker); db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker);
}
} }
} }
@@ -227,17 +340,67 @@ class DbmsHandler {
* @param ic global InterpreterContext * @param ic global InterpreterContext
*/ */
void RestoreStreams(query::InterpreterContext *ic) { void RestoreStreams(query::InterpreterContext *ic) {
#ifdef MG_ENTERPRISE
std::lock_guard<LockT> wr(lock_); std::lock_guard<LockT> wr(lock_);
for (auto &[_, db_gk] : db_handler_) { for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc = db_gk.access(); auto db_acc = db_gk.access();
if (!db_acc) continue; if (db_acc) {
auto *db = db_acc->get(); auto *db = db_acc->get();
spdlog::debug("Restoring streams for database \"{}\"", db->id()); spdlog::debug("Restoring streams for database \"{}\"", db->id());
db->streams()->RestoreStreams(*db_acc, ic); db->streams()->RestoreStreams(*db_acc, ic);
}
} }
} }
/**
* @brief todo
*
* @param f
*/
void ForEach(auto f) {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc = db_gk.access();
if (db_acc) { // This isn't an error, just a defunct db
f(db_acc->get());
}
}
}
/**
* @brief todo
*
* @param f
*/
void ForOne(auto f) {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
auto db_acc = db_gk.access();
if (db_acc) { // This isn't an error, just a defunct db
if (f(db_acc->get())) break; // Run until the first successful one
}
}
#else
{
auto db_acc = db_gatekeeper_.access();
MG_ASSERT(db_acc, "Should always have the database");
f(db_acc->get());
}
#endif
}
private: private:
#ifdef MG_ENTERPRISE
/** /**
* @brief return the storage directory of the associated database * @brief return the storage directory of the associated database
* *
@@ -269,13 +432,9 @@ class DbmsHandler {
* @return NewResultT context on success, error on failure * @return NewResultT context on success, error on failure
*/ */
NewResultT New_(const std::string &name, std::filesystem::path storage_subdir) { NewResultT New_(const std::string &name, std::filesystem::path storage_subdir) {
if (default_config_) { auto config_copy = default_config_;
auto config_copy = *default_config_; storage::UpdatePaths(config_copy, default_config_.durability.storage_directory / storage_subdir);
storage::UpdatePaths(config_copy, default_config_->durability.storage_directory / storage_subdir); return New_(name, config_copy);
return New_(name, config_copy);
}
spdlog::info("Trying to generate session context without any configurations.");
return NewError::NO_CONFIGS;
} }
/** /**
@@ -292,7 +451,7 @@ class DbmsHandler {
return NewError::DEFUNCT; return NewError::DEFUNCT;
} }
auto new_db = db_handler_.New(name, storage_config); auto new_db = db_handler_.New(name, storage_config, repl_state_);
if (new_db.HasValue()) { if (new_db.HasValue()) {
// Success // Success
if (durability_) durability_->Put(name, "ok"); // TODO: Serialize the configuration? if (durability_) durability_->Put(name, "ok"); // TODO: Serialize the configuration?
@@ -377,14 +536,16 @@ class DbmsHandler {
throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name); throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name);
} }
// Should storage objects ever be deleted? mutable LockT lock_; //!< protective lock
mutable LockT lock_; //!< protective lock storage::Config default_config_; //!< Storage configuration used when creating new databases
DatabaseHandler db_handler_; //!< multi-tenancy storage handler const replication::ReplicationState &repl_state_; //!< Global replication state
std::optional<storage::Config> default_config_; //!< Storage configuration used when creating new databases DatabaseHandler db_handler_; //!< multi-tenancy storage handler
std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation) std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation)
std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory
bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures
}; #else
mutable utils::Gatekeeper<Database> db_gatekeeper_; //!< Single databases gatekeeper
#endif #endif
};
} // namespace memgraph::dbms } // namespace memgraph::dbms

View File

@@ -19,6 +19,7 @@
namespace memgraph::dbms { namespace memgraph::dbms {
#ifdef MG_ENTERPRISE
enum class DeleteError : uint8_t { enum class DeleteError : uint8_t {
DEFAULT_DB, DEFAULT_DB,
USING, USING,
@@ -34,11 +35,7 @@ enum class NewError : uint8_t {
GENERIC, GENERIC,
}; };
enum class SetForResult : uint8_t { #endif
SUCCESS,
ALREADY_SET,
FAIL,
};
/** /**
* UnknownSession Exception * UnknownSession Exception
@@ -48,6 +45,7 @@ enum class SetForResult : uint8_t {
class UnknownSessionException : public utils::BasicException { class UnknownSessionException : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(UnknownSessionException)
}; };
/** /**
@@ -58,6 +56,7 @@ class UnknownSessionException : public utils::BasicException {
class UnknownDatabaseException : public utils::BasicException { class UnknownDatabaseException : public utils::BasicException {
public: public:
using utils::BasicException::BasicException; using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(UnknownDatabaseException)
}; };
} // namespace memgraph::dbms } // namespace memgraph::dbms

View File

@@ -49,7 +49,7 @@ class Handler {
* @return NewResult * @return NewResult
*/ */
template <typename... Args> template <typename... Args>
NewResult New(std::piecewise_construct_t /* marker */, std::string_view name, Args... args) { NewResult New(std::piecewise_construct_t /* marker */, std::string_view name, Args &&...args) {
// Make sure the emplace will succeed, since we don't want to create temporary objects that could break something // Make sure the emplace will succeed, since we don't want to create temporary objects that could break something
if (!Has(name)) { if (!Has(name)) {
auto [itr, _] = items_.emplace(std::piecewise_construct, std::forward_as_tuple(name), auto [itr, _] = items_.emplace(std::piecewise_construct, std::forward_as_tuple(name),

View File

@@ -9,7 +9,11 @@
// by the Apache License, Version 2.0, included in the file // by the Apache License, Version 2.0, included in the file
// licenses/APL.txt. // licenses/APL.txt.
#include "storage/v2/inmemory/replication/replication_server.hpp" #include "dbms/inmemory/replication_handlers.hpp"
#include "dbms/constants.hpp"
#include "dbms/dbms_handler.hpp"
#include "replication/replication_server.hpp"
#include "spdlog/spdlog.h"
#include "storage/v2/durability/durability.hpp" #include "storage/v2/durability/durability.hpp"
#include "storage/v2/durability/snapshot.hpp" #include "storage/v2/durability/snapshot.hpp"
#include "storage/v2/durability/version.hpp" #include "storage/v2/durability/version.hpp"
@@ -17,9 +21,20 @@
#include "storage/v2/inmemory/storage.hpp" #include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/inmemory/unique_constraints.hpp" #include "storage/v2/inmemory/unique_constraints.hpp"
namespace memgraph::storage { using memgraph::storage::Delta;
using memgraph::storage::EdgeAccessor;
using memgraph::storage::EdgeRef;
using memgraph::storage::EdgeTypeId;
using memgraph::storage::LabelIndexStats;
using memgraph::storage::LabelPropertyIndexStats;
using memgraph::storage::PropertyId;
using memgraph::storage::UniqueConstraints;
using memgraph::storage::View;
using memgraph::storage::durability::WalDeltaData;
namespace memgraph::dbms {
namespace { namespace {
std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder *decoder) { std::pair<uint64_t, WalDeltaData> ReadDelta(storage::durability::BaseDecoder *decoder) {
try { try {
auto timestamp = ReadWalDeltaHeader(decoder); auto timestamp = ReadWalDeltaHeader(decoder);
SPDLOG_INFO(" Timestamp {}", timestamp); SPDLOG_INFO(" Timestamp {}", timestamp);
@@ -27,225 +42,278 @@ std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder
return {timestamp, delta}; return {timestamp, delta};
} catch (const slk::SlkReaderException &) { } catch (const slk::SlkReaderException &) {
throw utils::BasicException("Missing data!"); throw utils::BasicException("Missing data!");
} catch (const durability::RecoveryFailure &) { } catch (const storage::durability::RecoveryFailure &) {
throw utils::BasicException("Invalid data!"); throw utils::BasicException("Invalid data!");
} }
}; };
std::optional<DatabaseAccess> GetDatabaseAccessor(dbms::DbmsHandler *dbms_handler, std::string_view db_name) {
try {
#ifdef MG_ENTERPRISE
auto acc = dbms_handler->Get(db_name);
#else
if (db_name != dbms::kDefaultDB) {
spdlog::warn("Trying to replicate a non-default database on a community replica.");
return std::nullopt;
}
auto acc = dbms_handler->Get();
#endif
if (!acc) {
spdlog::error("Failed to get access to ", db_name);
return std::nullopt;
}
auto *inmem_storage = dynamic_cast<storage::InMemoryStorage *>(acc.get()->storage());
if (!inmem_storage || inmem_storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
spdlog::error("Database \"{}\" is not IN_MEMORY_TRANSACTIONAL.", db_name);
return std::nullopt;
}
return std::optional{std::move(acc)};
} catch (const dbms::UnknownDatabaseException &e) {
spdlog::warn("No database \"{}\" on replica!", db_name);
return std::nullopt;
}
}
} // namespace } // namespace
InMemoryReplicationServer::InMemoryReplicationServer(InMemoryStorage *storage, void InMemoryReplicationHandlers::Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server) {
const replication::ReplicationServerConfig &config) server.rpc_server_.Register<storage::replication::HeartbeatRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
: ReplicationServer{config}, storage_(storage) {
rpc_server_.Register<replication::HeartbeatRpc>([this](auto *req_reader, auto *res_builder) {
spdlog::debug("Received HeartbeatRpc"); spdlog::debug("Received HeartbeatRpc");
this->HeartbeatHandler(req_reader, res_builder); InMemoryReplicationHandlers::HeartbeatHandler(dbms_handler, req_reader, res_builder);
}); });
server.rpc_server_.Register<storage::replication::AppendDeltasRpc>(
rpc_server_.Register<replication::AppendDeltasRpc>([this](auto *req_reader, auto *res_builder) { [dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received AppendDeltasRpc"); spdlog::debug("Received AppendDeltasRpc");
this->AppendDeltasHandler(req_reader, res_builder); InMemoryReplicationHandlers::AppendDeltasHandler(dbms_handler, req_reader, res_builder);
}); });
rpc_server_.Register<replication::SnapshotRpc>([this](auto *req_reader, auto *res_builder) { server.rpc_server_.Register<storage::replication::SnapshotRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received SnapshotRpc"); spdlog::debug("Received SnapshotRpc");
this->SnapshotHandler(req_reader, res_builder); InMemoryReplicationHandlers::SnapshotHandler(dbms_handler, req_reader, res_builder);
}); });
rpc_server_.Register<replication::WalFilesRpc>([this](auto *req_reader, auto *res_builder) { server.rpc_server_.Register<storage::replication::WalFilesRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received WalFilesRpc"); spdlog::debug("Received WalFilesRpc");
this->WalFilesHandler(req_reader, res_builder); InMemoryReplicationHandlers::WalFilesHandler(dbms_handler, req_reader, res_builder);
}); });
rpc_server_.Register<replication::CurrentWalRpc>([this](auto *req_reader, auto *res_builder) { server.rpc_server_.Register<storage::replication::CurrentWalRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received CurrentWalRpc"); spdlog::debug("Received CurrentWalRpc");
this->CurrentWalHandler(req_reader, res_builder); InMemoryReplicationHandlers::CurrentWalHandler(dbms_handler, req_reader, res_builder);
}); });
rpc_server_.Register<replication::TimestampRpc>([this](auto *req_reader, auto *res_builder) { server.rpc_server_.Register<storage::replication::TimestampRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received TimestampRpc"); spdlog::debug("Received TimestampRpc");
this->TimestampHandler(req_reader, res_builder); InMemoryReplicationHandlers::TimestampHandler(dbms_handler, req_reader, res_builder);
}); });
} }
void InMemoryReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) { void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
replication::HeartbeatReq req; slk::Builder *res_builder) {
storage::replication::HeartbeatReq req;
slk::Load(&req, req_reader); slk::Load(&req, req_reader);
replication::HeartbeatRes res{true, storage_->replication_state_.last_commit_timestamp_.load(), auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
storage_->replication_state_.GetEpoch().id}; if (!db_acc) return;
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
auto const *storage = db_acc->get()->storage();
storage::replication::HeartbeatRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load(),
std::string{storage->repl_storage_state_.epoch_.id()}};
slk::Save(res, res_builder); slk::Save(res, res_builder);
} }
void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) { void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
replication::AppendDeltasReq req; slk::Builder *res_builder) {
storage::replication::AppendDeltasReq req;
slk::Load(&req, req_reader); slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::Decoder decoder(req_reader); storage::replication::Decoder decoder(req_reader);
auto maybe_epoch_id = decoder.ReadString(); auto maybe_epoch_id = decoder.ReadString();
MG_ASSERT(maybe_epoch_id, "Invalid replication message"); MG_ASSERT(maybe_epoch_id, "Invalid replication message");
if (*maybe_epoch_id != storage_->replication_state_.GetEpoch().id) { auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
storage_->replication_state_.AppendEpoch(*maybe_epoch_id); auto &repl_storage_state = storage->repl_storage_state_;
if (*maybe_epoch_id != storage->repl_storage_state_.epoch_.id()) {
auto prev_epoch = storage->repl_storage_state_.epoch_.SetEpoch(*maybe_epoch_id);
repl_storage_state.AddEpochToHistoryForce(prev_epoch);
} }
if (storage_->wal_file_) { if (storage->wal_file_) {
if (req.seq_num > storage_->wal_file_->SequenceNumber() || if (req.seq_num > storage->wal_file_->SequenceNumber() ||
*maybe_epoch_id != storage_->replication_state_.GetEpoch().id) { *maybe_epoch_id != storage->repl_storage_state_.epoch_.id()) {
storage_->wal_file_->FinalizeWal(); storage->wal_file_->FinalizeWal();
storage_->wal_file_.reset(); storage->wal_file_.reset();
storage_->wal_seq_num_ = req.seq_num; storage->wal_seq_num_ = req.seq_num;
spdlog::trace("Finalized WAL file"); spdlog::trace("Finalized WAL file");
} else { } else {
MG_ASSERT(storage_->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file"); MG_ASSERT(storage->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
storage_->wal_seq_num_ = req.seq_num + 1; storage->wal_seq_num_ = req.seq_num + 1;
} }
} else { } else {
storage_->wal_seq_num_ = req.seq_num; storage->wal_seq_num_ = req.seq_num;
} }
if (req.previous_commit_timestamp != storage_->replication_state_.last_commit_timestamp_.load()) { if (req.previous_commit_timestamp != repl_storage_state.last_commit_timestamp_.load()) {
// Empty the stream // Empty the stream
bool transaction_complete = false; bool transaction_complete = false;
while (!transaction_complete) { while (!transaction_complete) {
SPDLOG_INFO("Skipping delta"); SPDLOG_INFO("Skipping delta");
const auto [timestamp, delta] = ReadDelta(&decoder); const auto [timestamp, delta] = ReadDelta(&decoder);
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd( transaction_complete = storage::durability::IsWalDeltaDataTypeTransactionEnd(
delta.type, durability::kVersion); // TODO: Check if we are always using the latest version when replicating delta.type,
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
} }
replication::AppendDeltasRes res{false, storage_->replication_state_.last_commit_timestamp_.load()}; storage::replication::AppendDeltasRes res{storage->id(), false, repl_storage_state.last_commit_timestamp_.load()};
slk::Save(res, res_builder); slk::Save(res, res_builder);
return; return;
} }
ReadAndApplyDelta(storage_, &decoder, ReadAndApplyDelta(
durability::kVersion); // TODO: Check if we are always using the latest version when replicating storage, &decoder,
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
replication::AppendDeltasRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; storage::replication::AppendDeltasRes res{storage->id(), true, repl_storage_state.last_commit_timestamp_.load()};
slk::Save(res, res_builder); slk::Save(res, res_builder);
spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!"); spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!");
} }
void InMemoryReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) { void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
replication::SnapshotReq req; slk::Builder *res_builder) {
storage::replication::SnapshotReq req;
slk::Load(&req, req_reader); slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::Decoder decoder(req_reader); storage::replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->snapshot_directory_); auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
utils::EnsureDirOrDie(storage->snapshot_directory_);
const auto maybe_snapshot_path = decoder.ReadFile(storage_->snapshot_directory_); const auto maybe_snapshot_path = decoder.ReadFile(storage->snapshot_directory_);
MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!"); MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!");
spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path); spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path);
auto storage_guard = std::unique_lock{storage_->main_lock_}; auto storage_guard = std::unique_lock{storage->main_lock_};
spdlog::trace("Clearing database since recovering from snapshot."); spdlog::trace("Clearing database since recovering from snapshot.");
// Clear the database // Clear the database
storage_->vertices_.clear(); storage->vertices_.clear();
storage_->edges_.clear(); storage->edges_.clear();
storage_->constraints_.existence_constraints_ = std::make_unique<ExistenceConstraints>(); storage->constraints_.existence_constraints_ = std::make_unique<storage::ExistenceConstraints>();
storage_->constraints_.unique_constraints_ = std::make_unique<InMemoryUniqueConstraints>(); storage->constraints_.unique_constraints_ = std::make_unique<storage::InMemoryUniqueConstraints>();
storage_->indices_.label_index_ = std::make_unique<InMemoryLabelIndex>(&storage_->indices_, storage_->config_); storage->indices_.label_index_ = std::make_unique<storage::InMemoryLabelIndex>();
storage_->indices_.label_property_index_ = storage->indices_.label_property_index_ = std::make_unique<storage::InMemoryLabelPropertyIndex>();
std::make_unique<InMemoryLabelPropertyIndex>(&storage_->indices_, storage_->config_);
try { try {
spdlog::debug("Loading snapshot"); spdlog::debug("Loading snapshot");
auto &epoch = auto recovered_snapshot = storage::durability::LoadSnapshot(
storage_->replication_state_ *maybe_snapshot_path, &storage->vertices_, &storage->edges_, &storage->repl_storage_state_.history,
.GetEpoch(); // This needs to be a non-const ref since we are updating it in LoadSnapshot TODO fix storage->name_id_mapper_.get(), &storage->edge_count_, storage->config_);
auto recovered_snapshot = durability::LoadSnapshot(
*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_, &storage_->replication_state_.history,
storage_->name_id_mapper_.get(), &storage_->edge_count_, storage_->config_);
spdlog::debug("Snapshot loaded successfully"); spdlog::debug("Snapshot loaded successfully");
// If this step is present it should always be the first step of // If this step is present it should always be the first step of
// the recovery so we use the UUID we read from snasphost // the recovery so we use the UUID we read from snasphost
storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid); storage->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
epoch.id = std::move(recovered_snapshot.snapshot_info.epoch_id); storage->repl_storage_state_.epoch_.SetEpoch(std::move(recovered_snapshot.snapshot_info.epoch_id));
const auto &recovery_info = recovered_snapshot.recovery_info; const auto &recovery_info = recovered_snapshot.recovery_info;
storage_->vertex_id_ = recovery_info.next_vertex_id; storage->vertex_id_ = recovery_info.next_vertex_id;
storage_->edge_id_ = recovery_info.next_edge_id; storage->edge_id_ = recovery_info.next_edge_id;
storage_->timestamp_ = std::max(storage_->timestamp_, recovery_info.next_timestamp); storage->timestamp_ = std::max(storage->timestamp_, recovery_info.next_timestamp);
spdlog::trace("Recovering indices and constraints from snapshot."); spdlog::trace("Recovering indices and constraints from snapshot.");
durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage_->indices_, storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_,
&storage_->constraints_, &storage_->vertices_); &storage->constraints_, &storage->vertices_);
} catch (const durability::RecoveryFailure &e) { } catch (const storage::durability::RecoveryFailure &e) {
LOG_FATAL("Couldn't load the snapshot because of: {}", e.what()); LOG_FATAL("Couldn't load the snapshot because of: {}", e.what());
} }
storage_guard.unlock(); storage_guard.unlock();
replication::SnapshotRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; storage::replication::SnapshotRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder); slk::Save(res, res_builder);
spdlog::trace("Deleting old snapshot files due to snapshot recovery."); spdlog::trace("Deleting old snapshot files due to snapshot recovery.");
// Delete other durability files // Delete other durability files
auto snapshot_files = durability::GetSnapshotFiles(storage_->snapshot_directory_, storage_->uuid_); auto snapshot_files = storage::durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_);
for (const auto &[path, uuid, _] : snapshot_files) { for (const auto &[path, uuid, _] : snapshot_files) {
if (path != *maybe_snapshot_path) { if (path != *maybe_snapshot_path) {
spdlog::trace("Deleting snapshot file {}", path); spdlog::trace("Deleting snapshot file {}", path);
storage_->file_retainer_.DeleteFile(path); storage->file_retainer_.DeleteFile(path);
} }
} }
spdlog::trace("Deleting old WAL files due to snapshot recovery."); spdlog::trace("Deleting old WAL files due to snapshot recovery.");
auto wal_files = durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_); auto wal_files = storage::durability::GetWalFiles(storage->wal_directory_, storage->uuid_);
if (wal_files) { if (wal_files) {
for (const auto &wal_file : *wal_files) { for (const auto &wal_file : *wal_files) {
spdlog::trace("Deleting WAL file {}", wal_file.path); spdlog::trace("Deleting WAL file {}", wal_file.path);
storage_->file_retainer_.DeleteFile(wal_file.path); storage->file_retainer_.DeleteFile(wal_file.path);
} }
storage_->wal_file_.reset(); storage->wal_file_.reset();
} }
spdlog::debug("Replication recovery from snapshot finished!"); spdlog::debug("Replication recovery from snapshot finished!");
} }
void InMemoryReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) { void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
replication::WalFilesReq req; slk::Builder *res_builder) {
storage::replication::WalFilesReq req;
slk::Load(&req, req_reader); slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
const auto wal_file_number = req.file_number; const auto wal_file_number = req.file_number;
spdlog::debug("Received WAL files: {}", wal_file_number); spdlog::debug("Received WAL files: {}", wal_file_number);
replication::Decoder decoder(req_reader); storage::replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->wal_directory_); auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
utils::EnsureDirOrDie(storage->wal_directory_);
for (auto i = 0; i < wal_file_number; ++i) { for (auto i = 0; i < wal_file_number; ++i) {
LoadWal(storage_, &decoder); LoadWal(storage, &decoder);
} }
replication::WalFilesRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; storage::replication::WalFilesRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder); slk::Save(res, res_builder);
spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!"); spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!");
} }
void InMemoryReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) { void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
replication::CurrentWalReq req; slk::Builder *res_builder) {
storage::replication::CurrentWalReq req;
slk::Load(&req, req_reader); slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::Decoder decoder(req_reader); storage::replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->wal_directory_); auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
utils::EnsureDirOrDie(storage->wal_directory_);
LoadWal(storage_, &decoder); LoadWal(storage, &decoder);
replication::CurrentWalRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; storage::replication::CurrentWalRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder); slk::Save(res, res_builder);
spdlog::debug("Replication recovery from current WAL ended successfully, replica is now up to date!"); spdlog::debug("Replication recovery from current WAL ended successfully, replica is now up to date!");
} }
void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, replication::Decoder *decoder) { void InMemoryReplicationHandlers::LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder) {
const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory; const auto temp_wal_directory =
std::filesystem::temp_directory_path() / "memgraph" / storage::durability::kWalDirectory;
utils::EnsureDir(temp_wal_directory); utils::EnsureDir(temp_wal_directory);
auto maybe_wal_path = decoder->ReadFile(temp_wal_directory); auto maybe_wal_path = decoder->ReadFile(temp_wal_directory);
MG_ASSERT(maybe_wal_path, "Failed to load WAL!"); MG_ASSERT(maybe_wal_path, "Failed to load WAL!");
spdlog::trace("Received WAL saved to {}", *maybe_wal_path); spdlog::trace("Received WAL saved to {}", *maybe_wal_path);
try { try {
auto wal_info = durability::ReadWalInfo(*maybe_wal_path); auto wal_info = storage::durability::ReadWalInfo(*maybe_wal_path);
if (wal_info.seq_num == 0) { if (wal_info.seq_num == 0) {
storage->uuid_ = wal_info.uuid; storage->uuid_ = wal_info.uuid;
} }
auto &replica_epoch = storage->repl_storage_state_.epoch_;
if (wal_info.epoch_id != storage->replication_state_.GetEpoch().id) { if (wal_info.epoch_id != replica_epoch.id()) {
storage->replication_state_.AppendEpoch(wal_info.epoch_id); auto prev_epoch = replica_epoch.SetEpoch(wal_info.epoch_id);
storage->repl_storage_state_.AddEpochToHistoryForce(prev_epoch);
} }
if (storage->wal_file_) { if (storage->wal_file_) {
@@ -259,11 +327,12 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, replication::D
storage->wal_seq_num_ = wal_info.seq_num; storage->wal_seq_num_ = wal_info.seq_num;
} }
spdlog::trace("Loading WAL deltas from {}", *maybe_wal_path); spdlog::trace("Loading WAL deltas from {}", *maybe_wal_path);
durability::Decoder wal; storage::durability::Decoder wal;
const auto version = wal.Initialize(*maybe_wal_path, durability::kWalMagic); const auto version = wal.Initialize(*maybe_wal_path, storage::durability::kWalMagic);
spdlog::debug("WAL file {} loaded successfully", *maybe_wal_path); spdlog::debug("WAL file {} loaded successfully", *maybe_wal_path);
if (!version) throw durability::RecoveryFailure("Couldn't read WAL magic and/or version!"); if (!version) throw storage::durability::RecoveryFailure("Couldn't read WAL magic and/or version!");
if (!durability::IsVersionSupported(*version)) throw durability::RecoveryFailure("Invalid WAL version!"); if (!storage::durability::IsVersionSupported(*version))
throw storage::durability::RecoveryFailure("Invalid WAL version!");
wal.SetPosition(wal_info.offset_deltas); wal.SetPosition(wal_info.offset_deltas);
for (size_t i = 0; i < wal_info.num_deltas;) { for (size_t i = 0; i < wal_info.num_deltas;) {
@@ -271,38 +340,46 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, replication::D
} }
spdlog::debug("Replication from current WAL successful!"); spdlog::debug("Replication from current WAL successful!");
} catch (const durability::RecoveryFailure &e) { } catch (const storage::durability::RecoveryFailure &e) {
LOG_FATAL("Couldn't recover WAL deltas from {} because of: {}", *maybe_wal_path, e.what()); LOG_FATAL("Couldn't recover WAL deltas from {} because of: {}", *maybe_wal_path, e.what());
} }
} }
void InMemoryReplicationServer::TimestampHandler(slk::Reader *req_reader, slk::Builder *res_builder) { void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
replication::TimestampReq req; slk::Builder *res_builder) {
storage::replication::TimestampReq req;
slk::Load(&req, req_reader); slk::Load(&req, req_reader);
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::TimestampRes res{true, storage_->replication_state_.last_commit_timestamp_.load()}; // TODO: this handler is agnostic of InMemory, move to be reused by on-disk
auto const *storage = db_acc->get()->storage();
storage::replication::TimestampRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder); slk::Save(res, res_builder);
} }
uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder, uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage *storage,
const uint64_t version) { storage::durability::BaseDecoder *decoder,
const uint64_t version) {
auto edge_acc = storage->edges_.access(); auto edge_acc = storage->edges_.access();
auto vertex_acc = storage->vertices_.access(); auto vertex_acc = storage->vertices_.access();
constexpr bool kUniqueAccess = true; constexpr bool kUniqueAccess = true;
constexpr bool kSharedAccess = false;
std::optional<std::pair<uint64_t, InMemoryStorage::ReplicationAccessor>> commit_timestamp_and_accessor; std::optional<std::pair<uint64_t, storage::InMemoryStorage::ReplicationAccessor>> commit_timestamp_and_accessor;
auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp, auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp,
bool unique = !kUniqueAccess) { bool unique = kSharedAccess) {
if (!commit_timestamp_and_accessor) { if (!commit_timestamp_and_accessor) {
std::unique_ptr<Storage::Accessor> acc = nullptr; std::unique_ptr<storage::Storage::Accessor> acc = nullptr;
if (unique) { if (unique) {
acc = storage->UniqueAccess(std::nullopt); acc = storage->UniqueAccess(std::nullopt, false /*not main*/);
} else { } else {
acc = storage->Access(std::nullopt); acc = storage->Access(std::nullopt, false /*not main*/);
} }
auto inmem_acc = std::unique_ptr<InMemoryStorage::InMemoryAccessor>( auto inmem_acc = std::unique_ptr<storage::InMemoryStorage::InMemoryAccessor>(
static_cast<InMemoryStorage::InMemoryAccessor *>(acc.release())); static_cast<storage::InMemoryStorage::InMemoryAccessor *>(acc.release()));
commit_timestamp_and_accessor.emplace(commit_timestamp, std::move(*inmem_acc)); commit_timestamp_and_accessor.emplace(commit_timestamp, std::move(*inmem_acc));
} else if (commit_timestamp_and_accessor->first != commit_timestamp) { } else if (commit_timestamp_and_accessor->first != commit_timestamp) {
throw utils::BasicException("Received more than one transaction!"); throw utils::BasicException("Received more than one transaction!");
@@ -311,7 +388,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
}; };
uint64_t applied_deltas = 0; uint64_t applied_deltas = 0;
auto max_commit_timestamp = storage->replication_state_.last_commit_timestamp_.load(); auto max_commit_timestamp = storage->repl_storage_state_.last_commit_timestamp_.load();
for (bool transaction_complete = false; !transaction_complete; ++applied_deltas) { for (bool transaction_complete = false; !transaction_complete; ++applied_deltas) {
const auto [timestamp, delta] = ReadDelta(decoder); const auto [timestamp, delta] = ReadDelta(decoder);
@@ -319,7 +396,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
max_commit_timestamp = timestamp; max_commit_timestamp = timestamp;
} }
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version); transaction_complete = storage::durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version);
if (timestamp < storage->timestamp_) { if (timestamp < storage->timestamp_) {
continue; continue;
@@ -327,13 +404,13 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
SPDLOG_INFO(" Delta {}", applied_deltas); SPDLOG_INFO(" Delta {}", applied_deltas);
switch (delta.type) { switch (delta.type) {
case durability::WalDeltaData::Type::VERTEX_CREATE: { case WalDeltaData::Type::VERTEX_CREATE: {
spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint()); spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint());
auto *transaction = get_transaction(timestamp); auto *transaction = get_transaction(timestamp);
transaction->CreateVertexEx(delta.vertex_create_delete.gid); transaction->CreateVertexEx(delta.vertex_create_delete.gid);
break; break;
} }
case durability::WalDeltaData::Type::VERTEX_DELETE: { case WalDeltaData::Type::VERTEX_DELETE: {
spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint()); spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint());
auto *transaction = get_transaction(timestamp); auto *transaction = get_transaction(timestamp);
auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW); auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW);
@@ -342,7 +419,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: { case WalDeltaData::Type::VERTEX_ADD_LABEL: {
spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(), spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(),
delta.vertex_add_remove_label.label); delta.vertex_add_remove_label.label);
auto *transaction = get_transaction(timestamp); auto *transaction = get_transaction(timestamp);
@@ -352,7 +429,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: { case WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(), spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(),
delta.vertex_add_remove_label.label); delta.vertex_add_remove_label.label);
auto *transaction = get_transaction(timestamp); auto *transaction = get_transaction(timestamp);
@@ -362,7 +439,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: { case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(), spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value); delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
auto *transaction = get_transaction(timestamp); auto *transaction = get_transaction(timestamp);
@@ -373,7 +450,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::EDGE_CREATE: { case WalDeltaData::Type::EDGE_CREATE: {
spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}", spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}",
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type, delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint()); delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
@@ -388,7 +465,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (edge.HasError()) throw utils::BasicException("Invalid transaction!"); if (edge.HasError()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::EDGE_DELETE: { case WalDeltaData::Type::EDGE_DELETE: {
spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}", spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}",
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type, delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint()); delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
@@ -406,7 +483,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: { case WalDeltaData::Type::EDGE_SET_PROPERTY: {
spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(), spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value); delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
if (!storage->config_.items.properties_on_edges) if (!storage->config_.items.properties_on_edges)
@@ -460,14 +537,8 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
// type and invalid from/to pointers because we don't know them // type and invalid from/to pointers because we don't know them
// here, but that isn't an issue because we won't use that part of // here, but that isn't an issue because we won't use that part of
// the API here. // the API here.
auto ea = EdgeAccessor{edge_ref, auto ea = EdgeAccessor{edge_ref, EdgeTypeId::FromUint(0UL), nullptr, nullptr,
EdgeTypeId::FromUint(0UL), storage, &transaction->GetTransaction()};
nullptr,
nullptr,
&transaction->GetTransaction(),
&storage->indices_,
&storage->constraints_,
storage->config_.items};
auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property), auto ret = ea.SetProperty(transaction->NameToProperty(delta.vertex_edge_set_property.property),
delta.vertex_edge_set_property.value); delta.vertex_edge_set_property.value);
@@ -475,17 +546,18 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
break; break;
} }
case durability::WalDeltaData::Type::TRANSACTION_END: { case WalDeltaData::Type::TRANSACTION_END: {
spdlog::trace(" Transaction end"); spdlog::trace(" Transaction end");
if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp) if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
throw utils::BasicException("Invalid commit data!"); throw utils::BasicException("Invalid commit data!");
auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first); auto ret =
commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first, false /* not main */);
if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
commit_timestamp_and_accessor = std::nullopt; commit_timestamp_and_accessor = std::nullopt;
break; break;
} }
case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: { case WalDeltaData::Type::LABEL_INDEX_CREATE: {
spdlog::trace(" Create label index on :{}", delta.operation_label.label); spdlog::trace(" Create label index on :{}", delta.operation_label.label);
// Need to send the timestamp // Need to send the timestamp
auto *transaction = get_transaction(timestamp, kUniqueAccess); auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -493,14 +565,14 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!"); throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::LABEL_INDEX_DROP: { case WalDeltaData::Type::LABEL_INDEX_DROP: {
spdlog::trace(" Drop label index on :{}", delta.operation_label.label); spdlog::trace(" Drop label index on :{}", delta.operation_label.label);
auto *transaction = get_transaction(timestamp, kUniqueAccess); auto *transaction = get_transaction(timestamp, kUniqueAccess);
if (transaction->DropIndex(storage->NameToLabel(delta.operation_label.label)).HasError()) if (transaction->DropIndex(storage->NameToLabel(delta.operation_label.label)).HasError())
throw utils::BasicException("Invalid transaction!"); throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::LABEL_INDEX_STATS_SET: { case WalDeltaData::Type::LABEL_INDEX_STATS_SET: {
spdlog::trace(" Set label index statistics on :{}", delta.operation_label_stats.label); spdlog::trace(" Set label index statistics on :{}", delta.operation_label_stats.label);
// Need to send the timestamp // Need to send the timestamp
auto *transaction = get_transaction(timestamp); auto *transaction = get_transaction(timestamp);
@@ -512,7 +584,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->SetIndexStats(label, stats); transaction->SetIndexStats(label, stats);
break; break;
} }
case durability::WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: { case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: {
const auto &info = delta.operation_label; const auto &info = delta.operation_label;
spdlog::trace(" Clear label index statistics on :{}", info.label); spdlog::trace(" Clear label index statistics on :{}", info.label);
// Need to send the timestamp // Need to send the timestamp
@@ -520,7 +592,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->DeleteLabelIndexStats(storage->NameToLabel(info.label)); transaction->DeleteLabelIndexStats(storage->NameToLabel(info.label));
break; break;
} }
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: { case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label, spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property); delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess); auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -531,7 +603,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!"); throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: { case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label, spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property); delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess); auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -542,7 +614,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!"); throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: { case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: {
const auto &info = delta.operation_label_property_stats; const auto &info = delta.operation_label_property_stats;
spdlog::trace(" Set label-property index statistics on :{}", info.label); spdlog::trace(" Set label-property index statistics on :{}", info.label);
// Need to send the timestamp // Need to send the timestamp
@@ -556,7 +628,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->SetIndexStats(label, property, stats); transaction->SetIndexStats(label, property, stats);
break; break;
} }
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: { case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: {
const auto &info = delta.operation_label; const auto &info = delta.operation_label;
spdlog::trace(" Clear label-property index statistics on :{}", info.label); spdlog::trace(" Clear label-property index statistics on :{}", info.label);
// Need to send the timestamp // Need to send the timestamp
@@ -564,7 +636,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label)); transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label));
break; break;
} }
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: { case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label, spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property); delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess); auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -574,7 +646,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError()) throw utils::BasicException("Invalid transaction!"); if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: { case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label, spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property); delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess); auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -585,7 +657,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!"); throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: { case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
std::stringstream ss; std::stringstream ss;
utils::PrintIterable(ss, delta.operation_label_properties.properties); utils::PrintIterable(ss, delta.operation_label_properties.properties);
spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str()); spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
@@ -600,7 +672,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!"); throw utils::BasicException("Invalid transaction!");
break; break;
} }
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: { case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
std::stringstream ss; std::stringstream ss;
utils::PrintIterable(ss, delta.operation_label_properties.properties); utils::PrintIterable(ss, delta.operation_label_properties.properties);
spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str()); spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
@@ -621,10 +693,10 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (commit_timestamp_and_accessor) throw utils::BasicException("Did not finish the transaction!"); if (commit_timestamp_and_accessor) throw utils::BasicException("Did not finish the transaction!");
storage->replication_state_.last_commit_timestamp_ = max_commit_timestamp; storage->repl_storage_state_.last_commit_timestamp_ = max_commit_timestamp;
spdlog::debug("Applied {} deltas", applied_deltas); spdlog::debug("Applied {} deltas", applied_deltas);
return applied_deltas; return applied_deltas;
} }
} // namespace memgraph::storage } // namespace memgraph::dbms

View File

@@ -0,0 +1,49 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "replication/replication_server.hpp"
#include "replication/state.hpp"
#include "storage/v2/replication/serialization.hpp"
namespace memgraph::storage {
class InMemoryStorage;
}
namespace memgraph::dbms {
class DbmsHandler;
class InMemoryReplicationHandlers {
public:
static void Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server);
private:
// RPC handlers
static void HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder);
static uint64_t ReadAndApplyDelta(storage::InMemoryStorage *storage, storage::durability::BaseDecoder *decoder,
uint64_t version);
};
} // namespace memgraph::dbms

View File

@@ -0,0 +1,67 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <variant>
#include "dbms/constants.hpp"
#include "dbms/replication_handler.hpp"
#include "replication/state.hpp"
#include "storage/v2/config.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/storage.hpp"
namespace memgraph::dbms {
#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY
constexpr bool allow_mt_repl = true;
#else
constexpr bool allow_mt_repl = false;
#endif
inline std::unique_ptr<storage::Storage> CreateInMemoryStorage(
storage::Config config, const ::memgraph::replication::ReplicationState &repl_state) {
const auto wal_mode = config.durability.snapshot_wal_mode;
const auto name = config.name;
auto storage = std::make_unique<storage::InMemoryStorage>(std::move(config));
// Connect replication state and storage
storage->CreateSnapshotHandler(
[storage = storage.get(), &repl_state]() -> utils::BasicResult<storage::InMemoryStorage::CreateSnapshotError> {
if (repl_state.IsReplica()) {
return storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica;
}
return storage->CreateSnapshot();
});
if (allow_mt_repl || name == dbms::kDefaultDB) {
// Handle global replication state
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
// RECOVER REPLICA CONNECTIONS
memgraph::dbms::RestoreReplication(repl_state, *storage);
} else if (const ::memgraph::replication::RoleMainData *data =
std::get_if<::memgraph::replication::RoleMainData>(&repl_state.ReplicationData());
data && !data->registered_replicas_.empty()) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
if (wal_mode == storage::Config::Durability::SnapshotWalMode::DISABLED && repl_state.IsMain()) {
spdlog::warn(
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider "
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
"without write-ahead logs this instance is not replicating any data.");
}
return std::move(storage);
}
} // namespace memgraph::dbms

View File

@@ -0,0 +1,234 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "dbms/replication_handler.hpp"
#include "dbms/constants.hpp"
#include "dbms/dbms_handler.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "dbms/inmemory/storage_helper.hpp"
#include "replication/state.hpp"
using memgraph::replication::ReplicationClientConfig;
using memgraph::replication::ReplicationState;
using memgraph::replication::RoleMainData;
using memgraph::replication::RoleReplicaData;
namespace memgraph::dbms {
namespace {
std::string RegisterReplicaErrorToString(RegisterReplicaError error) {
switch (error) {
using enum RegisterReplicaError;
case NAME_EXISTS:
return "NAME_EXISTS";
case END_POINT_EXISTS:
return "END_POINT_EXISTS";
case CONNECTION_FAILED:
return "CONNECTION_FAILED";
case COULD_NOT_BE_PERSISTED:
return "COULD_NOT_BE_PERSISTED";
}
}
} // namespace
bool ReplicationHandler::SetReplicationRoleMain() {
auto const main_handler = [](RoleMainData const &) {
// If we are already MAIN, we don't want to change anything
return false;
};
auto const replica_handler = [this](RoleReplicaData const &) {
// STEP 1) bring down all REPLICA servers
dbms_handler_.ForEach([](Database *db) {
auto *storage = db->storage();
// Remember old epoch + storage timestamp association
storage->PrepareForNewEpoch();
});
// STEP 2) Change to MAIN
// TODO: restore replication servers if false?
if (!repl_state_.SetReplicationRoleMain()) {
// TODO: Handle recovery on failure???
return false;
}
// STEP 3) We are now MAIN, update storage local epoch
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
storage->repl_storage_state_.epoch_ = std::get<RoleMainData>(std::as_const(repl_state_).ReplicationData()).epoch_;
});
return true;
};
// TODO: under lock
return std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData());
}
bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) {
// We don't want to restart the server if we're already a REPLICA
if (repl_state_.IsReplica()) {
return false;
}
// Remove registered replicas
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); });
});
// Creates the server
repl_state_.SetReplicationRoleReplica(config);
// Start
const auto success =
std::visit(utils::Overloaded{[](auto) {
// ASSERT
return false;
},
[this](RoleReplicaData const &data) {
// Register handlers
InMemoryReplicationHandlers::Register(&dbms_handler_, *data.server);
if (!data.server->Start()) {
spdlog::error("Unable to start the replication server.");
return false;
}
return true;
}},
repl_state_.ReplicationData());
// TODO Handle error (restore to main?)
return success;
}
auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
-> memgraph::utils::BasicResult<RegisterReplicaError> {
MG_ASSERT(repl_state_.IsMain(), "Only main instance can register a replica!");
auto res = repl_state_.RegisterReplica(config);
switch (res) {
case memgraph::replication::RegisterReplicaError::NOT_MAIN:
MG_ASSERT(false, "Only main instance can register a replica!");
return {};
case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
return memgraph::dbms::RegisterReplicaError::NAME_EXISTS;
case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS:
return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS;
case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
case memgraph::replication::RegisterReplicaError::SUCCESS:
break;
}
bool all_clients_good = true;
if (!allow_mt_repl && dbms_handler_.All().size() > 1) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
if (!allow_mt_repl && storage->id() != kDefaultDB) {
return;
}
// TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
all_clients_good &=
storage->repl_storage_state_.replication_clients_.WithLock([storage, &config](auto &clients) -> bool {
auto client = storage->CreateReplicationClient(config, &storage->repl_storage_state_.epoch_);
client->Start();
if (client->State() == storage::replication::ReplicaState::INVALID) {
return false;
}
clients.push_back(std::move(client));
return true;
});
});
if (!all_clients_good) return RegisterReplicaError::CONNECTION_FAILED; // TODO: this happen to 1 or many...what to do
return {};
}
auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterReplicaResult {
auto const replica_handler = [](RoleReplicaData const &) -> UnregisterReplicaResult {
return UnregisterReplicaResult::NOT_MAIN;
};
auto const main_handler = [this, name](RoleMainData &mainData) -> UnregisterReplicaResult {
if (!repl_state_.TryPersistUnregisterReplica(name)) {
return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED;
}
auto const n_unregistered =
std::erase_if(mainData.registered_replicas_,
[&](ReplicationClientConfig const &registered_config) { return registered_config.name == name; });
dbms_handler_.ForEach([&](Database *db) {
db->storage()->repl_storage_state_.replication_clients_.WithLock(
[&](auto &clients) { std::erase_if(clients, [&](const auto &client) { return client->Name() == name; }); });
});
return n_unregistered != 0 ? UnregisterReplicaResult::SUCCESS : UnregisterReplicaResult::CAN_NOT_UNREGISTER;
};
return std::visit(utils::Overloaded{main_handler, replica_handler}, repl_state_.ReplicationData());
}
auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole { return repl_state_.GetRole(); }
bool ReplicationHandler::IsMain() const { return repl_state_.IsMain(); }
bool ReplicationHandler::IsReplica() const { return repl_state_.IsReplica(); }
void RestoreReplication(const replication::ReplicationState &repl_state, storage::Storage &storage) {
spdlog::info("Restoring replication role.");
/// MAIN
auto const recover_main = [&storage](RoleMainData const &mainData) {
for (const auto &config : mainData.registered_replicas_) {
spdlog::info("Replica {} restoration started for {}.", config.name, storage.id());
auto register_replica = [&storage](const memgraph::replication::ReplicationClientConfig &config)
-> memgraph::utils::BasicResult<RegisterReplicaError> {
return storage.repl_storage_state_.replication_clients_.WithLock(
[&storage, &config](auto &clients) -> utils::BasicResult<RegisterReplicaError> {
auto client = storage.CreateReplicationClient(config, &storage.repl_storage_state_.epoch_);
client->Start();
if (client->State() == storage::replication::ReplicaState::INVALID) {
spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.",
client->Name());
}
clients.push_back(std::move(client));
return {};
});
};
auto ret = register_replica(config);
if (ret.HasError()) {
MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError());
LOG_FATAL("Failure when restoring replica {}: {}.", config.name, RegisterReplicaErrorToString(ret.GetError()));
}
spdlog::info("Replica {} restored for {}.", config.name, storage.id());
}
spdlog::info("Replication role restored to MAIN.");
};
/// REPLICA
auto const recover_replica = [](RoleReplicaData const &data) { /*nothing to do*/ };
std::visit(
utils::Overloaded{
recover_main,
recover_replica,
},
std::as_const(repl_state).ReplicationData());
}
} // namespace memgraph::dbms

View File

@@ -0,0 +1,69 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "replication/role.hpp"
#include "storage/v2/storage.hpp"
#include "utils/result.hpp"
// BEGIN fwd declares
namespace memgraph::replication {
struct ReplicationState;
struct ReplicationServerConfig;
struct ReplicationClientConfig;
} // namespace memgraph::replication
namespace memgraph::dbms {
class DbmsHandler;
enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
enum class UnregisterReplicaResult : uint8_t {
NOT_MAIN,
COULD_NOT_BE_PERSISTED,
CAN_NOT_UNREGISTER,
SUCCESS,
};
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
/// TODO: extend to do multiple storages
struct ReplicationHandler {
ReplicationHandler(memgraph::replication::ReplicationState &replState, DbmsHandler &dbms_handler)
: repl_state_(replState), dbms_handler_(dbms_handler) {}
// as REPLICA, become MAIN
bool SetReplicationRoleMain();
// as MAIN, become REPLICA
bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config);
// as MAIN, define and connect to REPLICAs
auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
-> utils::BasicResult<RegisterReplicaError>;
// as MAIN, remove a REPLICA connection
auto UnregisterReplica(std::string_view name) -> UnregisterReplicaResult;
// Helper pass-through (TODO: remove)
auto GetRole() const -> memgraph::replication::ReplicationRole;
bool IsMain() const;
bool IsReplica() const;
private:
memgraph::replication::ReplicationState &repl_state_;
DbmsHandler &dbms_handler_;
};
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
/// TODO: extend to do multiple storages
void RestoreReplication(const replication::ReplicationState &repl_state, storage::Storage &storage);
} // namespace memgraph::dbms

View File

@@ -5,6 +5,7 @@ add_library(mg-flags STATIC audit.cpp
log_level.cpp log_level.cpp
memory_limit.cpp memory_limit.cpp
run_time_configurable.cpp run_time_configurable.cpp
storage_mode.cpp) storage_mode.cpp
query.cpp)
target_include_directories(mg-flags PUBLIC ${CMAKE_SOURCE_DIR}/include) target_include_directories(mg-flags PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-flags PUBLIC spdlog::spdlog mg-settings mg-utils) target_link_libraries(mg-flags PUBLIC spdlog::spdlog mg-settings mg-utils)

View File

@@ -16,5 +16,6 @@
#include "flags/isolation_level.hpp" #include "flags/isolation_level.hpp"
#include "flags/log_level.hpp" #include "flags/log_level.hpp"
#include "flags/memory_limit.hpp" #include "flags/memory_limit.hpp"
#include "flags/query.hpp"
#include "flags/run_time_configurable.hpp" #include "flags/run_time_configurable.hpp"
#include "flags/storage_mode.hpp" #include "flags/storage_mode.hpp"

View File

@@ -36,7 +36,3 @@ DEFINE_VALIDATED_int32(bolt_session_inactivity_timeout, 1800,
DEFINE_string(bolt_cert_file, "", "Certificate file which should be used for the Bolt server."); DEFINE_string(bolt_cert_file, "", "Certificate file which should be used for the Bolt server.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(bolt_key_file, "", "Key file which should be used for the Bolt server."); DEFINE_string(bolt_key_file, "", "Key file which should be used for the Bolt server.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(bolt_server_name_for_init, "",
"Server name which the database should send to the client in the "
"Bolt INIT message.");

View File

@@ -26,4 +26,4 @@ DECLARE_string(bolt_cert_file);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(bolt_key_file); DECLARE_string(bolt_key_file);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(bolt_server_name_for_init); // DECLARE_string(bolt_server_name_for_init); Moved to run_time_configurable

View File

@@ -145,11 +145,6 @@ DEFINE_string(pulsar_service_url, "", "Default URL used while connecting to Puls
// Query flags. // Query flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_double(query_execution_timeout_sec, -1,
"Maximum allowed query execution time. Queries exceeding this "
"limit will be aborted. Value of 0 means no limit.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(replication_replica_check_frequency_sec, 1, DEFINE_uint64(replication_replica_check_frequency_sec, 1,
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: " "The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "

View File

@@ -100,7 +100,7 @@ DECLARE_string(pulsar_service_url);
// Query flags. // Query flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_double(query_execution_timeout_sec); // DECLARE_double(query_execution_timeout_sec); Moved to run_time_configurable
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_string(query_modules_directory); DECLARE_string(query_modules_directory);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)

View File

@@ -26,7 +26,6 @@ using namespace std::string_view_literals;
// Logging flags // Logging flags
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(also_log_to_stderr, false, "Log messages go to stderr in addition to logfiles");
DEFINE_string(log_file, "", "Path to where the log should be stored."); DEFINE_string(log_file, "", "Path to where the log should be stored.");
inline constexpr std::array log_level_mappings{ inline constexpr std::array log_level_mappings{
@@ -34,11 +33,8 @@ inline constexpr std::array log_level_mappings{
std::pair{"INFO"sv, spdlog::level::info}, std::pair{"WARNING"sv, spdlog::level::warn}, std::pair{"INFO"sv, spdlog::level::info}, std::pair{"WARNING"sv, spdlog::level::warn},
std::pair{"ERROR"sv, spdlog::level::err}, std::pair{"CRITICAL"sv, spdlog::level::critical}}; std::pair{"ERROR"sv, spdlog::level::err}, std::pair{"CRITICAL"sv, spdlog::level::critical}};
const std::string log_level_help_string = fmt::format("Minimum log level. Allowed values: {}", const std::string memgraph::flags::log_level_help_string = fmt::format(
memgraph::utils::GetAllowedEnumValuesString(log_level_mappings)); "Minimum log level. Allowed values: {}", memgraph::utils::GetAllowedEnumValuesString(log_level_mappings));
DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(),
{ return memgraph::flags::ValidLogLevel(value); });
bool memgraph::flags::ValidLogLevel(std::string_view value) { bool memgraph::flags::ValidLogLevel(std::string_view value) {
if (const auto result = memgraph::utils::IsValidEnumValueString(value, log_level_mappings); result.HasError()) { if (const auto result = memgraph::utils::IsValidEnumValueString(value, log_level_mappings); result.HasError()) {
@@ -65,7 +61,9 @@ std::optional<spdlog::level::level_enum> memgraph::flags::LogLevelToEnum(std::st
} }
spdlog::level::level_enum ParseLogLevel() { spdlog::level::level_enum ParseLogLevel() {
const auto log_level = memgraph::flags::LogLevelToEnum(FLAGS_log_level); std::string ll;
gflags::GetCommandLineOption("log_level", &ll);
const auto log_level = memgraph::flags::LogLevelToEnum(ll);
MG_ASSERT(log_level, "Invalid log level"); MG_ASSERT(log_level, "Invalid log level");
return *log_level; return *log_level;
} }
@@ -77,10 +75,6 @@ void CreateLoggerFromSink(const auto &sinks, const auto log_level) {
logger->set_level(log_level); logger->set_level(log_level);
logger->flush_on(spdlog::level::trace); logger->flush_on(spdlog::level::trace);
spdlog::set_default_logger(std::move(logger)); spdlog::set_default_logger(std::move(logger));
// Enable stderr sink
if (FLAGS_also_log_to_stderr) {
memgraph::flags::LogToStderr(log_level);
}
} }
void memgraph::flags::InitializeLogger() { void memgraph::flags::InitializeLogger() {
@@ -117,6 +111,14 @@ void memgraph::flags::AddLoggerSink(spdlog::sink_ptr new_sink) {
// NOTE: default_logger is not thread-safe and shouldn't be changed during application lifetime // NOTE: default_logger is not thread-safe and shouldn't be changed during application lifetime
void memgraph::flags::LogToStderr(spdlog::level::level_enum log_level) { void memgraph::flags::LogToStderr(spdlog::level::level_enum log_level) {
auto default_logger = spdlog::default_logger(); auto default_logger = spdlog::default_logger();
auto sink = default_logger->sinks().front(); auto stderr = default_logger->sinks().front();
sink->set_level(log_level); stderr->set_level(log_level);
}
void memgraph::flags::UpdateStderr(spdlog::level::level_enum log_level) {
auto default_logger = spdlog::default_logger();
auto stderr = default_logger->sinks().front();
if (stderr->level() != spdlog::level::off) {
stderr->set_level(log_level);
}
} }

View File

@@ -14,16 +14,19 @@
#include <optional> #include <optional>
#include "gflags/gflags.h" #include "gflags/gflags.h"
DECLARE_string(log_level); // DECLARE_string(log_level); Moved to run_time_configurable
DECLARE_bool(also_log_to_stderr); // DECLARE_bool(also_log_to_stderr); Moved to run_time_configurable
namespace memgraph::flags { namespace memgraph::flags {
extern const std::string log_level_help_string;
bool ValidLogLevel(std::string_view value); bool ValidLogLevel(std::string_view value);
std::optional<spdlog::level::level_enum> LogLevelToEnum(std::string_view value); std::optional<spdlog::level::level_enum> LogLevelToEnum(std::string_view value);
void InitializeLogger(); void InitializeLogger();
void AddLoggerSink(spdlog::sink_ptr new_sink); void AddLoggerSink(spdlog::sink_ptr new_sink);
void LogToStderr(spdlog::level::level_enum log_level); void LogToStderr(spdlog::level::level_enum log_level);
void UpdateStderr(spdlog::level::level_enum log_level);
} // namespace memgraph::flags } // namespace memgraph::flags

14
src/flags/query.cpp Normal file
View File

@@ -0,0 +1,14 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "flags/query.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
// DEFINE_bool(cartesian_product_enabled, true, "Enable cartesian product expansion."); Moved to run_time_configurable

16
src/flags/query.hpp Normal file
View File

@@ -0,0 +1,16 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "gflags/gflags.h"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
// DECLARE_bool(cartesian_product_enabled); Moved to run_time_configurable

View File

@@ -10,102 +10,193 @@
// licenses/APL.txt. // licenses/APL.txt.
#include "flags/run_time_configurable.hpp" #include "flags/run_time_configurable.hpp"
#include <string> #include <string>
#include <tuple>
#include "gflags/gflags.h"
#include "flags/bolt.hpp" #include "flags/bolt.hpp"
#include "flags/general.hpp" #include "flags/general.hpp"
#include "flags/log_level.hpp" #include "flags/log_level.hpp"
#include "flags/query.hpp"
#include "spdlog/cfg/helpers-inl.h" #include "spdlog/cfg/helpers-inl.h"
#include "spdlog/spdlog.h" #include "spdlog/spdlog.h"
#include "utils/exceptions.hpp" #include "utils/exceptions.hpp"
#include "utils/flag_validation.hpp"
#include "utils/settings.hpp" #include "utils/settings.hpp"
#include "utils/string.hpp" #include "utils/string.hpp"
/*
* Setup GFlags
*/
// Bolt server flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(bolt_server_name_for_init, "Neo4j/v5.11.0 compatible graph database server - Memgraph",
"Server name which the database should send to the client in the "
"Bolt INIT message.");
// Logging flags
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(also_log_to_stderr, false, "Log messages go to stderr in addition to logfiles");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
DEFINE_VALIDATED_string(log_level, "WARNING", memgraph::flags::log_level_help_string.c_str(),
{ return memgraph::flags::ValidLogLevel(value); });
// Query flags
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_double(query_execution_timeout_sec, 600,
"Maximum allowed query execution time. Queries exceeding this "
"limit will be aborted. Value of 0 means no limit.");
// Query plan flags
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(cartesian_product_enabled, true, "Enable cartesian product expansion.");
namespace { namespace {
// Bolt server name // Bolt server name
constexpr auto kServerNameSettingKey = "server.name"; constexpr auto kServerNameSettingKey = "server.name";
constexpr auto kDefaultServerName = "Neo4j/v5.11.0 compatible graph database server - Memgraph"; constexpr auto kServerNameGFlagsKey = "bolt_server_name_for_init";
// Query timeout // Query timeout
constexpr auto kQueryTxSettingKey = "query.timeout"; constexpr auto kQueryTxSettingKey = "query.timeout";
constexpr auto kDefaultQueryTx = "600"; // seconds constexpr auto kQueryTxGFlagsKey = "query_execution_timeout_sec";
// Log level // Log level
// No default value because it is not persistent // No default value because it is not persistent
constexpr auto kLogLevelSettingKey = "log.level"; constexpr auto kLogLevelSettingKey = "log.level";
constexpr auto kLogLevelGFlagsKey = "log_level";
// Log to stderr // Log to stderr
// No default value because it is not persistent // No default value because it is not persistent
constexpr auto kLogToStderrSettingKey = "log.to_stderr"; constexpr auto kLogToStderrSettingKey = "log.to_stderr";
constexpr auto kLogToStderrGFlagsKey = "also_log_to_stderr";
constexpr auto kCartesianProductEnabledSettingKey = "cartesian-product-enabled";
constexpr auto kCartesianProductEnabledGFlagsKey = "cartesian-product-enabled";
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
std::atomic<double> execution_timeout_sec_; // Local cache-like thing
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
std::atomic<bool> cartesian_product_enabled_{true}; // Local cache-like thing
auto ToLLEnum(std::string_view val) {
const auto ll_enum = memgraph::flags::LogLevelToEnum(val);
if (!ll_enum) {
throw memgraph::utils::BasicException("Unsupported log level {}", val);
}
return *ll_enum;
}
bool ValidBoolStr(std::string_view in) {
const auto lc = memgraph::utils::ToLowerCase(in);
return lc == "false" || lc == "true";
}
auto GenHandler(std::string flag, std::string key) {
return [key = std::move(key), flag = std::move(flag)]() -> std::string {
const auto &val = memgraph::utils::global_settings.GetValue(key);
MG_ASSERT(val, "Failed to read value at '{}' from settings.", key);
gflags::SetCommandLineOption(flag.c_str(), val->c_str());
return *val;
};
}
} // namespace } // namespace
namespace memgraph::flags::run_time { namespace memgraph::flags::run_time {
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
memgraph::utils::Synchronized<std::string, memgraph::utils::SpinLock> bolt_server_name_;
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
std::atomic<double> execution_timeout_sec_;
void Initialize() { void Initialize() {
// Register bolt server name settings constexpr bool kRestore = true; //!< run-time flag is persistent between Memgraph restarts
memgraph::utils::global_settings.RegisterSetting(kServerNameSettingKey, kDefaultServerName, [&] {
const auto server_name = memgraph::utils::global_settings.GetValue(kServerNameSettingKey);
MG_ASSERT(server_name, "Bolt server name is missing from the settings");
*(bolt_server_name_.Lock()) = *server_name;
});
// Update value from read settings
const auto &name = memgraph::utils::global_settings.GetValue(kServerNameSettingKey);
MG_ASSERT(name, "Failed to read server name from settings.");
*(bolt_server_name_.Lock()) = *name;
// Override server name if passed via command line argument
if (!FLAGS_bolt_server_name_for_init.empty()) {
memgraph::utils::global_settings.SetValue(kServerNameSettingKey, FLAGS_bolt_server_name_for_init);
}
// Register query timeout /**
memgraph::utils::global_settings.RegisterSetting(kQueryTxSettingKey, kDefaultQueryTx, [&] { * @brief Helper function that registers a run-time flag
const auto query_tx = memgraph::utils::global_settings.GetValue(kQueryTxSettingKey); *
MG_ASSERT(query_tx, "Query timeout is missing from the settings"); * @param flag - GFlag name
execution_timeout_sec_ = std::stod(*query_tx); * @param key - Settings key used to store the flag
}); * @param restore - true if the flag is persistent between restarts
// Update value from read settings * @param post_update - user defined callback executed post flag update
const auto &tx = memgraph::utils::global_settings.GetValue(kQueryTxSettingKey); * @param validator - user defined value correctness checker
MG_ASSERT(tx, "Failed to read query timeout from settings."); */
execution_timeout_sec_ = std::stod(*tx); auto register_flag = [&](
// Override query timeout if passed via command line argument const std::string &flag, const std::string &key, bool restore,
if (FLAGS_query_execution_timeout_sec != -1) { std::function<void(const std::string &)> post_update = [](auto) {},
memgraph::utils::global_settings.SetValue(kQueryTxSettingKey, std::to_string(FLAGS_query_execution_timeout_sec)); std::function<bool(std::string_view)> validator = [](std::string_view) { return true; }) {
} // Get flag info
gflags::CommandLineFlagInfo info;
gflags::GetCommandLineFlagInfo(flag.c_str(), &info);
// Register setting
auto update = GenHandler(flag, key);
memgraph::utils::global_settings.RegisterSetting(
key, info.default_value,
[update, post_update = std::move(post_update)] {
const auto &val = update();
post_update(val);
},
validator);
// Register log level if (restore && info.is_default) {
auto get_global_log_level = []() { // No input from the user, restore persistent value from settings
const auto log_level = memgraph::utils::global_settings.GetValue(kLogLevelSettingKey); update();
MG_ASSERT(log_level, "Log level is missing from the settings"); } else {
const auto ll_enum = memgraph::flags::LogLevelToEnum(*log_level); // Override with current value - user defined a new value or the run-time flag is not persistent between starts
if (!ll_enum) { memgraph::utils::global_settings.SetValue(key, info.current_value);
throw utils::BasicException("Unsupported log level {}", *log_level);
} }
return *ll_enum;
}; };
memgraph::utils::global_settings.RegisterSetting(
kLogLevelSettingKey, FLAGS_log_level, [&] { spdlog::set_level(get_global_log_level()); },
memgraph::flags::ValidLogLevel);
// Always override log level with command line argument
memgraph::utils::global_settings.SetValue(kLogLevelSettingKey, FLAGS_log_level);
// Register logging to stderr /*
auto bool_to_str = [](bool in) { return in ? "true" : "false"; }; * Register bolt server name settings
const std::string log_to_stderr_s = bool_to_str(FLAGS_also_log_to_stderr); */
memgraph::utils::global_settings.RegisterSetting( register_flag(kServerNameGFlagsKey, kServerNameSettingKey, kRestore);
kLogToStderrSettingKey, log_to_stderr_s,
[&] { /*
const auto enable = memgraph::utils::global_settings.GetValue(kLogToStderrSettingKey); * Register query timeout
if (enable == "true") { */
LogToStderr(get_global_log_level()); register_flag(kQueryTxGFlagsKey, kQueryTxSettingKey, !kRestore, [&](const std::string &val) {
execution_timeout_sec_ = std::stod(val); // Cache for faster reads
});
/*
* Register log level
*/
register_flag(
kLogLevelGFlagsKey, kLogLevelSettingKey, !kRestore,
[](const std::string &val) {
const auto ll_enum = ToLLEnum(val);
spdlog::set_level(ll_enum);
UpdateStderr(ll_enum); // Updates level if active
},
memgraph::flags::ValidLogLevel);
/*
* Register logging to stderr
*/
register_flag(
kLogToStderrGFlagsKey, kLogToStderrSettingKey, !kRestore,
[](const std::string &val) {
if (val == "true") {
// No need to check if ll_val exists, we got here, so the log_level must exist already
const auto &ll_val = memgraph::utils::global_settings.GetValue(kLogLevelSettingKey);
LogToStderr(ToLLEnum(*ll_val));
} else { } else {
LogToStderr(spdlog::level::off); LogToStderr(spdlog::level::off);
} }
}, },
[](std::string_view in) { ValidBoolStr);
const auto lc = memgraph::utils::ToLowerCase(in);
return lc == "false" || lc == "true"; register_flag(
}); kCartesianProductEnabledGFlagsKey, kCartesianProductEnabledSettingKey, !kRestore,
// Always override log to stderr with command line argument [](const std::string &val) { cartesian_product_enabled_ = val == "true"; }, ValidBoolStr);
memgraph::utils::global_settings.SetValue(kLogToStderrSettingKey, log_to_stderr_s);
} }
std::string GetServerName() {
std::string s;
// Thread safe read of gflag
gflags::GetCommandLineOption(kServerNameGFlagsKey, &s);
return s;
}
double GetExecutionTimeout() { return execution_timeout_sec_; }
bool GetCartesianProductEnabled() { return cartesian_product_enabled_; }
} // namespace memgraph::flags::run_time } // namespace memgraph::flags::run_time

View File

@@ -15,12 +15,31 @@
namespace memgraph::flags::run_time { namespace memgraph::flags::run_time {
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) /**
extern utils::Synchronized<std::string, utils::SpinLock> bolt_server_name_; * @brief Initialize the run-time flags (must be done before run-time flags are used).
*
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) */
extern std::atomic<double> execution_timeout_sec_;
void Initialize(); void Initialize();
/**
* @brief Get the bolt server name value
*
* @return std::string
*/
std::string GetServerName();
/**
* @brief Get the query execution timeout value
*
* @return double
*/
double GetExecutionTimeout();
/**
* @brief Get the cartesian product enabled value
*
* @return bool
*/
bool GetCartesianProductEnabled();
} // namespace memgraph::flags::run_time } // namespace memgraph::flags::run_time

View File

@@ -7,5 +7,4 @@ target_sources(mg-glue PRIVATE auth.cpp
ServerT.cpp ServerT.cpp
MonitoringServerT.cpp MonitoringServerT.cpp
run_id.cpp) run_id.cpp)
target_link_libraries(mg-glue mg-query mg-auth mg-audit) target_link_libraries(mg-glue mg-query mg-auth mg-audit mg-flags)
target_precompile_headers(mg-glue INTERFACE auth_checker.hpp auth_handler.hpp)

View File

@@ -22,6 +22,7 @@
#include "license/license.hpp" #include "license/license.hpp"
#include "query/discard_value_stream.hpp" #include "query/discard_value_stream.hpp"
#include "query/interpreter_context.hpp" #include "query/interpreter_context.hpp"
#include "utils/event_map.hpp"
#include "utils/spin_lock.hpp" #include "utils/spin_lock.hpp"
namespace memgraph::metrics { namespace memgraph::metrics {
@@ -121,8 +122,8 @@ std::string SessionHL::GetCurrentDB() const {
} }
std::optional<std::string> SessionHL::GetServerNameForInit() { std::optional<std::string> SessionHL::GetServerNameForInit() {
auto locked_name = flags::run_time::bolt_server_name_.Lock(); const auto &name = flags::run_time::GetServerName();
return locked_name->empty() ? std::nullopt : std::make_optional(*locked_name); return name.empty() ? std::nullopt : std::make_optional(name);
} }
bool SessionHL::Authenticate(const std::string &username, const std::string &password) { bool SessionHL::Authenticate(const std::string &username, const std::string &password) {
@@ -155,6 +156,8 @@ std::map<std::string, memgraph::communication::bolt::Value> SessionHL::Discard(s
memgraph::query::DiscardValueResultStream stream; memgraph::query::DiscardValueResultStream stream;
return DecodeSummary(interpreter_.Pull(&stream, n, qid)); return DecodeSummary(interpreter_.Pull(&stream, n, qid));
} catch (const memgraph::query::QueryException &e) { } catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
// Wrap QueryException into ClientError, because we want to allow the // Wrap QueryException into ClientError, because we want to allow the
// client to fix their query. // client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what()); throw memgraph::communication::bolt::ClientError(e.what());
@@ -169,9 +172,16 @@ std::map<std::string, memgraph::communication::bolt::Value> SessionHL::Pull(Sess
TypedValueResultStream<TEncoder> stream(encoder, db->storage()); TypedValueResultStream<TEncoder> stream(encoder, db->storage());
return DecodeSummary(interpreter_.Pull(&stream, n, qid)); return DecodeSummary(interpreter_.Pull(&stream, n, qid));
} catch (const memgraph::query::QueryException &e) { } catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
// Wrap QueryException into ClientError, because we want to allow the // Wrap QueryException into ClientError, because we want to allow the
// client to fix their query. // client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what()); throw memgraph::communication::bolt::ClientError(e.what());
} catch (const utils::BasicException &) {
// Exceptions inheriting from BasicException will result in a TransientError
// i. e. client will be encouraged to retry execution because it
// could succeed if executed again.
throw;
} }
} }
@@ -212,10 +222,14 @@ std::pair<std::vector<std::string>, std::optional<int>> SessionHL::Interpret(
return {std::move(result.headers), result.qid}; return {std::move(result.headers), result.qid};
} catch (const memgraph::query::QueryException &e) { } catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
// Wrap QueryException into ClientError, because we want to allow the // Wrap QueryException into ClientError, because we want to allow the
// client to fix their query. // client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what()); throw memgraph::communication::bolt::ClientError(e.what());
} catch (const memgraph::query::ReplicationException &e) { } catch (const memgraph::query::ReplicationException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
throw memgraph::communication::bolt::ClientError(e.what()); throw memgraph::communication::bolt::ClientError(e.what());
} }
} }

View File

@@ -11,6 +11,8 @@
#pragma once #pragma once
#include <string>
namespace memgraph::glue { namespace memgraph::glue {
extern const std::string run_id_; extern const std::string run_id_;
} // namespace memgraph::glue } // namespace memgraph::glue

Some files were not shown because too many files have changed in this diff Show More