Compare commits

...

51 Commits

Author SHA1 Message Date
antonio2368
999b3ef79f Fix memory tracking issues (#150) 2021-05-18 18:42:19 +02:00
antonio2368
30413a7b4f Ignore carriage return at the end of line for a CSV file (#151) 2021-05-18 13:44:29 +02:00
Marko Budiselić
1def0c9104 Fix libs/setup.sh nocached download (#148) 2021-05-13 12:05:36 +02:00
Marko Budiselić
782c377f5d Add benchmarking GHA Diff job (#147) 2021-05-12 17:17:57 +02:00
Marko Budiselić
cc27a04139 Add bench-graph-client inside tools (#139) 2021-05-07 15:54:09 +02:00
Marko Budiselić
b71345655f Add balancing of GHA jobs across runners (#146) 2021-05-07 13:37:45 +02:00
antonio2368
ccdd58b336 Fix parsing nested complex types (#142)
* Fix parsing of types for mgp.List
2021-04-27 08:31:18 +02:00
antonio2368
50b6afd73d Fix clang tidy diff (#141)
* Ignore header files for clang-tidy-diff

* Rename code analysis job
2021-04-21 16:26:13 +02:00
Marko Budiselić
59105f68bd Add primary and secondary dependency links (#137)
* Add explicit exit in case of git checkout or cherry-pick
2021-04-20 10:54:54 +02:00
antonio2368
8de31092ad Add back the query memory limit logic (#134) 2021-04-13 10:41:50 +02:00
antonio2368
5c93f81881 Disable failing tests and add logs for replication e2e (#132)
* Disable sequential test

* Remove parent build and benchmark

* Save test data

* Save e2e logs in build folder

* Define different recovery time for each test
2021-04-02 12:29:10 +02:00
János Benjamin Antal
6d4fe5cdd5 Explore and implement the usage of clang-tidy and sanitizers (#125)
* Run clang-tidy on the full code base

* Run clang-tidy on diffs

* Enable ASAN in coverage build

* Add UBSAN to code analysis
2021-04-01 17:08:40 +02:00
Marko Budiselić
7b5263d300 Fix release package script (#131) 2021-03-30 12:48:45 +02:00
Marko Budiselić
e8a41e4457 Add automated build of all Memgraph packages (#123) 2021-03-30 12:08:51 +02:00
antonio2368
27f09e1c0a Make LOAD CSV query part separator (#130) 2021-03-29 16:40:55 +02:00
Josip Seljan
6dd9d32721 Fix scoped profile operator names for ScanAll operators (#129) 2021-03-28 09:27:02 +02:00
antonio2368
276e09d7d3 Fix reading rows with empty columns at the end (#127)
* Fix reading rows with empty columns at the end

* Update CHANGELOG for the recovery logs
2021-03-27 09:47:41 +01:00
Marko Budiselić
92dfc93b20 Update community license (#128) 2021-03-26 18:19:48 +01:00
János Benjamin Antal
06f761bdf9 Add logs for loading snapshot and WAL files (#121)
* Add logs for loading snapshot and WAL files
2021-03-26 15:02:35 +01:00
antonio2368
50ddd59450 Fix Centos8 build (#126)
* Increase server wait

* Use standard shebang for python scripts
2021-03-26 12:05:58 +01:00
antonio2368
60da033010 Use maximum number of threads for test (#124) 2021-03-25 08:19:00 +01:00
antonio2368
ee555b0c0d Link jemalloc with Threads lib (#122) 2021-03-24 15:21:18 +01:00
antonio2368
e8e4cd7f97 Memory control e2e test (#115)
* Add memory control e2e test

* Fix cmake for jemalloc
2021-03-24 15:21:18 +01:00
antonio2368
ad4c80af13 Add memory limit clause for cypher queries (#106)
* Add memory limit clause for cypher queries

* Add PROCEDURE and QUERY keywords

* Improve memory limit logs

* Update CHANGELOG
2021-03-24 15:21:11 +01:00
antonio2368
9c6bf4b1b8 Updated storage info (#102)
* Refactor readable size function

* Log set memory limit

* Add memory allocation info to storage

* Updated changelog
2021-03-24 15:10:28 +01:00
antonio2368
cc56ac3dd8 Expose query for cleaning memory and add memory limit flag (#100)
* Add memory flag, add additional meminfo utilities

* Add free memory query
2021-03-24 15:10:28 +01:00
antonio2368
dee885d69c Add exception enablers and blockers (#99)
* Throw OOMException while creating vertices and edges

* Throw on indices creation

* Throw on setting a property

* Throw oom exception while recovering

* Throw exception when query engine asks for extra memory

* Block out of memor exception during skip list GC
2021-03-24 14:53:15 +01:00
antonio2368
bbed7a2397 Add memory tracker definition (#93)
* Allow size 0 in MemoryTracker
* Block only exception throwing
* Subtract unsuccessfully allocated size in memory tracker
* Add oom exception enablers and blockers
2021-03-24 14:53:15 +01:00
antonio2368
e8810a4152 Add and use jemalloc lib (#90)
* Add and use jemalloc lib

* Add autoconf

* Silence macro redefinition warning
2021-03-24 14:53:15 +01:00
Josip Seljan
25eb2c147a LOAD CSV implementation fixes (#120)
* Change how csv::Reader handles memory resources
* Add multiline quoted string test
2021-03-24 12:21:07 +01:00
Josip Seljan
b3914c6b5d Update CHANGELOG.md (#117)
* Update CHANGELOG.md

Co-authored-by: jseljan <josip.seljan@memgraph.io>
2021-03-24 12:21:07 +01:00
antonio2368
d913a67e16 Add parsing improvements (#108)
* Add parsing improvements
2021-03-24 12:21:07 +01:00
Josip Seljan
f950a91732 Add LOAD CSV clause infrastructure (#101)
* Add LOAD CSV clause infrastructure
* Add LoadCsv operator
* Update csv::Reader class
* Support csv files with and without header

Co-authored-by: jseljan <josip.seljan@memgraph.io>
2021-03-24 12:21:07 +01:00
Josip Seljan
f6d5f576d5 CSV reader class (#98)
* CSV parsing class
* Basic CSV parsing tests
2021-03-24 12:21:07 +01:00
antonio2368
dc5eb4befd Fix module reload (#114)
* Fix module reload
2021-03-22 09:37:35 +01:00
antonio2368
593f7a3499 Add query module counters to telemetry (#112) 2021-03-18 11:03:42 +01:00
Marko Budiselić
77a0d7b8fa Remove test_all_workers workflow (#111) 2021-03-16 17:01:59 +01:00
antonio2368
c240b5b564 Remove v1 driver test for python (#109) 2021-03-16 11:42:36 +01:00
Marko Budiselić
866ed45562 Add daily build for all major latest OSs (#110) 2021-03-16 09:13:50 +01:00
János Benjamin Antal
1598cb24ea Add directory argument to get_version.py (#107)
* Add root-dir option to get_version.py

* Make CMake callable outside from the repository
2021-03-15 11:55:39 +01:00
antonio2368
35d789c56b Allow multiple folders for query modules (#105)
* Allow multiple query modules directories as arg
2021-03-10 12:18:09 +01:00
antonio2368
16715d5005 Enable replication in community (#104)
* Enable replication in community
2021-03-10 10:36:38 +01:00
antonio2368
a9f5f45b3d Improve telemetry data (#89)
* Add query type count to telemetry

* Extract telemetry data in different structure

* Add global event counter
2021-03-09 19:55:58 +01:00
antonio2368
2e0dd19bac Add machine-id to telemetry if not docker (#103)
* Add machine-id to telemetry if not docker

* Improve telemetry integration test
2021-03-08 15:15:58 +01:00
antonio2368
f807b495ab Update commit log after recovery (#96)
* Define additional commit log constructor which takes an oldest active id

* Delay commit log construction until the recovery process is finished

* Add test for commit log with initial id

* Silence the macro redefinition warning
2021-02-19 11:00:10 +01:00
antonio2368
3f3c55a4aa Format all the memgraph and test source files (#97) 2021-02-18 15:32:43 +01:00
antonio2368
435af8b833 Remove clang tidy from hooks (#94)
* Removed clang-tidy from hooks

* Improved clang-format output

* Remove lcp from types to format
2021-02-16 11:22:59 +01:00
antonio2368
cc1c1513ef Increase column width (#91)
* Increase column width

* Prevent clang-format/tidy from triggering diff build
2021-02-12 16:58:55 +01:00
Marko Budiselić
fae407d3fe Add implementation of average number of equals estimate in SkipList (#9)
* Implement average number of equals estimate in SkipList
2021-02-10 14:38:54 +01:00
Marko Budiselić
42c245df8a Fix CentOS8 packaging (#87) 2021-01-29 14:48:55 +01:00
Marko Budiselić
e4852cc5e3 Migrate release docs to Notion (#86) 2021-01-28 16:29:00 +01:00
410 changed files with 18563 additions and 22937 deletions

View File

@@ -1,10 +1,10 @@
---
Language: Cpp
BasedOnStyle: Google
Standard: "C++11"
Standard: "c++20"
UseTab: Never
DerivePointerAlignment: false
PointerAlignment: Right
ColumnLimit : 80
ColumnLimit : 120
IncludeBlocks: Preserve
...

View File

@@ -54,7 +54,7 @@ Checks: '*,
-readability-magic-numbers,
-readability-named-parameter'
WarningsAsErrors: ''
HeaderFilterRegex: ''
HeaderFilterRegex: 'src/.*'
AnalyzeTemporaryDtors: false
FormatStyle: none
CheckOptions:

View File

@@ -15,7 +15,7 @@ exec 1>&2
tmpdir=$(mktemp -d repo-XXXXXXXX)
trap "rm -rf $tmpdir" EXIT INT
modified_files=$(git diff --cached --name-only --diff-filter=AM $against | sed -nE "/.*\.(cpp|cc|cxx|c|h|hpp|lcp)$/p")
modified_files=$(git diff --cached --name-only --diff-filter=AM $against | sed -nE "/.*\.(cpp|cc|cxx|c|h|hpp)$/p")
for file in $modified_files; do
echo "Checking $file..."
@@ -30,14 +30,6 @@ for file in $modified_files; do
if [ $code -ne 0 ]; then
break
fi
echo "Running clang-tidy..."
$project_folder/tools/git-clang-tidy $tmpdir/$file
code=$?
if [ $code -ne 0 ]; then
break
fi
done;
return $code

View File

@@ -5,11 +5,12 @@ on:
paths-ignore:
- 'docs/**'
- '**/*.md'
- '.clang-*'
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -41,7 +42,7 @@ jobs:
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run stress test
run: |
@@ -64,9 +65,9 @@ jobs:
name: "Community DEB package"
path: build/output/memgraph*.deb
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
code_analysis:
name: "Code analysis"
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -78,7 +79,7 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build coverage binaries
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
@@ -86,9 +87,8 @@ jobs:
# Initialize dependencies.
./init
# Build coverage binaries.
cd build
cmake -DTEST_COVERAGE=ON ..
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
- name: Run unit tests
@@ -96,9 +96,9 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run unit tests.
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
cd build
ctest -R memgraph__unit --output-on-failure
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
- name: Compute code coverage
run: |
@@ -119,9 +119,19 @@ jobs:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
debug_build:
name: "Debug build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -195,7 +205,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -207,21 +217,6 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Set up parent
run: |
# Remove parent folder (if it exists).
cd ..
if [ -d parent ]; then
rm -rf parent
fi
# Copy untouched repository to parent folder.
cp -r memgraph parent
# Checkout previous commit
cd parent
git checkout HEAD~1
- name: Build release binaries
run: |
# Activate toolchain.
@@ -235,47 +230,6 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Build parent binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
cd ../parent
./init
# Build parent binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph memgraph__macro_benchmark
- name: Run macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Run parent macro benchmark tests
run: |
cd ../parent/tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Compute macro benchmark summary
run: |
./tools/github/macro_benchmark_summary \
--current tests/macro_benchmark/.harness_summary \
--previous ../parent/tests/macro_benchmark/.harness_summary \
--output macro_benchmark_summary.txt
- name: Save macro benchmark summary
uses: actions/upload-artifact@v2
with:
name: "Macro benchmark summary"
path: macro_benchmark_summary.txt
- name: Run GQL Behave tests
run: |
cd tests/gql_behave
@@ -298,6 +252,14 @@ jobs:
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
- name: Run e2e memory control tests
run: |
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
- name: Run stress test (plain)
run: |
cd tests/stress
@@ -330,6 +292,15 @@ jobs:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
path: |
# multiple paths could be defined
build/logs
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
@@ -369,3 +340,64 @@ jobs:
with:
name: "Jepsen Report"
path: tests/jepsen/Jepsen.tar.gz
release_benchmarks:
name: "Release benchmarks"
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build only memgraph release binarie.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run macro benchmarks
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"

44
.github/workflows/full_clang_tidy.yaml vendored Normal file
View File

@@ -0,0 +1,44 @@
name: Run clang-tidy on the full codebase
on:
workflow_dispatch:
jobs:
clang_tidy_check:
name: "Clang-tidy check"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results
run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

248
.github/workflows/package_all.yaml vendored Normal file
View File

@@ -0,0 +1,248 @@
name: Package All
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
on: workflow_dispatch
jobs:
centos-7_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-7_community
path: build/output/centos-7/memgraph*.rpm
centos-8_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-8_community
path: build/output/centos-8/memgraph*.rpm
debian-9_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9_community
path: build/output/debian-9/memgraph*.deb
debian-10_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10_community
path: build/output/debian-10/memgraph*.deb
docker_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package community debian-10 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: docker_community
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804_community
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004_community
path: build/output/ubuntu-20.04/memgraph*.deb
centos-7_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-7_enterprise
path: build/output/centos-7/memgraph*.rpm
centos-8_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-8_enterprise
path: build/output/centos-8/memgraph*.rpm
debian-9_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9_enterprise
path: build/output/debian-9/memgraph*.deb
debian-10_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10_enterprise
path: build/output/debian-10/memgraph*.deb
docker_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package enterprise debian-10 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: docker_enterprise
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804_enterprise
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004_enterprise
path: build/output/ubuntu-20.04/memgraph*.deb

View File

@@ -1,11 +1,14 @@
name: Release CentOS
name: Release CentOS 8
on: [workflow_dispatch]
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, General, Linux, X64, CentOS7]
runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
timeout-minutes: 960
@@ -69,7 +72,7 @@ jobs:
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, General, Linux, X64, CentOS7]
runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
@@ -124,7 +127,7 @@ jobs:
debug_build:
name: "Debug build"
runs-on: [self-hosted, General, Linux, X64, CentOS7]
runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
@@ -198,7 +201,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, General, Linux, X64, CentOS7]
runs-on: [self-hosted, Linux, X64, CentOS8]
env:
THREADS: 24
timeout-minutes: 960
@@ -289,6 +292,14 @@ jobs:
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
- name: Run e2e memory control tests
run: |
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
- name: Run stress test (plain)
run: |
cd tests/stress

View File

@@ -1,4 +1,4 @@
name: Release
name: Release Debian 10
on:
workflow_dispatch:
@@ -8,7 +8,7 @@ on:
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
timeout-minutes: 960
@@ -71,7 +71,7 @@ jobs:
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
@@ -126,7 +126,7 @@ jobs:
debug_build:
name: "Debug build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
@@ -200,7 +200,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Debian10]
env:
THREADS: 24
timeout-minutes: 960
@@ -290,6 +290,14 @@ jobs:
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
- name: Run e2e memory control tests
run: |
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
- name: Run stress test (plain)
run: |
cd tests/stress

View File

@@ -0,0 +1,326 @@
name: Release Ubuntu 20.04
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
timeout-minutes: 960
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
make -j$THREADS
- name: Create community DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create community DEB package.
cd build
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
- name: Save community DEB package
uses: actions/upload-artifact@v2
with:
name: "Community DEB package"
path: build/output/memgraph*.deb
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure
- name: Run stress test (plain)
run: |
cd tests/stress
./continuous_integration
- name: Run stress test (large)
run: |
cd tests/stress
./continuous_integration --large-dataset
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build coverage binaries.
cd build
cmake -DTEST_COVERAGE=ON ..
make -j$THREADS memgraph__unit
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run unit tests.
cd build
ctest -R memgraph__unit --output-on-failure
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Compute code coverage.
cd tools/github
./coverage_convert
# Package code coverage.
cd generated
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
debug_build:
name: "Debug build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
ctest -E "(memgraph__unit|memgraph__benchmark)" --output-on-failure
- name: Run drivers tests
run: |
./tests/drivers/run.sh
- name: Run integration tests
run: |
cd tests/integration
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run cppcheck and clang-format.
cd tools/github
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
release_build:
name: "Release build"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
timeout-minutes: 960
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Create enterprise DEB package.
cd build
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run micro benchmark tests.
cd build
# The `eval` benchmark needs a large stack limit.
ulimit -s 262144
ctest -R memgraph__benchmark -V
- name: Run macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Run parallel macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QueryParallelSuite MemgraphRunner \
--groups aggregation_parallel create_parallel bfs_parallel \
--num-database-workers 9 --num-clients-workers 30 \
--no-strict
- name: Run GQL Behave tests
run: |
cd tests/gql_behave
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run e2e replication tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path replication/workloads.yaml
- name: Run e2e memory control tests
run: |
cd tests
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-path memory/workloads.yaml
- name: Run stress test (plain)
run: |
cd tests/stress
./continuous_integration
- name: Run stress test (SSL)
run: |
cd tests/stress
./continuous_integration --use-ssl
- name: Run stress test (large)
run: |
cd tests/stress
./continuous_integration --large-dataset
- name: Run durability test (plain)
run: |
cd tests/stress
source ve3/bin/activate
python3 durability --num-steps 5
- name: Run durability test (large)
run: |
cd tests/stress
source ve3/bin/activate
python3 durability --num-steps 20

View File

@@ -1,118 +0,0 @@
name: Test All Workers
on: [workflow_dispatch]
jobs:
HP-DL360G6-1:
name: "HP-DL360G6-1"
runs-on: [HP-DL360G6-1]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-2:
name: "HP-DL360G6-2"
runs-on: [HP-DL360G6-2]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-3:
name: "HP-DL360G6-3"
runs-on: [HP-DL360G6-3]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
docker --version
docker ps | grep jepsen
HP-DL360G6-v2-1:
name: "HP-DL360G6-v2-1"
runs-on: [HP-DL360G6-v2-1]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-v2-2:
name: "HP-DL360G6-v2-2"
runs-on: [HP-DL360G6-v2-2]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-v2-3:
name: "HP-DL360G6-v2-3"
runs-on: [HP-DL360G6-v2-3]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-v2-4:
name: "HP-DL360G6-v2-4"
runs-on: [HP-DL360G6-v2-4]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-v2-5:
name: "HP-DL360G6-v2-5"
runs-on: [HP-DL360G6-v2-5]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-v2-6:
name: "HP-DL360G6-v2-6"
runs-on: [HP-DL360G6-v2-6]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system
HP-DL360G6-v2-7:
name: "HP-DL360G6-v2-7"
runs-on: [HP-DL360G6-v2-7]
timeout-minutes: 10
steps:
- name: Set up repository
uses: actions/checkout@v2
- name: Check the system
run: |
source /opt/toolchain-v2/activate
./tools/check-build-system

View File

@@ -2,7 +2,45 @@
## Future
TODO: Don't forget to add items on the fly.
### Bug Fixes
* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
For example, parsing of `mgp.List[mgp.Map]` works now.
* Fixed memory tracking issues. Some of the allocation and deallocation weren't
tracked during the query execution.
* Fixed reading CSV files that are using CRLF as the newline symbol.
## v1.4.0
### Breaking Changes
* Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.
The functionality is still the same.
### Major Feature and Improvements
* Added replication to community version.
* Added support for multiple query modules directories at the same time.
You can now define multiple, comma-separated paths to directories from
which the modules will be loaded using the `--query-modules-directory` flag.
* Added support for programatically reading in data from CSV files through the
`LOAD CSV` clause. We support CSV files with and without a header, the
supported dialect being Excel.
* Added a new flag `--memory-limit` which enables the user to set the maximum total amount of memory
memgraph can allocate during its runtime.
* Added `FREE MEMORY` query which tries to free unusued memory chunks in different parts of storage.
* Added the memory limit and amount of currently allocated bytes in the result of `SHOW STORAGE INFO` query.
* Added `QUERY MEMORY LIMIT num (KB|MB)` to Cypher queries which allows you to limit memory allocation for
the entire query. It can be added only at the end of the entire Cypher query.
* Added logs for the different parts of the recovery process. `INFO`, `DEBUG` and `TRACE` level all contain
additional information that is printed out while the recovery is in progress.
### Bug Fixes
* Fixed garbage collector by correctly marking the oldest current timestamp
after the database was recovered using the durability files.
* Fixed reloading of the modules with changed result names.
* Fixed profile query to show the correct name of the ScanAll operator variant.
## v1.3.0

View File

@@ -35,6 +35,9 @@ else()
message(FATAL_ERROR "Couldn't find clang and/or clang++!")
endif()
option(BUILD_FOR_DOCKER "Build Memgraph binary for docker." OFF)
message(STATUS "BUILD_FOR_DOCKER: ${BUILD_FOR_DOCKER}")
# -----------------------------------------------------------------------------
project(memgraph)
@@ -71,6 +74,8 @@ execute_process(
COMMAND "${get_version_script}" ${get_version_enterprise}
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph version.")
@@ -86,6 +91,8 @@ execute_process(
--variant deb
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
@@ -101,6 +108,8 @@ execute_process(
--variant rpm
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
@@ -303,8 +312,9 @@ if (UBSAN)
# runtime library and c++ standard libraries are present.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
# Make sure llvm-symbolizer binary is in path
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
# Make sure llvm-symbolizer binary is in path.
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
endif()
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")

View File

@@ -6,8 +6,8 @@ data structures, multi-version concurrency control and asynchronous IO.
## Development Documentation
Please continue
[here](https://www.notion.so/memgraph/memgraph-0428591638604c8385550e214ea9f3e6).
Please continue in
[Notion](https://www.notion.so/memgraph/memgraph-0428591638604c8385550e214ea9f3e6).
## User Documentation

View File

@@ -83,6 +83,10 @@ modifications:
value: "/usr/lib/memgraph/auth_module/example.py"
override: false
- name: "memory_limit"
value: "0"
override: true
undocumented:
- "flag_file"
- "also_log_to_stderr"

View File

@@ -1,4 +1,4 @@
# Memgraph Build and Run Environments
Please continue
[here](https://www.notion.so/memgraph/Tools-05e0baafb78a49b386e0063b4833d23d).
Please continue in
[Notion](https://www.notion.so/memgraph/Tools-05e0baafb78a49b386e0063b4833d23d).

View File

@@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconfig # build system
@@ -46,10 +48,13 @@ MEMGRAPH_BUILD_DEPS=(
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
@@ -74,16 +79,13 @@ check() {
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
if [ "$SUDO_USER" == "" ]; then
echo "Please run as sudo."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -117,11 +119,16 @@ install() {
continue
fi
if [ "$pkg" == PyYAML ]; then
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
@@ -25,6 +26,7 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
@@ -45,10 +47,13 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
@@ -67,16 +72,13 @@ check() {
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
if [ "$SUDO_USER" == "" ]; then
echo "Please run as sudo."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -85,6 +87,7 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
dnf install -y epel-release
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --set-enabled powertools # Required to install texinfo.
dnf update -y
dnf install -y wget git python36 python3-pip
@@ -134,11 +137,16 @@ install() {
continue
fi
if [ "$pkg" == PyYAML ]; then
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -43,13 +45,17 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cat >/etc/apt/sources.list <<EOF
deb http://deb.debian.org/debian/ buster main non-free contrib
@@ -82,5 +88,6 @@ EOF
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -41,15 +43,20 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
install_all_apt "$1"
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -8,23 +8,29 @@ source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
pkg
)
TOOLCHAIN_RUN_DEPS=(
pkg
)
MEMGRAPH_BUILD_DEPS=(
pkg
)
list() {
echo "$1"
}
check() {
echo "TODO: Implement ${FUNCNAME[0]}."
exit 1
}
install() {
echo "TODO: Implement ${FUNCNAME[0]}."
exit 1
}
# http://ahmed.amayem.com/bash-indirect-expansion-exploration
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # llvm
libedit-dev libpcre3-dev automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -27,6 +28,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -42,15 +44,20 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
apt install -y $1
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -43,13 +45,17 @@ MEMGRAPH_BUILD_DEPS=(
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cd "$DIR"
apt update
@@ -74,5 +80,6 @@ install() {
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -683,7 +683,15 @@ def _typing_to_cypher_type(type_):
return _mgp.type_nullable(simple_type)
return _mgp.type_nullable(parse_typing(type_arg_as_str))
elif type_as_str.startswith('typing.List'):
type_arg_as_str, = parse_type_args(type_as_str)
type_arg_as_str = parse_type_args(type_as_str)
if len(type_arg_as_str) > 1:
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
# so we need to join the parts.
type_arg_as_str = ', '.join(type_arg_as_str)
else:
type_arg_as_str = type_arg_as_str[0]
simple_type = get_simple_type(type_arg_as_str)
if simple_type is not None:
return _mgp.type_list(simple_type)

1
libs/.gitignore vendored
View File

@@ -4,3 +4,4 @@
!cleanup.sh
!CMakeLists.txt
!__main.cpp
!jemalloc.cmake

View File

@@ -8,6 +8,8 @@ if (NPROC EQUAL 0)
set(NPROC 1)
endif()
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
# convenience functions
function(import_header_library name include_dir)
add_library(${name} INTERFACE IMPORTED GLOBAL)
@@ -212,3 +214,5 @@ import_external_library(spdlog STATIC
${CMAKE_CURRENT_SOURCE_DIR}/spdlog/${CMAKE_INSTALL_LIBDIR}/libspdlog.a
${CMAKE_CURRENT_SOURCE_DIR}/spdlog/include
BUILD_COMMAND $(MAKE) spdlog)
include(jemalloc.cmake)

55
libs/jemalloc.cmake Normal file
View File

@@ -0,0 +1,55 @@
set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
set(JEMALLOC_SRCS
${JEMALLOC_DIR}/src/arena.c
${JEMALLOC_DIR}/src/background_thread.c
${JEMALLOC_DIR}/src/base.c
${JEMALLOC_DIR}/src/bin.c
${JEMALLOC_DIR}/src/bitmap.c
${JEMALLOC_DIR}/src/ckh.c
${JEMALLOC_DIR}/src/ctl.c
${JEMALLOC_DIR}/src/div.c
${JEMALLOC_DIR}/src/extent.c
${JEMALLOC_DIR}/src/extent_dss.c
${JEMALLOC_DIR}/src/extent_mmap.c
${JEMALLOC_DIR}/src/hash.c
${JEMALLOC_DIR}/src/hook.c
${JEMALLOC_DIR}/src/jemalloc.c
${JEMALLOC_DIR}/src/large.c
${JEMALLOC_DIR}/src/log.c
${JEMALLOC_DIR}/src/malloc_io.c
${JEMALLOC_DIR}/src/mutex.c
${JEMALLOC_DIR}/src/mutex_pool.c
${JEMALLOC_DIR}/src/nstime.c
${JEMALLOC_DIR}/src/pages.c
${JEMALLOC_DIR}/src/prng.c
${JEMALLOC_DIR}/src/prof.c
${JEMALLOC_DIR}/src/rtree.c
${JEMALLOC_DIR}/src/sc.c
${JEMALLOC_DIR}/src/stats.c
${JEMALLOC_DIR}/src/sz.c
${JEMALLOC_DIR}/src/tcache.c
${JEMALLOC_DIR}/src/test_hooks.c
${JEMALLOC_DIR}/src/ticker.c
${JEMALLOC_DIR}/src/tsd.c
${JEMALLOC_DIR}/src/witness.c
${JEMALLOC_DIR}/src/safety_check.c
)
add_library(jemalloc ${JEMALLOC_SRCS})
target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
find_package(Threads REQUIRED)
target_link_libraries(jemalloc PUBLIC Threads::Threads)
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
endif()
target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
# for RTLD_NEXT
target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)

View File

@@ -2,8 +2,9 @@
# Download external dependencies.
local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd ${working_dir}
cd "${working_dir}"
# Clones a git repository and optionally cherry picks additional commits. The
# function will try to preserve any local changes in the repo.
@@ -15,7 +16,11 @@ clone () {
shift 3
# Clone if there's no repo.
if [[ ! -d "$dir_name" ]]; then
git clone "$git_repo" "$dir_name"
echo "Cloning from $git_repo"
# If the clone fails, it doesn't make sense to continue with the function
# execution but the whole script should continue executing because we might
# clone the same repo from a different source.
git clone "$git_repo" "$dir_name" || return 1
fi
pushd "$dir_name"
# Just fetch new commits from remote repository. Don't merge/pull them in, so
@@ -29,12 +34,17 @@ clone () {
# Stash regardless of local_changes, so that a user gets a message on stdout.
git stash
# Checkout the primary commit (there's no need to pull/merge).
git checkout $checkout_id
# The checkout fail should exit this script immediately because the target
# commit is not there and that will most likely create build-time errors.
git checkout "$checkout_id" || exit 1
# Apply any optional cherry pick fixes.
while [[ $# -ne 0 ]]; do
local cherry_pick_id=$1
shift
git cherry-pick -n $cherry_pick_id
# The cherry-pick fail should exit this script immediately because the
# target commit is not there and that will most likely create build-time
# errors.
git cherry-pick -n "$cherry_pick_id" || exit 1
done
# Reapply any local changes.
if [[ $local_changes == true ]]; then
@@ -43,12 +53,95 @@ clone () {
popd
}
file_get_try_double () {
primary_url="$1"
secondary_url="$2"
echo "Download primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
filename="$(basename "$secondary_url")"
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
echo ""
}
repo_clone_try_double () {
primary_url="$1"
secondary_url="$2"
folder_name="$3"
ref="$4"
echo "Cloning primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
echo ""
}
# List all dependencies.
# The reason for introducing primary and secondary urls are:
# * HTTPS is hard to cache
# * Remote development workflow is more flexible if people don't have to connect to VPN
# * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
# * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
# The goal of having primary urls is to have links to the "local" cache of
# dependencies where these dependencies could be downloaded as fast as
# possible. The actual cache server could be on your local machine, on a
# dedicated machine inside the build cluster or on the actual build machine.
# Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["fmt"]="http://$local_cache_host/git/fmt.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
["gtest"]="http://$local_cache_host/git/googletest.git"
["gflags"]="http://$local_cache_host/git/gflags.git"
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
["bzip2"]="http://$local_cache_host/git/bzip2.git"
["zlib"]="http://$local_cache_host/git/zlib.git"
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
["mgclient"]="http://$local_cache_host/git/mgclient.git"
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
)
# The goal of secondary urls is to have links to the "source of truth" of
# dependencies, e.g., Github or S3. Download from secondary urls, if happens
# at all, should never fail. In other words, if it fails, the whole build
# should fail.
declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["fmt"]="https://github.com/fmtlib/fmt.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
["gtest"]="https://github.com/google/googletest.git"
["gflags"]="https://github.com/memgraph/gflags.git"
["libbcrypt"]="https://github.com/rg3/libbcrypt"
["bzip2"]="https://github.com/VFR-maniac/bzip2"
["zlib"]="https://github.com/madler/zlib.git"
["rocksdb"]="https://github.com/facebook/rocksdb.git"
["mgclient"]="https://github.com/memgraph/mgclient.git"
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["spdlog"]="https://github.com/gabime/spdlog"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
)
# antlr
antlr_generator_filename="antlr-4.6-complete.jar"
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
# fix missing include
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
# remove shared library from install dependencies
@@ -56,68 +149,86 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
# fmt
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
# rapidcheck
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
# google benchmark
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
clone https://github.com/google/benchmark.git benchmark $benchmark_tag
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
# google test
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
clone https://github.com/google/googletest.git googletest $googletest_tag
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
# google flags
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
clone https://github.com/memgraph/gflags.git gflags $gflags_tag
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
# libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
# neo4j
wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
tar -xzf neo4j.tar.gz
rm -rf neo4j
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
tar -xzf neo4j-community-3.2.3-unix.tar.gz
mv neo4j-community-3.2.3 neo4j
rm neo4j.tar.gz
rm neo4j-community-3.2.3-unix.tar.gz
# nlohmann json
# We wget header instead of cloning repo since repo is huge (lots of test data).
# We use head on Sep 1, 2017 instead of last release since it was long time ago.
mkdir -p json
cd json
wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
clone https://github.com/madler/zlib.git zlib $zlib_tag
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
# remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
# remove shared library from install dependencies
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
# mgclient
mgclient_tag="v1.2.0" # (2021-01-14)
clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
# pymgclient
pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
clone https://github.com/gabime/spdlog spdlog $spdlog_tag
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
pushd jemalloc
# ThreadPool select job randomly, and there can be some threads that had been
# performed some memory heavy task before and will be inactive for some time,
# but until it will became active again, the memory will not be freed since by
# default each thread has it's own arena, but there should be not more then
# 4*CPU arenas (see opt.nareans description).
#
# By enabling percpu_arena number of arenas limited to number of CPUs and hence
# this problem should go away.
#
# muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
popd

View File

@@ -1,33 +1,40 @@
# User License Agreement
# Memgraph Community User License Agreement
1. Description
This License Agreement governs your use of the Memgraph Community Release (the
"Software") and documentation ("Documentation").
THIS LICENSE AGREEMENT GOVERNS LICENSEES USE OF THE MEMGRAPH COMMUNITY
RELEASE AND DOCUMENTATION.
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU ("LICENSEE") AGREE TO THESE
TERMS.
2. License Grant
1. License Grant
The Software and Documentation are provided to Licensee at no charge and are
licensed, not sold to Licensee. No ownership of any part of the Software and
Documentation is hereby transferred to Licensee. Subject to (i) the terms and
conditions of this License Agreement, (ii) any additional license restrictions
and parameters contained on Licensors quotation, website, or order form
(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
conditions of this License Agreement, and (ii) any additional license
restrictions and parameters contained on Licensors quotation, website, or
order form, Licensor hereby grants Licensee a personal, non-assignable,
non-transferable and non-exclusive license to install, access and use the
Software (in object code form only) and Documentation for Licensees internal
business purposes only. All rights relating to the Software and Documentation
that are not expressly licensed in this License Agreement, whether now existing
or which may hereafter come into existence are reserved for Licensor. Licensee
shall not remove, obscure, or alter any proprietary rights notices (including
without limitation copyright and trademark notices), which may be affixed to or
contained within the Software or Documentation.
business purposes (including for use in a production environment) only. All
rights relating to the Software and Documentation that are not expressly
licensed in this License Agreement, whether now existing or which may hereafter
come into existence are reserved for Licensor. Licensee shall not remove,
obscure, or alter any proprietary rights notices (including without limitation
copyright and trademark notices), which may be affixed to or contained within
the Software or Documentation.
3. Restrictions
Licensor may terminate this License Agreement with immediate effect upon
written notice to the Licensee. Upon termination Licensee shall delete all
electronic copies of all or any part of the Software and/or the Documentation
resident in its systems or elsewhere.
2. Restrictions
Licensee will not, directly or indirectly, (a) copy the Software or
Documentation in any manner or for any purpose; (b) install, access or use any
component of the Software or Documentation for any purpose not expressly
granted in Section 2 above; (c) resell, distribute, publicly display or
granted in Section 1 above; (c) resell, distribute, publicly display or
publicly perform the Software or Documentation or any component thereof, by
transfer, lease, loan or any other means, or make it available for use by
others in any time-sharing, service bureau or similar arrangement; (d)
@@ -37,25 +44,55 @@ algorithms or techniques incorporated in the Software; (e) export the Software
or Documentation in violation of any applicable laws or regulations; (f)
modify, translate, adapt, or create derivative works from the Software or
Documentation; (g) circumvent, disable or otherwise interfere with
security-related features of the Software or Documentation; (h)
reverse-engineer, disassemble, attempt to derive the source code; (i) use the
security-related features of the Software or Documentation; (h) use the
Software or Documentation for any illegal purpose, in any manner that is
inconsistent with the terms of this License Agreement, or to engage in illegal
activity; (j) remove or alter any trademark, logo, copyright or other
activity; (i) remove or alter any trademark, logo, copyright or other
proprietary notices, legends, symbols or labels on, or embedded in, the
Software or Documentation; or (k) provide access to the Software or
Software or Documentation; or (j) provide access to the Software or
Documentation to third parties.
4. Warranty Disclaimer
3. Warranty Disclaimer
THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED AS IS” FOR
DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
THIRD PARTIES INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" AND LICENSOR MAKES NO
WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
INFRINGEMENT OF THIRD PARTIES INTELLECTUAL PROPERTY RIGHTS OR OTHER
PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.
4. Limitation of Liability
Licensor shall not in any circumstances be liable, whether in tort (including
for negligence or breach of statutory duty howsoever arising), contract,
misrepresentation (whether innocent or negligent) or otherwise for: loss of
profits, loss of business, depletion of goodwill or similar losses, loss of
anticipated savings, loss of goods, loss or corruption of data or computer
downtime, or any special, indirect, consequential or pure economic loss, costs,
damages, charges or expenses.
Licensor's total aggregate liability in contract, tort (including without
limitation negligence or breach of statutory duty howsoever arising),
misrepresentation (whether innocent or negligent), restitution or otherwise,
arising in connection with the performance or contemplated performance of this
License Agreement shall in all circumstances be limited to GBP10.00 (ten pounds
sterling).
Nothing in this License Agreement shall limit Licensors liability in the case
of death or personal injury caused by negligence, fraud, or fraudulent
misrepresentation, or where it otherwise cannot be limited by law.
5. Technical Data
Licensor may collect and use technical information (such as usage patterns)
gathered when the Licensee downloads and uses the Software. This is generally
statistical data which does not identify an identified or identifiable
individual. It may also include Licensees IP address which is personal data
and is processed in accordance with our Privacy Policy. We only use this
technical information to improve our products.
6. Law and Jurisdiction
This License Agreement is governed by the laws of England and is subject to the
non-exclusive jurisdiction of the courts of England.

View File

@@ -1,43 +1,4 @@
# Memgraph Release Packaging
# Memgraph Release
Various tools and packaging configuration files should be put under this
directory. Common files for all packages can be kept in the root of this
directory, for example `memgraph.service`. If the common stuff should be
grouped, it should be in a subdirectory. `examples` directory is one such
case. Packaging specific stuff must have its own directory.
Currently we support distributing Memgraph binary through the following
packages.
* Debian package
* RPM package
* Docker image
* ArchLinux package
## Release process
While releasing an official version of Memgraph, there are two possible
scenarios:
* First release in new major.minor series
* Patch release in existing major.minor series
To release a new major.minor release of Memgraph you should execute the
following steps:
1. Merge all PRs that must be in the new release
2. Document all changes in `CHANGELOG.md` and merge them
3. From the `master` branch, create a branch named `release/X.Y` and push it
to `origin`
4. Create the release packages triggering a `Release {{Operating System}}`
workflow using branch `release/X.Y` on Github Actions
5. Enjoy
To release a new patch release in an existing major.minor series you should
execute the following steps:
1. Checkout to the `release/X.Y` branch
2. Cherry-pick all landed commits that should be included in the patch version
3. Document all changes in `CHANGELOG.md` and commit them
4. Edit the root `CMakeLists.txt` and set `MEMGRAPH_OVERRIDE_VERSION` to
`X.Y.patch` and commit the change
5. Create the release packages triggering a `Release {{Operating System}}`
workflow using branch `release/X.Y` on Github Actions
6. Enjoy
Please continue in
[Notion](https://www.notion.so/memgraph/Release-621733518c82456ebc119d2a025b5dd3).

View File

@@ -1,4 +1,5 @@
FROM debian:buster
# NOTE: If you change the base distro update release/package as well.
ARG deb_release

View File

@@ -1,4 +1,5 @@
FROM debian:buster
# NOTE: If you change the base distro update release/package as well.
ARG deb_release

View File

@@ -3,6 +3,7 @@ import argparse
import re
import subprocess
import sys
import os
# This script is used to determine the current version of Memgraph. The script
@@ -161,8 +162,17 @@ parser.add_argument(
parser.add_argument(
"--variant", choices=("binary", "deb", "rpm"), default="binary",
help="which variant of the version string should be generated")
parser.add_argument(
"--memgraph-root-dir", help="The root directory of the checked out "
"Memgraph repository.", default=".")
args = parser.parse_args()
if not os.path.isdir(args.memgraph_root_dir):
raise Exception("The root directory ({}) is not a valid directory".format(
args.memgraph_root_dir))
os.chdir(args.memgraph_root_dir)
offering = "enterprise" if args.enterprise else "community"
# Check whether the version was manually supplied.
@@ -182,7 +192,19 @@ if args.version:
try:
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
if current_branch != "master":
get_output("git", "fetch", "origin", "master:master")
branches = get_output("git", "branch")
if "master" in branches:
# If master is present locally, the fetch is allowed to fail
# because this script will still be able to compare against the
# master branch.
try:
get_output("git", "fetch", "origin", "master:master")
except Exception:
pass
else:
# If master is not present locally, the fetch command has to
# succeed because something else will fail otherwise.
get_output("git", "fetch", "origin", "master:master")
except Exception:
print("Fatal error while ensuring local master branch.")
sys.exit(1)

View File

@@ -0,0 +1,12 @@
FROM centos:7
RUN yum -y update \
&& yum install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-7.tar.gz \
-O toolchain-v2-binaries-centos-7.tar.gz \
&& tar xzvf toolchain-v2-binaries-centos-7.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,12 @@
FROM centos:8
RUN dnf -y update \
&& dnf install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-8.tar.gz \
-O toolchain-v2-binaries-centos-8.tar.gz \
&& tar xzvf toolchain-v2-binaries-centos-8.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,15 @@
FROM debian:10
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-10.tar.gz \
-O toolchain-v2-binaries-debian-10.tar.gz \
&& tar xzvf toolchain-v2-binaries-debian-10.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,15 @@
FROM debian:9
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-9.tar.gz \
-O toolchain-v2-binaries-debian-9.tar.gz \
&& tar xzvf toolchain-v2-binaries-debian-9.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,26 @@
version: "3"
services:
mgbuild_centos-7:
build:
context: centos-7
container_name: "mgbuild_centos-7"
mgbuild_centos-8:
build:
context: centos-8
container_name: "mgbuild_centos-8"
mgbuild_debian-9:
build:
context: debian-9
container_name: "mgbuild_debian-9"
mgbuild_debian-10:
build:
context: debian-10
container_name: "mgbuild_debian-10"
mgbuild_ubuntu-18.04:
build:
context: ubuntu-18.04
container_name: "mgbuild_ubuntu-18.04"
mgbuild_ubuntu-20.04:
build:
context: ubuntu-20.04
container_name: "mgbuild_ubuntu-20.04"

152
release/package/run.sh Executable file
View File

@@ -0,0 +1,152 @@
#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OFFERING=(community enterprise)
SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04)
PROJECT_ROOT="$SCRIPT_DIR/../.."
ACTIVATE_TOOLCHAIN="source /opt/toolchain-v2/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () {
echo "$0 init|package {offering} {os} [--for-docker]|docker|test"
echo ""
echo " offerings: ${SUPPORTED_OFFERING[*]}"
echo " OSs: ${SUPPORTED_OS[*]}"
exit 1
}
make_package () {
offering="$1"
offering_flag=" -DMG_ENTERPRISE=OFF "
if [[ "$offering" == "enterprise" ]]; then
offering_flag=" -DMG_ENTERPRISE=ON "
fi
if [[ "$offering" == "community" ]]; then
offering_flag=" -DMG_ENTERPRISE=OFF "
fi
os="$2"
package_command=""
if [[ "$os" =~ ^"centos".* ]]; then
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
docker_flag=" -DBUILD_FOR_DOCKER=OFF "
if [[ "$#" -gt 2 ]]; then
if [[ "$3" == "--for-docker" ]]; then
docker_flag=" -DBUILD_FOR_DOCKER=ON "
fi
fi
build_container="mgbuild_$os"
echo "Building Memgraph $offering for $os on $build_container..."
echo "Copying project files..."
# If master is not the current branch, fetch it, because the get_version
# script depends on it. If we are on master, the fetch command is going to
# fail so that's why there is the explicit check.
# Required here because Docker build container can't access remote.
cd "$PROJECT_ROOT"
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
git fetch origin master:master
fi
docker exec "$build_container" mkdir -p /memgraph
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
container_build_dir="/memgraph/build"
container_output_dir="$container_build_dir/output"
# TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
# image build phase, but that is not easy at this point because the
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
echo "Installing dependencies..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
docker exec "$build_container" bash -c "cd /memgraph && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $offering_flag $docker_flag .."
# ' is used instead of " because we need to run make within the allowed
# container resources.
# shellcheck disable=SC2016
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc)'
docker exec "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
echo "Copying targeted package to host..."
last_package_name=$(docker exec "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
# The operating system folder is introduced because multiple different
# packages could be preserved during the same build "session".
mkdir -p "$HOST_OUTPUT_DIR/$os"
package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
echo "Package saved to $package_host_destination."
}
case "$1" in
init)
cd "$SCRIPT_DIR"
docker-compose build
docker-compose up -d
;;
docker)
# NOTE: Docker is build on top of Debian 10 package.
based_on_os="debian-10"
# shellcheck disable=SC2012
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
# shellcheck disable=SC2012
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
docker_host_folder="$HOST_OUTPUT_DIR/docker"
docker_host_image_path="$docker_host_folder/$docker_image_name"
mkdir -p "$docker_host_folder"
cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
echo "Docker images saved to $docker_host_image_path."
;;
package)
shift 1
if [[ "$#" -lt 2 ]]; then
print_help
fi
offering="$1"
shift 1
is_offering_ok=false
for supported_offering in "${SUPPORTED_OFFERING[@]}"; do
if [[ "$supported_offering" == "${offering}" ]]; then
is_offering_ok=true
fi
done
os="$1"
shift 1
is_os_ok=false
for supported_os in "${SUPPORTED_OS[@]}"; do
if [[ "$supported_os" == "${os}" ]]; then
is_os_ok=true
fi
done
if [[ "$is_offering_ok" == true ]] && [[ "$is_os_ok" == true ]]; then
make_package "$offering" "$os" "$@"
else
print_help
fi
;;
test)
echo "TODO(gitbuda): Test all packages on mgtest containers."
;;
*)
print_help
;;
esac

View File

@@ -0,0 +1,15 @@
FROM ubuntu:18.04
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-18.04.tar.gz \
-O toolchain-v2-binaries-ubuntu-18.04.tar.gz \
&& tar xzvf toolchain-v2-binaries-ubuntu-18.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,15 @@
FROM ubuntu:20.04
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-20.04.tar.gz \
-O toolchain-v2-binaries-ubuntu-20.04.tar.gz \
&& tar xzvf toolchain-v2-binaries-ubuntu-20.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -30,6 +30,7 @@ BuildRequires: systemd
# This is needed to prevent Python compilation errors when building the RPM
# package
# https://github.com/scylladb/scylla/issues/2235
%if 0%{?rhel} < 8
%global __os_install_post \
/usr/lib/rpm/redhat/brp-compress \
%{!?__debug_package:\
@@ -39,6 +40,16 @@ BuildRequires: systemd
/usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
%{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \
%{nil}
%else
%global __os_install_post \
/usr/lib/rpm/brp-compress \
%{!?__debug_package:\
/usr/lib/rpm/brp-strip %{__strip} \
/usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump} \
} \
/usr/lib/rpm/brp-strip-static-archive %{__strip} \
%{nil}
%endif
%define _rpmdir %_topdir/RPMS
%define _srcrpmdir %_topdir/SRPMS

View File

@@ -30,6 +30,7 @@ BuildRequires: systemd
# This is needed to prevent Python compilation errors when building the RPM
# package
# https://github.com/scylladb/scylla/issues/2235
%if 0%{?rhel} < 8
%global __os_install_post \
/usr/lib/rpm/redhat/brp-compress \
%{!?__debug_package:\
@@ -39,6 +40,16 @@ BuildRequires: systemd
/usr/lib/rpm/redhat/brp-strip-static-archive %{__strip} \
%{!?__jar_repack:/usr/lib/rpm/redhat/brp-java-repack-jars} \
%{nil}
%else
%global __os_install_post \
/usr/lib/rpm/brp-compress \
%{!?__debug_package:\
/usr/lib/rpm/brp-strip %{__strip} \
/usr/lib/rpm/brp-strip-comment-note %{__strip} %{__objdump} \
} \
/usr/lib/rpm/brp-strip-static-archive %{__strip} \
%{nil}
%endif
%define _rpmdir %_topdir/RPMS
%define _srcrpmdir %_topdir/SRPMS

View File

@@ -10,11 +10,11 @@ add_subdirectory(telemetry)
add_subdirectory(communication)
add_subdirectory(storage/v2)
add_subdirectory(query)
add_subdirectory(slk)
add_subdirectory(rpc)
if (MG_ENTERPRISE)
add_subdirectory(audit)
add_subdirectory(auth)
add_subdirectory(slk)
add_subdirectory(rpc)
endif()
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -37,17 +37,17 @@ if (MG_ENTERPRISE)
glue/auth.cpp)
endif()
set(MG_SINGLE_NODE_V2_LIBS stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication mg-new-delete mg-utils)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(MG_SINGLE_NODE_V2_LIBS ${MG_SINGLE_NODE_V2_LIBS} mg-auth mg-audit)
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-auth mg-audit)
endif()
# memgraph main executable
add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph ${MG_SINGLE_NODE_V2_LIBS})
target_link_libraries(memgraph ${mg_single_node_v2_libs})
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the
# symbols in custom procedure module libraries.

View File

@@ -38,8 +38,7 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) {
case storage::PropertyValue::Type::Map: {
ret = nlohmann::json::object();
for (const auto &item : pv.ValueMap()) {
ret.push_back(nlohmann::json::object_t::value_type(
item.first, PropertyValueToJson(item.second)));
ret.push_back(nlohmann::json::object_t::value_type(item.first, PropertyValueToJson(item.second)));
}
break;
}
@@ -47,8 +46,7 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) {
return ret;
}
Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size,
int32_t buffer_flush_interval_millis)
Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
: storage_directory_(storage_directory),
buffer_size_(buffer_size),
buffer_flush_interval_millis_(buffer_flush_interval_millis),
@@ -63,9 +61,7 @@ void Log::Start() {
started_ = true;
ReopenLog();
scheduler_.Run("Audit",
std::chrono::milliseconds(buffer_flush_interval_millis_),
[&] { Flush(); });
scheduler_.Run("Audit", std::chrono::milliseconds(buffer_flush_interval_millis_), [&] { Flush(); });
}
Log::~Log() {
@@ -78,13 +74,12 @@ Log::~Log() {
Flush();
}
void Log::Record(const std::string &address, const std::string &username,
const std::string &query,
void Log::Record(const std::string &address, const std::string &username, const std::string &query,
const storage::PropertyValue &params) {
if (!started_.load(std::memory_order_relaxed)) return;
auto timestamp = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch())
.count();
auto timestamp =
std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
buffer_->emplace(Item{timestamp, address, username, query, params});
}
@@ -92,8 +87,7 @@ void Log::ReopenLog() {
if (!started_.load(std::memory_order_relaxed)) return;
std::lock_guard<std::mutex> guard(lock_);
if (log_.IsOpen()) log_.Close();
log_.Open(storage_directory_ / "audit.log",
utils::OutputFile::Mode::APPEND_TO_EXISTING);
log_.Open(storage_directory_ / "audit.log", utils::OutputFile::Mode::APPEND_TO_EXISTING);
}
void Log::Flush() {
@@ -101,11 +95,9 @@ void Log::Flush() {
for (uint64_t i = 0; i < buffer_size_; ++i) {
auto item = buffer_->pop();
if (!item) break;
log_.Write(
fmt::format("{}.{:06d},{},{},{},{}\n", item->timestamp / 1000000,
item->timestamp % 1000000, item->address, item->username,
utils::Escape(item->query),
utils::Escape(PropertyValueToJson(item->params).dump())));
log_.Write(fmt::format("{}.{:06d},{},{},{},{}\n", item->timestamp / 1000000, item->timestamp % 1000000,
item->address, item->username, utils::Escape(item->query),
utils::Escape(PropertyValueToJson(item->params).dump())));
}
log_.Sync();
}

View File

@@ -27,8 +27,7 @@ class Log {
};
public:
Log(const std::filesystem::path &storage_directory, int32_t buffer_size,
int32_t buffer_flush_interval_millis);
Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
~Log();
@@ -43,8 +42,8 @@ class Log {
void Start();
/// Adds an entry to the audit log. Thread-safe.
void Record(const std::string &address, const std::string &username,
const std::string &query, const storage::PropertyValue &params);
void Record(const std::string &address, const std::string &username, const std::string &query,
const storage::PropertyValue &params);
/// Reopens the log file. Used for log file rotation. Thread-safe.
void ReopenLog();

View File

@@ -12,26 +12,20 @@
#include "utils/logging.hpp"
#include "utils/string.hpp"
DEFINE_VALIDATED_string(
auth_module_executable, "",
"Absolute path to the auth module executable that should be used.", {
if (value.empty()) return true;
// Check the file status, following symlinks.
auto status = std::filesystem::status(value);
if (!std::filesystem::is_regular_file(status)) {
std::cerr << "The auth module path doesn't exist or isn't a file!"
<< std::endl;
return false;
}
return true;
});
DEFINE_bool(auth_module_create_missing_user, true,
"Set to false to disable creation of missing users.");
DEFINE_bool(auth_module_create_missing_role, true,
"Set to false to disable creation of missing roles.");
DEFINE_bool(
auth_module_manage_roles, true,
"Set to false to disable management of roles through the auth module.");
DEFINE_VALIDATED_string(auth_module_executable, "", "Absolute path to the auth module executable that should be used.",
{
if (value.empty()) return true;
// Check the file status, following symlinks.
auto status = std::filesystem::status(value);
if (!std::filesystem::is_regular_file(status)) {
std::cerr << "The auth module path doesn't exist or isn't a file!" << std::endl;
return false;
}
return true;
});
DEFINE_bool(auth_module_create_missing_user, true, "Set to false to disable creation of missing users.");
DEFINE_bool(auth_module_create_missing_role, true, "Set to false to disable creation of missing roles.");
DEFINE_bool(auth_module_manage_roles, true, "Set to false to disable management of roles through the auth module.");
DEFINE_VALIDATED_int32(auth_module_timeout_ms, 10000,
"Timeout (in milliseconds) used when waiting for a "
"response from the auth module.",
@@ -60,11 +54,9 @@ const std::string kLinkPrefix = "link:";
* key="link:<username>", value="<rolename>"
*/
Auth::Auth(const std::string &storage_directory)
: storage_(storage_directory), module_(FLAGS_auth_module_executable) {}
Auth::Auth(const std::string &storage_directory) : storage_(storage_directory), module_(FLAGS_auth_module_executable) {}
std::optional<User> Auth::Authenticate(const std::string &username,
const std::string &password) {
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
if (module_.IsUsed()) {
nlohmann::json params = nlohmann::json::object();
params["username"] = username;
@@ -73,8 +65,7 @@ std::optional<User> Auth::Authenticate(const std::string &username,
auto ret = module_.Call(params, FLAGS_auth_module_timeout_ms);
// Verify response integrity.
if (!ret.is_object() || ret.find("authenticated") == ret.end() ||
ret.find("role") == ret.end()) {
if (!ret.is_object() || ret.find("authenticated") == ret.end() || ret.find("role") == ret.end()) {
return std::nullopt;
}
const auto &ret_authenticated = ret.at("authenticated");
@@ -142,9 +133,7 @@ std::optional<User> Auth::Authenticate(const std::string &username,
} else {
auto user = GetUser(username);
if (!user) {
spdlog::warn(
"Couldn't authenticate user '{}' because the user doesn't exist",
username);
spdlog::warn("Couldn't authenticate user '{}' because the user doesn't exist", username);
return std::nullopt;
}
if (!user->CheckPassword(password)) {
@@ -182,21 +171,18 @@ std::optional<User> Auth::GetUser(const std::string &username_orig) {
void Auth::SaveUser(const User &user) {
bool success = false;
if (user.role()) {
success = storage_.PutMultiple(
{{kUserPrefix + user.username(), user.Serialize().dump()},
{kLinkPrefix + user.username(), user.role()->rolename()}});
success = storage_.PutMultiple({{kUserPrefix + user.username(), user.Serialize().dump()},
{kLinkPrefix + user.username(), user.role()->rolename()}});
} else {
success = storage_.PutAndDeleteMultiple(
{{kUserPrefix + user.username(), user.Serialize().dump()}},
{kLinkPrefix + user.username()});
success = storage_.PutAndDeleteMultiple({{kUserPrefix + user.username(), user.Serialize().dump()}},
{kLinkPrefix + user.username()});
}
if (!success) {
throw AuthException("Couldn't save user '{}'!", user.username());
}
}
std::optional<User> Auth::AddUser(const std::string &username,
const std::optional<std::string> &password) {
std::optional<User> Auth::AddUser(const std::string &username, const std::optional<std::string> &password) {
auto existing_user = GetUser(username);
if (existing_user) return std::nullopt;
auto existing_role = GetRole(username);
@@ -210,8 +196,7 @@ std::optional<User> Auth::AddUser(const std::string &username,
bool Auth::RemoveUser(const std::string &username_orig) {
auto username = utils::ToLowerCase(username_orig);
if (!storage_.Get(kUserPrefix + username)) return false;
std::vector<std::string> keys(
{kLinkPrefix + username, kUserPrefix + username});
std::vector<std::string> keys({kLinkPrefix + username, kUserPrefix + username});
if (!storage_.DeleteMultiple(keys)) {
throw AuthException("Couldn't remove user '{}'!", username);
}
@@ -220,8 +205,7 @@ bool Auth::RemoveUser(const std::string &username_orig) {
std::vector<auth::User> Auth::AllUsers() {
std::vector<auth::User> ret;
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix);
++it) {
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
auto username = it->first.substr(kUserPrefix.size());
if (username != utils::ToLowerCase(username)) continue;
auto user = GetUser(username);
@@ -232,9 +216,7 @@ std::vector<auth::User> Auth::AllUsers() {
return ret;
}
bool Auth::HasUsers() {
return storage_.begin(kUserPrefix) != storage_.end(kUserPrefix);
}
bool Auth::HasUsers() { return storage_.begin(kUserPrefix) != storage_.end(kUserPrefix); }
std::optional<Role> Auth::GetRole(const std::string &rolename_orig) {
auto rolename = utils::ToLowerCase(rolename_orig);
@@ -271,8 +253,7 @@ bool Auth::RemoveRole(const std::string &rolename_orig) {
auto rolename = utils::ToLowerCase(rolename_orig);
if (!storage_.Get(kRolePrefix + rolename)) return false;
std::vector<std::string> keys;
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix);
++it) {
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix); ++it) {
if (utils::ToLowerCase(it->second) == rolename) {
keys.push_back(it->first);
}
@@ -286,8 +267,7 @@ bool Auth::RemoveRole(const std::string &rolename_orig) {
std::vector<auth::Role> Auth::AllRoles() {
std::vector<auth::Role> ret;
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix);
++it) {
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
auto rolename = it->first.substr(kRolePrefix.size());
if (rolename != utils::ToLowerCase(rolename)) continue;
auto role = GetRole(rolename);
@@ -300,12 +280,10 @@ std::vector<auth::Role> Auth::AllRoles() {
return ret;
}
std::vector<auth::User> Auth::AllUsersForRole(
const std::string &rolename_orig) {
std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig) {
auto rolename = utils::ToLowerCase(rolename_orig);
std::vector<auth::User> ret;
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix);
++it) {
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix); ++it) {
auto username = it->first.substr(kLinkPrefix.size());
if (username != utils::ToLowerCase(username)) continue;
if (it->second != utils::ToLowerCase(it->second)) continue;

View File

@@ -32,8 +32,7 @@ class Auth final {
* @return a user when the username and password match, nullopt otherwise
* @throw AuthException if unable to authenticate for whatever reason.
*/
std::optional<User> Authenticate(const std::string &username,
const std::string &password);
std::optional<User> Authenticate(const std::string &username, const std::string &password);
/**
* Gets a user from the storage.
@@ -63,9 +62,7 @@ class Auth final {
* @return a user when the user is created, nullopt if the user exists
* @throw AuthException if unable to save the user.
*/
std::optional<User> AddUser(
const std::string &username,
const std::optional<std::string> &password = std::nullopt);
std::optional<User> AddUser(const std::string &username, const std::optional<std::string> &password = std::nullopt);
/**
* Removes a user from the storage.

View File

@@ -9,8 +9,7 @@
#include "utils/cast.hpp"
#include "utils/string.hpp"
DEFINE_bool(auth_password_permit_null, true,
"Set to false to disable null passwords.");
DEFINE_bool(auth_password_permit_null, true, "Set to false to disable null passwords.");
DEFINE_string(auth_password_strength_regex, ".+",
"The regular expression that should be used to match the entire "
@@ -44,6 +43,10 @@ std::string PermissionToString(Permission permission) {
return "REPLICATION";
case Permission::LOCK_PATH:
return "LOCK_PATH";
case Permission::READ_FILE:
return "READ_FILE";
case Permission::FREE_MEMORY:
return "FREE_MEMORY";
case Permission::AUTH:
return "AUTH";
}
@@ -129,8 +132,7 @@ Permissions Permissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!");
}
if (!data["grants"].is_number_unsigned() ||
!data["denies"].is_number_unsigned()) {
if (!data["grants"].is_number_unsigned() || !data["denies"].is_number_unsigned()) {
throw AuthException("Couldn't load permissions data!");
}
return {data["grants"], data["denies"]};
@@ -143,12 +145,9 @@ bool operator==(const Permissions &first, const Permissions &second) {
return first.grants() == second.grants() && first.denies() == second.denies();
}
bool operator!=(const Permissions &first, const Permissions &second) {
return !(first == second);
}
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
Role::Role(const std::string &rolename)
: rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
@@ -176,18 +175,13 @@ Role Role::Deserialize(const nlohmann::json &data) {
}
bool operator==(const Role &first, const Role &second) {
return first.rolename_ == second.rolename_ &&
first.permissions_ == second.permissions_;
return first.rolename_ == second.rolename_ && first.permissions_ == second.permissions_;
}
User::User(const std::string &username)
: username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, const std::string &password_hash,
const Permissions &permissions)
: username_(utils::ToLowerCase(username)),
password_hash_(password_hash),
permissions_(permissions) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
bool User::CheckPassword(const std::string &password) {
if (password_hash_ == "") return true;
@@ -244,8 +238,7 @@ User User::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load user data!");
}
if (!data["username"].is_string() || !data["password_hash"].is_string() ||
!data["permissions"].is_object()) {
if (!data["username"].is_string() || !data["password_hash"].is_string() || !data["permissions"].is_object()) {
throw AuthException("Couldn't load user data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
@@ -253,9 +246,7 @@ User User::Deserialize(const nlohmann::json &data) {
}
bool operator==(const User &first, const User &second) {
return first.username_ == second.username_ &&
first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ &&
first.role_ == second.role_;
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
first.permissions_ == second.permissions_ && first.role_ == second.role_;
}
} // namespace auth

View File

@@ -23,17 +23,18 @@ enum class Permission : uint64_t {
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
LOCK_PATH = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
AUTH = 1U << 16U
};
// clang-format on
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {
Permission::MATCH, Permission::CREATE, Permission::MERGE,
Permission::DELETE, Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS, Permission::CONSTRAINT,
Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::LOCK_PATH};
const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permission::CREATE, Permission::MERGE,
Permission::DELETE, Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS, Permission::CONSTRAINT,
Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::LOCK_PATH, Permission::READ_FILE, Permission::FREE_MEMORY};
// Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission);
@@ -110,15 +111,13 @@ class User final {
public:
User(const std::string &username);
User(const std::string &username, const std::string &password_hash,
const Permissions &permissions);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
/// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password);
/// @throw AuthException if unable to set the password.
void UpdatePassword(
const std::optional<std::string> &password = std::nullopt);
void UpdatePassword(const std::optional<std::string> &password = std::nullopt);
void SetRole(const Role &role);

View File

@@ -86,54 +86,22 @@ class CharPP final {
////////////////////////////////////
const std::vector<int> kSeccompSyscallsBlacklist = {
SCMP_SYS(mknod),
SCMP_SYS(mount),
SCMP_SYS(setuid),
SCMP_SYS(stime),
SCMP_SYS(ptrace),
SCMP_SYS(setgid),
SCMP_SYS(acct),
SCMP_SYS(umount),
SCMP_SYS(setpgid),
SCMP_SYS(chroot),
SCMP_SYS(setreuid),
SCMP_SYS(setregid),
SCMP_SYS(sethostname),
SCMP_SYS(settimeofday),
SCMP_SYS(setgroups),
SCMP_SYS(swapon),
SCMP_SYS(reboot),
SCMP_SYS(setpriority),
SCMP_SYS(ioperm),
SCMP_SYS(syslog),
SCMP_SYS(iopl),
SCMP_SYS(vhangup),
SCMP_SYS(vm86old),
SCMP_SYS(swapoff),
SCMP_SYS(setdomainname),
SCMP_SYS(adjtimex),
SCMP_SYS(init_module),
SCMP_SYS(delete_module),
SCMP_SYS(setfsuid),
SCMP_SYS(setfsgid),
SCMP_SYS(setresuid),
SCMP_SYS(vm86),
SCMP_SYS(setresgid),
SCMP_SYS(capset),
SCMP_SYS(setreuid),
SCMP_SYS(setregid),
SCMP_SYS(setgroups),
SCMP_SYS(setresuid),
SCMP_SYS(setresgid),
SCMP_SYS(setuid),
SCMP_SYS(setgid),
SCMP_SYS(setfsuid),
SCMP_SYS(setfsgid),
SCMP_SYS(pivot_root),
SCMP_SYS(sched_setaffinity),
SCMP_SYS(clock_settime),
SCMP_SYS(kexec_load),
SCMP_SYS(mknodat),
SCMP_SYS(mknod), SCMP_SYS(mount), SCMP_SYS(setuid),
SCMP_SYS(stime), SCMP_SYS(ptrace), SCMP_SYS(setgid),
SCMP_SYS(acct), SCMP_SYS(umount), SCMP_SYS(setpgid),
SCMP_SYS(chroot), SCMP_SYS(setreuid), SCMP_SYS(setregid),
SCMP_SYS(sethostname), SCMP_SYS(settimeofday), SCMP_SYS(setgroups),
SCMP_SYS(swapon), SCMP_SYS(reboot), SCMP_SYS(setpriority),
SCMP_SYS(ioperm), SCMP_SYS(syslog), SCMP_SYS(iopl),
SCMP_SYS(vhangup), SCMP_SYS(vm86old), SCMP_SYS(swapoff),
SCMP_SYS(setdomainname), SCMP_SYS(adjtimex), SCMP_SYS(init_module),
SCMP_SYS(delete_module), SCMP_SYS(setfsuid), SCMP_SYS(setfsgid),
SCMP_SYS(setresuid), SCMP_SYS(vm86), SCMP_SYS(setresgid),
SCMP_SYS(capset), SCMP_SYS(setreuid), SCMP_SYS(setregid),
SCMP_SYS(setgroups), SCMP_SYS(setresuid), SCMP_SYS(setresgid),
SCMP_SYS(setuid), SCMP_SYS(setgid), SCMP_SYS(setfsuid),
SCMP_SYS(setfsgid), SCMP_SYS(pivot_root), SCMP_SYS(sched_setaffinity),
SCMP_SYS(clock_settime), SCMP_SYS(kexec_load), SCMP_SYS(mknodat),
SCMP_SYS(unshare),
#ifdef SYS_seccomp
SCMP_SYS(seccomp),
@@ -182,24 +150,20 @@ int Target(void *arg) {
// Redirect `stdin` to `/dev/null`.
int fd = open("/dev/null", O_RDONLY | O_CLOEXEC);
if (fd == -1) {
std::cerr
<< "Couldn't open \"/dev/null\" for auth module stdin because of: "
<< strerror(errno) << " (" << errno << ")!" << std::endl;
std::cerr << "Couldn't open \"/dev/null\" for auth module stdin because of: " << strerror(errno) << " (" << errno
<< ")!" << std::endl;
return EXIT_FAILURE;
}
if (dup2(fd, STDIN_FILENO) != STDIN_FILENO) {
std::cerr
<< "Couldn't attach \"/dev/null\" to auth module stdin because of: "
<< strerror(errno) << " (" << errno << ")!" << std::endl;
std::cerr << "Couldn't attach \"/dev/null\" to auth module stdin because of: " << strerror(errno) << " (" << errno
<< ")!" << std::endl;
return EXIT_FAILURE;
}
// Change the current directory to the module directory.
if (chdir(ta->module_executable_path.parent_path().c_str()) != 0) {
std::cerr << "Couldn't change directory to "
<< ta->module_executable_path.parent_path()
<< " for auth module stdin because of: " << strerror(errno)
<< " (" << errno << ")!" << std::endl;
std::cerr << "Couldn't change directory to " << ta->module_executable_path.parent_path()
<< " for auth module stdin because of: " << strerror(errno) << " (" << errno << ")!" << std::endl;
return EXIT_FAILURE;
}
@@ -214,8 +178,7 @@ int Target(void *arg) {
}
// Connect the communication input pipe.
if (dup2(ta->pipe_to_module, kCommunicationToModuleFd) !=
kCommunicationToModuleFd) {
if (dup2(ta->pipe_to_module, kCommunicationToModuleFd) != kCommunicationToModuleFd) {
std::cerr << "Couldn't attach communication to module pipe to auth module "
"because of: "
<< strerror(errno) << " (" << errno << ")!" << std::endl;
@@ -223,8 +186,7 @@ int Target(void *arg) {
}
// Connect the communication output pipe.
if (dup2(ta->pipe_from_module, kCommunicationFromModuleFd) !=
kCommunicationFromModuleFd) {
if (dup2(ta->pipe_from_module, kCommunicationFromModuleFd) != kCommunicationFromModuleFd) {
std::cerr << "Couldn't attach communication from module pipe to auth "
"module because of: "
<< strerror(errno) << " (" << errno << ")!" << std::endl;
@@ -246,8 +208,8 @@ int Target(void *arg) {
sigemptyset(&action.sa_mask);
action.sa_flags = 0;
if (sigaction(SIGINT, &action, nullptr) != 0) {
std::cerr << "Couldn't ignore SIGINT for auth module because of: "
<< strerror(errno) << " (" << errno << ")!" << std::endl;
std::cerr << "Couldn't ignore SIGINT for auth module because of: " << strerror(errno) << " (" << errno << ")!"
<< std::endl;
return EXIT_FAILURE;
}
@@ -261,8 +223,7 @@ int Target(void *arg) {
// If the `execve` call succeeded then the process will exit from that call
// and won't reach this piece of code ever.
std::cerr << "Couldn't start auth module because of: " << strerror(errno)
<< " (" << errno << ")!" << std::endl;
std::cerr << "Couldn't start auth module because of: " << strerror(errno) << " (" << errno << ")!" << std::endl;
return EXIT_FAILURE;
}
@@ -408,8 +369,7 @@ bool Module::Startup() {
return true;
}
nlohmann::json Module::Call(const nlohmann::json &params,
int timeout_millisec) {
nlohmann::json Module::Call(const nlohmann::json &params, int timeout_millisec) {
std::lock_guard<std::mutex> guard(lock_);
if (!params.is_object()) return {};

View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python3
import json
import io

View File

@@ -1,4 +1,4 @@
#!/usr/bin/env python3
#!/usr/bin/python3
import json
import io
import ssl

View File

@@ -43,16 +43,14 @@ class ClientFatalException : public utils::BasicException {
// only handle the `ClientFatalException`.
class ServerCommunicationException : public ClientFatalException {
public:
ServerCommunicationException()
: ClientFatalException("Couldn't communicate with the server!") {}
ServerCommunicationException() : ClientFatalException("Couldn't communicate with the server!") {}
};
// Internal exception used whenever a malformed data error occurs. You should
// only handle the `ClientFatalException`.
class ServerMalformedDataException : public ClientFatalException {
public:
ServerMalformedDataException()
: ClientFatalException("The server sent malformed data!") {}
ServerMalformedDataException() : ClientFatalException("The server sent malformed data!") {}
};
/// Structure that is used to return results from an executed query.
@@ -79,8 +77,7 @@ class Client final {
/// connection is set-up, multiple queries may be executed through a single
/// established connection.
/// @throws ClientFatalException when we couldn't connect to the server
void Connect(const io::network::Endpoint &endpoint,
const std::string &username, const std::string &password,
void Connect(const io::network::Endpoint &endpoint, const std::string &username, const std::string &password,
const std::string &client_name = "memgraph-bolt") {
if (!client_.Connect(endpoint)) {
throw ClientFatalException("Couldn't connect to {}!", endpoint);
@@ -103,14 +100,11 @@ class Client final {
}
if (memcmp(kProtocol, client_.GetData(), sizeof(kProtocol)) != 0) {
SPDLOG_ERROR("Server negotiated unsupported protocol version!");
throw ClientFatalException(
"The server negotiated an usupported protocol version!");
throw ClientFatalException("The server negotiated an usupported protocol version!");
}
client_.ShiftData(sizeof(kProtocol));
if (!encoder_.MessageInit(client_name, {{"scheme", "basic"},
{"principal", username},
{"credentials", password}})) {
if (!encoder_.MessageInit(client_name, {{"scheme", "basic"}, {"principal", username}, {"credentials", password}})) {
SPDLOG_ERROR("Couldn't send init message!");
throw ServerCommunicationException();
}
@@ -135,15 +129,12 @@ class Client final {
/// executing the query (eg. mistyped query,
/// etc.)
/// @throws ClientFatalException when we couldn't communicate with the server
QueryData Execute(const std::string &query,
const std::map<std::string, Value> &parameters) {
QueryData Execute(const std::string &query, const std::map<std::string, Value> &parameters) {
if (!client_.IsConnected()) {
throw ClientFatalException(
"You must first connect to the server before using the client!");
throw ClientFatalException("You must first connect to the server before using the client!");
}
SPDLOG_INFO("Sending run message with statement: '{}'; parameters: {}",
query, parameters);
SPDLOG_INFO("Sending run message with statement: '{}'; parameters: {}", query, parameters);
encoder_.MessageRun(query, parameters);
encoder_.MessagePullAll();
@@ -165,8 +156,7 @@ class Client final {
if (it != tmp.end()) {
auto it_code = tmp.find("code");
if (it_code != tmp.end()) {
throw ClientQueryException(it_code->second.ValueString(),
it->second.ValueString());
throw ClientQueryException(it_code->second.ValueString(), it->second.ValueString());
} else {
throw ClientQueryException("", it->second.ValueString());
}
@@ -209,8 +199,7 @@ class Client final {
if (it != tmp.end()) {
auto it_code = tmp.find("code");
if (it_code != tmp.end()) {
throw ClientQueryException(it_code->second.ValueString(),
it->second.ValueString());
throw ClientQueryException(it_code->second.ValueString(), it->second.ValueString());
} else {
throw ClientQueryException("", it->second.ValueString());
}
@@ -308,15 +297,11 @@ class Client final {
communication::ClientOutputStream output_stream_{client_};
// decoder objects
ChunkedDecoderBuffer<communication::ClientInputStream> decoder_buffer_{
input_stream_};
Decoder<ChunkedDecoderBuffer<communication::ClientInputStream>> decoder_{
decoder_buffer_};
ChunkedDecoderBuffer<communication::ClientInputStream> decoder_buffer_{input_stream_};
Decoder<ChunkedDecoderBuffer<communication::ClientInputStream>> decoder_{decoder_buffer_};
// encoder objects
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{
output_stream_};
ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>>
encoder_{encoder_buffer_};
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_};
ClientEncoder<ChunkedEncoderBuffer<communication::ClientOutputStream>> encoder_{encoder_buffer_};
};
} // namespace communication::bolt

View File

@@ -78,12 +78,8 @@ enum class Marker : uint8_t {
};
static constexpr uint8_t MarkerString = 0, MarkerList = 1, MarkerMap = 2;
static constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList,
Marker::TinyMap};
static constexpr Marker Marker8[3] = {Marker::String8, Marker::List8,
Marker::Map8};
static constexpr Marker Marker16[3] = {Marker::String16, Marker::List16,
Marker::Map16};
static constexpr Marker Marker32[3] = {Marker::String32, Marker::List32,
Marker::Map32};
static constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList, Marker::TinyMap};
static constexpr Marker Marker8[3] = {Marker::String8, Marker::List8, Marker::Map8};
static constexpr Marker Marker16[3] = {Marker::String16, Marker::List16, Marker::Map16};
static constexpr Marker Marker32[3] = {Marker::String32, Marker::List32, Marker::Map32};
} // namespace communication::bolt

View File

@@ -40,9 +40,7 @@ enum class ChunkState : uint8_t {
template <typename TBuffer>
class ChunkedDecoderBuffer {
public:
ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) {
data_.reserve(kChunkMaxDataSize);
}
ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
/**
* Reads data from the internal buffer.

View File

@@ -158,8 +158,7 @@ class Decoder {
}
bool ReadBool(const Marker &marker, Value *data) {
DMG_ASSERT(marker == Marker::False || marker == Marker::True,
"Received invalid marker!");
DMG_ASSERT(marker == Marker::False || marker == Marker::True, "Received invalid marker!");
if (marker == Marker::False) {
*data = Value(false);
} else {

View File

@@ -7,8 +7,7 @@
#include "utils/cast.hpp"
#include "utils/endian.hpp"
static_assert(std::is_same_v<std::uint8_t, char> ||
std::is_same_v<std::uint8_t, unsigned char>,
static_assert(std::is_same_v<std::uint8_t, char> || std::is_same_v<std::uint8_t, unsigned char>,
"communication::bolt::Encoder requires uint8_t to be "
"implemented as char or unsigned char.");
@@ -29,9 +28,7 @@ class BaseEncoder {
void WriteRAW(const uint8_t *data, uint64_t len) { buffer_.Write(data, len); }
void WriteRAW(const char *data, uint64_t len) {
WriteRAW((const uint8_t *)data, len);
}
void WriteRAW(const char *data, uint64_t len) { WriteRAW((const uint8_t *)data, len); }
void WriteRAW(const uint8_t data) { WriteRAW(&data, 1); }
@@ -126,8 +123,7 @@ class BaseEncoder {
void WriteEdge(const Edge &edge, bool unbound = false) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + (unbound ? 3 : 5));
WriteRAW(utils::UnderlyingCast(unbound ? Signature::UnboundRelationship
: Signature::Relationship));
WriteRAW(utils::UnderlyingCast(unbound ? Signature::UnboundRelationship : Signature::Relationship));
WriteInt(edge.id.AsInt());
if (!unbound) {

View File

@@ -37,8 +37,7 @@ namespace communication::bolt {
template <class TOutputStream>
class ChunkedEncoderBuffer {
public:
ChunkedEncoderBuffer(TOutputStream &output_stream)
: output_stream_(output_stream) {}
ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
/**
* Writes n values into the buffer. If n is bigger than whole chunk size
@@ -53,12 +52,10 @@ class ChunkedEncoderBuffer {
while (n > 0) {
// Define the number of bytes which will be copied into the chunk because
// the internal storage is a fixed length array.
size_t size =
n < kChunkMaxDataSize - have_ ? n : kChunkMaxDataSize - have_;
size_t size = n < kChunkMaxDataSize - have_ ? n : kChunkMaxDataSize - have_;
// Copy `size` values to the chunk array.
std::memcpy(chunk_.data() + kChunkHeaderSize + have_, values + written,
size);
std::memcpy(chunk_.data() + kChunkHeaderSize + have_, values + written, size);
// Update positions. The position pointer and incoming size have to be
// updated because all incoming values have to be processed.
@@ -87,8 +84,7 @@ class ChunkedEncoderBuffer {
chunk_[1] = have_ & 0xFF;
// Write the data to the stream.
auto ret = output_stream_.Write(chunk_.data(), kChunkHeaderSize + have_,
have_more);
auto ret = output_stream_.Write(chunk_.data(), kChunkHeaderSize + have_, have_more);
// Cleanup.
Clear();

View File

@@ -38,8 +38,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageInit(const std::string client_name,
const std::map<std::string, Value> &auth_token) {
bool MessageInit(const std::string client_name, const std::map<std::string, Value> &auth_token) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
WriteRAW(utils::UnderlyingCast(Signature::Init));
WriteString(client_name);
@@ -65,9 +64,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
* @returns true if the data was successfully sent to the client
* when flushing, false otherwise
*/
bool MessageRun(const std::string &statement,
const std::map<std::string, Value> &parameters,
bool have_more = true) {
bool MessageRun(const std::string &statement, const std::map<std::string, Value> &parameters, bool have_more = true) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct2));
WriteRAW(utils::UnderlyingCast(Signature::Run));
WriteString(statement);

View File

@@ -50,13 +50,10 @@ class VerboseError : public utils::BasicException {
};
template <class... Args>
VerboseError(Classification classification, const std::string &category,
const std::string &title, const std::string &format,
Args &&... args)
VerboseError(Classification classification, const std::string &category, const std::string &title,
const std::string &format, Args &&...args)
: BasicException(format, std::forward<Args>(args)...),
code_(fmt::format("Memgraph.{}.{}.{}",
ClassificationToString(classification), category,
title)) {}
code_(fmt::format("Memgraph.{}.{}.{}", ClassificationToString(classification), category, title)) {}
const std::string &code() const noexcept { return code_; }

View File

@@ -62,9 +62,7 @@ class Session {
* @param q If set, defines from which query to pull the results,
* otherwise the last query is used.
*/
virtual std::map<std::string, Value> Pull(TEncoder *encoder,
std::optional<int> n,
std::optional<int> qid) = 0;
virtual std::map<std::string, Value> Pull(TEncoder *encoder, std::optional<int> n, std::optional<int> qid) = 0;
/**
* Discard results of the processed query.
@@ -74,8 +72,7 @@ class Session {
* @param q If set, defines from which query to discard the results,
* otherwise the last query is used.
*/
virtual std::map<std::string, Value> Discard(std::optional<int> n,
std::optional<int> qid) = 0;
virtual std::map<std::string, Value> Discard(std::optional<int> n, std::optional<int> qid) = 0;
virtual void BeginTransaction() = 0;
virtual void CommitTransaction() = 0;
@@ -85,8 +82,7 @@ class Session {
virtual void Abort() = 0;
/** Return `true` if the user was successfully authenticated. */
virtual bool Authenticate(const std::string &username,
const std::string &password) = 0;
virtual bool Authenticate(const std::string &username, const std::string &password) = 0;
/** Return the name of the server that should be used for the Bolt INIT
* message. */
@@ -104,8 +100,7 @@ class Session {
// Receive the handshake.
if (input_stream_.size() < kHandshakeSize) {
spdlog::trace("Received partial handshake of size {}",
input_stream_.size());
spdlog::trace("Received partial handshake of size {}", input_stream_.size());
return;
}
state_ = StateHandshakeRun(*this);

View File

@@ -44,4 +44,4 @@ enum class State : uint8_t {
*/
Close
};
}
} // namespace communication::bolt

View File

@@ -26,8 +26,7 @@ State StateErrorRun(TSession &session, State state) {
return State::Close;
}
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 &&
session.version_.minor == 1)) {
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
spdlog::trace("Received NOOP message");
return state;
}
@@ -35,8 +34,7 @@ State StateErrorRun(TSession &session, State state) {
// Clear the data buffer if it has any leftover data.
session.encoder_buffer_.Clear();
if ((session.version_.major == 1 && signature == Signature::AckFailure) ||
signature == Signature::Reset) {
if ((session.version_.major == 1 && signature == Signature::AckFailure) || signature == Signature::Reset) {
if (signature == Signature::AckFailure) {
spdlog::trace("AckFailure received");
} else {
@@ -62,8 +60,7 @@ State StateErrorRun(TSession &session, State state) {
// All bolt client messages have less than 15 parameters so if we receive
// anything than a TinyStruct it's an error.
if ((value & 0xF0) != utils::UnderlyingCast(Marker::TinyStruct)) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!",
value);
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", value);
return State::Close;
}

View File

@@ -16,8 +16,7 @@
namespace communication::bolt {
// TODO (mferencevic): revise these error messages
inline std::pair<std::string, std::string> ExceptionToErrorMessage(
const std::exception &e) {
inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::exception &e) {
if (auto *verbose = dynamic_cast<const VerboseError *>(&e)) {
return {verbose->code(), verbose->what()};
}
@@ -54,8 +53,7 @@ inline std::pair<std::string, std::string> ExceptionToErrorMessage(
// All exceptions used in memgraph are derived from BasicException. Since
// we caught some other exception we don't know what is going on. Return
// DatabaseError, log real message and return generic string.
spdlog::error("Unknown exception occurred during query execution {}",
e.what());
spdlog::error("Unknown exception occurred during query execution {}", e.what());
return {"Memgraph.DatabaseError.MemgraphError.MemgraphError",
"An unknown exception occurred, this is unexpected. Real message "
"should be in database logs."};
@@ -69,8 +67,7 @@ inline State HandleFailure(TSession &session, const std::exception &e) {
}
session.encoder_buffer_.Clear();
auto code_message = ExceptionToErrorMessage(e);
bool fail_sent = session.encoder_.MessageFailure(
{{"code", code_message.first}, {"message", code_message.second}});
bool fail_sent = session.encoder_.MessageFailure({{"code", code_message.first}, {"message", code_message.second}});
if (!fail_sent) {
spdlog::trace("Couldn't send failure message!");
return State::Close;
@@ -80,15 +77,12 @@ inline State HandleFailure(TSession &session, const std::exception &e) {
template <typename TSession>
State HandleRun(TSession &session, State state, Marker marker) {
const std::map<std::string, Value> kEmptyFields = {
{"fields", std::vector<Value>{}}};
const std::map<std::string, Value> kEmptyFields = {{"fields", std::vector<Value>{}}};
const auto expected_marker =
session.version_.major == 1 ? Marker::TinyStruct2 : Marker::TinyStruct3;
const auto expected_marker = session.version_.major == 1 ? Marker::TinyStruct2 : Marker::TinyStruct3;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!",
session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3",
utils::UnderlyingCast(marker));
session.version_.major == 1 ? "TinyStruct2" : "TinyStruct3", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -117,15 +111,13 @@ State HandleRun(TSession &session, State state, Marker marker) {
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(),
"There should be no data to write in this state");
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
auto [header, qid] =
session.Interpret(query.ValueString(), params.ValueMap());
auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
@@ -146,12 +138,10 @@ State HandleRun(TSession &session, State state, Marker marker) {
namespace detail {
template <bool is_pull, typename TSession>
State HandlePullDiscard(TSession &session, State state, Marker marker) {
const auto expected_marker =
session.version_.major == 1 ? Marker::TinyStruct : Marker::TinyStruct1;
const auto expected_marker = session.version_.major == 1 ? Marker::TinyStruct : Marker::TinyStruct1;
if (marker != expected_marker) {
spdlog::trace("Expected {} marker, but received 0x{:02X}!",
session.version_.major == 1 ? "TinyStruct" : "TinyStruct1",
utils::UnderlyingCast(marker));
session.version_.major == 1 ? "TinyStruct" : "TinyStruct1", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -176,15 +166,13 @@ State HandlePullDiscard(TSession &session, State state, Marker marker) {
}
const auto &extra_map = extra.ValueMap();
if (extra_map.count("n")) {
if (const auto n_value = extra_map.at("n").ValueInt();
n_value != kPullAll) {
if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
n = n_value;
}
}
if (extra_map.count("qid")) {
if (const auto qid_value = extra_map.at("qid").ValueInt();
qid_value != kPullLast) {
if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
qid = qid_value;
}
}
@@ -236,8 +224,7 @@ State HandleReset(Session &session, State, Marker marker) {
// now this command only resets the session to a clean state. It
// does not IGNORE running and pending commands as it should.
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!",
utils::UnderlyingCast(marker));
spdlog::trace("Expected TinyStruct marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -262,8 +249,7 @@ State HandleBegin(Session &session, State state, Marker marker) {
}
if (marker != Marker::TinyStruct1) {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02x}!",
utils::UnderlyingCast(marker));
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -278,8 +264,7 @@ State HandleBegin(Session &session, State state, Marker marker) {
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(),
"There should be no data to write in this state");
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
if (!session.encoder_.MessageSuccess({})) {
spdlog::trace("Couldn't send success message!");
@@ -303,8 +288,7 @@ State HandleCommit(Session &session, State state, Marker marker) {
}
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!",
utils::UnderlyingCast(marker));
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -313,8 +297,7 @@ State HandleCommit(Session &session, State state, Marker marker) {
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(),
"There should be no data to write in this state");
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
try {
if (!session.encoder_.MessageSuccess({})) {
@@ -336,8 +319,7 @@ State HandleRollback(Session &session, State state, Marker marker) {
}
if (marker != Marker::TinyStruct) {
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!",
utils::UnderlyingCast(marker));
spdlog::trace("Expected TinyStruct marker, but received 0x{:02x}!", utils::UnderlyingCast(marker));
return State::Close;
}
@@ -346,8 +328,7 @@ State HandleRollback(Session &session, State state, Marker marker) {
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(),
"There should be no data to write in this state");
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
try {
if (!session.encoder_.MessageSuccess({})) {
@@ -376,8 +357,7 @@ State StateExecutingRun(Session &session, State state) {
return State::Close;
}
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 &&
session.version_.minor == 1)) {
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
spdlog::trace("Received NOOP message");
return state;
}
@@ -399,8 +379,7 @@ State StateExecutingRun(Session &session, State state) {
} else if (signature == Signature::Goodbye && session.version_.major != 1) {
throw SessionClosedException("Closing connection.");
} else {
spdlog::trace("Unrecognized signature received (0x{:02X})!",
utils::UnderlyingCast(signature));
spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
return State::Close;
}
}

View File

@@ -17,15 +17,13 @@ namespace communication::bolt {
*/
template <typename TSession>
State StateHandshakeRun(TSession &session) {
auto precmp =
std::memcmp(session.input_stream_.data(), kPreamble, sizeof(kPreamble));
auto precmp = std::memcmp(session.input_stream_.data(), kPreamble, sizeof(kPreamble));
if (UNLIKELY(precmp != 0)) {
spdlog::trace("Received a wrong preamble!");
return State::Close;
}
DMG_ASSERT(session.input_stream_.size() >= kHandshakeSize,
"Wrong size of the handshake data!");
DMG_ASSERT(session.input_stream_.size() >= kHandshakeSize, "Wrong size of the handshake data!");
auto dataPosition = session.input_stream_.data() + sizeof(kPreamble);
@@ -61,8 +59,7 @@ State StateHandshakeRun(TSession &session) {
return State::Close;
}
spdlog::info("Using version {}.{} of protocol", session.version_.major,
session.version_.minor);
spdlog::info("Using version {}.{} of protocol", session.version_.major, session.version_.minor);
// Delete data from the input stream. It is guaranteed that there will more
// than, or equal to 20 bytes (kHandshakeSize) in the buffer.

View File

@@ -15,8 +15,7 @@ namespace detail {
template <typename TSession>
std::optional<Value> StateInitRunV1(TSession &session, const Marker marker) {
if (UNLIKELY(marker != Marker::TinyStruct2)) {
spdlog::trace("Expected TinyStruct2 marker, but received 0x{:02X}!",
utils::UnderlyingCast(marker));
spdlog::trace("Expected TinyStruct2 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace(
"The client sent malformed data, but we are continuing "
"because the official Neo4j Java driver sends malformed "
@@ -45,8 +44,7 @@ std::optional<Value> StateInitRunV1(TSession &session, const Marker marker) {
template <typename TSession>
std::optional<Value> StateInitRunV4(TSession &session, const Marker marker) {
if (UNLIKELY(marker != Marker::TinyStruct1)) {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!",
utils::UnderlyingCast(marker));
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace(
"The client sent malformed data, but we are continuing "
"because the official Neo4j Java driver sends malformed "
@@ -80,8 +78,7 @@ std::optional<Value> StateInitRunV4(TSession &session, const Marker marker) {
*/
template <typename Session>
State StateInitRun(Session &session) {
DMG_ASSERT(!session.encoder_buffer_.HasData(),
"There should be no data to write in this state");
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
Marker marker;
Signature signature;
@@ -90,21 +87,18 @@ State StateInitRun(Session &session) {
return State::Close;
}
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 &&
session.version_.minor == 1)) {
if (UNLIKELY(signature == Signature::Noop && session.version_.major == 4 && session.version_.minor == 1)) {
SPDLOG_DEBUG("Received NOOP message");
return State::Init;
}
if (UNLIKELY(signature != Signature::Init)) {
spdlog::trace("Expected Init signature, but received 0x{:02X}!",
utils::UnderlyingCast(signature));
spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
return State::Close;
}
auto maybeMetadata = session.version_.major == 1
? detail::StateInitRunV1(session, marker)
: detail::StateInitRunV4(session, marker);
auto maybeMetadata =
session.version_.major == 1 ? detail::StateInitRunV1(session, marker) : detail::StateInitRunV4(session, marker);
if (!maybeMetadata) {
return State::Close;
@@ -126,16 +120,14 @@ State StateInitRun(Session &session) {
username = data["principal"].ValueString();
password = data["credentials"].ValueString();
} else if (data["scheme"].ValueString() != "none") {
spdlog::warn("Unsupported authentication scheme: {}",
data["scheme"].ValueString());
spdlog::warn("Unsupported authentication scheme: {}", data["scheme"].ValueString());
return State::Close;
}
// Authenticate the user.
if (!session.Authenticate(username, password)) {
if (!session.encoder_.MessageFailure(
{{"code", "Memgraph.ClientError.Security.Unauthenticated"},
{"message", "Authentication failure"}})) {
{{"code", "Memgraph.ClientError.Security.Unauthenticated"}, {"message", "Authentication failure"}})) {
spdlog::trace("Couldn't send failure message to the client!");
}
// Throw an exception to indicate to the network stack that the session

View File

@@ -198,8 +198,7 @@ Value &Value::operator=(Value &&other) noexcept {
new (&edge_v) Edge(std::move(other.edge_v));
break;
case Type::UnboundedEdge:
new (&unbounded_edge_v)
UnboundedEdge(std::move(other.unbounded_edge_v));
new (&unbounded_edge_v) UnboundedEdge(std::move(other.unbounded_edge_v));
break;
case Type::Path:
new (&path_v) Path(std::move(other.path_v));
@@ -258,17 +257,14 @@ std::ostream &operator<<(std::ostream &os, const Vertex &vertex) {
if (vertex.labels.size() > 0) {
os << ":";
}
utils::PrintIterable(os, vertex.labels, ":",
[&](auto &stream, auto label) { stream << label; });
utils::PrintIterable(os, vertex.labels, ":", [&](auto &stream, auto label) { stream << label; });
if (vertex.labels.size() > 0 && vertex.properties.size() > 0) {
os << " ";
}
if (vertex.properties.size() > 0) {
os << "{";
utils::PrintIterable(os, vertex.properties, ", ",
[&](auto &stream, const auto &pair) {
stream << pair.first << ": " << pair.second;
});
[&](auto &stream, const auto &pair) { stream << pair.first << ": " << pair.second; });
os << "}";
}
return os << ")";
@@ -279,9 +275,7 @@ std::ostream &operator<<(std::ostream &os, const Edge &edge) {
if (edge.properties.size() > 0) {
os << " {";
utils::PrintIterable(os, edge.properties, ", ",
[&](auto &stream, const auto &pair) {
stream << pair.first << ": " << pair.second;
});
[&](auto &stream, const auto &pair) { stream << pair.first << ": " << pair.second; });
os << "}";
}
return os << "]";
@@ -292,9 +286,7 @@ std::ostream &operator<<(std::ostream &os, const UnboundedEdge &edge) {
if (edge.properties.size() > 0) {
os << " {";
utils::PrintIterable(os, edge.properties, ", ",
[&](auto &stream, const auto &pair) {
stream << pair.first << ": " << pair.second;
});
[&](auto &stream, const auto &pair) { stream << pair.first << ": " << pair.second; });
os << "}";
}
return os << "]";
@@ -339,9 +331,7 @@ std::ostream &operator<<(std::ostream &os, const Value &value) {
case Value::Type::Map:
os << "{";
utils::PrintIterable(os, value.ValueMap(), ", ",
[](auto &stream, const auto &pair) {
stream << pair.first << ": " << pair.second;
});
[](auto &stream, const auto &pair) { stream << pair.first << ": " << pair.second; });
return os << "}";
case Value::Type::Vertex:
return os << value.ValueVertex();

View File

@@ -33,9 +33,7 @@ class Id {
int64_t id_;
};
inline bool operator==(const Id &id1, const Id &id2) {
return id1.AsInt() == id2.AsInt();
}
inline bool operator==(const Id &id1, const Id &id2) { return id1.AsInt() == id2.AsInt(); }
inline bool operator!=(const Id &id1, const Id &id2) { return !(id1 == id2); }
@@ -84,13 +82,10 @@ struct Path {
// into the collection and puts that index into `indices`. A multiplier is
// added to switch between positive and negative indices (that define edge
// direction).
auto add_element = [this](auto &collection, const auto &element,
int multiplier, int offset) {
auto add_element = [this](auto &collection, const auto &element, int multiplier, int offset) {
auto found =
std::find_if(collection.begin(), collection.end(),
[&](const auto &e) { return e.id == element.id; });
indices.emplace_back(multiplier *
(std::distance(collection.begin(), found) + offset));
std::find_if(collection.begin(), collection.end(), [&](const auto &e) { return e.id == element.id; });
indices.emplace_back(multiplier * (std::distance(collection.begin(), found) + offset));
if (found == collection.end()) collection.push_back(element);
};
@@ -125,19 +120,7 @@ class Value {
Value() : type_(Type::Null) {}
/** Types that can be stored in a Value. */
enum class Type : unsigned {
Null,
Bool,
Int,
Double,
String,
List,
Map,
Vertex,
Edge,
UnboundedEdge,
Path
};
enum class Type : unsigned { Null, Bool, Int, Double, String, List, Map, Vertex, Edge, UnboundedEdge, Path };
// constructors for primitive types
Value(bool value) : type_(Type::Bool) { bool_v = value; }
@@ -146,47 +129,29 @@ class Value {
Value(double value) : type_(Type::Double) { double_v = value; }
// constructors for non-primitive types
Value(const std::string &value) : type_(Type::String) {
new (&string_v) std::string(value);
}
Value(const std::string &value) : type_(Type::String) { new (&string_v) std::string(value); }
Value(const char *value) : Value(std::string(value)) {}
Value(const std::vector<Value> &value) : type_(Type::List) {
new (&list_v) std::vector<Value>(value);
}
Value(const std::vector<Value> &value) : type_(Type::List) { new (&list_v) std::vector<Value>(value); }
Value(const std::map<std::string, Value> &value) : type_(Type::Map) {
new (&map_v) std::map<std::string, Value>(value);
}
Value(const Vertex &value) : type_(Type::Vertex) {
new (&vertex_v) Vertex(value);
}
Value(const Vertex &value) : type_(Type::Vertex) { new (&vertex_v) Vertex(value); }
Value(const Edge &value) : type_(Type::Edge) { new (&edge_v) Edge(value); }
Value(const UnboundedEdge &value) : type_(Type::UnboundedEdge) {
new (&unbounded_edge_v) UnboundedEdge(value);
}
Value(const UnboundedEdge &value) : type_(Type::UnboundedEdge) { new (&unbounded_edge_v) UnboundedEdge(value); }
Value(const Path &value) : type_(Type::Path) { new (&path_v) Path(value); }
// move constructors for non-primitive values
Value(std::string &&value) noexcept : type_(Type::String) {
new (&string_v) std::string(std::move(value));
}
Value(std::vector<Value> &&value) noexcept : type_(Type::List) {
new (&list_v) std::vector<Value>(std::move(value));
}
Value(std::string &&value) noexcept : type_(Type::String) { new (&string_v) std::string(std::move(value)); }
Value(std::vector<Value> &&value) noexcept : type_(Type::List) { new (&list_v) std::vector<Value>(std::move(value)); }
Value(std::map<std::string, Value> &&value) noexcept : type_(Type::Map) {
new (&map_v) std::map<std::string, Value>(std::move(value));
}
Value(Vertex &&value) noexcept : type_(Type::Vertex) {
new (&vertex_v) Vertex(std::move(value));
}
Value(Edge &&value) noexcept : type_(Type::Edge) {
new (&edge_v) Edge(std::move(value));
}
Value(Vertex &&value) noexcept : type_(Type::Vertex) { new (&vertex_v) Vertex(std::move(value)); }
Value(Edge &&value) noexcept : type_(Type::Edge) { new (&edge_v) Edge(std::move(value)); }
Value(UnboundedEdge &&value) noexcept : type_(Type::UnboundedEdge) {
new (&unbounded_edge_v) UnboundedEdge(std::move(value));
}
Value(Path &&value) noexcept : type_(Type::Path) {
new (&path_v) Path(std::move(value));
}
Value(Path &&value) noexcept : type_(Type::Path) { new (&path_v) Path(std::move(value)); }
Value &operator=(const Value &other);
Value &operator=(Value &&other) noexcept;

View File

@@ -4,8 +4,7 @@
namespace communication {
Buffer::Buffer()
: data_(kBufferInitialSize, 0), read_end_(this), write_end_(this) {}
Buffer::Buffer() : data_(kBufferInitialSize, 0), read_end_(this), write_end_(this) {}
Buffer::ReadEnd::ReadEnd(Buffer *buffer) : buffer_(buffer) {}
@@ -21,9 +20,7 @@ void Buffer::ReadEnd::Clear() { buffer_->Clear(); }
Buffer::WriteEnd::WriteEnd(Buffer *buffer) : buffer_(buffer) {}
io::network::StreamBuffer Buffer::WriteEnd::Allocate() {
return buffer_->Allocate();
}
io::network::StreamBuffer Buffer::WriteEnd::Allocate() { return buffer_->Allocate(); }
void Buffer::WriteEnd::Written(size_t len) { buffer_->Written(len); }

View File

@@ -195,8 +195,7 @@ bool Client::Write(const uint8_t *data, size_t len, bool have_more) {
}
bool Client::Write(const std::string &str, bool have_more) {
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(),
have_more);
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(), have_more);
}
const io::network::Endpoint &Client::endpoint() { return socket_.endpoint(); }
@@ -224,12 +223,9 @@ void ClientInputStream::Clear() { client_.ClearData(); }
ClientOutputStream::ClientOutputStream(Client &client) : client_(client) {}
bool ClientOutputStream::Write(const uint8_t *data, size_t len,
bool have_more) {
bool ClientOutputStream::Write(const uint8_t *data, size_t len, bool have_more) {
return client_.Write(data, len, have_more);
}
bool ClientOutputStream::Write(const std::string &str, bool have_more) {
return client_.Write(str, have_more);
}
bool ClientOutputStream::Write(const std::string &str, bool have_more) { return client_.Write(str, have_more); }
} // namespace communication

View File

@@ -19,21 +19,16 @@ ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) {
}
}
ClientContext::ClientContext(const std::string &key_file,
const std::string &cert_file)
: ClientContext(true) {
ClientContext::ClientContext(const std::string &key_file, const std::string &cert_file) : ClientContext(true) {
if (key_file != "" && cert_file != "") {
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(),
SSL_FILETYPE_PEM) == 1,
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load client certificate from file: {}", cert_file);
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(),
SSL_FILETYPE_PEM) == 1,
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load client private key from file: ", key_file);
}
}
ClientContext::ClientContext(ClientContext &&other) noexcept
: use_ssl_(other.use_ssl_), ctx_(other.ctx_) {
ClientContext::ClientContext(ClientContext &&other) noexcept : use_ssl_(other.use_ssl_), ctx_(other.ctx_) {
other.use_ssl_ = false;
other.ctx_ = nullptr;
}
@@ -69,9 +64,8 @@ bool ClientContext::use_ssl() { return use_ssl_; }
ServerContext::ServerContext() : use_ssl_(false), ctx_(nullptr) {}
ServerContext::ServerContext(const std::string &key_file,
const std::string &cert_file,
const std::string &ca_file, bool verify_peer)
ServerContext::ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file,
bool verify_peer)
: use_ssl_(true),
#if OPENSSL_VERSION_NUMBER < 0x10100000L
ctx_(SSL_CTX_new(SSLv23_server_method()))
@@ -81,11 +75,9 @@ ServerContext::ServerContext(const std::string &key_file,
{
// TODO (mferencevic): add support for encrypted private keys
// TODO (mferencevic): add certificate revocation list (CRL)
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(),
SSL_FILETYPE_PEM) == 1,
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load server certificate from file: {}", cert_file);
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(),
SSL_FILETYPE_PEM) == 1,
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load server private key from file: {}", key_file);
// Disable legacy SSL support. Other options can be seen here:
@@ -94,29 +86,25 @@ ServerContext::ServerContext(const std::string &key_file,
if (ca_file != "") {
// Load the certificate authority file.
MG_ASSERT(
SSL_CTX_load_verify_locations(ctx_, ca_file.c_str(), nullptr) == 1,
"Couldn't load certificate authority from file: {}", ca_file);
MG_ASSERT(SSL_CTX_load_verify_locations(ctx_, ca_file.c_str(), nullptr) == 1,
"Couldn't load certificate authority from file: {}", ca_file);
if (verify_peer) {
// Add the CA to list of accepted CAs that is sent to the client.
STACK_OF(X509_NAME) *ca_names = SSL_load_client_CA_file(ca_file.c_str());
MG_ASSERT(ca_names != nullptr,
"Couldn't load certificate authority from file: {}", ca_file);
MG_ASSERT(ca_names != nullptr, "Couldn't load certificate authority from file: {}", ca_file);
// `ca_names` doesn' need to be free'd because we pass it to
// `SSL_CTX_set_client_CA_list`:
// https://mta.openssl.org/pipermail/openssl-users/2015-May/001363.html
SSL_CTX_set_client_CA_list(ctx_, ca_names);
// Enable verification of the client certificate.
SSL_CTX_set_verify(
ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
SSL_CTX_set_verify(ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
}
}
}
ServerContext::ServerContext(ServerContext &&other) noexcept
: use_ssl_(other.use_ssl_), ctx_(other.ctx_) {
ServerContext::ServerContext(ServerContext &&other) noexcept : use_ssl_(other.use_ssl_), ctx_(other.ctx_) {
other.use_ssl_ = false;
other.ctx_ = nullptr;
}

View File

@@ -72,8 +72,8 @@ class ServerContext final {
* to check that the client certificate is valid, then you need to supply a
* valid `ca_file` as well.
*/
ServerContext(const std::string &key_file, const std::string &cert_file,
const std::string &ca_file = "", bool verify_peer = false);
ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file = "",
bool verify_peer = false);
// This object can't be copied because the underlying SSL implementation is
// messy and ownership can't be handled correctly.

View File

@@ -28,10 +28,7 @@ void LockingFunction(int mode, int n, const char *file, int line) {
}
}
unsigned long IdFunction() {
return (unsigned long)std::hash<std::thread::id>()(
std::this_thread::get_id());
}
unsigned long IdFunction() { return (unsigned long)std::hash<std::thread::id>()(std::this_thread::get_id()); }
void SetupThreading() {
crypto_locks.resize(CRYPTO_num_locks());
@@ -58,8 +55,7 @@ SSLInit::SSLInit() {
ERR_load_crypto_strings();
// Ignore SIGPIPE.
MG_ASSERT(utils::SignalIgnore(utils::Signal::Pipe),
"Couldn't ignore SIGPIPE!");
MG_ASSERT(utils::SignalIgnore(utils::Signal::Pipe), "Couldn't ignore SIGPIPE!");
SetupThreading();
}

View File

@@ -40,8 +40,7 @@ class Listener final {
using SessionHandler = Session<TSession, TSessionData>;
public:
Listener(TSessionData *data, ServerContext *context,
int inactivity_timeout_sec, const std::string &service_name,
Listener(TSessionData *data, ServerContext *context, int inactivity_timeout_sec, const std::string &service_name,
size_t workers_count)
: data_(data),
alive_(false),
@@ -77,8 +76,8 @@ class Listener final {
int fd = connection.fd();
// Create a new Session for the connection.
sessions_.push_back(std::make_unique<SessionHandler>(
std::move(connection), data_, context_, inactivity_timeout_sec_));
sessions_.push_back(
std::make_unique<SessionHandler>(std::move(connection), data_, context_, inactivity_timeout_sec_));
// Register the connection in Epoll.
// We want to listen to an incoming event which is edge triggered and
@@ -86,8 +85,7 @@ class Listener final {
// concurrently and that is why we use `EPOLLONESHOT`, for a detailed
// description what are the problems and why this is correct see:
// https://idea.popcount.org/2017-02-20-epoll-is-fundamentally-broken-12/
epoll_.Add(fd, EPOLLIN | EPOLLET | EPOLLRDHUP | EPOLLONESHOT,
sessions_.back().get());
epoll_.Add(fd, EPOLLIN | EPOLLET | EPOLLRDHUP | EPOLLONESHOT, sessions_.back().get());
}
/**
@@ -117,8 +115,7 @@ class Listener final {
std::lock_guard<utils::SpinLock> guard(lock_);
for (auto &session : sessions_) {
if (session->TimedOut()) {
spdlog::warn("{} session associated with {} timed out",
service_name, session->socket().endpoint());
spdlog::warn("{} session associated with {} timed out", service_name, session->socket().endpoint());
// Here we shutdown the socket to terminate any leftover
// blocking `Write` calls and to signal an event that the
// session is closed. Session cleanup will be done in the event
@@ -178,8 +175,7 @@ class Listener final {
// dereference it here. It is safe to dereference the pointer because
// this design guarantees that there will never be an event that has
// a stale Session pointer.
SessionHandler &session =
*reinterpret_cast<SessionHandler *>(event.data.ptr);
SessionHandler &session = *reinterpret_cast<SessionHandler *>(event.data.ptr);
// Process epoll events. We use epoll in edge-triggered mode so we process
// all events here. Only one of the `if` statements must be executed
@@ -192,20 +188,16 @@ class Listener final {
;
} else if (event.events & EPOLLRDHUP) {
// The client closed the connection.
spdlog::info("{} client {} closed the connection.", service_name_,
session.socket().endpoint());
spdlog::info("{} client {} closed the connection.", service_name_, session.socket().endpoint());
CloseSession(session);
} else if (!(event.events & EPOLLIN) ||
event.events & (EPOLLHUP | EPOLLERR)) {
} else if (!(event.events & EPOLLIN) || event.events & (EPOLLHUP | EPOLLERR)) {
// There was an error on the server side.
spdlog::error("Error occured in {} session associated with {}",
service_name_, session.socket().endpoint());
spdlog::error("Error occured in {} session associated with {}", service_name_, session.socket().endpoint());
CloseSession(session);
} else {
// Unhandled epoll event.
spdlog::error(
"Unhandled event occured in {} session associated with {} events: {}",
service_name_, session.socket().endpoint(), event.events);
spdlog::error("Unhandled event occured in {} session associated with {} events: {}", service_name_,
session.socket().endpoint(), event.events);
CloseSession(session);
}
}
@@ -215,13 +207,11 @@ class Listener final {
if (session.Execute()) {
// Session execution done, rearm epoll to send events for this
// socket.
epoll_.Modify(session.socket().fd(),
EPOLLIN | EPOLLET | EPOLLRDHUP | EPOLLONESHOT, &session);
epoll_.Modify(session.socket().fd(), EPOLLIN | EPOLLET | EPOLLRDHUP | EPOLLONESHOT, &session);
return false;
}
} catch (const SessionClosedException &e) {
spdlog::info("{} client {} closed the connection.", service_name_,
session.socket().endpoint());
spdlog::info("{} client {} closed the connection.", service_name_, session.socket().endpoint());
CloseSession(session);
return false;
} catch (const std::exception &e) {
@@ -245,11 +235,9 @@ class Listener final {
epoll_.Delete(session.socket().fd());
std::lock_guard<utils::SpinLock> guard(lock_);
auto it = std::find_if(sessions_.begin(), sessions_.end(),
[&](const auto &l) { return l.get() == &session; });
auto it = std::find_if(sessions_.begin(), sessions_.end(), [&](const auto &l) { return l.get() == &session; });
MG_ASSERT(it != sessions_.end(),
"Trying to remove session that is not found in sessions!");
MG_ASSERT(it != sessions_.end(), "Trying to remove session that is not found in sessions!");
int i = it - sessions_.begin();
swap(sessions_[i], sessions_.back());

View File

@@ -27,9 +27,7 @@ class ResultStreamFaker {
void Header(const std::vector<std::string> &fields) { header_ = fields; }
void Result(const std::vector<communication::bolt::Value> &values) {
results_.push_back(values);
}
void Result(const std::vector<communication::bolt::Value> &values) { results_.push_back(values); }
void Result(const std::vector<query::TypedValue> &values) {
std::vector<communication::bolt::Value> bvalues;
@@ -42,16 +40,12 @@ class ResultStreamFaker {
results_.push_back(std::move(bvalues));
}
void Summary(
const std::map<std::string, communication::bolt::Value> &summary) {
summary_ = summary;
}
void Summary(const std::map<std::string, communication::bolt::Value> &summary) { summary_ = summary; }
void Summary(const std::map<std::string, query::TypedValue> &summary) {
std::map<std::string, communication::bolt::Value> bsummary;
for (const auto &item : summary) {
auto maybe_value =
glue::ToBoltValue(item.second, *store_, storage::View::NEW);
auto maybe_value = glue::ToBoltValue(item.second, *store_, storage::View::NEW);
MG_ASSERT(maybe_value.HasValue());
bsummary.insert({item.first, std::move(*maybe_value)});
}
@@ -64,8 +58,7 @@ class ResultStreamFaker {
const auto &GetSummary() const { return summary_; }
friend std::ostream &operator<<(std::ostream &os,
const ResultStreamFaker &results) {
friend std::ostream &operator<<(std::ostream &os, const ResultStreamFaker &results) {
auto decoded_value_to_string = [](const auto &value) {
std::stringstream ss;
ss << value;
@@ -73,21 +66,16 @@ class ResultStreamFaker {
};
const std::vector<std::string> &header = results.GetHeader();
std::vector<int> column_widths(header.size());
std::transform(header.begin(), header.end(), column_widths.begin(),
[](const auto &s) { return s.size(); });
std::transform(header.begin(), header.end(), column_widths.begin(), [](const auto &s) { return s.size(); });
// convert all the results into strings, and track max column width
auto &results_data = results.GetResults();
std::vector<std::vector<std::string>> result_strings(
results_data.size(), std::vector<std::string>(column_widths.size()));
for (int row_ind = 0; row_ind < static_cast<int>(results_data.size());
++row_ind) {
for (int col_ind = 0; col_ind < static_cast<int>(column_widths.size());
++col_ind) {
std::string string_val =
decoded_value_to_string(results_data[row_ind][col_ind]);
column_widths[col_ind] =
std::max(column_widths[col_ind], (int)string_val.size());
std::vector<std::vector<std::string>> result_strings(results_data.size(),
std::vector<std::string>(column_widths.size()));
for (int row_ind = 0; row_ind < static_cast<int>(results_data.size()); ++row_ind) {
for (int col_ind = 0; col_ind < static_cast<int>(column_widths.size()); ++col_ind) {
std::string string_val = decoded_value_to_string(results_data[row_ind][col_ind]);
column_widths[col_ind] = std::max(column_widths[col_ind], (int)string_val.size());
result_strings[row_ind][col_ind] = string_val;
}
}
@@ -96,15 +84,13 @@ class ResultStreamFaker {
// first define some helper functions
auto emit_horizontal_line = [&]() {
os << "+";
for (auto col_width : column_widths)
os << std::string((unsigned long)col_width + 2, '-') << "+";
for (auto col_width : column_widths) os << std::string((unsigned long)col_width + 2, '-') << "+";
os << std::endl;
};
auto emit_result_vec = [&](const std::vector<std::string> result_vec) {
os << "| ";
for (int col_ind = 0; col_ind < static_cast<int>(column_widths.size());
++col_ind) {
for (int col_ind = 0; col_ind < static_cast<int>(column_widths.size()); ++col_ind) {
const std::string &res = result_vec[col_ind];
os << res << std::string(column_widths[col_ind] - res.size(), ' ');
os << " | ";
@@ -123,9 +109,7 @@ class ResultStreamFaker {
// output the summary
os << "Query summary: {";
utils::PrintIterable(os, results.GetSummary(), ", ",
[&](auto &stream, const auto &kv) {
stream << kv.first << ": " << kv.second;
});
[&](auto &stream, const auto &kv) { stream << kv.first << ": " << kv.second; });
os << "}" << std::endl;
return os;

View File

@@ -46,14 +46,12 @@ class Server final {
* Constructs and binds server to endpoint, operates on session data and
* invokes workers_count workers
*/
Server(const io::network::Endpoint &endpoint, TSessionData *session_data,
ServerContext *context, int inactivity_timeout_sec,
const std::string &service_name,
Server(const io::network::Endpoint &endpoint, TSessionData *session_data, ServerContext *context,
int inactivity_timeout_sec, const std::string &service_name,
size_t workers_count = std::thread::hardware_concurrency())
: alive_(false),
endpoint_(endpoint),
listener_(session_data, context, inactivity_timeout_sec, service_name,
workers_count),
listener_(session_data, context, inactivity_timeout_sec, service_name, workers_count),
service_name_(service_name) {}
~Server() {
@@ -69,8 +67,7 @@ class Server final {
Server &operator=(Server &&) = delete;
const auto &endpoint() const {
MG_ASSERT(alive_,
"You can't get the server endpoint when it's not running!");
MG_ASSERT(alive_, "You can't get the server endpoint when it's not running!");
return socket_.endpoint();
}
@@ -138,8 +135,7 @@ class Server final {
// Connection is not available anymore or configuration failed.
return;
}
spdlog::info("Accepted a {} connection from {}", service_name_,
s->endpoint());
spdlog::info("Accepted a {} connection from {}", service_name_, s->endpoint());
listener_.AddConnection(std::move(*s));
}

View File

@@ -35,22 +35,17 @@ using InputStream = Buffer::ReadEnd;
*/
class OutputStream final {
public:
OutputStream(
std::function<bool(const uint8_t *, size_t, bool)> write_function)
: write_function_(write_function) {}
OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function) : write_function_(write_function) {}
OutputStream(const OutputStream &) = delete;
OutputStream(OutputStream &&) = delete;
OutputStream &operator=(const OutputStream &) = delete;
OutputStream &operator=(OutputStream &&) = delete;
bool Write(const uint8_t *data, size_t len, bool have_more = false) {
return write_function_(data, len, have_more);
}
bool Write(const uint8_t *data, size_t len, bool have_more = false) { return write_function_(data, len, have_more); }
bool Write(const std::string &str, bool have_more = false) {
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(),
have_more);
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(), have_more);
}
private:
@@ -65,14 +60,10 @@ class OutputStream final {
template <class TSession, class TSessionData>
class Session final {
public:
Session(io::network::Socket &&socket, TSessionData *data,
ServerContext *context, int inactivity_timeout_sec)
Session(io::network::Socket &&socket, TSessionData *data, ServerContext *context, int inactivity_timeout_sec)
: socket_(std::move(socket)),
output_stream_([this](const uint8_t *data, size_t len, bool have_more) {
return Write(data, len, have_more);
}),
session_(data, socket_.endpoint(), input_buffer_.read_end(),
&output_stream_),
output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
session_(data, socket_.endpoint(), input_buffer_.read_end(), &output_stream_),
inactivity_timeout_sec_(inactivity_timeout_sec) {
// Set socket options.
// The socket is set to be a non-blocking socket. We use the socket in a
@@ -243,8 +234,7 @@ class Session final {
bool TimedOut() {
std::unique_lock<utils::SpinLock> guard(lock_);
if (execution_active_) return false;
return last_event_time_ + std::chrono::seconds(inactivity_timeout_sec_) <
std::chrono::steady_clock::now();
return last_event_time_ + std::chrono::seconds(inactivity_timeout_sec_) < std::chrono::steady_clock::now();
}
/**
@@ -316,8 +306,7 @@ class Session final {
TSession session_;
// Time of the last event and associated lock.
std::chrono::time_point<std::chrono::steady_clock> last_event_time_{
std::chrono::steady_clock::now()};
std::chrono::time_point<std::chrono::steady_clock> last_event_time_{std::chrono::steady_clock::now()};
bool execution_active_{false};
utils::SpinLock lock_;
const int inactivity_timeout_sec_;

View File

@@ -20,9 +20,7 @@
template <typename TElement>
class RingBuffer {
public:
explicit RingBuffer(int capacity) : capacity_(capacity) {
buffer_ = std::make_unique<TElement[]>(capacity_);
}
explicit RingBuffer(int capacity) : capacity_(capacity) { buffer_ = std::make_unique<TElement[]>(capacity_); }
RingBuffer(const RingBuffer &) = delete;
RingBuffer(RingBuffer &&) = delete;

View File

@@ -28,6 +28,10 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::REPLICATION;
case query::AuthQuery::Privilege::LOCK_PATH:
return auth::Permission::LOCK_PATH;
case query::AuthQuery::Privilege::READ_FILE:
return auth::Permission::READ_FILE;
case query::AuthQuery::Privilege::FREE_MEMORY:
return auth::Permission::FREE_MEMORY;
case query::AuthQuery::Privilege::AUTH:
return auth::Permission::AUTH;
}

View File

@@ -32,34 +32,28 @@ query::TypedValue ToTypedValue(const Value &value) {
}
case Value::Type::Map: {
std::map<std::string, query::TypedValue> map;
for (const auto &kv : value.ValueMap())
map.emplace(kv.first, ToTypedValue(kv.second));
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToTypedValue(kv.second));
return query::TypedValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException(
"Unsupported conversion from Value to TypedValue");
throw communication::bolt::ValueException("Unsupported conversion from Value to TypedValue");
}
}
storage::Result<communication::bolt::Vertex> ToBoltVertex(
const query::VertexAccessor &vertex, const storage::Storage &db,
storage::View view) {
storage::Result<communication::bolt::Vertex> ToBoltVertex(const query::VertexAccessor &vertex,
const storage::Storage &db, storage::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
storage::Result<communication::bolt::Edge> ToBoltEdge(
const query::EdgeAccessor &edge, const storage::Storage &db,
storage::View view) {
storage::Result<communication::bolt::Edge> ToBoltEdge(const query::EdgeAccessor &edge, const storage::Storage &db,
storage::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
storage::Result<Value> ToBoltValue(const query::TypedValue &value,
const storage::Storage &db,
storage::View view) {
storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage::Storage &db, storage::View view) {
switch (value.type()) {
case query::TypedValue::Type::Null:
return Value();
@@ -90,20 +84,17 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value,
}
return Value(std::move(map));
}
case query::TypedValue::Type::Vertex:
{
case query::TypedValue::Type::Vertex: {
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::TypedValue::Type::Edge:
{
case query::TypedValue::Type::Edge: {
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::TypedValue::Type::Path:
{
case query::TypedValue::Type::Path: {
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
@@ -111,9 +102,8 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value,
}
}
storage::Result<communication::bolt::Vertex> ToBoltVertex(
const storage::VertexAccessor &vertex, const storage::Storage &db,
storage::View view) {
storage::Result<communication::bolt::Vertex> ToBoltVertex(const storage::VertexAccessor &vertex,
const storage::Storage &db, storage::View view) {
auto id = communication::bolt::Id::FromUint(vertex.Gid().AsUint());
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
@@ -131,12 +121,10 @@ storage::Result<communication::bolt::Vertex> ToBoltVertex(
return communication::bolt::Vertex{id, labels, properties};
}
storage::Result<communication::bolt::Edge> ToBoltEdge(
const storage::EdgeAccessor &edge, const storage::Storage &db,
storage::View view) {
storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccessor &edge, const storage::Storage &db,
storage::View view) {
auto id = communication::bolt::Id::FromUint(edge.Gid().AsUint());
auto from =
communication::bolt::Id::FromUint(edge.FromVertex().Gid().AsUint());
auto from = communication::bolt::Id::FromUint(edge.FromVertex().Gid().AsUint());
auto to = communication::bolt::Id::FromUint(edge.ToVertex().Gid().AsUint());
auto type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
@@ -148,8 +136,8 @@ storage::Result<communication::bolt::Edge> ToBoltEdge(
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::Result<communication::bolt::Path> ToBoltPath(
const query::Path &path, const storage::Storage &db, storage::View view) {
storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, const storage::Storage &db,
storage::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());
for (const auto &v : path.vertices()) {
@@ -182,22 +170,19 @@ storage::PropertyValue ToPropertyValue(const Value &value) {
case Value::Type::List: {
std::vector<storage::PropertyValue> vec;
vec.reserve(value.ValueList().size());
for (const auto &value : value.ValueList())
vec.emplace_back(ToPropertyValue(value));
for (const auto &value : value.ValueList()) vec.emplace_back(ToPropertyValue(value));
return storage::PropertyValue(std::move(vec));
}
case Value::Type::Map: {
std::map<std::string, storage::PropertyValue> map;
for (const auto &kv : value.ValueMap())
map.emplace(kv.first, ToPropertyValue(kv.second));
for (const auto &kv : value.ValueMap()) map.emplace(kv.first, ToPropertyValue(kv.second));
return storage::PropertyValue(std::move(map));
}
case Value::Type::Vertex:
case Value::Type::Edge:
case Value::Type::UnboundedEdge:
case Value::Type::Path:
throw communication::bolt::ValueException(
"Unsupported conversion from Value to PropertyValue");
throw communication::bolt::ValueException("Unsupported conversion from Value to PropertyValue");
}
}

View File

@@ -21,35 +21,32 @@ namespace glue {
/// @param storage::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
storage::Result<communication::bolt::Vertex> ToBoltVertex(
const storage::VertexAccessor &vertex, const storage::Storage &db,
storage::View view);
storage::Result<communication::bolt::Vertex> ToBoltVertex(const storage::VertexAccessor &vertex,
const storage::Storage &db, storage::View view);
/// @param storage::EdgeAccessor for converting to communication::bolt::Edge.
/// @param storage::Storage for getting edge type and property names.
/// @param storage::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
storage::Result<communication::bolt::Edge> ToBoltEdge(
const storage::EdgeAccessor &edge, const storage::Storage &db,
storage::View view);
storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccessor &edge, const storage::Storage &db,
storage::View view);
/// @param query::Path for converting to communication::bolt::Path.
/// @param storage::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::Result<communication::bolt::Path> ToBoltPath(
const query::Path &path, const storage::Storage &db, storage::View view);
storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, const storage::Storage &db,
storage::View view);
/// @param query::TypedValue for converting to communication::bolt::Value.
/// @param storage::Storage for ToBoltVertex and ToBoltEdge.
/// @param storage::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
storage::Result<communication::bolt::Value> ToBoltValue(
const query::TypedValue &value, const storage::Storage &db,
storage::View view);
storage::Result<communication::bolt::Value> ToBoltValue(const query::TypedValue &value, const storage::Storage &db,
storage::View view);
query::TypedValue ToTypedValue(const communication::bolt::Value &value);

View File

@@ -17,17 +17,13 @@
inline void LoadConfig(const std::string &product_name) {
namespace fs = std::filesystem;
std::vector<fs::path> configs = {fs::path("/etc/memgraph/memgraph.conf")};
if (getenv("HOME") != nullptr)
configs.emplace_back(fs::path(getenv("HOME")) /
fs::path(".memgraph/config"));
if (getenv("HOME") != nullptr) configs.emplace_back(fs::path(getenv("HOME")) / fs::path(".memgraph/config"));
{
auto memgraph_config = getenv("MEMGRAPH_CONFIG");
if (memgraph_config != nullptr) {
auto path = fs::path(memgraph_config);
MG_ASSERT(
fs::exists(path),
"MEMGRAPH_CONFIG environment variable set to nonexisting path: {}",
path.generic_string());
MG_ASSERT(fs::exists(path), "MEMGRAPH_CONFIG environment variable set to nonexisting path: {}",
path.generic_string());
configs.emplace_back(path);
}
}
@@ -35,8 +31,7 @@ inline void LoadConfig(const std::string &product_name) {
std::vector<std::string> flagfile_arguments;
for (const auto &config : configs)
if (fs::exists(config)) {
flagfile_arguments.emplace_back(
std::string("--flag-file=" + config.generic_string()));
flagfile_arguments.emplace_back(std::string("--flag-file=" + config.generic_string()));
}
int custom_argc = static_cast<int>(flagfile_arguments.size()) + 1;

View File

@@ -25,10 +25,8 @@ Endpoint::IpFamily Endpoint::GetIpFamily(const std::string &ip_address) {
}
}
std::optional<std::pair<std::string, uint16_t>>
Endpoint::ParseSocketOrIpAddress(
const std::string &address,
const std::optional<uint16_t> default_port = {}) {
std::optional<std::pair<std::string, uint16_t>> Endpoint::ParseSocketOrIpAddress(
const std::string &address, const std::optional<uint16_t> default_port = {}) {
/// expected address format:
/// - "ip_address:port_number"
/// - "ip_address"
@@ -80,8 +78,7 @@ std::string Endpoint::SocketAddress() const {
}
Endpoint::Endpoint() {}
Endpoint::Endpoint(std::string ip_address, uint16_t port)
: address(std::move(ip_address)), port(port) {
Endpoint::Endpoint(std::string ip_address, uint16_t port) : address(std::move(ip_address)), port(port) {
IpFamily ip_family = GetIpFamily(address);
if (ip_family == IpFamily::NONE) {
throw NetworkError("Not a valid IPv4 or IPv6 address: {}", ip_address);

View File

@@ -21,13 +21,11 @@ class Epoll {
public:
using Event = struct epoll_event;
Epoll(bool set_cloexec = false)
: epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) {
Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) {
// epoll_create1 returns an error if there is a logical error in our code
// (for example invalid flags) or if there is irrecoverable error. In both
// cases it is best to terminate.
MG_ASSERT(epoll_fd_ != -1, "Error on epoll create: ({}) {}", errno,
strerror(errno));
MG_ASSERT(epoll_fd_ != -1, "Error on epoll create: ({}) {}", errno, strerror(errno));
}
/**
@@ -42,15 +40,13 @@ class Epoll {
Event event;
event.events = events;
event.data.ptr = ptr;
int status = epoll_ctl(epoll_fd_, (modify ? EPOLL_CTL_MOD : EPOLL_CTL_ADD),
fd, &event);
int status = epoll_ctl(epoll_fd_, (modify ? EPOLL_CTL_MOD : EPOLL_CTL_ADD), fd, &event);
// epoll_ctl can return an error on our logical error or on irrecoverable
// error. There is a third possibility that some system limit is reached. In
// that case we could return an erorr and close connection. Chances of
// reaching system limit in normally working memgraph is extremely unlikely,
// so it is correct to terminate even in that case.
MG_ASSERT(!status, "Error on epoll {}: ({}) {}",
(modify ? "modify" : "add"), errno, strerror(errno));
MG_ASSERT(!status, "Error on epoll {}: ({}) {}", (modify ? "modify" : "add"), errno, strerror(errno));
}
/**
@@ -60,9 +56,7 @@ class Epoll {
* @param events epoll events mask
* @param ptr pointer to the associated event handler
*/
void Modify(int fd, uint32_t events, void *ptr) {
Add(fd, events, ptr, true);
}
void Modify(int fd, uint32_t events, void *ptr) { Add(fd, events, ptr, true); }
/**
* This function deletes a file descriptor that is listened for events.
@@ -76,8 +70,7 @@ class Epoll {
// that case we could return an erorr and close connection. Chances of
// reaching system limit in normally working memgraph is extremely unlikely,
// so it is correct to terminate even in that case.
MG_ASSERT(!status, "Error on epoll delete: ({}) {}", errno,
strerror(errno));
MG_ASSERT(!status, "Error on epoll delete: ({}) {}", errno, strerror(errno));
}
/**
@@ -91,8 +84,7 @@ class Epoll {
int Wait(Event *events, int max_events, int timeout) {
auto num_events = epoll_wait(epoll_fd_, events, max_events, timeout);
// If this check fails there was logical error in our code.
MG_ASSERT(num_events != -1 || errno == EINTR,
"Error on epoll wait: ({}) {}", errno, strerror(errno));
MG_ASSERT(num_events != -1 || errno == EINTR, "Error on epoll wait: ({}) {}", errno, strerror(errno));
// num_events can be -1 if errno was EINTR (epoll_wait interrupted by signal
// handler). We treat that as no events, so we return 0.
return num_events == -1 ? 0 : num_events;

View File

@@ -8,4 +8,4 @@ class NetworkError : public utils::StacktraceException {
public:
using utils::StacktraceException::StacktraceException;
};
}
} // namespace io::network

View File

@@ -59,8 +59,7 @@ bool Socket::IsOpen() const { return socket_ != -1; }
bool Socket::Connect(const Endpoint &endpoint) {
if (socket_ != -1) return false;
auto info = AddrInfo::Get(endpoint.address.c_str(),
std::to_string(endpoint.port).c_str());
auto info = AddrInfo::Get(endpoint.address.c_str(), std::to_string(endpoint.port).c_str());
for (struct addrinfo *it = info; it != nullptr; it = it->ai_next) {
int sfd = socket(it->ai_family, it->ai_socktype, it->ai_protocol);
@@ -83,8 +82,7 @@ bool Socket::Connect(const Endpoint &endpoint) {
bool Socket::Bind(const Endpoint &endpoint) {
if (socket_ != -1) return false;
auto info = AddrInfo::Get(endpoint.address.c_str(),
std::to_string(endpoint.port).c_str());
auto info = AddrInfo::Get(endpoint.address.c_str(), std::to_string(endpoint.port).c_str());
for (struct addrinfo *it = info; it != nullptr; it = it->ai_next) {
int sfd = socket(it->ai_family, it->ai_socktype, it->ai_protocol);
@@ -130,38 +128,30 @@ void Socket::SetNonBlocking() {
int flags = fcntl(socket_, F_GETFL, 0);
MG_ASSERT(flags != -1, "Can't get socket mode");
flags |= O_NONBLOCK;
MG_ASSERT(fcntl(socket_, F_SETFL, flags) != -1,
"Can't set socket nonblocking");
MG_ASSERT(fcntl(socket_, F_SETFL, flags) != -1, "Can't set socket nonblocking");
}
void Socket::SetKeepAlive() {
int optval = 1;
socklen_t optlen = sizeof(optval);
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen),
"Can't set socket keep alive");
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen), "Can't set socket keep alive");
optval = 20; // wait 20s before sending keep-alive packets
MG_ASSERT(
!setsockopt(socket_, SOL_TCP, TCP_KEEPIDLE, (void *)&optval, optlen),
"Can't set socket keep alive");
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPIDLE, (void *)&optval, optlen), "Can't set socket keep alive");
optval = 4; // 4 keep-alive packets must fail to close
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPCNT, (void *)&optval, optlen),
"Can't set socket keep alive");
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPCNT, (void *)&optval, optlen), "Can't set socket keep alive");
optval = 15; // send keep-alive packets every 15s
MG_ASSERT(
!setsockopt(socket_, SOL_TCP, TCP_KEEPINTVL, (void *)&optval, optlen),
"Can't set socket keep alive");
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPINTVL, (void *)&optval, optlen), "Can't set socket keep alive");
}
void Socket::SetNoDelay() {
int optval = 1;
socklen_t optlen = sizeof(optval);
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_NODELAY, (void *)&optval, optlen),
"Can't set socket no delay");
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_NODELAY, (void *)&optval, optlen), "Can't set socket no delay");
}
void Socket::SetTimeout(long sec, long usec) {
@@ -169,11 +159,9 @@ void Socket::SetTimeout(long sec, long usec) {
tv.tv_sec = sec;
tv.tv_usec = usec;
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)),
"Can't set socket timeout");
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)), "Can't set socket timeout");
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)),
"Can't set socket timeout");
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)), "Can't set socket timeout");
}
int Socket::ErrorStatus() const {
@@ -238,8 +226,7 @@ bool Socket::Write(const uint8_t *data, size_t len, bool have_more) {
}
bool Socket::Write(const std::string &s, bool have_more) {
return Write(reinterpret_cast<const uint8_t *>(s.data()), s.size(),
have_more);
return Write(reinterpret_cast<const uint8_t *>(s.data()), s.size(), have_more);
}
ssize_t Socket::Read(void *buffer, size_t len, bool nonblock) {

View File

@@ -14,4 +14,4 @@ struct StreamBuffer {
uint8_t *data;
size_t len;
};
}
} // namespace io::network

View File

@@ -23,9 +23,8 @@ std::string ResolveHostname(std::string hostname) {
int addr_result;
addrinfo *servinfo;
MG_ASSERT((addr_result =
getaddrinfo(hostname.c_str(), NULL, &hints, &servinfo)) == 0,
"Error with getaddrinfo: {}", gai_strerror(addr_result));
MG_ASSERT((addr_result = getaddrinfo(hostname.c_str(), NULL, &hints, &servinfo)) == 0, "Error with getaddrinfo: {}",
gai_strerror(addr_result));
MG_ASSERT(servinfo, "Could not resolve address: {}", hostname);
std::string address;

View File

@@ -12,18 +12,16 @@ struct KVStore::impl {
rocksdb::Options options;
};
KVStore::KVStore(std::filesystem::path storage)
: pimpl_(std::make_unique<impl>()) {
KVStore::KVStore(std::filesystem::path storage) : pimpl_(std::make_unique<impl>()) {
pimpl_->storage = storage;
if (!utils::EnsureDir(pimpl_->storage))
throw KVStoreError("Folder for the key-value store " +
pimpl_->storage.string() + " couldn't be initialized!");
throw KVStoreError("Folder for the key-value store " + pimpl_->storage.string() + " couldn't be initialized!");
pimpl_->options.create_if_missing = true;
rocksdb::DB *db = nullptr;
auto s = rocksdb::DB::Open(pimpl_->options, storage.c_str(), &db);
if (!s.ok())
throw KVStoreError("RocksDB couldn't be initialized inside " +
storage.string() + " -- " + std::string(s.ToString()));
throw KVStoreError("RocksDB couldn't be initialized inside " + storage.string() + " -- " +
std::string(s.ToString()));
pimpl_->db.reset(db);
}
@@ -72,19 +70,16 @@ bool KVStore::DeleteMultiple(const std::vector<std::string> &keys) {
}
bool KVStore::DeletePrefix(const std::string &prefix) {
std::unique_ptr<rocksdb::Iterator> iter = std::unique_ptr<rocksdb::Iterator>(
pimpl_->db->NewIterator(rocksdb::ReadOptions()));
for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix);
iter->Next()) {
if (!pimpl_->db->Delete(rocksdb::WriteOptions(), iter->key()).ok())
return false;
std::unique_ptr<rocksdb::Iterator> iter =
std::unique_ptr<rocksdb::Iterator>(pimpl_->db->NewIterator(rocksdb::ReadOptions()));
for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next()) {
if (!pimpl_->db->Delete(rocksdb::WriteOptions(), iter->key()).ok()) return false;
}
return true;
}
bool KVStore::PutAndDeleteMultiple(
const std::map<std::string, std::string> &items,
const std::vector<std::string> &keys) {
bool KVStore::PutAndDeleteMultiple(const std::map<std::string, std::string> &items,
const std::vector<std::string> &keys) {
rocksdb::WriteBatch batch;
for (const auto &item : items) {
batch.Put(item.first, item.second);
@@ -105,22 +100,16 @@ struct KVStore::iterator::impl {
std::pair<std::string, std::string> disk_prop;
};
KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix,
bool at_end)
KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix, bool at_end)
: pimpl_(std::make_unique<impl>()) {
pimpl_->kvstore = kvstore;
pimpl_->prefix = prefix;
pimpl_->it = std::unique_ptr<rocksdb::Iterator>(
pimpl_->kvstore->pimpl_->db->NewIterator(rocksdb::ReadOptions()));
pimpl_->it = std::unique_ptr<rocksdb::Iterator>(pimpl_->kvstore->pimpl_->db->NewIterator(rocksdb::ReadOptions()));
pimpl_->it->Seek(pimpl_->prefix);
if (!pimpl_->it->Valid() || !pimpl_->it->key().starts_with(pimpl_->prefix) ||
at_end)
pimpl_->it = nullptr;
if (!pimpl_->it->Valid() || !pimpl_->it->key().starts_with(pimpl_->prefix) || at_end) pimpl_->it = nullptr;
}
KVStore::iterator::iterator(KVStore::iterator &&other) {
pimpl_ = std::move(other.pimpl_);
}
KVStore::iterator::iterator(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); }
KVStore::iterator::~iterator() {}
@@ -131,24 +120,19 @@ KVStore::iterator &KVStore::iterator::operator=(KVStore::iterator &&other) {
KVStore::iterator &KVStore::iterator::operator++() {
pimpl_->it->Next();
if (!pimpl_->it->Valid() || !pimpl_->it->key().starts_with(pimpl_->prefix))
pimpl_->it = nullptr;
if (!pimpl_->it->Valid() || !pimpl_->it->key().starts_with(pimpl_->prefix)) pimpl_->it = nullptr;
return *this;
}
bool KVStore::iterator::operator==(const iterator &other) const {
return pimpl_->kvstore == other.pimpl_->kvstore &&
pimpl_->prefix == other.pimpl_->prefix &&
return pimpl_->kvstore == other.pimpl_->kvstore && pimpl_->prefix == other.pimpl_->prefix &&
pimpl_->it == other.pimpl_->it;
}
bool KVStore::iterator::operator!=(const iterator &other) const {
return !(*this == other);
}
bool KVStore::iterator::operator!=(const iterator &other) const { return !(*this == other); }
KVStore::iterator::reference KVStore::iterator::operator*() {
pimpl_->disk_prop = {pimpl_->it->key().ToString(),
pimpl_->it->value().ToString()};
pimpl_->disk_prop = {pimpl_->it->key().ToString(), pimpl_->it->value().ToString()};
return pimpl_->disk_prop;
}
@@ -166,8 +150,7 @@ size_t KVStore::Size(const std::string &prefix) {
return size;
}
bool KVStore::CompactRange(const std::string &begin_prefix,
const std::string &end_prefix) {
bool KVStore::CompactRange(const std::string &begin_prefix, const std::string &end_prefix) {
rocksdb::CompactRangeOptions options;
rocksdb::Slice begin(begin_prefix);
rocksdb::Slice end(end_prefix);

View File

@@ -114,8 +114,7 @@ class KVStore final {
* @return true if the items have been successfully stored and deleted.
* In case of any error false is going to be returned.
*/
bool PutAndDeleteMultiple(const std::map<std::string, std::string> &items,
const std::vector<std::string> &keys);
bool PutAndDeleteMultiple(const std::map<std::string, std::string> &items, const std::vector<std::string> &keys);
/**
* Returns total number of stored (key, value) pairs. The function takes an
@@ -140,8 +139,7 @@ class KVStore final {
*
* @return - true if the compaction finished successfully, false otherwise.
*/
bool CompactRange(const std::string &begin_prefix,
const std::string &end_prefix);
bool CompactRange(const std::string &begin_prefix, const std::string &end_prefix);
/**
* Custom prefix-based iterator over kvstore.
@@ -150,17 +148,14 @@ class KVStore final {
* and behaves as if all of those pairs are stored in a single iterable
* collection of std::pair<std::string, std::string>.
*/
class iterator final
: public std::iterator<
std::input_iterator_tag, // iterator_category
std::pair<std::string, std::string>, // value_type
long, // difference_type
const std::pair<std::string, std::string> *, // pointer
const std::pair<std::string, std::string> & // reference
> {
class iterator final : public std::iterator<std::input_iterator_tag, // iterator_category
std::pair<std::string, std::string>, // value_type
long, // difference_type
const std::pair<std::string, std::string> *, // pointer
const std::pair<std::string, std::string> & // reference
> {
public:
explicit iterator(const KVStore *kvstore, const std::string &prefix = "",
bool at_end = false);
explicit iterator(const KVStore *kvstore, const std::string &prefix = "", bool at_end = false);
iterator(const iterator &other) = delete;
@@ -191,13 +186,9 @@ class KVStore final {
std::unique_ptr<impl> pimpl_;
};
iterator begin(const std::string &prefix = "") {
return iterator(this, prefix);
}
iterator begin(const std::string &prefix = "") { return iterator(this, prefix); }
iterator end(const std::string &prefix = "") {
return iterator(this, prefix, true);
}
iterator end(const std::string &prefix = "") { return iterator(this, prefix, true); }
private:
struct impl;

View File

@@ -26,8 +26,7 @@ std::optional<std::string> KVStore::Get(const std::string &key) const noexcept {
}
bool KVStore::Delete(const std::string &key) {
LOG_FATAL(
"Unsupported operation (KVStore::Delete) -- this is a dummy kvstore");
LOG_FATAL("Unsupported operation (KVStore::Delete) -- this is a dummy kvstore");
}
bool KVStore::DeleteMultiple(const std::vector<std::string> &keys) {
@@ -42,9 +41,8 @@ bool KVStore::DeletePrefix(const std::string &prefix) {
"dummy kvstore");
}
bool KVStore::PutAndDeleteMultiple(
const std::map<std::string, std::string> &items,
const std::vector<std::string> &keys) {
bool KVStore::PutAndDeleteMultiple(const std::map<std::string, std::string> &items,
const std::vector<std::string> &keys) {
LOG_FATAL(
"Unsupported operation (KVStore::PutAndDeleteMultiple) -- this is a "
"dummy kvstore");
@@ -54,13 +52,9 @@ bool KVStore::PutAndDeleteMultiple(
struct KVStore::iterator::impl {};
KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix,
bool at_end)
: pimpl_(new impl()) {}
KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix, bool at_end) : pimpl_(new impl()) {}
KVStore::iterator::iterator(KVStore::iterator &&other) {
pimpl_ = std::move(other.pimpl_);
}
KVStore::iterator::iterator(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); }
KVStore::iterator::~iterator() {}
@@ -77,9 +71,7 @@ KVStore::iterator &KVStore::iterator::operator++() {
bool KVStore::iterator::operator==(const iterator &other) const { return true; }
bool KVStore::iterator::operator!=(const iterator &other) const {
return false;
}
bool KVStore::iterator::operator!=(const iterator &other) const { return false; }
KVStore::iterator::reference KVStore::iterator::operator*() {
LOG_FATAL(
@@ -99,8 +91,7 @@ bool KVStore::iterator::IsValid() { return false; }
size_t KVStore::Size(const std::string &prefix) { return 0; }
bool KVStore::CompactRange(const std::string &begin_prefix,
const std::string &end_prefix) {
bool KVStore::CompactRange(const std::string &begin_prefix, const std::string &end_prefix) {
LOG_FATAL(
"Unsupported operation (KVStore::Compact) -- this is a "
"dummy kvstore");

View File

@@ -1,4 +1,5 @@
#include <algorithm>
#include <atomic>
#include <chrono>
#include <csignal>
#include <cstdint>
@@ -23,15 +24,19 @@
#include "py/py.hpp"
#include "query/exceptions.hpp"
#include "query/interpreter.hpp"
#include "query/plan/operator.hpp"
#include "query/procedure/module.hpp"
#include "query/procedure/py_module.hpp"
#include "requests/requests.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/view.hpp"
#include "telemetry/telemetry.hpp"
#include "utils/event_counter.hpp"
#include "utils/file.hpp"
#include "utils/flag_validation.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/readable_size.hpp"
#include "utils/signals.hpp"
#include "utils/string.hpp"
#include "utils/sysinfo/memory.hpp"
@@ -63,25 +68,19 @@
#endif
// Bolt server flags.
DEFINE_string(bolt_address, "0.0.0.0",
"IP address on which the Bolt server should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687,
"Port on which the Bolt server should listen.",
DEFINE_string(bolt_address, "0.0.0.0", "IP address on which the Bolt server should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687, "Port on which the Bolt server should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
DEFINE_VALIDATED_int32(
bolt_num_workers, std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers used by the Bolt server. By default, this will be the "
"number of processing units available on the machine.",
FLAG_IN_RANGE(1, INT32_MAX));
DEFINE_VALIDATED_int32(
bolt_session_inactivity_timeout, 1800,
"Time in seconds after which inactive Bolt sessions will be "
"closed.",
FLAG_IN_RANGE(1, INT32_MAX));
DEFINE_string(bolt_cert_file, "",
"Certificate file which should be used for the Bolt server.");
DEFINE_string(bolt_key_file, "",
"Key file which should be used for the Bolt server.");
DEFINE_VALIDATED_int32(bolt_num_workers, std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers used by the Bolt server. By default, this will be the "
"number of processing units available on the machine.",
FLAG_IN_RANGE(1, INT32_MAX));
DEFINE_VALIDATED_int32(bolt_session_inactivity_timeout, 1800,
"Time in seconds after which inactive Bolt sessions will be "
"closed.",
FLAG_IN_RANGE(1, INT32_MAX));
DEFINE_string(bolt_cert_file, "", "Certificate file which should be used for the Bolt server.");
DEFINE_string(bolt_key_file, "", "Key file which should be used for the Bolt server.");
DEFINE_string(bolt_server_name_for_init, "",
"Server name which the database should send to the client in the "
"Bolt INIT message.");
@@ -89,26 +88,20 @@ DEFINE_string(bolt_server_name_for_init, "",
// General purpose flags.
// NOTE: The `data_directory` flag must be the same here and in
// `mg_import_csv`. If you change it, make sure to change it there as well.
DEFINE_string(data_directory, "mg_data",
"Path to directory in which to save all permanent data.");
DEFINE_HIDDEN_string(
log_link_basename, "",
"Basename used for symlink creation to the last log file.");
DEFINE_string(data_directory, "mg_data", "Path to directory in which to save all permanent data.");
DEFINE_HIDDEN_string(log_link_basename, "", "Basename used for symlink creation to the last log file.");
DEFINE_uint64(memory_warning_threshold, 1024,
"Memory warning threshold, in MB. If Memgraph detects there is "
"less available RAM it will log a warning. Set to 0 to "
"disable.");
// Storage flags.
DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30,
"Storage garbage collector interval (in seconds).",
DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).",
FLAG_IN_RANGE(1, 24 * 3600));
// NOTE: The `storage_properties_on_edges` flag must be the same here and in
// `mg_import_csv`. If you change it, make sure to change it there as well.
DEFINE_bool(storage_properties_on_edges, false,
"Controls whether edges have properties.");
DEFINE_bool(storage_recover_on_startup, false,
"Controls whether the storage recovers persisted data on startup.");
DEFINE_bool(storage_properties_on_edges, false, "Controls whether edges have properties.");
DEFINE_bool(storage_recover_on_startup, false, "Controls whether the storage recovers persisted data on startup.");
DEFINE_VALIDATED_uint64(storage_snapshot_interval_sec, 0,
"Storage snapshot creation interval (in seconds). Set "
"to 0 to disable periodic snapshot creation.",
@@ -116,21 +109,15 @@ DEFINE_VALIDATED_uint64(storage_snapshot_interval_sec, 0,
DEFINE_bool(storage_wal_enabled, false,
"Controls whether the storage uses write-ahead-logging. To enable "
"WAL periodic snapshots must be enabled.");
DEFINE_VALIDATED_uint64(storage_snapshot_retention_count, 3,
"The number of snapshots that should always be kept.",
DEFINE_VALIDATED_uint64(storage_snapshot_retention_count, 3, "The number of snapshots that should always be kept.",
FLAG_IN_RANGE(1, 1000000));
DEFINE_VALIDATED_uint64(storage_wal_file_size_kib,
storage::Config::Durability().wal_file_size_kibibytes,
"Minimum file size of each WAL file.",
FLAG_IN_RANGE(1, 1000 * 1024));
DEFINE_VALIDATED_uint64(
storage_wal_file_flush_every_n_tx,
storage::Config::Durability().wal_file_flush_every_n_tx,
"Issue a 'fsync' call after this amount of transactions are written to the "
"WAL file. Set to 1 for fully synchronous operation.",
FLAG_IN_RANGE(1, 1000000));
DEFINE_bool(storage_snapshot_on_exit, false,
"Controls whether the storage creates another snapshot on exit.");
DEFINE_VALIDATED_uint64(storage_wal_file_size_kib, storage::Config::Durability().wal_file_size_kibibytes,
"Minimum file size of each WAL file.", FLAG_IN_RANGE(1, 1000 * 1024));
DEFINE_VALIDATED_uint64(storage_wal_file_flush_every_n_tx, storage::Config::Durability().wal_file_flush_every_n_tx,
"Issue a 'fsync' call after this amount of transactions are written to the "
"WAL file. Set to 1 for fully synchronous operation.",
FLAG_IN_RANGE(1, 1000000));
DEFINE_bool(storage_snapshot_on_exit, false, "Controls whether the storage creates another snapshot on exit.");
DEFINE_bool(telemetry_enabled, false,
"Set to true to enable telemetry. We collect information about the "
@@ -141,13 +128,11 @@ DEFINE_bool(telemetry_enabled, false,
// Audit logging flags.
#ifdef MG_ENTERPRISE
DEFINE_bool(audit_enabled, false, "Set to true to enable audit logging.");
DEFINE_VALIDATED_int32(audit_buffer_size, audit::kBufferSizeDefault,
"Maximum number of items in the audit log buffer.",
DEFINE_VALIDATED_int32(audit_buffer_size, audit::kBufferSizeDefault, "Maximum number of items in the audit log buffer.",
FLAG_IN_RANGE(1, INT32_MAX));
DEFINE_VALIDATED_int32(
audit_buffer_flush_interval_ms, audit::kBufferFlushIntervalMillisDefault,
"Interval (in milliseconds) used for flushing the audit log buffer.",
FLAG_IN_RANGE(10, INT32_MAX));
DEFINE_VALIDATED_int32(audit_buffer_flush_interval_ms, audit::kBufferFlushIntervalMillisDefault,
"Interval (in milliseconds) used for flushing the audit log buffer.",
FLAG_IN_RANGE(10, INT32_MAX));
#endif
// Query flags.
@@ -155,41 +140,54 @@ DEFINE_uint64(query_execution_timeout_sec, 180,
"Maximum allowed query execution time. Queries exceeding this "
"limit will be aborted. Value of 0 means no limit.");
DEFINE_VALIDATED_string(
query_modules_directory, "",
"Directory where modules with custom query procedures are stored.", {
if (value.empty()) return true;
if (utils::DirExists(value)) return true;
std::cout << "Expected --" << flagname << " to point to a directory."
<< std::endl;
return false;
});
namespace {
std::vector<std::filesystem::path> query_modules_directories;
} // namespace
DEFINE_VALIDATED_string(query_modules_directory, "",
"Directory where modules with custom query procedures are stored. "
"NOTE: Multiple comma-separated directories can be defined.",
{
query_modules_directories.clear();
if (value.empty()) return true;
const auto directories = utils::Split(value, ",");
for (const auto &dir : directories) {
if (!utils::DirExists(dir)) {
std::cout << "Expected --" << flagname << " to point to directories." << std::endl;
std::cout << dir << " is not a directory." << std::endl;
return false;
}
}
query_modules_directories.reserve(directories.size());
std::transform(directories.begin(), directories.end(),
std::back_inserter(query_modules_directories),
[](const auto &dir) { return dir; });
return true;
});
// Logging flags
DEFINE_bool(also_log_to_stderr, false,
"Log messages go to stderr in addition to logfiles");
DEFINE_bool(also_log_to_stderr, false, "Log messages go to stderr in addition to logfiles");
DEFINE_string(log_file, "", "Path to where the log should be stored.");
DEFINE_uint64(
memory_limit, 0,
"Total memory limit in MiB. Set to 0 to use the default values which are 100\% of the phyisical memory if the swap "
"is enabled and 90\% of the physical memory otherwise.");
namespace {
constexpr std::array log_level_mappings{
std::pair{"TRACE", spdlog::level::trace},
std::pair{"DEBUG", spdlog::level::debug},
std::pair{"INFO", spdlog::level::info},
std::pair{"WARNING", spdlog::level::warn},
std::pair{"ERROR", spdlog::level::err},
std::pair{"CRITICAL", spdlog::level::critical}};
std::pair{"TRACE", spdlog::level::trace}, std::pair{"DEBUG", spdlog::level::debug},
std::pair{"INFO", spdlog::level::info}, std::pair{"WARNING", spdlog::level::warn},
std::pair{"ERROR", spdlog::level::err}, std::pair{"CRITICAL", spdlog::level::critical}};
std::string GetAllowedLogLevelsString() {
std::vector<std::string> allowed_log_levels;
allowed_log_levels.reserve(log_level_mappings.size());
std::transform(log_level_mappings.cbegin(), log_level_mappings.cend(),
std::back_inserter(allowed_log_levels),
std::transform(log_level_mappings.cbegin(), log_level_mappings.cend(), std::back_inserter(allowed_log_levels),
[](const auto &mapping) { return mapping.first; });
return utils::Join(allowed_log_levels, ", ");
}
const std::string log_level_help_string = fmt::format(
"Minimum log level. Allowed values: {}", GetAllowedLogLevelsString());
const std::string log_level_help_string =
fmt::format("Minimum log level. Allowed values: {}", GetAllowedLogLevelsString());
} // namespace
DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
@@ -199,11 +197,8 @@ DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
}
if (std::find_if(log_level_mappings.cbegin(), log_level_mappings.cend(),
[&](const auto &mapping) {
return mapping.first == value;
}) == log_level_mappings.cend()) {
std::cout << "Invalid value for log level. Allowed values: "
<< GetAllowedLogLevelsString() << std::endl;
[&](const auto &mapping) { return mapping.first == value; }) == log_level_mappings.cend()) {
std::cout << "Invalid value for log level. Allowed values: " << GetAllowedLogLevelsString() << std::endl;
return false;
}
@@ -212,9 +207,8 @@ DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
namespace {
void ParseLogLevel() {
const auto mapping_iter = std::find_if(
log_level_mappings.cbegin(), log_level_mappings.cend(),
[](const auto &mapping) { return mapping.first == FLAGS_log_level; });
const auto mapping_iter = std::find_if(log_level_mappings.cbegin(), log_level_mappings.cend(),
[](const auto &mapping) { return mapping.first == FLAGS_log_level; });
MG_ASSERT(mapping_iter != log_level_mappings.cend(), "Invalid log level");
spdlog::set_level(mapping_iter->second);
@@ -227,8 +221,7 @@ void ConfigureLogging() {
std::vector<spdlog::sink_ptr> loggers;
if (FLAGS_also_log_to_stderr) {
loggers.emplace_back(
std::make_shared<spdlog::sinks::stderr_color_sink_mt>());
loggers.emplace_back(std::make_shared<spdlog::sinks::stderr_color_sink_mt>());
}
if (!FLAGS_log_file.empty()) {
@@ -240,16 +233,33 @@ void ConfigureLogging() {
local_time = localtime(&current_time);
loggers.emplace_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(
FLAGS_log_file, local_time->tm_hour, local_time->tm_min, false,
log_retention_count));
FLAGS_log_file, local_time->tm_hour, local_time->tm_min, false, log_retention_count));
}
spdlog::set_default_logger(std::make_shared<spdlog::logger>(
"memgraph_log", loggers.begin(), loggers.end()));
spdlog::set_default_logger(std::make_shared<spdlog::logger>("memgraph_log", loggers.begin(), loggers.end()));
spdlog::flush_on(spdlog::level::trace);
ParseLogLevel();
}
int64_t GetMemoryLimit() {
if (FLAGS_memory_limit == 0) {
auto maybe_total_memory = utils::sysinfo::TotalMemory();
MG_ASSERT(maybe_total_memory, "Failed to fetch the total physical memory");
const auto maybe_swap_memory = utils::sysinfo::SwapTotalMemory();
MG_ASSERT(maybe_swap_memory, "Failed to fetch the total swap memory");
if (*maybe_swap_memory == 0) {
// take only 90% of the total memory
*maybe_total_memory *= 9;
*maybe_total_memory /= 10;
}
return *maybe_total_memory * 1024;
}
// We parse the memory as MiB every time
return FLAGS_memory_limit * 1024 * 1024;
}
} // namespace
/// Encapsulates Dbms and Interpreter that are passed through the network server
@@ -258,13 +268,9 @@ void ConfigureLogging() {
struct SessionData {
// Explicit constructor here to ensure that pointers to all objects are
// supplied.
SessionData(storage::Storage *db,
query::InterpreterContext *interpreter_context, auth::Auth *auth,
SessionData(storage::Storage *db, query::InterpreterContext *interpreter_context, auth::Auth *auth,
audit::Log *audit_log)
: db(db),
interpreter_context(interpreter_context),
auth(auth),
audit_log(audit_log) {}
: db(db), interpreter_context(interpreter_context), auth(auth), audit_log(audit_log) {}
storage::Storage *db;
query::InterpreterContext *interpreter_context;
auth::Auth *auth;
@@ -274,24 +280,19 @@ struct SessionData {
struct SessionData {
// Explicit constructor here to ensure that pointers to all objects are
// supplied.
SessionData(storage::Storage *db,
query::InterpreterContext *interpreter_context)
SessionData(storage::Storage *db, query::InterpreterContext *interpreter_context)
: db(db), interpreter_context(interpreter_context) {}
storage::Storage *db;
query::InterpreterContext *interpreter_context;
};
#endif
class BoltSession final
: public communication::bolt::Session<communication::InputStream,
communication::OutputStream> {
class BoltSession final : public communication::bolt::Session<communication::InputStream, communication::OutputStream> {
public:
BoltSession(SessionData *data, const io::network::Endpoint &endpoint,
communication::InputStream *input_stream,
BoltSession(SessionData *data, const io::network::Endpoint &endpoint, communication::InputStream *input_stream,
communication::OutputStream *output_stream)
: communication::bolt::Session<communication::InputStream,
communication::OutputStream>(
input_stream, output_stream),
: communication::bolt::Session<communication::InputStream, communication::OutputStream>(input_stream,
output_stream),
db_(data->db),
interpreter_(data->interpreter_context),
#ifdef MG_ENTERPRISE
@@ -301,8 +302,7 @@ class BoltSession final
endpoint_(endpoint) {
}
using communication::bolt::Session<communication::InputStream,
communication::OutputStream>::TEncoder;
using communication::bolt::Session<communication::InputStream, communication::OutputStream>::TEncoder;
void BeginTransaction() override { interpreter_.BeginTransaction(); }
@@ -311,15 +311,11 @@ class BoltSession final
void RollbackTransaction() override { interpreter_.RollbackTransaction(); }
std::pair<std::vector<std::string>, std::optional<int>> Interpret(
const std::string &query,
const std::map<std::string, communication::bolt::Value> &params)
override {
const std::string &query, const std::map<std::string, communication::bolt::Value> &params) override {
std::map<std::string, storage::PropertyValue> params_pv;
for (const auto &kv : params)
params_pv.emplace(kv.first, glue::ToPropertyValue(kv.second));
for (const auto &kv : params) params_pv.emplace(kv.first, glue::ToPropertyValue(kv.second));
#ifdef MG_ENTERPRISE
audit_log_->Record(endpoint_.address, user_ ? user_->username() : "", query,
storage::PropertyValue(params_pv));
audit_log_->Record(endpoint_.address, user_ ? user_->username() : "", query, storage::PropertyValue(params_pv));
#endif
try {
auto result = interpreter_.Prepare(query, params_pv);
@@ -327,8 +323,7 @@ class BoltSession final
if (user_) {
const auto &permissions = user_->GetPermissions();
for (const auto &privilege : result.privileges) {
if (permissions.Has(glue::PrivilegeToPermission(privilege)) !=
auth::PermissionLevel::GRANT) {
if (permissions.Has(glue::PrivilegeToPermission(privilege)) != auth::PermissionLevel::GRANT) {
interpreter_.Abort();
throw communication::bolt::ClientError(
"You are not authorized to execute this query! Please contact "
@@ -346,23 +341,20 @@ class BoltSession final
}
}
std::map<std::string, communication::bolt::Value> Pull(
TEncoder *encoder, std::optional<int> n,
std::optional<int> qid) override {
std::map<std::string, communication::bolt::Value> Pull(TEncoder *encoder, std::optional<int> n,
std::optional<int> qid) override {
TypedValueResultStream stream(encoder, db_);
return PullResults(stream, n, qid);
}
std::map<std::string, communication::bolt::Value> Discard(
std::optional<int> n, std::optional<int> qid) override {
std::map<std::string, communication::bolt::Value> Discard(std::optional<int> n, std::optional<int> qid) override {
DiscardValueResultStream stream;
return PullResults(stream, n, qid);
}
void Abort() override { interpreter_.Abort(); }
bool Authenticate(const std::string &username,
const std::string &password) override {
bool Authenticate(const std::string &username, const std::string &password) override {
#ifdef MG_ENTERPRISE
if (!auth_->HasUsers()) return true;
user_ = auth_->Authenticate(username, password);
@@ -379,14 +371,13 @@ class BoltSession final
private:
template <typename TStream>
std::map<std::string, communication::bolt::Value> PullResults(
TStream &stream, std::optional<int> n, std::optional<int> qid) {
std::map<std::string, communication::bolt::Value> PullResults(TStream &stream, std::optional<int> n,
std::optional<int> qid) {
try {
const auto &summary = interpreter_.Pull(&stream, n, qid);
std::map<std::string, communication::bolt::Value> decoded_summary;
for (const auto &kv : summary) {
auto maybe_value =
glue::ToBoltValue(kv.second, *db_, storage::View::NEW);
auto maybe_value = glue::ToBoltValue(kv.second, *db_, storage::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::Error::DELETED_OBJECT:
@@ -394,8 +385,7 @@ class BoltSession final
case storage::Error::VERTEX_HAS_EDGES:
case storage::Error::PROPERTIES_DISABLED:
case storage::Error::NONEXISTENT_OBJECT:
throw communication::bolt::ClientError(
"Unexpected storage error when streaming summary.");
throw communication::bolt::ClientError("Unexpected storage error when streaming summary.");
}
}
decoded_summary.emplace(kv.first, std::move(*maybe_value));
@@ -412,8 +402,7 @@ class BoltSession final
/// before forwarding the calls to original TEncoder.
class TypedValueResultStream {
public:
TypedValueResultStream(TEncoder *encoder, const storage::Storage *db)
: encoder_(encoder), db_(db) {}
TypedValueResultStream(TEncoder *encoder, const storage::Storage *db) : encoder_(encoder), db_(db) {}
void Result(const std::vector<query::TypedValue> &values) {
std::vector<communication::bolt::Value> decoded_values;
@@ -423,16 +412,13 @@ class BoltSession final
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case storage::Error::DELETED_OBJECT:
throw communication::bolt::ClientError(
"Returning a deleted object as a result.");
throw communication::bolt::ClientError("Returning a deleted object as a result.");
case storage::Error::NONEXISTENT_OBJECT:
throw communication::bolt::ClientError(
"Returning a nonexistent object as a result.");
throw communication::bolt::ClientError("Returning a nonexistent object as a result.");
case storage::Error::VERTEX_HAS_EDGES:
case storage::Error::SERIALIZATION_ERROR:
case storage::Error::PROPERTIES_DISABLED:
throw communication::bolt::ClientError(
"Unexpected storage error when streaming results.");
throw communication::bolt::ClientError("Unexpected storage error when streaming results.");
}
}
decoded_values.emplace_back(std::move(*maybe_value));
@@ -467,20 +453,17 @@ using ServerT = communication::Server<BoltSession, SessionData>;
using communication::ServerContext;
#ifdef MG_ENTERPRISE
DEFINE_string(
auth_user_or_role_name_regex, "[a-zA-Z0-9_.+-@]+",
"Set to the regular expression that each user or role name must fulfill.");
DEFINE_string(auth_user_or_role_name_regex, "[a-zA-Z0-9_.+-@]+",
"Set to the regular expression that each user or role name must fulfill.");
class AuthQueryHandler final : public query::AuthQueryHandler {
auth::Auth *auth_;
std::regex name_regex_;
public:
AuthQueryHandler(auth::Auth *auth, const std::regex &name_regex)
: auth_(auth), name_regex_(name_regex) {}
AuthQueryHandler(auth::Auth *auth, const std::regex &name_regex) : auth_(auth), name_regex_(name_regex) {}
bool CreateUser(const std::string &username,
const std::optional<std::string> &password) override {
bool CreateUser(const std::string &username, const std::optional<std::string> &password) override {
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
@@ -506,8 +489,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
}
}
void SetPassword(const std::string &username,
const std::optional<std::string> &password) override {
void SetPassword(const std::string &username, const std::optional<std::string> &password) override {
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
@@ -515,8 +497,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
std::lock_guard<std::mutex> lock(auth_->WithLock());
auto user = auth_->GetUser(username);
if (!user) {
throw query::QueryRuntimeException("User '{}' doesn't exist.",
username);
throw query::QueryRuntimeException("User '{}' doesn't exist.", username);
}
user->UpdatePassword(password);
auth_->SaveUser(*user);
@@ -581,8 +562,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
}
}
std::optional<std::string> GetRolenameForUser(
const std::string &username) override {
std::optional<std::string> GetRolenameForUser(const std::string &username) override {
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
@@ -590,8 +570,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
std::lock_guard<std::mutex> lock(auth_->WithLock());
auto user = auth_->GetUser(username);
if (!user) {
throw query::QueryRuntimeException("User '{}' doesn't exist .",
username);
throw query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
if (user->role()) return user->role()->rolename();
return std::nullopt;
@@ -600,8 +579,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
}
}
std::vector<query::TypedValue> GetUsernamesForRole(
const std::string &rolename) override {
std::vector<query::TypedValue> GetUsernamesForRole(const std::string &rolename) override {
if (!std::regex_match(rolename, name_regex_)) {
throw query::QueryRuntimeException("Invalid role name.");
}
@@ -609,8 +587,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
std::lock_guard<std::mutex> lock(auth_->WithLock());
auto role = auth_->GetRole(rolename);
if (!role) {
throw query::QueryRuntimeException("Role '{}' doesn't exist.",
rolename);
throw query::QueryRuntimeException("Role '{}' doesn't exist.", rolename);
}
std::vector<query::TypedValue> usernames;
const auto &users = auth_->AllUsersForRole(rolename);
@@ -624,8 +601,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
}
}
void SetRole(const std::string &username,
const std::string &rolename) override {
void SetRole(const std::string &username, const std::string &rolename) override {
if (!std::regex_match(username, name_regex_)) {
throw query::QueryRuntimeException("Invalid user name.");
}
@@ -636,18 +612,15 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
std::lock_guard<std::mutex> lock(auth_->WithLock());
auto user = auth_->GetUser(username);
if (!user) {
throw query::QueryRuntimeException("User '{}' doesn't exist .",
username);
throw query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
auto role = auth_->GetRole(rolename);
if (!role) {
throw query::QueryRuntimeException("Role '{}' doesn't exist .",
rolename);
throw query::QueryRuntimeException("Role '{}' doesn't exist .", rolename);
}
if (user->role()) {
throw query::QueryRuntimeException(
"User '{}' is already a member of role '{}'.", username,
user->role()->rolename());
throw query::QueryRuntimeException("User '{}' is already a member of role '{}'.", username,
user->role()->rolename());
}
user->SetRole(*role);
auth_->SaveUser(*user);
@@ -664,8 +637,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
std::lock_guard<std::mutex> lock(auth_->WithLock());
auto user = auth_->GetUser(username);
if (!user) {
throw query::QueryRuntimeException("User '{}' doesn't exist .",
username);
throw query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
user->ClearRole();
auth_->SaveUser(*user);
@@ -674,8 +646,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
}
}
std::vector<std::vector<query::TypedValue>> GetPrivileges(
const std::string &user_or_role) override {
std::vector<std::vector<query::TypedValue>> GetPrivileges(const std::string &user_or_role) override {
if (!std::regex_match(user_or_role, name_regex_)) {
throw query::QueryRuntimeException("Invalid user or role name.");
}
@@ -685,8 +656,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
auto user = auth_->GetUser(user_or_role);
auto role = auth_->GetRole(user_or_role);
if (!user && !role) {
throw query::QueryRuntimeException("User or role '{}' doesn't exist.",
user_or_role);
throw query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role);
}
if (user) {
const auto &permissions = user->GetPermissions();
@@ -709,10 +679,9 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
description.emplace_back("DENIED TO ROLE");
}
}
grants.push_back(
{query::TypedValue(auth::PermissionToString(permission)),
query::TypedValue(auth::PermissionLevelToString(effective)),
query::TypedValue(utils::Join(description, ", "))});
grants.push_back({query::TypedValue(auth::PermissionToString(permission)),
query::TypedValue(auth::PermissionLevelToString(effective)),
query::TypedValue(utils::Join(description, ", "))});
}
}
} else {
@@ -727,10 +696,9 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
} else if (effective == auth::PermissionLevel::DENY) {
description = "DENIED TO ROLE";
}
grants.push_back(
{query::TypedValue(auth::PermissionToString(permission)),
query::TypedValue(auth::PermissionLevelToString(effective)),
query::TypedValue(description)});
grants.push_back({query::TypedValue(auth::PermissionToString(permission)),
query::TypedValue(auth::PermissionLevelToString(effective)),
query::TypedValue(description)});
}
}
}
@@ -740,48 +708,40 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
}
}
void GrantPrivilege(
const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges,
[](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions->Grant(permission);
});
void GrantPrivilege(const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions->Grant(permission);
});
}
void DenyPrivilege(
const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges,
[](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions->Deny(permission);
});
void DenyPrivilege(const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions->Deny(permission);
});
}
void RevokePrivilege(
const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges,
[](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions->Revoke(permission);
});
void RevokePrivilege(const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
permissions->Revoke(permission);
});
}
private:
template <class TEditFun>
void EditPermissions(
const std::string &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges,
const TEditFun &edit_fun) {
void EditPermissions(const std::string &user_or_role, const std::vector<query::AuthQuery::Privilege> &privileges,
const TEditFun &edit_fun) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw query::QueryRuntimeException("Invalid user or role name.");
}
@@ -795,8 +755,7 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
auto user = auth_->GetUser(user_or_role);
auto role = auth_->GetRole(user_or_role);
if (!user && !role) {
throw query::QueryRuntimeException("User or role '{}' doesn't exist.",
user_or_role);
throw query::QueryRuntimeException("User or role '{}' doesn't exist.", user_or_role);
}
if (user) {
for (const auto &permission : permissions) {
@@ -818,71 +777,44 @@ class AuthQueryHandler final : public query::AuthQueryHandler {
class NoAuthInCommunity : public query::QueryRuntimeException {
public:
NoAuthInCommunity()
: query::QueryRuntimeException::QueryRuntimeException(
"Auth is not supported in Memgraph Community!") {}
: query::QueryRuntimeException::QueryRuntimeException("Auth is not supported in Memgraph Community!") {}
};
class AuthQueryHandler final : public query::AuthQueryHandler {
public:
bool CreateUser(const std::string &,
const std::optional<std::string> &) override {
throw NoAuthInCommunity();
}
bool CreateUser(const std::string &, const std::optional<std::string> &) override { throw NoAuthInCommunity(); }
bool DropUser(const std::string &) override { throw NoAuthInCommunity(); }
void SetPassword(const std::string &,
const std::optional<std::string> &) override {
throw NoAuthInCommunity();
}
void SetPassword(const std::string &, const std::optional<std::string> &) override { throw NoAuthInCommunity(); }
bool CreateRole(const std::string &) override { throw NoAuthInCommunity(); }
bool DropRole(const std::string &) override { throw NoAuthInCommunity(); }
std::vector<query::TypedValue> GetUsernames() override {
throw NoAuthInCommunity();
}
std::vector<query::TypedValue> GetUsernames() override { throw NoAuthInCommunity(); }
std::vector<query::TypedValue> GetRolenames() override {
throw NoAuthInCommunity();
}
std::vector<query::TypedValue> GetRolenames() override { throw NoAuthInCommunity(); }
std::optional<std::string> GetRolenameForUser(const std::string &) override {
throw NoAuthInCommunity();
}
std::optional<std::string> GetRolenameForUser(const std::string &) override { throw NoAuthInCommunity(); }
std::vector<query::TypedValue> GetUsernamesForRole(
const std::string &) override {
throw NoAuthInCommunity();
}
std::vector<query::TypedValue> GetUsernamesForRole(const std::string &) override { throw NoAuthInCommunity(); }
void SetRole(const std::string &, const std::string &) override {
throw NoAuthInCommunity();
}
void SetRole(const std::string &, const std::string &) override { throw NoAuthInCommunity(); }
void ClearRole(const std::string &) override { throw NoAuthInCommunity(); }
std::vector<std::vector<query::TypedValue>> GetPrivileges(
const std::string &) override {
std::vector<std::vector<query::TypedValue>> GetPrivileges(const std::string &) override { throw NoAuthInCommunity(); }
void GrantPrivilege(const std::string &, const std::vector<query::AuthQuery::Privilege> &) override {
throw NoAuthInCommunity();
}
void GrantPrivilege(
const std::string &,
const std::vector<query::AuthQuery::Privilege> &) override {
void DenyPrivilege(const std::string &, const std::vector<query::AuthQuery::Privilege> &) override {
throw NoAuthInCommunity();
}
void DenyPrivilege(
const std::string &,
const std::vector<query::AuthQuery::Privilege> &) override {
throw NoAuthInCommunity();
}
void RevokePrivilege(
const std::string &,
const std::vector<query::AuthQuery::Privilege> &) override {
void RevokePrivilege(const std::string &, const std::vector<query::AuthQuery::Privilege> &) override {
throw NoAuthInCommunity();
}
};
@@ -911,11 +843,9 @@ void InitSignalHandlers(const std::function<void()> &shutdown_fun) {
shutdown_fun();
};
MG_ASSERT(utils::SignalHandler::RegisterHandler(
utils::Signal::Terminate, shutdown, block_shutdown_signals),
MG_ASSERT(utils::SignalHandler::RegisterHandler(utils::Signal::Terminate, shutdown, block_shutdown_signals),
"Unable to register SIGTERM handler!");
MG_ASSERT(utils::SignalHandler::RegisterHandler(
utils::Signal::Interupt, shutdown, block_shutdown_signals),
MG_ASSERT(utils::SignalHandler::RegisterHandler(utils::Signal::Interupt, shutdown, block_shutdown_signals),
"Unable to register SIGINT handler!");
}
@@ -952,13 +882,10 @@ int main(int argc, char **argv) {
auto gil = py::EnsureGIL();
auto maybe_exc = py::AppendToSysPath(py_support_dir.c_str());
if (maybe_exc) {
spdlog::error("Unable to load support for embedded Python: {}",
*maybe_exc);
spdlog::error("Unable to load support for embedded Python: {}", *maybe_exc);
}
} else {
spdlog::error(
"Unable to load support for embedded Python: missing directory {}",
py_support_dir);
spdlog::error("Unable to load support for embedded Python: missing directory {}", py_support_dir);
}
} catch (const std::filesystem::filesystem_error &e) {
spdlog::error("Unable to load support for embedded Python: {}", e.what());
@@ -973,13 +900,12 @@ int main(int argc, char **argv) {
// Start memory warning logger.
utils::Scheduler mem_log_scheduler;
if (FLAGS_memory_warning_threshold > 0) {
auto free_ram = utils::sysinfo::AvailableMemoryKilobytes();
auto free_ram = utils::sysinfo::AvailableMemory();
if (free_ram) {
mem_log_scheduler.Run("Memory warning", std::chrono::seconds(3), [] {
auto free_ram = utils::sysinfo::AvailableMemoryKilobytes();
auto free_ram = utils::sysinfo::AvailableMemory();
if (free_ram && *free_ram / 1024 < FLAGS_memory_warning_threshold)
spdlog::warn("Running out of available RAM, only {} MB left",
*free_ram / 1024);
spdlog::warn("Running out of available RAM, only {} MB left", *free_ram / 1024);
});
} else {
// Kernel version for the `MemAvailable` value is from: man procfs
@@ -990,8 +916,7 @@ int main(int argc, char **argv) {
}
}
std::cout << "You are running Memgraph v" << gflags::VersionString()
<< std::endl;
std::cout << "You are running Memgraph v" << gflags::VersionString() << std::endl;
auto data_directory = std::filesystem::path(FLAGS_data_directory);
@@ -1011,73 +936,64 @@ int main(int argc, char **argv) {
auth::Auth auth{data_directory / "auth"};
// Audit log
audit::Log audit_log{data_directory / "audit", FLAGS_audit_buffer_size,
FLAGS_audit_buffer_flush_interval_ms};
audit::Log audit_log{data_directory / "audit", FLAGS_audit_buffer_size, FLAGS_audit_buffer_flush_interval_ms};
// Start the log if enabled.
if (FLAGS_audit_enabled) {
audit_log.Start();
}
// Setup SIGUSR2 to be used for reopening audit log files, when e.g. logrotate
// rotates our audit logs.
MG_ASSERT(
utils::SignalHandler::RegisterHandler(
utils::Signal::User2, [&audit_log]() { audit_log.ReopenLog(); }),
"Unable to register SIGUSR2 handler!");
MG_ASSERT(utils::SignalHandler::RegisterHandler(utils::Signal::User2, [&audit_log]() { audit_log.ReopenLog(); }),
"Unable to register SIGUSR2 handler!");
// End enterprise features initialization
#endif
// Main storage and execution engines initialization
const auto memory_limit = GetMemoryLimit();
spdlog::info("Memory limit set to {}", utils::GetReadableSize(memory_limit));
utils::total_memory_tracker.SetHardLimit(memory_limit);
// Main storage and execution engines initialization
storage::Config db_config{
.gc = {.type = storage::Config::Gc::Type::PERIODIC,
.interval = std::chrono::seconds(FLAGS_storage_gc_cycle_sec)},
.gc = {.type = storage::Config::Gc::Type::PERIODIC, .interval = std::chrono::seconds(FLAGS_storage_gc_cycle_sec)},
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
.durability = {
.storage_directory = FLAGS_data_directory,
.recover_on_startup = FLAGS_storage_recover_on_startup,
.snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit}};
.durability = {.storage_directory = FLAGS_data_directory,
.recover_on_startup = FLAGS_storage_recover_on_startup,
.snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit}};
if (FLAGS_storage_snapshot_interval_sec == 0) {
if (FLAGS_storage_wal_enabled) {
LOG_FATAL(
"In order to use write-ahead-logging you must enable "
"periodic snapshots by setting the snapshot interval to a "
"value larger than 0!");
db_config.durability.snapshot_wal_mode =
storage::Config::Durability::SnapshotWalMode::DISABLED;
db_config.durability.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::DISABLED;
}
} else {
if (FLAGS_storage_wal_enabled) {
db_config.durability.snapshot_wal_mode = storage::Config::Durability::
SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL;
db_config.durability.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL;
} else {
db_config.durability.snapshot_wal_mode =
storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT;
db_config.durability.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT;
}
db_config.durability.snapshot_interval =
std::chrono::seconds(FLAGS_storage_snapshot_interval_sec);
db_config.durability.snapshot_interval = std::chrono::seconds(FLAGS_storage_snapshot_interval_sec);
}
storage::Storage db(db_config);
query::InterpreterContext interpreter_context{&db};
query::SetExecutionTimeout(&interpreter_context,
FLAGS_query_execution_timeout_sec);
query::SetExecutionTimeout(&interpreter_context, FLAGS_query_execution_timeout_sec);
#ifdef MG_ENTERPRISE
SessionData session_data{&db, &interpreter_context, &auth, &audit_log};
#else
SessionData session_data{&db, &interpreter_context};
#endif
query::procedure::gModuleRegistry.SetModulesDirectory(
FLAGS_query_modules_directory);
query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectory();
query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories);
query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectories();
#ifdef MG_ENTERPRISE
AuthQueryHandler auth_handler(&auth,
std::regex(FLAGS_auth_user_or_role_name_regex));
AuthQueryHandler auth_handler(&auth, std::regex(FLAGS_auth_user_or_role_name_regex));
#else
AuthQueryHandler auth_handler;
#endif
@@ -1093,20 +1009,27 @@ int main(int argc, char **argv) {
spdlog::warn("Using non-secure Bolt connection (without SSL)");
}
ServerT server({FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)},
&session_data, &context, FLAGS_bolt_session_inactivity_timeout,
service_name, FLAGS_bolt_num_workers);
ServerT server({FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)}, &session_data, &context,
FLAGS_bolt_session_inactivity_timeout, service_name, FLAGS_bolt_num_workers);
// Setup telemetry
std::optional<telemetry::Telemetry> telemetry;
if (FLAGS_telemetry_enabled) {
telemetry.emplace(
"https://telemetry.memgraph.com/88b5e7e8-746a-11e8-9f85-538a9e9690cc/",
data_directory / "telemetry", std::chrono::minutes(10));
telemetry->AddCollector("db", [&db]() -> nlohmann::json {
telemetry.emplace("https://telemetry.memgraph.com/88b5e7e8-746a-11e8-9f85-538a9e9690cc/",
data_directory / "telemetry", std::chrono::minutes(10));
telemetry->AddCollector("storage", [&db]() -> nlohmann::json {
auto info = db.GetInfo();
return {{"vertices", info.vertex_count}, {"edges", info.edge_count}};
});
telemetry->AddCollector("event_counters", []() -> nlohmann::json {
nlohmann::json ret;
for (size_t i = 0; i < EventCounter::End(); ++i) {
ret[EventCounter::GetName(i)] = EventCounter::global_counters[i].load(std::memory_order_relaxed);
}
return ret;
});
telemetry->AddCollector("query_module_counters",
[]() -> nlohmann::json { return query::plan::CallProcedure::GetAndResetCounters(); });
}
// Handler for regular termination signals
@@ -1129,5 +1052,7 @@ int main(int argc, char **argv) {
// Shutdown Python
Py_Finalize();
PyMem_RawFree(program_name);
utils::total_memory_tracker.LogPeakMemoryUsage();
return 0;
}

View File

@@ -42,30 +42,22 @@ bool ValidateIdTypeOptions(const char *flagname, const std::string &value) {
// They are used to automatically load the same configuration as the main
// Memgraph binary so that the flags don't need to be specified when importing a
// CSV file on a correctly set-up Memgraph installation.
DEFINE_string(data_directory, "mg_data",
"Path to directory in which to save all permanent data.");
DEFINE_bool(storage_properties_on_edges, false,
"Controls whether relationships have properties.");
DEFINE_string(data_directory, "mg_data", "Path to directory in which to save all permanent data.");
DEFINE_bool(storage_properties_on_edges, false, "Controls whether relationships have properties.");
// CSV import flags.
DEFINE_string(array_delimiter, ";",
"Delimiter between elements of array values.");
DEFINE_string(array_delimiter, ";", "Delimiter between elements of array values.");
DEFINE_validator(array_delimiter, &ValidateControlCharacter);
DEFINE_string(delimiter, ",", "Delimiter between each field in the CSV.");
DEFINE_validator(delimiter, &ValidateControlCharacter);
DEFINE_string(quote, "\"",
"Quotation character for data in the CSV. Cannot contain '\n'");
DEFINE_string(quote, "\"", "Quotation character for data in the CSV. Cannot contain '\n'");
DEFINE_validator(quote, &ValidateControlCharacter);
DEFINE_bool(skip_duplicate_nodes, false,
"Set to true to skip duplicate nodes instead of raising an error.");
DEFINE_bool(skip_duplicate_nodes, false, "Set to true to skip duplicate nodes instead of raising an error.");
DEFINE_bool(skip_bad_relationships, false,
"Set to true to skip relationships that connect nodes that don't "
"exist instead of raising an error.");
DEFINE_bool(ignore_empty_strings, false,
"Set to true to treat empty strings as null values.");
DEFINE_bool(
ignore_extra_columns, false,
"Set to true to ignore columns that aren't specified in the header.");
DEFINE_bool(ignore_empty_strings, false, "Set to true to treat empty strings as null values.");
DEFINE_bool(ignore_extra_columns, false, "Set to true to ignore columns that aren't specified in the header.");
DEFINE_bool(trim_strings, false,
"Set to true to trim leading/trailing whitespace from all fields "
"that are loaded from the CSV file.");
@@ -75,25 +67,22 @@ DEFINE_string(id_type, "STRING",
DEFINE_validator(id_type, &ValidateIdTypeOptions);
// Arguments `--nodes` and `--relationships` can be input multiple times and are
// handled with custom parsing.
DEFINE_string(
nodes, "",
"Files that should be parsed for nodes. The CSV header will be loaded from "
"the first supplied file, all other files supplied in a single flag will "
"be treated as data files. Additional labels can be specified for the node "
"files. The flag can be specified multiple times (useful for differently "
"formatted node files). The format of this argument is: "
"[<label>[:<label>]...=]<file>[,<file>][,<file>]...");
DEFINE_string(
relationships, "",
"Files that should be parsed for relationships. The CSV header will be "
"loaded from the first supplied file, all other files supplied in a single "
"flag will be treated as data files. The relationship type can be "
"specified for the relationship files. The flag can be specified multiple "
"times (useful for differently formatted relationship files). The format "
"of this argument is: [<type>=]<file>[,<file>][,<file>]...");
DEFINE_string(nodes, "",
"Files that should be parsed for nodes. The CSV header will be loaded from "
"the first supplied file, all other files supplied in a single flag will "
"be treated as data files. Additional labels can be specified for the node "
"files. The flag can be specified multiple times (useful for differently "
"formatted node files). The format of this argument is: "
"[<label>[:<label>]...=]<file>[,<file>][,<file>]...");
DEFINE_string(relationships, "",
"Files that should be parsed for relationships. The CSV header will be "
"loaded from the first supplied file, all other files supplied in a single "
"flag will be treated as data files. The relationship type can be "
"specified for the relationship files. The flag can be specified multiple "
"times (useful for differently formatted relationship files). The format "
"of this argument is: [<type>=]<file>[,<file>][,<file>]...");
std::vector<std::string> ParseRepeatedFlag(const std::string &flagname,
int argc, char *argv[]) {
std::vector<std::string> ParseRepeatedFlag(const std::string &flagname, int argc, char *argv[]) {
std::vector<std::string> values;
for (int i = 1; i < argc; ++i) {
std::string flag(argv[i]);
@@ -132,9 +121,7 @@ struct NodeId {
std::string id_space;
};
bool operator==(const NodeId &a, const NodeId &b) {
return a.id == b.id && a.id_space == b.id_space;
}
bool operator==(const NodeId &a, const NodeId &b) { return a.id == b.id && a.id_space == b.id_space; }
std::ostream &operator<<(std::ostream &stream, const NodeId &node_id) {
if (!node_id.id_space.empty()) {
@@ -171,8 +158,7 @@ enum class CsvParserState {
EXPECT_DELIMITER,
};
bool SubstringStartsWith(const std::string_view &str, size_t pos,
const std::string_view &what) {
bool SubstringStartsWith(const std::string_view &str, size_t pos, const std::string_view &what) {
return utils::StartsWith(utils::Substr(str, pos), what);
}
@@ -262,8 +248,7 @@ std::pair<std::vector<std::string>, uint64_t> ReadRow(std::istream &stream) {
}
case CsvParserState::QUOTING: {
auto quote_now = SubstringStartsWith(line, i, FLAGS_quote);
auto quote_next =
SubstringStartsWith(line, i + FLAGS_quote.size(), FLAGS_quote);
auto quote_next = SubstringStartsWith(line, i + FLAGS_quote.size(), FLAGS_quote);
if (quote_now && quote_next) {
// This is an escaped quote character.
column += FLAGS_quote;
@@ -293,8 +278,7 @@ std::pair<std::vector<std::string>, uint64_t> ReadRow(std::istream &stream) {
state = CsvParserState::NEXT_FIELD;
i += FLAGS_delimiter.size() - 1;
} else {
throw LoadException("Expected '{}' after '{}', but got '{}'",
FLAGS_delimiter, FLAGS_quote, c);
throw LoadException("Expected '{}' after '{}', but got '{}'", FLAGS_delimiter, FLAGS_quote, c);
}
break;
}
@@ -326,8 +310,7 @@ std::pair<std::vector<std::string>, uint64_t> ReadRow(std::istream &stream) {
}
if (FLAGS_trim_strings) {
std::transform(std::begin(row), std::end(row), std::begin(row),
[](const auto &item) { return utils::Trim(item); });
std::transform(std::begin(row), std::end(row), std::begin(row), [](const auto &item) { return utils::Trim(item); });
}
return {std::move(row), lines_count};
@@ -373,13 +356,10 @@ double StringToDouble(const std::string &value) {
}
/// @throw LoadException
storage::PropertyValue StringToValue(const std::string &str,
const std::string &type) {
if (FLAGS_ignore_empty_strings && str.empty())
return storage::PropertyValue();
storage::PropertyValue StringToValue(const std::string &str, const std::string &type) {
if (FLAGS_ignore_empty_strings && str.empty()) return storage::PropertyValue();
auto convert = [](const auto &str, const auto &type) {
if (type == "integer" || type == "int" || type == "long" ||
type == "byte" || type == "short") {
if (type == "integer" || type == "int" || type == "long" || type == "byte" || type == "short") {
return storage::PropertyValue(StringToInt(str));
} else if (type == "float" || type == "double") {
return storage::PropertyValue(StringToDouble(str));
@@ -411,8 +391,7 @@ storage::PropertyValue StringToValue(const std::string &str,
std::string GetIdSpace(const std::string &type) {
// The format of this field is as follows:
// [START_|END_]ID[(<id_space>)]
std::regex format(R"(^(START_|END_)?ID(\(([^\(\)]+)\))?$)",
std::regex::extended);
std::regex format(R"(^(START_|END_)?ID(\(([^\(\)]+)\))?$)", std::regex::extended);
std::smatch res;
if (!std::regex_match(type, res, format))
throw LoadException(
@@ -424,8 +403,7 @@ std::string GetIdSpace(const std::string &type) {
}
/// @throw LoadException
void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields,
const std::vector<std::string> &row,
void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields, const std::vector<std::string> &row,
const std::vector<std::string> &additional_labels,
std::unordered_map<NodeId, storage::Gid> *node_id_map) {
std::optional<NodeId> id;
@@ -458,45 +436,32 @@ void ProcessNodeRow(storage::Storage *store, const std::vector<Field> &fields,
} else {
pv_id = storage::PropertyValue(node_id.id);
}
auto node_property =
node.SetProperty(acc.NameToProperty(field.name), pv_id);
if (!node_property.HasValue())
throw LoadException("Couldn't add property '{}' to the node",
field.name);
if (!*node_property)
throw LoadException("The property '{}' already exists", field.name);
auto node_property = node.SetProperty(acc.NameToProperty(field.name), pv_id);
if (!node_property.HasValue()) throw LoadException("Couldn't add property '{}' to the node", field.name);
if (!*node_property) throw LoadException("The property '{}' already exists", field.name);
}
id = node_id;
} else if (field.type == "LABEL") {
for (const auto &label : utils::Split(value, FLAGS_array_delimiter)) {
auto node_label = node.AddLabel(acc.NameToLabel(label));
if (!node_label.HasValue())
throw LoadException("Couldn't add label '{}' to the node", label);
if (!*node_label)
throw LoadException("The label '{}' already exists", label);
if (!node_label.HasValue()) throw LoadException("Couldn't add label '{}' to the node", label);
if (!*node_label) throw LoadException("The label '{}' already exists", label);
}
} else if (field.type != "IGNORE") {
auto node_property = node.SetProperty(acc.NameToProperty(field.name),
StringToValue(value, field.type));
if (!node_property.HasValue())
throw LoadException("Couldn't add property '{}' to the node",
field.name);
if (!*node_property)
throw LoadException("The property '{}' already exists", field.name);
auto node_property = node.SetProperty(acc.NameToProperty(field.name), StringToValue(value, field.type));
if (!node_property.HasValue()) throw LoadException("Couldn't add property '{}' to the node", field.name);
if (!*node_property) throw LoadException("The property '{}' already exists", field.name);
}
}
for (const auto &label : additional_labels) {
auto node_label = node.AddLabel(acc.NameToLabel(label));
if (!node_label.HasValue())
throw LoadException("Couldn't add label '{}' to the node", label);
if (!*node_label)
throw LoadException("The label '{}' already exists", label);
if (!node_label.HasValue()) throw LoadException("Couldn't add label '{}' to the node", label);
if (!*node_label) throw LoadException("The label '{}' already exists", label);
}
if (acc.Commit().HasError()) throw LoadException("Couldn't store the node");
}
void ProcessNodes(storage::Storage *store, const std::string &nodes_path,
std::optional<std::vector<Field>> *header,
void ProcessNodes(storage::Storage *store, const std::string &nodes_path, std::optional<std::vector<Field>> *header,
std::unordered_map<NodeId, storage::Gid> *node_id_map,
const std::vector<std::string> &additional_labels) {
std::ifstream nodes_file(nodes_path);
@@ -524,17 +489,14 @@ void ProcessNodes(storage::Storage *store, const std::string &nodes_path,
row_number += lines_count;
}
} catch (const LoadException &e) {
LOG_FATAL("Couldn't process row {} of '{}' because of: {}", row_number,
nodes_path, e.what());
LOG_FATAL("Couldn't process row {} of '{}' because of: {}", row_number, nodes_path, e.what());
}
}
/// @throw LoadException
void ProcessRelationshipsRow(
storage::Storage *store, const std::vector<Field> &fields,
const std::vector<std::string> &row,
std::optional<std::string> relationship_type,
const std::unordered_map<NodeId, storage::Gid> &node_id_map) {
void ProcessRelationshipsRow(storage::Storage *store, const std::vector<Field> &fields,
const std::vector<std::string> &row, std::optional<std::string> relationship_type,
const std::unordered_map<NodeId, storage::Gid> &node_id_map) {
std::optional<storage::Gid> start_id;
std::optional<storage::Gid> end_id;
std::map<std::string, storage::PropertyValue> properties;
@@ -576,14 +538,11 @@ void ProcessRelationshipsRow(
}
end_id = it->second;
} else if (field.type == "TYPE") {
if (relationship_type)
throw LoadException("Only one relationship TYPE must be specified");
if (relationship_type) throw LoadException("Only one relationship TYPE must be specified");
relationship_type = value;
} else if (field.type != "IGNORE") {
auto [it, inserted] =
properties.emplace(field.name, StringToValue(value, field.type));
if (!inserted)
throw LoadException("The property '{}' already exists", field.name);
auto [it, inserted] = properties.emplace(field.name, StringToValue(value, field.type));
if (!inserted) throw LoadException("The property '{}' already exists", field.name);
}
}
if (!start_id) throw LoadException("START_ID must be set");
@@ -596,18 +555,14 @@ void ProcessRelationshipsRow(
auto to_node = acc.FindVertex(*end_id, storage::View::NEW);
if (!to_node) throw LoadException("To node must be in the storage");
auto relationship = acc.CreateEdge(&*from_node, &*to_node,
acc.NameToEdgeType(*relationship_type));
if (!relationship.HasValue())
throw LoadException("Couldn't create the relationship");
auto relationship = acc.CreateEdge(&*from_node, &*to_node, acc.NameToEdgeType(*relationship_type));
if (!relationship.HasValue()) throw LoadException("Couldn't create the relationship");
for (const auto &property : properties) {
auto ret = relationship->SetProperty(acc.NameToProperty(property.first),
property.second);
auto ret = relationship->SetProperty(acc.NameToProperty(property.first), property.second);
if (!ret.HasValue()) {
if (ret.GetError() != storage::Error::PROPERTIES_DISABLED) {
throw LoadException("Couldn't add property '{}' to the relationship",
property.first);
throw LoadException("Couldn't add property '{}' to the relationship", property.first);
} else {
throw LoadException(
"Couldn't add property '{}' to the relationship because properties "
@@ -617,15 +572,13 @@ void ProcessRelationshipsRow(
}
}
if (acc.Commit().HasError())
throw LoadException("Couldn't store the relationship");
if (acc.Commit().HasError()) throw LoadException("Couldn't store the relationship");
}
void ProcessRelationships(
storage::Storage *store, const std::string &relationships_path,
const std::optional<std::string> &relationship_type,
std::optional<std::vector<Field>> *header,
const std::unordered_map<NodeId, storage::Gid> &node_id_map) {
void ProcessRelationships(storage::Storage *store, const std::string &relationships_path,
const std::optional<std::string> &relationship_type,
std::optional<std::vector<Field>> *header,
const std::unordered_map<NodeId, storage::Gid> &node_id_map) {
std::ifstream relationships_file(relationships_path);
MG_ASSERT(relationships_file, "Unable to open '{}'", relationships_path);
uint64_t row_number = 1;
@@ -647,13 +600,11 @@ void ProcessRelationships(
if (row.size() > (*header)->size()) {
row.resize((*header)->size());
}
ProcessRelationshipsRow(store, **header, row, relationship_type,
node_id_map);
ProcessRelationshipsRow(store, **header, row, relationship_type, node_id_map);
row_number += lines_count;
}
} catch (const LoadException &e) {
LOG_FATAL("Couldn't process row {} of '{}' because of: {}", row_number,
relationships_path, e.what());
LOG_FATAL("Couldn't process row {} of '{}' because of: {}", row_number, relationships_path, e.what());
}
}
@@ -735,8 +686,7 @@ int main(int argc, char *argv[]) {
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
.durability = {.storage_directory = FLAGS_data_directory,
.recover_on_startup = false,
.snapshot_wal_mode =
storage::Config::Durability::SnapshotWalMode::DISABLED,
.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::DISABLED,
.snapshot_on_exit = true},
}};
@@ -748,8 +698,7 @@ int main(int argc, char *argv[]) {
std::optional<std::vector<Field>> header;
for (const auto &nodes_file : files) {
spdlog::info("Loading {}", nodes_file);
ProcessNodes(&store, nodes_file, &header, &node_id_map,
additional_labels);
ProcessNodes(&store, nodes_file, &header, &node_id_map, additional_labels);
}
}
@@ -759,8 +708,7 @@ int main(int argc, char *argv[]) {
std::optional<std::vector<Field>> header;
for (const auto &relationships_file : files) {
spdlog::info("Loading {}", relationships_file);
ProcessRelationships(&store, relationships_file, type, &header,
node_id_map);
ProcessRelationships(&store, relationships_file, type, &header, node_id_map);
}
}

View File

@@ -104,34 +104,26 @@ class [[nodiscard]] Object final {
/// This function always succeeds, meaning that exceptions that occur while
/// calling __getattr__ and __getattribute__ will get suppressed. To get error
/// reporting, use GetAttr instead.
bool HasAttr(const char *attr_name) const {
return PyObject_HasAttrString(ptr_, attr_name);
}
bool HasAttr(const char *attr_name) const { return PyObject_HasAttrString(ptr_, attr_name); }
/// Equivalent to `hasattr(this, attr_name)` in Python.
///
/// This function always succeeds, meaning that exceptions that occur while
/// calling __getattr__ and __getattribute__ will get suppressed. To get error
/// reporting, use GetAttr instead.
bool HasAttr(PyObject *attr_name) const {
return PyObject_HasAttr(ptr_, attr_name);
}
bool HasAttr(PyObject *attr_name) const { return PyObject_HasAttr(ptr_, attr_name); }
/// Equivalent to `this.attr_name` in Python.
///
/// Returned Object is nullptr if an error occurred.
/// @sa FetchError
Object GetAttr(const char *attr_name) const {
return Object(PyObject_GetAttrString(ptr_, attr_name));
}
Object GetAttr(const char *attr_name) const { return Object(PyObject_GetAttrString(ptr_, attr_name)); }
/// Equivalent to `this.attr_name` in Python.
///
/// Returned Object is nullptr if an error occurred.
/// @sa FetchError
Object GetAttr(PyObject *attr_name) const {
return Object(PyObject_GetAttr(ptr_, attr_name));
}
Object GetAttr(PyObject *attr_name) const { return Object(PyObject_GetAttr(ptr_, attr_name)); }
/// Equivalent to `this.attr_name = v` in Python.
///
@@ -145,9 +137,7 @@ class [[nodiscard]] Object final {
///
/// False is returned if an error occurred.
/// @sa FetchError
[[nodiscard]] bool SetAttr(PyObject *attr_name, PyObject *v) {
return PyObject_SetAttr(ptr_, attr_name, v) == 0;
}
[[nodiscard]] bool SetAttr(PyObject *attr_name, PyObject *v) { return PyObject_SetAttr(ptr_, attr_name, v) == 0; }
/// Equivalent to `callable()` in Python.
///
@@ -161,8 +151,7 @@ class [[nodiscard]] Object final {
/// @sa FetchError
template <class... TArgs>
Object Call(const TArgs &...args) const {
return Object(PyObject_CallFunctionObjArgs(
ptr_, static_cast<PyObject *>(args)..., nullptr));
return Object(PyObject_CallFunctionObjArgs(ptr_, static_cast<PyObject *>(args)..., nullptr));
}
/// Equivalent to `obj.meth_name()` in Python.
@@ -170,8 +159,7 @@ class [[nodiscard]] Object final {
/// Returned Object is nullptr if an error occurred.
/// @sa FetchError
Object CallMethod(std::string_view meth_name) const {
Object name(
PyUnicode_FromStringAndSize(meth_name.data(), meth_name.size()));
Object name(PyUnicode_FromStringAndSize(meth_name.data(), meth_name.size()));
return Object(PyObject_CallMethodObjArgs(ptr_, name.Ptr(), nullptr));
}
@@ -181,10 +169,8 @@ class [[nodiscard]] Object final {
/// @sa FetchError
template <class... TArgs>
Object CallMethod(std::string_view meth_name, const TArgs &...args) const {
Object name(
PyUnicode_FromStringAndSize(meth_name.data(), meth_name.size()));
return Object(PyObject_CallMethodObjArgs(
ptr_, name.Ptr(), static_cast<PyObject *>(args)..., nullptr));
Object name(PyUnicode_FromStringAndSize(meth_name.data(), meth_name.size()));
return Object(PyObject_CallMethodObjArgs(ptr_, name.Ptr(), static_cast<PyObject *>(args)..., nullptr));
}
};
@@ -210,8 +196,7 @@ struct [[nodiscard]] ExceptionInfo final {
/// argument `skip_first_line` allows the user to skip the first line of the
/// traceback. It is useful if the first line in the traceback always prints
/// some internal wrapper function.
[[nodiscard]] inline std::string FormatException(const ExceptionInfo &exc_info,
bool skip_first_line = false) {
[[nodiscard]] inline std::string FormatException(const ExceptionInfo &exc_info, bool skip_first_line = false) {
if (!exc_info.type) return "";
Object traceback_mod(PyImport_ImportModule("traceback"));
MG_ASSERT(traceback_mod);
@@ -221,9 +206,8 @@ struct [[nodiscard]] ExceptionInfo final {
if (skip_first_line && traceback_root) {
traceback_root = traceback_root.GetAttr("tb_next");
}
auto list = format_exception_fn.Call(
exc_info.type, exc_info.value ? exc_info.value.Ptr() : Py_None,
traceback_root ? traceback_root.Ptr() : Py_None);
auto list = format_exception_fn.Call(exc_info.type, exc_info.value ? exc_info.value.Ptr() : Py_None,
traceback_root ? traceback_root.Ptr() : Py_None);
MG_ASSERT(list);
std::stringstream ss;
auto len = PyList_GET_SIZE(list.Ptr());
@@ -235,8 +219,7 @@ struct [[nodiscard]] ExceptionInfo final {
}
/// Write ExceptionInfo to stream just like the Python interpreter would.
inline std::ostream &operator<<(std::ostream &os,
const ExceptionInfo &exc_info) {
inline std::ostream &operator<<(std::ostream &os, const ExceptionInfo &exc_info) {
os << FormatException(exc_info);
return os;
}
@@ -259,16 +242,14 @@ inline std::ostream &operator<<(std::ostream &os,
}
inline void RestoreError(ExceptionInfo exc_info) {
PyErr_Restore(exc_info.type.Steal(), exc_info.value.Steal(),
exc_info.traceback.Steal());
PyErr_Restore(exc_info.type.Steal(), exc_info.value.Steal(), exc_info.traceback.Steal());
}
/// Append `dir` to Python's `sys.path`.
///
/// The function does not check whether the directory exists, or is readable.
/// ExceptionInfo is returned if an error occurred.
[[nodiscard]] inline std::optional<ExceptionInfo> AppendToSysPath(
const char *dir) {
[[nodiscard]] inline std::optional<ExceptionInfo> AppendToSysPath(const char *dir) {
MG_ASSERT(dir);
auto *py_path = PySys_GetObject("path");
MG_ASSERT(py_path);

View File

@@ -17,6 +17,7 @@ set(mg_query_sources
frontend/semantic/symbol_generator.cpp
frontend/stripped.cpp
interpret/awesome_memgraph_functions.cpp
interpret/eval.cpp
interpreter.cpp
plan/operator.cpp
plan/preprocess.cpp
@@ -35,7 +36,7 @@ add_library(mg-query STATIC ${mg_query_sources})
add_dependencies(mg-query generate_lcp_query)
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query dl cppitertools)
target_link_libraries(mg-query mg-storage-v2)
target_link_libraries(mg-query mg-storage-v2 mg-utils)
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()

View File

@@ -15,9 +15,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
// comparisons are from this point legal only between values of
// the same type, or int+float combinations
if ((a.type() != b.type() && !(a.IsNumeric() && b.IsNumeric())))
throw QueryRuntimeException(
"Can't compare value of type {} to value of type {}.", a.type(),
b.type());
throw QueryRuntimeException("Can't compare value of type {} to value of type {}.", a.type(), b.type());
switch (a.type()) {
case TypedValue::Type::Bool:
@@ -39,8 +37,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
case TypedValue::Type::Vertex:
case TypedValue::Type::Edge:
case TypedValue::Type::Path:
throw QueryRuntimeException(
"Comparison is not defined for values of type {}.", a.type());
throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type());
default:
LOG_FATAL("Unhandled comparison for types");
}

View File

@@ -27,12 +27,10 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b);
class TypedValueVectorCompare final {
public:
TypedValueVectorCompare() {}
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering)
: ordering_(ordering) {}
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
template <class TAllocator>
bool operator()(const std::vector<TypedValue, TAllocator> &c1,
const std::vector<TypedValue, TAllocator> &c2) const {
bool operator()(const std::vector<TypedValue, TAllocator> &c1, const std::vector<TypedValue, TAllocator> &c2) const {
// ordering is invalid if there are more elements in the collections
// then there are in the ordering_ vector
MG_ASSERT(c1.size() <= ordering_.size() && c2.size() <= ordering_.size(),
@@ -41,12 +39,9 @@ class TypedValueVectorCompare final {
auto c1_it = c1.begin();
auto c2_it = c2.begin();
auto ordering_it = ordering_.begin();
for (; c1_it != c1.end() && c2_it != c2.end();
c1_it++, c2_it++, ordering_it++) {
if (impl::TypedValueCompare(*c1_it, *c2_it))
return *ordering_it == Ordering::ASC;
if (impl::TypedValueCompare(*c2_it, *c1_it))
return *ordering_it == Ordering::DESC;
for (; c1_it != c1.end() && c2_it != c2.end(); c1_it++, c2_it++, ordering_it++) {
if (impl::TypedValueCompare(*c1_it, *c2_it)) return *ordering_it == Ordering::ASC;
if (impl::TypedValueCompare(*c2_it, *c1_it)) return *ordering_it == Ordering::DESC;
}
// at least one collection is exhausted
@@ -61,41 +56,33 @@ class TypedValueVectorCompare final {
};
/// Raise QueryRuntimeException if the value for symbol isn't of expected type.
inline void ExpectType(const Symbol &symbol, const TypedValue &value,
TypedValue::Type expected) {
inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue::Type expected) {
if (value.type() != expected)
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected,
symbol.name(), value.type());
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
/// Set a property `value` mapped with given `key` on a `record`.
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <class TRecordAccessor>
void PropsSetChecked(TRecordAccessor *record, const storage::PropertyId &key,
const TypedValue &value) {
void PropsSetChecked(TRecordAccessor *record, const storage::PropertyId &key, const TypedValue &value) {
try {
auto maybe_error = record->SetProperty(key, storage::PropertyValue(value));
if (maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case storage::Error::SERIALIZATION_ERROR:
throw QueryRuntimeException(
"Can't serialize due to concurrent operations.");
throw QueryRuntimeException("Can't serialize due to concurrent operations.");
case storage::Error::DELETED_OBJECT:
throw QueryRuntimeException(
"Trying to set properties on a deleted object.");
throw QueryRuntimeException("Trying to set properties on a deleted object.");
case storage::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException(
"Can't set property because properties on edges are disabled.");
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::Error::VERTEX_HAS_EDGES:
case storage::Error::NONEXISTENT_OBJECT:
throw QueryRuntimeException(
"Unexpected error when setting a property.");
throw QueryRuntimeException("Unexpected error when setting a property.");
}
}
} catch (const TypedValueException &) {
throw QueryRuntimeException("'{}' cannot be used as a property value.",
value.type());
throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
}
}

View File

@@ -26,8 +26,8 @@ struct EvaluationContext {
mutable std::unordered_map<std::string, int64_t> counters;
};
inline std::vector<storage::PropertyId> NamesToProperties(
const std::vector<std::string> &property_names, DbAccessor *dba) {
inline std::vector<storage::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
DbAccessor *dba) {
std::vector<storage::PropertyId> properties;
properties.reserve(property_names.size());
for (const auto &name : property_names) {
@@ -36,8 +36,7 @@ inline std::vector<storage::PropertyId> NamesToProperties(
return properties;
}
inline std::vector<storage::LabelId> NamesToLabels(
const std::vector<std::string> &label_names, DbAccessor *dba) {
inline std::vector<storage::LabelId> NamesToLabels(const std::vector<std::string> &label_names, DbAccessor *dba) {
std::vector<storage::LabelId> labels;
labels.reserve(label_names.size());
for (const auto &name : label_names) {
@@ -60,11 +59,9 @@ struct ExecutionContext {
};
inline bool MustAbort(const ExecutionContext &context) {
return (context.is_shutting_down &&
context.is_shutting_down->load(std::memory_order_acquire)) ||
return (context.is_shutting_down && context.is_shutting_down->load(std::memory_order_acquire)) ||
(context.max_execution_time_sec > 0 &&
context.execution_tsc_timer.Elapsed() >=
context.max_execution_time_sec);
context.execution_tsc_timer.Elapsed() >= context.max_execution_time_sec);
}
} // namespace query

View File

@@ -47,19 +47,15 @@ class EdgeAccessor final {
auto Properties(storage::View view) const { return impl_.Properties(view); }
storage::Result<storage::PropertyValue> GetProperty(storage::View view,
storage::PropertyId key) const {
storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::Result<bool> SetProperty(storage::PropertyId key,
const storage::PropertyValue &value) {
storage::Result<bool> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::Result<bool> RemoveProperty(storage::PropertyId key) {
return SetProperty(key, storage::PropertyValue());
}
storage::Result<bool> RemoveProperty(storage::PropertyId key) { return SetProperty(key, storage::PropertyValue()); }
utils::BasicResult<storage::Error, void> ClearProperties() {
auto ret = impl_.ClearProperties();
@@ -86,44 +82,32 @@ class VertexAccessor final {
public:
storage::VertexAccessor impl_;
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) {
return EdgeAccessor(impl);
}
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); }
public:
explicit VertexAccessor(storage::VertexAccessor impl)
: impl_(std::move(impl)) {}
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(std::move(impl)) {}
auto Labels(storage::View view) const { return impl_.Labels(view); }
storage::Result<bool> AddLabel(storage::LabelId label) {
return impl_.AddLabel(label);
}
storage::Result<bool> AddLabel(storage::LabelId label) { return impl_.AddLabel(label); }
storage::Result<bool> RemoveLabel(storage::LabelId label) {
return impl_.RemoveLabel(label);
}
storage::Result<bool> RemoveLabel(storage::LabelId label) { return impl_.RemoveLabel(label); }
storage::Result<bool> HasLabel(storage::View view,
storage::LabelId label) const {
storage::Result<bool> HasLabel(storage::View view, storage::LabelId label) const {
return impl_.HasLabel(label, view);
}
auto Properties(storage::View view) const { return impl_.Properties(view); }
storage::Result<storage::PropertyValue> GetProperty(storage::View view,
storage::PropertyId key) const {
storage::Result<storage::PropertyValue> GetProperty(storage::View view, storage::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::Result<bool> SetProperty(storage::PropertyId key,
const storage::PropertyValue &value) {
storage::Result<bool> SetProperty(storage::PropertyId key, const storage::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::Result<bool> RemoveProperty(storage::PropertyId key) {
return SetProperty(key, storage::PropertyValue());
}
storage::Result<bool> RemoveProperty(storage::PropertyId key) { return SetProperty(key, storage::PropertyValue()); }
utils::BasicResult<storage::Error, void> ClearProperties() {
auto ret = impl_.ClearProperties();
@@ -131,10 +115,8 @@ class VertexAccessor final {
return {};
}
auto InEdges(storage::View view,
const std::vector<storage::EdgeTypeId> &edge_types) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.InEdges(view)))> {
auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -142,20 +124,15 @@ class VertexAccessor final {
auto InEdges(storage::View view) const { return InEdges(view, {}); }
auto InEdges(storage::View view,
const std::vector<storage::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.InEdges(view)))> {
auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
auto OutEdges(storage::View view,
const std::vector<storage::EdgeTypeId> &edge_types) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.OutEdges(view)))> {
auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -163,23 +140,17 @@ class VertexAccessor final {
auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
auto OutEdges(storage::View view,
const std::vector<storage::EdgeTypeId> &edge_types,
auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor,
*impl_.OutEdges(view)))> {
-> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
storage::Result<size_t> InDegree(storage::View view) const {
return impl_.InDegree(view);
}
storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
storage::Result<size_t> OutDegree(storage::View view) const {
return impl_.OutDegree(view);
}
storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
int64_t CypherId() const { return impl_.Gid().AsInt(); }
@@ -190,13 +161,9 @@ class VertexAccessor final {
bool operator!=(const VertexAccessor &v) const { return !(*this == v); }
};
inline VertexAccessor EdgeAccessor::To() const {
return VertexAccessor(impl_.ToVertex());
}
inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
inline VertexAccessor EdgeAccessor::From() const {
return VertexAccessor(impl_.FromVertex());
}
inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
@@ -225,8 +192,7 @@ class DbAccessor final {
bool operator!=(const Iterator &other) const { return !(other == *this); }
};
explicit VerticesIterable(storage::VerticesIterable iterable)
: iterable_(std::move(iterable)) {}
explicit VerticesIterable(storage::VerticesIterable iterable) : iterable_(std::move(iterable)) {}
Iterator begin() { return Iterator(iterable_.begin()); }
@@ -234,60 +200,45 @@ class DbAccessor final {
};
public:
explicit DbAccessor(storage::Storage::Accessor *accessor)
: accessor_(accessor) {}
explicit DbAccessor(storage::Storage::Accessor *accessor) : accessor_(accessor) {}
std::optional<VertexAccessor> FindVertex(storage::Gid gid,
storage::View view) {
std::optional<VertexAccessor> FindVertex(storage::Gid gid, storage::View view) {
auto maybe_vertex = accessor_->FindVertex(gid, view);
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
return std::nullopt;
}
VerticesIterable Vertices(storage::View view) {
return VerticesIterable(accessor_->Vertices(view));
}
VerticesIterable Vertices(storage::View view) { return VerticesIterable(accessor_->Vertices(view)); }
VerticesIterable Vertices(storage::View view, storage::LabelId label) {
return VerticesIterable(accessor_->Vertices(label, view));
}
VerticesIterable Vertices(storage::View view, storage::LabelId label,
storage::PropertyId property) {
VerticesIterable Vertices(storage::View view, storage::LabelId label, storage::PropertyId property) {
return VerticesIterable(accessor_->Vertices(label, property, view));
}
VerticesIterable Vertices(storage::View view, storage::LabelId label,
storage::PropertyId property,
VerticesIterable Vertices(storage::View view, storage::LabelId label, storage::PropertyId property,
const storage::PropertyValue &value) {
return VerticesIterable(accessor_->Vertices(label, property, value, view));
}
VerticesIterable Vertices(
storage::View view, storage::LabelId label, storage::PropertyId property,
const std::optional<utils::Bound<storage::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::PropertyValue>> &upper) {
return VerticesIterable(
accessor_->Vertices(label, property, lower, upper, view));
VerticesIterable Vertices(storage::View view, storage::LabelId label, storage::PropertyId property,
const std::optional<utils::Bound<storage::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::PropertyValue>> &upper) {
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
}
VertexAccessor InsertVertex() {
return VertexAccessor(accessor_->CreateVertex());
}
VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex()); }
storage::Result<EdgeAccessor> InsertEdge(VertexAccessor *from,
VertexAccessor *to,
storage::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::EdgeTypeId &edge_type) {
auto maybe_edge =
accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
if (maybe_edge.HasError())
return storage::Result<EdgeAccessor>(maybe_edge.GetError());
auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
if (maybe_edge.HasError()) return storage::Result<EdgeAccessor>(maybe_edge.GetError());
return EdgeAccessor(std::move(*maybe_edge));
}
storage::Result<bool> RemoveEdge(EdgeAccessor *edge) {
return accessor_->DeleteEdge(&edge->impl_);
}
storage::Result<bool> RemoveEdge(EdgeAccessor *edge) { return accessor_->DeleteEdge(&edge->impl_); }
storage::Result<bool> DetachRemoveVertex(VertexAccessor *vertex_accessor) {
return accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
@@ -297,55 +248,35 @@ class DbAccessor final {
return accessor_->DeleteVertex(&vertex_accessor->impl_);
}
storage::PropertyId NameToProperty(const std::string_view &name) {
return accessor_->NameToProperty(name);
}
storage::PropertyId NameToProperty(const std::string_view &name) { return accessor_->NameToProperty(name); }
storage::LabelId NameToLabel(const std::string_view &name) {
return accessor_->NameToLabel(name);
}
storage::LabelId NameToLabel(const std::string_view &name) { return accessor_->NameToLabel(name); }
storage::EdgeTypeId NameToEdgeType(const std::string_view &name) {
return accessor_->NameToEdgeType(name);
}
storage::EdgeTypeId NameToEdgeType(const std::string_view &name) { return accessor_->NameToEdgeType(name); }
const std::string &PropertyToName(storage::PropertyId prop) const {
return accessor_->PropertyToName(prop);
}
const std::string &PropertyToName(storage::PropertyId prop) const { return accessor_->PropertyToName(prop); }
const std::string &LabelToName(storage::LabelId label) const {
return accessor_->LabelToName(label);
}
const std::string &LabelToName(storage::LabelId label) const { return accessor_->LabelToName(label); }
const std::string &EdgeTypeToName(storage::EdgeTypeId type) const {
return accessor_->EdgeTypeToName(type);
}
const std::string &EdgeTypeToName(storage::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); }
void AdvanceCommand() { accessor_->AdvanceCommand(); }
utils::BasicResult<storage::ConstraintViolation, void> Commit() {
return accessor_->Commit();
}
utils::BasicResult<storage::ConstraintViolation, void> Commit() { return accessor_->Commit(); }
void Abort() { accessor_->Abort(); }
bool LabelIndexExists(storage::LabelId label) const {
return accessor_->LabelIndexExists(label);
}
bool LabelIndexExists(storage::LabelId label) const { return accessor_->LabelIndexExists(label); }
bool LabelPropertyIndexExists(storage::LabelId label,
storage::PropertyId prop) const {
bool LabelPropertyIndexExists(storage::LabelId label, storage::PropertyId prop) const {
return accessor_->LabelPropertyIndexExists(label, prop);
}
int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); }
int64_t VerticesCount(storage::LabelId label) const {
return accessor_->ApproximateVertexCount(label);
}
int64_t VerticesCount(storage::LabelId label) const { return accessor_->ApproximateVertexCount(label); }
int64_t VerticesCount(storage::LabelId label,
storage::PropertyId property) const {
int64_t VerticesCount(storage::LabelId label, storage::PropertyId property) const {
return accessor_->ApproximateVertexCount(label, property);
}
@@ -354,20 +285,15 @@ class DbAccessor final {
return accessor_->ApproximateVertexCount(label, property, value);
}
int64_t VerticesCount(
storage::LabelId label, storage::PropertyId property,
const std::optional<utils::Bound<storage::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::PropertyValue>> &upper) const {
int64_t VerticesCount(storage::LabelId label, storage::PropertyId property,
const std::optional<utils::Bound<storage::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::PropertyValue>> &upper) const {
return accessor_->ApproximateVertexCount(label, property, lower, upper);
}
storage::IndicesInfo ListAllIndices() const {
return accessor_->ListAllIndices();
}
storage::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); }
storage::ConstraintsInfo ListAllConstraints() const {
return accessor_->ListAllConstraints();
}
storage::ConstraintsInfo ListAllConstraints() const { return accessor_->ListAllConstraints(); }
};
} // namespace query
@@ -376,16 +302,12 @@ namespace std {
template <>
struct hash<query::VertexAccessor> {
size_t operator()(const query::VertexAccessor &v) const {
return std::hash<decltype(v.impl_)>{}(v.impl_);
}
size_t operator()(const query::VertexAccessor &v) const { return std::hash<decltype(v.impl_)>{}(v.impl_); }
};
template <>
struct hash<query::EdgeAccessor> {
size_t operator()(const query::EdgeAccessor &e) const {
return std::hash<decltype(e.impl_)>{}(e.impl_);
}
size_t operator()(const query::EdgeAccessor &e) const { return std::hash<decltype(e.impl_)>{}(e.impl_); }
};
} // namespace std

View File

@@ -50,8 +50,7 @@ void DumpPreciseDouble(std::ostream *os, double value) {
// A temporary stream is used to keep precision of the original output
// stream unchanged.
std::ostringstream temp_oss;
temp_oss << std::setprecision(std::numeric_limits<double>::max_digits10)
<< value;
temp_oss << std::setprecision(std::numeric_limits<double>::max_digits10) << value;
*os << temp_oss.str();
}
@@ -75,9 +74,7 @@ void DumpPropertyValue(std::ostream *os, const storage::PropertyValue &value) {
case storage::PropertyValue::Type::List: {
*os << "[";
const auto &list = value.ValueList();
utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) {
DumpPropertyValue(&os, item);
});
utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); });
*os << "]";
return;
}
@@ -94,10 +91,9 @@ void DumpPropertyValue(std::ostream *os, const storage::PropertyValue &value) {
}
}
void DumpProperties(
std::ostream *os, query::DbAccessor *dba,
const std::map<storage::PropertyId, storage::PropertyValue> &store,
std::optional<int64_t> property_id = std::nullopt) {
void DumpProperties(std::ostream *os, query::DbAccessor *dba,
const std::map<storage::PropertyId, storage::PropertyValue> &store,
std::optional<int64_t> property_id = std::nullopt) {
*os << "{";
if (property_id) {
*os << kInternalPropertyId << ": " << *property_id;
@@ -110,24 +106,20 @@ void DumpProperties(
*os << "}";
}
void DumpVertex(std::ostream *os, query::DbAccessor *dba,
const query::VertexAccessor &vertex) {
void DumpVertex(std::ostream *os, query::DbAccessor *dba, const query::VertexAccessor &vertex) {
*os << "CREATE (";
*os << ":" << kInternalVertexLabel;
auto maybe_labels = vertex.Labels(storage::View::OLD);
if (maybe_labels.HasError()) {
switch (maybe_labels.GetError()) {
case storage::Error::DELETED_OBJECT:
throw query::QueryRuntimeException(
"Trying to get labels from a deleted node.");
throw query::QueryRuntimeException("Trying to get labels from a deleted node.");
case storage::Error::NONEXISTENT_OBJECT:
throw query::QueryRuntimeException(
"Trying to get labels from a node that doesn't exist.");
throw query::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
case storage::Error::SERIALIZATION_ERROR:
case storage::Error::VERTEX_HAS_EDGES:
case storage::Error::PROPERTIES_DISABLED:
throw query::QueryRuntimeException(
"Unexpected error when getting labels.");
throw query::QueryRuntimeException("Unexpected error when getting labels.");
}
}
for (const auto &label : *maybe_labels) {
@@ -138,24 +130,20 @@ void DumpVertex(std::ostream *os, query::DbAccessor *dba,
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::Error::DELETED_OBJECT:
throw query::QueryRuntimeException(
"Trying to get properties from a deleted object.");
throw query::QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::Error::NONEXISTENT_OBJECT:
throw query::QueryRuntimeException(
"Trying to get properties from a node that doesn't exist.");
throw query::QueryRuntimeException("Trying to get properties from a node that doesn't exist.");
case storage::Error::SERIALIZATION_ERROR:
case storage::Error::VERTEX_HAS_EDGES:
case storage::Error::PROPERTIES_DISABLED:
throw query::QueryRuntimeException(
"Unexpected error when getting properties.");
throw query::QueryRuntimeException("Unexpected error when getting properties.");
}
}
DumpProperties(os, dba, *maybe_props, vertex.CypherId());
*os << ");";
}
void DumpEdge(std::ostream *os, query::DbAccessor *dba,
const query::EdgeAccessor &edge) {
void DumpEdge(std::ostream *os, query::DbAccessor *dba, const query::EdgeAccessor &edge) {
*os << "MATCH ";
*os << "(u:" << kInternalVertexLabel << "), ";
*os << "(v:" << kInternalVertexLabel << ")";
@@ -169,16 +157,13 @@ void DumpEdge(std::ostream *os, query::DbAccessor *dba,
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::Error::DELETED_OBJECT:
throw query::QueryRuntimeException(
"Trying to get properties from a deleted object.");
throw query::QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::Error::NONEXISTENT_OBJECT:
throw query::QueryRuntimeException(
"Trying to get properties from an edge that doesn't exist.");
throw query::QueryRuntimeException("Trying to get properties from an edge that doesn't exist.");
case storage::Error::SERIALIZATION_ERROR:
case storage::Error::VERTEX_HAS_EDGES:
case storage::Error::PROPERTIES_DISABLED:
throw query::QueryRuntimeException(
"Unexpected error when getting properties.");
throw query::QueryRuntimeException("Unexpected error when getting properties.");
}
}
if (maybe_props->size() > 0) {
@@ -188,35 +173,28 @@ void DumpEdge(std::ostream *os, query::DbAccessor *dba,
*os << "]->(v);";
}
void DumpLabelIndex(std::ostream *os, query::DbAccessor *dba,
const storage::LabelId label) {
void DumpLabelIndex(std::ostream *os, query::DbAccessor *dba, const storage::LabelId label) {
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";";
}
void DumpLabelPropertyIndex(std::ostream *os, query::DbAccessor *dba,
storage::LabelId label,
void DumpLabelPropertyIndex(std::ostream *os, query::DbAccessor *dba, storage::LabelId label,
storage::PropertyId property) {
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "("
<< EscapeName(dba->PropertyToName(property)) << ");";
}
void DumpExistenceConstraint(std::ostream *os, query::DbAccessor *dba,
storage::LabelId label,
storage::PropertyId property) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label))
<< ") ASSERT EXISTS (u." << EscapeName(dba->PropertyToName(property))
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property))
<< ");";
}
void DumpUniqueConstraint(std::ostream *os, query::DbAccessor *dba,
storage::LabelId label,
void DumpExistenceConstraint(std::ostream *os, query::DbAccessor *dba, storage::LabelId label,
storage::PropertyId property) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u."
<< EscapeName(dba->PropertyToName(property)) << ");";
}
void DumpUniqueConstraint(std::ostream *os, query::DbAccessor *dba, storage::LabelId label,
const std::set<storage::PropertyId> &properties) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label))
<< ") ASSERT ";
utils::PrintIterable(
*os, properties, ", ", [&dba](auto &stream, const auto &property) {
stream << "u." << EscapeName(dba->PropertyToName(property));
});
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT ";
utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) {
stream << "u." << EscapeName(dba->PropertyToName(property));
});
*os << " IS UNIQUE;";
}
@@ -250,8 +228,7 @@ bool PullPlanDump::Pull(AnyStream *stream, std::optional<int> n) {
// finishes. If the function did not finish streaming all the results,
// std::nullopt should be returned because n results have already been sent.
while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) {
const auto maybe_streamed_count =
pull_chunks_[current_chunk_index_](stream, n);
const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n);
if (!maybe_streamed_count) {
// n wasn't large enough to stream all the results from the current chunk
@@ -273,9 +250,7 @@ bool PullPlanDump::Pull(AnyStream *stream, std::optional<int> n) {
PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() {
// Dump all label indices
return [this, global_index = 0U](
AnyStream *stream,
std::optional<int> n) mutable -> std::optional<size_t> {
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of indices vectors
if (!indices_info_) {
indices_info_.emplace(dba_->ListAllIndices());
@@ -301,9 +276,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() {
}
PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
return [this, global_index = 0U](
AnyStream *stream,
std::optional<int> n) mutable -> std::optional<size_t> {
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of indices vectors
if (!indices_info_) {
indices_info_.emplace(dba_->ListAllIndices());
@@ -314,8 +287,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
while (global_index < label_property.size() && (!n || local_counter < *n)) {
std::ostringstream os;
const auto &label_property_index = label_property[global_index];
DumpLabelPropertyIndex(&os, dba_, label_property_index.first,
label_property_index.second);
DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second);
stream->Result({TypedValue(os.str())});
++global_index;
@@ -331,9 +303,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
}
PullPlanDump::PullChunk PullPlanDump::CreateExistenceConstraintsPullChunk() {
return [this, global_index = 0U](
AnyStream *stream,
std::optional<int> n) mutable -> std::optional<size_t> {
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of constraint vectors
if (!constraints_info_) {
constraints_info_.emplace(dba_->ListAllConstraints());
@@ -360,9 +330,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateExistenceConstraintsPullChunk() {
}
PullPlanDump::PullChunk PullPlanDump::CreateUniqueConstraintsPullChunk() {
return [this, global_index = 0U](
AnyStream *stream,
std::optional<int> n) mutable -> std::optional<size_t> {
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of constraint vectors
if (!constraints_info_) {
constraints_info_.emplace(dba_->ListAllConstraints());
@@ -389,12 +357,10 @@ PullPlanDump::PullChunk PullPlanDump::CreateUniqueConstraintsPullChunk() {
}
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() {
return [this](AnyStream *stream,
std::optional<int>) mutable -> std::optional<size_t> {
return [this](AnyStream *stream, std::optional<int>) mutable -> std::optional<size_t> {
if (vertices_iterable_.begin() != vertices_iterable_.end()) {
std::ostringstream os;
os << "CREATE INDEX ON :" << kInternalVertexLabel << "("
<< kInternalPropertyId << ");";
os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
stream->Result({TypedValue(os.str())});
internal_index_created_ = true;
return 1;
@@ -404,10 +370,8 @@ PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() {
}
PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
return [this,
maybe_current_iter = std::optional<VertexAccessorIterableIterator>{}](
AnyStream *stream,
std::optional<int> n) mutable -> std::optional<size_t> {
return [this, maybe_current_iter = std::optional<VertexAccessorIterableIterator>{}](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
@@ -419,8 +383,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
auto &current_iter{*maybe_current_iter};
size_t local_counter = 0;
while (current_iter != vertices_iterable_.end() &&
(!n || local_counter < *n)) {
while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) {
std::ostringstream os;
DumpVertex(&os, dba_, *current_iter);
stream->Result({TypedValue(os.str())});
@@ -436,74 +399,62 @@ PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
}
PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
return
[this,
maybe_current_vertex_iter =
std::optional<VertexAccessorIterableIterator>{},
// we need to save the iterable which contains list of accessor so
// our saved iterator is valid in the next run
maybe_edge_iterable = std::shared_ptr<EdgeAccessorIterable>{nullptr},
maybe_current_edge_iter = std::optional<EdgeAccessorIterableIterator>{}](
AnyStream *stream,
std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
// states.
if (!maybe_current_vertex_iter) {
maybe_current_vertex_iter.emplace(vertices_iterable_.begin());
}
return [this, maybe_current_vertex_iter = std::optional<VertexAccessorIterableIterator>{},
// we need to save the iterable which contains list of accessor so
// our saved iterator is valid in the next run
maybe_edge_iterable = std::shared_ptr<EdgeAccessorIterable>{nullptr},
maybe_current_edge_iter = std::optional<EdgeAccessorIterableIterator>{}](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
// states.
if (!maybe_current_vertex_iter) {
maybe_current_vertex_iter.emplace(vertices_iterable_.begin());
}
auto &current_vertex_iter{*maybe_current_vertex_iter};
size_t local_counter = 0U;
for (; current_vertex_iter != vertices_iterable_.end() &&
(!n || local_counter < *n);
++current_vertex_iter) {
const auto &vertex = *current_vertex_iter;
// If we have a saved iterable from a previous pull
// we need to use the same iterable
if (!maybe_edge_iterable) {
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(
vertex.OutEdges(storage::View::OLD));
}
auto &maybe_edges = *maybe_edge_iterable;
MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
auto current_edge_iter = maybe_current_edge_iter
? *maybe_current_edge_iter
: maybe_edges->begin();
for (; current_edge_iter != maybe_edges->end() &&
(!n || local_counter < *n);
++current_edge_iter) {
std::ostringstream os;
DumpEdge(&os, dba_, *current_edge_iter);
stream->Result({TypedValue(os.str())});
auto &current_vertex_iter{*maybe_current_vertex_iter};
size_t local_counter = 0U;
for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) {
const auto &vertex = *current_vertex_iter;
// If we have a saved iterable from a previous pull
// we need to use the same iterable
if (!maybe_edge_iterable) {
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::View::OLD));
}
auto &maybe_edges = *maybe_edge_iterable;
MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin();
for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) {
std::ostringstream os;
DumpEdge(&os, dba_, *current_edge_iter);
stream->Result({TypedValue(os.str())});
++local_counter;
}
if (current_edge_iter != maybe_edges->end()) {
maybe_current_edge_iter.emplace(current_edge_iter);
return std::nullopt;
}
maybe_current_edge_iter = std::nullopt;
maybe_edge_iterable = nullptr;
}
if (current_vertex_iter == vertices_iterable_.end()) {
return local_counter;
}
++local_counter;
}
if (current_edge_iter != maybe_edges->end()) {
maybe_current_edge_iter.emplace(current_edge_iter);
return std::nullopt;
};
}
maybe_current_edge_iter = std::nullopt;
maybe_edge_iterable = nullptr;
}
if (current_vertex_iter == vertices_iterable_.end()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() {
return [this](AnyStream *stream, std::optional<int>) {
if (internal_index_created_) {
std::ostringstream os;
os << "DROP INDEX ON :" << kInternalVertexLabel << "("
<< kInternalPropertyId << ");";
os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
stream->Result({TypedValue(os.str())});
return 1;
}
@@ -515,8 +466,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() {
return [this](AnyStream *stream, std::optional<int>) {
if (internal_index_created_) {
std::ostringstream os;
os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u."
<< kInternalPropertyId << ";";
os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";";
stream->Result({TypedValue(os.str())});
return 1;
}
@@ -524,8 +474,6 @@ PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() {
};
}
void DumpDatabaseToCypherQueries(query::DbAccessor *dba, AnyStream *stream) {
PullPlanDump(dba).Pull(stream, {});
}
void DumpDatabaseToCypherQueries(query::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); }
} // namespace query

Some files were not shown because too many files have changed in this diff Show More