Compare commits
20 Commits
tmp-fix-je
...
E088-MG-Qu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
999b3ef79f | ||
|
|
30413a7b4f | ||
|
|
1def0c9104 | ||
|
|
782c377f5d | ||
|
|
cc27a04139 | ||
|
|
b71345655f | ||
|
|
ccdd58b336 | ||
|
|
50b6afd73d | ||
|
|
59105f68bd | ||
|
|
8de31092ad | ||
|
|
5c93f81881 | ||
|
|
6d4fe5cdd5 | ||
|
|
7b5263d300 | ||
|
|
e8a41e4457 | ||
|
|
27f09e1c0a | ||
|
|
6dd9d32721 | ||
|
|
276e09d7d3 | ||
|
|
92dfc93b20 | ||
|
|
06f761bdf9 | ||
|
|
50ddd59450 |
@@ -54,7 +54,7 @@ Checks: '*,
|
||||
-readability-magic-numbers,
|
||||
-readability-named-parameter'
|
||||
WarningsAsErrors: ''
|
||||
HeaderFilterRegex: ''
|
||||
HeaderFilterRegex: 'src/.*'
|
||||
AnalyzeTemporaryDtors: false
|
||||
FormatStyle: none
|
||||
CheckOptions:
|
||||
|
||||
159
.github/workflows/diff.yaml
vendored
159
.github/workflows/diff.yaml
vendored
@@ -10,7 +10,7 @@ on:
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -65,9 +65,9 @@ jobs:
|
||||
name: "Community DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
coverage_build:
|
||||
name: "Coverage build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
code_analysis:
|
||||
name: "Code analysis"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build coverage binaries
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
@@ -87,9 +87,8 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build coverage binaries.
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON ..
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
|
||||
- name: Run unit tests
|
||||
@@ -97,9 +96,9 @@ jobs:
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Run unit tests.
|
||||
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
|
||||
|
||||
- name: Compute code coverage
|
||||
run: |
|
||||
@@ -120,9 +119,19 @@ jobs:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -196,7 +205,7 @@ jobs:
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -208,21 +217,6 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up parent
|
||||
run: |
|
||||
# Remove parent folder (if it exists).
|
||||
cd ..
|
||||
if [ -d parent ]; then
|
||||
rm -rf parent
|
||||
fi
|
||||
|
||||
# Copy untouched repository to parent folder.
|
||||
cp -r memgraph parent
|
||||
|
||||
# Checkout previous commit
|
||||
cd parent
|
||||
git checkout HEAD~1
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -236,47 +230,6 @@ jobs:
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Build parent binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
cd ../parent
|
||||
./init
|
||||
|
||||
# Build parent binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS memgraph memgraph__macro_benchmark
|
||||
|
||||
- name: Run macro benchmark tests
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Run parent macro benchmark tests
|
||||
run: |
|
||||
cd ../parent/tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Compute macro benchmark summary
|
||||
run: |
|
||||
./tools/github/macro_benchmark_summary \
|
||||
--current tests/macro_benchmark/.harness_summary \
|
||||
--previous ../parent/tests/macro_benchmark/.harness_summary \
|
||||
--output macro_benchmark_summary.txt
|
||||
|
||||
- name: Save macro benchmark summary
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Macro benchmark summary"
|
||||
path: macro_benchmark_summary.txt
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests/gql_behave
|
||||
@@ -339,9 +292,18 @@ jobs:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, HP-DL360G6-v2-3]
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
#continue-on-error: true
|
||||
env:
|
||||
THREADS: 24
|
||||
@@ -378,3 +340,64 @@ jobs:
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Upload macro benchmark results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}"
|
||||
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}"
|
||||
|
||||
44
.github/workflows/full_clang_tidy.yaml
vendored
Normal file
44
.github/workflows/full_clang_tidy.yaml
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
name: Run clang-tidy on the full codebase
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
clang_tidy_check:
|
||||
name: "Clang-tidy check"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build debug binaries.
|
||||
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# The results are also written to standard output in order to retain them in the logs
|
||||
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
|
||||
tee ./build/full_clang_tidy_output.txt
|
||||
|
||||
- name: Summarize clang-tidy results
|
||||
run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh
|
||||
248
.github/workflows/package_all.yaml
vendored
Normal file
248
.github/workflows/package_all.yaml
vendored
Normal file
@@ -0,0 +1,248 @@
|
||||
name: Package All
|
||||
|
||||
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
|
||||
|
||||
on: workflow_dispatch
|
||||
|
||||
jobs:
|
||||
centos-7_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package community centos-7
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-7_community
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
centos-8_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package community centos-8
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-8_community
|
||||
path: build/output/centos-8/memgraph*.rpm
|
||||
|
||||
debian-9_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package community debian-9
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-9_community
|
||||
path: build/output/debian-9/memgraph*.deb
|
||||
|
||||
debian-10_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package community debian-10
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-10_community
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
docker_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
cd release/package
|
||||
./run.sh package community debian-10 --for-docker
|
||||
./run.sh docker
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: docker_community
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
ubuntu-1804_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package community ubuntu-18.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-1804_community
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004_community:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package community ubuntu-20.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-2004_community
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
centos-7_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package enterprise centos-7
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-7_enterprise
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
centos-8_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package enterprise centos-8
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-8_enterprise
|
||||
path: build/output/centos-8/memgraph*.rpm
|
||||
|
||||
debian-9_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package enterprise debian-9
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-9_enterprise
|
||||
path: build/output/debian-9/memgraph*.deb
|
||||
|
||||
debian-10_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package enterprise debian-10
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-10_enterprise
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
docker_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
cd release/package
|
||||
./run.sh package enterprise debian-10 --for-docker
|
||||
./run.sh docker
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: docker_enterprise
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
ubuntu-1804_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package enterprise ubuntu-18.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-1804_enterprise
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004_enterprise:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package enterprise ubuntu-20.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-2004_enterprise
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
2
.github/workflows/release_debian10.yaml
vendored
2
.github/workflows/release_debian10.yaml
vendored
@@ -1,4 +1,4 @@
|
||||
name: Release Debian10
|
||||
name: Release Debian 10
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
2
.github/workflows/release_ubuntu2004.yaml
vendored
2
.github/workflows/release_ubuntu2004.yaml
vendored
@@ -1,4 +1,4 @@
|
||||
name: Release Ubuntu20.04
|
||||
name: Release Ubuntu 20.04
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
15
CHANGELOG.md
15
CHANGELOG.md
@@ -2,6 +2,16 @@
|
||||
|
||||
## Future
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
|
||||
For example, parsing of `mgp.List[mgp.Map]` works now.
|
||||
* Fixed memory tracking issues. Some of the allocation and deallocation weren't
|
||||
tracked during the query execution.
|
||||
* Fixed reading CSV files that are using CRLF as the newline symbol.
|
||||
|
||||
## v1.4.0
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
* Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.
|
||||
@@ -10,7 +20,7 @@
|
||||
### Major Feature and Improvements
|
||||
|
||||
* Added replication to community version.
|
||||
* Add support for multiple query modules directories at the same time.
|
||||
* Added support for multiple query modules directories at the same time.
|
||||
You can now define multiple, comma-separated paths to directories from
|
||||
which the modules will be loaded using the `--query-modules-directory` flag.
|
||||
* Added support for programatically reading in data from CSV files through the
|
||||
@@ -22,12 +32,15 @@
|
||||
* Added the memory limit and amount of currently allocated bytes in the result of `SHOW STORAGE INFO` query.
|
||||
* Added `QUERY MEMORY LIMIT num (KB|MB)` to Cypher queries which allows you to limit memory allocation for
|
||||
the entire query. It can be added only at the end of the entire Cypher query.
|
||||
* Added logs for the different parts of the recovery process. `INFO`, `DEBUG` and `TRACE` level all contain
|
||||
additional information that is printed out while the recovery is in progress.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixed garbage collector by correctly marking the oldest current timestamp
|
||||
after the database was recovered using the durability files.
|
||||
* Fixed reloading of the modules with changed result names.
|
||||
* Fixed profile query to show the correct name of the ScanAll operator variant.
|
||||
|
||||
## v1.3.0
|
||||
|
||||
|
||||
@@ -312,8 +312,9 @@ if (UBSAN)
|
||||
# runtime library and c++ standard libraries are present.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
|
||||
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
|
||||
# Make sure llvm-symbolizer binary is in path
|
||||
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
|
||||
# Make sure llvm-symbolizer binary is in path.
|
||||
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
|
||||
endif()
|
||||
|
||||
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
|
||||
|
||||
@@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
|
||||
libedit-devel pcre-devel automake bison # swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz # used for archive unpacking
|
||||
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconfig # build system
|
||||
@@ -48,9 +50,11 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
|
||||
autoconf # for jemalloc code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
@@ -75,16 +79,13 @@ check() {
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$SUDO_USER" == "" ]; then
|
||||
echo "Please run as sudo."
|
||||
exit 1
|
||||
fi
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
@@ -118,11 +119,16 @@ install() {
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == PyYAML ]; then
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user PyYAML
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
yum install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel automake bison # for swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz # used for archive unpacking
|
||||
@@ -25,6 +26,7 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconf-pkg-config # build system
|
||||
@@ -47,9 +49,11 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
autoconf # for jemalloc code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
@@ -68,16 +72,13 @@ check() {
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$SUDO_USER" == "" ]; then
|
||||
echo "Please run as sudo."
|
||||
exit 1
|
||||
fi
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
@@ -86,6 +87,7 @@ install() {
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
dnf install -y epel-release
|
||||
dnf install -y 'dnf-command(config-manager)'
|
||||
dnf config-manager --set-enabled powertools # Required to install texinfo.
|
||||
dnf update -y
|
||||
dnf install -y wget git python36 python3-pip
|
||||
@@ -135,11 +137,16 @@ install() {
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == PyYAML ]; then
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user PyYAML
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
dnf install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
libreadline7 # for cmake and llvm
|
||||
libffi6 libxml2 # for llvm
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
@@ -45,12 +47,15 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
dotnet-sdk-3.1 golang nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
cat >/etc/apt/sources.list <<EOF
|
||||
deb http://deb.debian.org/debian/ buster main non-free contrib
|
||||
@@ -83,5 +88,6 @@ EOF
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
libreadline7 # for cmake and llvm
|
||||
libffi6 libxml2 # for llvm
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
@@ -43,14 +45,18 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
|
||||
autoconf # for jemalloc code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
install_all_apt "$1"
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -8,23 +8,29 @@ source "$DIR/../util.sh"
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
pkg
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
echo "TODO: Implement ${FUNCNAME[0]}."
|
||||
exit 1
|
||||
}
|
||||
|
||||
install() {
|
||||
echo "TODO: Implement ${FUNCNAME[0]}."
|
||||
exit 1
|
||||
}
|
||||
|
||||
# http://ahmed.amayem.com/bash-indirect-expansion-exploration
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -18,6 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libffi-dev libxml2-dev # llvm
|
||||
libedit-dev libpcre3-dev automake bison # swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
@@ -27,6 +28,7 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
libreadline7 # for cmake and llvm
|
||||
libffi6 libxml2 # for llvm
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
@@ -44,14 +46,18 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
|
||||
autoconf # for jemalloc code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
apt install -y $1
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -17,6 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
@@ -26,6 +27,7 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi7 libxml2 # for llvm
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
@@ -45,12 +47,15 @@ MEMGRAPH_BUILD_DEPS=(
|
||||
dotnet-sdk-3.1 golang nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
apt update
|
||||
@@ -75,5 +80,6 @@ install() {
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
|
||||
@@ -683,7 +683,15 @@ def _typing_to_cypher_type(type_):
|
||||
return _mgp.type_nullable(simple_type)
|
||||
return _mgp.type_nullable(parse_typing(type_arg_as_str))
|
||||
elif type_as_str.startswith('typing.List'):
|
||||
type_arg_as_str, = parse_type_args(type_as_str)
|
||||
type_arg_as_str = parse_type_args(type_as_str)
|
||||
|
||||
if len(type_arg_as_str) > 1:
|
||||
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
|
||||
# so we need to join the parts.
|
||||
type_arg_as_str = ', '.join(type_arg_as_str)
|
||||
else:
|
||||
type_arg_as_str = type_arg_as_str[0]
|
||||
|
||||
simple_type = get_simple_type(type_arg_as_str)
|
||||
if simple_type is not None:
|
||||
return _mgp.type_list(simple_type)
|
||||
|
||||
150
libs/setup.sh
150
libs/setup.sh
@@ -2,8 +2,9 @@
|
||||
|
||||
# Download external dependencies.
|
||||
|
||||
local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
|
||||
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
cd ${working_dir}
|
||||
cd "${working_dir}"
|
||||
|
||||
# Clones a git repository and optionally cherry picks additional commits. The
|
||||
# function will try to preserve any local changes in the repo.
|
||||
@@ -15,7 +16,11 @@ clone () {
|
||||
shift 3
|
||||
# Clone if there's no repo.
|
||||
if [[ ! -d "$dir_name" ]]; then
|
||||
git clone "$git_repo" "$dir_name"
|
||||
echo "Cloning from $git_repo"
|
||||
# If the clone fails, it doesn't make sense to continue with the function
|
||||
# execution but the whole script should continue executing because we might
|
||||
# clone the same repo from a different source.
|
||||
git clone "$git_repo" "$dir_name" || return 1
|
||||
fi
|
||||
pushd "$dir_name"
|
||||
# Just fetch new commits from remote repository. Don't merge/pull them in, so
|
||||
@@ -29,12 +34,17 @@ clone () {
|
||||
# Stash regardless of local_changes, so that a user gets a message on stdout.
|
||||
git stash
|
||||
# Checkout the primary commit (there's no need to pull/merge).
|
||||
git checkout $checkout_id
|
||||
# The checkout fail should exit this script immediately because the target
|
||||
# commit is not there and that will most likely create build-time errors.
|
||||
git checkout "$checkout_id" || exit 1
|
||||
# Apply any optional cherry pick fixes.
|
||||
while [[ $# -ne 0 ]]; do
|
||||
local cherry_pick_id=$1
|
||||
shift
|
||||
git cherry-pick -n $cherry_pick_id
|
||||
# The cherry-pick fail should exit this script immediately because the
|
||||
# target commit is not there and that will most likely create build-time
|
||||
# errors.
|
||||
git cherry-pick -n "$cherry_pick_id" || exit 1
|
||||
done
|
||||
# Reapply any local changes.
|
||||
if [[ $local_changes == true ]]; then
|
||||
@@ -43,12 +53,95 @@ clone () {
|
||||
popd
|
||||
}
|
||||
|
||||
file_get_try_double () {
|
||||
primary_url="$1"
|
||||
secondary_url="$2"
|
||||
echo "Download primary from $primary_url secondary from $secondary_url"
|
||||
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
|
||||
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
|
||||
filename="$(basename "$secondary_url")"
|
||||
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
|
||||
echo ""
|
||||
}
|
||||
|
||||
repo_clone_try_double () {
|
||||
primary_url="$1"
|
||||
secondary_url="$2"
|
||||
folder_name="$3"
|
||||
ref="$4"
|
||||
echo "Cloning primary from $primary_url secondary from $secondary_url"
|
||||
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
|
||||
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
|
||||
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
|
||||
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
|
||||
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
|
||||
echo ""
|
||||
}
|
||||
|
||||
# List all dependencies.
|
||||
|
||||
# The reason for introducing primary and secondary urls are:
|
||||
# * HTTPS is hard to cache
|
||||
# * Remote development workflow is more flexible if people don't have to connect to VPN
|
||||
# * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
|
||||
# * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
|
||||
|
||||
# The goal of having primary urls is to have links to the "local" cache of
|
||||
# dependencies where these dependencies could be downloaded as fast as
|
||||
# possible. The actual cache server could be on your local machine, on a
|
||||
# dedicated machine inside the build cluster or on the actual build machine.
|
||||
# Download from primary_urls might fail because the cache is not installed.
|
||||
declare -A primary_urls=(
|
||||
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
|
||||
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
|
||||
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
|
||||
["fmt"]="http://$local_cache_host/git/fmt.git"
|
||||
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
|
||||
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
|
||||
["gtest"]="http://$local_cache_host/git/googletest.git"
|
||||
["gflags"]="http://$local_cache_host/git/gflags.git"
|
||||
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
|
||||
["bzip2"]="http://$local_cache_host/git/bzip2.git"
|
||||
["zlib"]="http://$local_cache_host/git/zlib.git"
|
||||
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
|
||||
["mgclient"]="http://$local_cache_host/git/mgclient.git"
|
||||
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
|
||||
["spdlog"]="http://$local_cache_host/git/spdlog"
|
||||
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
|
||||
)
|
||||
|
||||
# The goal of secondary urls is to have links to the "source of truth" of
|
||||
# dependencies, e.g., Github or S3. Download from secondary urls, if happens
|
||||
# at all, should never fail. In other words, if it fails, the whole build
|
||||
# should fail.
|
||||
declare -A secondary_urls=(
|
||||
["antlr4-code"]="https://github.com/antlr/antlr4.git"
|
||||
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
|
||||
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
|
||||
["fmt"]="https://github.com/fmtlib/fmt.git"
|
||||
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
|
||||
["gbenchmark"]="https://github.com/google/benchmark.git"
|
||||
["gtest"]="https://github.com/google/googletest.git"
|
||||
["gflags"]="https://github.com/memgraph/gflags.git"
|
||||
["libbcrypt"]="https://github.com/rg3/libbcrypt"
|
||||
["bzip2"]="https://github.com/VFR-maniac/bzip2"
|
||||
["zlib"]="https://github.com/madler/zlib.git"
|
||||
["rocksdb"]="https://github.com/facebook/rocksdb.git"
|
||||
["mgclient"]="https://github.com/memgraph/mgclient.git"
|
||||
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
|
||||
["spdlog"]="https://github.com/gabime/spdlog"
|
||||
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
|
||||
)
|
||||
|
||||
# antlr
|
||||
antlr_generator_filename="antlr-4.6-complete.jar"
|
||||
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
|
||||
wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
|
||||
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
|
||||
|
||||
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
|
||||
clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
|
||||
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
|
||||
# fix missing include
|
||||
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
|
||||
# remove shared library from install dependencies
|
||||
@@ -56,74 +149,73 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
|
||||
|
||||
# cppitertools v2.0 2019-12-23
|
||||
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
|
||||
clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
|
||||
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
|
||||
|
||||
# fmt
|
||||
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
|
||||
clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
|
||||
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
|
||||
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
|
||||
|
||||
# rapidcheck
|
||||
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
|
||||
clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
|
||||
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
|
||||
|
||||
# google benchmark
|
||||
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
|
||||
clone https://github.com/google/benchmark.git benchmark $benchmark_tag
|
||||
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
|
||||
|
||||
# google test
|
||||
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
|
||||
clone https://github.com/google/googletest.git googletest $googletest_tag
|
||||
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
|
||||
|
||||
# google flags
|
||||
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
|
||||
clone https://github.com/memgraph/gflags.git gflags $gflags_tag
|
||||
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
|
||||
|
||||
# libbcrypt
|
||||
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
|
||||
clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
|
||||
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
|
||||
|
||||
# neo4j
|
||||
wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
|
||||
tar -xzf neo4j.tar.gz
|
||||
rm -rf neo4j
|
||||
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
|
||||
tar -xzf neo4j-community-3.2.3-unix.tar.gz
|
||||
mv neo4j-community-3.2.3 neo4j
|
||||
rm neo4j.tar.gz
|
||||
rm neo4j-community-3.2.3-unix.tar.gz
|
||||
|
||||
# nlohmann json
|
||||
# We wget header instead of cloning repo since repo is huge (lots of test data).
|
||||
# We use head on Sep 1, 2017 instead of last release since it was long time ago.
|
||||
mkdir -p json
|
||||
cd json
|
||||
wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
|
||||
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
|
||||
cd ..
|
||||
|
||||
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
|
||||
clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
|
||||
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
|
||||
|
||||
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
|
||||
clone https://github.com/madler/zlib.git zlib $zlib_tag
|
||||
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
|
||||
# remove shared library from install dependencies
|
||||
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
|
||||
|
||||
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
|
||||
clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
|
||||
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
|
||||
# remove shared library from install dependencies
|
||||
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
|
||||
|
||||
# mgclient
|
||||
mgclient_tag="v1.2.0" # (2021-01-14)
|
||||
clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
|
||||
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
|
||||
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
|
||||
|
||||
# pymgclient
|
||||
pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
|
||||
clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
|
||||
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
|
||||
|
||||
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
|
||||
clone https://github.com/gabime/spdlog spdlog $spdlog_tag
|
||||
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
|
||||
|
||||
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
|
||||
clone https://github.com/jemalloc/jemalloc.git jemalloc $jemalloc_tag
|
||||
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
|
||||
pushd jemalloc
|
||||
# ThreadPool select job randomly, and there can be some threads that had been
|
||||
# performed some memory heavy task before and will be inactive for some time,
|
||||
@@ -138,5 +230,5 @@ pushd jemalloc
|
||||
# avoid spurious latencies and additional work associated with
|
||||
# MADV_DONTNEED. See
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
|
||||
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000"
|
||||
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
popd
|
||||
|
||||
@@ -1,33 +1,40 @@
|
||||
# User License Agreement
|
||||
# Memgraph Community User License Agreement
|
||||
|
||||
1. Description
|
||||
This License Agreement governs your use of the Memgraph Community Release (the
|
||||
"Software") and documentation ("Documentation").
|
||||
|
||||
THIS LICENSE AGREEMENT GOVERNS LICENSEE’S USE OF THE MEMGRAPH COMMUNITY
|
||||
RELEASE AND DOCUMENTATION.
|
||||
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU ("LICENSEE") AGREE TO THESE
|
||||
TERMS.
|
||||
|
||||
2. License Grant
|
||||
1. License Grant
|
||||
|
||||
The Software and Documentation are provided to Licensee at no charge and are
|
||||
licensed, not sold to Licensee. No ownership of any part of the Software and
|
||||
Documentation is hereby transferred to Licensee. Subject to (i) the terms and
|
||||
conditions of this License Agreement, (ii) any additional license restrictions
|
||||
and parameters contained on Licensor’s quotation, website, or order form
|
||||
(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
|
||||
conditions of this License Agreement, and (ii) any additional license
|
||||
restrictions and parameters contained on Licensor’s quotation, website, or
|
||||
order form, Licensor hereby grants Licensee a personal, non-assignable,
|
||||
non-transferable and non-exclusive license to install, access and use the
|
||||
Software (in object code form only) and Documentation for Licensee’s internal
|
||||
business purposes only. All rights relating to the Software and Documentation
|
||||
that are not expressly licensed in this License Agreement, whether now existing
|
||||
or which may hereafter come into existence are reserved for Licensor. Licensee
|
||||
shall not remove, obscure, or alter any proprietary rights notices (including
|
||||
without limitation copyright and trademark notices), which may be affixed to or
|
||||
contained within the Software or Documentation.
|
||||
business purposes (including for use in a production environment) only. All
|
||||
rights relating to the Software and Documentation that are not expressly
|
||||
licensed in this License Agreement, whether now existing or which may hereafter
|
||||
come into existence are reserved for Licensor. Licensee shall not remove,
|
||||
obscure, or alter any proprietary rights notices (including without limitation
|
||||
copyright and trademark notices), which may be affixed to or contained within
|
||||
the Software or Documentation.
|
||||
|
||||
3. Restrictions
|
||||
Licensor may terminate this License Agreement with immediate effect upon
|
||||
written notice to the Licensee. Upon termination Licensee shall delete all
|
||||
electronic copies of all or any part of the Software and/or the Documentation
|
||||
resident in its systems or elsewhere.
|
||||
|
||||
2. Restrictions
|
||||
|
||||
Licensee will not, directly or indirectly, (a) copy the Software or
|
||||
Documentation in any manner or for any purpose; (b) install, access or use any
|
||||
component of the Software or Documentation for any purpose not expressly
|
||||
granted in Section 2 above; (c) resell, distribute, publicly display or
|
||||
granted in Section 1 above; (c) resell, distribute, publicly display or
|
||||
publicly perform the Software or Documentation or any component thereof, by
|
||||
transfer, lease, loan or any other means, or make it available for use by
|
||||
others in any time-sharing, service bureau or similar arrangement; (d)
|
||||
@@ -37,25 +44,55 @@ algorithms or techniques incorporated in the Software; (e) export the Software
|
||||
or Documentation in violation of any applicable laws or regulations; (f)
|
||||
modify, translate, adapt, or create derivative works from the Software or
|
||||
Documentation; (g) circumvent, disable or otherwise interfere with
|
||||
security-related features of the Software or Documentation; (h)
|
||||
reverse-engineer, disassemble, attempt to derive the source code; (i) use the
|
||||
security-related features of the Software or Documentation; (h) use the
|
||||
Software or Documentation for any illegal purpose, in any manner that is
|
||||
inconsistent with the terms of this License Agreement, or to engage in illegal
|
||||
activity; (j) remove or alter any trademark, logo, copyright or other
|
||||
activity; (i) remove or alter any trademark, logo, copyright or other
|
||||
proprietary notices, legends, symbols or labels on, or embedded in, the
|
||||
Software or Documentation; or (k) provide access to the Software or
|
||||
Software or Documentation; or (j) provide access to the Software or
|
||||
Documentation to third parties.
|
||||
|
||||
4. Warranty Disclaimer
|
||||
3. Warranty Disclaimer
|
||||
|
||||
THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED “AS IS” FOR
|
||||
DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
|
||||
PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
|
||||
IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
|
||||
THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
|
||||
NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
|
||||
INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
|
||||
WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
|
||||
THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" AND LICENSOR MAKES NO
|
||||
WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
|
||||
INFRINGEMENT OF THIRD PARTIES’ INTELLECTUAL PROPERTY RIGHTS OR OTHER
|
||||
PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
|
||||
FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
|
||||
OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
|
||||
|
||||
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.
|
||||
4. Limitation of Liability
|
||||
|
||||
Licensor shall not in any circumstances be liable, whether in tort (including
|
||||
for negligence or breach of statutory duty howsoever arising), contract,
|
||||
misrepresentation (whether innocent or negligent) or otherwise for: loss of
|
||||
profits, loss of business, depletion of goodwill or similar losses, loss of
|
||||
anticipated savings, loss of goods, loss or corruption of data or computer
|
||||
downtime, or any special, indirect, consequential or pure economic loss, costs,
|
||||
damages, charges or expenses.
|
||||
|
||||
Licensor's total aggregate liability in contract, tort (including without
|
||||
limitation negligence or breach of statutory duty howsoever arising),
|
||||
misrepresentation (whether innocent or negligent), restitution or otherwise,
|
||||
arising in connection with the performance or contemplated performance of this
|
||||
License Agreement shall in all circumstances be limited to GBP10.00 (ten pounds
|
||||
sterling).
|
||||
|
||||
Nothing in this License Agreement shall limit Licensor’s liability in the case
|
||||
of death or personal injury caused by negligence, fraud, or fraudulent
|
||||
misrepresentation, or where it otherwise cannot be limited by law.
|
||||
|
||||
5. Technical Data
|
||||
|
||||
Licensor may collect and use technical information (such as usage patterns)
|
||||
gathered when the Licensee downloads and uses the Software. This is generally
|
||||
statistical data which does not identify an identified or identifiable
|
||||
individual. It may also include Licensee’s IP address which is personal data
|
||||
and is processed in accordance with our Privacy Policy. We only use this
|
||||
technical information to improve our products.
|
||||
|
||||
6. Law and Jurisdiction
|
||||
|
||||
This License Agreement is governed by the laws of England and is subject to the
|
||||
non-exclusive jurisdiction of the courts of England.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
FROM debian:buster
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG deb_release
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
FROM debian:buster
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG deb_release
|
||||
|
||||
|
||||
@@ -192,7 +192,19 @@ if args.version:
|
||||
try:
|
||||
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
|
||||
if current_branch != "master":
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
branches = get_output("git", "branch")
|
||||
if "master" in branches:
|
||||
# If master is present locally, the fetch is allowed to fail
|
||||
# because this script will still be able to compare against the
|
||||
# master branch.
|
||||
try:
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# If master is not present locally, the fetch command has to
|
||||
# succeed because something else will fail otherwise.
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
except Exception:
|
||||
print("Fatal error while ensuring local master branch.")
|
||||
sys.exit(1)
|
||||
|
||||
12
release/package/centos-7/Dockerfile
Normal file
12
release/package/centos-7/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM centos:7
|
||||
|
||||
RUN yum -y update \
|
||||
&& yum install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-7.tar.gz \
|
||||
-O toolchain-v2-binaries-centos-7.tar.gz \
|
||||
&& tar xzvf toolchain-v2-binaries-centos-7.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
12
release/package/centos-8/Dockerfile
Normal file
12
release/package/centos-8/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM centos:8
|
||||
|
||||
RUN dnf -y update \
|
||||
&& dnf install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-8.tar.gz \
|
||||
-O toolchain-v2-binaries-centos-8.tar.gz \
|
||||
&& tar xzvf toolchain-v2-binaries-centos-8.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
15
release/package/debian-10/Dockerfile
Normal file
15
release/package/debian-10/Dockerfile
Normal file
@@ -0,0 +1,15 @@
|
||||
FROM debian:10
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-10.tar.gz \
|
||||
-O toolchain-v2-binaries-debian-10.tar.gz \
|
||||
&& tar xzvf toolchain-v2-binaries-debian-10.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
15
release/package/debian-9/Dockerfile
Normal file
15
release/package/debian-9/Dockerfile
Normal file
@@ -0,0 +1,15 @@
|
||||
FROM debian:9
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-9.tar.gz \
|
||||
-O toolchain-v2-binaries-debian-9.tar.gz \
|
||||
&& tar xzvf toolchain-v2-binaries-debian-9.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
26
release/package/docker-compose.yml
Normal file
26
release/package/docker-compose.yml
Normal file
@@ -0,0 +1,26 @@
|
||||
version: "3"
|
||||
services:
|
||||
mgbuild_centos-7:
|
||||
build:
|
||||
context: centos-7
|
||||
container_name: "mgbuild_centos-7"
|
||||
mgbuild_centos-8:
|
||||
build:
|
||||
context: centos-8
|
||||
container_name: "mgbuild_centos-8"
|
||||
mgbuild_debian-9:
|
||||
build:
|
||||
context: debian-9
|
||||
container_name: "mgbuild_debian-9"
|
||||
mgbuild_debian-10:
|
||||
build:
|
||||
context: debian-10
|
||||
container_name: "mgbuild_debian-10"
|
||||
mgbuild_ubuntu-18.04:
|
||||
build:
|
||||
context: ubuntu-18.04
|
||||
container_name: "mgbuild_ubuntu-18.04"
|
||||
mgbuild_ubuntu-20.04:
|
||||
build:
|
||||
context: ubuntu-20.04
|
||||
container_name: "mgbuild_ubuntu-20.04"
|
||||
152
release/package/run.sh
Executable file
152
release/package/run.sh
Executable file
@@ -0,0 +1,152 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
SUPPORTED_OFFERING=(community enterprise)
|
||||
SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04)
|
||||
PROJECT_ROOT="$SCRIPT_DIR/../.."
|
||||
ACTIVATE_TOOLCHAIN="source /opt/toolchain-v2/activate"
|
||||
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
|
||||
|
||||
print_help () {
|
||||
echo "$0 init|package {offering} {os} [--for-docker]|docker|test"
|
||||
echo ""
|
||||
echo " offerings: ${SUPPORTED_OFFERING[*]}"
|
||||
echo " OSs: ${SUPPORTED_OS[*]}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
make_package () {
|
||||
offering="$1"
|
||||
offering_flag=" -DMG_ENTERPRISE=OFF "
|
||||
if [[ "$offering" == "enterprise" ]]; then
|
||||
offering_flag=" -DMG_ENTERPRISE=ON "
|
||||
fi
|
||||
if [[ "$offering" == "community" ]]; then
|
||||
offering_flag=" -DMG_ENTERPRISE=OFF "
|
||||
fi
|
||||
os="$2"
|
||||
package_command=""
|
||||
if [[ "$os" =~ ^"centos".* ]]; then
|
||||
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
|
||||
fi
|
||||
if [[ "$os" =~ ^"debian".* ]]; then
|
||||
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
|
||||
fi
|
||||
if [[ "$os" =~ ^"ubuntu".* ]]; then
|
||||
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
|
||||
fi
|
||||
docker_flag=" -DBUILD_FOR_DOCKER=OFF "
|
||||
if [[ "$#" -gt 2 ]]; then
|
||||
if [[ "$3" == "--for-docker" ]]; then
|
||||
docker_flag=" -DBUILD_FOR_DOCKER=ON "
|
||||
fi
|
||||
fi
|
||||
build_container="mgbuild_$os"
|
||||
echo "Building Memgraph $offering for $os on $build_container..."
|
||||
|
||||
echo "Copying project files..."
|
||||
# If master is not the current branch, fetch it, because the get_version
|
||||
# script depends on it. If we are on master, the fetch command is going to
|
||||
# fail so that's why there is the explicit check.
|
||||
# Required here because Docker build container can't access remote.
|
||||
cd "$PROJECT_ROOT"
|
||||
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
|
||||
git fetch origin master:master
|
||||
fi
|
||||
docker exec "$build_container" mkdir -p /memgraph
|
||||
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
|
||||
|
||||
container_build_dir="/memgraph/build"
|
||||
container_output_dir="$container_build_dir/output"
|
||||
|
||||
# TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
|
||||
# image build phase, but that is not easy at this point because the
|
||||
# environment/os/{os}.sh does not come within the toolchain package. When
|
||||
# migrating to the next version of toolchain do that, and remove the
|
||||
# TOOLCHAIN_RUN_DEPS installation from here.
|
||||
echo "Installing dependencies..."
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
|
||||
|
||||
echo "Building targeted package..."
|
||||
docker exec "$build_container" bash -c "cd /memgraph && ./init"
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $offering_flag $docker_flag .."
|
||||
# ' is used instead of " because we need to run make within the allowed
|
||||
# container resources.
|
||||
# shellcheck disable=SC2016
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc)'
|
||||
docker exec "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
|
||||
|
||||
echo "Copying targeted package to host..."
|
||||
last_package_name=$(docker exec "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
|
||||
# The operating system folder is introduced because multiple different
|
||||
# packages could be preserved during the same build "session".
|
||||
mkdir -p "$HOST_OUTPUT_DIR/$os"
|
||||
package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
|
||||
docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
|
||||
echo "Package saved to $package_host_destination."
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
init)
|
||||
cd "$SCRIPT_DIR"
|
||||
docker-compose build
|
||||
docker-compose up -d
|
||||
;;
|
||||
|
||||
docker)
|
||||
# NOTE: Docker is build on top of Debian 10 package.
|
||||
based_on_os="debian-10"
|
||||
# shellcheck disable=SC2012
|
||||
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
|
||||
docker_build_folder="$PROJECT_ROOT/release/docker"
|
||||
cd "$docker_build_folder"
|
||||
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
|
||||
# shellcheck disable=SC2012
|
||||
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
|
||||
docker_host_folder="$HOST_OUTPUT_DIR/docker"
|
||||
docker_host_image_path="$docker_host_folder/$docker_image_name"
|
||||
mkdir -p "$docker_host_folder"
|
||||
cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
|
||||
echo "Docker images saved to $docker_host_image_path."
|
||||
;;
|
||||
|
||||
package)
|
||||
shift 1
|
||||
if [[ "$#" -lt 2 ]]; then
|
||||
print_help
|
||||
fi
|
||||
offering="$1"
|
||||
shift 1
|
||||
is_offering_ok=false
|
||||
for supported_offering in "${SUPPORTED_OFFERING[@]}"; do
|
||||
if [[ "$supported_offering" == "${offering}" ]]; then
|
||||
is_offering_ok=true
|
||||
fi
|
||||
done
|
||||
os="$1"
|
||||
shift 1
|
||||
is_os_ok=false
|
||||
for supported_os in "${SUPPORTED_OS[@]}"; do
|
||||
if [[ "$supported_os" == "${os}" ]]; then
|
||||
is_os_ok=true
|
||||
fi
|
||||
done
|
||||
if [[ "$is_offering_ok" == true ]] && [[ "$is_os_ok" == true ]]; then
|
||||
make_package "$offering" "$os" "$@"
|
||||
else
|
||||
print_help
|
||||
fi
|
||||
;;
|
||||
|
||||
test)
|
||||
echo "TODO(gitbuda): Test all packages on mgtest containers."
|
||||
;;
|
||||
|
||||
*)
|
||||
print_help
|
||||
;;
|
||||
esac
|
||||
15
release/package/ubuntu-18.04/Dockerfile
Normal file
15
release/package/ubuntu-18.04/Dockerfile
Normal file
@@ -0,0 +1,15 @@
|
||||
FROM ubuntu:18.04
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-18.04.tar.gz \
|
||||
-O toolchain-v2-binaries-ubuntu-18.04.tar.gz \
|
||||
&& tar xzvf toolchain-v2-binaries-ubuntu-18.04.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
15
release/package/ubuntu-20.04/Dockerfile
Normal file
15
release/package/ubuntu-20.04/Dockerfile
Normal file
@@ -0,0 +1,15 @@
|
||||
FROM ubuntu:20.04
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-20.04.tar.gz \
|
||||
-O toolchain-v2-binaries-ubuntu-20.04.tar.gz \
|
||||
&& tar xzvf toolchain-v2-binaries-ubuntu-20.04.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
#!/usr/bin/python3
|
||||
import json
|
||||
import io
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env python3
|
||||
#!/usr/bin/python3
|
||||
import json
|
||||
import io
|
||||
import ssl
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/readable_size.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/tsc.hpp"
|
||||
|
||||
@@ -603,8 +604,8 @@ struct PullPlanVector {
|
||||
|
||||
struct PullPlan {
|
||||
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory);
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
std::optional<size_t> memory_limit = {});
|
||||
std::optional<ExecutionContext> Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
@@ -614,6 +615,7 @@ struct PullPlan {
|
||||
plan::UniqueCursorPtr cursor_ = nullptr;
|
||||
Frame frame_;
|
||||
ExecutionContext ctx_;
|
||||
std::optional<size_t> memory_limit_;
|
||||
|
||||
// As it's possible to query execution using multiple pulls
|
||||
// we need the keep track of the total execution time across
|
||||
@@ -630,11 +632,12 @@ struct PullPlan {
|
||||
};
|
||||
|
||||
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory)
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
const std::optional<size_t> memory_limit)
|
||||
: plan_(plan),
|
||||
cursor_(plan->plan().MakeCursor(execution_memory)),
|
||||
frame_(plan->symbol_table().max_position(), execution_memory) {
|
||||
frame_(plan->symbol_table().max_position(), execution_memory),
|
||||
memory_limit_(memory_limit) {
|
||||
ctx_.db_accessor = dba;
|
||||
ctx_.symbol_table = plan->symbol_table();
|
||||
ctx_.evaluation_context.timestamp =
|
||||
@@ -657,21 +660,25 @@ std::optional<ExecutionContext> PullPlan::Pull(AnyStream *stream, std::optional<
|
||||
// single `Pull`.
|
||||
constexpr size_t stack_size = 256 * 1024;
|
||||
char stack_data[stack_size];
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
|
||||
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
|
||||
|
||||
if (memory_limit_) {
|
||||
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
|
||||
ctx_.evaluation_context.memory = &*maybe_limited_resource;
|
||||
} else {
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
}
|
||||
|
||||
// Returns true if a result was pulled.
|
||||
const auto pull_result = [&]() -> bool {
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
|
||||
return cursor_->Pull(frame_, ctx_);
|
||||
};
|
||||
const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); };
|
||||
|
||||
const auto stream_values = [&]() {
|
||||
// TODO: The streamed values should also probably use the above memory.
|
||||
@@ -828,10 +835,24 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
|
||||
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage),
|
||||
utils::Downcast<CypherQuery>(parsed_query.query), parsed_query.parameters,
|
||||
&interpreter_context->plan_cache, dba, parsed_query.is_cacheable);
|
||||
utils::MemoryResource *execution_memory) {
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
|
||||
|
||||
Frame frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
evaluation_context.parameters = parsed_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
|
||||
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
if (memory_limit) {
|
||||
spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit));
|
||||
}
|
||||
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
|
||||
parsed_query.parameters, &interpreter_context->plan_cache, dba);
|
||||
|
||||
summary->insert_or_assign("cost_estimate", plan->cost());
|
||||
auto rw_type_checker = plan::ReadWriteTypeChecker();
|
||||
@@ -850,8 +871,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
|
||||
}
|
||||
|
||||
auto pull_plan =
|
||||
std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory);
|
||||
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
|
||||
execution_memory, memory_limit);
|
||||
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
|
||||
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
@@ -865,7 +886,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
|
||||
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
utils::MemoryResource *execution_memory) {
|
||||
const std::string kExplainQueryStart = "explain ";
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
|
||||
"Expected stripped query to start with '{}'", kExplainQueryStart);
|
||||
@@ -911,7 +932,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
|
||||
|
||||
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory) {
|
||||
const std::string kProfileQueryStart = "profile ";
|
||||
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
|
||||
@@ -949,6 +970,15 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
|
||||
Frame frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
evaluation_context.parameters = parsed_inner_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
|
||||
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
|
||||
auto cypher_query_plan = CypherQueryToPlan(
|
||||
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
|
||||
@@ -960,14 +990,14 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), summary, dba,
|
||||
interpreter_context, execution_memory,
|
||||
interpreter_context, execution_memory, memory_limit,
|
||||
// We want to execute the query we are profiling lazily, so we delay
|
||||
// the construction of the corresponding context.
|
||||
ctx = std::optional<ExecutionContext>{}, pull_plan = std::shared_ptr<PullPlanVector>(nullptr)](
|
||||
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
|
||||
// No output symbols are given so that nothing is streamed.
|
||||
if (!ctx) {
|
||||
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory)
|
||||
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, memory_limit)
|
||||
.Pull(stream, {}, {}, summary);
|
||||
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(ctx->stats, ctx->profile_execution_time));
|
||||
}
|
||||
@@ -985,7 +1015,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
}
|
||||
|
||||
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
utils::MemoryResource *execution_memory) {
|
||||
return PreparedQuery{{"QUERY"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::make_shared<PullPlanDump>(dba)](
|
||||
@@ -1000,7 +1030,7 @@ PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, T
|
||||
|
||||
PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw IndexInMulticommandTxException();
|
||||
}
|
||||
@@ -1069,7 +1099,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
|
||||
|
||||
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw UserModificationInMulticommandTxException();
|
||||
}
|
||||
@@ -1122,6 +1152,8 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_ex
|
||||
return std::nullopt;
|
||||
},
|
||||
RWType::NONE};
|
||||
// False positive report for the std::make_shared above
|
||||
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
|
||||
}
|
||||
|
||||
PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
@@ -1180,7 +1212,7 @@ PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_exp
|
||||
|
||||
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
storage::Storage *db, utils::MonotonicBufferResource *execution_memory) {
|
||||
storage::Storage *db, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw InfoInMulticommandTxException();
|
||||
}
|
||||
@@ -1268,8 +1300,7 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
|
||||
|
||||
PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw ConstraintInMulticommandTxException();
|
||||
}
|
||||
@@ -1434,10 +1465,12 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
|
||||
|
||||
// Handle transaction control queries.
|
||||
auto query_upper = utils::Trim(utils::ToUpperCase(query_string));
|
||||
|
||||
if (query_upper == "BEGIN" || query_upper == "COMMIT" || query_upper == "ROLLBACK") {
|
||||
query_execution->prepared_query.emplace(PrepareTransactionQuery(query_upper));
|
||||
const auto upper_case_query = utils::ToUpperCase(query_string);
|
||||
const auto trimmed_query = utils::Trim(upper_case_query);
|
||||
|
||||
if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") {
|
||||
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
|
||||
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
|
||||
}
|
||||
|
||||
|
||||
@@ -317,7 +317,9 @@ class Interpreter final {
|
||||
private:
|
||||
struct QueryExecution {
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
|
||||
utils::MonotonicBufferResource execution_monotonic_memory{kExecutionMemoryBlockSize};
|
||||
utils::ResourceWithOutOfMemoryException execution_memory{&execution_monotonic_memory};
|
||||
|
||||
std::map<std::string, TypedValue> summary;
|
||||
|
||||
explicit QueryExecution() = default;
|
||||
@@ -331,7 +333,7 @@ class Interpreter final {
|
||||
// destroy the prepared query which is using that instance
|
||||
// of execution memory.
|
||||
prepared_query.reset();
|
||||
execution_memory.Release();
|
||||
execution_monotonic_memory.Release();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -324,11 +324,15 @@ VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, Exec
|
||||
template <class TVerticesFun>
|
||||
class ScanAllCursor : public Cursor {
|
||||
public:
|
||||
explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices)
|
||||
: output_symbol_(output_symbol), input_cursor_(std::move(input_cursor)), get_vertices_(std::move(get_vertices)) {}
|
||||
explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices,
|
||||
const char *op_name)
|
||||
: output_symbol_(output_symbol),
|
||||
input_cursor_(std::move(input_cursor)),
|
||||
get_vertices_(std::move(get_vertices)),
|
||||
op_name_(op_name) {}
|
||||
|
||||
bool Pull(Frame &frame, ExecutionContext &context) override {
|
||||
SCOPED_PROFILE_OP("ScanAll");
|
||||
SCOPED_PROFILE_OP(op_name_);
|
||||
|
||||
if (MustAbort(context)) throw HintedAbortError();
|
||||
|
||||
@@ -364,6 +368,7 @@ class ScanAllCursor : public Cursor {
|
||||
TVerticesFun get_vertices_;
|
||||
std::optional<typename std::result_of<TVerticesFun(Frame &, ExecutionContext &)>::type::value_type> vertices_;
|
||||
std::optional<decltype(vertices_.value().begin())> vertices_it_;
|
||||
const char *op_name_;
|
||||
};
|
||||
|
||||
ScanAll::ScanAll(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, storage::View view)
|
||||
@@ -379,7 +384,7 @@ UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const {
|
||||
return std::make_optional(db->Vertices(view_));
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
|
||||
std::move(vertices));
|
||||
std::move(vertices), "ScanAll");
|
||||
}
|
||||
|
||||
std::vector<Symbol> ScanAll::ModifiedSymbols(const SymbolTable &table) const {
|
||||
@@ -402,7 +407,7 @@ UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const {
|
||||
return std::make_optional(db->Vertices(view_, label_));
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
|
||||
std::move(vertices));
|
||||
std::move(vertices), "ScanAllByLabel");
|
||||
}
|
||||
|
||||
// TODO(buda): Implement ScanAllByLabelProperty operator to iterate over
|
||||
@@ -466,7 +471,7 @@ UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *m
|
||||
return std::make_optional(db->Vertices(view_, label_, property_, maybe_lower, maybe_upper));
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
|
||||
std::move(vertices));
|
||||
std::move(vertices), "ScanAllByLabelPropertyRange");
|
||||
}
|
||||
|
||||
ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr<LogicalOperator> &input,
|
||||
@@ -498,7 +503,7 @@ UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *m
|
||||
return std::make_optional(db->Vertices(view_, label_, property_, storage::PropertyValue(value)));
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
|
||||
std::move(vertices));
|
||||
std::move(vertices), "ScanAllByLabelPropertyValue");
|
||||
}
|
||||
|
||||
ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol,
|
||||
@@ -516,7 +521,7 @@ UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) c
|
||||
return std::make_optional(db->Vertices(view_, label_, property_));
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
|
||||
std::move(vertices));
|
||||
std::move(vertices), "ScanAllByLabelProperty");
|
||||
}
|
||||
|
||||
ScanAllById::ScanAllById(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, Expression *expression,
|
||||
@@ -542,7 +547,7 @@ UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const {
|
||||
return std::vector<VertexAccessor>{*maybe_vertex};
|
||||
};
|
||||
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
|
||||
std::move(vertices));
|
||||
std::move(vertices), "ScanAllById");
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -522,7 +522,8 @@ std::vector<SingleQueryPart> CollectSingleQueryParts(SymbolTable &symbol_table,
|
||||
query_part->merge_matching.emplace_back(Matching{});
|
||||
AddMatching({merge->pattern_}, nullptr, symbol_table, storage, query_part->merge_matching.back());
|
||||
} else if (utils::IsSubtype(*clause, With::kType) || utils::IsSubtype(*clause, query::Unwind::kType) ||
|
||||
utils::IsSubtype(*clause, query::CallProcedure::kType)) {
|
||||
utils::IsSubtype(*clause, query::CallProcedure::kType) ||
|
||||
utils::IsSubtype(*clause, query::LoadCsv::kType)) {
|
||||
// This query part is done, continue with a new one.
|
||||
query_parts.emplace_back(SingleQueryPart{});
|
||||
query_part = &query_parts.back();
|
||||
|
||||
@@ -102,30 +102,47 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
|
||||
// recovery process.
|
||||
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
|
||||
Constraints *constraints, utils::SkipList<Vertex> *vertices) {
|
||||
spdlog::info("Recreating indices from metadata.");
|
||||
// Recover label indices.
|
||||
spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
|
||||
for (const auto &item : indices_constraints.indices.label) {
|
||||
if (!indices->label_index.CreateIndex(item, vertices->access()))
|
||||
throw RecoveryFailure("The label index must be created here!");
|
||||
spdlog::info("A label index is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label indices are recreated.");
|
||||
|
||||
// Recover label+property indices.
|
||||
spdlog::info("Recreating {} label+property indices from metadata.",
|
||||
indices_constraints.indices.label_property.size());
|
||||
for (const auto &item : indices_constraints.indices.label_property) {
|
||||
if (!indices->label_property_index.CreateIndex(item.first, item.second, vertices->access()))
|
||||
throw RecoveryFailure("The label+property index must be created here!");
|
||||
spdlog::info("A label+property index is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label+property indices are recreated.");
|
||||
spdlog::info("Indices are recreated.");
|
||||
|
||||
spdlog::info("Recreating constraints from metadata.");
|
||||
// Recover existence constraints.
|
||||
spdlog::info("Recreating {} existence constraints from metadata.", indices_constraints.constraints.existence.size());
|
||||
for (const auto &item : indices_constraints.constraints.existence) {
|
||||
auto ret = CreateExistenceConstraint(constraints, item.first, item.second, vertices->access());
|
||||
if (ret.HasError() || !ret.GetValue()) throw RecoveryFailure("The existence constraint must be created here!");
|
||||
spdlog::info("A existence constraint is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Existence constraints are recreated from metadata.");
|
||||
|
||||
// Recover unique constraints.
|
||||
spdlog::info("Recreating {} unique constraints from metadata.", indices_constraints.constraints.unique.size());
|
||||
for (const auto &item : indices_constraints.constraints.unique) {
|
||||
auto ret = constraints->unique_constraints.CreateConstraint(item.first, item.second, vertices->access());
|
||||
if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
|
||||
throw RecoveryFailure("The unique constraint must be created here!");
|
||||
spdlog::info("A unique constraint is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Unique constraints are recreated from metadata.");
|
||||
spdlog::info("Constraints are recreated from metadata.");
|
||||
}
|
||||
|
||||
std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_directory,
|
||||
@@ -137,7 +154,12 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
Indices *indices, Constraints *constraints, Config::Items items,
|
||||
uint64_t *wal_seq_num) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) return std::nullopt;
|
||||
spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
|
||||
wal_directory);
|
||||
if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) {
|
||||
spdlog::warn("Snapshot or WAL directory don't exist, there is nothing to recover.");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto snapshot_files = GetSnapshotFiles(snapshot_directory);
|
||||
|
||||
@@ -145,6 +167,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
RecoveredIndicesAndConstraints indices_constraints;
|
||||
std::optional<uint64_t> snapshot_timestamp;
|
||||
if (!snapshot_files.empty()) {
|
||||
spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory);
|
||||
// Order the files by name
|
||||
std::sort(snapshot_files.begin(), snapshot_files.end());
|
||||
|
||||
@@ -157,13 +180,13 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
spdlog::warn("The snapshot file {} isn't related to the latest snapshot file!", path);
|
||||
continue;
|
||||
}
|
||||
spdlog::info("Starting snapshot recovery from {}", path);
|
||||
spdlog::info("Starting snapshot recovery from {}.", path);
|
||||
try {
|
||||
recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history, name_id_mapper, edge_count, items);
|
||||
spdlog::info("Snapshot recovery successful!");
|
||||
break;
|
||||
} catch (const RecoveryFailure &e) {
|
||||
spdlog::warn("Couldn't recover snapshot from {} because of: {}", path, e.what());
|
||||
spdlog::warn("Couldn't recover snapshot from {} because of: {}.", path, e.what());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
@@ -181,6 +204,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
return recovered_snapshot->recovery_info;
|
||||
}
|
||||
} else {
|
||||
spdlog::info("No snapshot file was found, collecting information from WAL directory {}.", wal_directory);
|
||||
std::error_code error_code;
|
||||
if (!utils::DirExists(wal_directory)) return std::nullopt;
|
||||
// We use this smaller struct that contains only a subset of information
|
||||
@@ -206,7 +230,10 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
}
|
||||
}
|
||||
MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message());
|
||||
if (wal_files.empty()) return std::nullopt;
|
||||
if (wal_files.empty()) {
|
||||
spdlog::warn("No snapshot or WAL file found!");
|
||||
return std::nullopt;
|
||||
}
|
||||
std::sort(wal_files.begin(), wal_files.end());
|
||||
// UUID used for durability is the UUID of the last WAL file.
|
||||
// Same for the epoch id.
|
||||
@@ -215,7 +242,10 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
}
|
||||
|
||||
auto maybe_wal_files = GetWalFiles(wal_directory, *uuid);
|
||||
if (!maybe_wal_files) return std::nullopt;
|
||||
if (!maybe_wal_files) {
|
||||
spdlog::warn("Couldn't get WAL file info from the WAL directory!");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
// Array of all discovered WAL files, ordered by sequence number.
|
||||
auto &wal_files = *maybe_wal_files;
|
||||
@@ -232,6 +262,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
"files that match the last WAL file!");
|
||||
|
||||
if (!wal_files.empty()) {
|
||||
spdlog::info("Checking WAL files.");
|
||||
{
|
||||
const auto &first_wal = wal_files[0];
|
||||
if (first_wal.seq_num != 0) {
|
||||
@@ -255,6 +286,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
}
|
||||
std::optional<uint64_t> previous_seq_num;
|
||||
auto last_loaded_timestamp = snapshot_timestamp;
|
||||
spdlog::info("Trying to load WAL files.");
|
||||
for (auto &wal_file : wal_files) {
|
||||
if (previous_seq_num && (wal_file.seq_num - *previous_seq_num) > 1) {
|
||||
LOG_FATAL("You are missing a WAL file with the sequence number {}!", *previous_seq_num + 1);
|
||||
@@ -290,6 +322,8 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
// The sequence number needs to be recovered even though `LoadWal` didn't
|
||||
// load any deltas from that file.
|
||||
*wal_seq_num = *previous_seq_num + 1;
|
||||
|
||||
spdlog::info("All necessary WAL files are loaded successfully.");
|
||||
}
|
||||
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices);
|
||||
|
||||
@@ -168,14 +168,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
});
|
||||
|
||||
// Read snapshot info.
|
||||
auto info = ReadSnapshotInfo(path);
|
||||
|
||||
const auto info = ReadSnapshotInfo(path);
|
||||
spdlog::info("Recovering {} vertices and {} edges.", info.vertices_count, info.edges_count);
|
||||
// Check for edges.
|
||||
bool snapshot_has_edges = info.offset_edges != 0;
|
||||
|
||||
// Recover mapper.
|
||||
std::unordered_map<uint64_t, uint64_t> snapshot_id_map;
|
||||
{
|
||||
spdlog::info("Recovering mapper metadata.");
|
||||
if (!snapshot.SetPosition(info.offset_mapper)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto marker = snapshot.ReadMarker();
|
||||
@@ -191,6 +192,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (!name) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto my_id = name_id_mapper->NameToId(*name);
|
||||
snapshot_id_map.emplace(*id, my_id);
|
||||
SPDLOG_TRACE("Mapping \"{}\"from snapshot id {} to actual id {}.", *name, *id, my_id);
|
||||
}
|
||||
}
|
||||
auto get_label_from_id = [&snapshot_id_map](uint64_t snapshot_id) {
|
||||
@@ -217,10 +219,11 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
auto edge_acc = edges->access();
|
||||
uint64_t last_edge_gid = 0;
|
||||
if (snapshot_has_edges) {
|
||||
spdlog::info("Recovering {} edges.", info.edges_count);
|
||||
if (!snapshot.SetPosition(info.offset_edges)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
for (uint64_t i = 0; i < info.edges_count; ++i) {
|
||||
{
|
||||
auto marker = snapshot.ReadMarker();
|
||||
const auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_EDGE) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
@@ -230,6 +233,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
last_edge_gid = *gid;
|
||||
spdlog::debug("Recovering edge {} with properties.", *gid);
|
||||
auto [it, inserted] = edge_acc.insert(Edge{Gid::FromUint(*gid), nullptr});
|
||||
if (!inserted) throw RecoveryFailure("The edge must be inserted here!");
|
||||
|
||||
@@ -243,6 +247,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (!key) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto value = snapshot.ReadPropertyValue();
|
||||
if (!value) throw RecoveryFailure("Invalid snapshot data!");
|
||||
SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for edge {}.",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
|
||||
props.SetProperty(get_property_from_id(*key), *value);
|
||||
}
|
||||
}
|
||||
@@ -253,6 +259,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
last_edge_gid = *gid;
|
||||
|
||||
spdlog::debug("Ensuring edge {} doesn't have any properties.", *gid);
|
||||
// Read properties.
|
||||
{
|
||||
auto props_size = snapshot.ReadUint();
|
||||
@@ -264,12 +271,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
}
|
||||
}
|
||||
}
|
||||
spdlog::info("Edges are recovered.");
|
||||
}
|
||||
|
||||
// Recover vertices (labels and properties).
|
||||
if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
auto vertex_acc = vertices->access();
|
||||
uint64_t last_vertex_gid = 0;
|
||||
spdlog::info("Recovering {} vertices.", info.vertices_count);
|
||||
for (uint64_t i = 0; i < info.vertices_count; ++i) {
|
||||
{
|
||||
auto marker = snapshot.ReadMarker();
|
||||
@@ -283,10 +292,12 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
last_vertex_gid = *gid;
|
||||
spdlog::debug("Recovering vertex {}.", *gid);
|
||||
auto [it, inserted] = vertex_acc.insert(Vertex{Gid::FromUint(*gid), nullptr});
|
||||
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
|
||||
|
||||
// Recover labels.
|
||||
spdlog::trace("Recovering labels for vertex {}.", *gid);
|
||||
{
|
||||
auto labels_size = snapshot.ReadUint();
|
||||
if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
@@ -295,11 +306,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
for (uint64_t j = 0; j < *labels_size; ++j) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
SPDLOG_TRACE("Recovered label \"{}\" for vertex {}.", name_id_mapper->IdToName(snapshot_id_map.at(*label)),
|
||||
*gid);
|
||||
labels.emplace_back(get_label_from_id(*label));
|
||||
}
|
||||
}
|
||||
|
||||
// Recover properties.
|
||||
spdlog::trace("Recovering properties for vertex {}.", *gid);
|
||||
{
|
||||
auto props_size = snapshot.ReadUint();
|
||||
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
@@ -309,6 +323,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (!key) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto value = snapshot.ReadPropertyValue();
|
||||
if (!value) throw RecoveryFailure("Invalid snapshot data!");
|
||||
SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for vertex {}.",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
|
||||
props.SetProperty(get_property_from_id(*key), *value);
|
||||
}
|
||||
}
|
||||
@@ -339,8 +355,10 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
}
|
||||
spdlog::info("Vertices are recovered.");
|
||||
|
||||
// Recover vertices (in/out edges).
|
||||
spdlog::info("Recovering connectivity.");
|
||||
if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
for (auto &vertex : vertex_acc) {
|
||||
{
|
||||
@@ -348,6 +366,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
spdlog::trace("Recovering connectivity for vertex {}.", vertex.gid.AsUint());
|
||||
// Check vertex.
|
||||
auto gid = snapshot.ReadUint();
|
||||
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
@@ -377,6 +396,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
|
||||
// Recover in edges.
|
||||
{
|
||||
spdlog::trace("Recovering inbound edges for vertex {}.", vertex.gid.AsUint());
|
||||
auto in_size = snapshot.ReadUint();
|
||||
if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
vertex.in_edges.reserve(*in_size);
|
||||
@@ -404,12 +424,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
edge_ref = EdgeRef(&*edge);
|
||||
}
|
||||
}
|
||||
SPDLOG_TRACE("Recovered inbound edge {} with label \"{}\" from vertex {}.", *edge_gid,
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), from_vertex->gid.AsUint());
|
||||
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &*from_vertex, edge_ref);
|
||||
}
|
||||
}
|
||||
|
||||
// Recover out edges.
|
||||
{
|
||||
spdlog::trace("Recovering outbound edges for vertex {}.", vertex.gid.AsUint());
|
||||
auto out_size = snapshot.ReadUint();
|
||||
if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
vertex.out_edges.reserve(*out_size);
|
||||
@@ -437,6 +460,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
edge_ref = EdgeRef(&*edge);
|
||||
}
|
||||
}
|
||||
SPDLOG_TRACE("Recovered outbound edge {} with label \"{}\" to vertex {}.", *edge_gid,
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), to_vertex->gid.AsUint());
|
||||
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &*to_vertex, edge_ref);
|
||||
}
|
||||
// Increment edge count. We only increment the count here because the
|
||||
@@ -444,6 +469,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
edge_count->fetch_add(*out_size, std::memory_order_acq_rel);
|
||||
}
|
||||
}
|
||||
spdlog::info("Connectivity is recovered.");
|
||||
|
||||
// Set initial values for edge/vertex ID generators.
|
||||
ret.next_edge_id = last_edge_gid + 1;
|
||||
@@ -452,6 +478,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
|
||||
// Recover indices.
|
||||
{
|
||||
spdlog::info("Recovering metadata of indices.");
|
||||
if (!snapshot.SetPosition(info.offset_indices)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto marker = snapshot.ReadMarker();
|
||||
@@ -461,18 +488,22 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
{
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} label indices.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
AddRecoveredIndexConstraint(&indices_constraints.indices.label, get_label_from_id(*label),
|
||||
"The label index already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of label index for :{}", name_id_mapper->IdToName(snapshot_id_map.at(*label)));
|
||||
}
|
||||
spdlog::info("Metadata of label indices are recovered.");
|
||||
}
|
||||
|
||||
// Recover label+property indices.
|
||||
{
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} label+property indices.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
@@ -481,12 +512,18 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
AddRecoveredIndexConstraint(&indices_constraints.indices.label_property,
|
||||
{get_label_from_id(*label), get_property_from_id(*property)},
|
||||
"The label+property index already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of label+property index for :{}({})",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*label)),
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*property)));
|
||||
}
|
||||
spdlog::info("Metadata of label+property indices are recovered.");
|
||||
}
|
||||
spdlog::info("Metadata of indices are recovered.");
|
||||
}
|
||||
|
||||
// Recover constraints.
|
||||
{
|
||||
spdlog::info("Recovering metadata of constraints.");
|
||||
if (!snapshot.SetPosition(info.offset_constraints)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto marker = snapshot.ReadMarker();
|
||||
@@ -496,6 +533,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
{
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} existence constraints.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
@@ -504,7 +542,11 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
AddRecoveredIndexConstraint(&indices_constraints.constraints.existence,
|
||||
{get_label_from_id(*label), get_property_from_id(*property)},
|
||||
"The existence constraint already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of existence constraint for :{}({})",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*label)),
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*property)));
|
||||
}
|
||||
spdlog::info("Metadata of existence constraints are recovered.");
|
||||
}
|
||||
|
||||
// Recover unique constraints.
|
||||
@@ -513,6 +555,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
if (*version >= kUniqueConstraintVersion) {
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} unique constraints.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
@@ -526,10 +569,15 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
}
|
||||
AddRecoveredIndexConstraint(&indices_constraints.constraints.unique, {get_label_from_id(*label), properties},
|
||||
"The unique constraint already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of unique constraints for :{}",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*label)));
|
||||
}
|
||||
spdlog::info("Metadata of unique constraints are recovered.");
|
||||
}
|
||||
spdlog::info("Metadata of constraints are recovered.");
|
||||
}
|
||||
|
||||
spdlog::info("Recovering metadata.");
|
||||
// Recover epoch history
|
||||
{
|
||||
if (!snapshot.SetPosition(info.offset_epoch_history)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
@@ -555,6 +603,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::info("Metadata recovered.");
|
||||
// Recover timestamp.
|
||||
ret.next_timestamp = info.start_timestamp + 1;
|
||||
|
||||
|
||||
@@ -610,6 +610,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
|
||||
const std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
|
||||
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
|
||||
Config::Items items) {
|
||||
spdlog::info("Trying to load WAL file {}.", path);
|
||||
RecoveryInfo ret;
|
||||
|
||||
Decoder wal;
|
||||
@@ -622,13 +623,17 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
|
||||
ret.last_commit_timestamp = info.to_timestamp;
|
||||
|
||||
// Check timestamp.
|
||||
if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) return ret;
|
||||
if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) {
|
||||
spdlog::info("Skip loading WAL file because it is too old.");
|
||||
return ret;
|
||||
}
|
||||
|
||||
// Recover deltas.
|
||||
wal.SetPosition(info.offset_deltas);
|
||||
uint64_t deltas_applied = 0;
|
||||
auto edge_acc = edges->access();
|
||||
auto vertex_acc = vertices->access();
|
||||
spdlog::info("WAL file contains {} deltas.", info.num_deltas);
|
||||
for (uint64_t i = 0; i < info.num_deltas; ++i) {
|
||||
// Read WAL delta header to find out the delta timestamp.
|
||||
auto timestamp = ReadWalDeltaHeader(&wal);
|
||||
@@ -839,7 +844,8 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::info("Applied {} deltas from WAL", deltas_applied, path);
|
||||
spdlog::info("Applied {} deltas from WAL. Skipped {} deltas, because they were too old.", deltas_applied,
|
||||
info.num_deltas - deltas_applied);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -61,12 +61,7 @@ void Reader::TryInitializeHeader() {
|
||||
const Reader::Header &Reader::GetHeader() const { return header_; }
|
||||
|
||||
namespace {
|
||||
enum class CsvParserState : uint8_t {
|
||||
INITIAL_FIELD,
|
||||
NEXT_FIELD,
|
||||
QUOTING,
|
||||
EXPECT_DELIMITER,
|
||||
};
|
||||
enum class CsvParserState : uint8_t { INITIAL_FIELD, NEXT_FIELD, QUOTING, EXPECT_DELIMITER, DONE };
|
||||
|
||||
} // namespace
|
||||
|
||||
@@ -89,7 +84,12 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
|
||||
|
||||
std::string_view line_string_view = *maybe_line;
|
||||
|
||||
while (!line_string_view.empty()) {
|
||||
// remove '\r' from the end in case we have dos file format
|
||||
if (line_string_view.back() == '\r') {
|
||||
line_string_view.remove_suffix(1);
|
||||
}
|
||||
|
||||
while (state != CsvParserState::DONE && !line_string_view.empty()) {
|
||||
const auto c = line_string_view[0];
|
||||
|
||||
// Line feeds and carriage returns are ignored in CSVs.
|
||||
@@ -120,11 +120,11 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
|
||||
const auto delimiter_idx = line_string_view.find(*read_config_.delimiter);
|
||||
row.emplace_back(line_string_view.substr(0, delimiter_idx));
|
||||
if (delimiter_idx == std::string_view::npos) {
|
||||
line_string_view.remove_prefix(line_string_view.size());
|
||||
state = CsvParserState::DONE;
|
||||
} else {
|
||||
line_string_view.remove_prefix(delimiter_idx + read_config_.delimiter->size());
|
||||
state = CsvParserState::NEXT_FIELD;
|
||||
}
|
||||
state = CsvParserState::NEXT_FIELD;
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -159,15 +159,21 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case CsvParserState::DONE: {
|
||||
LOG_FATAL("Invalid state of the CSV parser!");
|
||||
}
|
||||
}
|
||||
}
|
||||
} while (state == CsvParserState::QUOTING);
|
||||
|
||||
switch (state) {
|
||||
case CsvParserState::INITIAL_FIELD:
|
||||
case CsvParserState::NEXT_FIELD:
|
||||
case CsvParserState::DONE:
|
||||
case CsvParserState::EXPECT_DELIMITER:
|
||||
break;
|
||||
case CsvParserState::NEXT_FIELD:
|
||||
row.emplace_back("");
|
||||
break;
|
||||
case CsvParserState::QUOTING: {
|
||||
return ParseError(ParseError::ErrorCode::NO_CLOSING_QUOTE,
|
||||
"There is no more data left to load while inside a quoted string. "
|
||||
|
||||
@@ -65,7 +65,7 @@ void Fatal(const char *msg, const Args &...msg_args) {
|
||||
do { \
|
||||
spdlog::critical(__VA_ARGS__); \
|
||||
std::terminate(); \
|
||||
} while (0);
|
||||
} while (0)
|
||||
|
||||
#ifndef NDEBUG
|
||||
#define DLOG_FATAL(...) LOG_FATAL(__VA_ARGS__)
|
||||
|
||||
@@ -23,6 +23,9 @@ size_t GrowMonotonicBuffer(size_t current_size, size_t max_size) {
|
||||
return std::ceil(next_size);
|
||||
}
|
||||
|
||||
__attribute__((no_sanitize("pointer-overflow"))) void CheckAllocationSizeOverflow(void *aligned_ptr, size_t bytes) {
|
||||
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
MonotonicBufferResource::MonotonicBufferResource(size_t initial_size) : initial_size_(initial_size) {}
|
||||
@@ -121,7 +124,7 @@ void *MonotonicBufferResource::DoAllocate(size_t bytes, size_t alignment) {
|
||||
next_buffer_size_ = GrowMonotonicBuffer(next_buffer_size_, std::numeric_limits<size_t>::max() - sizeof(Buffer));
|
||||
}
|
||||
if (reinterpret_cast<char *>(aligned_ptr) < buffer_head) throw BadAlloc("Allocation alignment overflow");
|
||||
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
|
||||
CheckAllocationSizeOverflow(aligned_ptr, bytes);
|
||||
allocated_ = reinterpret_cast<char *>(aligned_ptr) - data + bytes;
|
||||
return aligned_ptr;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#include <cstddef>
|
||||
#include <new>
|
||||
|
||||
#if USE_JEMALLOC
|
||||
@@ -10,7 +11,7 @@
|
||||
#include "utils/memory_tracker.hpp"
|
||||
|
||||
namespace {
|
||||
void *newImpl(std::size_t size) {
|
||||
void *newImpl(const std::size_t size) {
|
||||
auto *ptr = malloc(size);
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
return ptr;
|
||||
@@ -19,11 +20,26 @@ void *newImpl(std::size_t size) {
|
||||
throw std::bad_alloc{};
|
||||
}
|
||||
|
||||
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
|
||||
void *newImpl(const std::size_t size, const std::align_val_t align) {
|
||||
auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deleteImpl(void *ptr) noexcept { free(ptr); }
|
||||
throw std::bad_alloc{};
|
||||
}
|
||||
|
||||
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
|
||||
void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept {
|
||||
return aligned_alloc(size, static_cast<std::size_t>(align));
|
||||
}
|
||||
|
||||
#if USE_JEMALLOC
|
||||
void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
|
||||
|
||||
void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
|
||||
dallocx(ptr, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t size) noexcept {
|
||||
if (UNLIKELY(ptr == nullptr)) {
|
||||
@@ -33,24 +49,43 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
|
||||
sdallocx(ptr, size, 0);
|
||||
}
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
|
||||
if (UNLIKELY(ptr == nullptr)) {
|
||||
return;
|
||||
}
|
||||
|
||||
sdallocx(ptr, size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
|
||||
#else
|
||||
void deleteImpl(void *ptr) noexcept { free(ptr); }
|
||||
|
||||
void deleteImpl(void *ptr, const std::align_val_t /*unused*/) noexcept { free(ptr); }
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t /*unused*/) noexcept { free(ptr); }
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t /*unused*/) noexcept { free(ptr); }
|
||||
#endif
|
||||
|
||||
void TrackMemory(const size_t size) {
|
||||
size_t actual_size = size;
|
||||
|
||||
void TrackMemory(std::size_t size) {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(size != 0)) {
|
||||
actual_size = nallocx(size, 0);
|
||||
size = nallocx(size, 0);
|
||||
}
|
||||
#endif
|
||||
utils::total_memory_tracker.Alloc(actual_size);
|
||||
utils::total_memory_tracker.Alloc(size);
|
||||
}
|
||||
|
||||
bool TrackMemoryNoExcept(const size_t size) {
|
||||
void TrackMemory(std::size_t size, const std::align_val_t align) {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(size != 0)) {
|
||||
size = nallocx(size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
#endif
|
||||
utils::total_memory_tracker.Alloc(size);
|
||||
}
|
||||
|
||||
bool TrackMemoryNoExcept(const std::size_t size) {
|
||||
try {
|
||||
TrackMemory(size);
|
||||
} catch (...) {
|
||||
@@ -60,7 +95,17 @@ bool TrackMemoryNoExcept(const size_t size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0) noexcept {
|
||||
bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
|
||||
try {
|
||||
TrackMemory(size, align);
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
|
||||
try {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
@@ -78,32 +123,74 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0)
|
||||
}
|
||||
}
|
||||
|
||||
void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
|
||||
try {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align))); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
#else
|
||||
if (size) {
|
||||
utils::total_memory_tracker.Free(size);
|
||||
} else {
|
||||
// Innaccurate because malloc_usable_size() result is greater or equal to allocated size.
|
||||
utils::total_memory_tracker.Free(malloc_usable_size(ptr));
|
||||
}
|
||||
#endif
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void *operator new(std::size_t size) {
|
||||
void *operator new(const std::size_t size) {
|
||||
TrackMemory(size);
|
||||
return newImpl(size);
|
||||
}
|
||||
|
||||
void *operator new[](std::size_t size) {
|
||||
void *operator new[](const std::size_t size) {
|
||||
TrackMemory(size);
|
||||
return newImpl(size);
|
||||
}
|
||||
|
||||
void *operator new(std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
void *operator new(const std::size_t size, const std::align_val_t align) {
|
||||
TrackMemory(size, align);
|
||||
return newImpl(size, align);
|
||||
}
|
||||
|
||||
void *operator new[](const std::size_t size, const std::align_val_t align) {
|
||||
TrackMemory(size, align);
|
||||
return newImpl(size, align);
|
||||
}
|
||||
|
||||
void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size))) {
|
||||
return newNoExcept(size);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void *operator new[](std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size))) {
|
||||
return newNoExcept(size);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size, align))) {
|
||||
return newNoExcept(size, align);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size, align))) {
|
||||
return newNoExcept(size, align);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void operator delete(void *ptr) noexcept {
|
||||
UntrackMemory(ptr);
|
||||
deleteImpl(ptr);
|
||||
@@ -114,12 +201,52 @@ void operator delete[](void *ptr) noexcept {
|
||||
deleteImpl(ptr);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, std::size_t size) noexcept {
|
||||
void operator delete(void *ptr, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::size_t size) noexcept {
|
||||
UntrackMemory(ptr, size);
|
||||
deleteSized(ptr, size);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, std::size_t size) noexcept {
|
||||
void operator delete[](void *ptr, const std::size_t size) noexcept {
|
||||
UntrackMemory(ptr, size);
|
||||
deleteSized(ptr, size);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align, size);
|
||||
deleteSized(ptr, size, align);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align, size);
|
||||
deleteSized(ptr, size, align);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr);
|
||||
deleteImpl(ptr);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr);
|
||||
deleteImpl(ptr);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
@@ -38,19 +38,5 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
|
||||
spdlog::info("Memgraph is out of memory");
|
||||
|
||||
spdlog::info("Cleaning up unused memory");
|
||||
client->Execute("MATCH (n) DETACH DELETE n;");
|
||||
client->DiscardAll();
|
||||
client->Execute("FREE MEMORY;");
|
||||
client->DiscardAll();
|
||||
|
||||
// now it should succeed
|
||||
spdlog::info("Retrying the query with the memory cleaned up");
|
||||
client->Execute(create_query);
|
||||
if (!client->FetchOne()) {
|
||||
LOG_FATAL("Memgraph is still out of memory");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,8 @@ bolt_port: &bolt_port "7687"
|
||||
template_cluster: &template_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"]
|
||||
args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"]
|
||||
log_file: "memory-e2e.log"
|
||||
setup_queries: []
|
||||
validation_queries: []
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ int main(int argc, char **argv) {
|
||||
const auto label_name = (*data)[0][1].ValueString();
|
||||
const auto property_name = (*data)[0][2].ValueList()[0].ValueString();
|
||||
if (label_name != "Node" || property_name != "id") {
|
||||
LOG_FATAL("{} does NOT hava valid constraint created.", database_endpoint)
|
||||
LOG_FATAL("{} does NOT hava valid constraint created.", database_endpoint);
|
||||
}
|
||||
} else {
|
||||
LOG_FATAL("Unable to get CONSTRAINT INFO from {}", database_endpoint);
|
||||
|
||||
@@ -11,19 +11,23 @@ template_validation_queries: &template_validation_queries
|
||||
template_cluster: &template_cluster
|
||||
cluster:
|
||||
replica_1:
|
||||
args: ["--bolt-port", "7688"]
|
||||
args: ["--bolt-port", "7688", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-replica1.log"
|
||||
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
|
||||
<<: *template_validation_queries
|
||||
replica_2:
|
||||
args: ["--bolt-port", "7689"]
|
||||
args: ["--bolt-port", "7689", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-replica2.log"
|
||||
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
|
||||
<<: *template_validation_queries
|
||||
replica_3:
|
||||
args: ["--bolt-port", "7690"]
|
||||
args: ["--bolt-port", "7690", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-replica3.log"
|
||||
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10003;"]
|
||||
<<: *template_validation_queries
|
||||
main:
|
||||
args: ["--bolt-port", "7687"]
|
||||
args: ["--bolt-port", "7687", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-main.log"
|
||||
setup_queries: [
|
||||
"REGISTER REPLICA replica_1 SYNC WITH TIMEOUT 0 TO '127.0.0.1:10001'",
|
||||
"REGISTER REPLICA replica_2 SYNC WITH TIMEOUT 1 TO '127.0.0.1:10002'",
|
||||
|
||||
@@ -44,7 +44,9 @@ def run(args):
|
||||
for name, config in workload['cluster'].items():
|
||||
mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY)
|
||||
mg_instances[name] = mg_instance
|
||||
mg_instance.start(args=config['args'])
|
||||
log_file_path = os.path.join(BUILD_DIR, 'logs', config['log_file'])
|
||||
binary_args = config['args'] + ["--log-file", log_file_path]
|
||||
mg_instance.start(args=binary_args)
|
||||
for query in config['setup_queries']:
|
||||
mg_instance.query(query)
|
||||
# Test.
|
||||
|
||||
@@ -73,7 +73,7 @@ class Memgraph:
|
||||
virtualenv_bin = os.path.join(SCRIPT_DIR, "ve3", "bin", "python3")
|
||||
with open(script_file) as fin:
|
||||
data = fin.read()
|
||||
data = data.replace("/usr/bin/env python3", virtualenv_bin)
|
||||
data = data.replace("/usr/bin/python3", virtualenv_bin)
|
||||
data = data.replace("/etc/memgraph/auth/ldap.yaml",
|
||||
self._auth_config)
|
||||
with open(self._auth_module, "w") as fout:
|
||||
|
||||
@@ -45,7 +45,7 @@ def execute_test(**kwargs):
|
||||
server = None
|
||||
if start_server:
|
||||
server = subprocess.Popen(list(map(str, server_args)))
|
||||
time.sleep(0.1)
|
||||
time.sleep(0.4)
|
||||
assert server.poll() is None, "Server process died prematurely!"
|
||||
|
||||
try:
|
||||
|
||||
@@ -164,4 +164,4 @@
|
||||
{:bank (bank-checker)
|
||||
:timeline (timeline/html)})
|
||||
:generator (c/replication-gen (gen/mix [read-balances valid-transfer]))
|
||||
:final-generator (gen/once read-balances)})
|
||||
:final-generator {:gen (gen/once read-balances) :recovery-time 20}})
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
"A map of workload names to functions that can take opts and construct
|
||||
workloads."
|
||||
{:bank bank/workload
|
||||
:sequential sequential/workload
|
||||
;; :sequential sequential/workload (T0532-MG)
|
||||
:large large/workload})
|
||||
|
||||
(def nemesis-configuration
|
||||
@@ -45,8 +45,8 @@
|
||||
(gen/log "Healing cluster.")
|
||||
(gen/nemesis (:final-generator nemesis))
|
||||
(gen/log "Waiting for recovery")
|
||||
(gen/sleep 20)
|
||||
(gen/clients final-generator))
|
||||
(gen/sleep (:recovery-time final-generator))
|
||||
(gen/clients (:gen final-generator)))
|
||||
gen)]
|
||||
(merge tests/noop-test
|
||||
opts
|
||||
|
||||
@@ -103,4 +103,4 @@
|
||||
:timeline (timeline/html)})
|
||||
:generator (c/replication-gen
|
||||
(gen/mix [read-nodes add-nodes]))
|
||||
:final-generator (gen/once read-nodes)})
|
||||
:final-generator {:gen (gen/once read-nodes) :recovery-time 40}})
|
||||
|
||||
@@ -383,15 +383,16 @@ TEST(BoltSession, ExecuteRunWrongMarker) {
|
||||
}
|
||||
|
||||
TEST(BoltSession, ExecuteRunMissingData) {
|
||||
std::array<uint8_t, 6> run_req_without_parameters{
|
||||
run_req_header[0], run_req_header[1], run_req_header[2], 0x00, 0x00, 0x00};
|
||||
// test lengths, they test the following situations:
|
||||
// missing header data, missing query data, missing parameters
|
||||
int len[] = {1, 2, 37};
|
||||
|
||||
int len[] = {1, 2, run_req_without_parameters.size()};
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
INIT_VARS;
|
||||
ExecuteHandshake(input_stream, session, output);
|
||||
ExecuteInit(input_stream, session, output);
|
||||
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_header, len[i]), SessionException);
|
||||
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_without_parameters.data(), len[i]), SessionException);
|
||||
|
||||
ASSERT_EQ(session.state_, State::Close);
|
||||
CheckFailureMessage(output);
|
||||
@@ -871,7 +872,7 @@ TEST(BoltSession, Noop) {
|
||||
CheckFailureMessage(output);
|
||||
|
||||
session.state_ = State::Result;
|
||||
ExecuteCommand(input_stream, session, pullall_req, sizeof(v4::pullall_req));
|
||||
ExecuteCommand(input_stream, session, pullall_req, sizeof(pullall_req));
|
||||
CheckSuccessMessage(output);
|
||||
|
||||
ASSERT_THROW(ExecuteCommand(input_stream, session, v4_1::noop, sizeof(v4_1::noop)), SessionException);
|
||||
|
||||
@@ -73,6 +73,12 @@ class TestPlanner : public ::testing::Test {};
|
||||
|
||||
using PlannerTypes = ::testing::Types<Planner>;
|
||||
|
||||
void DeleteListContent(std::list<BaseOpChecker *> *list) {
|
||||
for (BaseOpChecker *ptr : *list) {
|
||||
delete ptr;
|
||||
}
|
||||
}
|
||||
|
||||
TYPED_TEST_CASE(TestPlanner, PlannerTypes);
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchNodeReturn) {
|
||||
@@ -223,6 +229,7 @@ TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) {
|
||||
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
|
||||
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()};
|
||||
CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchWhereReturn) {
|
||||
@@ -549,10 +556,8 @@ TYPED_TEST(TestPlanner, MatchMerge) {
|
||||
auto acc = ExpectAccumulate({symbol_table.at(*ident_n)});
|
||||
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
|
||||
CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce());
|
||||
for (auto &op : on_match) delete op;
|
||||
on_match.clear();
|
||||
for (auto &op : on_create) delete op;
|
||||
on_create.clear();
|
||||
DeleteListContent(&on_match);
|
||||
DeleteListContent(&on_create);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
|
||||
@@ -564,6 +569,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
|
||||
WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r")));
|
||||
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchUnwindReturn) {
|
||||
@@ -705,6 +711,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) {
|
||||
// optional ScanAll.
|
||||
std::list<BaseOpChecker *> optional{new ExpectFilter(), new ExpectScanAll()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchReturnAsterisk) {
|
||||
@@ -763,8 +770,8 @@ TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) {
|
||||
std::list<BaseOpChecker *> on_match{new ExpectScanAll(), new ExpectFilter()};
|
||||
std::list<BaseOpChecker *> on_create{new ExpectCreateNode()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create));
|
||||
for (auto &op : on_match) delete op;
|
||||
for (auto &op : on_create) delete op;
|
||||
DeleteListContent(&on_match);
|
||||
DeleteListContent(&on_create);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
|
||||
@@ -774,6 +781,7 @@ TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
|
||||
QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n")));
|
||||
std::list<BaseOpChecker *> optional{new ExpectScanAll()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, FunctionAggregationReturn) {
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
|
||||
#include "test_utils.hpp"
|
||||
|
||||
static void DummyCallback(const mgp_list *, const mgp_graph *, mgp_result *, mgp_memory *) {}
|
||||
|
||||
TEST(Module, InvalidProcedureRegistration) {
|
||||
@@ -53,7 +55,8 @@ TEST(Module, ProcedureSignature) {
|
||||
CheckSignature(proc, "proc() :: ()");
|
||||
mgp_proc_add_arg(proc, "arg1", mgp_type_number());
|
||||
CheckSignature(proc, "proc(arg1 :: NUMBER) :: ()");
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
|
||||
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
|
||||
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: ()");
|
||||
mgp_proc_add_result(proc, "res1", mgp_type_list(mgp_type_int()));
|
||||
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: (res1 :: LIST OF INTEGER)");
|
||||
@@ -69,7 +72,8 @@ TEST(Module, ProcedureSignature) {
|
||||
"(res1 :: LIST OF INTEGER, DEPRECATED res2 :: STRING)");
|
||||
EXPECT_FALSE(mgp_proc_add_result(proc, "res2", mgp_type_any()));
|
||||
EXPECT_FALSE(mgp_proc_add_deprecated_result(proc, "res1", mgp_type_any()));
|
||||
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(), mgp_value_make_string("string=\"value\"", &memory));
|
||||
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(),
|
||||
test_utils::CreateValueOwningPtr(mgp_value_make_string("string=\"value\"", &memory)).get());
|
||||
CheckSignature(proc,
|
||||
"proc(arg1 :: NUMBER, opt1 = Null :: ANY?, "
|
||||
"opt2 = \"string=\\\"value\\\"\" :: STRING) :: "
|
||||
@@ -80,6 +84,7 @@ TEST(Module, ProcedureSignatureOnlyOptArg) {
|
||||
mgp_memory memory{utils::NewDeleteResource()};
|
||||
mgp_module module(utils::NewDeleteResource());
|
||||
auto *proc = mgp_module_add_read_procedure(&module, "proc", DummyCallback);
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
|
||||
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
|
||||
CheckSignature(proc, "proc(opt1 = Null :: ANY?) :: ()");
|
||||
}
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
|
||||
#include "test_utils.hpp"
|
||||
|
||||
TEST(CypherType, PresentableNameSimpleTypes) {
|
||||
EXPECT_EQ(mgp_type_any()->impl->GetPresentableName(), "ANY");
|
||||
EXPECT_EQ(mgp_type_bool()->impl->GetPresentableName(), "BOOLEAN");
|
||||
@@ -66,6 +72,7 @@ TEST(CypherType, NullSatisfiesType) {
|
||||
EXPECT_TRUE(null_type->impl->SatisfiesType(tv_null));
|
||||
}
|
||||
}
|
||||
mgp_value_destroy(mgp_null);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,6 +108,7 @@ TEST(CypherType, BoolSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_bool, tv_bool,
|
||||
{mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
|
||||
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_bool);
|
||||
}
|
||||
|
||||
TEST(CypherType, IntSatisfiesType) {
|
||||
@@ -111,6 +119,7 @@ TEST(CypherType, IntSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_int, tv_int,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_int);
|
||||
}
|
||||
|
||||
TEST(CypherType, DoubleSatisfiesType) {
|
||||
@@ -121,6 +130,7 @@ TEST(CypherType, DoubleSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_double, tv_double,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_double);
|
||||
}
|
||||
|
||||
TEST(CypherType, StringSatisfiesType) {
|
||||
@@ -131,12 +141,13 @@ TEST(CypherType, StringSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_string, tv_string,
|
||||
{mgp_type_bool(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
|
||||
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_string);
|
||||
}
|
||||
|
||||
TEST(CypherType, MapSatisfiesType) {
|
||||
mgp_memory memory{utils::NewDeleteResource()};
|
||||
auto *map = mgp_map_make_empty(&memory);
|
||||
mgp_map_insert(map, "key", mgp_value_make_int(42, &memory));
|
||||
mgp_map_insert(map, "key", test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get());
|
||||
auto *mgp_map_v = mgp_value_make_map(map);
|
||||
const query::TypedValue tv_map(std::map<std::string, query::TypedValue>{{"key", query::TypedValue(42)}});
|
||||
CheckSatisfiesTypesAndNullable(mgp_map_v, tv_map, {mgp_type_any(), mgp_type_map()});
|
||||
@@ -144,6 +155,7 @@ TEST(CypherType, MapSatisfiesType) {
|
||||
mgp_map_v, tv_map,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_node(),
|
||||
mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_map_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, VertexSatisfiesType) {
|
||||
@@ -160,6 +172,7 @@ TEST(CypherType, VertexSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_vertex_v, tv_vertex,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
|
||||
mgp_type_number(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_vertex_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, EdgeSatisfiesType) {
|
||||
@@ -178,6 +191,7 @@ TEST(CypherType, EdgeSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_edge_v, tv_edge,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
|
||||
mgp_type_number(), mgp_type_node(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_edge_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, PathSatisfiesType) {
|
||||
@@ -190,9 +204,13 @@ TEST(CypherType, PathSatisfiesType) {
|
||||
mgp_memory memory{utils::NewDeleteResource()};
|
||||
utils::Allocator<mgp_path> alloc(memory.impl);
|
||||
mgp_graph graph{&dba, storage::View::NEW};
|
||||
auto *path = mgp_path_make_with_start(alloc.new_object<mgp_vertex>(v1, &graph), &memory);
|
||||
auto *mgp_vertex_v = alloc.new_object<mgp_vertex>(v1, &graph);
|
||||
auto path = mgp_path_make_with_start(mgp_vertex_v, &memory);
|
||||
ASSERT_TRUE(path);
|
||||
ASSERT_TRUE(mgp_path_expand(path, alloc.new_object<mgp_edge>(edge, &graph)));
|
||||
alloc.delete_object(mgp_vertex_v);
|
||||
auto mgp_edge_v = alloc.new_object<mgp_edge>(edge, &graph);
|
||||
ASSERT_TRUE(mgp_path_expand(path, mgp_edge_v));
|
||||
alloc.delete_object(mgp_edge_v);
|
||||
auto *mgp_path_v = mgp_value_make_path(path);
|
||||
const query::TypedValue tv_path(query::Path(v1, edge, v2));
|
||||
CheckSatisfiesTypesAndNullable(mgp_path_v, tv_path, {mgp_type_any(), mgp_type_path()});
|
||||
@@ -200,6 +218,7 @@ TEST(CypherType, PathSatisfiesType) {
|
||||
mgp_path_v, tv_path,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship()});
|
||||
mgp_value_destroy(mgp_path_v);
|
||||
}
|
||||
|
||||
static std::vector<const mgp_type *> MakeListTypes(const std::vector<const mgp_type *> &element_types) {
|
||||
@@ -224,6 +243,7 @@ TEST(CypherType, EmptyListSatisfiesType) {
|
||||
auto all_types = MakeListTypes(primitive_types);
|
||||
all_types.push_back(mgp_type_any());
|
||||
CheckSatisfiesTypesAndNullable(mgp_list_v, tv_list, all_types);
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, ListOfIntSatisfiesType) {
|
||||
@@ -233,7 +253,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
|
||||
auto *mgp_list_v = mgp_value_make_list(list);
|
||||
query::TypedValue tv_list(std::vector<query::TypedValue>{});
|
||||
for (int64_t i = 0; i < elem_count; ++i) {
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(i, &memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(i, &memory)).get()));
|
||||
tv_list.ValueList().emplace_back(i);
|
||||
auto valid_types = MakeListTypes({mgp_type_any(), mgp_type_int(), mgp_type_number()});
|
||||
valid_types.push_back(mgp_type_any());
|
||||
@@ -242,6 +262,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
}
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
|
||||
@@ -251,10 +272,10 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
|
||||
auto *mgp_list_v = mgp_value_make_list(list);
|
||||
query::TypedValue tv_list(std::vector<query::TypedValue>{});
|
||||
// Add an int
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(42, &memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get()));
|
||||
tv_list.ValueList().emplace_back(42);
|
||||
// Add a boolean
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_bool(1, &memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_bool(1, &memory)).get()));
|
||||
tv_list.ValueList().emplace_back(true);
|
||||
auto valid_types = MakeListTypes({mgp_type_any()});
|
||||
valid_types.push_back(mgp_type_any());
|
||||
@@ -264,6 +285,7 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
|
||||
mgp_list_v, tv_list,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, ListOfNullSatisfiesType) {
|
||||
@@ -271,7 +293,7 @@ TEST(CypherType, ListOfNullSatisfiesType) {
|
||||
auto *list = mgp_list_make_empty(1, &memory);
|
||||
auto *mgp_list_v = mgp_value_make_list(list);
|
||||
query::TypedValue tv_list(std::vector<query::TypedValue>{});
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_null(&memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get()));
|
||||
tv_list.ValueList().emplace_back();
|
||||
// List with Null satisfies all nullable list element types
|
||||
std::vector<const mgp_type *> primitive_types{
|
||||
@@ -295,4 +317,5 @@ TEST(CypherType, ListOfNullSatisfiesType) {
|
||||
EXPECT_FALSE(null_type->impl->SatisfiesType(*mgp_list_v)) << null_type->impl->GetPresentableName();
|
||||
EXPECT_FALSE(null_type->impl->SatisfiesType(tv_list));
|
||||
}
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
@@ -254,6 +254,7 @@ TEST(PyModule, PyObjectToMgpValue) {
|
||||
const mgp_value *v2 = mgp_map_at(map, "four");
|
||||
ASSERT_TRUE(mgp_value_is_double(v2));
|
||||
EXPECT_EQ(mgp_value_get_double(v2), 4.0);
|
||||
mgp_value_destroy(value);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
|
||||
@@ -709,14 +709,15 @@ TEST_P(DurabilityTest, SnapshotFallback) {
|
||||
{.items = {.properties_on_edges = GetParam()},
|
||||
.durability = {.storage_directory = storage_directory,
|
||||
.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT,
|
||||
.snapshot_interval = std::chrono::milliseconds(2000)}});
|
||||
.snapshot_interval = std::chrono::milliseconds(3000)}});
|
||||
CreateBaseDataset(&store, GetParam());
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(3500));
|
||||
ASSERT_EQ(GetSnapshotsList().size(), 1);
|
||||
CreateExtendedDataset(&store);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(3000));
|
||||
}
|
||||
|
||||
ASSERT_GE(GetSnapshotsList().size(), 2);
|
||||
ASSERT_EQ(GetSnapshotsList().size(), 2);
|
||||
ASSERT_EQ(GetBackupSnapshotsList().size(), 0);
|
||||
ASSERT_EQ(GetWalsList().size(), 0);
|
||||
ASSERT_EQ(GetBackupWalsList().size(), 0);
|
||||
@@ -724,7 +725,7 @@ TEST_P(DurabilityTest, SnapshotFallback) {
|
||||
// Destroy last snapshot.
|
||||
{
|
||||
auto snapshots = GetSnapshotsList();
|
||||
ASSERT_GE(snapshots.size(), 2);
|
||||
ASSERT_EQ(snapshots.size(), 2);
|
||||
DestroySnapshot(*snapshots.begin());
|
||||
}
|
||||
|
||||
|
||||
9
tests/unit/test_utils.hpp
Normal file
9
tests/unit/test_utils.hpp
Normal file
@@ -0,0 +1,9 @@
|
||||
#include <memory>
|
||||
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
|
||||
namespace test_utils {
|
||||
using MgpValueOwningPtr = std::unique_ptr<mgp_value, void (*)(mgp_value *)>;
|
||||
|
||||
MgpValueOwningPtr CreateValueOwningPtr(mgp_value *value) { return MgpValueOwningPtr(value, &mgp_value_destroy); }
|
||||
} // namespace test_utils
|
||||
@@ -397,8 +397,8 @@ TEST_F(TypedValueLogicTest, LogicalXor) {
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
TEST_F(AllTypesFixture, ConstructionWithMemoryResource) {
|
||||
std::vector<TypedValue> values_with_custom_memory;
|
||||
utils::MonotonicBufferResource monotonic_memory(1024);
|
||||
std::vector<TypedValue> values_with_custom_memory;
|
||||
for (const auto &value : values_) {
|
||||
EXPECT_EQ(value.GetMemoryResource(), utils::NewDeleteResource());
|
||||
TypedValue copy_constructed_value(value, &monotonic_memory);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#include "utils/string.hpp"
|
||||
|
||||
class CsvReaderTest : public ::testing::Test {
|
||||
class CsvReaderTest : public ::testing::TestWithParam<const char *> {
|
||||
protected:
|
||||
const std::filesystem::path csv_directory{std::filesystem::temp_directory_path() / "csv_testing"};
|
||||
|
||||
@@ -30,7 +30,9 @@ class CsvReaderTest : public ::testing::Test {
|
||||
namespace {
|
||||
class FileWriter {
|
||||
public:
|
||||
explicit FileWriter(const std::filesystem::path path) { stream_.open(path); }
|
||||
explicit FileWriter(const std::filesystem::path path, std::string newline = "\n") : newline_{std::move(newline)} {
|
||||
stream_.open(path);
|
||||
}
|
||||
|
||||
FileWriter(const FileWriter &) = delete;
|
||||
FileWriter &operator=(const FileWriter &) = delete;
|
||||
@@ -45,7 +47,7 @@ class FileWriter {
|
||||
return 0;
|
||||
}
|
||||
|
||||
stream_ << line << std::endl;
|
||||
stream_ << line << newline_;
|
||||
|
||||
// including the newline character
|
||||
return line.size() + 1;
|
||||
@@ -53,6 +55,7 @@ class FileWriter {
|
||||
|
||||
private:
|
||||
std::ofstream stream_;
|
||||
std::string newline_;
|
||||
};
|
||||
|
||||
std::string CreateRow(const std::vector<std::string> &columns, const std::string_view delim) {
|
||||
@@ -69,10 +72,10 @@ auto ToPmrColumns(const std::vector<std::string> &columns) {
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_F(CsvReaderTest, CommaDelimiter) {
|
||||
TEST_P(CsvReaderTest, CommaDelimiter) {
|
||||
// create a file with a single valid row;
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
const std::vector<std::string> columns{"A", "B", "C"};
|
||||
writer.WriteLine(CreateRow(columns, ","));
|
||||
@@ -93,9 +96,9 @@ TEST_F(CsvReaderTest, CommaDelimiter) {
|
||||
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, SemicolonDelimiter) {
|
||||
TEST_P(CsvReaderTest, SemicolonDelimiter) {
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -116,12 +119,12 @@ TEST_F(CsvReaderTest, SemicolonDelimiter) {
|
||||
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, SkipBad) {
|
||||
TEST_P(CsvReaderTest, SkipBad) {
|
||||
// create a file with invalid first two rows (containing a string with a
|
||||
// missing closing quote);
|
||||
// the last row is valid;
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -161,11 +164,11 @@ TEST_F(CsvReaderTest, SkipBad) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, AllRowsValid) {
|
||||
TEST_P(CsvReaderTest, AllRowsValid) {
|
||||
// create a file with all rows valid;
|
||||
// parser should return 'std::nullopt'
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -190,11 +193,11 @@ TEST_F(CsvReaderTest, AllRowsValid) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, SkipAllRows) {
|
||||
TEST_P(CsvReaderTest, SkipAllRows) {
|
||||
// create a file with all rows invalid (containing a string with a missing closing quote);
|
||||
// parser should return 'std::nullopt'
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -217,9 +220,9 @@ TEST_F(CsvReaderTest, SkipAllRows) {
|
||||
ASSERT_EQ(parsed_row, std::nullopt);
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, WithHeader) {
|
||||
TEST_P(CsvReaderTest, WithHeader) {
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -249,12 +252,12 @@ TEST_F(CsvReaderTest, WithHeader) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, MultilineQuotedString) {
|
||||
TEST_P(CsvReaderTest, MultilineQuotedString) {
|
||||
// create a file with first row valid and the second row containing a quoted
|
||||
// string spanning two lines;
|
||||
// parser should return two valid rows
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -283,3 +286,37 @@ TEST_F(CsvReaderTest, MultilineQuotedString) {
|
||||
parsed_row = reader.GetNextRow(mem);
|
||||
ASSERT_EQ(*parsed_row, ToPmrColumns(expected_multiline));
|
||||
}
|
||||
|
||||
TEST_P(CsvReaderTest, EmptyColumns) {
|
||||
// create a file with all rows valid;
|
||||
// parser should return 'std::nullopt'
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
const utils::pmr::string delimiter{",", mem};
|
||||
const utils::pmr::string quote{"\"", mem};
|
||||
|
||||
std::vector<std::vector<std::string>> expected_rows{{"", "B", "C"}, {"A", "", "C"}, {"A", "B", ""}};
|
||||
|
||||
for (const auto &row : expected_rows) {
|
||||
writer.WriteLine(CreateRow(row, delimiter));
|
||||
}
|
||||
|
||||
writer.Close();
|
||||
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = false;
|
||||
const csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = csv::Reader(filepath, cfg);
|
||||
|
||||
for (const auto &expected_row : expected_rows) {
|
||||
const auto pmr_expected_row = ToPmrColumns(expected_row);
|
||||
const auto parsed_row = reader.GetNextRow(mem);
|
||||
ASSERT_TRUE(parsed_row.has_value());
|
||||
ASSERT_EQ(*parsed_row, pmr_expected_row);
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n"));
|
||||
|
||||
@@ -12,6 +12,7 @@ class TestMemory final : public utils::MemoryResource {
|
||||
size_t delete_count_{0};
|
||||
|
||||
private:
|
||||
static constexpr size_t kPadSize = 32;
|
||||
void *DoAllocate(size_t bytes, size_t alignment) override {
|
||||
new_count_++;
|
||||
EXPECT_TRUE(alignment != 0U && (alignment & (alignment - 1U)) == 0U) << "Alignment must be power of 2";
|
||||
@@ -20,11 +21,11 @@ class TestMemory final : public utils::MemoryResource {
|
||||
EXPECT_TRUE(bytes + pad_size > bytes) << "TestMemory size overflow";
|
||||
EXPECT_TRUE(bytes + pad_size + alignment > bytes + alignment) << "TestMemory size overflow";
|
||||
EXPECT_TRUE(2U * alignment > alignment) << "TestMemory alignment overflow";
|
||||
// Allocate a block containing extra alignment and pad_size bytes, but
|
||||
// Allocate a block containing extra alignment and kPadSize bytes, but
|
||||
// aligned to 2 * alignment. Then we can offset the ptr so that it's never
|
||||
// aligned to 2 * alignment. This ought to make allocator alignment issues
|
||||
// more obvious.
|
||||
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + pad_size, 2U * alignment);
|
||||
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + kPadSize, 2U * alignment);
|
||||
// Clear allocated memory to 0xFF, marking the invalid region.
|
||||
memset(ptr, 0xFF, alignment + bytes + pad_size);
|
||||
// Offset the ptr so it's not aligned to 2 * alignment, but still aligned to
|
||||
@@ -39,7 +40,8 @@ class TestMemory final : public utils::MemoryResource {
|
||||
void DoDeallocate(void *ptr, size_t bytes, size_t alignment) override {
|
||||
delete_count_++;
|
||||
// Deallocate the original ptr, before alignment adjustment.
|
||||
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, bytes, alignment);
|
||||
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, alignment + bytes + kPadSize,
|
||||
2U * alignment);
|
||||
}
|
||||
|
||||
bool DoIsEqual(const utils::MemoryResource &other) const noexcept override { return this == &other; }
|
||||
|
||||
68
tools/bench-graph-client/main.py
Executable file
68
tools/bench-graph-client/main.py
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Bench Graph client responsible for sending benchmarking data in JSON format to
|
||||
the Bench Graph server.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from argparse import ArgumentParser
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "")
|
||||
GITHUB_SHA = os.getenv("GITHUB_SHA", "")
|
||||
GITHUB_REF = os.getenv("GITHUB_REF", "")
|
||||
|
||||
BENCH_GRAPH_SERVER_ENDPOINT = os.getenv(
|
||||
"BENCH_GRAPH_SERVER_ENDPOINT",
|
||||
"http://bench-graph-api:9001")
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_args():
|
||||
argp = ArgumentParser(description=__doc__)
|
||||
argp.add_argument("--benchmark-name", type=str, required=True)
|
||||
argp.add_argument("--benchmark-results-path", type=str, required=True)
|
||||
argp.add_argument("--github-run-id", type=int, required=True)
|
||||
argp.add_argument("--github-run-number", type=int, required=True)
|
||||
return argp.parse_args()
|
||||
|
||||
|
||||
def post_measurement(args):
|
||||
with open(args.benchmark_results_path, "r") as f:
|
||||
data = json.load(f)
|
||||
timestamp = datetime.now().timestamp()
|
||||
branch = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
stdout=subprocess.PIPE,
|
||||
check=True).stdout.decode("utf-8").strip()
|
||||
req = requests.post(
|
||||
f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements",
|
||||
json={
|
||||
"name": args.benchmark_name,
|
||||
"timestamp": timestamp,
|
||||
"git_repo": GITHUB_REPOSITORY,
|
||||
"git_ref": GITHUB_REF,
|
||||
"git_sha": GITHUB_SHA,
|
||||
"github_run_id": args.github_run_id,
|
||||
"github_run_number": args.github_run_number,
|
||||
"results": data,
|
||||
"git_branch": branch},
|
||||
timeout=1)
|
||||
assert req.status_code == 200, \
|
||||
f"Uploading {args.benchmark_name} data failed."
|
||||
log.info(f"{args.benchmark_name} data sent to "
|
||||
f"{BENCH_GRAPH_SERVER_ENDPOINT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
post_measurement(args)
|
||||
1
tools/bench-graph-client/requirements.txt
Normal file
1
tools/bench-graph-client/requirements.txt
Normal file
@@ -0,0 +1 @@
|
||||
requests==2.25.1
|
||||
269
tools/github/clang-tidy/clang-tidy-diff.py
Executable file
269
tools/github/clang-tidy/clang-tidy-diff.py
Executable file
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
#===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
|
||||
#
|
||||
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
# See https://llvm.org/LICENSE.txt for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
#
|
||||
#===-----------------------------------------------------------------------===#
|
||||
|
||||
r"""
|
||||
ClangTidy Diff Checker
|
||||
======================
|
||||
|
||||
This script reads input from a unified diff, runs clang-tidy on all changed
|
||||
files and outputs clang-tidy warnings in changed lines only. This is useful to
|
||||
detect clang-tidy regressions in the lines touched by a specific patch.
|
||||
Example usage for git/svn users:
|
||||
|
||||
git diff -U0 HEAD^ | clang-tidy-diff.py -p1
|
||||
svn diff --diff-cmd=diff -x-U0 | \
|
||||
clang-tidy-diff.py -fix -checks=-*,modernize-use-override
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
is_py2 = sys.version[0] == '2'
|
||||
|
||||
if is_py2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue as queue
|
||||
|
||||
|
||||
def run_tidy(task_queue, lock, timeout):
|
||||
watchdog = None
|
||||
while True:
|
||||
command = task_queue.get()
|
||||
try:
|
||||
proc = subprocess.Popen(command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
|
||||
if timeout is not None:
|
||||
watchdog = threading.Timer(timeout, proc.kill)
|
||||
watchdog.start()
|
||||
|
||||
stdout, stderr = proc.communicate()
|
||||
|
||||
with lock:
|
||||
sys.stdout.write(stdout.decode('utf-8') + '\n')
|
||||
sys.stdout.flush()
|
||||
if stderr:
|
||||
sys.stderr.write(stderr.decode('utf-8') + '\n')
|
||||
sys.stderr.flush()
|
||||
except Exception as e:
|
||||
with lock:
|
||||
sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
|
||||
finally:
|
||||
with lock:
|
||||
if not (timeout is None or watchdog is None):
|
||||
if not watchdog.is_alive():
|
||||
sys.stderr.write('Terminated by timeout: ' +
|
||||
' '.join(command) + '\n')
|
||||
watchdog.cancel()
|
||||
task_queue.task_done()
|
||||
|
||||
|
||||
def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
|
||||
for _ in range(max_tasks):
|
||||
t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
|
||||
def merge_replacement_files(tmpdir, mergefile):
|
||||
"""Merge all replacement files in a directory into a single file"""
|
||||
# The fixes suggested by clang-tidy >= 4.0.0 are given under
|
||||
# the top level key 'Diagnostics' in the output yaml files
|
||||
mergekey = "Diagnostics"
|
||||
merged = []
|
||||
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
|
||||
content = yaml.safe_load(open(replacefile, 'r'))
|
||||
if not content:
|
||||
continue # Skip empty files.
|
||||
merged.extend(content.get(mergekey, []))
|
||||
|
||||
if merged:
|
||||
# MainSourceFile: The key is required by the definition inside
|
||||
# include/clang/Tooling/ReplacementsYaml.h, but the value
|
||||
# is actually never used inside clang-apply-replacements,
|
||||
# so we set it to '' here.
|
||||
output = {'MainSourceFile': '', mergekey: merged}
|
||||
with open(mergefile, 'w') as out:
|
||||
yaml.safe_dump(output, out)
|
||||
else:
|
||||
# Empty the file:
|
||||
open(mergefile, 'w').close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=
|
||||
'Run clang-tidy against changed files, and '
|
||||
'output diagnostics only for modified '
|
||||
'lines.')
|
||||
parser.add_argument('-clang-tidy-binary', metavar='PATH',
|
||||
default='clang-tidy',
|
||||
help='path to clang-tidy binary')
|
||||
parser.add_argument('-p', metavar='NUM', default=0,
|
||||
help='strip the smallest prefix containing P slashes')
|
||||
parser.add_argument('-regex', metavar='PATTERN', default=None,
|
||||
help='custom pattern selecting file paths to check '
|
||||
'(case sensitive, overrides -iregex)')
|
||||
parser.add_argument('-iregex', metavar='PATTERN', default=
|
||||
r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
|
||||
help='custom pattern selecting file paths to check '
|
||||
'(case insensitive, overridden by -regex)')
|
||||
parser.add_argument('-j', type=int, default=1,
|
||||
help='number of tidy instances to be run in parallel.')
|
||||
parser.add_argument('-timeout', type=int, default=None,
|
||||
help='timeout per each file in seconds.')
|
||||
parser.add_argument('-fix', action='store_true', default=False,
|
||||
help='apply suggested fixes')
|
||||
parser.add_argument('-checks',
|
||||
help='checks filter, when not specified, use clang-tidy '
|
||||
'default',
|
||||
default='')
|
||||
parser.add_argument('-path', dest='build_path',
|
||||
help='Path used to read a compile command database.')
|
||||
if yaml:
|
||||
parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
|
||||
help='Create a yaml file to store suggested fixes in, '
|
||||
'which can be applied with clang-apply-replacements.')
|
||||
parser.add_argument('-extra-arg', dest='extra_arg',
|
||||
action='append', default=[],
|
||||
help='Additional argument to append to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
|
||||
action='append', default=[],
|
||||
help='Additional argument to prepend to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-quiet', action='store_true', default=False,
|
||||
help='Run clang-tidy in quiet mode')
|
||||
clang_tidy_args = []
|
||||
argv = sys.argv[1:]
|
||||
if '--' in argv:
|
||||
clang_tidy_args.extend(argv[argv.index('--'):])
|
||||
argv = argv[:argv.index('--')]
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# Extract changed lines for each file.
|
||||
filename = None
|
||||
lines_by_file = {}
|
||||
for line in sys.stdin:
|
||||
match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
|
||||
if match:
|
||||
filename = match.group(2)
|
||||
if filename is None:
|
||||
continue
|
||||
|
||||
if args.regex is not None:
|
||||
if not re.match('^%s$' % args.regex, filename):
|
||||
continue
|
||||
else:
|
||||
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
|
||||
continue
|
||||
|
||||
match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
|
||||
if match:
|
||||
start_line = int(match.group(1))
|
||||
line_count = 1
|
||||
if match.group(3):
|
||||
line_count = int(match.group(3))
|
||||
if line_count == 0:
|
||||
continue
|
||||
end_line = start_line + line_count - 1
|
||||
lines_by_file.setdefault(filename, []).append([start_line, end_line])
|
||||
|
||||
if not any(lines_by_file):
|
||||
print("No relevant changes found.")
|
||||
sys.exit(0)
|
||||
|
||||
max_task_count = args.j
|
||||
if max_task_count == 0:
|
||||
max_task_count = multiprocessing.cpu_count()
|
||||
max_task_count = min(len(lines_by_file), max_task_count)
|
||||
|
||||
tmpdir = None
|
||||
if yaml and args.export_fixes:
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
# Tasks for clang-tidy.
|
||||
task_queue = queue.Queue(max_task_count)
|
||||
# A lock for console output.
|
||||
lock = threading.Lock()
|
||||
|
||||
# Run a pool of clang-tidy workers.
|
||||
start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
|
||||
|
||||
# Form the common args list.
|
||||
common_clang_tidy_args = []
|
||||
if args.fix:
|
||||
common_clang_tidy_args.append('-fix')
|
||||
if args.checks != '':
|
||||
common_clang_tidy_args.append('-checks=' + args.checks)
|
||||
if args.quiet:
|
||||
common_clang_tidy_args.append('-quiet')
|
||||
if args.build_path is not None:
|
||||
common_clang_tidy_args.append('-p=%s' % args.build_path)
|
||||
for arg in args.extra_arg:
|
||||
common_clang_tidy_args.append('-extra-arg=%s' % arg)
|
||||
for arg in args.extra_arg_before:
|
||||
common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
|
||||
|
||||
for name in lines_by_file:
|
||||
line_filter_json = json.dumps(
|
||||
[{"name": name, "lines": lines_by_file[name]}],
|
||||
separators=(',', ':'))
|
||||
|
||||
# Run clang-tidy on files containing changes.
|
||||
command = [args.clang_tidy_binary]
|
||||
command.append('-line-filter=' + line_filter_json)
|
||||
if yaml and args.export_fixes:
|
||||
# Get a temporary file. We immediately close the handle so clang-tidy can
|
||||
# overwrite it.
|
||||
(handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
|
||||
os.close(handle)
|
||||
command.append('-export-fixes=' + tmp_name)
|
||||
command.extend(common_clang_tidy_args)
|
||||
command.append(name)
|
||||
command.extend(clang_tidy_args)
|
||||
|
||||
task_queue.put(command)
|
||||
|
||||
# Wait for all threads to be done.
|
||||
task_queue.join()
|
||||
|
||||
if yaml and args.export_fixes:
|
||||
print('Writing fixes to ' + args.export_fixes + ' ...')
|
||||
try:
|
||||
merge_replacement_files(tmpdir, args.export_fixes)
|
||||
except:
|
||||
sys.stderr.write('Error exporting fixes.\n')
|
||||
traceback.print_exc()
|
||||
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
9
tools/github/clang-tidy/count_errors.sh
Executable file
9
tools/github/clang-tidy/count_errors.sh
Executable file
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
|
||||
# the first sort | uniq is necessary, because the same occurrence of the same error
|
||||
# can be reported from headers when they are included in multiple source files
|
||||
`dirname ${BASH_SOURCE[0]}`/grep_error_lines.sh |
|
||||
sort | uniq |
|
||||
sed -E 's/.*\[(.*)\]\r?$/\1/g' | # extract the check name from [check-name]
|
||||
sort | uniq -c | # count each type of check
|
||||
sort -nr # sort them into descending order
|
||||
12
tools/github/clang-tidy/grep_error_lines.sh
Executable file
12
tools/github/clang-tidy/grep_error_lines.sh
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Matches timestamp like "2021-03-25T17:06:42.2621697Z"
|
||||
TIMESTAMP_PATTERN="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z"
|
||||
|
||||
# Matches absolute file pathes with line and column identifier like
|
||||
# "/opt/actions-runner/_work/memgraph/memgraph/src/utils/exceptions.hpp:71:11:"
|
||||
FILE_ABSOLUTE_PATH_PATTERN="/[^:]+:\d+:\d+:"
|
||||
|
||||
ERROR_OR_WARNING_PATTERN="(error|warning):"
|
||||
|
||||
grep -P "^($TIMESTAMP_PATTERN )?$FILE_ABSOLUTE_PATH_PATTERN $ERROR_OR_WARNING_PATTERN.*$"
|
||||
337
tools/github/clang-tidy/run-clang-tidy.py
Executable file
337
tools/github/clang-tidy/run-clang-tidy.py
Executable file
@@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
#===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
|
||||
#
|
||||
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
# See https://llvm.org/LICENSE.txt for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
#
|
||||
#===-----------------------------------------------------------------------===#
|
||||
# FIXME: Integrate with clang-tidy-diff.py
|
||||
|
||||
|
||||
"""
|
||||
Parallel clang-tidy runner
|
||||
==========================
|
||||
|
||||
Runs clang-tidy over all files in a compilation database. Requires clang-tidy
|
||||
and clang-apply-replacements in $PATH.
|
||||
|
||||
Example invocations.
|
||||
- Run clang-tidy on all files in the current working directory with a default
|
||||
set of checks and show warnings in the cpp files and all project headers.
|
||||
run-clang-tidy.py $PWD
|
||||
|
||||
- Fix all header guards.
|
||||
run-clang-tidy.py -fix -checks=-*,llvm-header-guard
|
||||
|
||||
- Fix all header guards included from clang-tidy and header guards
|
||||
for clang-tidy headers.
|
||||
run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
|
||||
-header-filter=extra/clang-tidy
|
||||
|
||||
Compilation database setup:
|
||||
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
is_py2 = sys.version[0] == '2'
|
||||
|
||||
if is_py2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue as queue
|
||||
|
||||
|
||||
def find_compilation_database(path):
|
||||
"""Adjusts the directory until a compilation database is found."""
|
||||
result = './'
|
||||
while not os.path.isfile(os.path.join(result, path)):
|
||||
if os.path.realpath(result) == '/':
|
||||
print('Error: could not find compilation database.')
|
||||
sys.exit(1)
|
||||
result += '../'
|
||||
return os.path.realpath(result)
|
||||
|
||||
|
||||
def make_absolute(f, directory):
|
||||
if os.path.isabs(f):
|
||||
return f
|
||||
return os.path.normpath(os.path.join(directory, f))
|
||||
|
||||
|
||||
def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
|
||||
header_filter, allow_enabling_alpha_checkers,
|
||||
extra_arg, extra_arg_before, quiet, config):
|
||||
"""Gets a command line for clang-tidy."""
|
||||
start = [clang_tidy_binary]
|
||||
if allow_enabling_alpha_checkers:
|
||||
start.append('-allow-enabling-analyzer-alpha-checkers')
|
||||
if header_filter is not None:
|
||||
start.append('-header-filter=' + header_filter)
|
||||
if checks:
|
||||
start.append('-checks=' + checks)
|
||||
if tmpdir is not None:
|
||||
start.append('-export-fixes')
|
||||
# Get a temporary file. We immediately close the handle so clang-tidy can
|
||||
# overwrite it.
|
||||
(handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
|
||||
os.close(handle)
|
||||
start.append(name)
|
||||
for arg in extra_arg:
|
||||
start.append('-extra-arg=%s' % arg)
|
||||
for arg in extra_arg_before:
|
||||
start.append('-extra-arg-before=%s' % arg)
|
||||
start.append('-p=' + build_path)
|
||||
if quiet:
|
||||
start.append('-quiet')
|
||||
if config:
|
||||
start.append('-config=' + config)
|
||||
start.append(f)
|
||||
return start
|
||||
|
||||
|
||||
def merge_replacement_files(tmpdir, mergefile):
|
||||
"""Merge all replacement files in a directory into a single file"""
|
||||
# The fixes suggested by clang-tidy >= 4.0.0 are given under
|
||||
# the top level key 'Diagnostics' in the output yaml files
|
||||
mergekey = "Diagnostics"
|
||||
merged=[]
|
||||
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
|
||||
content = yaml.safe_load(open(replacefile, 'r'))
|
||||
if not content:
|
||||
continue # Skip empty files.
|
||||
merged.extend(content.get(mergekey, []))
|
||||
|
||||
if merged:
|
||||
# MainSourceFile: The key is required by the definition inside
|
||||
# include/clang/Tooling/ReplacementsYaml.h, but the value
|
||||
# is actually never used inside clang-apply-replacements,
|
||||
# so we set it to '' here.
|
||||
output = {'MainSourceFile': '', mergekey: merged}
|
||||
with open(mergefile, 'w') as out:
|
||||
yaml.safe_dump(output, out)
|
||||
else:
|
||||
# Empty the file:
|
||||
open(mergefile, 'w').close()
|
||||
|
||||
|
||||
def check_clang_apply_replacements_binary(args):
|
||||
"""Checks if invoking supplied clang-apply-replacements binary works."""
|
||||
try:
|
||||
subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
|
||||
except:
|
||||
print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
|
||||
'binary correctly specified?', file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def apply_fixes(args, tmpdir):
|
||||
"""Calls clang-apply-fixes on a given directory."""
|
||||
invocation = [args.clang_apply_replacements_binary]
|
||||
if args.format:
|
||||
invocation.append('-format')
|
||||
if args.style:
|
||||
invocation.append('-style=' + args.style)
|
||||
invocation.append(tmpdir)
|
||||
subprocess.call(invocation)
|
||||
|
||||
|
||||
def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
|
||||
"""Takes filenames out of queue and runs clang-tidy on them."""
|
||||
while True:
|
||||
name = queue.get()
|
||||
invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
|
||||
tmpdir, build_path, args.header_filter,
|
||||
args.allow_enabling_alpha_checkers,
|
||||
args.extra_arg, args.extra_arg_before,
|
||||
args.quiet, args.config)
|
||||
|
||||
proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
output, err = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
failed_files.append(name)
|
||||
with lock:
|
||||
sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
|
||||
if len(err) > 0:
|
||||
sys.stdout.flush()
|
||||
sys.stderr.write(err.decode('utf-8'))
|
||||
queue.task_done()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
|
||||
'in a compilation database. Requires '
|
||||
'clang-tidy and clang-apply-replacements in '
|
||||
'$PATH.')
|
||||
parser.add_argument('-allow-enabling-alpha-checkers',
|
||||
action='store_true', help='allow alpha checkers from '
|
||||
'clang-analyzer.')
|
||||
parser.add_argument('-clang-tidy-binary', metavar='PATH',
|
||||
default='clang-tidy-11',
|
||||
help='path to clang-tidy binary')
|
||||
parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
|
||||
default='clang-apply-replacements-11',
|
||||
help='path to clang-apply-replacements binary')
|
||||
parser.add_argument('-checks', default=None,
|
||||
help='checks filter, when not specified, use clang-tidy '
|
||||
'default')
|
||||
parser.add_argument('-config', default=None,
|
||||
help='Specifies a configuration in YAML/JSON format: '
|
||||
' -config="{Checks: \'*\', '
|
||||
' CheckOptions: [{key: x, '
|
||||
' value: y}]}" '
|
||||
'When the value is empty, clang-tidy will '
|
||||
'attempt to find a file named .clang-tidy for '
|
||||
'each source file in its parent directories.')
|
||||
parser.add_argument('-header-filter', default=None,
|
||||
help='regular expression matching the names of the '
|
||||
'headers to output diagnostics from. Diagnostics from '
|
||||
'the main file of each translation unit are always '
|
||||
'displayed.')
|
||||
if yaml:
|
||||
parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
|
||||
help='Create a yaml file to store suggested fixes in, '
|
||||
'which can be applied with clang-apply-replacements.')
|
||||
parser.add_argument('-j', type=int, default=0,
|
||||
help='number of tidy instances to be run in parallel.')
|
||||
parser.add_argument('files', nargs='*', default=['.*'],
|
||||
help='files to be processed (regex on path)')
|
||||
parser.add_argument('-fix', action='store_true', help='apply fix-its')
|
||||
parser.add_argument('-format', action='store_true', help='Reformat code '
|
||||
'after applying fixes')
|
||||
parser.add_argument('-style', default='file', help='The style of reformat '
|
||||
'code after applying fixes')
|
||||
parser.add_argument('-p', dest='build_path',
|
||||
help='Path used to read a compile command database.')
|
||||
parser.add_argument('-extra-arg', dest='extra_arg',
|
||||
action='append', default=[],
|
||||
help='Additional argument to append to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
|
||||
action='append', default=[],
|
||||
help='Additional argument to prepend to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-quiet', action='store_true',
|
||||
help='Run clang-tidy in quiet mode')
|
||||
args = parser.parse_args()
|
||||
|
||||
db_path = 'compile_commands.json'
|
||||
|
||||
if args.build_path is not None:
|
||||
build_path = args.build_path
|
||||
else:
|
||||
# Find our database
|
||||
build_path = find_compilation_database(db_path)
|
||||
|
||||
try:
|
||||
invocation = [args.clang_tidy_binary, '-list-checks']
|
||||
if args.allow_enabling_alpha_checkers:
|
||||
invocation.append('-allow-enabling-analyzer-alpha-checkers')
|
||||
invocation.append('-p=' + build_path)
|
||||
if args.checks:
|
||||
invocation.append('-checks=' + args.checks)
|
||||
invocation.append('-')
|
||||
if args.quiet:
|
||||
# Even with -quiet we still want to check if we can call clang-tidy.
|
||||
with open(os.devnull, 'w') as dev_null:
|
||||
subprocess.check_call(invocation, stdout=dev_null)
|
||||
else:
|
||||
subprocess.check_call(invocation)
|
||||
except:
|
||||
print("Unable to run clang-tidy.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Load the database and extract all files.
|
||||
database = json.load(open(os.path.join(build_path, db_path)))
|
||||
files = [make_absolute(entry['file'], entry['directory'])
|
||||
for entry in database]
|
||||
|
||||
max_task = args.j
|
||||
if max_task == 0:
|
||||
max_task = multiprocessing.cpu_count()
|
||||
|
||||
tmpdir = None
|
||||
if args.fix or (yaml and args.export_fixes):
|
||||
check_clang_apply_replacements_binary(args)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
# Build up a big regexy filter from all command line arguments.
|
||||
file_name_re = re.compile('|'.join(args.files))
|
||||
|
||||
return_code = 0
|
||||
try:
|
||||
# Spin up a bunch of tidy-launching threads.
|
||||
task_queue = queue.Queue(max_task)
|
||||
# List of files with a non-zero return code.
|
||||
failed_files = []
|
||||
lock = threading.Lock()
|
||||
for _ in range(max_task):
|
||||
t = threading.Thread(target=run_tidy,
|
||||
args=(args, tmpdir, build_path, task_queue, lock, failed_files))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
# Fill the queue with files.
|
||||
for name in files:
|
||||
if file_name_re.search(name):
|
||||
task_queue.put(name)
|
||||
|
||||
# Wait for all threads to be done.
|
||||
task_queue.join()
|
||||
if len(failed_files):
|
||||
return_code = 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
# This is a sad hack. Unfortunately subprocess goes
|
||||
# bonkers with ctrl-c and we start forking merrily.
|
||||
print('\nCtrl-C detected, goodbye.')
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
os.kill(0, 9)
|
||||
|
||||
if yaml and args.export_fixes:
|
||||
print('Writing fixes to ' + args.export_fixes + ' ...')
|
||||
try:
|
||||
merge_replacement_files(tmpdir, args.export_fixes)
|
||||
except:
|
||||
print('Error exporting fixes.\n', file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
return_code=1
|
||||
|
||||
if args.fix:
|
||||
print('Applying fixes ...')
|
||||
try:
|
||||
apply_fixes(args, tmpdir)
|
||||
except:
|
||||
print('Error applying fixes.\n', file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
return_code = 1
|
||||
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
sys.exit(return_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
12
tools/lsan.supp
Normal file
12
tools/lsan.supp
Normal file
@@ -0,0 +1,12 @@
|
||||
leak:antlr4::atn::ArrayPredictionContext::ArrayPredictionContext
|
||||
leak:std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<antlr4::atn::SingletonPredictionContext, std::allocator<antlr4::atn::SingletonPredictionContext>, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&>(antlr4::atn::SingletonPredictionContext*&, std::_Sp_alloc_shared_tag<std::allocator<antlr4::atn::SingletonPredictionContext> >, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&)
|
||||
leak:antlr4::atn::PredictionContext::mergeSingletons(std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
|
||||
leak:void std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::PredictionContext> >(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::PredictionContext>*, std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > > >, std::shared_ptr<antlr4::atn::PredictionContext>&&)
|
||||
leak:antlr4::atn::ParserATNSimulator::closureCheckingStopState(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::ATNConfigSet*, std::unordered_set<std::shared_ptr<antlr4::atn::ATNConfig>, antlr4::atn::ATNConfig::Hasher, antlr4::atn::ATNConfig::Comparer, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >&, bool, bool, int, bool)
|
||||
leak:antlr4::atn::ParserATNSimulator::computeReachSet(antlr4::atn::ATNConfigSet*, unsigned long, bool)
|
||||
leak:std::_Hashtable<unsigned long, std::pair<unsigned long const, antlr4::atn::ATNConfig*>, std::allocator<std::pair<unsigned long const, antlr4::atn::ATNConfig*> >, std::__detail::_Select1st, std::equal_to<unsigned long>, std::hash<unsigned long>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node(unsigned long const&, unsigned long, unsigned long, std::__detail::_Hash_node<std::pair<unsigned long const, antlr4::atn::ATNConfig*>, false>*, unsigned long)
|
||||
leak:void std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::ATNConfig> const&>(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::ATNConfig>*, std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > > >, std::shared_ptr<antlr4::atn::ATNConfig> const&)
|
||||
leak:antlr4::atn::ATNConfigSet::add(std::shared_ptr<antlr4::atn::ATNConfig> const&, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
|
||||
leak:antlr4::atn::ParserATNSimulator::getEpsilonTarget(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::Transition*, bool, bool, bool, bool)
|
||||
leak:antlr4::atn::PredictionContext::mergeArrays(std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
|
||||
leak:/lib/x86_64-linux-gnu/libpython3.
|
||||
Reference in New Issue
Block a user