Compare commits
12 Commits
release/1.
...
E088-MG-Qu
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
999b3ef79f | ||
|
|
30413a7b4f | ||
|
|
1def0c9104 | ||
|
|
782c377f5d | ||
|
|
cc27a04139 | ||
|
|
b71345655f | ||
|
|
ccdd58b336 | ||
|
|
50b6afd73d | ||
|
|
59105f68bd | ||
|
|
8de31092ad | ||
|
|
5c93f81881 | ||
|
|
6d4fe5cdd5 |
@@ -54,7 +54,7 @@ Checks: '*,
|
||||
-readability-magic-numbers,
|
||||
-readability-named-parameter'
|
||||
WarningsAsErrors: ''
|
||||
HeaderFilterRegex: ''
|
||||
HeaderFilterRegex: 'src/.*'
|
||||
AnalyzeTemporaryDtors: false
|
||||
FormatStyle: none
|
||||
CheckOptions:
|
||||
|
||||
157
.github/workflows/diff.yaml
vendored
157
.github/workflows/diff.yaml
vendored
@@ -10,7 +10,7 @@ on:
|
||||
jobs:
|
||||
community_build:
|
||||
name: "Community build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -65,9 +65,9 @@ jobs:
|
||||
name: "Community DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
coverage_build:
|
||||
name: "Coverage build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
code_analysis:
|
||||
name: "Code analysis"
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build coverage binaries
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
@@ -87,9 +87,8 @@ jobs:
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build coverage binaries.
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON ..
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
|
||||
- name: Run unit tests
|
||||
@@ -97,9 +96,9 @@ jobs:
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Run unit tests.
|
||||
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
|
||||
cd build
|
||||
ctest -R memgraph__unit --output-on-failure -j$THREADS
|
||||
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
|
||||
|
||||
- name: Compute code coverage
|
||||
run: |
|
||||
@@ -120,9 +119,19 @@ jobs:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
|
||||
debug_build:
|
||||
name: "Debug build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -196,7 +205,7 @@ jobs:
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
runs-on: [self-hosted, General, Linux, X64, Debian10]
|
||||
runs-on: [self-hosted, Linux, X64, Diff]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
@@ -208,21 +217,6 @@ jobs:
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up parent
|
||||
run: |
|
||||
# Remove parent folder (if it exists).
|
||||
cd ..
|
||||
if [ -d parent ]; then
|
||||
rm -rf parent
|
||||
fi
|
||||
|
||||
# Copy untouched repository to parent folder.
|
||||
cp -r memgraph parent
|
||||
|
||||
# Checkout previous commit
|
||||
cd parent
|
||||
git checkout HEAD~1
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -236,47 +230,6 @@ jobs:
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Build parent binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
cd ../parent
|
||||
./init
|
||||
|
||||
# Build parent binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS memgraph memgraph__macro_benchmark
|
||||
|
||||
- name: Run macro benchmark tests
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Run parent macro benchmark tests
|
||||
run: |
|
||||
cd ../parent/tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Compute macro benchmark summary
|
||||
run: |
|
||||
./tools/github/macro_benchmark_summary \
|
||||
--current tests/macro_benchmark/.harness_summary \
|
||||
--previous ../parent/tests/macro_benchmark/.harness_summary \
|
||||
--output macro_benchmark_summary.txt
|
||||
|
||||
- name: Save macro benchmark summary
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Macro benchmark summary"
|
||||
path: macro_benchmark_summary.txt
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests/gql_behave
|
||||
@@ -339,6 +292,15 @@ jobs:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
@@ -378,3 +340,64 @@ jobs:
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Upload macro benchmark results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}"
|
||||
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}"
|
||||
|
||||
44
.github/workflows/full_clang_tidy.yaml
vendored
Normal file
44
.github/workflows/full_clang_tidy.yaml
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
name: Run clang-tidy on the full codebase
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
clang_tidy_check:
|
||||
name: "Clang-tidy check"
|
||||
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
|
||||
env:
|
||||
THREADS: 24
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build debug binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build debug binaries.
|
||||
|
||||
cd build
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
source /opt/toolchain-v2/activate
|
||||
|
||||
# The results are also written to standard output in order to retain them in the logs
|
||||
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
|
||||
tee ./build/full_clang_tidy_output.txt
|
||||
|
||||
- name: Summarize clang-tidy results
|
||||
run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh
|
||||
10
CHANGELOG.md
10
CHANGELOG.md
@@ -2,6 +2,16 @@
|
||||
|
||||
## Future
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
|
||||
For example, parsing of `mgp.List[mgp.Map]` works now.
|
||||
* Fixed memory tracking issues. Some of the allocation and deallocation weren't
|
||||
tracked during the query execution.
|
||||
* Fixed reading CSV files that are using CRLF as the newline symbol.
|
||||
|
||||
## v1.4.0
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
* Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.
|
||||
|
||||
@@ -312,8 +312,9 @@ if (UBSAN)
|
||||
# runtime library and c++ standard libraries are present.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
|
||||
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
|
||||
# Make sure llvm-symbolizer binary is in path
|
||||
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
|
||||
# Make sure llvm-symbolizer binary is in path.
|
||||
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
|
||||
endif()
|
||||
|
||||
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
|
||||
|
||||
@@ -683,7 +683,15 @@ def _typing_to_cypher_type(type_):
|
||||
return _mgp.type_nullable(simple_type)
|
||||
return _mgp.type_nullable(parse_typing(type_arg_as_str))
|
||||
elif type_as_str.startswith('typing.List'):
|
||||
type_arg_as_str, = parse_type_args(type_as_str)
|
||||
type_arg_as_str = parse_type_args(type_as_str)
|
||||
|
||||
if len(type_arg_as_str) > 1:
|
||||
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
|
||||
# so we need to join the parts.
|
||||
type_arg_as_str = ', '.join(type_arg_as_str)
|
||||
else:
|
||||
type_arg_as_str = type_arg_as_str[0]
|
||||
|
||||
simple_type = get_simple_type(type_arg_as_str)
|
||||
if simple_type is not None:
|
||||
return _mgp.type_list(simple_type)
|
||||
|
||||
150
libs/setup.sh
150
libs/setup.sh
@@ -2,8 +2,9 @@
|
||||
|
||||
# Download external dependencies.
|
||||
|
||||
local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
|
||||
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
cd ${working_dir}
|
||||
cd "${working_dir}"
|
||||
|
||||
# Clones a git repository and optionally cherry picks additional commits. The
|
||||
# function will try to preserve any local changes in the repo.
|
||||
@@ -15,7 +16,11 @@ clone () {
|
||||
shift 3
|
||||
# Clone if there's no repo.
|
||||
if [[ ! -d "$dir_name" ]]; then
|
||||
git clone "$git_repo" "$dir_name"
|
||||
echo "Cloning from $git_repo"
|
||||
# If the clone fails, it doesn't make sense to continue with the function
|
||||
# execution but the whole script should continue executing because we might
|
||||
# clone the same repo from a different source.
|
||||
git clone "$git_repo" "$dir_name" || return 1
|
||||
fi
|
||||
pushd "$dir_name"
|
||||
# Just fetch new commits from remote repository. Don't merge/pull them in, so
|
||||
@@ -29,12 +34,17 @@ clone () {
|
||||
# Stash regardless of local_changes, so that a user gets a message on stdout.
|
||||
git stash
|
||||
# Checkout the primary commit (there's no need to pull/merge).
|
||||
git checkout $checkout_id
|
||||
# The checkout fail should exit this script immediately because the target
|
||||
# commit is not there and that will most likely create build-time errors.
|
||||
git checkout "$checkout_id" || exit 1
|
||||
# Apply any optional cherry pick fixes.
|
||||
while [[ $# -ne 0 ]]; do
|
||||
local cherry_pick_id=$1
|
||||
shift
|
||||
git cherry-pick -n $cherry_pick_id
|
||||
# The cherry-pick fail should exit this script immediately because the
|
||||
# target commit is not there and that will most likely create build-time
|
||||
# errors.
|
||||
git cherry-pick -n "$cherry_pick_id" || exit 1
|
||||
done
|
||||
# Reapply any local changes.
|
||||
if [[ $local_changes == true ]]; then
|
||||
@@ -43,12 +53,95 @@ clone () {
|
||||
popd
|
||||
}
|
||||
|
||||
file_get_try_double () {
|
||||
primary_url="$1"
|
||||
secondary_url="$2"
|
||||
echo "Download primary from $primary_url secondary from $secondary_url"
|
||||
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
|
||||
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
|
||||
filename="$(basename "$secondary_url")"
|
||||
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
|
||||
echo ""
|
||||
}
|
||||
|
||||
repo_clone_try_double () {
|
||||
primary_url="$1"
|
||||
secondary_url="$2"
|
||||
folder_name="$3"
|
||||
ref="$4"
|
||||
echo "Cloning primary from $primary_url secondary from $secondary_url"
|
||||
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
|
||||
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
|
||||
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
|
||||
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
|
||||
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
|
||||
echo ""
|
||||
}
|
||||
|
||||
# List all dependencies.
|
||||
|
||||
# The reason for introducing primary and secondary urls are:
|
||||
# * HTTPS is hard to cache
|
||||
# * Remote development workflow is more flexible if people don't have to connect to VPN
|
||||
# * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
|
||||
# * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
|
||||
|
||||
# The goal of having primary urls is to have links to the "local" cache of
|
||||
# dependencies where these dependencies could be downloaded as fast as
|
||||
# possible. The actual cache server could be on your local machine, on a
|
||||
# dedicated machine inside the build cluster or on the actual build machine.
|
||||
# Download from primary_urls might fail because the cache is not installed.
|
||||
declare -A primary_urls=(
|
||||
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
|
||||
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
|
||||
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
|
||||
["fmt"]="http://$local_cache_host/git/fmt.git"
|
||||
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
|
||||
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
|
||||
["gtest"]="http://$local_cache_host/git/googletest.git"
|
||||
["gflags"]="http://$local_cache_host/git/gflags.git"
|
||||
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
|
||||
["bzip2"]="http://$local_cache_host/git/bzip2.git"
|
||||
["zlib"]="http://$local_cache_host/git/zlib.git"
|
||||
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
|
||||
["mgclient"]="http://$local_cache_host/git/mgclient.git"
|
||||
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
|
||||
["spdlog"]="http://$local_cache_host/git/spdlog"
|
||||
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
|
||||
)
|
||||
|
||||
# The goal of secondary urls is to have links to the "source of truth" of
|
||||
# dependencies, e.g., Github or S3. Download from secondary urls, if happens
|
||||
# at all, should never fail. In other words, if it fails, the whole build
|
||||
# should fail.
|
||||
declare -A secondary_urls=(
|
||||
["antlr4-code"]="https://github.com/antlr/antlr4.git"
|
||||
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
|
||||
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
|
||||
["fmt"]="https://github.com/fmtlib/fmt.git"
|
||||
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
|
||||
["gbenchmark"]="https://github.com/google/benchmark.git"
|
||||
["gtest"]="https://github.com/google/googletest.git"
|
||||
["gflags"]="https://github.com/memgraph/gflags.git"
|
||||
["libbcrypt"]="https://github.com/rg3/libbcrypt"
|
||||
["bzip2"]="https://github.com/VFR-maniac/bzip2"
|
||||
["zlib"]="https://github.com/madler/zlib.git"
|
||||
["rocksdb"]="https://github.com/facebook/rocksdb.git"
|
||||
["mgclient"]="https://github.com/memgraph/mgclient.git"
|
||||
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
|
||||
["spdlog"]="https://github.com/gabime/spdlog"
|
||||
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
|
||||
)
|
||||
|
||||
# antlr
|
||||
antlr_generator_filename="antlr-4.6-complete.jar"
|
||||
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
|
||||
wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
|
||||
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
|
||||
|
||||
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
|
||||
clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
|
||||
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
|
||||
# fix missing include
|
||||
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
|
||||
# remove shared library from install dependencies
|
||||
@@ -56,74 +149,73 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
|
||||
|
||||
# cppitertools v2.0 2019-12-23
|
||||
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
|
||||
clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
|
||||
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
|
||||
|
||||
# fmt
|
||||
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
|
||||
clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
|
||||
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
|
||||
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
|
||||
|
||||
# rapidcheck
|
||||
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
|
||||
clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
|
||||
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
|
||||
|
||||
# google benchmark
|
||||
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
|
||||
clone https://github.com/google/benchmark.git benchmark $benchmark_tag
|
||||
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
|
||||
|
||||
# google test
|
||||
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
|
||||
clone https://github.com/google/googletest.git googletest $googletest_tag
|
||||
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
|
||||
|
||||
# google flags
|
||||
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
|
||||
clone https://github.com/memgraph/gflags.git gflags $gflags_tag
|
||||
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
|
||||
|
||||
# libbcrypt
|
||||
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
|
||||
clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
|
||||
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
|
||||
|
||||
# neo4j
|
||||
wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
|
||||
tar -xzf neo4j.tar.gz
|
||||
rm -rf neo4j
|
||||
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
|
||||
tar -xzf neo4j-community-3.2.3-unix.tar.gz
|
||||
mv neo4j-community-3.2.3 neo4j
|
||||
rm neo4j.tar.gz
|
||||
rm neo4j-community-3.2.3-unix.tar.gz
|
||||
|
||||
# nlohmann json
|
||||
# We wget header instead of cloning repo since repo is huge (lots of test data).
|
||||
# We use head on Sep 1, 2017 instead of last release since it was long time ago.
|
||||
mkdir -p json
|
||||
cd json
|
||||
wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
|
||||
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
|
||||
cd ..
|
||||
|
||||
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
|
||||
clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
|
||||
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
|
||||
|
||||
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
|
||||
clone https://github.com/madler/zlib.git zlib $zlib_tag
|
||||
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
|
||||
# remove shared library from install dependencies
|
||||
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
|
||||
|
||||
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
|
||||
clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
|
||||
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
|
||||
# remove shared library from install dependencies
|
||||
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
|
||||
|
||||
# mgclient
|
||||
mgclient_tag="v1.2.0" # (2021-01-14)
|
||||
clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
|
||||
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
|
||||
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
|
||||
|
||||
# pymgclient
|
||||
pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
|
||||
clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
|
||||
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
|
||||
|
||||
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
|
||||
clone https://github.com/gabime/spdlog spdlog $spdlog_tag
|
||||
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
|
||||
|
||||
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
|
||||
clone https://github.com/jemalloc/jemalloc.git jemalloc $jemalloc_tag
|
||||
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
|
||||
pushd jemalloc
|
||||
# ThreadPool select job randomly, and there can be some threads that had been
|
||||
# performed some memory heavy task before and will be inactive for some time,
|
||||
@@ -138,5 +230,5 @@ pushd jemalloc
|
||||
# avoid spurious latencies and additional work associated with
|
||||
# MADV_DONTNEED. See
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
|
||||
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000"
|
||||
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
|
||||
popd
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/readable_size.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/tsc.hpp"
|
||||
|
||||
@@ -603,8 +604,8 @@ struct PullPlanVector {
|
||||
|
||||
struct PullPlan {
|
||||
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory);
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
std::optional<size_t> memory_limit = {});
|
||||
std::optional<ExecutionContext> Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
@@ -614,6 +615,7 @@ struct PullPlan {
|
||||
plan::UniqueCursorPtr cursor_ = nullptr;
|
||||
Frame frame_;
|
||||
ExecutionContext ctx_;
|
||||
std::optional<size_t> memory_limit_;
|
||||
|
||||
// As it's possible to query execution using multiple pulls
|
||||
// we need the keep track of the total execution time across
|
||||
@@ -630,11 +632,12 @@ struct PullPlan {
|
||||
};
|
||||
|
||||
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory)
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
const std::optional<size_t> memory_limit)
|
||||
: plan_(plan),
|
||||
cursor_(plan->plan().MakeCursor(execution_memory)),
|
||||
frame_(plan->symbol_table().max_position(), execution_memory) {
|
||||
frame_(plan->symbol_table().max_position(), execution_memory),
|
||||
memory_limit_(memory_limit) {
|
||||
ctx_.db_accessor = dba;
|
||||
ctx_.symbol_table = plan->symbol_table();
|
||||
ctx_.evaluation_context.timestamp =
|
||||
@@ -657,21 +660,25 @@ std::optional<ExecutionContext> PullPlan::Pull(AnyStream *stream, std::optional<
|
||||
// single `Pull`.
|
||||
constexpr size_t stack_size = 256 * 1024;
|
||||
char stack_data[stack_size];
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
|
||||
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
|
||||
|
||||
if (memory_limit_) {
|
||||
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
|
||||
ctx_.evaluation_context.memory = &*maybe_limited_resource;
|
||||
} else {
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
}
|
||||
|
||||
// Returns true if a result was pulled.
|
||||
const auto pull_result = [&]() -> bool {
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
|
||||
return cursor_->Pull(frame_, ctx_);
|
||||
};
|
||||
const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); };
|
||||
|
||||
const auto stream_values = [&]() {
|
||||
// TODO: The streamed values should also probably use the above memory.
|
||||
@@ -828,10 +835,24 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
|
||||
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage),
|
||||
utils::Downcast<CypherQuery>(parsed_query.query), parsed_query.parameters,
|
||||
&interpreter_context->plan_cache, dba, parsed_query.is_cacheable);
|
||||
utils::MemoryResource *execution_memory) {
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
|
||||
|
||||
Frame frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
evaluation_context.parameters = parsed_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
|
||||
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
if (memory_limit) {
|
||||
spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit));
|
||||
}
|
||||
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
|
||||
parsed_query.parameters, &interpreter_context->plan_cache, dba);
|
||||
|
||||
summary->insert_or_assign("cost_estimate", plan->cost());
|
||||
auto rw_type_checker = plan::ReadWriteTypeChecker();
|
||||
@@ -850,8 +871,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
|
||||
}
|
||||
|
||||
auto pull_plan =
|
||||
std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory);
|
||||
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
|
||||
execution_memory, memory_limit);
|
||||
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
|
||||
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
@@ -865,7 +886,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
|
||||
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
utils::MemoryResource *execution_memory) {
|
||||
const std::string kExplainQueryStart = "explain ";
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
|
||||
"Expected stripped query to start with '{}'", kExplainQueryStart);
|
||||
@@ -911,7 +932,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
|
||||
|
||||
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory) {
|
||||
const std::string kProfileQueryStart = "profile ";
|
||||
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
|
||||
@@ -949,6 +970,15 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
|
||||
Frame frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
evaluation_context.parameters = parsed_inner_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
|
||||
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
|
||||
auto cypher_query_plan = CypherQueryToPlan(
|
||||
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
|
||||
@@ -960,14 +990,14 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), summary, dba,
|
||||
interpreter_context, execution_memory,
|
||||
interpreter_context, execution_memory, memory_limit,
|
||||
// We want to execute the query we are profiling lazily, so we delay
|
||||
// the construction of the corresponding context.
|
||||
ctx = std::optional<ExecutionContext>{}, pull_plan = std::shared_ptr<PullPlanVector>(nullptr)](
|
||||
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
|
||||
// No output symbols are given so that nothing is streamed.
|
||||
if (!ctx) {
|
||||
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory)
|
||||
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, memory_limit)
|
||||
.Pull(stream, {}, {}, summary);
|
||||
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(ctx->stats, ctx->profile_execution_time));
|
||||
}
|
||||
@@ -985,7 +1015,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
}
|
||||
|
||||
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
utils::MemoryResource *execution_memory) {
|
||||
return PreparedQuery{{"QUERY"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::make_shared<PullPlanDump>(dba)](
|
||||
@@ -1000,7 +1030,7 @@ PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, T
|
||||
|
||||
PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw IndexInMulticommandTxException();
|
||||
}
|
||||
@@ -1069,7 +1099,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
|
||||
|
||||
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw UserModificationInMulticommandTxException();
|
||||
}
|
||||
@@ -1122,6 +1152,8 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_ex
|
||||
return std::nullopt;
|
||||
},
|
||||
RWType::NONE};
|
||||
// False positive report for the std::make_shared above
|
||||
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
|
||||
}
|
||||
|
||||
PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
@@ -1180,7 +1212,7 @@ PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_exp
|
||||
|
||||
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
storage::Storage *db, utils::MonotonicBufferResource *execution_memory) {
|
||||
storage::Storage *db, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw InfoInMulticommandTxException();
|
||||
}
|
||||
@@ -1268,8 +1300,7 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
|
||||
|
||||
PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context,
|
||||
utils::MonotonicBufferResource *execution_memory) {
|
||||
InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw ConstraintInMulticommandTxException();
|
||||
}
|
||||
@@ -1434,10 +1465,12 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
|
||||
|
||||
// Handle transaction control queries.
|
||||
auto query_upper = utils::Trim(utils::ToUpperCase(query_string));
|
||||
|
||||
if (query_upper == "BEGIN" || query_upper == "COMMIT" || query_upper == "ROLLBACK") {
|
||||
query_execution->prepared_query.emplace(PrepareTransactionQuery(query_upper));
|
||||
const auto upper_case_query = utils::ToUpperCase(query_string);
|
||||
const auto trimmed_query = utils::Trim(upper_case_query);
|
||||
|
||||
if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") {
|
||||
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
|
||||
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
|
||||
}
|
||||
|
||||
|
||||
@@ -317,7 +317,9 @@ class Interpreter final {
|
||||
private:
|
||||
struct QueryExecution {
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
|
||||
utils::MonotonicBufferResource execution_monotonic_memory{kExecutionMemoryBlockSize};
|
||||
utils::ResourceWithOutOfMemoryException execution_memory{&execution_monotonic_memory};
|
||||
|
||||
std::map<std::string, TypedValue> summary;
|
||||
|
||||
explicit QueryExecution() = default;
|
||||
@@ -331,7 +333,7 @@ class Interpreter final {
|
||||
// destroy the prepared query which is using that instance
|
||||
// of execution memory.
|
||||
prepared_query.reset();
|
||||
execution_memory.Release();
|
||||
execution_monotonic_memory.Release();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -84,6 +84,11 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
|
||||
|
||||
std::string_view line_string_view = *maybe_line;
|
||||
|
||||
// remove '\r' from the end in case we have dos file format
|
||||
if (line_string_view.back() == '\r') {
|
||||
line_string_view.remove_suffix(1);
|
||||
}
|
||||
|
||||
while (state != CsvParserState::DONE && !line_string_view.empty()) {
|
||||
const auto c = line_string_view[0];
|
||||
|
||||
|
||||
@@ -23,6 +23,9 @@ size_t GrowMonotonicBuffer(size_t current_size, size_t max_size) {
|
||||
return std::ceil(next_size);
|
||||
}
|
||||
|
||||
__attribute__((no_sanitize("pointer-overflow"))) void CheckAllocationSizeOverflow(void *aligned_ptr, size_t bytes) {
|
||||
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
MonotonicBufferResource::MonotonicBufferResource(size_t initial_size) : initial_size_(initial_size) {}
|
||||
@@ -121,7 +124,7 @@ void *MonotonicBufferResource::DoAllocate(size_t bytes, size_t alignment) {
|
||||
next_buffer_size_ = GrowMonotonicBuffer(next_buffer_size_, std::numeric_limits<size_t>::max() - sizeof(Buffer));
|
||||
}
|
||||
if (reinterpret_cast<char *>(aligned_ptr) < buffer_head) throw BadAlloc("Allocation alignment overflow");
|
||||
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
|
||||
CheckAllocationSizeOverflow(aligned_ptr, bytes);
|
||||
allocated_ = reinterpret_cast<char *>(aligned_ptr) - data + bytes;
|
||||
return aligned_ptr;
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
#include <cstddef>
|
||||
#include <new>
|
||||
|
||||
#if USE_JEMALLOC
|
||||
@@ -10,7 +11,7 @@
|
||||
#include "utils/memory_tracker.hpp"
|
||||
|
||||
namespace {
|
||||
void *newImpl(std::size_t size) {
|
||||
void *newImpl(const std::size_t size) {
|
||||
auto *ptr = malloc(size);
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
return ptr;
|
||||
@@ -19,11 +20,26 @@ void *newImpl(std::size_t size) {
|
||||
throw std::bad_alloc{};
|
||||
}
|
||||
|
||||
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
|
||||
void *newImpl(const std::size_t size, const std::align_val_t align) {
|
||||
auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
return ptr;
|
||||
}
|
||||
|
||||
void deleteImpl(void *ptr) noexcept { free(ptr); }
|
||||
throw std::bad_alloc{};
|
||||
}
|
||||
|
||||
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
|
||||
void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept {
|
||||
return aligned_alloc(size, static_cast<std::size_t>(align));
|
||||
}
|
||||
|
||||
#if USE_JEMALLOC
|
||||
void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
|
||||
|
||||
void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
|
||||
dallocx(ptr, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t size) noexcept {
|
||||
if (UNLIKELY(ptr == nullptr)) {
|
||||
@@ -33,24 +49,43 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
|
||||
sdallocx(ptr, size, 0);
|
||||
}
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
|
||||
if (UNLIKELY(ptr == nullptr)) {
|
||||
return;
|
||||
}
|
||||
|
||||
sdallocx(ptr, size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
|
||||
#else
|
||||
void deleteImpl(void *ptr) noexcept { free(ptr); }
|
||||
|
||||
void deleteImpl(void *ptr, const std::align_val_t /*unused*/) noexcept { free(ptr); }
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t /*unused*/) noexcept { free(ptr); }
|
||||
|
||||
void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t /*unused*/) noexcept { free(ptr); }
|
||||
#endif
|
||||
|
||||
void TrackMemory(const size_t size) {
|
||||
size_t actual_size = size;
|
||||
|
||||
void TrackMemory(std::size_t size) {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(size != 0)) {
|
||||
actual_size = nallocx(size, 0);
|
||||
size = nallocx(size, 0);
|
||||
}
|
||||
#endif
|
||||
utils::total_memory_tracker.Alloc(actual_size);
|
||||
utils::total_memory_tracker.Alloc(size);
|
||||
}
|
||||
|
||||
bool TrackMemoryNoExcept(const size_t size) {
|
||||
void TrackMemory(std::size_t size, const std::align_val_t align) {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(size != 0)) {
|
||||
size = nallocx(size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
#endif
|
||||
utils::total_memory_tracker.Alloc(size);
|
||||
}
|
||||
|
||||
bool TrackMemoryNoExcept(const std::size_t size) {
|
||||
try {
|
||||
TrackMemory(size);
|
||||
} catch (...) {
|
||||
@@ -60,7 +95,17 @@ bool TrackMemoryNoExcept(const size_t size) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0) noexcept {
|
||||
bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
|
||||
try {
|
||||
TrackMemory(size, align);
|
||||
} catch (...) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
|
||||
try {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
@@ -78,32 +123,74 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0)
|
||||
}
|
||||
}
|
||||
|
||||
void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
|
||||
try {
|
||||
#if USE_JEMALLOC
|
||||
if (LIKELY(ptr != nullptr)) {
|
||||
utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align))); // NOLINT(hicpp-signed-bitwise)
|
||||
}
|
||||
#else
|
||||
if (size) {
|
||||
utils::total_memory_tracker.Free(size);
|
||||
} else {
|
||||
// Innaccurate because malloc_usable_size() result is greater or equal to allocated size.
|
||||
utils::total_memory_tracker.Free(malloc_usable_size(ptr));
|
||||
}
|
||||
#endif
|
||||
} catch (...) {
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void *operator new(std::size_t size) {
|
||||
void *operator new(const std::size_t size) {
|
||||
TrackMemory(size);
|
||||
return newImpl(size);
|
||||
}
|
||||
|
||||
void *operator new[](std::size_t size) {
|
||||
void *operator new[](const std::size_t size) {
|
||||
TrackMemory(size);
|
||||
return newImpl(size);
|
||||
}
|
||||
|
||||
void *operator new(std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
void *operator new(const std::size_t size, const std::align_val_t align) {
|
||||
TrackMemory(size, align);
|
||||
return newImpl(size, align);
|
||||
}
|
||||
|
||||
void *operator new[](const std::size_t size, const std::align_val_t align) {
|
||||
TrackMemory(size, align);
|
||||
return newImpl(size, align);
|
||||
}
|
||||
|
||||
void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size))) {
|
||||
return newNoExcept(size);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void *operator new[](std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size))) {
|
||||
return newNoExcept(size);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size, align))) {
|
||||
return newNoExcept(size, align);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
if (LIKELY(TrackMemoryNoExcept(size, align))) {
|
||||
return newNoExcept(size, align);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void operator delete(void *ptr) noexcept {
|
||||
UntrackMemory(ptr);
|
||||
deleteImpl(ptr);
|
||||
@@ -114,12 +201,52 @@ void operator delete[](void *ptr) noexcept {
|
||||
deleteImpl(ptr);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, std::size_t size) noexcept {
|
||||
void operator delete(void *ptr, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::size_t size) noexcept {
|
||||
UntrackMemory(ptr, size);
|
||||
deleteSized(ptr, size);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, std::size_t size) noexcept {
|
||||
void operator delete[](void *ptr, const std::size_t size) noexcept {
|
||||
UntrackMemory(ptr, size);
|
||||
deleteSized(ptr, size);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align, size);
|
||||
deleteSized(ptr, size, align);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
|
||||
UntrackMemory(ptr, align, size);
|
||||
deleteSized(ptr, size, align);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr);
|
||||
deleteImpl(ptr);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr);
|
||||
deleteImpl(ptr);
|
||||
}
|
||||
|
||||
void operator delete(void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
void operator delete[](void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
|
||||
UntrackMemory(ptr, align);
|
||||
deleteImpl(ptr, align);
|
||||
}
|
||||
|
||||
@@ -38,19 +38,5 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
|
||||
spdlog::info("Memgraph is out of memory");
|
||||
|
||||
spdlog::info("Cleaning up unused memory");
|
||||
client->Execute("MATCH (n) DETACH DELETE n;");
|
||||
client->DiscardAll();
|
||||
client->Execute("FREE MEMORY;");
|
||||
client->DiscardAll();
|
||||
|
||||
// now it should succeed
|
||||
spdlog::info("Retrying the query with the memory cleaned up");
|
||||
client->Execute(create_query);
|
||||
if (!client->FetchOne()) {
|
||||
LOG_FATAL("Memgraph is still out of memory");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -2,7 +2,8 @@ bolt_port: &bolt_port "7687"
|
||||
template_cluster: &template_cluster
|
||||
cluster:
|
||||
main:
|
||||
args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"]
|
||||
args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"]
|
||||
log_file: "memory-e2e.log"
|
||||
setup_queries: []
|
||||
validation_queries: []
|
||||
|
||||
|
||||
@@ -11,19 +11,23 @@ template_validation_queries: &template_validation_queries
|
||||
template_cluster: &template_cluster
|
||||
cluster:
|
||||
replica_1:
|
||||
args: ["--bolt-port", "7688"]
|
||||
args: ["--bolt-port", "7688", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-replica1.log"
|
||||
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
|
||||
<<: *template_validation_queries
|
||||
replica_2:
|
||||
args: ["--bolt-port", "7689"]
|
||||
args: ["--bolt-port", "7689", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-replica2.log"
|
||||
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
|
||||
<<: *template_validation_queries
|
||||
replica_3:
|
||||
args: ["--bolt-port", "7690"]
|
||||
args: ["--bolt-port", "7690", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-replica3.log"
|
||||
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10003;"]
|
||||
<<: *template_validation_queries
|
||||
main:
|
||||
args: ["--bolt-port", "7687"]
|
||||
args: ["--bolt-port", "7687", "--log-level=TRACE"]
|
||||
log_file: "replication-e2e-main.log"
|
||||
setup_queries: [
|
||||
"REGISTER REPLICA replica_1 SYNC WITH TIMEOUT 0 TO '127.0.0.1:10001'",
|
||||
"REGISTER REPLICA replica_2 SYNC WITH TIMEOUT 1 TO '127.0.0.1:10002'",
|
||||
|
||||
@@ -44,7 +44,9 @@ def run(args):
|
||||
for name, config in workload['cluster'].items():
|
||||
mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY)
|
||||
mg_instances[name] = mg_instance
|
||||
mg_instance.start(args=config['args'])
|
||||
log_file_path = os.path.join(BUILD_DIR, 'logs', config['log_file'])
|
||||
binary_args = config['args'] + ["--log-file", log_file_path]
|
||||
mg_instance.start(args=binary_args)
|
||||
for query in config['setup_queries']:
|
||||
mg_instance.query(query)
|
||||
# Test.
|
||||
|
||||
@@ -164,4 +164,4 @@
|
||||
{:bank (bank-checker)
|
||||
:timeline (timeline/html)})
|
||||
:generator (c/replication-gen (gen/mix [read-balances valid-transfer]))
|
||||
:final-generator (gen/once read-balances)})
|
||||
:final-generator {:gen (gen/once read-balances) :recovery-time 20}})
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
"A map of workload names to functions that can take opts and construct
|
||||
workloads."
|
||||
{:bank bank/workload
|
||||
:sequential sequential/workload
|
||||
;; :sequential sequential/workload (T0532-MG)
|
||||
:large large/workload})
|
||||
|
||||
(def nemesis-configuration
|
||||
@@ -45,8 +45,8 @@
|
||||
(gen/log "Healing cluster.")
|
||||
(gen/nemesis (:final-generator nemesis))
|
||||
(gen/log "Waiting for recovery")
|
||||
(gen/sleep 20)
|
||||
(gen/clients final-generator))
|
||||
(gen/sleep (:recovery-time final-generator))
|
||||
(gen/clients (:gen final-generator)))
|
||||
gen)]
|
||||
(merge tests/noop-test
|
||||
opts
|
||||
|
||||
@@ -103,4 +103,4 @@
|
||||
:timeline (timeline/html)})
|
||||
:generator (c/replication-gen
|
||||
(gen/mix [read-nodes add-nodes]))
|
||||
:final-generator (gen/once read-nodes)})
|
||||
:final-generator {:gen (gen/once read-nodes) :recovery-time 40}})
|
||||
|
||||
@@ -383,15 +383,16 @@ TEST(BoltSession, ExecuteRunWrongMarker) {
|
||||
}
|
||||
|
||||
TEST(BoltSession, ExecuteRunMissingData) {
|
||||
std::array<uint8_t, 6> run_req_without_parameters{
|
||||
run_req_header[0], run_req_header[1], run_req_header[2], 0x00, 0x00, 0x00};
|
||||
// test lengths, they test the following situations:
|
||||
// missing header data, missing query data, missing parameters
|
||||
int len[] = {1, 2, 37};
|
||||
|
||||
int len[] = {1, 2, run_req_without_parameters.size()};
|
||||
for (int i = 0; i < 3; ++i) {
|
||||
INIT_VARS;
|
||||
ExecuteHandshake(input_stream, session, output);
|
||||
ExecuteInit(input_stream, session, output);
|
||||
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_header, len[i]), SessionException);
|
||||
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_without_parameters.data(), len[i]), SessionException);
|
||||
|
||||
ASSERT_EQ(session.state_, State::Close);
|
||||
CheckFailureMessage(output);
|
||||
@@ -871,7 +872,7 @@ TEST(BoltSession, Noop) {
|
||||
CheckFailureMessage(output);
|
||||
|
||||
session.state_ = State::Result;
|
||||
ExecuteCommand(input_stream, session, pullall_req, sizeof(v4::pullall_req));
|
||||
ExecuteCommand(input_stream, session, pullall_req, sizeof(pullall_req));
|
||||
CheckSuccessMessage(output);
|
||||
|
||||
ASSERT_THROW(ExecuteCommand(input_stream, session, v4_1::noop, sizeof(v4_1::noop)), SessionException);
|
||||
|
||||
@@ -73,6 +73,12 @@ class TestPlanner : public ::testing::Test {};
|
||||
|
||||
using PlannerTypes = ::testing::Types<Planner>;
|
||||
|
||||
void DeleteListContent(std::list<BaseOpChecker *> *list) {
|
||||
for (BaseOpChecker *ptr : *list) {
|
||||
delete ptr;
|
||||
}
|
||||
}
|
||||
|
||||
TYPED_TEST_CASE(TestPlanner, PlannerTypes);
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchNodeReturn) {
|
||||
@@ -223,6 +229,7 @@ TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) {
|
||||
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
|
||||
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()};
|
||||
CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchWhereReturn) {
|
||||
@@ -549,10 +556,8 @@ TYPED_TEST(TestPlanner, MatchMerge) {
|
||||
auto acc = ExpectAccumulate({symbol_table.at(*ident_n)});
|
||||
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
|
||||
CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce());
|
||||
for (auto &op : on_match) delete op;
|
||||
on_match.clear();
|
||||
for (auto &op : on_create) delete op;
|
||||
on_create.clear();
|
||||
DeleteListContent(&on_match);
|
||||
DeleteListContent(&on_create);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
|
||||
@@ -564,6 +569,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
|
||||
WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r")));
|
||||
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchUnwindReturn) {
|
||||
@@ -705,6 +711,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) {
|
||||
// optional ScanAll.
|
||||
std::list<BaseOpChecker *> optional{new ExpectFilter(), new ExpectScanAll()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MatchReturnAsterisk) {
|
||||
@@ -763,8 +770,8 @@ TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) {
|
||||
std::list<BaseOpChecker *> on_match{new ExpectScanAll(), new ExpectFilter()};
|
||||
std::list<BaseOpChecker *> on_create{new ExpectCreateNode()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create));
|
||||
for (auto &op : on_match) delete op;
|
||||
for (auto &op : on_create) delete op;
|
||||
DeleteListContent(&on_match);
|
||||
DeleteListContent(&on_create);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
|
||||
@@ -774,6 +781,7 @@ TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
|
||||
QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n")));
|
||||
std::list<BaseOpChecker *> optional{new ExpectScanAll()};
|
||||
CheckPlan<TypeParam>(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce());
|
||||
DeleteListContent(&optional);
|
||||
}
|
||||
|
||||
TYPED_TEST(TestPlanner, FunctionAggregationReturn) {
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
|
||||
#include "test_utils.hpp"
|
||||
|
||||
static void DummyCallback(const mgp_list *, const mgp_graph *, mgp_result *, mgp_memory *) {}
|
||||
|
||||
TEST(Module, InvalidProcedureRegistration) {
|
||||
@@ -53,7 +55,8 @@ TEST(Module, ProcedureSignature) {
|
||||
CheckSignature(proc, "proc() :: ()");
|
||||
mgp_proc_add_arg(proc, "arg1", mgp_type_number());
|
||||
CheckSignature(proc, "proc(arg1 :: NUMBER) :: ()");
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
|
||||
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
|
||||
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: ()");
|
||||
mgp_proc_add_result(proc, "res1", mgp_type_list(mgp_type_int()));
|
||||
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: (res1 :: LIST OF INTEGER)");
|
||||
@@ -69,7 +72,8 @@ TEST(Module, ProcedureSignature) {
|
||||
"(res1 :: LIST OF INTEGER, DEPRECATED res2 :: STRING)");
|
||||
EXPECT_FALSE(mgp_proc_add_result(proc, "res2", mgp_type_any()));
|
||||
EXPECT_FALSE(mgp_proc_add_deprecated_result(proc, "res1", mgp_type_any()));
|
||||
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(), mgp_value_make_string("string=\"value\"", &memory));
|
||||
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(),
|
||||
test_utils::CreateValueOwningPtr(mgp_value_make_string("string=\"value\"", &memory)).get());
|
||||
CheckSignature(proc,
|
||||
"proc(arg1 :: NUMBER, opt1 = Null :: ANY?, "
|
||||
"opt2 = \"string=\\\"value\\\"\" :: STRING) :: "
|
||||
@@ -80,6 +84,7 @@ TEST(Module, ProcedureSignatureOnlyOptArg) {
|
||||
mgp_memory memory{utils::NewDeleteResource()};
|
||||
mgp_module module(utils::NewDeleteResource());
|
||||
auto *proc = mgp_module_add_read_procedure(&module, "proc", DummyCallback);
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
|
||||
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
|
||||
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
|
||||
CheckSignature(proc, "proc(opt1 = Null :: ANY?) :: ()");
|
||||
}
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
|
||||
#include "test_utils.hpp"
|
||||
|
||||
TEST(CypherType, PresentableNameSimpleTypes) {
|
||||
EXPECT_EQ(mgp_type_any()->impl->GetPresentableName(), "ANY");
|
||||
EXPECT_EQ(mgp_type_bool()->impl->GetPresentableName(), "BOOLEAN");
|
||||
@@ -66,6 +72,7 @@ TEST(CypherType, NullSatisfiesType) {
|
||||
EXPECT_TRUE(null_type->impl->SatisfiesType(tv_null));
|
||||
}
|
||||
}
|
||||
mgp_value_destroy(mgp_null);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,6 +108,7 @@ TEST(CypherType, BoolSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_bool, tv_bool,
|
||||
{mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
|
||||
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_bool);
|
||||
}
|
||||
|
||||
TEST(CypherType, IntSatisfiesType) {
|
||||
@@ -111,6 +119,7 @@ TEST(CypherType, IntSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_int, tv_int,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_int);
|
||||
}
|
||||
|
||||
TEST(CypherType, DoubleSatisfiesType) {
|
||||
@@ -121,6 +130,7 @@ TEST(CypherType, DoubleSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_double, tv_double,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_double);
|
||||
}
|
||||
|
||||
TEST(CypherType, StringSatisfiesType) {
|
||||
@@ -131,12 +141,13 @@ TEST(CypherType, StringSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_string, tv_string,
|
||||
{mgp_type_bool(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
|
||||
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_string);
|
||||
}
|
||||
|
||||
TEST(CypherType, MapSatisfiesType) {
|
||||
mgp_memory memory{utils::NewDeleteResource()};
|
||||
auto *map = mgp_map_make_empty(&memory);
|
||||
mgp_map_insert(map, "key", mgp_value_make_int(42, &memory));
|
||||
mgp_map_insert(map, "key", test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get());
|
||||
auto *mgp_map_v = mgp_value_make_map(map);
|
||||
const query::TypedValue tv_map(std::map<std::string, query::TypedValue>{{"key", query::TypedValue(42)}});
|
||||
CheckSatisfiesTypesAndNullable(mgp_map_v, tv_map, {mgp_type_any(), mgp_type_map()});
|
||||
@@ -144,6 +155,7 @@ TEST(CypherType, MapSatisfiesType) {
|
||||
mgp_map_v, tv_map,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_node(),
|
||||
mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_map_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, VertexSatisfiesType) {
|
||||
@@ -160,6 +172,7 @@ TEST(CypherType, VertexSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_vertex_v, tv_vertex,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
|
||||
mgp_type_number(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_vertex_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, EdgeSatisfiesType) {
|
||||
@@ -178,6 +191,7 @@ TEST(CypherType, EdgeSatisfiesType) {
|
||||
CheckNotSatisfiesTypesAndListAndNullable(mgp_edge_v, tv_edge,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
|
||||
mgp_type_number(), mgp_type_node(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_edge_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, PathSatisfiesType) {
|
||||
@@ -190,9 +204,13 @@ TEST(CypherType, PathSatisfiesType) {
|
||||
mgp_memory memory{utils::NewDeleteResource()};
|
||||
utils::Allocator<mgp_path> alloc(memory.impl);
|
||||
mgp_graph graph{&dba, storage::View::NEW};
|
||||
auto *path = mgp_path_make_with_start(alloc.new_object<mgp_vertex>(v1, &graph), &memory);
|
||||
auto *mgp_vertex_v = alloc.new_object<mgp_vertex>(v1, &graph);
|
||||
auto path = mgp_path_make_with_start(mgp_vertex_v, &memory);
|
||||
ASSERT_TRUE(path);
|
||||
ASSERT_TRUE(mgp_path_expand(path, alloc.new_object<mgp_edge>(edge, &graph)));
|
||||
alloc.delete_object(mgp_vertex_v);
|
||||
auto mgp_edge_v = alloc.new_object<mgp_edge>(edge, &graph);
|
||||
ASSERT_TRUE(mgp_path_expand(path, mgp_edge_v));
|
||||
alloc.delete_object(mgp_edge_v);
|
||||
auto *mgp_path_v = mgp_value_make_path(path);
|
||||
const query::TypedValue tv_path(query::Path(v1, edge, v2));
|
||||
CheckSatisfiesTypesAndNullable(mgp_path_v, tv_path, {mgp_type_any(), mgp_type_path()});
|
||||
@@ -200,6 +218,7 @@ TEST(CypherType, PathSatisfiesType) {
|
||||
mgp_path_v, tv_path,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship()});
|
||||
mgp_value_destroy(mgp_path_v);
|
||||
}
|
||||
|
||||
static std::vector<const mgp_type *> MakeListTypes(const std::vector<const mgp_type *> &element_types) {
|
||||
@@ -224,6 +243,7 @@ TEST(CypherType, EmptyListSatisfiesType) {
|
||||
auto all_types = MakeListTypes(primitive_types);
|
||||
all_types.push_back(mgp_type_any());
|
||||
CheckSatisfiesTypesAndNullable(mgp_list_v, tv_list, all_types);
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, ListOfIntSatisfiesType) {
|
||||
@@ -233,7 +253,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
|
||||
auto *mgp_list_v = mgp_value_make_list(list);
|
||||
query::TypedValue tv_list(std::vector<query::TypedValue>{});
|
||||
for (int64_t i = 0; i < elem_count; ++i) {
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(i, &memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(i, &memory)).get()));
|
||||
tv_list.ValueList().emplace_back(i);
|
||||
auto valid_types = MakeListTypes({mgp_type_any(), mgp_type_int(), mgp_type_number()});
|
||||
valid_types.push_back(mgp_type_any());
|
||||
@@ -242,6 +262,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
}
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
|
||||
@@ -251,10 +272,10 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
|
||||
auto *mgp_list_v = mgp_value_make_list(list);
|
||||
query::TypedValue tv_list(std::vector<query::TypedValue>{});
|
||||
// Add an int
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(42, &memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get()));
|
||||
tv_list.ValueList().emplace_back(42);
|
||||
// Add a boolean
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_bool(1, &memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_bool(1, &memory)).get()));
|
||||
tv_list.ValueList().emplace_back(true);
|
||||
auto valid_types = MakeListTypes({mgp_type_any()});
|
||||
valid_types.push_back(mgp_type_any());
|
||||
@@ -264,6 +285,7 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
|
||||
mgp_list_v, tv_list,
|
||||
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
|
||||
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
TEST(CypherType, ListOfNullSatisfiesType) {
|
||||
@@ -271,7 +293,7 @@ TEST(CypherType, ListOfNullSatisfiesType) {
|
||||
auto *list = mgp_list_make_empty(1, &memory);
|
||||
auto *mgp_list_v = mgp_value_make_list(list);
|
||||
query::TypedValue tv_list(std::vector<query::TypedValue>{});
|
||||
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_null(&memory)));
|
||||
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get()));
|
||||
tv_list.ValueList().emplace_back();
|
||||
// List with Null satisfies all nullable list element types
|
||||
std::vector<const mgp_type *> primitive_types{
|
||||
@@ -295,4 +317,5 @@ TEST(CypherType, ListOfNullSatisfiesType) {
|
||||
EXPECT_FALSE(null_type->impl->SatisfiesType(*mgp_list_v)) << null_type->impl->GetPresentableName();
|
||||
EXPECT_FALSE(null_type->impl->SatisfiesType(tv_list));
|
||||
}
|
||||
mgp_value_destroy(mgp_list_v);
|
||||
}
|
||||
|
||||
@@ -254,6 +254,7 @@ TEST(PyModule, PyObjectToMgpValue) {
|
||||
const mgp_value *v2 = mgp_map_at(map, "four");
|
||||
ASSERT_TRUE(mgp_value_is_double(v2));
|
||||
EXPECT_EQ(mgp_value_get_double(v2), 4.0);
|
||||
mgp_value_destroy(value);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
|
||||
@@ -709,14 +709,15 @@ TEST_P(DurabilityTest, SnapshotFallback) {
|
||||
{.items = {.properties_on_edges = GetParam()},
|
||||
.durability = {.storage_directory = storage_directory,
|
||||
.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT,
|
||||
.snapshot_interval = std::chrono::milliseconds(2000)}});
|
||||
.snapshot_interval = std::chrono::milliseconds(3000)}});
|
||||
CreateBaseDataset(&store, GetParam());
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(3500));
|
||||
ASSERT_EQ(GetSnapshotsList().size(), 1);
|
||||
CreateExtendedDataset(&store);
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(3000));
|
||||
}
|
||||
|
||||
ASSERT_GE(GetSnapshotsList().size(), 2);
|
||||
ASSERT_EQ(GetSnapshotsList().size(), 2);
|
||||
ASSERT_EQ(GetBackupSnapshotsList().size(), 0);
|
||||
ASSERT_EQ(GetWalsList().size(), 0);
|
||||
ASSERT_EQ(GetBackupWalsList().size(), 0);
|
||||
@@ -724,7 +725,7 @@ TEST_P(DurabilityTest, SnapshotFallback) {
|
||||
// Destroy last snapshot.
|
||||
{
|
||||
auto snapshots = GetSnapshotsList();
|
||||
ASSERT_GE(snapshots.size(), 2);
|
||||
ASSERT_EQ(snapshots.size(), 2);
|
||||
DestroySnapshot(*snapshots.begin());
|
||||
}
|
||||
|
||||
|
||||
9
tests/unit/test_utils.hpp
Normal file
9
tests/unit/test_utils.hpp
Normal file
@@ -0,0 +1,9 @@
|
||||
#include <memory>
|
||||
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
|
||||
namespace test_utils {
|
||||
using MgpValueOwningPtr = std::unique_ptr<mgp_value, void (*)(mgp_value *)>;
|
||||
|
||||
MgpValueOwningPtr CreateValueOwningPtr(mgp_value *value) { return MgpValueOwningPtr(value, &mgp_value_destroy); }
|
||||
} // namespace test_utils
|
||||
@@ -397,8 +397,8 @@ TEST_F(TypedValueLogicTest, LogicalXor) {
|
||||
|
||||
// NOLINTNEXTLINE(hicpp-special-member-functions)
|
||||
TEST_F(AllTypesFixture, ConstructionWithMemoryResource) {
|
||||
std::vector<TypedValue> values_with_custom_memory;
|
||||
utils::MonotonicBufferResource monotonic_memory(1024);
|
||||
std::vector<TypedValue> values_with_custom_memory;
|
||||
for (const auto &value : values_) {
|
||||
EXPECT_EQ(value.GetMemoryResource(), utils::NewDeleteResource());
|
||||
TypedValue copy_constructed_value(value, &monotonic_memory);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#include "utils/string.hpp"
|
||||
|
||||
class CsvReaderTest : public ::testing::Test {
|
||||
class CsvReaderTest : public ::testing::TestWithParam<const char *> {
|
||||
protected:
|
||||
const std::filesystem::path csv_directory{std::filesystem::temp_directory_path() / "csv_testing"};
|
||||
|
||||
@@ -30,7 +30,9 @@ class CsvReaderTest : public ::testing::Test {
|
||||
namespace {
|
||||
class FileWriter {
|
||||
public:
|
||||
explicit FileWriter(const std::filesystem::path path) { stream_.open(path); }
|
||||
explicit FileWriter(const std::filesystem::path path, std::string newline = "\n") : newline_{std::move(newline)} {
|
||||
stream_.open(path);
|
||||
}
|
||||
|
||||
FileWriter(const FileWriter &) = delete;
|
||||
FileWriter &operator=(const FileWriter &) = delete;
|
||||
@@ -45,7 +47,7 @@ class FileWriter {
|
||||
return 0;
|
||||
}
|
||||
|
||||
stream_ << line << std::endl;
|
||||
stream_ << line << newline_;
|
||||
|
||||
// including the newline character
|
||||
return line.size() + 1;
|
||||
@@ -53,6 +55,7 @@ class FileWriter {
|
||||
|
||||
private:
|
||||
std::ofstream stream_;
|
||||
std::string newline_;
|
||||
};
|
||||
|
||||
std::string CreateRow(const std::vector<std::string> &columns, const std::string_view delim) {
|
||||
@@ -69,10 +72,10 @@ auto ToPmrColumns(const std::vector<std::string> &columns) {
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_F(CsvReaderTest, CommaDelimiter) {
|
||||
TEST_P(CsvReaderTest, CommaDelimiter) {
|
||||
// create a file with a single valid row;
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
const std::vector<std::string> columns{"A", "B", "C"};
|
||||
writer.WriteLine(CreateRow(columns, ","));
|
||||
@@ -93,9 +96,9 @@ TEST_F(CsvReaderTest, CommaDelimiter) {
|
||||
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, SemicolonDelimiter) {
|
||||
TEST_P(CsvReaderTest, SemicolonDelimiter) {
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -116,12 +119,12 @@ TEST_F(CsvReaderTest, SemicolonDelimiter) {
|
||||
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, SkipBad) {
|
||||
TEST_P(CsvReaderTest, SkipBad) {
|
||||
// create a file with invalid first two rows (containing a string with a
|
||||
// missing closing quote);
|
||||
// the last row is valid;
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -161,11 +164,11 @@ TEST_F(CsvReaderTest, SkipBad) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, AllRowsValid) {
|
||||
TEST_P(CsvReaderTest, AllRowsValid) {
|
||||
// create a file with all rows valid;
|
||||
// parser should return 'std::nullopt'
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -190,11 +193,11 @@ TEST_F(CsvReaderTest, AllRowsValid) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, SkipAllRows) {
|
||||
TEST_P(CsvReaderTest, SkipAllRows) {
|
||||
// create a file with all rows invalid (containing a string with a missing closing quote);
|
||||
// parser should return 'std::nullopt'
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -217,9 +220,9 @@ TEST_F(CsvReaderTest, SkipAllRows) {
|
||||
ASSERT_EQ(parsed_row, std::nullopt);
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, WithHeader) {
|
||||
TEST_P(CsvReaderTest, WithHeader) {
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -249,12 +252,12 @@ TEST_F(CsvReaderTest, WithHeader) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, MultilineQuotedString) {
|
||||
TEST_P(CsvReaderTest, MultilineQuotedString) {
|
||||
// create a file with first row valid and the second row containing a quoted
|
||||
// string spanning two lines;
|
||||
// parser should return two valid rows
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -284,11 +287,11 @@ TEST_F(CsvReaderTest, MultilineQuotedString) {
|
||||
ASSERT_EQ(*parsed_row, ToPmrColumns(expected_multiline));
|
||||
}
|
||||
|
||||
TEST_F(CsvReaderTest, EmptyColumns) {
|
||||
TEST_P(CsvReaderTest, EmptyColumns) {
|
||||
// create a file with all rows valid;
|
||||
// parser should return 'std::nullopt'
|
||||
const auto filepath = csv_directory / "bla.csv";
|
||||
auto writer = FileWriter(filepath);
|
||||
auto writer = FileWriter(filepath, GetParam());
|
||||
|
||||
utils::MemoryResource *mem(utils::NewDeleteResource());
|
||||
|
||||
@@ -315,3 +318,5 @@ TEST_F(CsvReaderTest, EmptyColumns) {
|
||||
ASSERT_EQ(*parsed_row, pmr_expected_row);
|
||||
}
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n"));
|
||||
|
||||
@@ -12,6 +12,7 @@ class TestMemory final : public utils::MemoryResource {
|
||||
size_t delete_count_{0};
|
||||
|
||||
private:
|
||||
static constexpr size_t kPadSize = 32;
|
||||
void *DoAllocate(size_t bytes, size_t alignment) override {
|
||||
new_count_++;
|
||||
EXPECT_TRUE(alignment != 0U && (alignment & (alignment - 1U)) == 0U) << "Alignment must be power of 2";
|
||||
@@ -20,11 +21,11 @@ class TestMemory final : public utils::MemoryResource {
|
||||
EXPECT_TRUE(bytes + pad_size > bytes) << "TestMemory size overflow";
|
||||
EXPECT_TRUE(bytes + pad_size + alignment > bytes + alignment) << "TestMemory size overflow";
|
||||
EXPECT_TRUE(2U * alignment > alignment) << "TestMemory alignment overflow";
|
||||
// Allocate a block containing extra alignment and pad_size bytes, but
|
||||
// Allocate a block containing extra alignment and kPadSize bytes, but
|
||||
// aligned to 2 * alignment. Then we can offset the ptr so that it's never
|
||||
// aligned to 2 * alignment. This ought to make allocator alignment issues
|
||||
// more obvious.
|
||||
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + pad_size, 2U * alignment);
|
||||
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + kPadSize, 2U * alignment);
|
||||
// Clear allocated memory to 0xFF, marking the invalid region.
|
||||
memset(ptr, 0xFF, alignment + bytes + pad_size);
|
||||
// Offset the ptr so it's not aligned to 2 * alignment, but still aligned to
|
||||
@@ -39,7 +40,8 @@ class TestMemory final : public utils::MemoryResource {
|
||||
void DoDeallocate(void *ptr, size_t bytes, size_t alignment) override {
|
||||
delete_count_++;
|
||||
// Deallocate the original ptr, before alignment adjustment.
|
||||
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, bytes, alignment);
|
||||
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, alignment + bytes + kPadSize,
|
||||
2U * alignment);
|
||||
}
|
||||
|
||||
bool DoIsEqual(const utils::MemoryResource &other) const noexcept override { return this == &other; }
|
||||
|
||||
68
tools/bench-graph-client/main.py
Executable file
68
tools/bench-graph-client/main.py
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Bench Graph client responsible for sending benchmarking data in JSON format to
|
||||
the Bench Graph server.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from argparse import ArgumentParser
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "")
|
||||
GITHUB_SHA = os.getenv("GITHUB_SHA", "")
|
||||
GITHUB_REF = os.getenv("GITHUB_REF", "")
|
||||
|
||||
BENCH_GRAPH_SERVER_ENDPOINT = os.getenv(
|
||||
"BENCH_GRAPH_SERVER_ENDPOINT",
|
||||
"http://bench-graph-api:9001")
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_args():
|
||||
argp = ArgumentParser(description=__doc__)
|
||||
argp.add_argument("--benchmark-name", type=str, required=True)
|
||||
argp.add_argument("--benchmark-results-path", type=str, required=True)
|
||||
argp.add_argument("--github-run-id", type=int, required=True)
|
||||
argp.add_argument("--github-run-number", type=int, required=True)
|
||||
return argp.parse_args()
|
||||
|
||||
|
||||
def post_measurement(args):
|
||||
with open(args.benchmark_results_path, "r") as f:
|
||||
data = json.load(f)
|
||||
timestamp = datetime.now().timestamp()
|
||||
branch = subprocess.run(
|
||||
["git", "rev-parse", "--abbrev-ref", "HEAD"],
|
||||
stdout=subprocess.PIPE,
|
||||
check=True).stdout.decode("utf-8").strip()
|
||||
req = requests.post(
|
||||
f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements",
|
||||
json={
|
||||
"name": args.benchmark_name,
|
||||
"timestamp": timestamp,
|
||||
"git_repo": GITHUB_REPOSITORY,
|
||||
"git_ref": GITHUB_REF,
|
||||
"git_sha": GITHUB_SHA,
|
||||
"github_run_id": args.github_run_id,
|
||||
"github_run_number": args.github_run_number,
|
||||
"results": data,
|
||||
"git_branch": branch},
|
||||
timeout=1)
|
||||
assert req.status_code == 200, \
|
||||
f"Uploading {args.benchmark_name} data failed."
|
||||
log.info(f"{args.benchmark_name} data sent to "
|
||||
f"{BENCH_GRAPH_SERVER_ENDPOINT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
post_measurement(args)
|
||||
1
tools/bench-graph-client/requirements.txt
Normal file
1
tools/bench-graph-client/requirements.txt
Normal file
@@ -0,0 +1 @@
|
||||
requests==2.25.1
|
||||
269
tools/github/clang-tidy/clang-tidy-diff.py
Executable file
269
tools/github/clang-tidy/clang-tidy-diff.py
Executable file
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
#===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
|
||||
#
|
||||
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
# See https://llvm.org/LICENSE.txt for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
#
|
||||
#===-----------------------------------------------------------------------===#
|
||||
|
||||
r"""
|
||||
ClangTidy Diff Checker
|
||||
======================
|
||||
|
||||
This script reads input from a unified diff, runs clang-tidy on all changed
|
||||
files and outputs clang-tidy warnings in changed lines only. This is useful to
|
||||
detect clang-tidy regressions in the lines touched by a specific patch.
|
||||
Example usage for git/svn users:
|
||||
|
||||
git diff -U0 HEAD^ | clang-tidy-diff.py -p1
|
||||
svn diff --diff-cmd=diff -x-U0 | \
|
||||
clang-tidy-diff.py -fix -checks=-*,modernize-use-override
|
||||
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
is_py2 = sys.version[0] == '2'
|
||||
|
||||
if is_py2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue as queue
|
||||
|
||||
|
||||
def run_tidy(task_queue, lock, timeout):
|
||||
watchdog = None
|
||||
while True:
|
||||
command = task_queue.get()
|
||||
try:
|
||||
proc = subprocess.Popen(command,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
|
||||
if timeout is not None:
|
||||
watchdog = threading.Timer(timeout, proc.kill)
|
||||
watchdog.start()
|
||||
|
||||
stdout, stderr = proc.communicate()
|
||||
|
||||
with lock:
|
||||
sys.stdout.write(stdout.decode('utf-8') + '\n')
|
||||
sys.stdout.flush()
|
||||
if stderr:
|
||||
sys.stderr.write(stderr.decode('utf-8') + '\n')
|
||||
sys.stderr.flush()
|
||||
except Exception as e:
|
||||
with lock:
|
||||
sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
|
||||
finally:
|
||||
with lock:
|
||||
if not (timeout is None or watchdog is None):
|
||||
if not watchdog.is_alive():
|
||||
sys.stderr.write('Terminated by timeout: ' +
|
||||
' '.join(command) + '\n')
|
||||
watchdog.cancel()
|
||||
task_queue.task_done()
|
||||
|
||||
|
||||
def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
|
||||
for _ in range(max_tasks):
|
||||
t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
|
||||
def merge_replacement_files(tmpdir, mergefile):
|
||||
"""Merge all replacement files in a directory into a single file"""
|
||||
# The fixes suggested by clang-tidy >= 4.0.0 are given under
|
||||
# the top level key 'Diagnostics' in the output yaml files
|
||||
mergekey = "Diagnostics"
|
||||
merged = []
|
||||
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
|
||||
content = yaml.safe_load(open(replacefile, 'r'))
|
||||
if not content:
|
||||
continue # Skip empty files.
|
||||
merged.extend(content.get(mergekey, []))
|
||||
|
||||
if merged:
|
||||
# MainSourceFile: The key is required by the definition inside
|
||||
# include/clang/Tooling/ReplacementsYaml.h, but the value
|
||||
# is actually never used inside clang-apply-replacements,
|
||||
# so we set it to '' here.
|
||||
output = {'MainSourceFile': '', mergekey: merged}
|
||||
with open(mergefile, 'w') as out:
|
||||
yaml.safe_dump(output, out)
|
||||
else:
|
||||
# Empty the file:
|
||||
open(mergefile, 'w').close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=
|
||||
'Run clang-tidy against changed files, and '
|
||||
'output diagnostics only for modified '
|
||||
'lines.')
|
||||
parser.add_argument('-clang-tidy-binary', metavar='PATH',
|
||||
default='clang-tidy',
|
||||
help='path to clang-tidy binary')
|
||||
parser.add_argument('-p', metavar='NUM', default=0,
|
||||
help='strip the smallest prefix containing P slashes')
|
||||
parser.add_argument('-regex', metavar='PATTERN', default=None,
|
||||
help='custom pattern selecting file paths to check '
|
||||
'(case sensitive, overrides -iregex)')
|
||||
parser.add_argument('-iregex', metavar='PATTERN', default=
|
||||
r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
|
||||
help='custom pattern selecting file paths to check '
|
||||
'(case insensitive, overridden by -regex)')
|
||||
parser.add_argument('-j', type=int, default=1,
|
||||
help='number of tidy instances to be run in parallel.')
|
||||
parser.add_argument('-timeout', type=int, default=None,
|
||||
help='timeout per each file in seconds.')
|
||||
parser.add_argument('-fix', action='store_true', default=False,
|
||||
help='apply suggested fixes')
|
||||
parser.add_argument('-checks',
|
||||
help='checks filter, when not specified, use clang-tidy '
|
||||
'default',
|
||||
default='')
|
||||
parser.add_argument('-path', dest='build_path',
|
||||
help='Path used to read a compile command database.')
|
||||
if yaml:
|
||||
parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
|
||||
help='Create a yaml file to store suggested fixes in, '
|
||||
'which can be applied with clang-apply-replacements.')
|
||||
parser.add_argument('-extra-arg', dest='extra_arg',
|
||||
action='append', default=[],
|
||||
help='Additional argument to append to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
|
||||
action='append', default=[],
|
||||
help='Additional argument to prepend to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-quiet', action='store_true', default=False,
|
||||
help='Run clang-tidy in quiet mode')
|
||||
clang_tidy_args = []
|
||||
argv = sys.argv[1:]
|
||||
if '--' in argv:
|
||||
clang_tidy_args.extend(argv[argv.index('--'):])
|
||||
argv = argv[:argv.index('--')]
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
# Extract changed lines for each file.
|
||||
filename = None
|
||||
lines_by_file = {}
|
||||
for line in sys.stdin:
|
||||
match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
|
||||
if match:
|
||||
filename = match.group(2)
|
||||
if filename is None:
|
||||
continue
|
||||
|
||||
if args.regex is not None:
|
||||
if not re.match('^%s$' % args.regex, filename):
|
||||
continue
|
||||
else:
|
||||
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
|
||||
continue
|
||||
|
||||
match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
|
||||
if match:
|
||||
start_line = int(match.group(1))
|
||||
line_count = 1
|
||||
if match.group(3):
|
||||
line_count = int(match.group(3))
|
||||
if line_count == 0:
|
||||
continue
|
||||
end_line = start_line + line_count - 1
|
||||
lines_by_file.setdefault(filename, []).append([start_line, end_line])
|
||||
|
||||
if not any(lines_by_file):
|
||||
print("No relevant changes found.")
|
||||
sys.exit(0)
|
||||
|
||||
max_task_count = args.j
|
||||
if max_task_count == 0:
|
||||
max_task_count = multiprocessing.cpu_count()
|
||||
max_task_count = min(len(lines_by_file), max_task_count)
|
||||
|
||||
tmpdir = None
|
||||
if yaml and args.export_fixes:
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
# Tasks for clang-tidy.
|
||||
task_queue = queue.Queue(max_task_count)
|
||||
# A lock for console output.
|
||||
lock = threading.Lock()
|
||||
|
||||
# Run a pool of clang-tidy workers.
|
||||
start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
|
||||
|
||||
# Form the common args list.
|
||||
common_clang_tidy_args = []
|
||||
if args.fix:
|
||||
common_clang_tidy_args.append('-fix')
|
||||
if args.checks != '':
|
||||
common_clang_tidy_args.append('-checks=' + args.checks)
|
||||
if args.quiet:
|
||||
common_clang_tidy_args.append('-quiet')
|
||||
if args.build_path is not None:
|
||||
common_clang_tidy_args.append('-p=%s' % args.build_path)
|
||||
for arg in args.extra_arg:
|
||||
common_clang_tidy_args.append('-extra-arg=%s' % arg)
|
||||
for arg in args.extra_arg_before:
|
||||
common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
|
||||
|
||||
for name in lines_by_file:
|
||||
line_filter_json = json.dumps(
|
||||
[{"name": name, "lines": lines_by_file[name]}],
|
||||
separators=(',', ':'))
|
||||
|
||||
# Run clang-tidy on files containing changes.
|
||||
command = [args.clang_tidy_binary]
|
||||
command.append('-line-filter=' + line_filter_json)
|
||||
if yaml and args.export_fixes:
|
||||
# Get a temporary file. We immediately close the handle so clang-tidy can
|
||||
# overwrite it.
|
||||
(handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
|
||||
os.close(handle)
|
||||
command.append('-export-fixes=' + tmp_name)
|
||||
command.extend(common_clang_tidy_args)
|
||||
command.append(name)
|
||||
command.extend(clang_tidy_args)
|
||||
|
||||
task_queue.put(command)
|
||||
|
||||
# Wait for all threads to be done.
|
||||
task_queue.join()
|
||||
|
||||
if yaml and args.export_fixes:
|
||||
print('Writing fixes to ' + args.export_fixes + ' ...')
|
||||
try:
|
||||
merge_replacement_files(tmpdir, args.export_fixes)
|
||||
except:
|
||||
sys.stderr.write('Error exporting fixes.\n')
|
||||
traceback.print_exc()
|
||||
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
9
tools/github/clang-tidy/count_errors.sh
Executable file
9
tools/github/clang-tidy/count_errors.sh
Executable file
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
|
||||
# the first sort | uniq is necessary, because the same occurrence of the same error
|
||||
# can be reported from headers when they are included in multiple source files
|
||||
`dirname ${BASH_SOURCE[0]}`/grep_error_lines.sh |
|
||||
sort | uniq |
|
||||
sed -E 's/.*\[(.*)\]\r?$/\1/g' | # extract the check name from [check-name]
|
||||
sort | uniq -c | # count each type of check
|
||||
sort -nr # sort them into descending order
|
||||
12
tools/github/clang-tidy/grep_error_lines.sh
Executable file
12
tools/github/clang-tidy/grep_error_lines.sh
Executable file
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Matches timestamp like "2021-03-25T17:06:42.2621697Z"
|
||||
TIMESTAMP_PATTERN="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z"
|
||||
|
||||
# Matches absolute file pathes with line and column identifier like
|
||||
# "/opt/actions-runner/_work/memgraph/memgraph/src/utils/exceptions.hpp:71:11:"
|
||||
FILE_ABSOLUTE_PATH_PATTERN="/[^:]+:\d+:\d+:"
|
||||
|
||||
ERROR_OR_WARNING_PATTERN="(error|warning):"
|
||||
|
||||
grep -P "^($TIMESTAMP_PATTERN )?$FILE_ABSOLUTE_PATH_PATTERN $ERROR_OR_WARNING_PATTERN.*$"
|
||||
337
tools/github/clang-tidy/run-clang-tidy.py
Executable file
337
tools/github/clang-tidy/run-clang-tidy.py
Executable file
@@ -0,0 +1,337 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
#===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
|
||||
#
|
||||
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
# See https://llvm.org/LICENSE.txt for license information.
|
||||
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
#
|
||||
#===-----------------------------------------------------------------------===#
|
||||
# FIXME: Integrate with clang-tidy-diff.py
|
||||
|
||||
|
||||
"""
|
||||
Parallel clang-tidy runner
|
||||
==========================
|
||||
|
||||
Runs clang-tidy over all files in a compilation database. Requires clang-tidy
|
||||
and clang-apply-replacements in $PATH.
|
||||
|
||||
Example invocations.
|
||||
- Run clang-tidy on all files in the current working directory with a default
|
||||
set of checks and show warnings in the cpp files and all project headers.
|
||||
run-clang-tidy.py $PWD
|
||||
|
||||
- Fix all header guards.
|
||||
run-clang-tidy.py -fix -checks=-*,llvm-header-guard
|
||||
|
||||
- Fix all header guards included from clang-tidy and header guards
|
||||
for clang-tidy headers.
|
||||
run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
|
||||
-header-filter=extra/clang-tidy
|
||||
|
||||
Compilation database setup:
|
||||
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
yaml = None
|
||||
|
||||
is_py2 = sys.version[0] == '2'
|
||||
|
||||
if is_py2:
|
||||
import Queue as queue
|
||||
else:
|
||||
import queue as queue
|
||||
|
||||
|
||||
def find_compilation_database(path):
|
||||
"""Adjusts the directory until a compilation database is found."""
|
||||
result = './'
|
||||
while not os.path.isfile(os.path.join(result, path)):
|
||||
if os.path.realpath(result) == '/':
|
||||
print('Error: could not find compilation database.')
|
||||
sys.exit(1)
|
||||
result += '../'
|
||||
return os.path.realpath(result)
|
||||
|
||||
|
||||
def make_absolute(f, directory):
|
||||
if os.path.isabs(f):
|
||||
return f
|
||||
return os.path.normpath(os.path.join(directory, f))
|
||||
|
||||
|
||||
def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
|
||||
header_filter, allow_enabling_alpha_checkers,
|
||||
extra_arg, extra_arg_before, quiet, config):
|
||||
"""Gets a command line for clang-tidy."""
|
||||
start = [clang_tidy_binary]
|
||||
if allow_enabling_alpha_checkers:
|
||||
start.append('-allow-enabling-analyzer-alpha-checkers')
|
||||
if header_filter is not None:
|
||||
start.append('-header-filter=' + header_filter)
|
||||
if checks:
|
||||
start.append('-checks=' + checks)
|
||||
if tmpdir is not None:
|
||||
start.append('-export-fixes')
|
||||
# Get a temporary file. We immediately close the handle so clang-tidy can
|
||||
# overwrite it.
|
||||
(handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
|
||||
os.close(handle)
|
||||
start.append(name)
|
||||
for arg in extra_arg:
|
||||
start.append('-extra-arg=%s' % arg)
|
||||
for arg in extra_arg_before:
|
||||
start.append('-extra-arg-before=%s' % arg)
|
||||
start.append('-p=' + build_path)
|
||||
if quiet:
|
||||
start.append('-quiet')
|
||||
if config:
|
||||
start.append('-config=' + config)
|
||||
start.append(f)
|
||||
return start
|
||||
|
||||
|
||||
def merge_replacement_files(tmpdir, mergefile):
|
||||
"""Merge all replacement files in a directory into a single file"""
|
||||
# The fixes suggested by clang-tidy >= 4.0.0 are given under
|
||||
# the top level key 'Diagnostics' in the output yaml files
|
||||
mergekey = "Diagnostics"
|
||||
merged=[]
|
||||
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
|
||||
content = yaml.safe_load(open(replacefile, 'r'))
|
||||
if not content:
|
||||
continue # Skip empty files.
|
||||
merged.extend(content.get(mergekey, []))
|
||||
|
||||
if merged:
|
||||
# MainSourceFile: The key is required by the definition inside
|
||||
# include/clang/Tooling/ReplacementsYaml.h, but the value
|
||||
# is actually never used inside clang-apply-replacements,
|
||||
# so we set it to '' here.
|
||||
output = {'MainSourceFile': '', mergekey: merged}
|
||||
with open(mergefile, 'w') as out:
|
||||
yaml.safe_dump(output, out)
|
||||
else:
|
||||
# Empty the file:
|
||||
open(mergefile, 'w').close()
|
||||
|
||||
|
||||
def check_clang_apply_replacements_binary(args):
|
||||
"""Checks if invoking supplied clang-apply-replacements binary works."""
|
||||
try:
|
||||
subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
|
||||
except:
|
||||
print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
|
||||
'binary correctly specified?', file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def apply_fixes(args, tmpdir):
|
||||
"""Calls clang-apply-fixes on a given directory."""
|
||||
invocation = [args.clang_apply_replacements_binary]
|
||||
if args.format:
|
||||
invocation.append('-format')
|
||||
if args.style:
|
||||
invocation.append('-style=' + args.style)
|
||||
invocation.append(tmpdir)
|
||||
subprocess.call(invocation)
|
||||
|
||||
|
||||
def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
|
||||
"""Takes filenames out of queue and runs clang-tidy on them."""
|
||||
while True:
|
||||
name = queue.get()
|
||||
invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
|
||||
tmpdir, build_path, args.header_filter,
|
||||
args.allow_enabling_alpha_checkers,
|
||||
args.extra_arg, args.extra_arg_before,
|
||||
args.quiet, args.config)
|
||||
|
||||
proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
output, err = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
failed_files.append(name)
|
||||
with lock:
|
||||
sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
|
||||
if len(err) > 0:
|
||||
sys.stdout.flush()
|
||||
sys.stderr.write(err.decode('utf-8'))
|
||||
queue.task_done()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
|
||||
'in a compilation database. Requires '
|
||||
'clang-tidy and clang-apply-replacements in '
|
||||
'$PATH.')
|
||||
parser.add_argument('-allow-enabling-alpha-checkers',
|
||||
action='store_true', help='allow alpha checkers from '
|
||||
'clang-analyzer.')
|
||||
parser.add_argument('-clang-tidy-binary', metavar='PATH',
|
||||
default='clang-tidy-11',
|
||||
help='path to clang-tidy binary')
|
||||
parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
|
||||
default='clang-apply-replacements-11',
|
||||
help='path to clang-apply-replacements binary')
|
||||
parser.add_argument('-checks', default=None,
|
||||
help='checks filter, when not specified, use clang-tidy '
|
||||
'default')
|
||||
parser.add_argument('-config', default=None,
|
||||
help='Specifies a configuration in YAML/JSON format: '
|
||||
' -config="{Checks: \'*\', '
|
||||
' CheckOptions: [{key: x, '
|
||||
' value: y}]}" '
|
||||
'When the value is empty, clang-tidy will '
|
||||
'attempt to find a file named .clang-tidy for '
|
||||
'each source file in its parent directories.')
|
||||
parser.add_argument('-header-filter', default=None,
|
||||
help='regular expression matching the names of the '
|
||||
'headers to output diagnostics from. Diagnostics from '
|
||||
'the main file of each translation unit are always '
|
||||
'displayed.')
|
||||
if yaml:
|
||||
parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
|
||||
help='Create a yaml file to store suggested fixes in, '
|
||||
'which can be applied with clang-apply-replacements.')
|
||||
parser.add_argument('-j', type=int, default=0,
|
||||
help='number of tidy instances to be run in parallel.')
|
||||
parser.add_argument('files', nargs='*', default=['.*'],
|
||||
help='files to be processed (regex on path)')
|
||||
parser.add_argument('-fix', action='store_true', help='apply fix-its')
|
||||
parser.add_argument('-format', action='store_true', help='Reformat code '
|
||||
'after applying fixes')
|
||||
parser.add_argument('-style', default='file', help='The style of reformat '
|
||||
'code after applying fixes')
|
||||
parser.add_argument('-p', dest='build_path',
|
||||
help='Path used to read a compile command database.')
|
||||
parser.add_argument('-extra-arg', dest='extra_arg',
|
||||
action='append', default=[],
|
||||
help='Additional argument to append to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
|
||||
action='append', default=[],
|
||||
help='Additional argument to prepend to the compiler '
|
||||
'command line.')
|
||||
parser.add_argument('-quiet', action='store_true',
|
||||
help='Run clang-tidy in quiet mode')
|
||||
args = parser.parse_args()
|
||||
|
||||
db_path = 'compile_commands.json'
|
||||
|
||||
if args.build_path is not None:
|
||||
build_path = args.build_path
|
||||
else:
|
||||
# Find our database
|
||||
build_path = find_compilation_database(db_path)
|
||||
|
||||
try:
|
||||
invocation = [args.clang_tidy_binary, '-list-checks']
|
||||
if args.allow_enabling_alpha_checkers:
|
||||
invocation.append('-allow-enabling-analyzer-alpha-checkers')
|
||||
invocation.append('-p=' + build_path)
|
||||
if args.checks:
|
||||
invocation.append('-checks=' + args.checks)
|
||||
invocation.append('-')
|
||||
if args.quiet:
|
||||
# Even with -quiet we still want to check if we can call clang-tidy.
|
||||
with open(os.devnull, 'w') as dev_null:
|
||||
subprocess.check_call(invocation, stdout=dev_null)
|
||||
else:
|
||||
subprocess.check_call(invocation)
|
||||
except:
|
||||
print("Unable to run clang-tidy.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Load the database and extract all files.
|
||||
database = json.load(open(os.path.join(build_path, db_path)))
|
||||
files = [make_absolute(entry['file'], entry['directory'])
|
||||
for entry in database]
|
||||
|
||||
max_task = args.j
|
||||
if max_task == 0:
|
||||
max_task = multiprocessing.cpu_count()
|
||||
|
||||
tmpdir = None
|
||||
if args.fix or (yaml and args.export_fixes):
|
||||
check_clang_apply_replacements_binary(args)
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
|
||||
# Build up a big regexy filter from all command line arguments.
|
||||
file_name_re = re.compile('|'.join(args.files))
|
||||
|
||||
return_code = 0
|
||||
try:
|
||||
# Spin up a bunch of tidy-launching threads.
|
||||
task_queue = queue.Queue(max_task)
|
||||
# List of files with a non-zero return code.
|
||||
failed_files = []
|
||||
lock = threading.Lock()
|
||||
for _ in range(max_task):
|
||||
t = threading.Thread(target=run_tidy,
|
||||
args=(args, tmpdir, build_path, task_queue, lock, failed_files))
|
||||
t.daemon = True
|
||||
t.start()
|
||||
|
||||
# Fill the queue with files.
|
||||
for name in files:
|
||||
if file_name_re.search(name):
|
||||
task_queue.put(name)
|
||||
|
||||
# Wait for all threads to be done.
|
||||
task_queue.join()
|
||||
if len(failed_files):
|
||||
return_code = 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
# This is a sad hack. Unfortunately subprocess goes
|
||||
# bonkers with ctrl-c and we start forking merrily.
|
||||
print('\nCtrl-C detected, goodbye.')
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
os.kill(0, 9)
|
||||
|
||||
if yaml and args.export_fixes:
|
||||
print('Writing fixes to ' + args.export_fixes + ' ...')
|
||||
try:
|
||||
merge_replacement_files(tmpdir, args.export_fixes)
|
||||
except:
|
||||
print('Error exporting fixes.\n', file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
return_code=1
|
||||
|
||||
if args.fix:
|
||||
print('Applying fixes ...')
|
||||
try:
|
||||
apply_fixes(args, tmpdir)
|
||||
except:
|
||||
print('Error applying fixes.\n', file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
return_code = 1
|
||||
|
||||
if tmpdir:
|
||||
shutil.rmtree(tmpdir)
|
||||
sys.exit(return_code)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
12
tools/lsan.supp
Normal file
12
tools/lsan.supp
Normal file
@@ -0,0 +1,12 @@
|
||||
leak:antlr4::atn::ArrayPredictionContext::ArrayPredictionContext
|
||||
leak:std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<antlr4::atn::SingletonPredictionContext, std::allocator<antlr4::atn::SingletonPredictionContext>, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&>(antlr4::atn::SingletonPredictionContext*&, std::_Sp_alloc_shared_tag<std::allocator<antlr4::atn::SingletonPredictionContext> >, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&)
|
||||
leak:antlr4::atn::PredictionContext::mergeSingletons(std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
|
||||
leak:void std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::PredictionContext> >(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::PredictionContext>*, std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > > >, std::shared_ptr<antlr4::atn::PredictionContext>&&)
|
||||
leak:antlr4::atn::ParserATNSimulator::closureCheckingStopState(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::ATNConfigSet*, std::unordered_set<std::shared_ptr<antlr4::atn::ATNConfig>, antlr4::atn::ATNConfig::Hasher, antlr4::atn::ATNConfig::Comparer, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >&, bool, bool, int, bool)
|
||||
leak:antlr4::atn::ParserATNSimulator::computeReachSet(antlr4::atn::ATNConfigSet*, unsigned long, bool)
|
||||
leak:std::_Hashtable<unsigned long, std::pair<unsigned long const, antlr4::atn::ATNConfig*>, std::allocator<std::pair<unsigned long const, antlr4::atn::ATNConfig*> >, std::__detail::_Select1st, std::equal_to<unsigned long>, std::hash<unsigned long>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node(unsigned long const&, unsigned long, unsigned long, std::__detail::_Hash_node<std::pair<unsigned long const, antlr4::atn::ATNConfig*>, false>*, unsigned long)
|
||||
leak:void std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::ATNConfig> const&>(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::ATNConfig>*, std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > > >, std::shared_ptr<antlr4::atn::ATNConfig> const&)
|
||||
leak:antlr4::atn::ATNConfigSet::add(std::shared_ptr<antlr4::atn::ATNConfig> const&, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
|
||||
leak:antlr4::atn::ParserATNSimulator::getEpsilonTarget(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::Transition*, bool, bool, bool, bool)
|
||||
leak:antlr4::atn::PredictionContext::mergeArrays(std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
|
||||
leak:/lib/x86_64-linux-gnu/libpython3.
|
||||
Reference in New Issue
Block a user