Compare commits

...

12 Commits

Author SHA1 Message Date
antonio2368
999b3ef79f Fix memory tracking issues (#150) 2021-05-18 18:42:19 +02:00
antonio2368
30413a7b4f Ignore carriage return at the end of line for a CSV file (#151) 2021-05-18 13:44:29 +02:00
Marko Budiselić
1def0c9104 Fix libs/setup.sh nocached download (#148) 2021-05-13 12:05:36 +02:00
Marko Budiselić
782c377f5d Add benchmarking GHA Diff job (#147) 2021-05-12 17:17:57 +02:00
Marko Budiselić
cc27a04139 Add bench-graph-client inside tools (#139) 2021-05-07 15:54:09 +02:00
Marko Budiselić
b71345655f Add balancing of GHA jobs across runners (#146) 2021-05-07 13:37:45 +02:00
antonio2368
ccdd58b336 Fix parsing nested complex types (#142)
* Fix parsing of types for mgp.List
2021-04-27 08:31:18 +02:00
antonio2368
50b6afd73d Fix clang tidy diff (#141)
* Ignore header files for clang-tidy-diff

* Rename code analysis job
2021-04-21 16:26:13 +02:00
Marko Budiselić
59105f68bd Add primary and secondary dependency links (#137)
* Add explicit exit in case of git checkout or cherry-pick
2021-04-20 10:54:54 +02:00
antonio2368
8de31092ad Add back the query memory limit logic (#134) 2021-04-13 10:41:50 +02:00
antonio2368
5c93f81881 Disable failing tests and add logs for replication e2e (#132)
* Disable sequential test

* Remove parent build and benchmark

* Save test data

* Save e2e logs in build folder

* Define different recovery time for each test
2021-04-02 12:29:10 +02:00
János Benjamin Antal
6d4fe5cdd5 Explore and implement the usage of clang-tidy and sanitizers (#125)
* Run clang-tidy on the full code base

* Run clang-tidy on diffs

* Enable ASAN in coverage build

* Add UBSAN to code analysis
2021-04-01 17:08:40 +02:00
36 changed files with 1333 additions and 229 deletions

View File

@@ -54,7 +54,7 @@ Checks: '*,
-readability-magic-numbers,
-readability-named-parameter'
WarningsAsErrors: ''
HeaderFilterRegex: ''
HeaderFilterRegex: 'src/.*'
AnalyzeTemporaryDtors: false
FormatStyle: none
CheckOptions:

View File

@@ -10,7 +10,7 @@ on:
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -65,9 +65,9 @@ jobs:
name: "Community DEB package"
path: build/output/memgraph*.deb
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
code_analysis:
name: "Code analysis"
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -79,7 +79,7 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build coverage binaries
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
@@ -87,9 +87,8 @@ jobs:
# Initialize dependencies.
./init
# Build coverage binaries.
cd build
cmake -DTEST_COVERAGE=ON ..
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
- name: Run unit tests
@@ -97,9 +96,9 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run unit tests.
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
- name: Compute code coverage
run: |
@@ -120,9 +119,19 @@ jobs:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
debug_build:
name: "Debug build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -196,7 +205,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
@@ -208,21 +217,6 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Set up parent
run: |
# Remove parent folder (if it exists).
cd ..
if [ -d parent ]; then
rm -rf parent
fi
# Copy untouched repository to parent folder.
cp -r memgraph parent
# Checkout previous commit
cd parent
git checkout HEAD~1
- name: Build release binaries
run: |
# Activate toolchain.
@@ -236,47 +230,6 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Build parent binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
cd ../parent
./init
# Build parent binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph memgraph__macro_benchmark
- name: Run macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Run parent macro benchmark tests
run: |
cd ../parent/tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Compute macro benchmark summary
run: |
./tools/github/macro_benchmark_summary \
--current tests/macro_benchmark/.harness_summary \
--previous ../parent/tests/macro_benchmark/.harness_summary \
--output macro_benchmark_summary.txt
- name: Save macro benchmark summary
uses: actions/upload-artifact@v2
with:
name: "Macro benchmark summary"
path: macro_benchmark_summary.txt
- name: Run GQL Behave tests
run: |
cd tests/gql_behave
@@ -339,6 +292,15 @@ jobs:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
path: |
# multiple paths could be defined
build/logs
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
@@ -378,3 +340,64 @@ jobs:
with:
name: "Jepsen Report"
path: tests/jepsen/Jepsen.tar.gz
release_benchmarks:
name: "Release benchmarks"
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build only memgraph release binarie.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run macro benchmarks
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"

44
.github/workflows/full_clang_tidy.yaml vendored Normal file
View File

@@ -0,0 +1,44 @@
name: Run clang-tidy on the full codebase
on:
workflow_dispatch:
jobs:
clang_tidy_check:
name: "Clang-tidy check"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results
run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

View File

@@ -2,6 +2,16 @@
## Future
### Bug Fixes
* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
For example, parsing of `mgp.List[mgp.Map]` works now.
* Fixed memory tracking issues. Some of the allocation and deallocation weren't
tracked during the query execution.
* Fixed reading CSV files that are using CRLF as the newline symbol.
## v1.4.0
### Breaking Changes
* Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.

View File

@@ -312,8 +312,9 @@ if (UBSAN)
# runtime library and c++ standard libraries are present.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
# Make sure llvm-symbolizer binary is in path
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
# Make sure llvm-symbolizer binary is in path.
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
endif()
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")

View File

@@ -683,7 +683,15 @@ def _typing_to_cypher_type(type_):
return _mgp.type_nullable(simple_type)
return _mgp.type_nullable(parse_typing(type_arg_as_str))
elif type_as_str.startswith('typing.List'):
type_arg_as_str, = parse_type_args(type_as_str)
type_arg_as_str = parse_type_args(type_as_str)
if len(type_arg_as_str) > 1:
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
# so we need to join the parts.
type_arg_as_str = ', '.join(type_arg_as_str)
else:
type_arg_as_str = type_arg_as_str[0]
simple_type = get_simple_type(type_arg_as_str)
if simple_type is not None:
return _mgp.type_list(simple_type)

View File

@@ -2,8 +2,9 @@
# Download external dependencies.
local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd ${working_dir}
cd "${working_dir}"
# Clones a git repository and optionally cherry picks additional commits. The
# function will try to preserve any local changes in the repo.
@@ -15,7 +16,11 @@ clone () {
shift 3
# Clone if there's no repo.
if [[ ! -d "$dir_name" ]]; then
git clone "$git_repo" "$dir_name"
echo "Cloning from $git_repo"
# If the clone fails, it doesn't make sense to continue with the function
# execution but the whole script should continue executing because we might
# clone the same repo from a different source.
git clone "$git_repo" "$dir_name" || return 1
fi
pushd "$dir_name"
# Just fetch new commits from remote repository. Don't merge/pull them in, so
@@ -29,12 +34,17 @@ clone () {
# Stash regardless of local_changes, so that a user gets a message on stdout.
git stash
# Checkout the primary commit (there's no need to pull/merge).
git checkout $checkout_id
# The checkout fail should exit this script immediately because the target
# commit is not there and that will most likely create build-time errors.
git checkout "$checkout_id" || exit 1
# Apply any optional cherry pick fixes.
while [[ $# -ne 0 ]]; do
local cherry_pick_id=$1
shift
git cherry-pick -n $cherry_pick_id
# The cherry-pick fail should exit this script immediately because the
# target commit is not there and that will most likely create build-time
# errors.
git cherry-pick -n "$cherry_pick_id" || exit 1
done
# Reapply any local changes.
if [[ $local_changes == true ]]; then
@@ -43,12 +53,95 @@ clone () {
popd
}
file_get_try_double () {
primary_url="$1"
secondary_url="$2"
echo "Download primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
filename="$(basename "$secondary_url")"
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
echo ""
}
repo_clone_try_double () {
primary_url="$1"
secondary_url="$2"
folder_name="$3"
ref="$4"
echo "Cloning primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
echo ""
}
# List all dependencies.
# The reason for introducing primary and secondary urls are:
# * HTTPS is hard to cache
# * Remote development workflow is more flexible if people don't have to connect to VPN
# * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
# * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
# The goal of having primary urls is to have links to the "local" cache of
# dependencies where these dependencies could be downloaded as fast as
# possible. The actual cache server could be on your local machine, on a
# dedicated machine inside the build cluster or on the actual build machine.
# Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["fmt"]="http://$local_cache_host/git/fmt.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
["gtest"]="http://$local_cache_host/git/googletest.git"
["gflags"]="http://$local_cache_host/git/gflags.git"
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
["bzip2"]="http://$local_cache_host/git/bzip2.git"
["zlib"]="http://$local_cache_host/git/zlib.git"
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
["mgclient"]="http://$local_cache_host/git/mgclient.git"
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
)
# The goal of secondary urls is to have links to the "source of truth" of
# dependencies, e.g., Github or S3. Download from secondary urls, if happens
# at all, should never fail. In other words, if it fails, the whole build
# should fail.
declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["fmt"]="https://github.com/fmtlib/fmt.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
["gtest"]="https://github.com/google/googletest.git"
["gflags"]="https://github.com/memgraph/gflags.git"
["libbcrypt"]="https://github.com/rg3/libbcrypt"
["bzip2"]="https://github.com/VFR-maniac/bzip2"
["zlib"]="https://github.com/madler/zlib.git"
["rocksdb"]="https://github.com/facebook/rocksdb.git"
["mgclient"]="https://github.com/memgraph/mgclient.git"
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["spdlog"]="https://github.com/gabime/spdlog"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
)
# antlr
antlr_generator_filename="antlr-4.6-complete.jar"
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
# fix missing include
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
# remove shared library from install dependencies
@@ -56,74 +149,73 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
# fmt
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
# rapidcheck
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
# google benchmark
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
clone https://github.com/google/benchmark.git benchmark $benchmark_tag
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
# google test
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
clone https://github.com/google/googletest.git googletest $googletest_tag
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
# google flags
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
clone https://github.com/memgraph/gflags.git gflags $gflags_tag
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
# libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
# neo4j
wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
tar -xzf neo4j.tar.gz
rm -rf neo4j
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
tar -xzf neo4j-community-3.2.3-unix.tar.gz
mv neo4j-community-3.2.3 neo4j
rm neo4j.tar.gz
rm neo4j-community-3.2.3-unix.tar.gz
# nlohmann json
# We wget header instead of cloning repo since repo is huge (lots of test data).
# We use head on Sep 1, 2017 instead of last release since it was long time ago.
mkdir -p json
cd json
wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
clone https://github.com/madler/zlib.git zlib $zlib_tag
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
# remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
# remove shared library from install dependencies
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
# mgclient
mgclient_tag="v1.2.0" # (2021-01-14)
clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
# pymgclient
pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
clone https://github.com/gabime/spdlog spdlog $spdlog_tag
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
clone https://github.com/jemalloc/jemalloc.git jemalloc $jemalloc_tag
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
pushd jemalloc
# ThreadPool select job randomly, and there can be some threads that had been
# performed some memory heavy task before and will be inactive for some time,
@@ -138,5 +230,5 @@ pushd jemalloc
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000"
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
popd

View File

@@ -27,6 +27,7 @@
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/readable_size.hpp"
#include "utils/string.hpp"
#include "utils/tsc.hpp"
@@ -603,8 +604,8 @@ struct PullPlanVector {
struct PullPlan {
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory);
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<size_t> memory_limit = {});
std::optional<ExecutionContext> Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary);
@@ -614,6 +615,7 @@ struct PullPlan {
plan::UniqueCursorPtr cursor_ = nullptr;
Frame frame_;
ExecutionContext ctx_;
std::optional<size_t> memory_limit_;
// As it's possible to query execution using multiple pulls
// we need the keep track of the total execution time across
@@ -630,11 +632,12 @@ struct PullPlan {
};
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory)
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
const std::optional<size_t> memory_limit)
: plan_(plan),
cursor_(plan->plan().MakeCursor(execution_memory)),
frame_(plan->symbol_table().max_position(), execution_memory) {
frame_(plan->symbol_table().max_position(), execution_memory),
memory_limit_(memory_limit) {
ctx_.db_accessor = dba;
ctx_.symbol_table = plan->symbol_table();
ctx_.evaluation_context.timestamp =
@@ -657,21 +660,25 @@ std::optional<ExecutionContext> PullPlan::Pull(AnyStream *stream, std::optional<
// single `Pull`.
constexpr size_t stack_size = 256 * 1024;
char stack_data[stack_size];
utils::ResourceWithOutOfMemoryException resource_with_exception;
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
// We can throw on every query because a simple queries for deleting will use only
// the stack allocated buffer.
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
if (memory_limit_) {
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
ctx_.evaluation_context.memory = &*maybe_limited_resource;
} else {
ctx_.evaluation_context.memory = &pool_memory;
}
// Returns true if a result was pulled.
const auto pull_result = [&]() -> bool {
// We can throw on every query because a simple queries for deleting will use only
// the stack allocated buffer.
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
utils::ResourceWithOutOfMemoryException resource_with_exception;
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
ctx_.evaluation_context.memory = &pool_memory;
return cursor_->Pull(frame_, ctx_);
};
const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); };
const auto stream_values = [&]() {
// TODO: The streamed values should also probably use the above memory.
@@ -828,10 +835,24 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MonotonicBufferResource *execution_memory) {
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage),
utils::Downcast<CypherQuery>(parsed_query.query), parsed_query.parameters,
&interpreter_context->plan_cache, dba, parsed_query.is_cacheable);
utils::MemoryResource *execution_memory) {
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
Frame frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
evaluation_context.parameters = parsed_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
if (memory_limit) {
spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit));
}
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
parsed_query.parameters, &interpreter_context->plan_cache, dba);
summary->insert_or_assign("cost_estimate", plan->cost());
auto rw_type_checker = plan::ReadWriteTypeChecker();
@@ -850,8 +871,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
}
auto pull_plan =
std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory);
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, memory_limit);
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
@@ -865,7 +886,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MonotonicBufferResource *execution_memory) {
utils::MemoryResource *execution_memory) {
const std::string kExplainQueryStart = "explain ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
"Expected stripped query to start with '{}'", kExplainQueryStart);
@@ -911,7 +932,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
DbAccessor *dba, utils::MemoryResource *execution_memory) {
const std::string kProfileQueryStart = "profile ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -949,6 +970,15 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
Frame frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
evaluation_context.parameters = parsed_inner_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
@@ -960,14 +990,14 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), summary, dba,
interpreter_context, execution_memory,
interpreter_context, execution_memory, memory_limit,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
ctx = std::optional<ExecutionContext>{}, pull_plan = std::shared_ptr<PullPlanVector>(nullptr)](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
// No output symbols are given so that nothing is streamed.
if (!ctx) {
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory)
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, memory_limit)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(ctx->stats, ctx->profile_execution_time));
}
@@ -985,7 +1015,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
}
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
utils::MonotonicBufferResource *execution_memory) {
utils::MemoryResource *execution_memory) {
return PreparedQuery{{"QUERY"},
std::move(parsed_query.required_privileges),
[pull_plan = std::make_shared<PullPlanDump>(dba)](
@@ -1000,7 +1030,7 @@ PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, T
PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory) {
utils::MemoryResource *execution_memory) {
if (in_explicit_transaction) {
throw IndexInMulticommandTxException();
}
@@ -1069,7 +1099,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
DbAccessor *dba, utils::MemoryResource *execution_memory) {
if (in_explicit_transaction) {
throw UserModificationInMulticommandTxException();
}
@@ -1122,6 +1152,8 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_ex
return std::nullopt;
},
RWType::NONE};
// False positive report for the std::make_shared above
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
}
PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
@@ -1180,7 +1212,7 @@ PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_exp
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
storage::Storage *db, utils::MonotonicBufferResource *execution_memory) {
storage::Storage *db, utils::MemoryResource *execution_memory) {
if (in_explicit_transaction) {
throw InfoInMulticommandTxException();
}
@@ -1268,8 +1300,7 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory) {
InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory) {
if (in_explicit_transaction) {
throw ConstraintInMulticommandTxException();
}
@@ -1434,10 +1465,12 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
// Handle transaction control queries.
auto query_upper = utils::Trim(utils::ToUpperCase(query_string));
if (query_upper == "BEGIN" || query_upper == "COMMIT" || query_upper == "ROLLBACK") {
query_execution->prepared_query.emplace(PrepareTransactionQuery(query_upper));
const auto upper_case_query = utils::ToUpperCase(query_string);
const auto trimmed_query = utils::Trim(upper_case_query);
if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") {
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
}

View File

@@ -317,7 +317,9 @@ class Interpreter final {
private:
struct QueryExecution {
std::optional<PreparedQuery> prepared_query;
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
utils::MonotonicBufferResource execution_monotonic_memory{kExecutionMemoryBlockSize};
utils::ResourceWithOutOfMemoryException execution_memory{&execution_monotonic_memory};
std::map<std::string, TypedValue> summary;
explicit QueryExecution() = default;
@@ -331,7 +333,7 @@ class Interpreter final {
// destroy the prepared query which is using that instance
// of execution memory.
prepared_query.reset();
execution_memory.Release();
execution_monotonic_memory.Release();
}
};

View File

@@ -84,6 +84,11 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
std::string_view line_string_view = *maybe_line;
// remove '\r' from the end in case we have dos file format
if (line_string_view.back() == '\r') {
line_string_view.remove_suffix(1);
}
while (state != CsvParserState::DONE && !line_string_view.empty()) {
const auto c = line_string_view[0];

View File

@@ -23,6 +23,9 @@ size_t GrowMonotonicBuffer(size_t current_size, size_t max_size) {
return std::ceil(next_size);
}
__attribute__((no_sanitize("pointer-overflow"))) void CheckAllocationSizeOverflow(void *aligned_ptr, size_t bytes) {
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
}
} // namespace
MonotonicBufferResource::MonotonicBufferResource(size_t initial_size) : initial_size_(initial_size) {}
@@ -121,7 +124,7 @@ void *MonotonicBufferResource::DoAllocate(size_t bytes, size_t alignment) {
next_buffer_size_ = GrowMonotonicBuffer(next_buffer_size_, std::numeric_limits<size_t>::max() - sizeof(Buffer));
}
if (reinterpret_cast<char *>(aligned_ptr) < buffer_head) throw BadAlloc("Allocation alignment overflow");
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
CheckAllocationSizeOverflow(aligned_ptr, bytes);
allocated_ = reinterpret_cast<char *>(aligned_ptr) - data + bytes;
return aligned_ptr;
}

View File

@@ -1,3 +1,4 @@
#include <cstddef>
#include <new>
#if USE_JEMALLOC
@@ -10,7 +11,7 @@
#include "utils/memory_tracker.hpp"
namespace {
void *newImpl(std::size_t size) {
void *newImpl(const std::size_t size) {
auto *ptr = malloc(size);
if (LIKELY(ptr != nullptr)) {
return ptr;
@@ -19,11 +20,26 @@ void *newImpl(std::size_t size) {
throw std::bad_alloc{};
}
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
void *newImpl(const std::size_t size, const std::align_val_t align) {
auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
if (LIKELY(ptr != nullptr)) {
return ptr;
}
void deleteImpl(void *ptr) noexcept { free(ptr); }
throw std::bad_alloc{};
}
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept {
return aligned_alloc(size, static_cast<std::size_t>(align));
}
#if USE_JEMALLOC
void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
dallocx(ptr, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
void deleteSized(void *ptr, const std::size_t size) noexcept {
if (UNLIKELY(ptr == nullptr)) {
@@ -33,24 +49,43 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
sdallocx(ptr, size, 0);
}
void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
if (UNLIKELY(ptr == nullptr)) {
return;
}
sdallocx(ptr, size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
#else
void deleteImpl(void *ptr) noexcept { free(ptr); }
void deleteImpl(void *ptr, const std::align_val_t /*unused*/) noexcept { free(ptr); }
void deleteSized(void *ptr, const std::size_t /*unused*/) noexcept { free(ptr); }
void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t /*unused*/) noexcept { free(ptr); }
#endif
void TrackMemory(const size_t size) {
size_t actual_size = size;
void TrackMemory(std::size_t size) {
#if USE_JEMALLOC
if (LIKELY(size != 0)) {
actual_size = nallocx(size, 0);
size = nallocx(size, 0);
}
#endif
utils::total_memory_tracker.Alloc(actual_size);
utils::total_memory_tracker.Alloc(size);
}
bool TrackMemoryNoExcept(const size_t size) {
void TrackMemory(std::size_t size, const std::align_val_t align) {
#if USE_JEMALLOC
if (LIKELY(size != 0)) {
size = nallocx(size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
#endif
utils::total_memory_tracker.Alloc(size);
}
bool TrackMemoryNoExcept(const std::size_t size) {
try {
TrackMemory(size);
} catch (...) {
@@ -60,7 +95,17 @@ bool TrackMemoryNoExcept(const size_t size) {
return true;
}
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0) noexcept {
bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
try {
TrackMemory(size, align);
} catch (...) {
return false;
}
return true;
}
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
try {
#if USE_JEMALLOC
if (LIKELY(ptr != nullptr)) {
@@ -78,32 +123,74 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0)
}
}
void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
try {
#if USE_JEMALLOC
if (LIKELY(ptr != nullptr)) {
utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align))); // NOLINT(hicpp-signed-bitwise)
}
#else
if (size) {
utils::total_memory_tracker.Free(size);
} else {
// Innaccurate because malloc_usable_size() result is greater or equal to allocated size.
utils::total_memory_tracker.Free(malloc_usable_size(ptr));
}
#endif
} catch (...) {
}
}
} // namespace
void *operator new(std::size_t size) {
void *operator new(const std::size_t size) {
TrackMemory(size);
return newImpl(size);
}
void *operator new[](std::size_t size) {
void *operator new[](const std::size_t size) {
TrackMemory(size);
return newImpl(size);
}
void *operator new(std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
void *operator new(const std::size_t size, const std::align_val_t align) {
TrackMemory(size, align);
return newImpl(size, align);
}
void *operator new[](const std::size_t size, const std::align_val_t align) {
TrackMemory(size, align);
return newImpl(size, align);
}
void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size))) {
return newNoExcept(size);
}
return nullptr;
}
void *operator new[](std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size))) {
return newNoExcept(size);
}
return nullptr;
}
void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size, align))) {
return newNoExcept(size, align);
}
return nullptr;
}
void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size, align))) {
return newNoExcept(size, align);
}
return nullptr;
}
void operator delete(void *ptr) noexcept {
UntrackMemory(ptr);
deleteImpl(ptr);
@@ -114,12 +201,52 @@ void operator delete[](void *ptr) noexcept {
deleteImpl(ptr);
}
void operator delete(void *ptr, std::size_t size) noexcept {
void operator delete(void *ptr, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}
void operator delete[](void *ptr, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}
void operator delete(void *ptr, const std::size_t size) noexcept {
UntrackMemory(ptr, size);
deleteSized(ptr, size);
}
void operator delete[](void *ptr, std::size_t size) noexcept {
void operator delete[](void *ptr, const std::size_t size) noexcept {
UntrackMemory(ptr, size);
deleteSized(ptr, size);
}
void operator delete(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align, size);
deleteSized(ptr, size, align);
}
void operator delete[](void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align, size);
deleteSized(ptr, size, align);
}
void operator delete(void *ptr, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr);
deleteImpl(ptr);
}
void operator delete[](void *ptr, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr);
deleteImpl(ptr);
}
void operator delete(void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}
void operator delete[](void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}

View File

@@ -38,19 +38,5 @@ int main(int argc, char **argv) {
}
spdlog::info("Memgraph is out of memory");
spdlog::info("Cleaning up unused memory");
client->Execute("MATCH (n) DETACH DELETE n;");
client->DiscardAll();
client->Execute("FREE MEMORY;");
client->DiscardAll();
// now it should succeed
spdlog::info("Retrying the query with the memory cleaned up");
client->Execute(create_query);
if (!client->FetchOne()) {
LOG_FATAL("Memgraph is still out of memory");
}
return 0;
}

View File

@@ -2,7 +2,8 @@ bolt_port: &bolt_port "7687"
template_cluster: &template_cluster
cluster:
main:
args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"]
args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"]
log_file: "memory-e2e.log"
setup_queries: []
validation_queries: []

View File

@@ -11,19 +11,23 @@ template_validation_queries: &template_validation_queries
template_cluster: &template_cluster
cluster:
replica_1:
args: ["--bolt-port", "7688"]
args: ["--bolt-port", "7688", "--log-level=TRACE"]
log_file: "replication-e2e-replica1.log"
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
<<: *template_validation_queries
replica_2:
args: ["--bolt-port", "7689"]
args: ["--bolt-port", "7689", "--log-level=TRACE"]
log_file: "replication-e2e-replica2.log"
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
<<: *template_validation_queries
replica_3:
args: ["--bolt-port", "7690"]
args: ["--bolt-port", "7690", "--log-level=TRACE"]
log_file: "replication-e2e-replica3.log"
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10003;"]
<<: *template_validation_queries
main:
args: ["--bolt-port", "7687"]
args: ["--bolt-port", "7687", "--log-level=TRACE"]
log_file: "replication-e2e-main.log"
setup_queries: [
"REGISTER REPLICA replica_1 SYNC WITH TIMEOUT 0 TO '127.0.0.1:10001'",
"REGISTER REPLICA replica_2 SYNC WITH TIMEOUT 1 TO '127.0.0.1:10002'",

View File

@@ -44,7 +44,9 @@ def run(args):
for name, config in workload['cluster'].items():
mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY)
mg_instances[name] = mg_instance
mg_instance.start(args=config['args'])
log_file_path = os.path.join(BUILD_DIR, 'logs', config['log_file'])
binary_args = config['args'] + ["--log-file", log_file_path]
mg_instance.start(args=binary_args)
for query in config['setup_queries']:
mg_instance.query(query)
# Test.

View File

@@ -164,4 +164,4 @@
{:bank (bank-checker)
:timeline (timeline/html)})
:generator (c/replication-gen (gen/mix [read-balances valid-transfer]))
:final-generator (gen/once read-balances)})
:final-generator {:gen (gen/once read-balances) :recovery-time 20}})

View File

@@ -22,7 +22,7 @@
"A map of workload names to functions that can take opts and construct
workloads."
{:bank bank/workload
:sequential sequential/workload
;; :sequential sequential/workload (T0532-MG)
:large large/workload})
(def nemesis-configuration
@@ -45,8 +45,8 @@
(gen/log "Healing cluster.")
(gen/nemesis (:final-generator nemesis))
(gen/log "Waiting for recovery")
(gen/sleep 20)
(gen/clients final-generator))
(gen/sleep (:recovery-time final-generator))
(gen/clients (:gen final-generator)))
gen)]
(merge tests/noop-test
opts

View File

@@ -103,4 +103,4 @@
:timeline (timeline/html)})
:generator (c/replication-gen
(gen/mix [read-nodes add-nodes]))
:final-generator (gen/once read-nodes)})
:final-generator {:gen (gen/once read-nodes) :recovery-time 40}})

View File

@@ -383,15 +383,16 @@ TEST(BoltSession, ExecuteRunWrongMarker) {
}
TEST(BoltSession, ExecuteRunMissingData) {
std::array<uint8_t, 6> run_req_without_parameters{
run_req_header[0], run_req_header[1], run_req_header[2], 0x00, 0x00, 0x00};
// test lengths, they test the following situations:
// missing header data, missing query data, missing parameters
int len[] = {1, 2, 37};
int len[] = {1, 2, run_req_without_parameters.size()};
for (int i = 0; i < 3; ++i) {
INIT_VARS;
ExecuteHandshake(input_stream, session, output);
ExecuteInit(input_stream, session, output);
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_header, len[i]), SessionException);
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_without_parameters.data(), len[i]), SessionException);
ASSERT_EQ(session.state_, State::Close);
CheckFailureMessage(output);
@@ -871,7 +872,7 @@ TEST(BoltSession, Noop) {
CheckFailureMessage(output);
session.state_ = State::Result;
ExecuteCommand(input_stream, session, pullall_req, sizeof(v4::pullall_req));
ExecuteCommand(input_stream, session, pullall_req, sizeof(pullall_req));
CheckSuccessMessage(output);
ASSERT_THROW(ExecuteCommand(input_stream, session, v4_1::noop, sizeof(v4_1::noop)), SessionException);

View File

@@ -73,6 +73,12 @@ class TestPlanner : public ::testing::Test {};
using PlannerTypes = ::testing::Types<Planner>;
void DeleteListContent(std::list<BaseOpChecker *> *list) {
for (BaseOpChecker *ptr : *list) {
delete ptr;
}
}
TYPED_TEST_CASE(TestPlanner, PlannerTypes);
TYPED_TEST(TestPlanner, MatchNodeReturn) {
@@ -223,6 +229,7 @@ TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) {
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()};
CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, MatchWhereReturn) {
@@ -549,10 +556,8 @@ TYPED_TEST(TestPlanner, MatchMerge) {
auto acc = ExpectAccumulate({symbol_table.at(*ident_n)});
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce());
for (auto &op : on_match) delete op;
on_match.clear();
for (auto &op : on_create) delete op;
on_create.clear();
DeleteListContent(&on_match);
DeleteListContent(&on_create);
}
TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
@@ -564,6 +569,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r")));
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()};
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, MatchUnwindReturn) {
@@ -705,6 +711,7 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) {
// optional ScanAll.
std::list<BaseOpChecker *> optional{new ExpectFilter(), new ExpectScanAll()};
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, MatchReturnAsterisk) {
@@ -763,8 +770,8 @@ TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) {
std::list<BaseOpChecker *> on_match{new ExpectScanAll(), new ExpectFilter()};
std::list<BaseOpChecker *> on_create{new ExpectCreateNode()};
CheckPlan<TypeParam>(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create));
for (auto &op : on_match) delete op;
for (auto &op : on_create) delete op;
DeleteListContent(&on_match);
DeleteListContent(&on_create);
}
TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
@@ -774,6 +781,7 @@ TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n")));
std::list<BaseOpChecker *> optional{new ExpectScanAll()};
CheckPlan<TypeParam>(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, FunctionAggregationReturn) {

View File

@@ -5,6 +5,8 @@
#include "query/procedure/mg_procedure_impl.hpp"
#include "test_utils.hpp"
static void DummyCallback(const mgp_list *, const mgp_graph *, mgp_result *, mgp_memory *) {}
TEST(Module, InvalidProcedureRegistration) {
@@ -53,7 +55,8 @@ TEST(Module, ProcedureSignature) {
CheckSignature(proc, "proc() :: ()");
mgp_proc_add_arg(proc, "arg1", mgp_type_number());
CheckSignature(proc, "proc(arg1 :: NUMBER) :: ()");
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: ()");
mgp_proc_add_result(proc, "res1", mgp_type_list(mgp_type_int()));
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: (res1 :: LIST OF INTEGER)");
@@ -69,7 +72,8 @@ TEST(Module, ProcedureSignature) {
"(res1 :: LIST OF INTEGER, DEPRECATED res2 :: STRING)");
EXPECT_FALSE(mgp_proc_add_result(proc, "res2", mgp_type_any()));
EXPECT_FALSE(mgp_proc_add_deprecated_result(proc, "res1", mgp_type_any()));
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(), mgp_value_make_string("string=\"value\"", &memory));
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(),
test_utils::CreateValueOwningPtr(mgp_value_make_string("string=\"value\"", &memory)).get());
CheckSignature(proc,
"proc(arg1 :: NUMBER, opt1 = Null :: ANY?, "
"opt2 = \"string=\\\"value\\\"\" :: STRING) :: "
@@ -80,6 +84,7 @@ TEST(Module, ProcedureSignatureOnlyOptArg) {
mgp_memory memory{utils::NewDeleteResource()};
mgp_module module(utils::NewDeleteResource());
auto *proc = mgp_module_add_read_procedure(&module, "proc", DummyCallback);
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
CheckSignature(proc, "proc(opt1 = Null :: ANY?) :: ()");
}

View File

@@ -1,7 +1,13 @@
#include <functional>
#include <memory>
#include <utility>
#include <gtest/gtest.h>
#include "query/procedure/mg_procedure_impl.hpp"
#include "test_utils.hpp"
TEST(CypherType, PresentableNameSimpleTypes) {
EXPECT_EQ(mgp_type_any()->impl->GetPresentableName(), "ANY");
EXPECT_EQ(mgp_type_bool()->impl->GetPresentableName(), "BOOLEAN");
@@ -66,6 +72,7 @@ TEST(CypherType, NullSatisfiesType) {
EXPECT_TRUE(null_type->impl->SatisfiesType(tv_null));
}
}
mgp_value_destroy(mgp_null);
}
}
@@ -101,6 +108,7 @@ TEST(CypherType, BoolSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_bool, tv_bool,
{mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_bool);
}
TEST(CypherType, IntSatisfiesType) {
@@ -111,6 +119,7 @@ TEST(CypherType, IntSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_int, tv_int,
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_int);
}
TEST(CypherType, DoubleSatisfiesType) {
@@ -121,6 +130,7 @@ TEST(CypherType, DoubleSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_double, tv_double,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_double);
}
TEST(CypherType, StringSatisfiesType) {
@@ -131,12 +141,13 @@ TEST(CypherType, StringSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_string, tv_string,
{mgp_type_bool(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_string);
}
TEST(CypherType, MapSatisfiesType) {
mgp_memory memory{utils::NewDeleteResource()};
auto *map = mgp_map_make_empty(&memory);
mgp_map_insert(map, "key", mgp_value_make_int(42, &memory));
mgp_map_insert(map, "key", test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get());
auto *mgp_map_v = mgp_value_make_map(map);
const query::TypedValue tv_map(std::map<std::string, query::TypedValue>{{"key", query::TypedValue(42)}});
CheckSatisfiesTypesAndNullable(mgp_map_v, tv_map, {mgp_type_any(), mgp_type_map()});
@@ -144,6 +155,7 @@ TEST(CypherType, MapSatisfiesType) {
mgp_map_v, tv_map,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_node(),
mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_map_v);
}
TEST(CypherType, VertexSatisfiesType) {
@@ -160,6 +172,7 @@ TEST(CypherType, VertexSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_vertex_v, tv_vertex,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
mgp_type_number(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_vertex_v);
}
TEST(CypherType, EdgeSatisfiesType) {
@@ -178,6 +191,7 @@ TEST(CypherType, EdgeSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_edge_v, tv_edge,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
mgp_type_number(), mgp_type_node(), mgp_type_path()});
mgp_value_destroy(mgp_edge_v);
}
TEST(CypherType, PathSatisfiesType) {
@@ -190,9 +204,13 @@ TEST(CypherType, PathSatisfiesType) {
mgp_memory memory{utils::NewDeleteResource()};
utils::Allocator<mgp_path> alloc(memory.impl);
mgp_graph graph{&dba, storage::View::NEW};
auto *path = mgp_path_make_with_start(alloc.new_object<mgp_vertex>(v1, &graph), &memory);
auto *mgp_vertex_v = alloc.new_object<mgp_vertex>(v1, &graph);
auto path = mgp_path_make_with_start(mgp_vertex_v, &memory);
ASSERT_TRUE(path);
ASSERT_TRUE(mgp_path_expand(path, alloc.new_object<mgp_edge>(edge, &graph)));
alloc.delete_object(mgp_vertex_v);
auto mgp_edge_v = alloc.new_object<mgp_edge>(edge, &graph);
ASSERT_TRUE(mgp_path_expand(path, mgp_edge_v));
alloc.delete_object(mgp_edge_v);
auto *mgp_path_v = mgp_value_make_path(path);
const query::TypedValue tv_path(query::Path(v1, edge, v2));
CheckSatisfiesTypesAndNullable(mgp_path_v, tv_path, {mgp_type_any(), mgp_type_path()});
@@ -200,6 +218,7 @@ TEST(CypherType, PathSatisfiesType) {
mgp_path_v, tv_path,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship()});
mgp_value_destroy(mgp_path_v);
}
static std::vector<const mgp_type *> MakeListTypes(const std::vector<const mgp_type *> &element_types) {
@@ -224,6 +243,7 @@ TEST(CypherType, EmptyListSatisfiesType) {
auto all_types = MakeListTypes(primitive_types);
all_types.push_back(mgp_type_any());
CheckSatisfiesTypesAndNullable(mgp_list_v, tv_list, all_types);
mgp_value_destroy(mgp_list_v);
}
TEST(CypherType, ListOfIntSatisfiesType) {
@@ -233,7 +253,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
auto *mgp_list_v = mgp_value_make_list(list);
query::TypedValue tv_list(std::vector<query::TypedValue>{});
for (int64_t i = 0; i < elem_count; ++i) {
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(i, &memory)));
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(i, &memory)).get()));
tv_list.ValueList().emplace_back(i);
auto valid_types = MakeListTypes({mgp_type_any(), mgp_type_int(), mgp_type_number()});
valid_types.push_back(mgp_type_any());
@@ -242,6 +262,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
}
mgp_value_destroy(mgp_list_v);
}
TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
@@ -251,10 +272,10 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
auto *mgp_list_v = mgp_value_make_list(list);
query::TypedValue tv_list(std::vector<query::TypedValue>{});
// Add an int
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(42, &memory)));
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get()));
tv_list.ValueList().emplace_back(42);
// Add a boolean
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_bool(1, &memory)));
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_bool(1, &memory)).get()));
tv_list.ValueList().emplace_back(true);
auto valid_types = MakeListTypes({mgp_type_any()});
valid_types.push_back(mgp_type_any());
@@ -264,6 +285,7 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
mgp_list_v, tv_list,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_list_v);
}
TEST(CypherType, ListOfNullSatisfiesType) {
@@ -271,7 +293,7 @@ TEST(CypherType, ListOfNullSatisfiesType) {
auto *list = mgp_list_make_empty(1, &memory);
auto *mgp_list_v = mgp_value_make_list(list);
query::TypedValue tv_list(std::vector<query::TypedValue>{});
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_null(&memory)));
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get()));
tv_list.ValueList().emplace_back();
// List with Null satisfies all nullable list element types
std::vector<const mgp_type *> primitive_types{
@@ -295,4 +317,5 @@ TEST(CypherType, ListOfNullSatisfiesType) {
EXPECT_FALSE(null_type->impl->SatisfiesType(*mgp_list_v)) << null_type->impl->GetPresentableName();
EXPECT_FALSE(null_type->impl->SatisfiesType(tv_list));
}
mgp_value_destroy(mgp_list_v);
}

View File

@@ -254,6 +254,7 @@ TEST(PyModule, PyObjectToMgpValue) {
const mgp_value *v2 = mgp_map_at(map, "four");
ASSERT_TRUE(mgp_value_is_double(v2));
EXPECT_EQ(mgp_value_get_double(v2), 4.0);
mgp_value_destroy(value);
}
int main(int argc, char **argv) {

View File

@@ -709,14 +709,15 @@ TEST_P(DurabilityTest, SnapshotFallback) {
{.items = {.properties_on_edges = GetParam()},
.durability = {.storage_directory = storage_directory,
.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT,
.snapshot_interval = std::chrono::milliseconds(2000)}});
.snapshot_interval = std::chrono::milliseconds(3000)}});
CreateBaseDataset(&store, GetParam());
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
std::this_thread::sleep_for(std::chrono::milliseconds(3500));
ASSERT_EQ(GetSnapshotsList().size(), 1);
CreateExtendedDataset(&store);
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
std::this_thread::sleep_for(std::chrono::milliseconds(3000));
}
ASSERT_GE(GetSnapshotsList().size(), 2);
ASSERT_EQ(GetSnapshotsList().size(), 2);
ASSERT_EQ(GetBackupSnapshotsList().size(), 0);
ASSERT_EQ(GetWalsList().size(), 0);
ASSERT_EQ(GetBackupWalsList().size(), 0);
@@ -724,7 +725,7 @@ TEST_P(DurabilityTest, SnapshotFallback) {
// Destroy last snapshot.
{
auto snapshots = GetSnapshotsList();
ASSERT_GE(snapshots.size(), 2);
ASSERT_EQ(snapshots.size(), 2);
DestroySnapshot(*snapshots.begin());
}

View File

@@ -0,0 +1,9 @@
#include <memory>
#include "query/procedure/mg_procedure_impl.hpp"
namespace test_utils {
using MgpValueOwningPtr = std::unique_ptr<mgp_value, void (*)(mgp_value *)>;
MgpValueOwningPtr CreateValueOwningPtr(mgp_value *value) { return MgpValueOwningPtr(value, &mgp_value_destroy); }
} // namespace test_utils

View File

@@ -397,8 +397,8 @@ TEST_F(TypedValueLogicTest, LogicalXor) {
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(AllTypesFixture, ConstructionWithMemoryResource) {
std::vector<TypedValue> values_with_custom_memory;
utils::MonotonicBufferResource monotonic_memory(1024);
std::vector<TypedValue> values_with_custom_memory;
for (const auto &value : values_) {
EXPECT_EQ(value.GetMemoryResource(), utils::NewDeleteResource());
TypedValue copy_constructed_value(value, &monotonic_memory);

View File

@@ -4,7 +4,7 @@
#include "utils/string.hpp"
class CsvReaderTest : public ::testing::Test {
class CsvReaderTest : public ::testing::TestWithParam<const char *> {
protected:
const std::filesystem::path csv_directory{std::filesystem::temp_directory_path() / "csv_testing"};
@@ -30,7 +30,9 @@ class CsvReaderTest : public ::testing::Test {
namespace {
class FileWriter {
public:
explicit FileWriter(const std::filesystem::path path) { stream_.open(path); }
explicit FileWriter(const std::filesystem::path path, std::string newline = "\n") : newline_{std::move(newline)} {
stream_.open(path);
}
FileWriter(const FileWriter &) = delete;
FileWriter &operator=(const FileWriter &) = delete;
@@ -45,7 +47,7 @@ class FileWriter {
return 0;
}
stream_ << line << std::endl;
stream_ << line << newline_;
// including the newline character
return line.size() + 1;
@@ -53,6 +55,7 @@ class FileWriter {
private:
std::ofstream stream_;
std::string newline_;
};
std::string CreateRow(const std::vector<std::string> &columns, const std::string_view delim) {
@@ -69,10 +72,10 @@ auto ToPmrColumns(const std::vector<std::string> &columns) {
} // namespace
TEST_F(CsvReaderTest, CommaDelimiter) {
TEST_P(CsvReaderTest, CommaDelimiter) {
// create a file with a single valid row;
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
const std::vector<std::string> columns{"A", "B", "C"};
writer.WriteLine(CreateRow(columns, ","));
@@ -93,9 +96,9 @@ TEST_F(CsvReaderTest, CommaDelimiter) {
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
}
TEST_F(CsvReaderTest, SemicolonDelimiter) {
TEST_P(CsvReaderTest, SemicolonDelimiter) {
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -116,12 +119,12 @@ TEST_F(CsvReaderTest, SemicolonDelimiter) {
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
}
TEST_F(CsvReaderTest, SkipBad) {
TEST_P(CsvReaderTest, SkipBad) {
// create a file with invalid first two rows (containing a string with a
// missing closing quote);
// the last row is valid;
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -161,11 +164,11 @@ TEST_F(CsvReaderTest, SkipBad) {
}
}
TEST_F(CsvReaderTest, AllRowsValid) {
TEST_P(CsvReaderTest, AllRowsValid) {
// create a file with all rows valid;
// parser should return 'std::nullopt'
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -190,11 +193,11 @@ TEST_F(CsvReaderTest, AllRowsValid) {
}
}
TEST_F(CsvReaderTest, SkipAllRows) {
TEST_P(CsvReaderTest, SkipAllRows) {
// create a file with all rows invalid (containing a string with a missing closing quote);
// parser should return 'std::nullopt'
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -217,9 +220,9 @@ TEST_F(CsvReaderTest, SkipAllRows) {
ASSERT_EQ(parsed_row, std::nullopt);
}
TEST_F(CsvReaderTest, WithHeader) {
TEST_P(CsvReaderTest, WithHeader) {
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -249,12 +252,12 @@ TEST_F(CsvReaderTest, WithHeader) {
}
}
TEST_F(CsvReaderTest, MultilineQuotedString) {
TEST_P(CsvReaderTest, MultilineQuotedString) {
// create a file with first row valid and the second row containing a quoted
// string spanning two lines;
// parser should return two valid rows
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -284,11 +287,11 @@ TEST_F(CsvReaderTest, MultilineQuotedString) {
ASSERT_EQ(*parsed_row, ToPmrColumns(expected_multiline));
}
TEST_F(CsvReaderTest, EmptyColumns) {
TEST_P(CsvReaderTest, EmptyColumns) {
// create a file with all rows valid;
// parser should return 'std::nullopt'
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath);
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -315,3 +318,5 @@ TEST_F(CsvReaderTest, EmptyColumns) {
ASSERT_EQ(*parsed_row, pmr_expected_row);
}
}
INSTANTIATE_TEST_CASE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n"));

View File

@@ -12,6 +12,7 @@ class TestMemory final : public utils::MemoryResource {
size_t delete_count_{0};
private:
static constexpr size_t kPadSize = 32;
void *DoAllocate(size_t bytes, size_t alignment) override {
new_count_++;
EXPECT_TRUE(alignment != 0U && (alignment & (alignment - 1U)) == 0U) << "Alignment must be power of 2";
@@ -20,11 +21,11 @@ class TestMemory final : public utils::MemoryResource {
EXPECT_TRUE(bytes + pad_size > bytes) << "TestMemory size overflow";
EXPECT_TRUE(bytes + pad_size + alignment > bytes + alignment) << "TestMemory size overflow";
EXPECT_TRUE(2U * alignment > alignment) << "TestMemory alignment overflow";
// Allocate a block containing extra alignment and pad_size bytes, but
// Allocate a block containing extra alignment and kPadSize bytes, but
// aligned to 2 * alignment. Then we can offset the ptr so that it's never
// aligned to 2 * alignment. This ought to make allocator alignment issues
// more obvious.
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + pad_size, 2U * alignment);
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + kPadSize, 2U * alignment);
// Clear allocated memory to 0xFF, marking the invalid region.
memset(ptr, 0xFF, alignment + bytes + pad_size);
// Offset the ptr so it's not aligned to 2 * alignment, but still aligned to
@@ -39,7 +40,8 @@ class TestMemory final : public utils::MemoryResource {
void DoDeallocate(void *ptr, size_t bytes, size_t alignment) override {
delete_count_++;
// Deallocate the original ptr, before alignment adjustment.
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, bytes, alignment);
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, alignment + bytes + kPadSize,
2U * alignment);
}
bool DoIsEqual(const utils::MemoryResource &other) const noexcept override { return this == &other; }

View File

@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""
Bench Graph client responsible for sending benchmarking data in JSON format to
the Bench Graph server.
"""
import json
import logging
import os
import requests
import subprocess
from datetime import datetime
from argparse import ArgumentParser
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "")
GITHUB_SHA = os.getenv("GITHUB_SHA", "")
GITHUB_REF = os.getenv("GITHUB_REF", "")
BENCH_GRAPH_SERVER_ENDPOINT = os.getenv(
"BENCH_GRAPH_SERVER_ENDPOINT",
"http://bench-graph-api:9001")
log = logging.getLogger(__name__)
def parse_args():
argp = ArgumentParser(description=__doc__)
argp.add_argument("--benchmark-name", type=str, required=True)
argp.add_argument("--benchmark-results-path", type=str, required=True)
argp.add_argument("--github-run-id", type=int, required=True)
argp.add_argument("--github-run-number", type=int, required=True)
return argp.parse_args()
def post_measurement(args):
with open(args.benchmark_results_path, "r") as f:
data = json.load(f)
timestamp = datetime.now().timestamp()
branch = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
stdout=subprocess.PIPE,
check=True).stdout.decode("utf-8").strip()
req = requests.post(
f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements",
json={
"name": args.benchmark_name,
"timestamp": timestamp,
"git_repo": GITHUB_REPOSITORY,
"git_ref": GITHUB_REF,
"git_sha": GITHUB_SHA,
"github_run_id": args.github_run_id,
"github_run_number": args.github_run_number,
"results": data,
"git_branch": branch},
timeout=1)
assert req.status_code == 200, \
f"Uploading {args.benchmark_name} data failed."
log.info(f"{args.benchmark_name} data sent to "
f"{BENCH_GRAPH_SERVER_ENDPOINT}")
if __name__ == "__main__":
args = parse_args()
logging.basicConfig(level=logging.INFO)
post_measurement(args)

View File

@@ -0,0 +1 @@
requests==2.25.1

View File

@@ -0,0 +1,269 @@
#!/usr/bin/env python3
#
#===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
r"""
ClangTidy Diff Checker
======================
This script reads input from a unified diff, runs clang-tidy on all changed
files and outputs clang-tidy warnings in changed lines only. This is useful to
detect clang-tidy regressions in the lines touched by a specific patch.
Example usage for git/svn users:
git diff -U0 HEAD^ | clang-tidy-diff.py -p1
svn diff --diff-cmd=diff -x-U0 | \
clang-tidy-diff.py -fix -checks=-*,modernize-use-override
"""
import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback
try:
import yaml
except ImportError:
yaml = None
is_py2 = sys.version[0] == '2'
if is_py2:
import Queue as queue
else:
import queue as queue
def run_tidy(task_queue, lock, timeout):
watchdog = None
while True:
command = task_queue.get()
try:
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if timeout is not None:
watchdog = threading.Timer(timeout, proc.kill)
watchdog.start()
stdout, stderr = proc.communicate()
with lock:
sys.stdout.write(stdout.decode('utf-8') + '\n')
sys.stdout.flush()
if stderr:
sys.stderr.write(stderr.decode('utf-8') + '\n')
sys.stderr.flush()
except Exception as e:
with lock:
sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
finally:
with lock:
if not (timeout is None or watchdog is None):
if not watchdog.is_alive():
sys.stderr.write('Terminated by timeout: ' +
' '.join(command) + '\n')
watchdog.cancel()
task_queue.task_done()
def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
for _ in range(max_tasks):
t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
t.daemon = True
t.start()
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged = []
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
content = yaml.safe_load(open(replacefile, 'r'))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {'MainSourceFile': '', mergekey: merged}
with open(mergefile, 'w') as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, 'w').close()
def main():
parser = argparse.ArgumentParser(description=
'Run clang-tidy against changed files, and '
'output diagnostics only for modified '
'lines.')
parser.add_argument('-clang-tidy-binary', metavar='PATH',
default='clang-tidy',
help='path to clang-tidy binary')
parser.add_argument('-p', metavar='NUM', default=0,
help='strip the smallest prefix containing P slashes')
parser.add_argument('-regex', metavar='PATTERN', default=None,
help='custom pattern selecting file paths to check '
'(case sensitive, overrides -iregex)')
parser.add_argument('-iregex', metavar='PATTERN', default=
r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
help='custom pattern selecting file paths to check '
'(case insensitive, overridden by -regex)')
parser.add_argument('-j', type=int, default=1,
help='number of tidy instances to be run in parallel.')
parser.add_argument('-timeout', type=int, default=None,
help='timeout per each file in seconds.')
parser.add_argument('-fix', action='store_true', default=False,
help='apply suggested fixes')
parser.add_argument('-checks',
help='checks filter, when not specified, use clang-tidy '
'default',
default='')
parser.add_argument('-path', dest='build_path',
help='Path used to read a compile command database.')
if yaml:
parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
help='Create a yaml file to store suggested fixes in, '
'which can be applied with clang-apply-replacements.')
parser.add_argument('-extra-arg', dest='extra_arg',
action='append', default=[],
help='Additional argument to append to the compiler '
'command line.')
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
action='append', default=[],
help='Additional argument to prepend to the compiler '
'command line.')
parser.add_argument('-quiet', action='store_true', default=False,
help='Run clang-tidy in quiet mode')
clang_tidy_args = []
argv = sys.argv[1:]
if '--' in argv:
clang_tidy_args.extend(argv[argv.index('--'):])
argv = argv[:argv.index('--')]
args = parser.parse_args(argv)
# Extract changed lines for each file.
filename = None
lines_by_file = {}
for line in sys.stdin:
match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
if match:
filename = match.group(2)
if filename is None:
continue
if args.regex is not None:
if not re.match('^%s$' % args.regex, filename):
continue
else:
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
continue
match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
if match:
start_line = int(match.group(1))
line_count = 1
if match.group(3):
line_count = int(match.group(3))
if line_count == 0:
continue
end_line = start_line + line_count - 1
lines_by_file.setdefault(filename, []).append([start_line, end_line])
if not any(lines_by_file):
print("No relevant changes found.")
sys.exit(0)
max_task_count = args.j
if max_task_count == 0:
max_task_count = multiprocessing.cpu_count()
max_task_count = min(len(lines_by_file), max_task_count)
tmpdir = None
if yaml and args.export_fixes:
tmpdir = tempfile.mkdtemp()
# Tasks for clang-tidy.
task_queue = queue.Queue(max_task_count)
# A lock for console output.
lock = threading.Lock()
# Run a pool of clang-tidy workers.
start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
# Form the common args list.
common_clang_tidy_args = []
if args.fix:
common_clang_tidy_args.append('-fix')
if args.checks != '':
common_clang_tidy_args.append('-checks=' + args.checks)
if args.quiet:
common_clang_tidy_args.append('-quiet')
if args.build_path is not None:
common_clang_tidy_args.append('-p=%s' % args.build_path)
for arg in args.extra_arg:
common_clang_tidy_args.append('-extra-arg=%s' % arg)
for arg in args.extra_arg_before:
common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
for name in lines_by_file:
line_filter_json = json.dumps(
[{"name": name, "lines": lines_by_file[name]}],
separators=(',', ':'))
# Run clang-tidy on files containing changes.
command = [args.clang_tidy_binary]
command.append('-line-filter=' + line_filter_json)
if yaml and args.export_fixes:
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
command.append('-export-fixes=' + tmp_name)
command.extend(common_clang_tidy_args)
command.append(name)
command.extend(clang_tidy_args)
task_queue.put(command)
# Wait for all threads to be done.
task_queue.join()
if yaml and args.export_fixes:
print('Writing fixes to ' + args.export_fixes + ' ...')
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
sys.stderr.write('Error exporting fixes.\n')
traceback.print_exc()
if tmpdir:
shutil.rmtree(tmpdir)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,9 @@
#!/bin/bash
# the first sort | uniq is necessary, because the same occurrence of the same error
# can be reported from headers when they are included in multiple source files
`dirname ${BASH_SOURCE[0]}`/grep_error_lines.sh |
sort | uniq |
sed -E 's/.*\[(.*)\]\r?$/\1/g' | # extract the check name from [check-name]
sort | uniq -c | # count each type of check
sort -nr # sort them into descending order

View File

@@ -0,0 +1,12 @@
#!/bin/bash
# Matches timestamp like "2021-03-25T17:06:42.2621697Z"
TIMESTAMP_PATTERN="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z"
# Matches absolute file pathes with line and column identifier like
# "/opt/actions-runner/_work/memgraph/memgraph/src/utils/exceptions.hpp:71:11:"
FILE_ABSOLUTE_PATH_PATTERN="/[^:]+:\d+:\d+:"
ERROR_OR_WARNING_PATTERN="(error|warning):"
grep -P "^($TIMESTAMP_PATTERN )?$FILE_ABSOLUTE_PATH_PATTERN $ERROR_OR_WARNING_PATTERN.*$"

View File

@@ -0,0 +1,337 @@
#!/usr/bin/env python3
#
#===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
# FIXME: Integrate with clang-tidy-diff.py
"""
Parallel clang-tidy runner
==========================
Runs clang-tidy over all files in a compilation database. Requires clang-tidy
and clang-apply-replacements in $PATH.
Example invocations.
- Run clang-tidy on all files in the current working directory with a default
set of checks and show warnings in the cpp files and all project headers.
run-clang-tidy.py $PWD
- Fix all header guards.
run-clang-tidy.py -fix -checks=-*,llvm-header-guard
- Fix all header guards included from clang-tidy and header guards
for clang-tidy headers.
run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
-header-filter=extra/clang-tidy
Compilation database setup:
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
"""
from __future__ import print_function
import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback
try:
import yaml
except ImportError:
yaml = None
is_py2 = sys.version[0] == '2'
if is_py2:
import Queue as queue
else:
import queue as queue
def find_compilation_database(path):
"""Adjusts the directory until a compilation database is found."""
result = './'
while not os.path.isfile(os.path.join(result, path)):
if os.path.realpath(result) == '/':
print('Error: could not find compilation database.')
sys.exit(1)
result += '../'
return os.path.realpath(result)
def make_absolute(f, directory):
if os.path.isabs(f):
return f
return os.path.normpath(os.path.join(directory, f))
def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
header_filter, allow_enabling_alpha_checkers,
extra_arg, extra_arg_before, quiet, config):
"""Gets a command line for clang-tidy."""
start = [clang_tidy_binary]
if allow_enabling_alpha_checkers:
start.append('-allow-enabling-analyzer-alpha-checkers')
if header_filter is not None:
start.append('-header-filter=' + header_filter)
if checks:
start.append('-checks=' + checks)
if tmpdir is not None:
start.append('-export-fixes')
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
start.append(name)
for arg in extra_arg:
start.append('-extra-arg=%s' % arg)
for arg in extra_arg_before:
start.append('-extra-arg-before=%s' % arg)
start.append('-p=' + build_path)
if quiet:
start.append('-quiet')
if config:
start.append('-config=' + config)
start.append(f)
return start
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged=[]
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
content = yaml.safe_load(open(replacefile, 'r'))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {'MainSourceFile': '', mergekey: merged}
with open(mergefile, 'w') as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, 'w').close()
def check_clang_apply_replacements_binary(args):
"""Checks if invoking supplied clang-apply-replacements binary works."""
try:
subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
except:
print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
'binary correctly specified?', file=sys.stderr)
traceback.print_exc()
sys.exit(1)
def apply_fixes(args, tmpdir):
"""Calls clang-apply-fixes on a given directory."""
invocation = [args.clang_apply_replacements_binary]
if args.format:
invocation.append('-format')
if args.style:
invocation.append('-style=' + args.style)
invocation.append(tmpdir)
subprocess.call(invocation)
def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
"""Takes filenames out of queue and runs clang-tidy on them."""
while True:
name = queue.get()
invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
tmpdir, build_path, args.header_filter,
args.allow_enabling_alpha_checkers,
args.extra_arg, args.extra_arg_before,
args.quiet, args.config)
proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = proc.communicate()
if proc.returncode != 0:
failed_files.append(name)
with lock:
sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
if len(err) > 0:
sys.stdout.flush()
sys.stderr.write(err.decode('utf-8'))
queue.task_done()
def main():
parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
'in a compilation database. Requires '
'clang-tidy and clang-apply-replacements in '
'$PATH.')
parser.add_argument('-allow-enabling-alpha-checkers',
action='store_true', help='allow alpha checkers from '
'clang-analyzer.')
parser.add_argument('-clang-tidy-binary', metavar='PATH',
default='clang-tidy-11',
help='path to clang-tidy binary')
parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
default='clang-apply-replacements-11',
help='path to clang-apply-replacements binary')
parser.add_argument('-checks', default=None,
help='checks filter, when not specified, use clang-tidy '
'default')
parser.add_argument('-config', default=None,
help='Specifies a configuration in YAML/JSON format: '
' -config="{Checks: \'*\', '
' CheckOptions: [{key: x, '
' value: y}]}" '
'When the value is empty, clang-tidy will '
'attempt to find a file named .clang-tidy for '
'each source file in its parent directories.')
parser.add_argument('-header-filter', default=None,
help='regular expression matching the names of the '
'headers to output diagnostics from. Diagnostics from '
'the main file of each translation unit are always '
'displayed.')
if yaml:
parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
help='Create a yaml file to store suggested fixes in, '
'which can be applied with clang-apply-replacements.')
parser.add_argument('-j', type=int, default=0,
help='number of tidy instances to be run in parallel.')
parser.add_argument('files', nargs='*', default=['.*'],
help='files to be processed (regex on path)')
parser.add_argument('-fix', action='store_true', help='apply fix-its')
parser.add_argument('-format', action='store_true', help='Reformat code '
'after applying fixes')
parser.add_argument('-style', default='file', help='The style of reformat '
'code after applying fixes')
parser.add_argument('-p', dest='build_path',
help='Path used to read a compile command database.')
parser.add_argument('-extra-arg', dest='extra_arg',
action='append', default=[],
help='Additional argument to append to the compiler '
'command line.')
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
action='append', default=[],
help='Additional argument to prepend to the compiler '
'command line.')
parser.add_argument('-quiet', action='store_true',
help='Run clang-tidy in quiet mode')
args = parser.parse_args()
db_path = 'compile_commands.json'
if args.build_path is not None:
build_path = args.build_path
else:
# Find our database
build_path = find_compilation_database(db_path)
try:
invocation = [args.clang_tidy_binary, '-list-checks']
if args.allow_enabling_alpha_checkers:
invocation.append('-allow-enabling-analyzer-alpha-checkers')
invocation.append('-p=' + build_path)
if args.checks:
invocation.append('-checks=' + args.checks)
invocation.append('-')
if args.quiet:
# Even with -quiet we still want to check if we can call clang-tidy.
with open(os.devnull, 'w') as dev_null:
subprocess.check_call(invocation, stdout=dev_null)
else:
subprocess.check_call(invocation)
except:
print("Unable to run clang-tidy.", file=sys.stderr)
sys.exit(1)
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
files = [make_absolute(entry['file'], entry['directory'])
for entry in database]
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
tmpdir = None
if args.fix or (yaml and args.export_fixes):
check_clang_apply_replacements_binary(args)
tmpdir = tempfile.mkdtemp()
# Build up a big regexy filter from all command line arguments.
file_name_re = re.compile('|'.join(args.files))
return_code = 0
try:
# Spin up a bunch of tidy-launching threads.
task_queue = queue.Queue(max_task)
# List of files with a non-zero return code.
failed_files = []
lock = threading.Lock()
for _ in range(max_task):
t = threading.Thread(target=run_tidy,
args=(args, tmpdir, build_path, task_queue, lock, failed_files))
t.daemon = True
t.start()
# Fill the queue with files.
for name in files:
if file_name_re.search(name):
task_queue.put(name)
# Wait for all threads to be done.
task_queue.join()
if len(failed_files):
return_code = 1
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print('\nCtrl-C detected, goodbye.')
if tmpdir:
shutil.rmtree(tmpdir)
os.kill(0, 9)
if yaml and args.export_fixes:
print('Writing fixes to ' + args.export_fixes + ' ...')
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
print('Error exporting fixes.\n', file=sys.stderr)
traceback.print_exc()
return_code=1
if args.fix:
print('Applying fixes ...')
try:
apply_fixes(args, tmpdir)
except:
print('Error applying fixes.\n', file=sys.stderr)
traceback.print_exc()
return_code = 1
if tmpdir:
shutil.rmtree(tmpdir)
sys.exit(return_code)
if __name__ == '__main__':
main()

12
tools/lsan.supp Normal file
View File

@@ -0,0 +1,12 @@
leak:antlr4::atn::ArrayPredictionContext::ArrayPredictionContext
leak:std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<antlr4::atn::SingletonPredictionContext, std::allocator<antlr4::atn::SingletonPredictionContext>, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&>(antlr4::atn::SingletonPredictionContext*&, std::_Sp_alloc_shared_tag<std::allocator<antlr4::atn::SingletonPredictionContext> >, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&)
leak:antlr4::atn::PredictionContext::mergeSingletons(std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
leak:void std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::PredictionContext> >(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::PredictionContext>*, std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > > >, std::shared_ptr<antlr4::atn::PredictionContext>&&)
leak:antlr4::atn::ParserATNSimulator::closureCheckingStopState(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::ATNConfigSet*, std::unordered_set<std::shared_ptr<antlr4::atn::ATNConfig>, antlr4::atn::ATNConfig::Hasher, antlr4::atn::ATNConfig::Comparer, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >&, bool, bool, int, bool)
leak:antlr4::atn::ParserATNSimulator::computeReachSet(antlr4::atn::ATNConfigSet*, unsigned long, bool)
leak:std::_Hashtable<unsigned long, std::pair<unsigned long const, antlr4::atn::ATNConfig*>, std::allocator<std::pair<unsigned long const, antlr4::atn::ATNConfig*> >, std::__detail::_Select1st, std::equal_to<unsigned long>, std::hash<unsigned long>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node(unsigned long const&, unsigned long, unsigned long, std::__detail::_Hash_node<std::pair<unsigned long const, antlr4::atn::ATNConfig*>, false>*, unsigned long)
leak:void std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::ATNConfig> const&>(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::ATNConfig>*, std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > > >, std::shared_ptr<antlr4::atn::ATNConfig> const&)
leak:antlr4::atn::ATNConfigSet::add(std::shared_ptr<antlr4::atn::ATNConfig> const&, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
leak:antlr4::atn::ParserATNSimulator::getEpsilonTarget(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::Transition*, bool, bool, bool, bool)
leak:antlr4::atn::PredictionContext::mergeArrays(std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
leak:/lib/x86_64-linux-gnu/libpython3.