Compare commits

..

2 Commits

Author SHA1 Message Date
Marko Budiselic
d1733bca91 Remove JepsenControl tag 2021-03-25 14:47:28 +01:00
Marko Budiselic
1e1b9d5c6c Run Jepsen job on black-panther 2021-03-25 14:46:26 +01:00
69 changed files with 309 additions and 2148 deletions

View File

@@ -54,7 +54,7 @@ Checks: '*,
-readability-magic-numbers,
-readability-named-parameter'
WarningsAsErrors: ''
HeaderFilterRegex: 'src/.*'
HeaderFilterRegex: ''
AnalyzeTemporaryDtors: false
FormatStyle: none
CheckOptions:

View File

@@ -10,7 +10,7 @@ on:
jobs:
community_build:
name: "Community build"
runs-on: [self-hosted, Linux, X64, Diff]
runs-on: [self-hosted, General, Linux, X64, Debian10]
env:
THREADS: 24
@@ -65,9 +65,9 @@ jobs:
name: "Community DEB package"
path: build/output/memgraph*.deb
code_analysis:
name: "Code analysis"
runs-on: [self-hosted, Linux, X64, Diff]
coverage_build:
name: "Coverage build"
runs-on: [self-hosted, General, Linux, X64, Debian10]
env:
THREADS: 24
@@ -79,7 +79,7 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build combined ASAN, UBSAN and coverage binaries
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
@@ -87,8 +87,9 @@ jobs:
# Initialize dependencies.
./init
# Build coverage binaries.
cd build
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
cmake -DTEST_COVERAGE=ON ..
make -j$THREADS memgraph__unit
- name: Run unit tests
@@ -96,9 +97,9 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v2/activate
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
# Run unit tests.
cd build
LSAN_OPTIONS=suppressions=$PWD/../tools/lsan.supp UBSAN_OPTIONS=halt_on_error=1 ctest -R memgraph__unit --output-on-failure -j2
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Compute code coverage
run: |
@@ -119,19 +120,9 @@ jobs:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
debug_build:
name: "Debug build"
runs-on: [self-hosted, Linux, X64, Diff]
runs-on: [self-hosted, General, Linux, X64, Debian10]
env:
THREADS: 24
@@ -205,7 +196,7 @@ jobs:
release_build:
name: "Release build"
runs-on: [self-hosted, Linux, X64, Diff]
runs-on: [self-hosted, General, Linux, X64, Debian10]
env:
THREADS: 24
@@ -217,6 +208,21 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
- name: Set up parent
run: |
# Remove parent folder (if it exists).
cd ..
if [ -d parent ]; then
rm -rf parent
fi
# Copy untouched repository to parent folder.
cp -r memgraph parent
# Checkout previous commit
cd parent
git checkout HEAD~1
- name: Build release binaries
run: |
# Activate toolchain.
@@ -230,6 +236,47 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Build parent binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
cd ../parent
./init
# Build parent binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph memgraph__macro_benchmark
- name: Run macro benchmark tests
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Run parent macro benchmark tests
run: |
cd ../parent/tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Compute macro benchmark summary
run: |
./tools/github/macro_benchmark_summary \
--current tests/macro_benchmark/.harness_summary \
--previous ../parent/tests/macro_benchmark/.harness_summary \
--output macro_benchmark_summary.txt
- name: Save macro benchmark summary
uses: actions/upload-artifact@v2
with:
name: "Macro benchmark summary"
path: macro_benchmark_summary.txt
- name: Run GQL Behave tests
run: |
cd tests/gql_behave
@@ -292,18 +339,9 @@ jobs:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
path: |
# multiple paths could be defined
build/logs
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
runs-on: [self-hosted, Linux, X64, Debian10, HP-DL360G6-v2-3]
#continue-on-error: true
env:
THREADS: 24
@@ -340,64 +378,3 @@ jobs:
with:
name: "Jepsen Report"
path: tests/jepsen/Jepsen.tar.gz
release_benchmarks:
name: "Release benchmarks"
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build only memgraph release binarie.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run macro benchmarks
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Upload macro benchmark results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}"

View File

@@ -1,44 +0,0 @@
name: Run clang-tidy on the full codebase
on:
workflow_dispatch:
jobs:
clang_tidy_check:
name: "Clang-tidy check"
runs-on: [self-hosted, Linux, X64, Ubuntu20.04]
env:
THREADS: 24
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v2/activate
# Initialize dependencies.
./init
# Build debug binaries.
cd build
cmake ..
make -j$THREADS
- name: Run clang-tidy
run: |
source /opt/toolchain-v2/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v2/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results
run: cat ./build/full_clang_tidy_output.txt | ./tools/github/clang-tidy/count_errors.sh

View File

@@ -1,248 +0,0 @@
name: Package All
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
on: workflow_dispatch
jobs:
centos-7_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-7_community
path: build/output/centos-7/memgraph*.rpm
centos-8_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-8_community
path: build/output/centos-8/memgraph*.rpm
debian-9_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9_community
path: build/output/debian-9/memgraph*.deb
debian-10_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10_community
path: build/output/debian-10/memgraph*.deb
docker_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package community debian-10 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: docker_community
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804_community
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004_community:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package community ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004_community
path: build/output/ubuntu-20.04/memgraph*.deb
centos-7_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise centos-7
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-7_enterprise
path: build/output/centos-7/memgraph*.rpm
centos-8_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-8_enterprise
path: build/output/centos-8/memgraph*.rpm
debian-9_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9_enterprise
path: build/output/debian-9/memgraph*.deb
debian-10_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise debian-10
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-10_enterprise
path: build/output/debian-10/memgraph*.deb
docker_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
cd release/package
./run.sh package enterprise debian-10 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: docker_enterprise
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804_enterprise
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004_enterprise:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package enterprise ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004_enterprise
path: build/output/ubuntu-20.04/memgraph*.deb

View File

@@ -1,4 +1,4 @@
name: Release Debian 10
name: Release Debian10
on:
workflow_dispatch:

View File

@@ -1,4 +1,4 @@
name: Release Ubuntu 20.04
name: Release Ubuntu20.04
on:
workflow_dispatch:

View File

@@ -2,16 +2,6 @@
## Future
### Bug Fixes
* Fixed parsing of types for Python procedures for types nested in `mgp.List`.
For example, parsing of `mgp.List[mgp.Map]` works now.
* Fixed memory tracking issues. Some of the allocation and deallocation weren't
tracked during the query execution.
* Fixed reading CSV files that are using CRLF as the newline symbol.
## v1.4.0
### Breaking Changes
* Changed `MEMORY LIMIT num (KB|MB)` clause in the procedure calls to `PROCEDURE MEMORY LIMIT num (KB|MB)`.
@@ -20,7 +10,7 @@
### Major Feature and Improvements
* Added replication to community version.
* Added support for multiple query modules directories at the same time.
* Add support for multiple query modules directories at the same time.
You can now define multiple, comma-separated paths to directories from
which the modules will be loaded using the `--query-modules-directory` flag.
* Added support for programatically reading in data from CSV files through the
@@ -32,15 +22,12 @@
* Added the memory limit and amount of currently allocated bytes in the result of `SHOW STORAGE INFO` query.
* Added `QUERY MEMORY LIMIT num (KB|MB)` to Cypher queries which allows you to limit memory allocation for
the entire query. It can be added only at the end of the entire Cypher query.
* Added logs for the different parts of the recovery process. `INFO`, `DEBUG` and `TRACE` level all contain
additional information that is printed out while the recovery is in progress.
### Bug Fixes
* Fixed garbage collector by correctly marking the oldest current timestamp
after the database was recovered using the durability files.
* Fixed reloading of the modules with changed result names.
* Fixed profile query to show the correct name of the ScanAll operator variant.
## v1.3.0

View File

@@ -312,9 +312,8 @@ if (UBSAN)
# runtime library and c++ standard libraries are present.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
# Make sure llvm-symbolizer binary is in path.
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1
# Make sure llvm-symbolizer binary is in path
endif()
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")

View File

@@ -18,7 +18,6 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
@@ -27,7 +26,6 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconfig # build system
@@ -50,11 +48,9 @@ MEMGRAPH_BUILD_DEPS=(
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
@@ -79,13 +75,16 @@ check() {
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
if [ "$SUDO_USER" == "" ]; then
echo "Please run as sudo."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -119,16 +118,11 @@ install() {
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,7 +17,6 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
@@ -26,7 +25,6 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
@@ -49,11 +47,9 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
@@ -72,13 +68,16 @@ check() {
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
if [ "$SUDO_USER" == "" ]; then
echo "Please run as sudo."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -87,7 +86,6 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
dnf install -y epel-release
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --set-enabled powertools # Required to install texinfo.
dnf update -y
dnf install -y wget git python36 python3-pip
@@ -137,16 +135,11 @@ install() {
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
continue
fi
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,7 +17,6 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -27,7 +26,6 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -47,15 +45,12 @@ MEMGRAPH_BUILD_DEPS=(
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cat >/etc/apt/sources.list <<EOF
deb http://deb.debian.org/debian/ buster main non-free contrib
@@ -88,6 +83,5 @@ EOF
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,7 +17,6 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -27,7 +26,6 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -45,18 +43,14 @@ MEMGRAPH_BUILD_DEPS=(
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
install_all_apt "$1"
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -8,29 +8,23 @@ source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
pkg
)
TOOLCHAIN_RUN_DEPS=(
pkg
)
MEMGRAPH_BUILD_DEPS=(
pkg
)
list() {
echo "$1"
}
check() {
echo "TODO: Implement ${FUNCNAME[0]}."
exit 1
}
install() {
echo "TODO: Implement ${FUNCNAME[0]}."
exit 1
}
# http://ahmed.amayem.com/bash-indirect-expansion-exploration
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -18,7 +18,6 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # llvm
libedit-dev libpcre3-dev automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -28,7 +27,6 @@ TOOLCHAIN_RUN_DEPS=(
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -46,18 +44,14 @@ MEMGRAPH_BUILD_DEPS=(
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
apt install -y $1
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -17,7 +17,6 @@ TOOLCHAIN_BUILD_DEPS=(
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
@@ -27,7 +26,6 @@ TOOLCHAIN_RUN_DEPS=(
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
@@ -47,15 +45,12 @@ MEMGRAPH_BUILD_DEPS=(
dotnet-sdk-3.1 golang nodejs npm
autoconf # for jemalloc code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cd "$DIR"
apt update
@@ -80,6 +75,5 @@ install() {
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -683,15 +683,7 @@ def _typing_to_cypher_type(type_):
return _mgp.type_nullable(simple_type)
return _mgp.type_nullable(parse_typing(type_arg_as_str))
elif type_as_str.startswith('typing.List'):
type_arg_as_str = parse_type_args(type_as_str)
if len(type_arg_as_str) > 1:
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
# so we need to join the parts.
type_arg_as_str = ', '.join(type_arg_as_str)
else:
type_arg_as_str = type_arg_as_str[0]
type_arg_as_str, = parse_type_args(type_as_str)
simple_type = get_simple_type(type_arg_as_str)
if simple_type is not None:
return _mgp.type_list(simple_type)

View File

@@ -2,9 +2,8 @@
# Download external dependencies.
local_cache_host=${MGDEPS_CACHE_HOST_PORT:-mgdeps-cache:8000}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "${working_dir}"
cd ${working_dir}
# Clones a git repository and optionally cherry picks additional commits. The
# function will try to preserve any local changes in the repo.
@@ -16,11 +15,7 @@ clone () {
shift 3
# Clone if there's no repo.
if [[ ! -d "$dir_name" ]]; then
echo "Cloning from $git_repo"
# If the clone fails, it doesn't make sense to continue with the function
# execution but the whole script should continue executing because we might
# clone the same repo from a different source.
git clone "$git_repo" "$dir_name" || return 1
git clone "$git_repo" "$dir_name"
fi
pushd "$dir_name"
# Just fetch new commits from remote repository. Don't merge/pull them in, so
@@ -34,17 +29,12 @@ clone () {
# Stash regardless of local_changes, so that a user gets a message on stdout.
git stash
# Checkout the primary commit (there's no need to pull/merge).
# The checkout fail should exit this script immediately because the target
# commit is not there and that will most likely create build-time errors.
git checkout "$checkout_id" || exit 1
git checkout $checkout_id
# Apply any optional cherry pick fixes.
while [[ $# -ne 0 ]]; do
local cherry_pick_id=$1
shift
# The cherry-pick fail should exit this script immediately because the
# target commit is not there and that will most likely create build-time
# errors.
git cherry-pick -n "$cherry_pick_id" || exit 1
git cherry-pick -n $cherry_pick_id
done
# Reapply any local changes.
if [[ $local_changes == true ]]; then
@@ -53,95 +43,12 @@ clone () {
popd
}
file_get_try_double () {
primary_url="$1"
secondary_url="$2"
echo "Download primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
filename="$(basename "$secondary_url")"
wget -nv "$primary_url" -O "$filename" || wget -nv "$secondary_url" -O "$filename" || exit 1
echo ""
}
repo_clone_try_double () {
primary_url="$1"
secondary_url="$2"
folder_name="$3"
ref="$4"
echo "Cloning primary from $primary_url secondary from $secondary_url"
if [ -z "$primary_url" ]; then echo "Primary should not be empty." && exit 1; fi
if [ -z "$secondary_url" ]; then echo "Secondary should not be empty." && exit 1; fi
if [ -z "$folder_name" ]; then echo "Clone folder should not be empty." && exit 1; fi
if [ -z "$ref" ]; then echo "Git clone ref should not be empty." && exit 1; fi
clone "$primary_url" "$folder_name" "$ref" || clone "$secondary_url" "$folder_name" "$ref" || exit 1
echo ""
}
# List all dependencies.
# The reason for introducing primary and secondary urls are:
# * HTTPS is hard to cache
# * Remote development workflow is more flexible if people don't have to connect to VPN
# * Direct download from the "source of truth" is slower and unreliable because of the whole internet in-between
# * When a new dependency has to be added, both urls could be the same, later someone could optimize if required
# The goal of having primary urls is to have links to the "local" cache of
# dependencies where these dependencies could be downloaded as fast as
# possible. The actual cache server could be on your local machine, on a
# dedicated machine inside the build cluster or on the actual build machine.
# Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.6-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["fmt"]="http://$local_cache_host/git/fmt.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
["gtest"]="http://$local_cache_host/git/googletest.git"
["gflags"]="http://$local_cache_host/git/gflags.git"
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
["bzip2"]="http://$local_cache_host/git/bzip2.git"
["zlib"]="http://$local_cache_host/git/zlib.git"
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
["mgclient"]="http://$local_cache_host/git/mgclient.git"
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
)
# The goal of secondary urls is to have links to the "source of truth" of
# dependencies, e.g., Github or S3. Download from secondary urls, if happens
# at all, should never fail. In other words, if it fails, the whole build
# should fail.
declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.6-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["fmt"]="https://github.com/fmtlib/fmt.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
["gtest"]="https://github.com/google/googletest.git"
["gflags"]="https://github.com/memgraph/gflags.git"
["libbcrypt"]="https://github.com/rg3/libbcrypt"
["bzip2"]="https://github.com/VFR-maniac/bzip2"
["zlib"]="https://github.com/madler/zlib.git"
["rocksdb"]="https://github.com/facebook/rocksdb.git"
["mgclient"]="https://github.com/memgraph/mgclient.git"
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["spdlog"]="https://github.com/gabime/spdlog"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
)
# antlr
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr_generator_filename="antlr-4.6-complete.jar"
# wget -O ${antlr_generator_filename} http://www.antlr.org/download/${antlr_generator_filename}
wget -nv -O ${antlr_generator_filename} https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${antlr_generator_filename}
antlr4_tag="aacd2a2c95816d8dc1c05814051d631bfec4cf3e" # v4.6
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag"
clone https://github.com/antlr/antlr4.git antlr4 $antlr4_tag
# fix missing include
sed -i 's/^#pragma once/#pragma once\n#include <functional>/' antlr4/runtime/Cpp/runtime/src/support/CPPUtils.h
# remove shared library from install dependencies
@@ -149,73 +56,74 @@ sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/'
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
clone https://github.com/ryanhaining/cppitertools.git cppitertools $cppitertools_ref
# fmt
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag"
fmt_tag="7bdf0628b1276379886c7f6dda2cef2b3b374f0b" # (2020-11-25)
clone https://github.com/fmtlib/fmt.git fmt $fmt_tag
# rapidcheck
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
clone https://github.com/emil-e/rapidcheck.git rapidcheck $rapidcheck_tag
# google benchmark
benchmark_tag="4f8bfeae470950ef005327973f15b0044eceaceb" # v1.1.0
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag"
clone https://github.com/google/benchmark.git benchmark $benchmark_tag
# google test
googletest_tag="ec44c6c1675c25b9827aacd08c02433cccde7780" # v1.8.0
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag"
clone https://github.com/google/googletest.git googletest $googletest_tag
# google flags
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
clone https://github.com/memgraph/gflags.git gflags $gflags_tag
# libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
clone https://github.com/rg3/libbcrypt libbcrypt $libbcrypt_tag
# neo4j
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
tar -xzf neo4j-community-3.2.3-unix.tar.gz
wget -nv https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz -O neo4j.tar.gz
tar -xzf neo4j.tar.gz
rm -rf neo4j
mv neo4j-community-3.2.3 neo4j
rm neo4j-community-3.2.3-unix.tar.gz
rm neo4j.tar.gz
# nlohmann json
# We wget header instead of cloning repo since repo is huge (lots of test data).
# We use head on Sep 1, 2017 instead of last release since it was long time ago.
mkdir -p json
cd json
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
wget "https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
clone https://github.com/VFR-maniac/bzip2 bzip2 $bzip2_tag
zlib_tag="cacf7f1d4e3d44d871b605da3b647f07d718623f" # v1.2.11.
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag"
clone https://github.com/madler/zlib.git zlib $zlib_tag
# remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="f3e33549c151f30ac4eb7c22356c6d0331f37652" # (2020-10-14)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag"
clone https://github.com/facebook/rocksdb.git rocksdb $rocksdb_tag
# remove shared library from install dependencies
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
# mgclient
mgclient_tag="v1.2.0" # (2021-01-14)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
clone https://github.com/memgraph/mgclient.git mgclient $mgclient_tag
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
# pymgclient
pymgclient_tag="4f85c179e56302d46a1e3e2cf43509db65f062b3" # (2021-01-15)
repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient]}" "pymgclient" "$pymgclient_tag"
clone https://github.com/memgraph/pymgclient.git pymgclient $pymgclient_tag
spdlog_tag="46d418164dd4cd9822cf8ca62a116a3f71569241" # (2020-12-01)
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag"
clone https://github.com/gabime/spdlog spdlog $spdlog_tag
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
clone https://github.com/jemalloc/jemalloc.git jemalloc $jemalloc_tag
pushd jemalloc
# ThreadPool select job randomly, and there can be some threads that had been
# performed some memory heavy task before and will be inactive for some time,
@@ -230,5 +138,5 @@ pushd jemalloc
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000"
popd

View File

@@ -1,40 +1,33 @@
# Memgraph Community User License Agreement
# User License Agreement
This License Agreement governs your use of the Memgraph Community Release (the
"Software") and documentation ("Documentation").
1. Description
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU ("LICENSEE") AGREE TO THESE
TERMS.
THIS LICENSE AGREEMENT GOVERNS LICENSEES USE OF THE MEMGRAPH COMMUNITY
RELEASE AND DOCUMENTATION.
1. License Grant
2. License Grant
The Software and Documentation are provided to Licensee at no charge and are
licensed, not sold to Licensee. No ownership of any part of the Software and
Documentation is hereby transferred to Licensee. Subject to (i) the terms and
conditions of this License Agreement, and (ii) any additional license
restrictions and parameters contained on Licensors quotation, website, or
order form, Licensor hereby grants Licensee a personal, non-assignable,
conditions of this License Agreement, (ii) any additional license restrictions
and parameters contained on Licensors quotation, website, or order form
(“Order Form”), Licensor hereby grants Licensee a personal, non-assignable,
non-transferable and non-exclusive license to install, access and use the
Software (in object code form only) and Documentation for Licensees internal
business purposes (including for use in a production environment) only. All
rights relating to the Software and Documentation that are not expressly
licensed in this License Agreement, whether now existing or which may hereafter
come into existence are reserved for Licensor. Licensee shall not remove,
obscure, or alter any proprietary rights notices (including without limitation
copyright and trademark notices), which may be affixed to or contained within
the Software or Documentation.
business purposes only. All rights relating to the Software and Documentation
that are not expressly licensed in this License Agreement, whether now existing
or which may hereafter come into existence are reserved for Licensor. Licensee
shall not remove, obscure, or alter any proprietary rights notices (including
without limitation copyright and trademark notices), which may be affixed to or
contained within the Software or Documentation.
Licensor may terminate this License Agreement with immediate effect upon
written notice to the Licensee. Upon termination Licensee shall delete all
electronic copies of all or any part of the Software and/or the Documentation
resident in its systems or elsewhere.
2. Restrictions
3. Restrictions
Licensee will not, directly or indirectly, (a) copy the Software or
Documentation in any manner or for any purpose; (b) install, access or use any
component of the Software or Documentation for any purpose not expressly
granted in Section 1 above; (c) resell, distribute, publicly display or
granted in Section 2 above; (c) resell, distribute, publicly display or
publicly perform the Software or Documentation or any component thereof, by
transfer, lease, loan or any other means, or make it available for use by
others in any time-sharing, service bureau or similar arrangement; (d)
@@ -44,55 +37,25 @@ algorithms or techniques incorporated in the Software; (e) export the Software
or Documentation in violation of any applicable laws or regulations; (f)
modify, translate, adapt, or create derivative works from the Software or
Documentation; (g) circumvent, disable or otherwise interfere with
security-related features of the Software or Documentation; (h) use the
security-related features of the Software or Documentation; (h)
reverse-engineer, disassemble, attempt to derive the source code; (i) use the
Software or Documentation for any illegal purpose, in any manner that is
inconsistent with the terms of this License Agreement, or to engage in illegal
activity; (i) remove or alter any trademark, logo, copyright or other
activity; (j) remove or alter any trademark, logo, copyright or other
proprietary notices, legends, symbols or labels on, or embedded in, the
Software or Documentation; or (j) provide access to the Software or
Software or Documentation; or (k) provide access to the Software or
Documentation to third parties.
3. Warranty Disclaimer
4. Warranty Disclaimer
THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" AND LICENSOR MAKES NO
WARRANTIES, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON
INFRINGEMENT OF THIRD PARTIES INTELLECTUAL PROPERTY RIGHTS OR OTHER
PROPRIETARY RIGHTS. NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION
FURNISHED UNDER IT IS INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE
OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
THE MEMGRAPH COMMUNITY RELEASE AND DOCUMENTATION ARE PROVIDED AS IS” FOR
DEVELOPMENT, TESTING AND EVALUATION PURPOSES ONLY. IT IS NOT LICENSED FOR
PRODUCTION USE AND LICENSOR MAKES NO AND DISCLAIMS ALL WARRANTIES, EXPRESS OR
IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NONINFRINGEMENT OF
THIRD PARTIES INTELLECTUAL PROPERTY RIGHTS OR OTHER PROPRIETARY RIGHTS.
NEITHER THIS LICENSE AGREEMENT NOR ANY DOCUMENTATION FURNISHED UNDER IT IS
INTENDED TO EXPRESS OR IMPLY ANY WARRANTY THAT THE OPERATION OF THE SOFTWARE
WILL BE UNINTERRUPTED, TIMELY, OR ERROR-FREE.
4. Limitation of Liability
Licensor shall not in any circumstances be liable, whether in tort (including
for negligence or breach of statutory duty howsoever arising), contract,
misrepresentation (whether innocent or negligent) or otherwise for: loss of
profits, loss of business, depletion of goodwill or similar losses, loss of
anticipated savings, loss of goods, loss or corruption of data or computer
downtime, or any special, indirect, consequential or pure economic loss, costs,
damages, charges or expenses.
Licensor's total aggregate liability in contract, tort (including without
limitation negligence or breach of statutory duty howsoever arising),
misrepresentation (whether innocent or negligent), restitution or otherwise,
arising in connection with the performance or contemplated performance of this
License Agreement shall in all circumstances be limited to GBP10.00 (ten pounds
sterling).
Nothing in this License Agreement shall limit Licensors liability in the case
of death or personal injury caused by negligence, fraud, or fraudulent
misrepresentation, or where it otherwise cannot be limited by law.
5. Technical Data
Licensor may collect and use technical information (such as usage patterns)
gathered when the Licensee downloads and uses the Software. This is generally
statistical data which does not identify an identified or identifiable
individual. It may also include Licensees IP address which is personal data
and is processed in accordance with our Privacy Policy. We only use this
technical information to improve our products.
6. Law and Jurisdiction
This License Agreement is governed by the laws of England and is subject to the
non-exclusive jurisdiction of the courts of England.
BY DOWNLOADING AND/OR ACCESSING THIS SOFTWARE, YOU AGREE TO SUCH TERMS.

View File

@@ -1,5 +1,4 @@
FROM debian:buster
# NOTE: If you change the base distro update release/package as well.
ARG deb_release

View File

@@ -1,5 +1,4 @@
FROM debian:buster
# NOTE: If you change the base distro update release/package as well.
ARG deb_release

View File

@@ -192,19 +192,7 @@ if args.version:
try:
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
if current_branch != "master":
branches = get_output("git", "branch")
if "master" in branches:
# If master is present locally, the fetch is allowed to fail
# because this script will still be able to compare against the
# master branch.
try:
get_output("git", "fetch", "origin", "master:master")
except Exception:
pass
else:
# If master is not present locally, the fetch command has to
# succeed because something else will fail otherwise.
get_output("git", "fetch", "origin", "master:master")
get_output("git", "fetch", "origin", "master:master")
except Exception:
print("Fatal error while ensuring local master branch.")
sys.exit(1)

View File

@@ -1,12 +0,0 @@
FROM centos:7
RUN yum -y update \
&& yum install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-7.tar.gz \
-O toolchain-v2-binaries-centos-7.tar.gz \
&& tar xzvf toolchain-v2-binaries-centos-7.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,12 +0,0 @@
FROM centos:8
RUN dnf -y update \
&& dnf install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-centos-8.tar.gz \
-O toolchain-v2-binaries-centos-8.tar.gz \
&& tar xzvf toolchain-v2-binaries-centos-8.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,15 +0,0 @@
FROM debian:10
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-10.tar.gz \
-O toolchain-v2-binaries-debian-10.tar.gz \
&& tar xzvf toolchain-v2-binaries-debian-10.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,15 +0,0 @@
FROM debian:9
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-debian-9.tar.gz \
-O toolchain-v2-binaries-debian-9.tar.gz \
&& tar xzvf toolchain-v2-binaries-debian-9.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,26 +0,0 @@
version: "3"
services:
mgbuild_centos-7:
build:
context: centos-7
container_name: "mgbuild_centos-7"
mgbuild_centos-8:
build:
context: centos-8
container_name: "mgbuild_centos-8"
mgbuild_debian-9:
build:
context: debian-9
container_name: "mgbuild_debian-9"
mgbuild_debian-10:
build:
context: debian-10
container_name: "mgbuild_debian-10"
mgbuild_ubuntu-18.04:
build:
context: ubuntu-18.04
container_name: "mgbuild_ubuntu-18.04"
mgbuild_ubuntu-20.04:
build:
context: ubuntu-20.04
container_name: "mgbuild_ubuntu-20.04"

View File

@@ -1,152 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OFFERING=(community enterprise)
SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 ubuntu-18.04 ubuntu-20.04)
PROJECT_ROOT="$SCRIPT_DIR/../.."
ACTIVATE_TOOLCHAIN="source /opt/toolchain-v2/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () {
echo "$0 init|package {offering} {os} [--for-docker]|docker|test"
echo ""
echo " offerings: ${SUPPORTED_OFFERING[*]}"
echo " OSs: ${SUPPORTED_OS[*]}"
exit 1
}
make_package () {
offering="$1"
offering_flag=" -DMG_ENTERPRISE=OFF "
if [[ "$offering" == "enterprise" ]]; then
offering_flag=" -DMG_ENTERPRISE=ON "
fi
if [[ "$offering" == "community" ]]; then
offering_flag=" -DMG_ENTERPRISE=OFF "
fi
os="$2"
package_command=""
if [[ "$os" =~ ^"centos".* ]]; then
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
docker_flag=" -DBUILD_FOR_DOCKER=OFF "
if [[ "$#" -gt 2 ]]; then
if [[ "$3" == "--for-docker" ]]; then
docker_flag=" -DBUILD_FOR_DOCKER=ON "
fi
fi
build_container="mgbuild_$os"
echo "Building Memgraph $offering for $os on $build_container..."
echo "Copying project files..."
# If master is not the current branch, fetch it, because the get_version
# script depends on it. If we are on master, the fetch command is going to
# fail so that's why there is the explicit check.
# Required here because Docker build container can't access remote.
cd "$PROJECT_ROOT"
if [[ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]]; then
git fetch origin master:master
fi
docker exec "$build_container" mkdir -p /memgraph
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
container_build_dir="/memgraph/build"
container_output_dir="$container_build_dir/output"
# TODO(gitbuda): TOOLCHAIN_RUN_DEPS should be installed during the Docker
# image build phase, but that is not easy at this point because the
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
echo "Installing dependencies..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
docker exec "$build_container" bash -c "cd /memgraph && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $offering_flag $docker_flag .."
# ' is used instead of " because we need to run make within the allowed
# container resources.
# shellcheck disable=SC2016
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN "'&& make -j$(nproc)'
docker exec "$build_container" bash -c "mkdir -p $container_output_dir && cd $container_output_dir && $ACTIVATE_TOOLCHAIN && $package_command"
echo "Copying targeted package to host..."
last_package_name=$(docker exec "$build_container" bash -c "cd $container_output_dir && ls -t memgraph* | head -1")
# The operating system folder is introduced because multiple different
# packages could be preserved during the same build "session".
mkdir -p "$HOST_OUTPUT_DIR/$os"
package_host_destination="$HOST_OUTPUT_DIR/$os/$last_package_name"
docker cp "$build_container:$container_output_dir/$last_package_name" "$package_host_destination"
echo "Package saved to $package_host_destination."
}
case "$1" in
init)
cd "$SCRIPT_DIR"
docker-compose build
docker-compose up -d
;;
docker)
# NOTE: Docker is build on top of Debian 10 package.
based_on_os="debian-10"
# shellcheck disable=SC2012
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
# shellcheck disable=SC2012
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
docker_host_folder="$HOST_OUTPUT_DIR/docker"
docker_host_image_path="$docker_host_folder/$docker_image_name"
mkdir -p "$docker_host_folder"
cp "$docker_build_folder/$docker_image_name" "$docker_host_image_path"
echo "Docker images saved to $docker_host_image_path."
;;
package)
shift 1
if [[ "$#" -lt 2 ]]; then
print_help
fi
offering="$1"
shift 1
is_offering_ok=false
for supported_offering in "${SUPPORTED_OFFERING[@]}"; do
if [[ "$supported_offering" == "${offering}" ]]; then
is_offering_ok=true
fi
done
os="$1"
shift 1
is_os_ok=false
for supported_os in "${SUPPORTED_OS[@]}"; do
if [[ "$supported_os" == "${os}" ]]; then
is_os_ok=true
fi
done
if [[ "$is_offering_ok" == true ]] && [[ "$is_os_ok" == true ]]; then
make_package "$offering" "$os" "$@"
else
print_help
fi
;;
test)
echo "TODO(gitbuda): Test all packages on mgtest containers."
;;
*)
print_help
;;
esac

View File

@@ -1,15 +0,0 @@
FROM ubuntu:18.04
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-18.04.tar.gz \
-O toolchain-v2-binaries-ubuntu-18.04.tar.gz \
&& tar xzvf toolchain-v2-binaries-ubuntu-18.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,15 +0,0 @@
FROM ubuntu:20.04
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/toolchain-v2/toolchain-v2-binaries-ubuntu-20.04.tar.gz \
-O toolchain-v2-binaries-ubuntu-20.04.tar.gz \
&& tar xzvf toolchain-v2-binaries-ubuntu-20.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,4 +1,4 @@
#!/usr/bin/python3
#!/usr/bin/env python3
import json
import io

View File

@@ -1,4 +1,4 @@
#!/usr/bin/python3
#!/usr/bin/env python3
import json
import io
import ssl

View File

@@ -27,7 +27,6 @@
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/readable_size.hpp"
#include "utils/string.hpp"
#include "utils/tsc.hpp"
@@ -604,8 +603,8 @@ struct PullPlanVector {
struct PullPlan {
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<size_t> memory_limit = {});
DbAccessor *dba, InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory);
std::optional<ExecutionContext> Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary);
@@ -615,7 +614,6 @@ struct PullPlan {
plan::UniqueCursorPtr cursor_ = nullptr;
Frame frame_;
ExecutionContext ctx_;
std::optional<size_t> memory_limit_;
// As it's possible to query execution using multiple pulls
// we need the keep track of the total execution time across
@@ -632,12 +630,11 @@ struct PullPlan {
};
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
const std::optional<size_t> memory_limit)
DbAccessor *dba, InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory)
: plan_(plan),
cursor_(plan->plan().MakeCursor(execution_memory)),
frame_(plan->symbol_table().max_position(), execution_memory),
memory_limit_(memory_limit) {
frame_(plan->symbol_table().max_position(), execution_memory) {
ctx_.db_accessor = dba;
ctx_.symbol_table = plan->symbol_table();
ctx_.evaluation_context.timestamp =
@@ -660,25 +657,21 @@ std::optional<ExecutionContext> PullPlan::Pull(AnyStream *stream, std::optional<
// single `Pull`.
constexpr size_t stack_size = 256 * 1024;
char stack_data[stack_size];
utils::ResourceWithOutOfMemoryException resource_with_exception;
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
// We can throw on every query because a simple queries for deleting will use only
// the stack allocated buffer.
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
if (memory_limit_) {
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
ctx_.evaluation_context.memory = &*maybe_limited_resource;
} else {
ctx_.evaluation_context.memory = &pool_memory;
}
// Returns true if a result was pulled.
const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); };
const auto pull_result = [&]() -> bool {
// We can throw on every query because a simple queries for deleting will use only
// the stack allocated buffer.
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
utils::ResourceWithOutOfMemoryException resource_with_exception;
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
ctx_.evaluation_context.memory = &pool_memory;
return cursor_->Pull(frame_, ctx_);
};
const auto stream_values = [&]() {
// TODO: The streamed values should also probably use the above memory.
@@ -835,24 +828,10 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory) {
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
Frame frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
evaluation_context.parameters = parsed_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
if (memory_limit) {
spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit));
}
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
parsed_query.parameters, &interpreter_context->plan_cache, dba);
utils::MonotonicBufferResource *execution_memory) {
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage),
utils::Downcast<CypherQuery>(parsed_query.query), parsed_query.parameters,
&interpreter_context->plan_cache, dba, parsed_query.is_cacheable);
summary->insert_or_assign("cost_estimate", plan->cost());
auto rw_type_checker = plan::ReadWriteTypeChecker();
@@ -871,8 +850,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
}
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, memory_limit);
auto pull_plan =
std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory);
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
@@ -886,7 +865,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory) {
utils::MonotonicBufferResource *execution_memory) {
const std::string kExplainQueryStart = "explain ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
"Expected stripped query to start with '{}'", kExplainQueryStart);
@@ -932,7 +911,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory) {
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
const std::string kProfileQueryStart = "profile ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -970,15 +949,6 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
Frame frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp =
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
evaluation_context.parameters = parsed_inner_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::View::OLD);
const auto memory_limit = EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
@@ -990,14 +960,14 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), summary, dba,
interpreter_context, execution_memory, memory_limit,
interpreter_context, execution_memory,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
ctx = std::optional<ExecutionContext>{}, pull_plan = std::shared_ptr<PullPlanVector>(nullptr)](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
// No output symbols are given so that nothing is streamed.
if (!ctx) {
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, memory_limit)
ctx = PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(ctx->stats, ctx->profile_execution_time));
}
@@ -1015,7 +985,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
}
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
utils::MemoryResource *execution_memory) {
utils::MonotonicBufferResource *execution_memory) {
return PreparedQuery{{"QUERY"},
std::move(parsed_query.required_privileges),
[pull_plan = std::make_shared<PullPlanDump>(dba)](
@@ -1030,7 +1000,7 @@ PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, T
PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
utils::MemoryResource *execution_memory) {
utils::MonotonicBufferResource *execution_memory) {
if (in_explicit_transaction) {
throw IndexInMulticommandTxException();
}
@@ -1099,7 +1069,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory) {
DbAccessor *dba, utils::MonotonicBufferResource *execution_memory) {
if (in_explicit_transaction) {
throw UserModificationInMulticommandTxException();
}
@@ -1152,8 +1122,6 @@ PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_ex
return std::nullopt;
},
RWType::NONE};
// False positive report for the std::make_shared above
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
}
PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
@@ -1212,7 +1180,7 @@ PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, const bool in_exp
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
storage::Storage *db, utils::MemoryResource *execution_memory) {
storage::Storage *db, utils::MonotonicBufferResource *execution_memory) {
if (in_explicit_transaction) {
throw InfoInMulticommandTxException();
}
@@ -1300,7 +1268,8 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory) {
InterpreterContext *interpreter_context,
utils::MonotonicBufferResource *execution_memory) {
if (in_explicit_transaction) {
throw ConstraintInMulticommandTxException();
}
@@ -1465,12 +1434,10 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
// Handle transaction control queries.
auto query_upper = utils::Trim(utils::ToUpperCase(query_string));
const auto upper_case_query = utils::ToUpperCase(query_string);
const auto trimmed_query = utils::Trim(upper_case_query);
if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") {
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
if (query_upper == "BEGIN" || query_upper == "COMMIT" || query_upper == "ROLLBACK") {
query_execution->prepared_query.emplace(PrepareTransactionQuery(query_upper));
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
}

View File

@@ -317,9 +317,7 @@ class Interpreter final {
private:
struct QueryExecution {
std::optional<PreparedQuery> prepared_query;
utils::MonotonicBufferResource execution_monotonic_memory{kExecutionMemoryBlockSize};
utils::ResourceWithOutOfMemoryException execution_memory{&execution_monotonic_memory};
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
std::map<std::string, TypedValue> summary;
explicit QueryExecution() = default;
@@ -333,7 +331,7 @@ class Interpreter final {
// destroy the prepared query which is using that instance
// of execution memory.
prepared_query.reset();
execution_monotonic_memory.Release();
execution_memory.Release();
}
};

View File

@@ -324,15 +324,11 @@ VertexAccessor &CreateExpand::CreateExpandCursor::OtherVertex(Frame &frame, Exec
template <class TVerticesFun>
class ScanAllCursor : public Cursor {
public:
explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices,
const char *op_name)
: output_symbol_(output_symbol),
input_cursor_(std::move(input_cursor)),
get_vertices_(std::move(get_vertices)),
op_name_(op_name) {}
explicit ScanAllCursor(Symbol output_symbol, UniqueCursorPtr input_cursor, TVerticesFun get_vertices)
: output_symbol_(output_symbol), input_cursor_(std::move(input_cursor)), get_vertices_(std::move(get_vertices)) {}
bool Pull(Frame &frame, ExecutionContext &context) override {
SCOPED_PROFILE_OP(op_name_);
SCOPED_PROFILE_OP("ScanAll");
if (MustAbort(context)) throw HintedAbortError();
@@ -368,7 +364,6 @@ class ScanAllCursor : public Cursor {
TVerticesFun get_vertices_;
std::optional<typename std::result_of<TVerticesFun(Frame &, ExecutionContext &)>::type::value_type> vertices_;
std::optional<decltype(vertices_.value().begin())> vertices_it_;
const char *op_name_;
};
ScanAll::ScanAll(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, storage::View view)
@@ -384,7 +379,7 @@ UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const {
return std::make_optional(db->Vertices(view_));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAll");
std::move(vertices));
}
std::vector<Symbol> ScanAll::ModifiedSymbols(const SymbolTable &table) const {
@@ -407,7 +402,7 @@ UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const {
return std::make_optional(db->Vertices(view_, label_));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAllByLabel");
std::move(vertices));
}
// TODO(buda): Implement ScanAllByLabelProperty operator to iterate over
@@ -471,7 +466,7 @@ UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *m
return std::make_optional(db->Vertices(view_, label_, property_, maybe_lower, maybe_upper));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAllByLabelPropertyRange");
std::move(vertices));
}
ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr<LogicalOperator> &input,
@@ -503,7 +498,7 @@ UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *m
return std::make_optional(db->Vertices(view_, label_, property_, storage::PropertyValue(value)));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAllByLabelPropertyValue");
std::move(vertices));
}
ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol,
@@ -521,7 +516,7 @@ UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) c
return std::make_optional(db->Vertices(view_, label_, property_));
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAllByLabelProperty");
std::move(vertices));
}
ScanAllById::ScanAllById(const std::shared_ptr<LogicalOperator> &input, Symbol output_symbol, Expression *expression,
@@ -547,7 +542,7 @@ UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const {
return std::vector<VertexAccessor>{*maybe_vertex};
};
return MakeUniqueCursorPtr<ScanAllCursor<decltype(vertices)>>(mem, output_symbol_, input_->MakeCursor(mem),
std::move(vertices), "ScanAllById");
std::move(vertices));
}
namespace {

View File

@@ -522,8 +522,7 @@ std::vector<SingleQueryPart> CollectSingleQueryParts(SymbolTable &symbol_table,
query_part->merge_matching.emplace_back(Matching{});
AddMatching({merge->pattern_}, nullptr, symbol_table, storage, query_part->merge_matching.back());
} else if (utils::IsSubtype(*clause, With::kType) || utils::IsSubtype(*clause, query::Unwind::kType) ||
utils::IsSubtype(*clause, query::CallProcedure::kType) ||
utils::IsSubtype(*clause, query::LoadCsv::kType)) {
utils::IsSubtype(*clause, query::CallProcedure::kType)) {
// This query part is done, continue with a new one.
query_parts.emplace_back(SingleQueryPart{});
query_part = &query_parts.back();

View File

@@ -102,47 +102,30 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
// recovery process.
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
Constraints *constraints, utils::SkipList<Vertex> *vertices) {
spdlog::info("Recreating indices from metadata.");
// Recover label indices.
spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
for (const auto &item : indices_constraints.indices.label) {
if (!indices->label_index.CreateIndex(item, vertices->access()))
throw RecoveryFailure("The label index must be created here!");
spdlog::info("A label index is recreated from metadata.");
}
spdlog::info("Label indices are recreated.");
// Recover label+property indices.
spdlog::info("Recreating {} label+property indices from metadata.",
indices_constraints.indices.label_property.size());
for (const auto &item : indices_constraints.indices.label_property) {
if (!indices->label_property_index.CreateIndex(item.first, item.second, vertices->access()))
throw RecoveryFailure("The label+property index must be created here!");
spdlog::info("A label+property index is recreated from metadata.");
}
spdlog::info("Label+property indices are recreated.");
spdlog::info("Indices are recreated.");
spdlog::info("Recreating constraints from metadata.");
// Recover existence constraints.
spdlog::info("Recreating {} existence constraints from metadata.", indices_constraints.constraints.existence.size());
for (const auto &item : indices_constraints.constraints.existence) {
auto ret = CreateExistenceConstraint(constraints, item.first, item.second, vertices->access());
if (ret.HasError() || !ret.GetValue()) throw RecoveryFailure("The existence constraint must be created here!");
spdlog::info("A existence constraint is recreated from metadata.");
}
spdlog::info("Existence constraints are recreated from metadata.");
// Recover unique constraints.
spdlog::info("Recreating {} unique constraints from metadata.", indices_constraints.constraints.unique.size());
for (const auto &item : indices_constraints.constraints.unique) {
auto ret = constraints->unique_constraints.CreateConstraint(item.first, item.second, vertices->access());
if (ret.HasError() || ret.GetValue() != UniqueConstraints::CreationStatus::SUCCESS)
throw RecoveryFailure("The unique constraint must be created here!");
spdlog::info("A unique constraint is recreated from metadata.");
}
spdlog::info("Unique constraints are recreated from metadata.");
spdlog::info("Constraints are recreated from metadata.");
}
std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_directory,
@@ -154,12 +137,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
Indices *indices, Constraints *constraints, Config::Items items,
uint64_t *wal_seq_num) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
wal_directory);
if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) {
spdlog::warn("Snapshot or WAL directory don't exist, there is nothing to recover.");
return std::nullopt;
}
if (!utils::DirExists(snapshot_directory) && !utils::DirExists(wal_directory)) return std::nullopt;
auto snapshot_files = GetSnapshotFiles(snapshot_directory);
@@ -167,7 +145,6 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
RecoveredIndicesAndConstraints indices_constraints;
std::optional<uint64_t> snapshot_timestamp;
if (!snapshot_files.empty()) {
spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory);
// Order the files by name
std::sort(snapshot_files.begin(), snapshot_files.end());
@@ -180,13 +157,13 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
spdlog::warn("The snapshot file {} isn't related to the latest snapshot file!", path);
continue;
}
spdlog::info("Starting snapshot recovery from {}.", path);
spdlog::info("Starting snapshot recovery from {}", path);
try {
recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history, name_id_mapper, edge_count, items);
spdlog::info("Snapshot recovery successful!");
break;
} catch (const RecoveryFailure &e) {
spdlog::warn("Couldn't recover snapshot from {} because of: {}.", path, e.what());
spdlog::warn("Couldn't recover snapshot from {} because of: {}", path, e.what());
continue;
}
}
@@ -204,7 +181,6 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
return recovered_snapshot->recovery_info;
}
} else {
spdlog::info("No snapshot file was found, collecting information from WAL directory {}.", wal_directory);
std::error_code error_code;
if (!utils::DirExists(wal_directory)) return std::nullopt;
// We use this smaller struct that contains only a subset of information
@@ -230,10 +206,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
}
}
MG_ASSERT(!error_code, "Couldn't recover data because an error occurred: {}!", error_code.message());
if (wal_files.empty()) {
spdlog::warn("No snapshot or WAL file found!");
return std::nullopt;
}
if (wal_files.empty()) return std::nullopt;
std::sort(wal_files.begin(), wal_files.end());
// UUID used for durability is the UUID of the last WAL file.
// Same for the epoch id.
@@ -242,10 +215,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
}
auto maybe_wal_files = GetWalFiles(wal_directory, *uuid);
if (!maybe_wal_files) {
spdlog::warn("Couldn't get WAL file info from the WAL directory!");
return std::nullopt;
}
if (!maybe_wal_files) return std::nullopt;
// Array of all discovered WAL files, ordered by sequence number.
auto &wal_files = *maybe_wal_files;
@@ -262,7 +232,6 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
"files that match the last WAL file!");
if (!wal_files.empty()) {
spdlog::info("Checking WAL files.");
{
const auto &first_wal = wal_files[0];
if (first_wal.seq_num != 0) {
@@ -286,7 +255,6 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
}
std::optional<uint64_t> previous_seq_num;
auto last_loaded_timestamp = snapshot_timestamp;
spdlog::info("Trying to load WAL files.");
for (auto &wal_file : wal_files) {
if (previous_seq_num && (wal_file.seq_num - *previous_seq_num) > 1) {
LOG_FATAL("You are missing a WAL file with the sequence number {}!", *previous_seq_num + 1);
@@ -322,8 +290,6 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
// The sequence number needs to be recovered even though `LoadWal` didn't
// load any deltas from that file.
*wal_seq_num = *previous_seq_num + 1;
spdlog::info("All necessary WAL files are loaded successfully.");
}
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices);

View File

@@ -168,15 +168,14 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
});
// Read snapshot info.
const auto info = ReadSnapshotInfo(path);
spdlog::info("Recovering {} vertices and {} edges.", info.vertices_count, info.edges_count);
auto info = ReadSnapshotInfo(path);
// Check for edges.
bool snapshot_has_edges = info.offset_edges != 0;
// Recover mapper.
std::unordered_map<uint64_t, uint64_t> snapshot_id_map;
{
spdlog::info("Recovering mapper metadata.");
if (!snapshot.SetPosition(info.offset_mapper)) throw RecoveryFailure("Couldn't read data from snapshot!");
auto marker = snapshot.ReadMarker();
@@ -192,7 +191,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (!name) throw RecoveryFailure("Invalid snapshot data!");
auto my_id = name_id_mapper->NameToId(*name);
snapshot_id_map.emplace(*id, my_id);
SPDLOG_TRACE("Mapping \"{}\"from snapshot id {} to actual id {}.", *name, *id, my_id);
}
}
auto get_label_from_id = [&snapshot_id_map](uint64_t snapshot_id) {
@@ -219,11 +217,10 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
auto edge_acc = edges->access();
uint64_t last_edge_gid = 0;
if (snapshot_has_edges) {
spdlog::info("Recovering {} edges.", info.edges_count);
if (!snapshot.SetPosition(info.offset_edges)) throw RecoveryFailure("Couldn't read data from snapshot!");
for (uint64_t i = 0; i < info.edges_count; ++i) {
{
const auto marker = snapshot.ReadMarker();
auto marker = snapshot.ReadMarker();
if (!marker || *marker != Marker::SECTION_EDGE) throw RecoveryFailure("Invalid snapshot data!");
}
@@ -233,7 +230,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
last_edge_gid = *gid;
spdlog::debug("Recovering edge {} with properties.", *gid);
auto [it, inserted] = edge_acc.insert(Edge{Gid::FromUint(*gid), nullptr});
if (!inserted) throw RecoveryFailure("The edge must be inserted here!");
@@ -247,8 +243,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (!key) throw RecoveryFailure("Invalid snapshot data!");
auto value = snapshot.ReadPropertyValue();
if (!value) throw RecoveryFailure("Invalid snapshot data!");
SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for edge {}.",
name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
props.SetProperty(get_property_from_id(*key), *value);
}
}
@@ -259,7 +253,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
last_edge_gid = *gid;
spdlog::debug("Ensuring edge {} doesn't have any properties.", *gid);
// Read properties.
{
auto props_size = snapshot.ReadUint();
@@ -271,14 +264,12 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
}
}
spdlog::info("Edges are recovered.");
}
// Recover vertices (labels and properties).
if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
auto vertex_acc = vertices->access();
uint64_t last_vertex_gid = 0;
spdlog::info("Recovering {} vertices.", info.vertices_count);
for (uint64_t i = 0; i < info.vertices_count; ++i) {
{
auto marker = snapshot.ReadMarker();
@@ -292,12 +283,10 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
throw RecoveryFailure("Invalid snapshot data!");
}
last_vertex_gid = *gid;
spdlog::debug("Recovering vertex {}.", *gid);
auto [it, inserted] = vertex_acc.insert(Vertex{Gid::FromUint(*gid), nullptr});
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
// Recover labels.
spdlog::trace("Recovering labels for vertex {}.", *gid);
{
auto labels_size = snapshot.ReadUint();
if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
@@ -306,14 +295,11 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
for (uint64_t j = 0; j < *labels_size; ++j) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
SPDLOG_TRACE("Recovered label \"{}\" for vertex {}.", name_id_mapper->IdToName(snapshot_id_map.at(*label)),
*gid);
labels.emplace_back(get_label_from_id(*label));
}
}
// Recover properties.
spdlog::trace("Recovering properties for vertex {}.", *gid);
{
auto props_size = snapshot.ReadUint();
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
@@ -323,8 +309,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (!key) throw RecoveryFailure("Invalid snapshot data!");
auto value = snapshot.ReadPropertyValue();
if (!value) throw RecoveryFailure("Invalid snapshot data!");
SPDLOG_TRACE("Recovered property \"{}\" with value \"{}\" for vertex {}.",
name_id_mapper->IdToName(snapshot_id_map.at(*key)), *value, *gid);
props.SetProperty(get_property_from_id(*key), *value);
}
}
@@ -355,10 +339,8 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
}
}
spdlog::info("Vertices are recovered.");
// Recover vertices (in/out edges).
spdlog::info("Recovering connectivity.");
if (!snapshot.SetPosition(info.offset_vertices)) throw RecoveryFailure("Couldn't read data from snapshot!");
for (auto &vertex : vertex_acc) {
{
@@ -366,7 +348,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
}
spdlog::trace("Recovering connectivity for vertex {}.", vertex.gid.AsUint());
// Check vertex.
auto gid = snapshot.ReadUint();
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
@@ -396,7 +377,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
// Recover in edges.
{
spdlog::trace("Recovering inbound edges for vertex {}.", vertex.gid.AsUint());
auto in_size = snapshot.ReadUint();
if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
vertex.in_edges.reserve(*in_size);
@@ -424,15 +404,12 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
edge_ref = EdgeRef(&*edge);
}
}
SPDLOG_TRACE("Recovered inbound edge {} with label \"{}\" from vertex {}.", *edge_gid,
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), from_vertex->gid.AsUint());
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &*from_vertex, edge_ref);
}
}
// Recover out edges.
{
spdlog::trace("Recovering outbound edges for vertex {}.", vertex.gid.AsUint());
auto out_size = snapshot.ReadUint();
if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
vertex.out_edges.reserve(*out_size);
@@ -460,8 +437,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
edge_ref = EdgeRef(&*edge);
}
}
SPDLOG_TRACE("Recovered outbound edge {} with label \"{}\" to vertex {}.", *edge_gid,
name_id_mapper->IdToName(snapshot_id_map.at(*edge_type)), to_vertex->gid.AsUint());
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &*to_vertex, edge_ref);
}
// Increment edge count. We only increment the count here because the
@@ -469,7 +444,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
edge_count->fetch_add(*out_size, std::memory_order_acq_rel);
}
}
spdlog::info("Connectivity is recovered.");
// Set initial values for edge/vertex ID generators.
ret.next_edge_id = last_edge_gid + 1;
@@ -478,7 +452,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
// Recover indices.
{
spdlog::info("Recovering metadata of indices.");
if (!snapshot.SetPosition(info.offset_indices)) throw RecoveryFailure("Couldn't read data from snapshot!");
auto marker = snapshot.ReadMarker();
@@ -488,22 +461,18 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
{
auto size = snapshot.ReadUint();
if (!size) throw RecoveryFailure("Invalid snapshot data!");
spdlog::info("Recovering metadata of {} label indices.", *size);
for (uint64_t i = 0; i < *size; ++i) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
AddRecoveredIndexConstraint(&indices_constraints.indices.label, get_label_from_id(*label),
"The label index already exists!");
SPDLOG_TRACE("Recovered metadata of label index for :{}", name_id_mapper->IdToName(snapshot_id_map.at(*label)));
}
spdlog::info("Metadata of label indices are recovered.");
}
// Recover label+property indices.
{
auto size = snapshot.ReadUint();
if (!size) throw RecoveryFailure("Invalid snapshot data!");
spdlog::info("Recovering metadata of {} label+property indices.", *size);
for (uint64_t i = 0; i < *size; ++i) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
@@ -512,18 +481,12 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
AddRecoveredIndexConstraint(&indices_constraints.indices.label_property,
{get_label_from_id(*label), get_property_from_id(*property)},
"The label+property index already exists!");
SPDLOG_TRACE("Recovered metadata of label+property index for :{}({})",
name_id_mapper->IdToName(snapshot_id_map.at(*label)),
name_id_mapper->IdToName(snapshot_id_map.at(*property)));
}
spdlog::info("Metadata of label+property indices are recovered.");
}
spdlog::info("Metadata of indices are recovered.");
}
// Recover constraints.
{
spdlog::info("Recovering metadata of constraints.");
if (!snapshot.SetPosition(info.offset_constraints)) throw RecoveryFailure("Couldn't read data from snapshot!");
auto marker = snapshot.ReadMarker();
@@ -533,7 +496,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
{
auto size = snapshot.ReadUint();
if (!size) throw RecoveryFailure("Invalid snapshot data!");
spdlog::info("Recovering metadata of {} existence constraints.", *size);
for (uint64_t i = 0; i < *size; ++i) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
@@ -542,11 +504,7 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
AddRecoveredIndexConstraint(&indices_constraints.constraints.existence,
{get_label_from_id(*label), get_property_from_id(*property)},
"The existence constraint already exists!");
SPDLOG_TRACE("Recovered metadata of existence constraint for :{}({})",
name_id_mapper->IdToName(snapshot_id_map.at(*label)),
name_id_mapper->IdToName(snapshot_id_map.at(*property)));
}
spdlog::info("Metadata of existence constraints are recovered.");
}
// Recover unique constraints.
@@ -555,7 +513,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
if (*version >= kUniqueConstraintVersion) {
auto size = snapshot.ReadUint();
if (!size) throw RecoveryFailure("Invalid snapshot data!");
spdlog::info("Recovering metadata of {} unique constraints.", *size);
for (uint64_t i = 0; i < *size; ++i) {
auto label = snapshot.ReadUint();
if (!label) throw RecoveryFailure("Invalid snapshot data!");
@@ -569,15 +526,10 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
AddRecoveredIndexConstraint(&indices_constraints.constraints.unique, {get_label_from_id(*label), properties},
"The unique constraint already exists!");
SPDLOG_TRACE("Recovered metadata of unique constraints for :{}",
name_id_mapper->IdToName(snapshot_id_map.at(*label)));
}
spdlog::info("Metadata of unique constraints are recovered.");
}
spdlog::info("Metadata of constraints are recovered.");
}
spdlog::info("Recovering metadata.");
// Recover epoch history
{
if (!snapshot.SetPosition(info.offset_epoch_history)) throw RecoveryFailure("Couldn't read data from snapshot!");
@@ -603,7 +555,6 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
}
}
spdlog::info("Metadata recovered.");
// Recover timestamp.
ret.next_timestamp = info.start_timestamp + 1;

View File

@@ -610,7 +610,6 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
const std::optional<uint64_t> last_loaded_timestamp, utils::SkipList<Vertex> *vertices,
utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
Config::Items items) {
spdlog::info("Trying to load WAL file {}.", path);
RecoveryInfo ret;
Decoder wal;
@@ -623,17 +622,13 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
ret.last_commit_timestamp = info.to_timestamp;
// Check timestamp.
if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) {
spdlog::info("Skip loading WAL file because it is too old.");
return ret;
}
if (last_loaded_timestamp && info.to_timestamp <= *last_loaded_timestamp) return ret;
// Recover deltas.
wal.SetPosition(info.offset_deltas);
uint64_t deltas_applied = 0;
auto edge_acc = edges->access();
auto vertex_acc = vertices->access();
spdlog::info("WAL file contains {} deltas.", info.num_deltas);
for (uint64_t i = 0; i < info.num_deltas; ++i) {
// Read WAL delta header to find out the delta timestamp.
auto timestamp = ReadWalDeltaHeader(&wal);
@@ -844,8 +839,7 @@ RecoveryInfo LoadWal(const std::filesystem::path &path, RecoveredIndicesAndConst
}
}
spdlog::info("Applied {} deltas from WAL. Skipped {} deltas, because they were too old.", deltas_applied,
info.num_deltas - deltas_applied);
spdlog::info("Applied {} deltas from WAL", deltas_applied, path);
return ret;
}

View File

@@ -61,7 +61,12 @@ void Reader::TryInitializeHeader() {
const Reader::Header &Reader::GetHeader() const { return header_; }
namespace {
enum class CsvParserState : uint8_t { INITIAL_FIELD, NEXT_FIELD, QUOTING, EXPECT_DELIMITER, DONE };
enum class CsvParserState : uint8_t {
INITIAL_FIELD,
NEXT_FIELD,
QUOTING,
EXPECT_DELIMITER,
};
} // namespace
@@ -84,12 +89,7 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
std::string_view line_string_view = *maybe_line;
// remove '\r' from the end in case we have dos file format
if (line_string_view.back() == '\r') {
line_string_view.remove_suffix(1);
}
while (state != CsvParserState::DONE && !line_string_view.empty()) {
while (!line_string_view.empty()) {
const auto c = line_string_view[0];
// Line feeds and carriage returns are ignored in CSVs.
@@ -120,11 +120,11 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
const auto delimiter_idx = line_string_view.find(*read_config_.delimiter);
row.emplace_back(line_string_view.substr(0, delimiter_idx));
if (delimiter_idx == std::string_view::npos) {
state = CsvParserState::DONE;
line_string_view.remove_prefix(line_string_view.size());
} else {
line_string_view.remove_prefix(delimiter_idx + read_config_.delimiter->size());
state = CsvParserState::NEXT_FIELD;
}
state = CsvParserState::NEXT_FIELD;
}
break;
}
@@ -159,20 +159,14 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
}
break;
}
case CsvParserState::DONE: {
LOG_FATAL("Invalid state of the CSV parser!");
}
}
}
} while (state == CsvParserState::QUOTING);
switch (state) {
case CsvParserState::INITIAL_FIELD:
case CsvParserState::DONE:
case CsvParserState::EXPECT_DELIMITER:
break;
case CsvParserState::NEXT_FIELD:
row.emplace_back("");
case CsvParserState::EXPECT_DELIMITER:
break;
case CsvParserState::QUOTING: {
return ParseError(ParseError::ErrorCode::NO_CLOSING_QUOTE,

View File

@@ -65,7 +65,7 @@ void Fatal(const char *msg, const Args &...msg_args) {
do { \
spdlog::critical(__VA_ARGS__); \
std::terminate(); \
} while (0)
} while (0);
#ifndef NDEBUG
#define DLOG_FATAL(...) LOG_FATAL(__VA_ARGS__)

View File

@@ -23,9 +23,6 @@ size_t GrowMonotonicBuffer(size_t current_size, size_t max_size) {
return std::ceil(next_size);
}
__attribute__((no_sanitize("pointer-overflow"))) void CheckAllocationSizeOverflow(void *aligned_ptr, size_t bytes) {
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
}
} // namespace
MonotonicBufferResource::MonotonicBufferResource(size_t initial_size) : initial_size_(initial_size) {}
@@ -124,7 +121,7 @@ void *MonotonicBufferResource::DoAllocate(size_t bytes, size_t alignment) {
next_buffer_size_ = GrowMonotonicBuffer(next_buffer_size_, std::numeric_limits<size_t>::max() - sizeof(Buffer));
}
if (reinterpret_cast<char *>(aligned_ptr) < buffer_head) throw BadAlloc("Allocation alignment overflow");
CheckAllocationSizeOverflow(aligned_ptr, bytes);
if (reinterpret_cast<char *>(aligned_ptr) + bytes <= aligned_ptr) throw BadAlloc("Allocation size overflow");
allocated_ = reinterpret_cast<char *>(aligned_ptr) - data + bytes;
return aligned_ptr;
}

View File

@@ -1,4 +1,3 @@
#include <cstddef>
#include <new>
#if USE_JEMALLOC
@@ -11,7 +10,7 @@
#include "utils/memory_tracker.hpp"
namespace {
void *newImpl(const std::size_t size) {
void *newImpl(std::size_t size) {
auto *ptr = malloc(size);
if (LIKELY(ptr != nullptr)) {
return ptr;
@@ -20,26 +19,11 @@ void *newImpl(const std::size_t size) {
throw std::bad_alloc{};
}
void *newImpl(const std::size_t size, const std::align_val_t align) {
auto *ptr = aligned_alloc(static_cast<std::size_t>(align), size);
if (LIKELY(ptr != nullptr)) {
return ptr;
}
throw std::bad_alloc{};
}
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept {
return aligned_alloc(size, static_cast<std::size_t>(align));
}
void deleteImpl(void *ptr) noexcept { free(ptr); }
#if USE_JEMALLOC
void deleteImpl(void *ptr) noexcept { dallocx(ptr, 0); }
void deleteImpl(void *ptr, const std::align_val_t align) noexcept {
dallocx(ptr, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
void deleteSized(void *ptr, const std::size_t size) noexcept {
if (UNLIKELY(ptr == nullptr)) {
@@ -49,43 +33,24 @@ void deleteSized(void *ptr, const std::size_t size) noexcept {
sdallocx(ptr, size, 0);
}
void deleteSized(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
if (UNLIKELY(ptr == nullptr)) {
return;
}
sdallocx(ptr, size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
#else
void deleteImpl(void *ptr) noexcept { free(ptr); }
void deleteImpl(void *ptr, const std::align_val_t /*unused*/) noexcept { free(ptr); }
void deleteSized(void *ptr, const std::size_t /*unused*/) noexcept { free(ptr); }
void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t /*unused*/) noexcept { free(ptr); }
#endif
void TrackMemory(std::size_t size) {
void TrackMemory(const size_t size) {
size_t actual_size = size;
#if USE_JEMALLOC
if (LIKELY(size != 0)) {
size = nallocx(size, 0);
actual_size = nallocx(size, 0);
}
#endif
utils::total_memory_tracker.Alloc(size);
utils::total_memory_tracker.Alloc(actual_size);
}
void TrackMemory(std::size_t size, const std::align_val_t align) {
#if USE_JEMALLOC
if (LIKELY(size != 0)) {
size = nallocx(size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
#endif
utils::total_memory_tracker.Alloc(size);
}
bool TrackMemoryNoExcept(const std::size_t size) {
bool TrackMemoryNoExcept(const size_t size) {
try {
TrackMemory(size);
} catch (...) {
@@ -95,17 +60,7 @@ bool TrackMemoryNoExcept(const std::size_t size) {
return true;
}
bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
try {
TrackMemory(size, align);
} catch (...) {
return false;
}
return true;
}
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size = 0) noexcept {
try {
#if USE_JEMALLOC
if (LIKELY(ptr != nullptr)) {
@@ -123,74 +78,32 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size
}
}
void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
try {
#if USE_JEMALLOC
if (LIKELY(ptr != nullptr)) {
utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align))); // NOLINT(hicpp-signed-bitwise)
}
#else
if (size) {
utils::total_memory_tracker.Free(size);
} else {
// Innaccurate because malloc_usable_size() result is greater or equal to allocated size.
utils::total_memory_tracker.Free(malloc_usable_size(ptr));
}
#endif
} catch (...) {
}
}
} // namespace
void *operator new(const std::size_t size) {
void *operator new(std::size_t size) {
TrackMemory(size);
return newImpl(size);
}
void *operator new[](const std::size_t size) {
void *operator new[](std::size_t size) {
TrackMemory(size);
return newImpl(size);
}
void *operator new(const std::size_t size, const std::align_val_t align) {
TrackMemory(size, align);
return newImpl(size, align);
}
void *operator new[](const std::size_t size, const std::align_val_t align) {
TrackMemory(size, align);
return newImpl(size, align);
}
void *operator new(const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
void *operator new(std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size))) {
return newNoExcept(size);
}
return nullptr;
}
void *operator new[](const std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
void *operator new[](std::size_t size, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size))) {
return newNoExcept(size);
}
return nullptr;
}
void *operator new(const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size, align))) {
return newNoExcept(size, align);
}
return nullptr;
}
void *operator new[](const std::size_t size, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
if (LIKELY(TrackMemoryNoExcept(size, align))) {
return newNoExcept(size, align);
}
return nullptr;
}
void operator delete(void *ptr) noexcept {
UntrackMemory(ptr);
deleteImpl(ptr);
@@ -201,52 +114,12 @@ void operator delete[](void *ptr) noexcept {
deleteImpl(ptr);
}
void operator delete(void *ptr, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}
void operator delete[](void *ptr, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}
void operator delete(void *ptr, const std::size_t size) noexcept {
void operator delete(void *ptr, std::size_t size) noexcept {
UntrackMemory(ptr, size);
deleteSized(ptr, size);
}
void operator delete[](void *ptr, const std::size_t size) noexcept {
void operator delete[](void *ptr, std::size_t size) noexcept {
UntrackMemory(ptr, size);
deleteSized(ptr, size);
}
void operator delete(void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align, size);
deleteSized(ptr, size, align);
}
void operator delete[](void *ptr, const std::size_t size, const std::align_val_t align) noexcept {
UntrackMemory(ptr, align, size);
deleteSized(ptr, size, align);
}
void operator delete(void *ptr, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr);
deleteImpl(ptr);
}
void operator delete[](void *ptr, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr);
deleteImpl(ptr);
}
void operator delete(void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}
void operator delete[](void *ptr, const std::align_val_t align, const std::nothrow_t & /*unused*/) noexcept {
UntrackMemory(ptr, align);
deleteImpl(ptr, align);
}

View File

@@ -38,5 +38,19 @@ int main(int argc, char **argv) {
}
spdlog::info("Memgraph is out of memory");
spdlog::info("Cleaning up unused memory");
client->Execute("MATCH (n) DETACH DELETE n;");
client->DiscardAll();
client->Execute("FREE MEMORY;");
client->DiscardAll();
// now it should succeed
spdlog::info("Retrying the query with the memory cleaned up");
client->Execute(create_query);
if (!client->FetchOne()) {
LOG_FATAL("Memgraph is still out of memory");
}
return 0;
}

View File

@@ -2,8 +2,7 @@ bolt_port: &bolt_port "7687"
template_cluster: &template_cluster
cluster:
main:
args: ["--bolt-port", *bolt_port, "--memory-limit=1000", "--storage-gc-cycle-sec=180", "--log-level=TRACE"]
log_file: "memory-e2e.log"
args: ["--bolt-port", *bolt_port, "--memory-limit=500", "--storage-gc-cycle-sec=180"]
setup_queries: []
validation_queries: []

View File

@@ -39,7 +39,7 @@ int main(int argc, char **argv) {
const auto label_name = (*data)[0][1].ValueString();
const auto property_name = (*data)[0][2].ValueList()[0].ValueString();
if (label_name != "Node" || property_name != "id") {
LOG_FATAL("{} does NOT hava valid constraint created.", database_endpoint);
LOG_FATAL("{} does NOT hava valid constraint created.", database_endpoint)
}
} else {
LOG_FATAL("Unable to get CONSTRAINT INFO from {}", database_endpoint);

View File

@@ -11,23 +11,19 @@ template_validation_queries: &template_validation_queries
template_cluster: &template_cluster
cluster:
replica_1:
args: ["--bolt-port", "7688", "--log-level=TRACE"]
log_file: "replication-e2e-replica1.log"
args: ["--bolt-port", "7688"]
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"]
<<: *template_validation_queries
replica_2:
args: ["--bolt-port", "7689", "--log-level=TRACE"]
log_file: "replication-e2e-replica2.log"
args: ["--bolt-port", "7689"]
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10002;"]
<<: *template_validation_queries
replica_3:
args: ["--bolt-port", "7690", "--log-level=TRACE"]
log_file: "replication-e2e-replica3.log"
args: ["--bolt-port", "7690"]
setup_queries: ["SET REPLICATION ROLE TO REPLICA WITH PORT 10003;"]
<<: *template_validation_queries
main:
args: ["--bolt-port", "7687", "--log-level=TRACE"]
log_file: "replication-e2e-main.log"
args: ["--bolt-port", "7687"]
setup_queries: [
"REGISTER REPLICA replica_1 SYNC WITH TIMEOUT 0 TO '127.0.0.1:10001'",
"REGISTER REPLICA replica_2 SYNC WITH TIMEOUT 1 TO '127.0.0.1:10002'",

View File

@@ -44,9 +44,7 @@ def run(args):
for name, config in workload['cluster'].items():
mg_instance = MemgraphInstanceRunner(MEMGRAPH_BINARY)
mg_instances[name] = mg_instance
log_file_path = os.path.join(BUILD_DIR, 'logs', config['log_file'])
binary_args = config['args'] + ["--log-file", log_file_path]
mg_instance.start(args=binary_args)
mg_instance.start(args=config['args'])
for query in config['setup_queries']:
mg_instance.query(query)
# Test.

View File

@@ -73,7 +73,7 @@ class Memgraph:
virtualenv_bin = os.path.join(SCRIPT_DIR, "ve3", "bin", "python3")
with open(script_file) as fin:
data = fin.read()
data = data.replace("/usr/bin/python3", virtualenv_bin)
data = data.replace("/usr/bin/env python3", virtualenv_bin)
data = data.replace("/etc/memgraph/auth/ldap.yaml",
self._auth_config)
with open(self._auth_module, "w") as fout:

View File

@@ -45,7 +45,7 @@ def execute_test(**kwargs):
server = None
if start_server:
server = subprocess.Popen(list(map(str, server_args)))
time.sleep(0.4)
time.sleep(0.1)
assert server.poll() is None, "Server process died prematurely!"
try:

View File

@@ -164,4 +164,4 @@
{:bank (bank-checker)
:timeline (timeline/html)})
:generator (c/replication-gen (gen/mix [read-balances valid-transfer]))
:final-generator {:gen (gen/once read-balances) :recovery-time 20}})
:final-generator (gen/once read-balances)})

View File

@@ -22,7 +22,7 @@
"A map of workload names to functions that can take opts and construct
workloads."
{:bank bank/workload
;; :sequential sequential/workload (T0532-MG)
:sequential sequential/workload
:large large/workload})
(def nemesis-configuration
@@ -45,8 +45,8 @@
(gen/log "Healing cluster.")
(gen/nemesis (:final-generator nemesis))
(gen/log "Waiting for recovery")
(gen/sleep (:recovery-time final-generator))
(gen/clients (:gen final-generator)))
(gen/sleep 20)
(gen/clients final-generator))
gen)]
(merge tests/noop-test
opts

View File

@@ -103,4 +103,4 @@
:timeline (timeline/html)})
:generator (c/replication-gen
(gen/mix [read-nodes add-nodes]))
:final-generator {:gen (gen/once read-nodes) :recovery-time 40}})
:final-generator (gen/once read-nodes)})

View File

@@ -383,16 +383,15 @@ TEST(BoltSession, ExecuteRunWrongMarker) {
}
TEST(BoltSession, ExecuteRunMissingData) {
std::array<uint8_t, 6> run_req_without_parameters{
run_req_header[0], run_req_header[1], run_req_header[2], 0x00, 0x00, 0x00};
// test lengths, they test the following situations:
// missing header data, missing query data, missing parameters
int len[] = {1, 2, run_req_without_parameters.size()};
int len[] = {1, 2, 37};
for (int i = 0; i < 3; ++i) {
INIT_VARS;
ExecuteHandshake(input_stream, session, output);
ExecuteInit(input_stream, session, output);
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_without_parameters.data(), len[i]), SessionException);
ASSERT_THROW(ExecuteCommand(input_stream, session, run_req_header, len[i]), SessionException);
ASSERT_EQ(session.state_, State::Close);
CheckFailureMessage(output);
@@ -872,7 +871,7 @@ TEST(BoltSession, Noop) {
CheckFailureMessage(output);
session.state_ = State::Result;
ExecuteCommand(input_stream, session, pullall_req, sizeof(pullall_req));
ExecuteCommand(input_stream, session, pullall_req, sizeof(v4::pullall_req));
CheckSuccessMessage(output);
ASSERT_THROW(ExecuteCommand(input_stream, session, v4_1::noop, sizeof(v4_1::noop)), SessionException);

View File

@@ -73,12 +73,6 @@ class TestPlanner : public ::testing::Test {};
using PlannerTypes = ::testing::Types<Planner>;
void DeleteListContent(std::list<BaseOpChecker *> *list) {
for (BaseOpChecker *ptr : *list) {
delete ptr;
}
}
TYPED_TEST_CASE(TestPlanner, PlannerTypes);
TYPED_TEST(TestPlanner, MatchNodeReturn) {
@@ -229,7 +223,6 @@ TYPED_TEST(TestPlanner, OptionalMatchNamedPatternReturn) {
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectConstructNamedPath()};
CheckPlan(planner.plan(), symbol_table, ExpectOptional(optional_symbols, optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, MatchWhereReturn) {
@@ -556,8 +549,10 @@ TYPED_TEST(TestPlanner, MatchMerge) {
auto acc = ExpectAccumulate({symbol_table.at(*ident_n)});
auto planner = MakePlanner<TypeParam>(&dba, storage, symbol_table, query);
CheckPlan(planner.plan(), symbol_table, ExpectScanAll(), ExpectMerge(on_match, on_create), acc, ExpectProduce());
DeleteListContent(&on_match);
DeleteListContent(&on_create);
for (auto &op : on_match) delete op;
on_match.clear();
for (auto &op : on_create) delete op;
on_create.clear();
}
TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
@@ -569,7 +564,6 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhereReturn) {
WHERE(LESS(PROPERTY_LOOKUP("m", prop), LITERAL(42))), RETURN("r")));
std::list<BaseOpChecker *> optional{new ExpectScanAll(), new ExpectExpand(), new ExpectFilter()};
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectOptional(optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, MatchUnwindReturn) {
@@ -711,7 +705,6 @@ TYPED_TEST(TestPlanner, MatchOptionalMatchWhere) {
// optional ScanAll.
std::list<BaseOpChecker *> optional{new ExpectFilter(), new ExpectScanAll()};
CheckPlan<TypeParam>(query, storage, ExpectScanAll(), ExpectExpand(), ExpectOptional(optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, MatchReturnAsterisk) {
@@ -770,8 +763,8 @@ TYPED_TEST(TestPlanner, UnwindMergeNodeProperty) {
std::list<BaseOpChecker *> on_match{new ExpectScanAll(), new ExpectFilter()};
std::list<BaseOpChecker *> on_create{new ExpectCreateNode()};
CheckPlan<TypeParam>(query, storage, ExpectUnwind(), ExpectMerge(on_match, on_create));
DeleteListContent(&on_match);
DeleteListContent(&on_create);
for (auto &op : on_match) delete op;
for (auto &op : on_create) delete op;
}
TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
@@ -781,7 +774,6 @@ TYPED_TEST(TestPlanner, MultipleOptionalMatchReturn) {
QUERY(SINGLE_QUERY(OPTIONAL_MATCH(PATTERN(NODE("n"))), OPTIONAL_MATCH(PATTERN(NODE("m"))), RETURN("n")));
std::list<BaseOpChecker *> optional{new ExpectScanAll()};
CheckPlan<TypeParam>(query, storage, ExpectOptional(optional), ExpectOptional(optional), ExpectProduce());
DeleteListContent(&optional);
}
TYPED_TEST(TestPlanner, FunctionAggregationReturn) {

View File

@@ -5,8 +5,6 @@
#include "query/procedure/mg_procedure_impl.hpp"
#include "test_utils.hpp"
static void DummyCallback(const mgp_list *, const mgp_graph *, mgp_result *, mgp_memory *) {}
TEST(Module, InvalidProcedureRegistration) {
@@ -55,8 +53,7 @@ TEST(Module, ProcedureSignature) {
CheckSignature(proc, "proc() :: ()");
mgp_proc_add_arg(proc, "arg1", mgp_type_number());
CheckSignature(proc, "proc(arg1 :: NUMBER) :: ()");
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: ()");
mgp_proc_add_result(proc, "res1", mgp_type_list(mgp_type_int()));
CheckSignature(proc, "proc(arg1 :: NUMBER, opt1 = Null :: ANY?) :: (res1 :: LIST OF INTEGER)");
@@ -72,8 +69,7 @@ TEST(Module, ProcedureSignature) {
"(res1 :: LIST OF INTEGER, DEPRECATED res2 :: STRING)");
EXPECT_FALSE(mgp_proc_add_result(proc, "res2", mgp_type_any()));
EXPECT_FALSE(mgp_proc_add_deprecated_result(proc, "res1", mgp_type_any()));
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(),
test_utils::CreateValueOwningPtr(mgp_value_make_string("string=\"value\"", &memory)).get());
mgp_proc_add_opt_arg(proc, "opt2", mgp_type_string(), mgp_value_make_string("string=\"value\"", &memory));
CheckSignature(proc,
"proc(arg1 :: NUMBER, opt1 = Null :: ANY?, "
"opt2 = \"string=\\\"value\\\"\" :: STRING) :: "
@@ -84,7 +80,6 @@ TEST(Module, ProcedureSignatureOnlyOptArg) {
mgp_memory memory{utils::NewDeleteResource()};
mgp_module module(utils::NewDeleteResource());
auto *proc = mgp_module_add_read_procedure(&module, "proc", DummyCallback);
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()),
test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get());
mgp_proc_add_opt_arg(proc, "opt1", mgp_type_nullable(mgp_type_any()), mgp_value_make_null(&memory));
CheckSignature(proc, "proc(opt1 = Null :: ANY?) :: ()");
}

View File

@@ -1,13 +1,7 @@
#include <functional>
#include <memory>
#include <utility>
#include <gtest/gtest.h>
#include "query/procedure/mg_procedure_impl.hpp"
#include "test_utils.hpp"
TEST(CypherType, PresentableNameSimpleTypes) {
EXPECT_EQ(mgp_type_any()->impl->GetPresentableName(), "ANY");
EXPECT_EQ(mgp_type_bool()->impl->GetPresentableName(), "BOOLEAN");
@@ -72,7 +66,6 @@ TEST(CypherType, NullSatisfiesType) {
EXPECT_TRUE(null_type->impl->SatisfiesType(tv_null));
}
}
mgp_value_destroy(mgp_null);
}
}
@@ -108,7 +101,6 @@ TEST(CypherType, BoolSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_bool, tv_bool,
{mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_bool);
}
TEST(CypherType, IntSatisfiesType) {
@@ -119,7 +111,6 @@ TEST(CypherType, IntSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_int, tv_int,
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_int);
}
TEST(CypherType, DoubleSatisfiesType) {
@@ -130,7 +121,6 @@ TEST(CypherType, DoubleSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_double, tv_double,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_double);
}
TEST(CypherType, StringSatisfiesType) {
@@ -141,13 +131,12 @@ TEST(CypherType, StringSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_string, tv_string,
{mgp_type_bool(), mgp_type_int(), mgp_type_float(), mgp_type_number(),
mgp_type_map(), mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_string);
}
TEST(CypherType, MapSatisfiesType) {
mgp_memory memory{utils::NewDeleteResource()};
auto *map = mgp_map_make_empty(&memory);
mgp_map_insert(map, "key", test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get());
mgp_map_insert(map, "key", mgp_value_make_int(42, &memory));
auto *mgp_map_v = mgp_value_make_map(map);
const query::TypedValue tv_map(std::map<std::string, query::TypedValue>{{"key", query::TypedValue(42)}});
CheckSatisfiesTypesAndNullable(mgp_map_v, tv_map, {mgp_type_any(), mgp_type_map()});
@@ -155,7 +144,6 @@ TEST(CypherType, MapSatisfiesType) {
mgp_map_v, tv_map,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_node(),
mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_map_v);
}
TEST(CypherType, VertexSatisfiesType) {
@@ -172,7 +160,6 @@ TEST(CypherType, VertexSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_vertex_v, tv_vertex,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
mgp_type_number(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_vertex_v);
}
TEST(CypherType, EdgeSatisfiesType) {
@@ -191,7 +178,6 @@ TEST(CypherType, EdgeSatisfiesType) {
CheckNotSatisfiesTypesAndListAndNullable(mgp_edge_v, tv_edge,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(),
mgp_type_number(), mgp_type_node(), mgp_type_path()});
mgp_value_destroy(mgp_edge_v);
}
TEST(CypherType, PathSatisfiesType) {
@@ -204,13 +190,9 @@ TEST(CypherType, PathSatisfiesType) {
mgp_memory memory{utils::NewDeleteResource()};
utils::Allocator<mgp_path> alloc(memory.impl);
mgp_graph graph{&dba, storage::View::NEW};
auto *mgp_vertex_v = alloc.new_object<mgp_vertex>(v1, &graph);
auto path = mgp_path_make_with_start(mgp_vertex_v, &memory);
auto *path = mgp_path_make_with_start(alloc.new_object<mgp_vertex>(v1, &graph), &memory);
ASSERT_TRUE(path);
alloc.delete_object(mgp_vertex_v);
auto mgp_edge_v = alloc.new_object<mgp_edge>(edge, &graph);
ASSERT_TRUE(mgp_path_expand(path, mgp_edge_v));
alloc.delete_object(mgp_edge_v);
ASSERT_TRUE(mgp_path_expand(path, alloc.new_object<mgp_edge>(edge, &graph)));
auto *mgp_path_v = mgp_value_make_path(path);
const query::TypedValue tv_path(query::Path(v1, edge, v2));
CheckSatisfiesTypesAndNullable(mgp_path_v, tv_path, {mgp_type_any(), mgp_type_path()});
@@ -218,7 +200,6 @@ TEST(CypherType, PathSatisfiesType) {
mgp_path_v, tv_path,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship()});
mgp_value_destroy(mgp_path_v);
}
static std::vector<const mgp_type *> MakeListTypes(const std::vector<const mgp_type *> &element_types) {
@@ -243,7 +224,6 @@ TEST(CypherType, EmptyListSatisfiesType) {
auto all_types = MakeListTypes(primitive_types);
all_types.push_back(mgp_type_any());
CheckSatisfiesTypesAndNullable(mgp_list_v, tv_list, all_types);
mgp_value_destroy(mgp_list_v);
}
TEST(CypherType, ListOfIntSatisfiesType) {
@@ -253,7 +233,7 @@ TEST(CypherType, ListOfIntSatisfiesType) {
auto *mgp_list_v = mgp_value_make_list(list);
query::TypedValue tv_list(std::vector<query::TypedValue>{});
for (int64_t i = 0; i < elem_count; ++i) {
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(i, &memory)).get()));
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(i, &memory)));
tv_list.ValueList().emplace_back(i);
auto valid_types = MakeListTypes({mgp_type_any(), mgp_type_int(), mgp_type_number()});
valid_types.push_back(mgp_type_any());
@@ -262,7 +242,6 @@ TEST(CypherType, ListOfIntSatisfiesType) {
{mgp_type_bool(), mgp_type_string(), mgp_type_float(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
}
mgp_value_destroy(mgp_list_v);
}
TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
@@ -272,10 +251,10 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
auto *mgp_list_v = mgp_value_make_list(list);
query::TypedValue tv_list(std::vector<query::TypedValue>{});
// Add an int
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_int(42, &memory)).get()));
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_int(42, &memory)));
tv_list.ValueList().emplace_back(42);
// Add a boolean
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_bool(1, &memory)).get()));
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_bool(1, &memory)));
tv_list.ValueList().emplace_back(true);
auto valid_types = MakeListTypes({mgp_type_any()});
valid_types.push_back(mgp_type_any());
@@ -285,7 +264,6 @@ TEST(CypherType, ListOfIntAndBoolSatisfiesType) {
mgp_list_v, tv_list,
{mgp_type_bool(), mgp_type_string(), mgp_type_int(), mgp_type_float(), mgp_type_number(), mgp_type_map(),
mgp_type_node(), mgp_type_relationship(), mgp_type_path()});
mgp_value_destroy(mgp_list_v);
}
TEST(CypherType, ListOfNullSatisfiesType) {
@@ -293,7 +271,7 @@ TEST(CypherType, ListOfNullSatisfiesType) {
auto *list = mgp_list_make_empty(1, &memory);
auto *mgp_list_v = mgp_value_make_list(list);
query::TypedValue tv_list(std::vector<query::TypedValue>{});
ASSERT_TRUE(mgp_list_append(list, test_utils::CreateValueOwningPtr(mgp_value_make_null(&memory)).get()));
ASSERT_TRUE(mgp_list_append(list, mgp_value_make_null(&memory)));
tv_list.ValueList().emplace_back();
// List with Null satisfies all nullable list element types
std::vector<const mgp_type *> primitive_types{
@@ -317,5 +295,4 @@ TEST(CypherType, ListOfNullSatisfiesType) {
EXPECT_FALSE(null_type->impl->SatisfiesType(*mgp_list_v)) << null_type->impl->GetPresentableName();
EXPECT_FALSE(null_type->impl->SatisfiesType(tv_list));
}
mgp_value_destroy(mgp_list_v);
}

View File

@@ -254,7 +254,6 @@ TEST(PyModule, PyObjectToMgpValue) {
const mgp_value *v2 = mgp_map_at(map, "four");
ASSERT_TRUE(mgp_value_is_double(v2));
EXPECT_EQ(mgp_value_get_double(v2), 4.0);
mgp_value_destroy(value);
}
int main(int argc, char **argv) {

View File

@@ -709,15 +709,14 @@ TEST_P(DurabilityTest, SnapshotFallback) {
{.items = {.properties_on_edges = GetParam()},
.durability = {.storage_directory = storage_directory,
.snapshot_wal_mode = storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT,
.snapshot_interval = std::chrono::milliseconds(3000)}});
.snapshot_interval = std::chrono::milliseconds(2000)}});
CreateBaseDataset(&store, GetParam());
std::this_thread::sleep_for(std::chrono::milliseconds(3500));
ASSERT_EQ(GetSnapshotsList().size(), 1);
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
CreateExtendedDataset(&store);
std::this_thread::sleep_for(std::chrono::milliseconds(3000));
std::this_thread::sleep_for(std::chrono::milliseconds(2500));
}
ASSERT_EQ(GetSnapshotsList().size(), 2);
ASSERT_GE(GetSnapshotsList().size(), 2);
ASSERT_EQ(GetBackupSnapshotsList().size(), 0);
ASSERT_EQ(GetWalsList().size(), 0);
ASSERT_EQ(GetBackupWalsList().size(), 0);
@@ -725,7 +724,7 @@ TEST_P(DurabilityTest, SnapshotFallback) {
// Destroy last snapshot.
{
auto snapshots = GetSnapshotsList();
ASSERT_EQ(snapshots.size(), 2);
ASSERT_GE(snapshots.size(), 2);
DestroySnapshot(*snapshots.begin());
}

View File

@@ -1,9 +0,0 @@
#include <memory>
#include "query/procedure/mg_procedure_impl.hpp"
namespace test_utils {
using MgpValueOwningPtr = std::unique_ptr<mgp_value, void (*)(mgp_value *)>;
MgpValueOwningPtr CreateValueOwningPtr(mgp_value *value) { return MgpValueOwningPtr(value, &mgp_value_destroy); }
} // namespace test_utils

View File

@@ -397,8 +397,8 @@ TEST_F(TypedValueLogicTest, LogicalXor) {
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST_F(AllTypesFixture, ConstructionWithMemoryResource) {
utils::MonotonicBufferResource monotonic_memory(1024);
std::vector<TypedValue> values_with_custom_memory;
utils::MonotonicBufferResource monotonic_memory(1024);
for (const auto &value : values_) {
EXPECT_EQ(value.GetMemoryResource(), utils::NewDeleteResource());
TypedValue copy_constructed_value(value, &monotonic_memory);

View File

@@ -4,7 +4,7 @@
#include "utils/string.hpp"
class CsvReaderTest : public ::testing::TestWithParam<const char *> {
class CsvReaderTest : public ::testing::Test {
protected:
const std::filesystem::path csv_directory{std::filesystem::temp_directory_path() / "csv_testing"};
@@ -30,9 +30,7 @@ class CsvReaderTest : public ::testing::TestWithParam<const char *> {
namespace {
class FileWriter {
public:
explicit FileWriter(const std::filesystem::path path, std::string newline = "\n") : newline_{std::move(newline)} {
stream_.open(path);
}
explicit FileWriter(const std::filesystem::path path) { stream_.open(path); }
FileWriter(const FileWriter &) = delete;
FileWriter &operator=(const FileWriter &) = delete;
@@ -47,7 +45,7 @@ class FileWriter {
return 0;
}
stream_ << line << newline_;
stream_ << line << std::endl;
// including the newline character
return line.size() + 1;
@@ -55,7 +53,6 @@ class FileWriter {
private:
std::ofstream stream_;
std::string newline_;
};
std::string CreateRow(const std::vector<std::string> &columns, const std::string_view delim) {
@@ -72,10 +69,10 @@ auto ToPmrColumns(const std::vector<std::string> &columns) {
} // namespace
TEST_P(CsvReaderTest, CommaDelimiter) {
TEST_F(CsvReaderTest, CommaDelimiter) {
// create a file with a single valid row;
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
const std::vector<std::string> columns{"A", "B", "C"};
writer.WriteLine(CreateRow(columns, ","));
@@ -96,9 +93,9 @@ TEST_P(CsvReaderTest, CommaDelimiter) {
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
}
TEST_P(CsvReaderTest, SemicolonDelimiter) {
TEST_F(CsvReaderTest, SemicolonDelimiter) {
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -119,12 +116,12 @@ TEST_P(CsvReaderTest, SemicolonDelimiter) {
ASSERT_EQ(*parsed_row, ToPmrColumns(columns));
}
TEST_P(CsvReaderTest, SkipBad) {
TEST_F(CsvReaderTest, SkipBad) {
// create a file with invalid first two rows (containing a string with a
// missing closing quote);
// the last row is valid;
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -164,11 +161,11 @@ TEST_P(CsvReaderTest, SkipBad) {
}
}
TEST_P(CsvReaderTest, AllRowsValid) {
TEST_F(CsvReaderTest, AllRowsValid) {
// create a file with all rows valid;
// parser should return 'std::nullopt'
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -193,11 +190,11 @@ TEST_P(CsvReaderTest, AllRowsValid) {
}
}
TEST_P(CsvReaderTest, SkipAllRows) {
TEST_F(CsvReaderTest, SkipAllRows) {
// create a file with all rows invalid (containing a string with a missing closing quote);
// parser should return 'std::nullopt'
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -220,9 +217,9 @@ TEST_P(CsvReaderTest, SkipAllRows) {
ASSERT_EQ(parsed_row, std::nullopt);
}
TEST_P(CsvReaderTest, WithHeader) {
TEST_F(CsvReaderTest, WithHeader) {
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -252,12 +249,12 @@ TEST_P(CsvReaderTest, WithHeader) {
}
}
TEST_P(CsvReaderTest, MultilineQuotedString) {
TEST_F(CsvReaderTest, MultilineQuotedString) {
// create a file with first row valid and the second row containing a quoted
// string spanning two lines;
// parser should return two valid rows
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
auto writer = FileWriter(filepath);
utils::MemoryResource *mem(utils::NewDeleteResource());
@@ -286,37 +283,3 @@ TEST_P(CsvReaderTest, MultilineQuotedString) {
parsed_row = reader.GetNextRow(mem);
ASSERT_EQ(*parsed_row, ToPmrColumns(expected_multiline));
}
TEST_P(CsvReaderTest, EmptyColumns) {
// create a file with all rows valid;
// parser should return 'std::nullopt'
const auto filepath = csv_directory / "bla.csv";
auto writer = FileWriter(filepath, GetParam());
utils::MemoryResource *mem(utils::NewDeleteResource());
const utils::pmr::string delimiter{",", mem};
const utils::pmr::string quote{"\"", mem};
std::vector<std::vector<std::string>> expected_rows{{"", "B", "C"}, {"A", "", "C"}, {"A", "B", ""}};
for (const auto &row : expected_rows) {
writer.WriteLine(CreateRow(row, delimiter));
}
writer.Close();
const bool with_header = false;
const bool ignore_bad = false;
const csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
auto reader = csv::Reader(filepath, cfg);
for (const auto &expected_row : expected_rows) {
const auto pmr_expected_row = ToPmrColumns(expected_row);
const auto parsed_row = reader.GetNextRow(mem);
ASSERT_TRUE(parsed_row.has_value());
ASSERT_EQ(*parsed_row, pmr_expected_row);
}
}
INSTANTIATE_TEST_CASE_P(NewlineParameterizedTest, CsvReaderTest, ::testing::Values("\n", "\r\n"));

View File

@@ -12,7 +12,6 @@ class TestMemory final : public utils::MemoryResource {
size_t delete_count_{0};
private:
static constexpr size_t kPadSize = 32;
void *DoAllocate(size_t bytes, size_t alignment) override {
new_count_++;
EXPECT_TRUE(alignment != 0U && (alignment & (alignment - 1U)) == 0U) << "Alignment must be power of 2";
@@ -21,11 +20,11 @@ class TestMemory final : public utils::MemoryResource {
EXPECT_TRUE(bytes + pad_size > bytes) << "TestMemory size overflow";
EXPECT_TRUE(bytes + pad_size + alignment > bytes + alignment) << "TestMemory size overflow";
EXPECT_TRUE(2U * alignment > alignment) << "TestMemory alignment overflow";
// Allocate a block containing extra alignment and kPadSize bytes, but
// Allocate a block containing extra alignment and pad_size bytes, but
// aligned to 2 * alignment. Then we can offset the ptr so that it's never
// aligned to 2 * alignment. This ought to make allocator alignment issues
// more obvious.
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + kPadSize, 2U * alignment);
void *ptr = utils::NewDeleteResource()->Allocate(alignment + bytes + pad_size, 2U * alignment);
// Clear allocated memory to 0xFF, marking the invalid region.
memset(ptr, 0xFF, alignment + bytes + pad_size);
// Offset the ptr so it's not aligned to 2 * alignment, but still aligned to
@@ -40,8 +39,7 @@ class TestMemory final : public utils::MemoryResource {
void DoDeallocate(void *ptr, size_t bytes, size_t alignment) override {
delete_count_++;
// Deallocate the original ptr, before alignment adjustment.
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, alignment + bytes + kPadSize,
2U * alignment);
return utils::NewDeleteResource()->Deallocate(static_cast<char *>(ptr) - alignment, bytes, alignment);
}
bool DoIsEqual(const utils::MemoryResource &other) const noexcept override { return this == &other; }

View File

@@ -1,68 +0,0 @@
#!/usr/bin/env python3
"""
Bench Graph client responsible for sending benchmarking data in JSON format to
the Bench Graph server.
"""
import json
import logging
import os
import requests
import subprocess
from datetime import datetime
from argparse import ArgumentParser
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "")
GITHUB_SHA = os.getenv("GITHUB_SHA", "")
GITHUB_REF = os.getenv("GITHUB_REF", "")
BENCH_GRAPH_SERVER_ENDPOINT = os.getenv(
"BENCH_GRAPH_SERVER_ENDPOINT",
"http://bench-graph-api:9001")
log = logging.getLogger(__name__)
def parse_args():
argp = ArgumentParser(description=__doc__)
argp.add_argument("--benchmark-name", type=str, required=True)
argp.add_argument("--benchmark-results-path", type=str, required=True)
argp.add_argument("--github-run-id", type=int, required=True)
argp.add_argument("--github-run-number", type=int, required=True)
return argp.parse_args()
def post_measurement(args):
with open(args.benchmark_results_path, "r") as f:
data = json.load(f)
timestamp = datetime.now().timestamp()
branch = subprocess.run(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
stdout=subprocess.PIPE,
check=True).stdout.decode("utf-8").strip()
req = requests.post(
f"{BENCH_GRAPH_SERVER_ENDPOINT}/measurements",
json={
"name": args.benchmark_name,
"timestamp": timestamp,
"git_repo": GITHUB_REPOSITORY,
"git_ref": GITHUB_REF,
"git_sha": GITHUB_SHA,
"github_run_id": args.github_run_id,
"github_run_number": args.github_run_number,
"results": data,
"git_branch": branch},
timeout=1)
assert req.status_code == 200, \
f"Uploading {args.benchmark_name} data failed."
log.info(f"{args.benchmark_name} data sent to "
f"{BENCH_GRAPH_SERVER_ENDPOINT}")
if __name__ == "__main__":
args = parse_args()
logging.basicConfig(level=logging.INFO)
post_measurement(args)

View File

@@ -1 +0,0 @@
requests==2.25.1

View File

@@ -1,269 +0,0 @@
#!/usr/bin/env python3
#
#===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
r"""
ClangTidy Diff Checker
======================
This script reads input from a unified diff, runs clang-tidy on all changed
files and outputs clang-tidy warnings in changed lines only. This is useful to
detect clang-tidy regressions in the lines touched by a specific patch.
Example usage for git/svn users:
git diff -U0 HEAD^ | clang-tidy-diff.py -p1
svn diff --diff-cmd=diff -x-U0 | \
clang-tidy-diff.py -fix -checks=-*,modernize-use-override
"""
import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback
try:
import yaml
except ImportError:
yaml = None
is_py2 = sys.version[0] == '2'
if is_py2:
import Queue as queue
else:
import queue as queue
def run_tidy(task_queue, lock, timeout):
watchdog = None
while True:
command = task_queue.get()
try:
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if timeout is not None:
watchdog = threading.Timer(timeout, proc.kill)
watchdog.start()
stdout, stderr = proc.communicate()
with lock:
sys.stdout.write(stdout.decode('utf-8') + '\n')
sys.stdout.flush()
if stderr:
sys.stderr.write(stderr.decode('utf-8') + '\n')
sys.stderr.flush()
except Exception as e:
with lock:
sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n')
finally:
with lock:
if not (timeout is None or watchdog is None):
if not watchdog.is_alive():
sys.stderr.write('Terminated by timeout: ' +
' '.join(command) + '\n')
watchdog.cancel()
task_queue.task_done()
def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout):
for _ in range(max_tasks):
t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout))
t.daemon = True
t.start()
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged = []
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
content = yaml.safe_load(open(replacefile, 'r'))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {'MainSourceFile': '', mergekey: merged}
with open(mergefile, 'w') as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, 'w').close()
def main():
parser = argparse.ArgumentParser(description=
'Run clang-tidy against changed files, and '
'output diagnostics only for modified '
'lines.')
parser.add_argument('-clang-tidy-binary', metavar='PATH',
default='clang-tidy',
help='path to clang-tidy binary')
parser.add_argument('-p', metavar='NUM', default=0,
help='strip the smallest prefix containing P slashes')
parser.add_argument('-regex', metavar='PATTERN', default=None,
help='custom pattern selecting file paths to check '
'(case sensitive, overrides -iregex)')
parser.add_argument('-iregex', metavar='PATTERN', default=
r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)',
help='custom pattern selecting file paths to check '
'(case insensitive, overridden by -regex)')
parser.add_argument('-j', type=int, default=1,
help='number of tidy instances to be run in parallel.')
parser.add_argument('-timeout', type=int, default=None,
help='timeout per each file in seconds.')
parser.add_argument('-fix', action='store_true', default=False,
help='apply suggested fixes')
parser.add_argument('-checks',
help='checks filter, when not specified, use clang-tidy '
'default',
default='')
parser.add_argument('-path', dest='build_path',
help='Path used to read a compile command database.')
if yaml:
parser.add_argument('-export-fixes', metavar='FILE', dest='export_fixes',
help='Create a yaml file to store suggested fixes in, '
'which can be applied with clang-apply-replacements.')
parser.add_argument('-extra-arg', dest='extra_arg',
action='append', default=[],
help='Additional argument to append to the compiler '
'command line.')
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
action='append', default=[],
help='Additional argument to prepend to the compiler '
'command line.')
parser.add_argument('-quiet', action='store_true', default=False,
help='Run clang-tidy in quiet mode')
clang_tidy_args = []
argv = sys.argv[1:]
if '--' in argv:
clang_tidy_args.extend(argv[argv.index('--'):])
argv = argv[:argv.index('--')]
args = parser.parse_args(argv)
# Extract changed lines for each file.
filename = None
lines_by_file = {}
for line in sys.stdin:
match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line)
if match:
filename = match.group(2)
if filename is None:
continue
if args.regex is not None:
if not re.match('^%s$' % args.regex, filename):
continue
else:
if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
continue
match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
if match:
start_line = int(match.group(1))
line_count = 1
if match.group(3):
line_count = int(match.group(3))
if line_count == 0:
continue
end_line = start_line + line_count - 1
lines_by_file.setdefault(filename, []).append([start_line, end_line])
if not any(lines_by_file):
print("No relevant changes found.")
sys.exit(0)
max_task_count = args.j
if max_task_count == 0:
max_task_count = multiprocessing.cpu_count()
max_task_count = min(len(lines_by_file), max_task_count)
tmpdir = None
if yaml and args.export_fixes:
tmpdir = tempfile.mkdtemp()
# Tasks for clang-tidy.
task_queue = queue.Queue(max_task_count)
# A lock for console output.
lock = threading.Lock()
# Run a pool of clang-tidy workers.
start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout)
# Form the common args list.
common_clang_tidy_args = []
if args.fix:
common_clang_tidy_args.append('-fix')
if args.checks != '':
common_clang_tidy_args.append('-checks=' + args.checks)
if args.quiet:
common_clang_tidy_args.append('-quiet')
if args.build_path is not None:
common_clang_tidy_args.append('-p=%s' % args.build_path)
for arg in args.extra_arg:
common_clang_tidy_args.append('-extra-arg=%s' % arg)
for arg in args.extra_arg_before:
common_clang_tidy_args.append('-extra-arg-before=%s' % arg)
for name in lines_by_file:
line_filter_json = json.dumps(
[{"name": name, "lines": lines_by_file[name]}],
separators=(',', ':'))
# Run clang-tidy on files containing changes.
command = [args.clang_tidy_binary]
command.append('-line-filter=' + line_filter_json)
if yaml and args.export_fixes:
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
command.append('-export-fixes=' + tmp_name)
command.extend(common_clang_tidy_args)
command.append(name)
command.extend(clang_tidy_args)
task_queue.put(command)
# Wait for all threads to be done.
task_queue.join()
if yaml and args.export_fixes:
print('Writing fixes to ' + args.export_fixes + ' ...')
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
sys.stderr.write('Error exporting fixes.\n')
traceback.print_exc()
if tmpdir:
shutil.rmtree(tmpdir)
if __name__ == '__main__':
main()

View File

@@ -1,9 +0,0 @@
#!/bin/bash
# the first sort | uniq is necessary, because the same occurrence of the same error
# can be reported from headers when they are included in multiple source files
`dirname ${BASH_SOURCE[0]}`/grep_error_lines.sh |
sort | uniq |
sed -E 's/.*\[(.*)\]\r?$/\1/g' | # extract the check name from [check-name]
sort | uniq -c | # count each type of check
sort -nr # sort them into descending order

View File

@@ -1,12 +0,0 @@
#!/bin/bash
# Matches timestamp like "2021-03-25T17:06:42.2621697Z"
TIMESTAMP_PATTERN="\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z"
# Matches absolute file pathes with line and column identifier like
# "/opt/actions-runner/_work/memgraph/memgraph/src/utils/exceptions.hpp:71:11:"
FILE_ABSOLUTE_PATH_PATTERN="/[^:]+:\d+:\d+:"
ERROR_OR_WARNING_PATTERN="(error|warning):"
grep -P "^($TIMESTAMP_PATTERN )?$FILE_ABSOLUTE_PATH_PATTERN $ERROR_OR_WARNING_PATTERN.*$"

View File

@@ -1,337 +0,0 @@
#!/usr/bin/env python3
#
#===- run-clang-tidy.py - Parallel clang-tidy runner --------*- python -*--===#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===-----------------------------------------------------------------------===#
# FIXME: Integrate with clang-tidy-diff.py
"""
Parallel clang-tidy runner
==========================
Runs clang-tidy over all files in a compilation database. Requires clang-tidy
and clang-apply-replacements in $PATH.
Example invocations.
- Run clang-tidy on all files in the current working directory with a default
set of checks and show warnings in the cpp files and all project headers.
run-clang-tidy.py $PWD
- Fix all header guards.
run-clang-tidy.py -fix -checks=-*,llvm-header-guard
- Fix all header guards included from clang-tidy and header guards
for clang-tidy headers.
run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
-header-filter=extra/clang-tidy
Compilation database setup:
http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
"""
from __future__ import print_function
import argparse
import glob
import json
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import traceback
try:
import yaml
except ImportError:
yaml = None
is_py2 = sys.version[0] == '2'
if is_py2:
import Queue as queue
else:
import queue as queue
def find_compilation_database(path):
"""Adjusts the directory until a compilation database is found."""
result = './'
while not os.path.isfile(os.path.join(result, path)):
if os.path.realpath(result) == '/':
print('Error: could not find compilation database.')
sys.exit(1)
result += '../'
return os.path.realpath(result)
def make_absolute(f, directory):
if os.path.isabs(f):
return f
return os.path.normpath(os.path.join(directory, f))
def get_tidy_invocation(f, clang_tidy_binary, checks, tmpdir, build_path,
header_filter, allow_enabling_alpha_checkers,
extra_arg, extra_arg_before, quiet, config):
"""Gets a command line for clang-tidy."""
start = [clang_tidy_binary]
if allow_enabling_alpha_checkers:
start.append('-allow-enabling-analyzer-alpha-checkers')
if header_filter is not None:
start.append('-header-filter=' + header_filter)
if checks:
start.append('-checks=' + checks)
if tmpdir is not None:
start.append('-export-fixes')
# Get a temporary file. We immediately close the handle so clang-tidy can
# overwrite it.
(handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
os.close(handle)
start.append(name)
for arg in extra_arg:
start.append('-extra-arg=%s' % arg)
for arg in extra_arg_before:
start.append('-extra-arg-before=%s' % arg)
start.append('-p=' + build_path)
if quiet:
start.append('-quiet')
if config:
start.append('-config=' + config)
start.append(f)
return start
def merge_replacement_files(tmpdir, mergefile):
"""Merge all replacement files in a directory into a single file"""
# The fixes suggested by clang-tidy >= 4.0.0 are given under
# the top level key 'Diagnostics' in the output yaml files
mergekey = "Diagnostics"
merged=[]
for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
content = yaml.safe_load(open(replacefile, 'r'))
if not content:
continue # Skip empty files.
merged.extend(content.get(mergekey, []))
if merged:
# MainSourceFile: The key is required by the definition inside
# include/clang/Tooling/ReplacementsYaml.h, but the value
# is actually never used inside clang-apply-replacements,
# so we set it to '' here.
output = {'MainSourceFile': '', mergekey: merged}
with open(mergefile, 'w') as out:
yaml.safe_dump(output, out)
else:
# Empty the file:
open(mergefile, 'w').close()
def check_clang_apply_replacements_binary(args):
"""Checks if invoking supplied clang-apply-replacements binary works."""
try:
subprocess.check_call([args.clang_apply_replacements_binary, '--version'])
except:
print('Unable to run clang-apply-replacements. Is clang-apply-replacements '
'binary correctly specified?', file=sys.stderr)
traceback.print_exc()
sys.exit(1)
def apply_fixes(args, tmpdir):
"""Calls clang-apply-fixes on a given directory."""
invocation = [args.clang_apply_replacements_binary]
if args.format:
invocation.append('-format')
if args.style:
invocation.append('-style=' + args.style)
invocation.append(tmpdir)
subprocess.call(invocation)
def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
"""Takes filenames out of queue and runs clang-tidy on them."""
while True:
name = queue.get()
invocation = get_tidy_invocation(name, args.clang_tidy_binary, args.checks,
tmpdir, build_path, args.header_filter,
args.allow_enabling_alpha_checkers,
args.extra_arg, args.extra_arg_before,
args.quiet, args.config)
proc = subprocess.Popen(invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output, err = proc.communicate()
if proc.returncode != 0:
failed_files.append(name)
with lock:
sys.stdout.write(' '.join(invocation) + '\n' + output.decode('utf-8'))
if len(err) > 0:
sys.stdout.flush()
sys.stderr.write(err.decode('utf-8'))
queue.task_done()
def main():
parser = argparse.ArgumentParser(description='Runs clang-tidy over all files '
'in a compilation database. Requires '
'clang-tidy and clang-apply-replacements in '
'$PATH.')
parser.add_argument('-allow-enabling-alpha-checkers',
action='store_true', help='allow alpha checkers from '
'clang-analyzer.')
parser.add_argument('-clang-tidy-binary', metavar='PATH',
default='clang-tidy-11',
help='path to clang-tidy binary')
parser.add_argument('-clang-apply-replacements-binary', metavar='PATH',
default='clang-apply-replacements-11',
help='path to clang-apply-replacements binary')
parser.add_argument('-checks', default=None,
help='checks filter, when not specified, use clang-tidy '
'default')
parser.add_argument('-config', default=None,
help='Specifies a configuration in YAML/JSON format: '
' -config="{Checks: \'*\', '
' CheckOptions: [{key: x, '
' value: y}]}" '
'When the value is empty, clang-tidy will '
'attempt to find a file named .clang-tidy for '
'each source file in its parent directories.')
parser.add_argument('-header-filter', default=None,
help='regular expression matching the names of the '
'headers to output diagnostics from. Diagnostics from '
'the main file of each translation unit are always '
'displayed.')
if yaml:
parser.add_argument('-export-fixes', metavar='filename', dest='export_fixes',
help='Create a yaml file to store suggested fixes in, '
'which can be applied with clang-apply-replacements.')
parser.add_argument('-j', type=int, default=0,
help='number of tidy instances to be run in parallel.')
parser.add_argument('files', nargs='*', default=['.*'],
help='files to be processed (regex on path)')
parser.add_argument('-fix', action='store_true', help='apply fix-its')
parser.add_argument('-format', action='store_true', help='Reformat code '
'after applying fixes')
parser.add_argument('-style', default='file', help='The style of reformat '
'code after applying fixes')
parser.add_argument('-p', dest='build_path',
help='Path used to read a compile command database.')
parser.add_argument('-extra-arg', dest='extra_arg',
action='append', default=[],
help='Additional argument to append to the compiler '
'command line.')
parser.add_argument('-extra-arg-before', dest='extra_arg_before',
action='append', default=[],
help='Additional argument to prepend to the compiler '
'command line.')
parser.add_argument('-quiet', action='store_true',
help='Run clang-tidy in quiet mode')
args = parser.parse_args()
db_path = 'compile_commands.json'
if args.build_path is not None:
build_path = args.build_path
else:
# Find our database
build_path = find_compilation_database(db_path)
try:
invocation = [args.clang_tidy_binary, '-list-checks']
if args.allow_enabling_alpha_checkers:
invocation.append('-allow-enabling-analyzer-alpha-checkers')
invocation.append('-p=' + build_path)
if args.checks:
invocation.append('-checks=' + args.checks)
invocation.append('-')
if args.quiet:
# Even with -quiet we still want to check if we can call clang-tidy.
with open(os.devnull, 'w') as dev_null:
subprocess.check_call(invocation, stdout=dev_null)
else:
subprocess.check_call(invocation)
except:
print("Unable to run clang-tidy.", file=sys.stderr)
sys.exit(1)
# Load the database and extract all files.
database = json.load(open(os.path.join(build_path, db_path)))
files = [make_absolute(entry['file'], entry['directory'])
for entry in database]
max_task = args.j
if max_task == 0:
max_task = multiprocessing.cpu_count()
tmpdir = None
if args.fix or (yaml and args.export_fixes):
check_clang_apply_replacements_binary(args)
tmpdir = tempfile.mkdtemp()
# Build up a big regexy filter from all command line arguments.
file_name_re = re.compile('|'.join(args.files))
return_code = 0
try:
# Spin up a bunch of tidy-launching threads.
task_queue = queue.Queue(max_task)
# List of files with a non-zero return code.
failed_files = []
lock = threading.Lock()
for _ in range(max_task):
t = threading.Thread(target=run_tidy,
args=(args, tmpdir, build_path, task_queue, lock, failed_files))
t.daemon = True
t.start()
# Fill the queue with files.
for name in files:
if file_name_re.search(name):
task_queue.put(name)
# Wait for all threads to be done.
task_queue.join()
if len(failed_files):
return_code = 1
except KeyboardInterrupt:
# This is a sad hack. Unfortunately subprocess goes
# bonkers with ctrl-c and we start forking merrily.
print('\nCtrl-C detected, goodbye.')
if tmpdir:
shutil.rmtree(tmpdir)
os.kill(0, 9)
if yaml and args.export_fixes:
print('Writing fixes to ' + args.export_fixes + ' ...')
try:
merge_replacement_files(tmpdir, args.export_fixes)
except:
print('Error exporting fixes.\n', file=sys.stderr)
traceback.print_exc()
return_code=1
if args.fix:
print('Applying fixes ...')
try:
apply_fixes(args, tmpdir)
except:
print('Error applying fixes.\n', file=sys.stderr)
traceback.print_exc()
return_code = 1
if tmpdir:
shutil.rmtree(tmpdir)
sys.exit(return_code)
if __name__ == '__main__':
main()

View File

@@ -1,12 +0,0 @@
leak:antlr4::atn::ArrayPredictionContext::ArrayPredictionContext
leak:std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<antlr4::atn::SingletonPredictionContext, std::allocator<antlr4::atn::SingletonPredictionContext>, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&>(antlr4::atn::SingletonPredictionContext*&, std::_Sp_alloc_shared_tag<std::allocator<antlr4::atn::SingletonPredictionContext> >, std::weak_ptr<antlr4::atn::PredictionContext>&, unsigned long&)
leak:antlr4::atn::PredictionContext::mergeSingletons(std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, std::shared_ptr<antlr4::atn::SingletonPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
leak:void std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::PredictionContext> >(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::PredictionContext>*, std::vector<std::shared_ptr<antlr4::atn::PredictionContext>, std::allocator<std::shared_ptr<antlr4::atn::PredictionContext> > > >, std::shared_ptr<antlr4::atn::PredictionContext>&&)
leak:antlr4::atn::ParserATNSimulator::closureCheckingStopState(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::ATNConfigSet*, std::unordered_set<std::shared_ptr<antlr4::atn::ATNConfig>, antlr4::atn::ATNConfig::Hasher, antlr4::atn::ATNConfig::Comparer, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >&, bool, bool, int, bool)
leak:antlr4::atn::ParserATNSimulator::computeReachSet(antlr4::atn::ATNConfigSet*, unsigned long, bool)
leak:std::_Hashtable<unsigned long, std::pair<unsigned long const, antlr4::atn::ATNConfig*>, std::allocator<std::pair<unsigned long const, antlr4::atn::ATNConfig*> >, std::__detail::_Select1st, std::equal_to<unsigned long>, std::hash<unsigned long>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >::_M_insert_unique_node(unsigned long const&, unsigned long, unsigned long, std::__detail::_Hash_node<std::pair<unsigned long const, antlr4::atn::ATNConfig*>, false>*, unsigned long)
leak:void std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > >::_M_realloc_insert<std::shared_ptr<antlr4::atn::ATNConfig> const&>(__gnu_cxx::__normal_iterator<std::shared_ptr<antlr4::atn::ATNConfig>*, std::vector<std::shared_ptr<antlr4::atn::ATNConfig>, std::allocator<std::shared_ptr<antlr4::atn::ATNConfig> > > >, std::shared_ptr<antlr4::atn::ATNConfig> const&)
leak:antlr4::atn::ATNConfigSet::add(std::shared_ptr<antlr4::atn::ATNConfig> const&, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
leak:antlr4::atn::ParserATNSimulator::getEpsilonTarget(std::shared_ptr<antlr4::atn::ATNConfig> const&, antlr4::atn::Transition*, bool, bool, bool, bool)
leak:antlr4::atn::PredictionContext::mergeArrays(std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, std::shared_ptr<antlr4::atn::ArrayPredictionContext> const&, bool, std::map<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> >, std::shared_ptr<antlr4::atn::PredictionContext>, std::less<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > >, std::allocator<std::pair<std::pair<std::shared_ptr<antlr4::atn::PredictionContext>, std::shared_ptr<antlr4::atn::PredictionContext> > const, std::shared_ptr<antlr4::atn::PredictionContext> > > >*)
leak:/lib/x86_64-linux-gnu/libpython3.