Compare commits

..

32 Commits

Author SHA1 Message Date
antoniofilipovic
40e94dc524 add poc memory tracker per procedure 2023-10-11 14:05:12 +02:00
antoniofilipovic
8ba34ef8f0 revert back cmakelists 2023-10-11 13:57:31 +02:00
antoniofilipovic
df312387cc revert back cmakelists 2023-10-11 13:56:49 +02:00
antoniofilipovic
94d5e22dcd introduce tracking per thread, add working test 2023-10-11 13:55:42 +02:00
antoniofilipovic
11ee19516a add limits 2023-10-10 18:51:24 +02:00
antoniofilipovic
e6f854396c enable memory tracker per thread id 2023-10-10 16:26:56 +02:00
antoniofilipovic
afdf38e0c8 add per query memory limit 2023-10-09 12:13:25 +02:00
antoniofilipovic
956b95a95c remove unnecessary includes 2023-10-06 16:59:46 +02:00
antoniofilipovic
46df649c65 revert checks 2023-10-06 16:55:08 +02:00
antoniofilipovic
379ab47866 cleanup memory_control, remove per query tracker 2023-10-06 16:42:44 +02:00
antoniofilipovic
80689a8337 remove virtual memory tracker 2023-10-06 16:33:57 +02:00
antoniofilipovic
771982be05 remove old memory tracker 2023-10-06 16:30:55 +02:00
antoniofilipovic
ba1a9d3045 fix cmakelists, remove comments 2023-10-06 16:26:23 +02:00
antoniofilipovic
0fc2355d8d fix tests 2023-10-06 13:46:01 +02:00
antoniofilipovic
76d4790b81 comment out interpreter jemalloc stats 2023-10-06 12:22:18 +02:00
antoniofilipovic
32682fa463 merge master 2023-10-06 10:56:06 +02:00
antoniofilipovic
f4e4fdb754 add jemalloc to libs 2023-10-06 10:47:47 +02:00
antoniofilipovic
2c2c55abf4 test memory control 2023-10-05 09:42:10 +02:00
antoniofilipovic
32e744f09f comment unnecessary atomics 2023-10-04 10:07:09 +02:00
antoniofilipovic
e10daca67a add few improvements 2023-09-29 19:22:11 +02:00
antoniofilipovic
5659ff21a8 fix flakly behavior on sigterm on tests 2023-09-28 17:35:55 +02:00
antoniofilipovic
817688d63d add working versions for asan tests 2023-09-28 15:54:13 +02:00
antoniofilipovic
3e2a5c2744 clean up memory control 2023-09-27 14:14:35 +02:00
antoniofilipovic
4f9c8b661e add arenas to cmakelist 2023-09-26 14:50:48 +02:00
antoniofilipovic
9e83a37873 add working tracker 2023-09-26 14:50:01 +02:00
antoniofilipovic
a0047672cb Add fully working version with jemalloc hook memory tracker
This commit introduces memory tracker with jemalloc extent hooks which fully works in case when jemalloc config is following:
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:1000000000000,muzzy_decay_ms:0,dirty_decay_ms:0" \
./configure \
    --disable-cxx \
    $COMMON_CONFIGURE_FLAGS \
    --with-malloc-conf="retain:false,percpu_arena:percpu,oversize_threshold:1000000000000,muzzy_decay_ms:0,dirty_decay_ms:0"

This config will for jemalloc not to use lazy purge or MADV_FREE(muzzy_decay_ms=0 and dirty_decay_ms=0), it will force jemalloc not to use
custom arena for huge allocations (oversize_threshold) and it will force jemalloc not extend virtual memory indefinitely (retain=false)
and therefore call alloc hook when allocation actually takes place.

Only problem is if we do huge allocations which are not mapped on alloc directly, in that case jemalloc uses cache and we can overcounter
allocation size.
2023-09-21 12:52:17 +02:00
antoniofilipovic
6a4780d2ac remove reducing memory usage on lazy purge 2023-09-20 12:06:40 +02:00
antoniofilipovic
362cbe8338 remove reducing memory usage on lazy purge 2023-09-20 12:05:35 +02:00
antoniofilipovic
e7dd60b1f0 add better version than current tracking 2023-09-19 17:02:47 +02:00
antoniofilipovic
fd9b653de9 add basic working version 1 2023-09-14 14:18:26 +02:00
antoniofilipovic
596760e655 add non working version of hooks alloc 2023-09-12 14:02:57 +02:00
antoniofilipovic
0f8ef3cdb2 add initial version of extent_hooks 2023-09-11 16:58:49 +02:00
262 changed files with 3729 additions and 13172 deletions

View File

@@ -33,4 +33,4 @@ for file in $modified_files; do
fi
done;
exit ${FAIL}
return ${FAIL}

View File

@@ -43,7 +43,7 @@ jobs:
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMG_ENTERPRISE=OFF ..
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
make -j$THREADS
- name: Run unit tests
@@ -244,14 +244,12 @@ jobs:
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run GQL Behave tests
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
cd gql_behave
cd tests/gql_behave
./continuous_integration
- name: Save quality assurance status
@@ -351,7 +349,7 @@ jobs:
./init
# Build only memgraph release binarie.
cd build
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph
- name: Run Jepsen tests

View File

@@ -8,13 +8,6 @@ on:
memgraph_version:
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
required: false
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
jobs:
centos-7:
@@ -27,7 +20,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-7 ${{ github.event.inputs.build_type }}
./release/package/run.sh package centos-7
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -44,7 +37,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-9 ${{ github.event.inputs.build_type }}
./release/package/run.sh package centos-9
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -61,7 +54,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-10 ${{ github.event.inputs.build_type }}
./release/package/run.sh package debian-10
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -78,7 +71,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }}
./release/package/run.sh package debian-11
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -96,7 +89,7 @@ jobs:
- name: "Build package"
run: |
cd release/package
./run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-docker
./run.sh package debian-11 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v3
@@ -114,7 +107,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
./release/package/run.sh package ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -131,7 +124,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
./release/package/run.sh package ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -148,7 +141,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
./release/package/run.sh package ubuntu-22.04
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -165,7 +158,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
./release/package/run.sh package debian-11 --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -182,7 +175,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package fedora-36 ${{ github.event.inputs.build_type }}
./release/package/run.sh package fedora-36
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -199,7 +192,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
./release/package/run.sh package amzn-2
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -216,7 +209,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
./release/package/run.sh package debian-11-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
@@ -233,7 +226,7 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04-arm ${{ github.event.inputs.build_type }}
./release/package/run.sh package ubuntu-22.04-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
with:

View File

@@ -30,7 +30,7 @@ jobs:
# Build only memgraph release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
cmake -DCMAKE_BUILD_TYPE=Release ..
make -j$THREADS
- name: Get branch name (merge)

View File

@@ -2,15 +2,6 @@ name: Release CentOS 8
on:
workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule:
- cron: "0 22 * * *"
@@ -42,7 +33,7 @@ jobs:
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} -DMG_ENTERPRISE=OFF ..
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
make -j$THREADS
- name: Run unit tests
@@ -199,7 +190,7 @@ jobs:
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} ..
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Create enterprise RPM package

View File

@@ -2,15 +2,6 @@ name: Release Debian 10
on:
workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule:
- cron: "0 22 * * *"
@@ -42,7 +33,7 @@ jobs:
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} -DMG_ENTERPRISE=OFF ..
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
make -j$THREADS
- name: Run unit tests
@@ -199,7 +190,7 @@ jobs:
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} ..
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Create enterprise DEB package
@@ -331,7 +322,7 @@ jobs:
./init
# Build only memgraph release binary.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} ..
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph
- name: Run Jepsen tests

View File

@@ -2,15 +2,6 @@ name: Release Ubuntu 20.04
on:
workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
schedule:
- cron: "0 22 * * *"
@@ -42,7 +33,7 @@ jobs:
# Build community binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} -DMG_ENTERPRISE=OFF ..
cmake -DCMAKE_BUILD_TYPE=release -DMG_ENTERPRISE=OFF ..
make -j$THREADS
- name: Run unit tests
@@ -199,7 +190,7 @@ jobs:
# Build release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=${{ github.event.inputs.build_type }} ..
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Create enterprise DEB package

View File

@@ -1,5 +1,5 @@
<p align="center">
<img src="https://public-assets.memgraph.com/github-readme-images/github-memgraph-repo-banner.png">
<img width="400px" src="https://uploads-ssl.webflow.com/5e7ceb09657a69bdab054b3a/5e7ceb09657a6937ab054bba_Black_Original%20_Logo.png">
</p>
---

View File

@@ -1,55 +0,0 @@
# Try to find jemalloc library
#
# Use this module as:
# find_package(Jemalloc)
#
# or:
# find_package(Jemalloc REQUIRED)
#
# This will define the following variables:
#
# Jemalloc_FOUND True if the system has the jemalloc library.
# Jemalloc_INCLUDE_DIRS Include directories needed to use jemalloc.
# Jemalloc_LIBRARIES Libraries needed to link to jemalloc.
#
# The following cache variables may also be set:
#
# Jemalloc_INCLUDE_DIR The directory containing jemalloc/jemalloc.h.
# Jemalloc_LIBRARY The path to the jemalloc static library.
find_path(Jemalloc_INCLUDE_DIR NAMES jemalloc/jemalloc.h PATH_SUFFIXES include)
find_library(Jemalloc_LIBRARY NAMES libjemalloc.a PATH_SUFFIXES lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Jemalloc
FOUND_VAR Jemalloc_FOUND
REQUIRED_VARS
Jemalloc_LIBRARY
Jemalloc_INCLUDE_DIR
)
if(Jemalloc_FOUND)
set(Jemalloc_LIBRARIES ${Jemalloc_LIBRARY})
set(Jemalloc_INCLUDE_DIRS ${Jemalloc_INCLUDE_DIR})
else()
if(Jemalloc_FIND_REQUIRED)
message(FATAL_ERROR "Cannot find jemalloc!")
else()
message(WARNING "jemalloc is not found!")
endif()
endif()
if(Jemalloc_FOUND AND NOT TARGET Jemalloc::Jemalloc)
add_library(Jemalloc::Jemalloc UNKNOWN IMPORTED)
set_target_properties(Jemalloc::Jemalloc
PROPERTIES
IMPORTED_LOCATION "${Jemalloc_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${Jemalloc_INCLUDE_DIR}"
)
endif()
mark_as_advanced(
Jemalloc_INCLUDE_DIR
Jemalloc_LIBRARY
)

67
cmake/Findjemalloc.cmake Normal file
View File

@@ -0,0 +1,67 @@
# Try to find jemalloc library
#
# Use this module as:
# find_package(jemalloc)
#
# or:
# find_package(jemalloc REQUIRED)
#
# This will define the following variables:
#
# JEMALLOC_FOUND True if the system has the jemalloc library.
# Jemalloc_INCLUDE_DIRS Include directories needed to use jemalloc.
# Jemalloc_LIBRARIES Libraries needed to link to jemalloc.
#
# The following cache variables may also be set:
#
# Jemalloc_INCLUDE_DIR The directory containing jemalloc/jemalloc.h.
# Jemalloc_LIBRARY The path to the jemalloc static library.
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(jemalloc
FOUND_VAR JEMALLOC_FOUND
REQUIRED_VARS
JEMALLOC_LIBRARY
JEMALLOC_INCLUDE_DIR
)
if(JEMALLOC_INCLUDE_DIR)
message(STATUS "Found jemalloc include dir: ${JEMALLOC_INCLUDE_DIR}")
else()
message(WARNING "jemalloc not found!")
endif()
if(JEMALLOC_LIBRARY)
message(STATUS "Found jemalloc library: ${JEMALLOC_LIBRARY}")
else()
message(WARNING "jemalloc library not found!")
endif()
if(JEMALLOC_FOUND)
set(Jemalloc_LIBRARIES ${JEMALLOC_LIBRARY})
set(Jemalloc_INCLUDE_DIRS ${JEMALLOC_INCLUDE_DIR})
else()
if(Jemalloc_FIND_REQUIRED)
message(FATAL_ERROR "Cannot find jemalloc!")
else()
message(WARNING "jemalloc is not found!")
endif()
endif()
if(JEMALLOC_FOUND AND NOT TARGET Jemalloc::Jemalloc)
message(STATUS "JEMALLOC NOT TARGET")
add_library(Jemalloc::Jemalloc UNKNOWN IMPORTED)
set_target_properties(Jemalloc::Jemalloc
PROPERTIES
IMPORTED_LOCATION "${JEMALLOC_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${JEMALLOC_INCLUDE_DIR}"
)
endif()
mark_as_advanced(
JEMALLOC_INCLUDE_DIR
JEMALLOC_LIBRARY
)

2
environment/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
archives
build

View File

@@ -1,10 +1,5 @@
# Memgraph Operating Environments
## Issues related to build toolchain
* GCC 11.2 (toolchain-v4) doesn't compile on Fedora 38, multiple definitions of enum issue
* spdlog 1.10/11 doesn't work with fmt 10.0.0
## os
Under the `os` directory, you can find scripts to install all required system

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel

View File

@@ -20,7 +20,7 @@ TOOLCHAIN_BUILD_DEPS=(
curl # snappy
readline-devel # cmake and llvm
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel pcre2-devel automake bison # swig
libedit-devel pcre-devel automake bison # swig
file
openssl-devel
gmp-devel

View File

@@ -17,7 +17,7 @@ TOOLCHAIN_BUILD_DEPS=(
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb
gperf # for proxygen
git # for fbthrift
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
libedit-dev libpcre3-dev automake bison # for swig
curl # snappy
file # for libunwind
libssl-dev # for libevent

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
libedit-dev libpcre3-dev automake bison # for swig
curl # snappy
file # for libunwind
libssl-dev # for libevent

View File

@@ -18,7 +18,7 @@ TOOLCHAIN_BUILD_DEPS=(
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel

View File

@@ -1,108 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "fedora-38"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
perl # for openssl
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
if [ -v LD_LIBRARY_PATH ]; then
# On Fedora 38 yum/dnf and python11 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
fi
local missing=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
if [ -v OLD_LD_LIBRARY_PATH ]; then
echo "Restoring LD_LIBRARY_PATH..."
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -25,7 +25,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre2-dev libpcre3-dev automake bison # swig
libedit-dev libpcre3-dev automake bison # swig
)
TOOLCHAIN_RUN_DEPS=(

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(

View File

@@ -24,7 +24,7 @@ TOOLCHAIN_BUILD_DEPS=(
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(

View File

@@ -1,4 +1 @@
archives
build
output
*.tar.gz

View File

@@ -4,7 +4,7 @@ diff -ur a/CMakeLists.txt b/CMakeLists.txt
@@ -52,9 +52,9 @@
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0)
- # Disable RTTI.
- string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
@@ -17,7 +17,7 @@ diff -ur a/CMakeLists.txt b/CMakeLists.txt
@@ -77,9 +77,9 @@
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
- # Disable RTTI.
- string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
@@ -25,5 +25,5 @@ diff -ur a/CMakeLists.txt b/CMakeLists.txt
+ # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make

View File

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR"
source "$DIR/../../util.sh"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
# toolchain version
@@ -30,10 +30,10 @@ LLVM_VERSION=11.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Check for the dependencies.
echo "ALL BUILD PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
# check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -442,7 +442,7 @@ In order to be able to run all of these tools you should install the following
packages:
\`\`\`
$($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
\`\`\`
## Usage

View File

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR"
source "$DIR/../../util.sh"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
# toolchain version
@@ -31,10 +31,10 @@ LLVM_VERSION_LONG=12.0.1-rc4
SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Check for the dependencies.
echo "ALL BUILD PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
echo "ALL BUILD PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_BUILD_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_BUILD_DEPS
echo "ALL RUN PACKAGES: $($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)"
$DIR/../os/$DISTRO.sh check TOOLCHAIN_RUN_DEPS
# check installation directory
NAME=toolchain-v$TOOLCHAIN_VERSION
@@ -452,7 +452,7 @@ In order to be able to run all of these tools you should install the following
packages:
\`\`\`
$($DIR/../../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
$($DIR/../os/$DISTRO.sh list TOOLCHAIN_RUN_DEPS)
\`\`\`
## Usage

View File

@@ -7,7 +7,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR"
source "$DIR/../../util.sh"
source "$DIR/../util.sh"
DISTRO="$(operating_system)"
function log_tool_name () {
@@ -51,13 +51,17 @@ CPPCHECK_VERSION=2.6
LLVM_VERSION=13.0.0
SWIG_VERSION=4.0.2 # used only for LLVM compilation
# Set the right operating system setup script.
ENV_SCRIPT="$DIR/../../os/$DISTRO.sh"
# Set the right env script
ENV_SCRIPT="$DIR/../os/$DISTRO.sh"
if [[ "$for_arm" = true ]]; then
ENV_SCRIPT="$DIR/../../os/$DISTRO-arm.sh"
ENV_SCRIPT="$DIR/../os/$DISTRO-arm.sh"
fi
# Check for the toolchain build dependencies.
echo "ALL BUILD PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_BUILD_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_BUILD_DEPS
# Check for the toolchain run dependencies.
echo "ALL RUN PACKAGES: $(${ENV_SCRIPT} list TOOLCHAIN_RUN_DEPS)"
${ENV_SCRIPT} check TOOLCHAIN_RUN_DEPS
@@ -654,7 +658,7 @@ In order to be able to run all of these tools you should install the following
packages:
\`\`\`
$($DIR/../../os/$ENV_SCRIPT.sh list TOOLCHAIN_RUN_DEPS)
$($DIR/../os/$ENV_SCRIPT.sh list TOOLCHAIN_RUN_DEPS)
\`\`\`
## Usage

File diff suppressed because it is too large Load Diff

View File

@@ -1,42 +0,0 @@
#!/bin/bash -ex
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PREFIX=/opt/toolchain-v5
# NOTE: Often times when versions in the build script are changes, something
# doesn't work. To avoid rebuild of the whole toolchain but rebuild specific
# lib from 0, just comment specific line under this cript and run it. Don't
# forget to comment back to avoid unnecessary deletes next time your run this
# cript.
# rm -rf "$DIR/build"
# rm -rf "$DIR/output"
# rm -rf "$PREFIX/bin/gcc"
# rm -rf "$PREFIX/bin/ld.gold"
# rm -rf "$PREFIX/bin/gdb"
# rm -rf "$PREFIX/bin/cmake"
# rm -rf "$PREFIX/bin/clang"
# rm -rf "$PREFIX/include/bzlib.h"
# rm -rf "$PREFIX/include/fmt"
# rm -rf "$PREFIX/include/lz4.h"
# rm -rf "$PREFIX/include/lzma.h"
# rm -rf "$PREFIX/include/zlib.h"
# rm -rf "$PREFIX/include/zstd.h"
# rm -rf "$PREFIX/include/jemalloc"
# rm -rf "$PREFIX/include/boost"
# rm -rf "$PREFIX/include/double-conversion"
# rm -rf "$PREFIX/include/gflags"
# rm -rf "$PREFIX/include/libunwind.h"
# rm -rf "$PREFIX/include/glog"
# rm -rf "$PREFIX/include/event2"
# rm -rf "$PREFIX/include/sodium.h"
# rm -rf "$PREFIX/include/libaio.h"
# rm -rf "$PREFIX/include/FlexLexer.h"
# rm -rf "$PREFIX/include/snappy.h"
# rm -rf "$PREFIX/include/fizz"
# rm -rf "$PREFIX/include/folly"
# rm -rf "$PREFIX/include/proxygen"
# rm -rf "$PREFIX/include/wangle"
# rm -rf "$PREFIX/include/thrift"
# rm -rf "$PREFIX"

View File

@@ -1,41 +0,0 @@
diff -ur a/folly/CMakeLists.txt b/folly/CMakeLists.txt
--- a/folly/CMakeLists.txt 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/CMakeLists.txt 2022-02-03 15:19:41.349693134 +0100
@@ -28,7 +28,6 @@
)
add_subdirectory(experimental/exception_tracer)
-add_subdirectory(logging/example)
if (PYTHON_EXTENSIONS)
# Create tree of symbolic links in structure required for successful
diff -ur a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp
--- a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2022-02-03 15:19:11.003368891 +0100
@@ -96,6 +96,7 @@
#define __builtin_unreachable()
#endif
+#if 0
namespace __cxxabiv1 {
void __cxa_throw(
@@ -154,5 +155,5 @@
}
} // namespace std
-
+#endif
#endif // defined(__GLIBCXX__)
diff -ur a/folly/Portability.h b/folly/Portability.h
--- a/folly/Portability.h 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/Portability.h 2022-02-03 15:19:11.003368891 +0100
@@ -566,7 +566,7 @@
#define FOLLY_HAS_COROUTINES 0
#elif (__cpp_coroutines >= 201703L || __cpp_impl_coroutine >= 201902L) && \
(__has_include(<coroutine>) || __has_include(<experimental/coroutine>))
-#define FOLLY_HAS_COROUTINES 1
+#define FOLLY_HAS_COROUTINES 0
// This is mainly to workaround bugs triggered by LTO, when stack allocated
// variables in await_suspend end up on a coroutine frame.
#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE

View File

@@ -1,26 +0,0 @@
diff --git a/folly/CMakeLists.txt b/folly/CMakeLists.txt
index e0e16df..471131e 100644
--- a/folly/CMakeLists.txt
+++ b/folly/CMakeLists.txt
@@ -28,7 +28,7 @@ install(
)
add_subdirectory(experimental/exception_tracer)
-add_subdirectory(logging/example)
+# add_subdirectory(logging/example)
if (PYTHON_EXTENSIONS)
# Create tree of symbolic links in structure required for successful
diff --git a/folly/Portability.h b/folly/Portability.h
index 365ef1b..42d24b8 100644
--- a/folly/Portability.h
+++ b/folly/Portability.h
@@ -560,7 +560,7 @@ constexpr auto kCpplibVer = 0;
(defined(__cpp_coroutines) && __cpp_coroutines >= 201703L) || \
(defined(__cpp_impl_coroutine) && __cpp_impl_coroutine >= 201902L)) && \
(__has_include(<coroutine>) || __has_include(<experimental/coroutine>))
-#define FOLLY_HAS_COROUTINES 1
+#define FOLLY_HAS_COROUTINES 0
// This is mainly to workaround bugs triggered by LTO, when stack allocated
// variables in await_suspend end up on a coroutine frame.
#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE

View File

@@ -1,29 +0,0 @@
diff -ur a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2021-05-05 00:53:34.000000000 +0200
+++ b/CMakeLists.txt 2022-01-27 17:18:34.758302398 +0100
@@ -52,9 +52,9 @@
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0)
- # Disable RTTI.
- string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Use -Wall for clang and gcc.
if(NOT CMAKE_CXX_FLAGS MATCHES "-Wall")
@@ -77,9 +77,9 @@
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
- # Disable RTTI.
- string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make

View File

@@ -1,75 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBEzEOZIBEACxg/IuXERlDB48JBWmF4NxNUuuup1IhJAJyFGFSKh3OGAO2Ard
sNuRLjANsFXA7m7P5eTFcG+BoHHuAVYmKnI3PPZtHVLnUt4pGItPczQZ2BE1WpcI
ayjGTBJeKItX3Npqg9D/odO9WWS1i3FQPVdrLn0YH37/BA66jeMQCRo7g7GLpaNf
IrvYGsqTbxCwsmA37rpE7oyU4Yrf74HT091WBsRIoq/MelhbxTDMR8eu/dUGZQVc
Kj3lN55RepwWwUUKyqarY0zMt4HkFJ7v7yRL+Cvzy92Ouv4Wf2FlhNtEs5LE4Tax
W0PO5AEmUoKjX87SezQK0f652018b4u6Ex52cY7p+n5TII/UyoowH6+tY8UHo9yb
fStrqgNE/mY2bhA6+AwCaOUGsFzVVPTbjtxL3HacUP/jlA1h78V8VTvTs5d55iG7
jSqR9o05wje8rwNiXXK0xtiJahyNzL97Kn/DgPSqPIi45G+8nxWSPFM5eunBKRl9
vAnsvwrdPRsR6YR3uMHTuVhQX9/CY891MHkaZJ6wydWtKt3yQwJLYqwo5d4DwnUX
CduUwSKv+6RmtWI5ZmTQYOcBRcZyGKml9X9Q8iSbm6cnpFXmLrNQwCJN+D3SiYGc
MtbltZo0ysPMa6Xj5xFaYqWk/BI4iLb2Gs+ByGo/+a0Eq4XYBMOpitNniQARAQAB
tCdMYXNzZSBDb2xsaW4gPGxhc3NlLmNvbGxpbkB0dWthYW5pLm9yZz6JAlEEEwEK
ADsCGwMCHgECF4AECwkIBwMVCggFFgIDAQAWIQQ2kMJAzlG0Zw0wrRw47nV9aRhG
IAUCYEt9dQUJFxeR4wAKCRA47nV9aRhGIBNDEACxD6vJ+enZwe3IgkJh5JtLsC9b
MWCQRlPW1EVMsg96Cb5Rtron1eN1pp1TlzENJu1/C7C/VEsr9WwOPg26Men7fNf/
O21QM9IBWd/uB0Pu333WqKh92ESS5x9ST9DrG39nVGSPkQQBMuia72VrA+crPnwT
/h/u1IN6/sff5VDIU24rUiqW2Npy733dANruj7Ny0scRXVPltnVdhqwPHt6qNjC1
t+/cCnwHgW1BR1RYXBPpB42z/m29dL9rPrG0YPGWs2Bc+EATUICfEE6eIvwfciue
IJTjKT9Y9DrogJC2AYFhjC7N04OKdCB2hFs4BjexJwr4X0GJO7LhFl03c951AsIE
GHwrucRPB5bo2vmvQ8IvZn7CmtdUJzXv9JlyU6p+MIK1pz7TK6GgSOSffQIXZn6e
nUPtm9mEwuncOfmW8/ODYPs1gCWYgyiFJx8h7eEu+M4MxHSFBs7MwXf/Ae2fSp+M
P/p198qB8fC5oVBnF95qb0Qi0uc1D+Gb+gpBF+ymMb+s/VBOR3QWiym7AzBrJ62g
UnbC9jMLGnSRI+7p7raUfMTgXr5/oQoBw7ExJVltSSRrim2YH/t4CV47mO6dR9J3
1RtsTFIRNhz+07XPsETcuCV/dgqeC8fOFLt9MY17Sufhb1DcGy4urZBOIhXcpTV7
vHVj5IYH5nYOT49NRYkCOAQTAQIAIgUCTMQ5kgIbAwYLCQgHAwIGFQgCCQoLBBYC
AwECHgECF4AACgkQOO51fWkYRiAg4A/7BXKwoRaXrMbMPOW7vuVF7c2IKB2Yqzn1
vLBCwuEHkqY237lDcXY4/5LR+1gcZ3Duw1n/BRSm0FBdvyX/JTWiWNSDUkKAO/0l
T2Tg44YLrDT3bzwu8dbU9xQt6kH+SCOHvv5Oe4k79l5mro6fF3H1M0bN63x/YoFY
ojy09D7/JptY82oR4f/VdKnfZLJcCViCb0wp8SD2NkDAudKg+K+7PD8HlTWklQQg
TZdRXxVZKIJeU42aJDqnRbAhJd64YHyClhqut9F5LUmiP5qfLfNhkKDhNOwk2Blr
BGBJkSd7wPyzcX4Mun/L6YspHjbeVMt9TD7HQlo+OOd2OjAHCx6pqwkXnzeLPEaE
cPdQ1SHgrBViAxX3DNPubLP0Knw8XwFu96EuhHZgexE1W7bB4LFsJyXAc5k1PqPD
CLsAauxmvI2OfI7opG/8wyxDvNgoPjG8fZNAgY0REqPC0JnTXChH31IxUmhNotH8
tD3DDTZOHw05n5MwwUrEE9xiETVDfFQcMLfxZ9KLz+BC2g1t5LYublRgnCMNJzFg
sNUMM02CphABzl/LCLnumr0eyQQ/weV4twEhLwSDmqLYHL0EdYW0Y3CnnU9vmYxQ
cXKbstS71sEJJYBBmSBbf9GxkOY8BRNtwVwY0kPgxv1WqdVBiAFvfB+pyAsrax9B
3UeB7ZSwRD6JAhwEEAEKAAYFAlS25GwACgkQlbYYGy0z6ew92Q//ZA9/6piQtoW4
PwP/1DtWGyKU8hwR+9FG669iPk/dAG+yoEJtFMOUpg/FUFmCX8Bc4oEHsCVyLxKt
DcCVUIRcYNSFi5hTZaBEbwsOlDT37gtlfIIu34hhHRccKaLnN/N9gNMNw8wGh9xg
Q/KtxZwcbk/bZIlDkKTJkFBRAekdEGAFDWb/AZOy+LQxS8ZAh1eWkfV0i8opmK9k
gPXtLE0WSsqtYyGs58z+BFE9NH3tEUwK6jSvtuLwQl4UrICNbKthcpb8WwH6UXzb
q3QNSYVOpf/cqRdBJA6bvb/ku/xyKVL08lGmxD9v1b137R7mafDAFPTsvH2Mt/0V
YuhtWav3r1Bl9QksDxt2DTS8wiWDUBetGqOVdcw7vBrXPEWDNBmxeJXsiJ7zJlR+
9wrJOm6RV2+l1IPxu96EaPS+kTNBijKrhxb67bww8BTEWTd0wcdJmgWRkM8SIstp
IKqd0L2TFYph2/NtrBhRg+DIEPJPpSTGsUMcCEXCZPQ+cIdlQKsWpk0tZ62DlvEl
r7E+wgUSQolRfx5KrpZifiS2zQlhzdXv28CJhsVbLyw5fUAWUKIH/dCo5NKsNLk2
Lc5DH9VWnFgxAAtW290FqeK/4ulMq7Vs1dQSwyHM2Ni3QqqeaiOrh8gbSY5CMLFN
Y3HYRwuTYPa3AobsozCzBj0Zdf/6AFe5Ag0ETMQ5kgEQAL/FwKdjxgPxtSpgq1SM
zgZtTTyLqhgGD3NZfadHWHYRIL38NDV3JeTA79Y2zj2dj7KQPDT+0aqeizTV2E3j
P3iCQ53VOT4consBaQAgKexpptnS+T1DobtICFJ0GGzf0HRj6KO2zSOuOitWPWlU
wbvX7M0LLI2+hqlx0jTPqbJFZ/Za6KTtbS6xdCPVUpUqYZQpokEZcwQmUp8Q+lGo
JD2sNYCZyap63X/aAOgCGr2RXYddOH5e8vGzGW+mwtCv+WQ9Ay35mGqI5MqkbZd1
Qbuv2b1647E/QEEucfRHVbJVKGGPpFMUJtcItyyIt5jo+r9CCL4Cs47dF/9/RNwu
NvpvHXUyqMBQdWNZRMx4k/NGD/WviPi9m6mIMui6rOQsSOaqYdcUX4Nq2Orr3Oaz
2JPQdUfeI23iot1vK8hxvUCQTV3HfJghizN6spVl0yQOKBiE8miJRgrjHilH3hTb
xoo42xDkNAq+CQo3QAm1ibDxKCDq0RcWPjcCRAN/Q5MmpcodpdKkzV0yGIS4g7s5
frVrgV/kox2r4/Yxsr8K909+4H82AjTKGX/BmsQFCTAqBk6p7I0zxjIqJ/w33TZB
Q0Pn4r3WIlUPafzY6a9/LAvN1fHRxf9SpCByJsszD03Qu5f5TB8gthsdnVmTo7jj
iordEKMtw2aEMLzdWWTQ/TNVABEBAAGJAjwEGAEKACYCGwwWIQQ2kMJAzlG0Zw0w
rRw47nV9aRhGIAUCYEt9YAUJFxeRzgAKCRA47nV9aRhGIMLtD/9HuKM4pngImcuz
YwzQmdv4j26YYyh4jVsKEmVWTiRcehEgUIlrWkCu3qzd5NK+RetS7kJ8MPnzEUfj
YbpdC6yrF6n1mSrZZ4VJMkV2ev37bIgXM+Wp1mCAGbjNxQnjn9RabT/gjIqmGuRn
AP7RsSeOSuO/gO9h2Pteciz23ussTilB+8cTooQEQQZe6Kv/zukvL+ccSehLHsZ7
qVfRUAmtt8nFkXXE+s8jfLfhqstaI2/RJu5witaPcXM8Mnz2E95aASAbZy0eQot9
0Pvf07n9yuC3tueTvzvlXx3h5U3yT44tIOmzANIQjay1TGdm+RBJ2ZYyhyLawlZ2
NVUXXSp4QZZXPA0UWbF+pb7Q9cdKDNFVuvGBljuea0Yd0T2o+ibDq43HziX9ll+l
SXk9mqvW1UcDOaxWrSsm1Gc1O9g3wqH5xHAhtY8GPh/7VgAawskPkmnlkMW6pYPy
zibbeISJL1gd1jIT63y6aoVrtNoo+wYJm280ROflh4+5QOo6QJ+jm70fkXSG/qJ5
a8/qCPTHkJc/rpkL6/TDQAJURi9RhDAC0gb40HtusbN1LZEA+i0cWTmYXap+DB4Y
R4pApilpaG87M+VUokR4xpnx7vTb2MPa7Mdenvi9FEGnKXadmT8038vlfzz5GGUT
MlVin9BQPTpdA+PpRiJvKJgVDeAFOg==
=asTC
-----END PGP PUBLIC KEY BLOCK-----

View File

@@ -111,6 +111,35 @@ enum mgp_error mgp_global_aligned_alloc(size_t size_in_bytes, size_t alignment,
/// The behavior is undefined if `ptr` is not a value returned from a prior
/// mgp_global_alloc() or mgp_global_aligned_alloc().
void mgp_global_free(void *p);
/// State of the graph database.
struct mgp_graph;
/// Allocations are tracked only for master thread. If new threads are spawned
/// inside procedure, by calling following function with thread id
/// you can start tracking allocations for that thread too. This
/// is important if you need query memory limit to work
/// for given procedure or per procedure memory limit.
enum mgp_error mgp_track_thread_allocations(struct mgp_graph *graph, const char *thread_id);
/// Once allocations are tracked for custom thread, you need to stop tracking allocations
/// for given thread, before thread finishes with execution, or is detached.
/// Otherwise it might result in slowdown of system due to unnecessary tracking of
/// allocations.
enum mgp_error mgp_untrack_thread_allocations(struct mgp_graph *graph, const char *thread_id);
/// Allocations are tracked only for master thread. If new threads are spawned
/// inside procedure, by calling following function
/// you can start tracking allocations for current thread too. This
/// is important if you need query memory limit to work
/// for given procedure or per procedure memory limit.
enum mgp_error mgp_track_current_thread_allocations(struct mgp_graph *graph);
/// Once allocations are tracked for current thread, you need to stop tracking allocations
/// for given thread, before thread finishes with execution, or is detached.
/// Otherwise it might result in slowdown of system due to unnecessary tracking of
/// allocations.
enum mgp_error mgp_untrack_current_thread_allocations(struct mgp_graph *graph);
///@}
/// @name Operations on mgp_value
@@ -851,9 +880,6 @@ enum mgp_error mgp_edge_set_properties(struct mgp_edge *e, struct mgp_map *prope
enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *memory,
struct mgp_properties_iterator **result);
/// State of the graph database.
struct mgp_graph;
/// Get the vertex corresponding to given ID, or NULL if no such vertex exists.
/// Resulting vertex must be freed using mgp_vertex_destroy.
/// Return mgp_error::MGP_ERROR_UNABLE_TO_ALLOCATE if unable to allocate the vertex.

View File

@@ -15,7 +15,6 @@ set(GFLAGS_NOTHREADS OFF)
# NOTE: config/generate.py depends on the gflags help XML format.
find_package(gflags REQUIRED)
find_package(fmt 8.0.1)
find_package(Jemalloc REQUIRED)
find_package(ZLIB 1.2.11 REQUIRED)
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
@@ -99,6 +98,17 @@ macro(import_external_library name type library_location include_dir)
import_library(${name} ${type} ${${_upper_name}_LIBRARY} ${${_upper_name}_INCLUDE_DIR})
endmacro(import_external_library)
macro(set_path_external_library name type library_location include_dir)
string(TOUPPER ${name} _upper_name)
set(${_upper_name}_LIBRARY ${library_location} CACHE FILEPATH
"Path to ${name} library" FORCE)
set(${_upper_name}_INCLUDE_DIR ${include_dir} CACHE FILEPATH
"Path to ${name} include directory" FORCE)
mark_as_advanced(${name}_LIBRARY ${name}_INCLUDE_DIR)
endmacro(set_path_external_library)
# setup antlr
import_external_library(antlr4 STATIC
${CMAKE_CURRENT_SOURCE_DIR}/antlr4/runtime/Cpp/lib/libantlr4-runtime.a
@@ -265,3 +275,8 @@ import_header_library(ctre ${CMAKE_CURRENT_SOURCE_DIR})
# setup absl (cmake sub_directory tolerant)
set(ABSL_PROPAGATE_CXX_STD ON)
add_subdirectory(absl EXCLUDE_FROM_ALL)
# set Jemalloc
set_path_external_library(jemalloc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/lib/libjemalloc.a
${CMAKE_CURRENT_SOURCE_DIR}/jemalloc/include/)

View File

@@ -124,6 +124,7 @@ declare -A primary_urls=(
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
["ctre"]="http://$local_cache_host/file/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
["absl"]="https://$local_cache_host/git/abseil-cpp.git"
["jemalloc"]="https://$local_cache_host/git/jemalloc.git"
)
# The goal of secondary urls is to have links to the "source of truth" of
@@ -151,6 +152,7 @@ declare -A secondary_urls=(
["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
["ctre"]="https://raw.githubusercontent.com/hanickadot/compile-time-regular-expressions/v3.7.2/single-header/ctre.hpp"
["absl"]="https://github.com/abseil/abseil-cpp.git"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
)
# antlr
@@ -252,3 +254,21 @@ cd ..
# abseil 20230125.3
absl_ref="20230125.3"
repo_clone_try_double "${primary_urls[absl]}" "${secondary_urls[absl]}" "absl" "$absl_ref"
# jemalloc ea6b3e973b477b8061e0076bb257dbd7f3faa756
JEMALLOC_COMMIT_VERSION="5.2.1"
repo_clone_try_double "${secondary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$JEMALLOC_COMMIT_VERSION"
# this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR
pushd jemalloc
./autogen.sh
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
./configure \
--disable-cxx \
--enable-shared=no --prefix=$working_dir \
--with-malloc-conf="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
make -j$CPUS install
popd

View File

@@ -11,12 +11,6 @@ SUPPORTED_OS=(
amzn-2
)
SUPPORTED_BUILD_TYPES=(
Debug
Release
RelWithDebInfo
)
PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
@@ -24,16 +18,14 @@ HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () {
# TODO(gitbuda): Update the release/package/run.sh help
echo "$0 init|package|docker|test {os} {build_type} [--for-docker|--for-platform]"
echo "$0 init|package|docker|test {os} [--for-docker|--for-platform]"
echo ""
echo " OSs: ${SUPPORTED_OS[*]}"
echo " Build types: ${SUPPORTED_BUILD_TYPES[*]}"
exit 1
}
make_package () {
os="$1"
build_type="$2"
build_container="mgbuild_$os"
echo "Building Memgraph for $os on $build_container..."
@@ -52,10 +44,10 @@ make_package () {
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
telemetry_id_override_flag=""
if [[ "$#" -gt 2 ]]; then
if [[ "$3" == "--for-docker" ]]; then
if [[ "$#" -gt 1 ]]; then
if [[ "$2" == "--for-docker" ]]; then
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER "
elif [[ "$3" == "--for-platform" ]]; then
elif [[ "$2" == "--for-platform" ]]; then
telemetry_id_override_flag=" -DMG_TELEMETRY_ID_OVERRIDE=DOCKER-PLATFORM"
else
print_help
@@ -97,9 +89,9 @@ make_package () {
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
# TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX
if [[ "$os" =~ "-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=$build_type -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=$build_type $telemetry_id_override_flag .."
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
fi
# ' is used instead of " because we need to run make within the allowed
# container resources.
@@ -149,34 +141,20 @@ case "$1" in
package)
shift 1
if [[ "$#" -lt 2 ]]; then
if [[ "$#" -lt 1 ]]; then
print_help
fi
os="$1"
build_type="$2"
shift 2
shift 1
is_os_ok=false
for supported_os in "${SUPPORTED_OS[@]}"; do
if [[ "$supported_os" == "${os}" ]]; then
is_os_ok=true
break
fi
done
is_build_type_ok=false
for supported_build_type in "${SUPPORTED_BUILD_TYPES[@]}"; do
if [[ "$supported_build_type" == "${build_type}" ]]; then
is_build_type_ok=true
break
fi
done
if [[ "$is_os_ok" == true && "$is_build_type_ok" == true ]]; then
make_package "$os" "$build_type" "$@"
if [[ "$is_os_ok" == true ]]; then
make_package "$os" "$@"
else
if [[ "$is_os_ok" == false ]]; then
echo "Unsupported OS: $os"
elif [[ "$is_build_type_ok" == false ]]; then
echo "Unsupported build type: $build_type"
fi
print_help
fi
;;

View File

@@ -21,7 +21,6 @@ add_subdirectory(audit)
add_subdirectory(dbms)
add_subdirectory(flags)
add_subdirectory(distributed)
add_subdirectory(replication)
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
@@ -41,7 +40,7 @@ set(mg_single_node_v2_sources
add_executable(memgraph ${mg_single_node_v2_sources})
target_include_directories(memgraph PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(memgraph stdc++fs Threads::Threads
mg-telemetry mg-communication mg-communication-metrics mg-memory mg-utils mg-license mg-settings mg-glue mg-flags)
mg-telemetry mg-communication mg-memory mg-utils mg-license mg-settings mg-glue mg-flags)
# NOTE: `include/mg_procedure.syms` describes a pattern match for symbols which
# should be dynamically exported, so that `dlopen` can correctly link the

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,6 +21,5 @@ namespace memgraph::auth {
class AuthException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(AuthException)
};
} // namespace memgraph::auth

View File

@@ -16,11 +16,8 @@ set(communication_src_files
find_package(Boost REQUIRED)
add_library(mg-communication-metrics STATIC metrics.cpp)
target_link_libraries(mg-communication-metrics json)
add_library(mg-communication STATIC ${communication_src_files})
target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io mg-auth fmt::fmt gflags mg-communication-metrics mg-events)
target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io mg-auth fmt::fmt gflags)
find_package(OpenSSL REQUIRED)
target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -264,5 +264,4 @@ bool Client::ReadMessageData(Marker marker, Value &ret) {
}
return false;
}
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -38,7 +38,6 @@ class FailureResponseException : public utils::BasicException {
: utils::BasicException{message}, code_{code} {}
const std::string &code() const { return code_; }
SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException)
private:
std::string code_;
@@ -50,7 +49,6 @@ class FailureResponseException : public utils::BasicException {
class ClientQueryException : public FailureResponseException {
public:
using FailureResponseException::FailureResponseException;
SPECIALIZE_GET_EXCEPTION_NAME(ClientQueryException)
};
/// This exception is thrown whenever a fatal error occurs during query
@@ -59,7 +57,6 @@ class ClientQueryException : public FailureResponseException {
class ClientFatalException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(ClientFatalException)
};
// Internal exception used whenever a communication error occurs. You should
@@ -67,7 +64,6 @@ class ClientFatalException : public utils::BasicException {
class ServerCommunicationException : public ClientFatalException {
public:
ServerCommunicationException() : ClientFatalException("Couldn't communicate with the server!") {}
SPECIALIZE_GET_EXCEPTION_NAME(ServerCommunicationException)
};
// Internal exception used whenever a malformed data error occurs. You should
@@ -75,7 +71,6 @@ class ServerCommunicationException : public ClientFatalException {
class ServerMalformedDataException : public ClientFatalException {
public:
ServerMalformedDataException() : ClientFatalException("The server sent malformed data!") {}
SPECIALIZE_GET_EXCEPTION_NAME(ServerMalformedDataException)
};
/// Structure that is used to return results from an executed query.
@@ -160,5 +155,4 @@ class Client final {
ChunkedEncoderBuffer<communication::ClientOutputStream> encoder_buffer_{output_stream_};
ClientEncoder encoder_{encoder_buffer_};
};
} // namespace memgraph::communication::bolt

View File

@@ -1,55 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "communication/metrics.hpp"
namespace memgraph::communication::bolt {
template <typename TSession>
inline void RegisterNewSession(TSession &session, Value &metadata) {
auto &data = metadata.ValueMap();
session.metrics_ = bolt_metrics.Add(data.contains("user_agent") ? data["user_agent"].ValueString() : "unknown",
fmt::format("{}.{}", session.version_.major, session.version_.minor),
session.client_supported_bolt_versions_);
++session.metrics_.value()->sessions;
auto conn_type = (!data.contains("scheme") || data["scheme"].ValueString() == "none")
? BoltMetrics::ConnectionType::kAnonymous
: BoltMetrics::ConnectionType::kBasic;
++session.metrics_.value()->connection_types[(int)conn_type];
}
template <typename TSession>
inline void TouchNewSession(TSession &session, Value &metadata) {
auto &data = metadata.ValueMap();
session.metrics_ = bolt_metrics.Add(data.contains("user_agent") ? data["user_agent"].ValueString() : "unknown");
}
template <typename TSession>
inline void UpdateNewSession(TSession &session, Value &metadata) {
auto &data = metadata.ValueMap();
session.metrics_.value()->bolt_v = fmt::format("{}.{}", session.version_.major, session.version_.minor);
session.metrics_.value()->supported_bolt_v = session.client_supported_bolt_versions_;
++session.metrics_.value()->sessions;
auto conn_type = (!data.contains("scheme") || data["scheme"].ValueString() == "none")
? BoltMetrics::ConnectionType::kAnonymous
: BoltMetrics::ConnectionType::kBasic;
++session.metrics_.value()->connection_types[(int)conn_type];
}
template <typename TSession>
inline void IncrementQueryMetrics(TSession &session) {
++session.metrics_.value()->queries;
}
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,7 +30,6 @@ namespace memgraph::communication::bolt {
class ClientError : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(ClientError)
};
/**
@@ -68,7 +67,6 @@ class VerboseError : public utils::BasicException {
code_(fmt::format("Memgraph.{}.{}.{}", ClassificationToString(classification), category, title)) {}
const std::string &code() const noexcept { return code_; }
SPECIALIZE_GET_EXCEPTION_NAME(VerboseError)
private:
std::string ClassificationToString(Classification classification) {

View File

@@ -27,7 +27,6 @@
#include "communication/bolt/v1/states/handshake.hpp"
#include "communication/bolt/v1/states/init.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/metrics.hpp"
#include "dbms/constants.hpp"
#include "dbms/global.hpp"
#include "utils/exceptions.hpp"
@@ -44,7 +43,6 @@ namespace memgraph::communication::bolt {
class SessionException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(SessionException)
};
/**
@@ -209,8 +207,6 @@ class Session {
};
Version version_;
std::vector<std::string> client_supported_bolt_versions_;
std::optional<BoltMetrics::Metrics> metrics_;
virtual std::string GetCurrentDB() const = 0;
std::string UUID() const { return session_uuid_; }

View File

@@ -18,7 +18,6 @@
#include <string_view>
#include <vector>
#include "communication/bolt/metrics.hpp"
#include "communication/bolt/v1/codes.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/bolt/v1/exceptions.hpp"
@@ -211,9 +210,6 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
// Increment number of queries in the metrics
IncrementQueryMetrics(session);
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap(), {});
@@ -278,9 +274,6 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
// Increment number of queries in the metrics
IncrementQueryMetrics(session);
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap(), extra.ValueMap());

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,31 +24,6 @@
namespace memgraph::communication::bolt {
inline std::vector<std::string> StringifySupportedVersions(uint8_t *data) {
std::vector<std::string> res;
uint8_t range_i = 1;
uint8_t minor_i = 2;
uint8_t major_i = 3;
uint8_t chunk_size = 4;
uint8_t n_chunks = 4;
auto stringify_version = [](uint8_t major, uint8_t minor) { return fmt::format("{}.{}", major, minor); };
for (uint8_t i = 0; i < n_chunks; ++i) {
const uint32_t full_version = *((uint32_t *)data);
if (full_version == 0) break;
if (data[1] != 0) { // Supports a range of versions
uint8_t range = data[range_i];
auto max_minor = data[minor_i];
for (uint8_t r = 0; r <= range; ++r) {
res.push_back(stringify_version(data[major_i], max_minor - r));
}
} else {
res.push_back(stringify_version(data[major_i], data[minor_i]));
}
data += chunk_size;
}
return res;
}
inline bool CopyProtocolInformationIfSupported(uint16_t version, uint8_t *protocol) {
const auto *supported_version = std::find(std::begin(kSupportedVersions), std::end(kSupportedVersions), version);
if (supported_version != std::end(kSupportedVersions)) {
@@ -96,8 +71,6 @@ State StateHandshakeRun(TSession &session) {
auto dataPosition = session.input_stream_.data() + sizeof(kPreamble);
uint8_t protocol[4] = {0x00};
session.client_supported_bolt_versions_ = std::move(StringifySupportedVersions(dataPosition));
for (int i = 0; i < 4 && !protocol[3]; ++i) {
// If there is an offset defined (e.g. 0x00 0x03 0x03 0x04) the second byte
// That would enable the client to pick between 4.0 and 4.3 versions

View File

@@ -11,7 +11,6 @@
#pragma once
#include <fmt/core.h>
#include <fmt/format.h>
#include <optional>
@@ -19,7 +18,6 @@
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
#include "communication/metrics.hpp"
#include "spdlog/spdlog.h"
#include "utils/likely.hpp"
#include "utils/logging.hpp"
@@ -203,9 +201,6 @@ State StateInitRunV1(TSession &session, const Marker marker, const Signature sig
return result.value();
}
// Register session to metrics
RegisterNewSession(session, *maybeMetadata);
return SendSuccessMessage(session);
}
@@ -232,9 +227,6 @@ State StateInitRunV4(TSession &session, Marker marker, Signature signature) {
return result.value();
}
// Register session to metrics
RegisterNewSession(session, *maybeMetadata);
return SendSuccessMessage(session);
}
@@ -255,10 +247,6 @@ State StateInitRunV5(TSession &session, Marker marker, Signature signature) {
if (SendSuccessMessage(session) == State::Close) {
return State::Close;
}
// Register session to metrics
TouchNewSession(session, *maybeMetadata);
// Stay in Init
return State::Init;
}
@@ -286,10 +274,6 @@ State StateInitRunV5(TSession &session, Marker marker, Signature signature) {
if (SendSuccessMessage(session) == State::Close) {
return State::Close;
}
// Register session to metrics
UpdateNewSession(session, *maybeMetadata);
return State::Idle;
}

View File

@@ -276,7 +276,6 @@ class ValueException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
ValueException() : BasicException("Incompatible template param and type!") {}
SPECIALIZE_GET_EXCEPTION_NAME(ValueException)
};
/**

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,6 +21,5 @@ namespace memgraph::communication {
*/
class SessionClosedException : public utils::BasicException {
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(SessionClosedException)
};
} // namespace memgraph::communication

View File

@@ -1,42 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "metrics.hpp"
namespace {
constexpr auto kName = "name";
constexpr auto kSupportedBoltVersions = "supported_bolt_versions";
constexpr auto kBoltVersion = "bolt_version";
constexpr auto kConnectionTypes = "connection_types";
constexpr auto kSessions = "sessions";
constexpr auto kQueries = "queries";
} // namespace
namespace memgraph::communication {
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
BoltMetrics bolt_metrics;
nlohmann::json BoltMetrics::Info::ToJson() const {
nlohmann::json res;
res[kName] = name;
res[kSupportedBoltVersions] = nlohmann::json::array();
for (const auto &sbv : supported_bolt_v) {
res[kSupportedBoltVersions].push_back(sbv);
}
res[kBoltVersion] = bolt_v;
res[kConnectionTypes] = {{ConnectionTypeStr((ConnectionType)0), connection_types[0].load()},
{ConnectionTypeStr((ConnectionType)1), connection_types[1].load()}};
res[kSessions] = sessions.load();
res[kQueries] = queries.load();
return res;
}
} // namespace memgraph::communication

View File

@@ -1,116 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <map>
#include <mutex>
#include <shared_mutex>
#include <string>
#include <vector>
#include <json/json.hpp>
namespace memgraph::communication {
class BoltMetrics {
public:
enum class ConnectionType { kAnonymous = 0, kBasic, Count };
static constexpr std::array<std::string_view, (int)ConnectionType::Count> ct_to_str = {"anonymous", "basic"};
static std::string ConnectionTypeStr(ConnectionType type) { return std::string(ct_to_str[(int)type]); }
class Metrics;
struct Info {
explicit Info(std::string name) : name(std::move(name)) {}
Info(std::string name, std::string bolt_v, std::vector<std::string> supported_bolt_v)
: name(std::move(name)), bolt_v(std::move(bolt_v)), supported_bolt_v(std::move(supported_bolt_v)) {}
const std::string name; //!< Driver name
std::string bolt_v; //!< Bolt version used
std::vector<std::string> supported_bolt_v; //!< Bolt versions supported by the driver
std::atomic_int connection_types[(int)ConnectionType::Count]; //!< Authentication used when connecting
std::atomic_int sessions; //!< Number of sessions using the same driver
std::atomic_int queries; //!< Queries executed by the driver
nlohmann::json ToJson() const;
private:
friend class Metrics;
std::atomic_int concurrent_users;
};
class Metrics {
friend class BoltMetrics;
explicit Metrics(Info &info) : info_(&info) { ++info_->concurrent_users; }
public:
~Metrics() {
if (info_) --info_->concurrent_users;
}
Metrics(const Metrics &other) : info_(other.info_) { ++info_->concurrent_users; }
Metrics(Metrics &&other) noexcept : info_(other.info_) { other.info_ = nullptr; }
Metrics &operator=(const Metrics &other) {
if (this != &other) {
if (info_) --info_->concurrent_users;
info_ = other.info_;
if (info_) ++info_->concurrent_users;
}
return *this;
}
Metrics &operator=(Metrics &&other) noexcept {
if (this != &other) {
if (info_) --info_->concurrent_users;
info_ = other.info_;
other.info_ = nullptr; // Invalidate the source object
}
return *this;
}
Info *operator->() { return info_; }
Info *info_;
};
Metrics Add(std::string name) {
std::unique_lock<std::mutex> l(mtx);
auto key = name;
auto [it, _] = info.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)),
std::forward_as_tuple(std::move(name)));
return Metrics(it->second);
}
Metrics Add(std::string name, std::string bolt_v, std::vector<std::string> supported_bolt_v) {
std::unique_lock<std::mutex> l(mtx);
auto key = name;
auto [it, _] = info.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)),
std::forward_as_tuple(std::move(name), std::move(bolt_v), std::move(supported_bolt_v)));
return Metrics(it->second);
}
nlohmann::json ToJson() {
std::unique_lock<std::mutex> l(mtx);
auto res = nlohmann::json::array();
for (const auto &[_, client_info] : info) {
res.push_back(client_info.ToJson());
}
return res;
}
private:
mutable std::mutex mtx;
std::map<std::string, Info> info;
};
extern BoltMetrics bolt_metrics;
} // namespace memgraph::communication

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -175,11 +175,6 @@ void Session::OnRead(const boost::beast::error_code ec, const size_t /*bytes_tra
return;
}
if (ec) {
LogError(ec, "read");
return;
}
if (!IsAuthenticated()) {
auto response = nlohmann::json();
auto auth_failed = [this, &response](const std::string &message) {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -35,7 +35,6 @@ namespace memgraph::csv {
class CsvReadException : public utils::BasicException {
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(CsvReadException)
};
class FileCsvSource {

View File

@@ -27,14 +27,6 @@
namespace memgraph::dbms {
struct DatabaseInfo {
storage::StorageInfo storage_info;
uint64_t triggers;
uint64_t streams;
};
static inline nlohmann::json ToJson(const DatabaseInfo &info) { return ToJson(info.storage_info); }
/**
* @brief Class containing everything associated with a single Database
*
@@ -97,16 +89,9 @@ class Database {
/**
* @brief Get the storage info
*
* @param force_directory Use the configured directory, do not try to decipher the multi-db version
* @return DatabaseInfo
* @return storage::StorageInfo
*/
DatabaseInfo GetInfo(bool force_directory = false) const {
DatabaseInfo info;
info.storage_info = storage_->GetInfo(force_directory);
info.triggers = trigger_store_.GetTriggerInfo().size();
info.streams = streams_.GetStreamInfo().size();
return info;
}
storage::StorageInfo GetInfo() const { return storage_->GetInfo(); }
/**
* @brief Switch storage to OnDisk

View File

@@ -25,7 +25,6 @@
#include "auth/auth.hpp"
#include "constants.hpp"
#include "dbms/database.hpp"
#include "dbms/database_handler.hpp"
#include "global.hpp"
#include "query/config.hpp"
@@ -33,7 +32,6 @@
#include "spdlog/spdlog.h"
#include "storage/v2/durability/durability.hpp"
#include "storage/v2/durability/paths.hpp"
#include "storage/v2/isolation_level.hpp"
#include "utils/exceptions.hpp"
#include "utils/file.hpp"
#include "utils/logging.hpp"
@@ -44,43 +42,6 @@
namespace memgraph::dbms {
struct Statistics {
uint64_t num_vertex; //!< Sum of vertexes in every database
uint64_t num_edges; //!< Sum of edges in every database
uint64_t triggers; //!< Sum of triggers in every database
uint64_t streams; //!< Sum of streams in every database
uint64_t users; //!< Number of defined users
uint64_t num_databases; //!< Number of isolated databases
uint64_t indices; //!< Sum of indices in every database
uint64_t constraints; //!< Sum of constraints in every database
uint64_t storage_modes[3]; //!< Number of databases in each storage mode [IN_MEM_TX, IN_MEM_ANA, ON_DISK_TX]
uint64_t isolation_levels[3]; //!< Number of databases in each isolation level [SNAPSHOT, READ_COMM, READ_UNC]
uint64_t snapshot_enabled; //!< Number of databases with snapshots enabled
uint64_t wal_enabled; //!< Number of databases with WAL enabled
};
static inline nlohmann::json ToJson(const Statistics &stats) {
nlohmann::json res;
res["edges"] = stats.num_edges;
res["vertices"] = stats.num_vertex;
res["triggers"] = stats.triggers;
res["streams"] = stats.streams;
res["users"] = stats.users;
res["databases"] = stats.num_databases;
res["indices"] = stats.indices;
res["constraints"] = stats.constraints;
res["storage_modes"] = {{storage::StorageModeToString((storage::StorageMode)0), stats.storage_modes[0]},
{storage::StorageModeToString((storage::StorageMode)1), stats.storage_modes[1]},
{storage::StorageModeToString((storage::StorageMode)2), stats.storage_modes[2]}};
res["isolation_levels"] = {{storage::IsolationLevelToString((storage::IsolationLevel)0), stats.isolation_levels[0]},
{storage::IsolationLevelToString((storage::IsolationLevel)1), stats.isolation_levels[1]},
{storage::IsolationLevelToString((storage::IsolationLevel)2), stats.isolation_levels[2]}};
res["durability"] = {{"snapshot_enabled", stats.snapshot_enabled}, {"WAL_enabled", stats.wal_enabled}};
return res;
}
#ifdef MG_ENTERPRISE
using DeleteResult = utils::BasicResult<DeleteError>;
@@ -93,6 +54,12 @@ class DbmsHandler {
using LockT = utils::RWLock;
using NewResultT = utils::BasicResult<NewError, DatabaseAccess>;
struct Statistics {
uint64_t num_vertex; //!< Sum of vertexes in every database
uint64_t num_edges; //!< Sum of edges in every database
uint64_t num_databases; //! number of isolated databases
};
/**
* @brief Initialize the handler.
*
@@ -213,51 +180,25 @@ class DbmsHandler {
}
/**
* @brief Return the statistics all databases.
* @brief Return the number of vertex across all databases.
*
* @return Statistics
* @return uint64_t
*/
Statistics Stats() {
Statistics stats{};
Statistics Info() {
// TODO: Handle overflow?
uint64_t nv = 0;
uint64_t ne = 0;
std::shared_lock<LockT> rd(lock_);
const uint64_t ndb = std::distance(db_handler_.cbegin(), db_handler_.cend());
for (auto &[_, db_gk] : db_handler_) {
auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue;
auto &db_acc = *db_acc_opt;
const auto &info = db_acc->GetInfo();
const auto &storage_info = info.storage_info;
stats.num_vertex += storage_info.vertex_count;
stats.num_edges += storage_info.edge_count;
stats.triggers += info.triggers;
stats.streams += info.streams;
++stats.num_databases;
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
++stats.storage_modes[(int)storage_info.storage_mode];
++stats.isolation_levels[(int)storage_info.isolation_level];
stats.snapshot_enabled += storage_info.durability_snapshot_enabled;
stats.wal_enabled += storage_info.durability_wal_enabled;
nv += info.vertex_count;
ne += info.edge_count;
}
return stats;
}
/**
* @brief Return a vector with all database info.
*
* @return std::vector<DatabaseInfo>
*/
std::vector<DatabaseInfo> Info() {
std::vector<DatabaseInfo> res;
res.reserve(std::distance(db_handler_.cbegin(), db_handler_.cend()));
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue;
auto &db_acc = *db_acc_opt;
res.push_back(db_acc->GetInfo());
}
return res;
return {nv, ne, ndb};
}
/**

View File

@@ -48,7 +48,6 @@ enum class SetForResult : uint8_t {
class UnknownSessionException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(UnknownSessionException)
};
/**
@@ -59,7 +58,6 @@ class UnknownSessionException : public utils::BasicException {
class UnknownDatabaseException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(UnknownDatabaseException)
};
} // namespace memgraph::dbms

View File

@@ -22,7 +22,6 @@
#include "license/license.hpp"
#include "query/discard_value_stream.hpp"
#include "query/interpreter_context.hpp"
#include "utils/event_map.hpp"
#include "utils/spin_lock.hpp"
namespace memgraph::metrics {
@@ -156,8 +155,6 @@ std::map<std::string, memgraph::communication::bolt::Value> SessionHL::Discard(s
memgraph::query::DiscardValueResultStream stream;
return DecodeSummary(interpreter_.Pull(&stream, n, qid));
} catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
// Wrap QueryException into ClientError, because we want to allow the
// client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what());
@@ -172,8 +169,6 @@ std::map<std::string, memgraph::communication::bolt::Value> SessionHL::Pull(Sess
TypedValueResultStream<TEncoder> stream(encoder, db->storage());
return DecodeSummary(interpreter_.Pull(&stream, n, qid));
} catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
// Wrap QueryException into ClientError, because we want to allow the
// client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what());
@@ -217,14 +212,10 @@ std::pair<std::vector<std::string>, std::optional<int>> SessionHL::Interpret(
return {std::move(result.headers), result.qid};
} catch (const memgraph::query::QueryException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
// Wrap QueryException into ClientError, because we want to allow the
// client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what());
} catch (const memgraph::query::ReplicationException &e) {
// Count the number of specific exceptions thrown
metrics::IncrementCounter(GetExceptionName(e));
throw memgraph::communication::bolt::ClientError(e.what());
}
}

View File

@@ -1,4 +1,4 @@
set(mg_http_handlers_sources)
add_library(mg-http-handlers STATIC ${mg_http_handlers_sources})
target_link_libraries(mg-http-handlers mg-query mg-storage-v2 mg-events)
target_link_libraries(mg-http-handlers mg-query mg-storage-v2)

View File

@@ -57,10 +57,10 @@ class MetricsService {
}
private:
storage::Storage *const db_;
const storage::Storage *db_;
MetricsResponse GetMetrics() {
auto info = db_->GetBaseInfo();
auto info = db_->GetInfo();
return MetricsResponse{.vertex_count = info.vertex_count,
.edge_count = info.edge_count,

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,14 +18,12 @@
namespace memgraph::integrations::kafka {
class KafkaStreamException : public utils::BasicException {
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(KafkaStreamException)
};
class ConsumerFailedToInitializeException : public KafkaStreamException {
public:
ConsumerFailedToInitializeException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Failed to initialize Kafka consumer {} : {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerFailedToInitializeException)
};
class SettingCustomConfigFailed : public ConsumerFailedToInitializeException {
@@ -35,55 +33,47 @@ class SettingCustomConfigFailed : public ConsumerFailedToInitializeException {
: ConsumerFailedToInitializeException(
consumer_name,
fmt::format(R"(failed to set custom config ("{}": "{}"), because of error {})", key, value, error)) {}
SPECIALIZE_GET_EXCEPTION_NAME(SettingCustomConfigFailed)
};
class ConsumerRunningException : public KafkaStreamException {
public:
explicit ConsumerRunningException(const std::string_view consumer_name)
: KafkaStreamException("Kafka consumer {} is already running", consumer_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerRunningException)
};
class ConsumerStoppedException : public KafkaStreamException {
public:
explicit ConsumerStoppedException(const std::string_view consumer_name)
: KafkaStreamException("Kafka consumer {} is already stopped", consumer_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerStoppedException)
};
class ConsumerCheckFailedException : public KafkaStreamException {
public:
explicit ConsumerCheckFailedException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Kafka consumer {} check failed: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerCheckFailedException)
};
class ConsumerStartFailedException : public KafkaStreamException {
public:
explicit ConsumerStartFailedException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Starting Kafka consumer {} failed: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerStartFailedException)
};
class TopicNotFoundException : public KafkaStreamException {
public:
TopicNotFoundException(const std::string_view consumer_name, const std::string_view topic_name)
: KafkaStreamException("Kafka consumer {} cannot find topic {}", consumer_name, topic_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(TopicNotFoundException)
};
class ConsumerCommitFailedException : public KafkaStreamException {
public:
ConsumerCommitFailedException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Committing offset of consumer {} failed: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerCommitFailedException)
};
class ConsumerReadMessagesFailedException : public KafkaStreamException {
public:
ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerReadMessagesFailedException)
};
} // namespace memgraph::integrations::kafka

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,62 +18,53 @@
namespace memgraph::integrations::pulsar {
class PulsarStreamException : public utils::BasicException {
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(PulsarStreamException)
};
class ConsumerFailedToInitializeException : public PulsarStreamException {
public:
ConsumerFailedToInitializeException(const std::string &consumer_name, const std::string &error)
: PulsarStreamException("Failed to initialize Pulsar consumer {} : {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerFailedToInitializeException)
};
class ConsumerRunningException : public PulsarStreamException {
public:
explicit ConsumerRunningException(const std::string &consumer_name)
: PulsarStreamException("Pulsar consumer {} is already running", consumer_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerRunningException)
};
class ConsumerStoppedException : public PulsarStreamException {
public:
explicit ConsumerStoppedException(const std::string &consumer_name)
: PulsarStreamException("Pulsar consumer {} is already stopped", consumer_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerStoppedException)
};
class ConsumerCheckFailedException : public PulsarStreamException {
public:
explicit ConsumerCheckFailedException(const std::string &consumer_name, const std::string &error)
: PulsarStreamException("Pulsar consumer {} check failed: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerCheckFailedException)
};
class ConsumerStartFailedException : public PulsarStreamException {
public:
explicit ConsumerStartFailedException(const std::string &consumer_name, const std::string &error)
: PulsarStreamException("Starting Pulsar consumer {} failed: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerStartFailedException)
};
class TopicNotFoundException : public PulsarStreamException {
public:
TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
: PulsarStreamException("Pulsar consumer {} cannot find topic {}", consumer_name, topic_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(TopicNotFoundException)
};
class ConsumerReadMessagesFailedException : public PulsarStreamException {
public:
ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
: PulsarStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerReadMessagesFailedException)
};
class ConsumerAcknowledgeMessagesFailedException : public PulsarStreamException {
public:
explicit ConsumerAcknowledgeMessagesFailedException(const std::string_view consumer_name)
: PulsarStreamException("Acknowledging a message of consumer {} has failed!", consumer_name) {}
SPECIALIZE_GET_EXCEPTION_NAME(ConsumerAcknowledgeMessagesFailedException)
};
} // namespace memgraph::integrations::pulsar

View File

@@ -4,5 +4,4 @@ find_package(ZLIB REQUIRED)
# STATIC library used to store key-value pairs
add_library(mg-kvstore STATIC kvstore.cpp)
add_library(mg::kvstore ALIAS mg-kvstore)
target_link_libraries(mg-kvstore stdc++fs mg-utils rocksdb BZip2::BZip2 ZLIB::ZLIB gflags)

View File

@@ -25,7 +25,6 @@ namespace memgraph::kvstore {
class KVStoreError : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(KVStoreError)
};
/**

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -10,7 +10,6 @@
// licenses/APL.txt.
#include "audit/log.hpp"
#include "communication/metrics.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "dbms/constants.hpp"
@@ -23,6 +22,7 @@
#include "glue/run_id.hpp"
#include "helpers.hpp"
#include "license/license_sender.hpp"
#include "memory/memory_control.hpp"
#include "query/config.hpp"
#include "query/discard_value_stream.hpp"
#include "query/interpreter.hpp"
@@ -32,7 +32,6 @@
#include "requests/requests.hpp"
#include "telemetry/telemetry.hpp"
#include "utils/signals.hpp"
#include "utils/skip_list.hpp"
#include "utils/sysinfo/memory.hpp"
#include "utils/system_info.hpp"
#include "utils/terminate_handler.hpp"
@@ -108,6 +107,7 @@ void InitSignalHandlers(const std::function<void()> &shutdown_fun) {
}
int main(int argc, char **argv) {
memgraph::memory::SetHooks();
google::SetUsageMessage("Memgraph database server");
gflags::SetVersionString(version_string);
@@ -199,7 +199,6 @@ int main(int argc, char **argv) {
"won't be available.");
}
}
std::cout << "You are running Memgraph v" << gflags::VersionString() << std::endl;
std::cout << "To get started with Memgraph, visit https://memgr.ph/start" << std::endl;
@@ -430,19 +429,31 @@ int main(int argc, char **argv) {
std::optional<memgraph::telemetry::Telemetry> telemetry;
if (FLAGS_telemetry_enabled) {
telemetry.emplace(telemetry_server, data_directory / "telemetry", memgraph::glue::run_id_, machine_id,
service_name == "BoltS", FLAGS_data_directory, std::chrono::minutes(10));
std::chrono::minutes(10));
#ifdef MG_ENTERPRISE
telemetry->AddStorageCollector(new_handler, auth_);
telemetry->AddDatabaseCollector(new_handler);
telemetry->AddCollector("storage", [&new_handler]() -> nlohmann::json {
const auto &info = new_handler.Info();
return {{"vertices", info.num_vertex}, {"edges", info.num_edges}, {"databases", info.num_databases}};
});
#else
telemetry->AddStorageCollector(db_gatekeeper, auth_);
telemetry->AddDatabaseCollector();
telemetry->AddCollector("storage", [gk = &db_gatekeeper]() -> nlohmann::json {
auto db_acc = gk->access();
MG_ASSERT(db_acc, "Failed to get access to the default database");
auto info = db_acc->get()->GetInfo();
return {{"vertices", info.vertex_count}, {"edges", info.edge_count}};
});
#endif
telemetry->AddClientCollector();
telemetry->AddEventsCollector();
telemetry->AddQueryModuleCollector();
telemetry->AddExceptionCollector();
telemetry->AddReplicationCollector();
telemetry->AddCollector("event_counters", []() -> nlohmann::json {
nlohmann::json ret;
for (size_t i = 0; i < memgraph::metrics::CounterEnd(); ++i) {
ret[memgraph::metrics::GetCounterName(i)] =
memgraph::metrics::global_counters[i].load(std::memory_order_relaxed);
}
return ret;
});
telemetry->AddCollector("query_module_counters", []() -> nlohmann::json {
return memgraph::query::plan::CallProcedure::GetAndResetCounters();
});
}
memgraph::license::LicenseInfoSender license_info_sender(telemetry_server, memgraph::glue::run_id_, machine_id,
memory_limit,

View File

@@ -2,7 +2,9 @@ set(memory_src_files
new_delete.cpp
memory_control.cpp)
find_package(Jemalloc REQUIRED)
find_package(jemalloc REQUIRED)
add_library(mg-memory STATIC ${memory_src_files})
target_link_libraries(mg-memory mg-utils fmt)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,6 +10,11 @@
// licenses/APL.txt.
#include "memory_control.hpp"
#include <cstdint>
#include <sstream>
#include <thread>
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#if USE_JEMALLOC
#include <jemalloc/jemalloc.h>
@@ -22,6 +27,274 @@ namespace memgraph::memory {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define STRINGIFY(x) STRINGIFY_HELPER(x)
std::string get_string_thread_id(const std::thread::id &thread_id) {
std::ostringstream oss;
oss << thread_id;
return oss.str();
}
// TODO (af) think if following implementation would make sense
/*
std::string get_thread_id() {
static thread_local std::thread::id current_thread_id = std::this_thread::get_id();
// figure out how to cache this part
std::ostringstream oss;
oss << current_thread_id;
return oss.str();
}
*/
std::string get_thread_id() { return get_string_thread_id(std::this_thread::get_id()); }
#if USE_JEMALLOC
static void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero,
bool *commit, unsigned arena_ind);
static bool my_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind);
static void my_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind);
static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind);
static bool my_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind);
static bool my_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind);
extent_hooks_t *old_hooks = nullptr;
static extent_hooks_t custom_hooks = {
.alloc = &my_alloc,
.dalloc = &my_dalloc,
.destroy = &my_destroy,
.commit = &my_commit,
.decommit = &my_decommit,
.purge_lazy = nullptr,
.purge_forced = &my_purge_forced,
.split = nullptr,
.merge = nullptr,
};
static const extent_hooks_t *new_hooks = &custom_hooks;
void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit,
unsigned arena_ind) {
// This needs to be before, to throw exception in case of too big alloc
if (*commit) [[likely]] {
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Alloc(static_cast<int64_t>(size));
}
}
auto *ptr = old_hooks->alloc(extent_hooks, new_addr, size, alignment, zero, commit, arena_ind);
if (ptr == nullptr) [[unlikely]] {
if (*commit) {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Free(static_cast<int64_t>(size));
}
}
return ptr;
}
return ptr;
}
static bool my_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) {
auto err = old_hooks->dalloc(extent_hooks, addr, size, committed, arena_ind);
if (err) [[unlikely]] {
return err;
}
if (committed) [[likely]] {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Free(static_cast<int64_t>(size));
}
}
return false;
}
static void my_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) {
if (committed) [[likely]] {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Free(static_cast<int64_t>(size));
}
}
old_hooks->destroy(extent_hooks, addr, size, committed, arena_ind);
}
static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind) {
auto err = old_hooks->commit(extent_hooks, addr, size, offset, length, arena_ind);
if (err) {
return err;
}
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Alloc(static_cast<int64_t>(size));
}
return false;
}
static bool my_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind) {
MG_ASSERT(old_hooks && old_hooks->decommit);
auto err = old_hooks->decommit(extent_hooks, addr, size, offset, length, arena_ind);
if (err) {
return err;
}
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(length));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Free(static_cast<int64_t>(size));
}
return false;
}
static bool my_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length,
unsigned arena_ind) {
MG_ASSERT(old_hooks && old_hooks->purge_forced);
auto err = old_hooks->purge_forced(extent_hooks, addr, size, offset, length, arena_ind);
if (err) [[unlikely]] {
return err;
}
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(length));
if (arena_tracking[arena_ind]) [[unlikely]] {
transaction_id_tracker[thread_id_to_transaction_id[get_thread_id()]].Free(static_cast<int64_t>(size));
}
return false;
}
#endif
void SetHooks() {
#if USE_JEMALLOC
uint64_t allocated{0};
uint64_t sz{sizeof(allocated)};
sz = sizeof(unsigned);
unsigned n_arenas{0};
int err = mallctl("opt.narenas", (void *)&n_arenas, &sz, nullptr, 0);
if (err) {
return;
}
spdlog::trace("n areanas {}", n_arenas);
if (nullptr != old_hooks) {
return;
}
for (int i = 0; i < n_arenas; i++) {
arena_tracking[i] = 0;
std::string func_name = "arena." + std::to_string(i) + ".extent_hooks";
size_t hooks_len = sizeof(old_hooks);
int err = mallctl(func_name.c_str(), &old_hooks, &hooks_len, nullptr, 0);
if (err) {
LOG_FATAL("Error getting hooks for jemalloc arena {}", i);
}
// Due to the way jemalloc works, we need first to set their hooks
// which will trigger creating arena, then we can set our custom hook wrappers
err = mallctl(func_name.c_str(), nullptr, nullptr, &old_hooks, sizeof(old_hooks));
MG_ASSERT(old_hooks);
MG_ASSERT(old_hooks->alloc);
MG_ASSERT(old_hooks->dalloc);
MG_ASSERT(old_hooks->destroy);
MG_ASSERT(old_hooks->commit);
MG_ASSERT(old_hooks->decommit);
MG_ASSERT(old_hooks->purge_forced);
MG_ASSERT(old_hooks->purge_lazy);
MG_ASSERT(old_hooks->split);
MG_ASSERT(old_hooks->merge);
custom_hooks.purge_lazy = old_hooks->purge_lazy;
custom_hooks.split = old_hooks->split;
custom_hooks.merge = old_hooks->merge;
if (err) {
LOG_FATAL("Error setting jemalloc hooks for jemalloc arena {}", i);
}
err = mallctl(func_name.c_str(), nullptr, nullptr, &new_hooks, sizeof(new_hooks));
if (err) {
LOG_FATAL("Error setting custom hooks for jemalloc arena {}", i);
}
}
#endif
}
unsigned GetArenaForThread() {
#if USE_JEMALLOC
unsigned thread_arena{0};
size_t size_thread_arena = sizeof(thread_arena);
int err = mallctl("thread.arena", &thread_arena, &size_thread_arena, nullptr, 0);
if (err) {
return -1;
}
return thread_arena;
#endif
return -1;
}
bool AddTrackingOnArena(unsigned arena_id) {
#if USE_JEMALLOC
arena_tracking[arena_id].fetch_add(1);
#endif
return false;
}
bool RemoveTrackingOnArena(unsigned arena_id) {
#if USE_JEMALLOC
arena_tracking[arena_id].fetch_sub(1);
#endif
return true;
}
void UpdateThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
thread_id_to_transaction_id[get_string_thread_id(thread_id)] = transaction_id;
}
void UpdateThreadToTransactionId(const char *thread_id, uint64_t transaction_id) {
thread_id_to_transaction_id[std::string(thread_id)] = transaction_id;
}
void ResetThreadToTransactionId(const std::thread::id &thread_id) {
thread_id_to_transaction_id.erase(get_string_thread_id(thread_id));
}
void ResetThreadToTransactionId(const char *thread_id) { thread_id_to_transaction_id.erase(std::string(thread_id)); }
void AddTrackingsOnCurrentThread(uint64_t transaction_id) {
UpdateThreadToTransactionId(std::this_thread::get_id(), transaction_id);
AddTrackingOnArena(memgraph::memory::GetArenaForThread());
}
void RemoveTrackingsOnCurrentThread() {
ResetThreadToTransactionId(std::this_thread::get_id());
RemoveTrackingOnArena(GetArenaForThread());
}
void PurgeUnusedMemory() {
#if USE_JEMALLOC
mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
@@ -30,4 +303,5 @@ void PurgeUnusedMemory() {
#undef STRINGIFY
#undef STRINGIFY_HELPER
} // namespace memgraph::memory

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,6 +11,34 @@
#pragma once
#include <cstddef>
#include <cstdint>
#include <unordered_map>
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include <thread>
namespace memgraph::memory {
void PurgeUnusedMemory();
void SetHooks();
// TODO(af) This should all be part of memgraph::memory::thread namespace, and moved to different file
// This should be part of class
unsigned GetArenaForThread();
bool AddTrackingOnArena(unsigned);
bool RemoveTrackingOnArena(unsigned);
void UpdateThreadToTransactionId(const std::thread::id &, uint64_t);
void ResetThreadToTransactionId(const std::thread::id &);
void UpdateThreadToTransactionId(const char *, uint64_t);
void ResetThreadToTransactionId(const char *);
void AddTrackingsOnCurrentThread(uint64_t);
void RemoveTrackingsOnCurrentThread();
inline std::unordered_map<std::string, uint64_t> thread_id_to_transaction_id;
// TODO(af): think if we need to solve issue of tracking allocations for arena
// if user forgets to unregister tracking for that thread before it dies.
inline std::unordered_map<unsigned, std::atomic<int>> arena_tracking;
inline std::unordered_map<uint64_t, utils::MemoryTracker> transaction_id_tracker;
} // namespace memgraph::memory

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -87,21 +87,15 @@ void deleteSized(void *ptr, const std::size_t /*unused*/, const std::align_val_t
#endif
void TrackMemory(std::size_t size) {
#if USE_JEMALLOC
if (size != 0) [[likely]] {
size = nallocx(size, 0);
}
#endif
#if !USE_JEMALLOC
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
#endif
}
void TrackMemory(std::size_t size, const std::align_val_t align) {
#if USE_JEMALLOC
if (size != 0) [[likely]] {
size = nallocx(size, MALLOCX_ALIGN(align)); // NOLINT(hicpp-signed-bitwise)
}
#endif
#if !USE_JEMALLOC
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
#endif
}
bool TrackMemoryNoExcept(const std::size_t size) {
@@ -126,11 +120,7 @@ bool TrackMemoryNoExcept(const std::size_t size, const std::align_val_t align) {
void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size = 0) noexcept {
try {
#if USE_JEMALLOC
if (ptr != nullptr) [[likely]] {
memgraph::utils::total_memory_tracker.Free(sallocx(ptr, 0));
}
#else
#if !USE_JEMALLOC
if (size) {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
} else {
@@ -144,11 +134,7 @@ void UntrackMemory([[maybe_unused]] void *ptr, [[maybe_unused]] std::size_t size
void UntrackMemory(void *ptr, const std::align_val_t align, [[maybe_unused]] std::size_t size = 0) noexcept {
try {
#if USE_JEMALLOC
if (ptr != nullptr) [[likely]] {
memgraph::utils::total_memory_tracker.Free(sallocx(ptr, MALLOCX_ALIGN(align))); // NOLINT(hicpp-signed-bitwise)
}
#else
#if !USE_JEMALLOC
if (size) {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
} else {

View File

@@ -162,7 +162,6 @@ struct hash<NodeId> {
class LoadException : public memgraph::utils::BasicException {
public:
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(LoadException)
};
enum class CsvParserState {

View File

@@ -55,8 +55,7 @@ target_link_libraries(mg-query PUBLIC dl
mg-memory
mg::csv
mg-flags
mg-dbms
mg-events)
mg-dbms)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()

View File

@@ -19,7 +19,7 @@ struct InterpreterConfig {
bool allow_load_csv{true};
} query;
// The same as \ref memgraph::replication::ReplicationClientConfig
// The same as \ref memgraph::storage::replication::ReplicationClientConfig
std::chrono::seconds replication_replica_check_frequency{1};
std::string default_kafka_bootstrap_servers;

View File

@@ -21,6 +21,18 @@ namespace memgraph::query {
SubgraphDbAccessor::SubgraphDbAccessor(query::DbAccessor db_accessor, Graph *graph)
: db_accessor_(db_accessor), graph_(graph) {}
void SubgraphDbAccessor::TrackThreadAllocations(const char *thread_id) {
return db_accessor_.TrackThreadAllocations(thread_id);
}
void SubgraphDbAccessor::TrackCurrentThreadAllocations() { return db_accessor_.TrackCurrentThreadAllocations(); }
void SubgraphDbAccessor::UntrackThreadAllocations(const char *thread_id) {
return db_accessor_.UntrackThreadAllocations(thread_id);
}
void SubgraphDbAccessor::UntrackCurrentThreadAllocations() { return db_accessor_.TrackCurrentThreadAllocations(); }
storage::PropertyId SubgraphDbAccessor::NameToProperty(const std::string_view name) {
return db_accessor_.NameToProperty(name);
}

View File

@@ -17,6 +17,7 @@
#include <cppitertools/filter.hpp>
#include <cppitertools/imap.hpp>
#include "memory/memory_control.hpp"
#include "query/exceptions.hpp"
#include "storage/v2/edge_accessor.hpp"
#include "storage/v2/id_types.hpp"
@@ -372,6 +373,26 @@ class DbAccessor final {
void FinalizeTransaction() { accessor_->FinalizeTransaction(); }
void TrackThreadAllocations(const char *thread_id) {
memgraph::memory::UpdateThreadToTransactionId(thread_id, *accessor_->GetTransactionId());
auto arena = memgraph::memory::GetArenaForThread();
memgraph::memory::AddTrackingOnArena(arena);
}
void TrackCurrentThreadAllocations() {
memgraph::memory::AddTrackingsOnCurrentThread(*accessor_->GetTransactionId());
}
void UntrackThreadAllocations(const char *thread_id) {
memgraph::memory::ResetThreadToTransactionId(thread_id);
auto arena = memgraph::memory::GetArenaForThread();
memgraph::memory::RemoveTrackingOnArena(arena);
}
void UntrackCurrentThreadAllocations() { memgraph::memory::RemoveTrackingsOnCurrentThread(); }
std::optional<uint64_t> GetTransactionId() { return accessor_->GetTransactionId(); }
VerticesIterable Vertices(storage::View view) { return VerticesIterable(accessor_->Vertices(view)); }
VerticesIterable Vertices(storage::View view, storage::LabelId label) {
@@ -395,6 +416,10 @@ class DbAccessor final {
VertexAccessor InsertVertex() { return VertexAccessor(accessor_->CreateVertex()); }
void PrefetchOutEdges(const VertexAccessor &vertex) const { accessor_->PrefetchOutEdges(vertex.impl_); }
void PrefetchInEdges(const VertexAccessor &vertex) const { accessor_->PrefetchInEdges(vertex.impl_); }
storage::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::EdgeTypeId &edge_type) {
auto maybe_edge = accessor_->CreateEdge(&from->impl_, &to->impl_, edge_type);
@@ -432,6 +457,8 @@ class DbAccessor final {
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
accessor_->PrefetchOutEdges(vertex_accessor->impl_);
accessor_->PrefetchInEdges(vertex_accessor->impl_);
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
@@ -477,6 +504,9 @@ class DbAccessor final {
edges_impl.reserve(edges.size());
for (auto &vertex_accessor : nodes) {
accessor_->PrefetchOutEdges(vertex_accessor.impl_);
accessor_->PrefetchInEdges(vertex_accessor.impl_);
nodes_impl.push_back(&vertex_accessor.impl_);
}
@@ -634,6 +664,14 @@ class SubgraphDbAccessor final {
static SubgraphDbAccessor *MakeSubgraphDbAccessor(DbAccessor *db_accessor, Graph *graph);
void TrackThreadAllocations(const char *thread_id);
void TrackCurrentThreadAllocations();
void UntrackThreadAllocations(const char *thread_id);
void UntrackCurrentThreadAllocations();
storage::PropertyId NameToProperty(std::string_view name);
storage::LabelId NameToLabel(std::string_view name);
@@ -646,6 +684,10 @@ class SubgraphDbAccessor final {
const std::string &EdgeTypeToName(storage::EdgeTypeId type) const;
void PrefetchOutEdges(const SubgraphVertexAccessor &vertex) const { db_accessor_.PrefetchOutEdges(vertex.impl_); }
void PrefetchInEdges(const SubgraphVertexAccessor &vertex) const { db_accessor_.PrefetchInEdges(vertex.impl_); }
storage::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge);
storage::Result<EdgeAccessor> InsertEdge(SubgraphVertexAccessor *from, SubgraphVertexAccessor *to,

View File

@@ -482,6 +482,7 @@ PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
// If we have a saved iterable from a previous pull
// we need to use the same iterable
if (!maybe_edge_iterable) {
dba_->PrefetchOutEdges(vertex);
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::View::OLD));
}
auto &maybe_edges = *maybe_edge_iterable;

View File

@@ -14,7 +14,6 @@
#include "utils/exceptions.hpp"
#include <fmt/format.h>
#include <exception>
namespace memgraph::query {
@@ -26,21 +25,18 @@ namespace memgraph::query {
*/
class QueryException : public utils::BasicException {
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(QueryException)
};
class LexingException : public QueryException {
public:
using QueryException::QueryException;
LexingException() : QueryException("") {}
SPECIALIZE_GET_EXCEPTION_NAME(LexingException)
};
class SyntaxException : public QueryException {
public:
using QueryException::QueryException;
SyntaxException() : QueryException("") {}
SPECIALIZE_GET_EXCEPTION_NAME(SyntaxException)
};
// TODO: Figure out what information to put in exception.
@@ -56,19 +52,16 @@ class SemanticException : public QueryException {
public:
using QueryException::QueryException;
SemanticException() : QueryException("") {}
SPECIALIZE_GET_EXCEPTION_NAME(SemanticException)
};
class UnboundVariableError : public SemanticException {
public:
explicit UnboundVariableError(const std::string &name) : SemanticException("Unbound variable: " + name + ".") {}
SPECIALIZE_GET_EXCEPTION_NAME(UnboundVariableError)
};
class RedeclareVariableError : public SemanticException {
public:
explicit RedeclareVariableError(const std::string &name) : SemanticException("Redeclaring variable: " + name + ".") {}
SPECIALIZE_GET_EXCEPTION_NAME(RedeclareVariableError)
};
class TypeMismatchError : public SemanticException {
@@ -76,27 +69,23 @@ class TypeMismatchError : public SemanticException {
TypeMismatchError(const std::string &name, const std::string &datum, const std::string &expected)
: SemanticException(fmt::format("Type mismatch: {} already defined as {}, expected {}.", name, datum, expected)) {
}
SPECIALIZE_GET_EXCEPTION_NAME(TypeMismatchError)
};
class UnprovidedParameterError : public QueryException {
public:
using QueryException::QueryException;
SPECIALIZE_GET_EXCEPTION_NAME(UnprovidedParameterError)
};
class ProfileInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
ProfileInMulticommandTxException() : QueryException("PROFILE not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ProfileInMulticommandTxException)
};
class IndexInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
IndexInMulticommandTxException() : QueryException("Index manipulation not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(IndexInMulticommandTxException)
};
class ConstraintInMulticommandTxException : public QueryException {
@@ -106,14 +95,12 @@ class ConstraintInMulticommandTxException : public QueryException {
: QueryException(
"Constraint manipulation not allowed in multicommand "
"transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ConstraintInMulticommandTxException)
};
class InfoInMulticommandTxException : public QueryException {
public:
using QueryException::QueryException;
InfoInMulticommandTxException() : QueryException("Info reporting not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(InfoInMulticommandTxException)
};
/**
@@ -123,7 +110,6 @@ class InfoInMulticommandTxException : public QueryException {
class QueryRuntimeException : public QueryException {
public:
using QueryException::QueryException;
SPECIALIZE_GET_EXCEPTION_NAME(QueryRuntimeException)
};
enum class AbortReason : uint8_t {
@@ -146,7 +132,6 @@ class HintedAbortError : public utils::BasicException {
public:
using utils::BasicException::BasicException;
explicit HintedAbortError(AbortReason reason) : utils::BasicException(AsMsg(reason)), reason_{reason} {}
SPECIALIZE_GET_EXCEPTION_NAME(HintedAbortError)
auto Reason() const -> AbortReason { return reason_; }
@@ -171,20 +156,17 @@ class HintedAbortError : public utils::BasicException {
class ExplicitTransactionUsageException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
SPECIALIZE_GET_EXCEPTION_NAME(ExplicitTransactionUsageException)
};
class DatabaseContextRequiredException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
SPECIALIZE_GET_EXCEPTION_NAME(DatabaseContextRequiredException)
};
class WriteVertexOperationInEdgeImportModeException : public QueryException {
public:
WriteVertexOperationInEdgeImportModeException()
: QueryException("Write operations on vertices are forbidden while the edge import mode is active.") {}
SPECIALIZE_GET_EXCEPTION_NAME(WriteVertexOperationInEdgeImportModeException)
};
class TransactionSerializationException : public QueryException {
@@ -194,7 +176,6 @@ class TransactionSerializationException : public QueryException {
: QueryException(
"Cannot resolve conflicting transactions. You can retry this transaction when the conflicting transaction "
"is finished") {}
SPECIALIZE_GET_EXCEPTION_NAME(TransactionSerializationException)
};
class ReconstructionException : public QueryException {
@@ -203,7 +184,6 @@ class ReconstructionException : public QueryException {
: QueryException(
"Record invalid after WITH clause. Most likely deleted by a "
"preceeding DELETE.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ReconstructionException)
};
class RemoveAttachedVertexException : public QueryRuntimeException {
@@ -212,89 +192,76 @@ class RemoveAttachedVertexException : public QueryRuntimeException {
: QueryRuntimeException(
"Failed to remove node because of it's existing "
"connections. Consider using DETACH DELETE.") {}
SPECIALIZE_GET_EXCEPTION_NAME(RemoveAttachedVertexException)
};
class UserModificationInMulticommandTxException : public QueryException {
public:
UserModificationInMulticommandTxException()
: QueryException("Authentication clause not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(UserModificationInMulticommandTxException)
};
class InvalidArgumentsException : public QueryException {
public:
InvalidArgumentsException(const std::string &argument_name, const std::string &message)
: QueryException(fmt::format("Invalid arguments sent: {} - {}", argument_name, message)) {}
SPECIALIZE_GET_EXCEPTION_NAME(InvalidArgumentsException)
};
class ReplicationModificationInMulticommandTxException : public QueryException {
public:
ReplicationModificationInMulticommandTxException()
: QueryException("Replication clause not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ReplicationModificationInMulticommandTxException)
};
class ReplicationDisabledOnDiskStorage : public QueryException {
public:
ReplicationDisabledOnDiskStorage() : QueryException("Replication is not supported while in on-disk storage mode.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ReplicationDisabledOnDiskStorage)
};
class LockPathModificationInMulticommandTxException : public QueryException {
public:
LockPathModificationInMulticommandTxException()
: QueryException("Lock path query not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(LockPathModificationInMulticommandTxException)
};
class LockPathDisabledOnDiskStorage : public QueryException {
public:
LockPathDisabledOnDiskStorage()
: QueryException("Lock path disabled on disk storage since all data is already persisted. ") {}
SPECIALIZE_GET_EXCEPTION_NAME(LockPathDisabledOnDiskStorage)
};
class FreeMemoryModificationInMulticommandTxException : public QueryException {
public:
FreeMemoryModificationInMulticommandTxException()
: QueryException("Free memory query not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(FreeMemoryModificationInMulticommandTxException)
};
class FreeMemoryDisabledOnDiskStorage : public QueryException {
public:
FreeMemoryDisabledOnDiskStorage() : QueryException("Free memory does nothing when using disk storage. ") {}
SPECIALIZE_GET_EXCEPTION_NAME(FreeMemoryDisabledOnDiskStorage)
};
class ShowConfigModificationInMulticommandTxException : public QueryException {
public:
ShowConfigModificationInMulticommandTxException()
: QueryException("Show config query not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ShowConfigModificationInMulticommandTxException)
};
class TriggerModificationInMulticommandTxException : public QueryException {
public:
TriggerModificationInMulticommandTxException()
: QueryException("Trigger queries not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ShowConfigModificationInMulticommandTxException)
};
class StreamQueryInMulticommandTxException : public QueryException {
public:
StreamQueryInMulticommandTxException()
: QueryException("Stream queries are not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(StreamQueryInMulticommandTxException)
};
class IsolationLevelModificationInMulticommandTxException : public QueryException {
public:
IsolationLevelModificationInMulticommandTxException()
: QueryException("Isolation level cannot be modified in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(IsolationLevelModificationInMulticommandTxException)
};
class IsolationLevelModificationInAnalyticsException : public QueryException {
@@ -304,62 +271,53 @@ class IsolationLevelModificationInAnalyticsException : public QueryException {
"Isolation level cannot be modified when storage mode is set to IN_MEMORY_ANALYTICAL."
"IN_MEMORY_ANALYTICAL mode doesn't provide any isolation guarantees, "
"you can think about it as an equivalent to READ_UNCOMMITED.") {}
SPECIALIZE_GET_EXCEPTION_NAME(IsolationLevelModificationInAnalyticsException)
};
class StorageModeModificationInMulticommandTxException : public QueryException {
public:
StorageModeModificationInMulticommandTxException()
: QueryException("Storage mode cannot be modified in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(StorageModeModificationInMulticommandTxException)
};
class EdgeImportModeModificationInMulticommandTxException : public QueryException {
public:
EdgeImportModeModificationInMulticommandTxException()
: QueryException("Edge import mode cannot be modified in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(EdgeImportModeModificationInMulticommandTxException)
};
class CreateSnapshotInMulticommandTxException final : public QueryException {
public:
CreateSnapshotInMulticommandTxException()
: QueryException("Snapshot cannot be created in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(CreateSnapshotInMulticommandTxException)
};
class CreateSnapshotDisabledOnDiskStorage final : public QueryException {
public:
CreateSnapshotDisabledOnDiskStorage() : QueryException("In the on-disk storage mode data is already persistent.") {}
SPECIALIZE_GET_EXCEPTION_NAME(CreateSnapshotDisabledOnDiskStorage)
};
class EdgeImportModeQueryDisabledOnDiskStorage final : public QueryException {
public:
EdgeImportModeQueryDisabledOnDiskStorage()
: QueryException("Edge import mode is only allowed for on-disk storage mode.") {}
SPECIALIZE_GET_EXCEPTION_NAME(EdgeImportModeQueryDisabledOnDiskStorage)
};
class SettingConfigInMulticommandTxException final : public QueryException {
public:
SettingConfigInMulticommandTxException()
: QueryException("Settings cannot be changed or fetched in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(SettingConfigInMulticommandTxException)
};
class VersionInfoInMulticommandTxException : public QueryException {
public:
VersionInfoInMulticommandTxException()
: QueryException("Version info query not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(VersionInfoInMulticommandTxException)
};
class AnalyzeGraphInMulticommandTxException : public QueryException {
public:
AnalyzeGraphInMulticommandTxException()
: QueryException("Analyze graph query not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(AnalyzeGraphInMulticommandTxException)
};
class ReplicationException : public utils::BasicException {
@@ -368,33 +326,28 @@ class ReplicationException : public utils::BasicException {
explicit ReplicationException(const std::string &message)
: utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICAS' query.",
message) {}
SPECIALIZE_GET_EXCEPTION_NAME(ReplicationException)
};
class TransactionQueueInMulticommandTxException : public QueryException {
public:
TransactionQueueInMulticommandTxException()
: QueryException("Transaction queue queries not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(TransactionQueueInMulticommandTxException)
};
class IndexPersistenceException : public QueryException {
public:
IndexPersistenceException() : QueryException("Persisting index on disk failed.") {}
SPECIALIZE_GET_EXCEPTION_NAME(IndexPersistenceException)
};
class ConstraintsPersistenceException : public QueryException {
public:
ConstraintsPersistenceException() : QueryException("Persisting constraints on disk failed.") {}
SPECIALIZE_GET_EXCEPTION_NAME(ConstraintsPersistenceException)
};
class MultiDatabaseQueryInMulticommandTxException : public QueryException {
public:
MultiDatabaseQueryInMulticommandTxException()
: QueryException("Multi-database queries are not allowed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(MultiDatabaseQueryInMulticommandTxException)
};
} // namespace memgraph::query

View File

@@ -518,10 +518,6 @@ bool SymbolGenerator::PreVisit(Exists &exists) {
throw utils::NotYetImplemented("WITH can not be used with exists, but only during matching!");
}
if (scope.in_return) {
throw utils::NotYetImplemented("RETURN can not be used with exists, but only during matching!");
}
scope.in_exists = true;
const auto &symbol = CreateAnonymousSymbol();

View File

@@ -472,6 +472,8 @@ TypedValue Degree(const TypedValue *args, int64_t nargs, const FunctionContext &
FType<Or<Null, Vertex>>("degree", args, nargs);
if (args[0].IsNull()) return TypedValue(ctx.memory);
const auto &vertex = args[0].ValueVertex();
ctx.db_accessor->PrefetchInEdges(vertex);
ctx.db_accessor->PrefetchOutEdges(vertex);
size_t out_degree = UnwrapDegreeResult(vertex.OutDegree(ctx.view));
size_t in_degree = UnwrapDegreeResult(vertex.InDegree(ctx.view));
return TypedValue(static_cast<int64_t>(out_degree + in_degree), ctx.memory);
@@ -481,6 +483,7 @@ TypedValue InDegree(const TypedValue *args, int64_t nargs, const FunctionContext
FType<Or<Null, Vertex>>("inDegree", args, nargs);
if (args[0].IsNull()) return TypedValue(ctx.memory);
const auto &vertex = args[0].ValueVertex();
ctx.db_accessor->PrefetchInEdges(vertex);
size_t in_degree = UnwrapDegreeResult(vertex.InDegree(ctx.view));
return TypedValue(static_cast<int64_t>(in_degree), ctx.memory);
}
@@ -489,6 +492,7 @@ TypedValue OutDegree(const TypedValue *args, int64_t nargs, const FunctionContex
FType<Or<Null, Vertex>>("outDegree", args, nargs);
if (args[0].IsNull()) return TypedValue(ctx.memory);
const auto &vertex = args[0].ValueVertex();
ctx.db_accessor->PrefetchOutEdges(vertex);
size_t out_degree = UnwrapDegreeResult(vertex.OutDegree(ctx.view));
return TypedValue(static_cast<int64_t>(out_degree), ctx.memory);
}

View File

@@ -26,6 +26,7 @@
#include <optional>
#include <stdexcept>
#include <thread>
#include <tuple>
#include <unordered_map>
#include <utility>
#include <variant>
@@ -62,11 +63,9 @@
#include "query/procedure/module.hpp"
#include "query/stream.hpp"
#include "query/stream/common.hpp"
#include "query/stream/sources.hpp"
#include "query/stream/streams.hpp"
#include "query/trigger.hpp"
#include "query/typed_value.hpp"
#include "replication/config.hpp"
#include "spdlog/spdlog.h"
#include "storage/v2/disk/storage.hpp"
#include "storage/v2/edge.hpp"
@@ -74,6 +73,7 @@
#include "storage/v2/id_types.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/replication/config.hpp"
#include "storage/v2/storage_error.hpp"
#include "storage/v2/storage_mode.hpp"
#include "utils/algorithm.hpp"
@@ -99,8 +99,6 @@
#include "dbms/dbms_handler.hpp"
#include "query/auth_query_handler.hpp"
#include "query/interpreter_context.hpp"
#include "replication/state.hpp"
#include "storage/v2/replication/replication_handler.hpp"
namespace memgraph::metrics {
extern Event ReadQuery;
@@ -147,6 +145,7 @@ constexpr auto kAlwaysFalse = false;
namespace {
template <typename T, typename K>
void Sort(std::vector<T, K> &vec) {
std::sort(vec.begin(), vec.end());
}
@@ -158,15 +157,13 @@ void Sort(std::vector<TypedValue, K> &vec) {
}
// NOLINTNEXTLINE (misc-unused-parameters)
[[maybe_unused]] bool Same(const TypedValue &lv, const TypedValue &rv) {
bool Same(const TypedValue &lv, const TypedValue &rv) {
return TypedValue(lv).ValueString() == TypedValue(rv).ValueString();
}
// NOLINTNEXTLINE (misc-unused-parameters)
bool Same(const TypedValue &lv, const std::string &rv) { return std::string(TypedValue(lv).ValueString()) == rv; }
// NOLINTNEXTLINE (misc-unused-parameters)
[[maybe_unused]] bool Same(const std::string &lv, const TypedValue &rv) {
return lv == std::string(TypedValue(rv).ValueString());
}
bool Same(const std::string &lv, const TypedValue &rv) { return lv == std::string(TypedValue(rv).ValueString()); }
// NOLINTNEXTLINE (misc-unused-parameters)
bool Same(const std::string &lv, const std::string &rv) { return lv == rv; }
@@ -252,40 +249,24 @@ bool IsAllShortestPathsQuery(const std::vector<memgraph::query::Clause *> &claus
return false;
}
inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode) -> replication::ReplicationMode {
switch (sync_mode) {
case ReplicationQuery::SyncMode::ASYNC: {
return replication::ReplicationMode::ASYNC;
}
case ReplicationQuery::SyncMode::SYNC: {
return replication::ReplicationMode::SYNC;
}
}
// TODO: C++23 std::unreachable()
return replication::ReplicationMode::ASYNC;
}
class ReplQueryHandler final : public query::ReplicationQueryHandler {
public:
explicit ReplQueryHandler(storage::Storage *db) : db_(db), handler_{db_->repl_state_, *db_} {}
explicit ReplQueryHandler(storage::Storage *db) : db_(db) {}
/// @throw QueryRuntimeException if an error ocurred.
void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional<int64_t> port) override {
if (replication_role == ReplicationQuery::ReplicationRole::MAIN) {
if (!handler_.SetReplicationRoleMain()) {
if (!db_->SetMainReplicationRole()) {
throw QueryRuntimeException("Couldn't set role to main!");
}
} else {
if (!port || *port < 0 || *port > std::numeric_limits<uint16_t>::max()) {
throw QueryRuntimeException("Port number invalid!");
}
auto const config = memgraph::replication::ReplicationServerConfig{
.ip_address = memgraph::replication::kDefaultReplicationServerIp,
.port = static_cast<uint16_t>(*port),
};
if (!handler_.SetReplicationRoleReplica(config)) {
if (!db_->SetReplicaRole(storage::replication::ReplicationServerConfig{
.ip_address = storage::replication::kDefaultReplicationServerIp,
.port = static_cast<uint16_t>(*port),
})) {
throw QueryRuntimeException("Couldn't set role to replica!");
}
}
@@ -293,10 +274,10 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
ReplicationQuery::ReplicationRole ShowReplicationRole() const override {
switch (handler_.GetRole()) {
case memgraph::replication::ReplicationRole::MAIN:
switch (db_->GetReplicationRole()) {
case storage::replication::ReplicationRole::MAIN:
return ReplicationQuery::ReplicationRole::MAIN;
case memgraph::replication::ReplicationRole::REPLICA:
case storage::replication::ReplicationRole::REPLICA:
return ReplicationQuery::ReplicationRole::REPLICA;
}
throw QueryRuntimeException("Couldn't show replication role - invalid role set!");
@@ -306,29 +287,39 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
void RegisterReplica(const std::string &name, const std::string &socket_address,
const ReplicationQuery::SyncMode sync_mode,
const std::chrono::seconds replica_check_frequency) override {
if (handler_.IsReplica()) {
if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) {
// replica can't register another replica
throw QueryRuntimeException("Replica can't register another replica!");
}
if (name == memgraph::replication::kReservedReplicationRoleName) {
if (name == storage::replication::kReservedReplicationRoleName) {
throw QueryRuntimeException("This replica name is reserved and can not be used as replica name!");
}
auto repl_mode = convertToReplicationMode(sync_mode);
storage::replication::ReplicationMode repl_mode;
switch (sync_mode) {
case ReplicationQuery::SyncMode::ASYNC: {
repl_mode = storage::replication::ReplicationMode::ASYNC;
break;
}
case ReplicationQuery::SyncMode::SYNC: {
repl_mode = storage::replication::ReplicationMode::SYNC;
break;
}
}
auto maybe_ip_and_port =
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, memgraph::replication::kDefaultReplicationPort);
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, storage::replication::kDefaultReplicationPort);
if (maybe_ip_and_port) {
auto [ip, port] = *maybe_ip_and_port;
auto config = replication::ReplicationClientConfig{.name = name,
.mode = repl_mode,
.ip_address = ip,
.port = port,
.replica_check_frequency = replica_check_frequency,
.ssl = std::nullopt};
using storage::RegistrationMode;
auto ret = handler_.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, config);
auto ret = db_->RegisterReplica(
storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID,
storage::replication::ReplicationClientConfig{.name = name,
.mode = repl_mode,
.ip_address = ip,
.port = port,
.replica_check_frequency = replica_check_frequency,
.ssl = std::nullopt});
if (ret.HasError()) {
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
}
@@ -337,26 +328,20 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
}
}
/// @throw QueryRuntimeException if an error occurred.
void DropReplica(std::string_view replica_name) override {
auto const result = handler_.UnregisterReplica(replica_name);
switch (result) {
using enum memgraph::storage::UnregisterReplicaResult;
case NOT_MAIN:
throw QueryRuntimeException("Replica can't unregister a replica!");
case COULD_NOT_BE_PERSISTED:
[[fallthrough]];
case CAN_NOT_UNREGISTER:
throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name));
case SUCCESS:
break;
/// @throw QueryRuntimeException if an error ocurred.
void DropReplica(const std::string &replica_name) override {
if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) {
// replica can't unregister a replica
throw QueryRuntimeException("Replica can't unregister a replica!");
}
if (!db_->UnregisterReplica(replica_name)) {
throw QueryRuntimeException(fmt::format("Couldn't unregister the replica '{}'", replica_name));
}
}
using Replica = ReplicationQueryHandler::Replica;
std::vector<Replica> ShowReplicas() const override {
auto const &replState = db_->repl_state_;
if (replState.IsReplica()) {
if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) {
// replica can't show registered replicas (it shouldn't have any)
throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!");
}
@@ -370,10 +355,10 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
replica.name = repl_info.name;
replica.socket_address = repl_info.endpoint.SocketAddress();
switch (repl_info.mode) {
case memgraph::replication::ReplicationMode::SYNC:
case storage::replication::ReplicationMode::SYNC:
replica.sync_mode = ReplicationQuery::SyncMode::SYNC;
break;
case memgraph::replication::ReplicationMode::ASYNC:
case storage::replication::ReplicationMode::ASYNC:
replica.sync_mode = ReplicationQuery::SyncMode::ASYNC;
break;
}
@@ -406,7 +391,6 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
private:
storage::Storage *db_;
storage::ReplicationHandler handler_;
};
/// returns false if the replication role can't be set
@@ -1282,6 +1266,29 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary) {
std::optional<uint64_t> transaction_id = ctx_.db_accessor->GetTransactionId();
MG_ASSERT(transaction_id.has_value());
unsigned arena_ind{0};
if (memory_limit_) {
// TODO (AF) think to isolate this in namespace or make a class
memgraph::memory::transaction_id_tracker.emplace(std::piecewise_construct, std::forward_as_tuple(*transaction_id),
std::forward_as_tuple());
auto &memory_tracker = memgraph::memory::transaction_id_tracker[*transaction_id];
memory_tracker.SetMaximumHardLimit(static_cast<int64_t>(*memory_limit_));
memory_tracker.SetHardLimit(static_cast<int64_t>(*memory_limit_));
arena_ind = memgraph::memory::GetArenaForThread();
memgraph::memory::AddTrackingOnArena(arena_ind);
memgraph::memory::UpdateThreadToTransactionId(std::this_thread::get_id(), *transaction_id);
}
utils::OnScopeExit<std::function<void()>> reset_query_limit{
[memory_limit = memory_limit_, transaction_id = *transaction_id, arena_ind]() {
if (memory_limit) {
// TODO (AF) think to isolate this in namespace or make a class
memgraph::memory::transaction_id_tracker.erase(transaction_id);
memgraph::memory::RemoveTrackingOnArena(arena_ind);
memgraph::memory::ResetThreadToTransactionId(std::this_thread::get_id());
}
}};
// Set up temporary memory for a single Pull. Initial memory comes from the
// stack. 256 KiB should fit on the stack and should be more than enough for a
// single `Pull`.
@@ -1305,13 +1312,7 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
pool_memory.emplace(kMaxBlockPerChunks, 1024, &monotonic_memory, &resource_with_exception);
}
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
if (memory_limit_) {
maybe_limited_resource.emplace(&*pool_memory, *memory_limit_);
ctx_.evaluation_context.memory = &*maybe_limited_resource;
} else {
ctx_.evaluation_context.memory = &*pool_memory;
}
ctx_.evaluation_context.memory = &*pool_memory;
// Returns true if a result was pulled.
const auto pull_result = [&]() -> bool { return cursor_->Pull(frame_, ctx_); };
@@ -1378,6 +1379,7 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
}
cursor_->Shutdown();
ctx_.profile_execution_time = execution_time_;
return GetStatsWithTotalTime(ctx_);
}
@@ -1387,19 +1389,18 @@ bool IsWriteQueryOnMainMemoryReplica(storage::Storage *storage,
const query::plan::ReadWriteTypeChecker::RWType query_type) {
if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL ||
storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
auto const &replState = storage->repl_state_;
return replState.IsReplica() && (query_type == RWType::W || query_type == RWType::RW);
return (storage->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) &&
(query_type == RWType::W || query_type == RWType::RW);
}
return false;
}
bool IsReplica(storage::Storage *storage) {
storage::replication::ReplicationRole GetReplicaRole(storage::Storage *storage) {
if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL ||
storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
auto const &replState = storage->repl_state_;
return replState.IsReplica();
return storage->GetReplicationRole();
}
return false;
return storage::replication::ReplicationRole::MAIN;
}
} // namespace
@@ -2642,14 +2643,14 @@ PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in
}
}
return PreparedQuery{{},
std::move(parsed_query.required_privileges),
[callback = std::move(callback)](AnyStream * /*stream*/,
std::optional<int> /*n*/) -> std::optional<QueryHandlerResult> {
callback();
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
return PreparedQuery{
{},
std::move(parsed_query.required_privileges),
[callback = std::move(callback)](AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
callback();
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
}
Callback SwitchMemoryDevice(storage::StorageMode current_mode, storage::StorageMode requested_mode,
@@ -2680,7 +2681,7 @@ Callback SwitchMemoryDevice(storage::StorageMode current_mode, storage::StorageM
}
std::unique_lock main_guard{in.storage()->main_lock_}; // do we need this?
if (auto vertex_cnt_approx = in.storage()->GetBaseInfo().vertex_count; vertex_cnt_approx > 0) {
if (auto vertex_cnt_approx = in.storage()->GetInfo().vertex_count; vertex_cnt_approx > 0) {
throw utils::BasicException(
"You cannot switch from an in-memory storage mode to the on-disk storage mode when the database "
"contains data. Delete all entries from the database, run FREE MEMORY and then repeat this "
@@ -2802,7 +2803,7 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
std::move(parsed_query.required_privileges),
[storage](AnyStream * /*stream*/, std::optional<int> /*n*/) -> std::optional<QueryHandlerResult> {
auto *mem_storage = static_cast<storage::InMemoryStorage *>(storage);
if (auto maybe_error = mem_storage->CreateSnapshot(storage->repl_state_, {}); maybe_error.HasError()) {
if (auto maybe_error = mem_storage->CreateSnapshot({}); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica:
throw utils::BasicException(
@@ -2996,7 +2997,6 @@ PreparedQuery PrepareDatabaseInfoQuery(ParsedQuery parsed_query, bool in_explici
results.push_back({TypedValue(label_property_index_mark), TypedValue(storage->LabelToName(item.first)),
TypedValue(storage->PropertyToName(item.second))});
}
std::sort(results.begin(), results.end(), [&label_index_mark](const auto &record_1, const auto &record_2) {
const auto type_1 = record_1[0].ValueString();
const auto type_2 = record_2[0].ValueString();
@@ -3078,18 +3078,18 @@ PreparedQuery PrepareSystemInfoQuery(ParsedQuery parsed_query, bool in_explicit_
header = {"storage info", "value"};
handler = [storage = current_db.db_acc_->get()->storage(), interpreter_isolation_level,
next_transaction_isolation_level] {
auto info = storage->GetBaseInfo();
auto info = storage->GetInfo();
std::vector<std::vector<TypedValue>> results{
{TypedValue("name"), TypedValue(storage->id())},
{TypedValue("vertex_count"), TypedValue(static_cast<int64_t>(info.vertex_count))},
{TypedValue("edge_count"), TypedValue(static_cast<int64_t>(info.edge_count))},
{TypedValue("average_degree"), TypedValue(info.average_degree)},
{TypedValue("memory_usage"), TypedValue(utils::GetReadableSize(static_cast<double>(info.memory_usage)))},
{TypedValue("disk_usage"), TypedValue(utils::GetReadableSize(static_cast<double>(info.disk_usage)))},
{TypedValue("memory_allocated"),
{TypedValue("memory_usage"), TypedValue(static_cast<int64_t>(info.memory_usage))},
{TypedValue("disk_usage"), TypedValue(static_cast<int64_t>(info.disk_usage))},
{TypedValue("readable_memory_allocated"),
TypedValue(utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.Amount())))},
{TypedValue("allocation_limit"),
TypedValue(utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.HardLimit())))},
{TypedValue("memory_allocated"), TypedValue(static_cast<int64_t>(utils::total_memory_tracker.Amount()))},
{TypedValue("allocation_limit"), TypedValue(static_cast<int64_t>(utils::total_memory_tracker.HardLimit()))},
{TypedValue("global_isolation_level"), TypedValue(IsolationLevelToString(storage->GetIsolationLevel()))},
{TypedValue("session_isolation_level"), TypedValue(IsolationLevelToString(interpreter_isolation_level))},
{TypedValue("next_session_isolation_level"),
@@ -3117,7 +3117,6 @@ PreparedQuery PrepareSystemInfoQuery(ParsedQuery parsed_query, bool in_explicit_
action = action_on_complete;
pull_plan = std::make_shared<PullPlanVector>(std::move(results));
}
if (pull_plan->Pull(stream, n)) {
return action;
}
@@ -3348,7 +3347,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
}
// TODO: Remove once replicas support multi-tenant replication
if (!current_db.db_acc_) throw DatabaseContextRequiredException("Multi database queries require a defined database.");
if (IsReplica(current_db.db_acc_->get()->storage())) {
if (GetReplicaRole(current_db.db_acc_->get()->storage()) == storage::replication::ReplicationRole::REPLICA) {
throw QueryException("Query forbidden on the replica!");
}
@@ -3491,8 +3490,7 @@ PreparedQuery PrepareShowDatabasesQuery(ParsedQuery parsed_query, CurrentDB &cur
throw QueryException("Trying to use enterprise feature without a valid license.");
}
// TODO: Remove once replicas support multi-tenant replication
auto &replState = storage->repl_state_;
if (replState.IsReplica()) {
if (GetReplicaRole(storage) == storage::replication::ReplicationRole::REPLICA) {
throw QueryException("SHOW DATABASES forbidden on the replica!");
}
@@ -3837,10 +3835,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid,
query_execution->prepared_query->db};
} catch (const utils::BasicException &) {
// Trigger first failed query
metrics::FirstFailedQuery();
memgraph::metrics::IncrementCounter(memgraph::metrics::FailedQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::FailedPrepare);
AbortCommand(query_execution_ptr);
throw;
}
@@ -3866,12 +3861,10 @@ std::vector<TypedValue> Interpreter::GetQueries() {
}
void Interpreter::Abort() {
bool decrement = true;
auto expected = TransactionStatus::ACTIVE;
while (!transaction_status_.compare_exchange_weak(expected, TransactionStatus::STARTED_ROLLBACK)) {
if (expected == TransactionStatus::TERMINATED || expected == TransactionStatus::IDLE) {
transaction_status_.store(TransactionStatus::STARTED_ROLLBACK);
decrement = false;
break;
}
expected = TransactionStatus::ACTIVE;
@@ -3887,10 +3880,7 @@ void Interpreter::Abort() {
current_timeout_timer_.reset();
current_transaction_.reset();
if (decrement) {
// Decrement only if the transaction was active when we started to Abort
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTransactions);
}
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTransactions);
// if (!current_db_.db_transactional_accessor_) return;
current_db_.CleanupDBTransaction(true);

View File

@@ -39,7 +39,6 @@
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/storage.hpp"
#include "utils/event_counter.hpp"
#include "utils/event_trigger.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/settings.hpp"
@@ -52,9 +51,6 @@
namespace memgraph::metrics {
extern const Event FailedQuery;
extern const Event FailedPrepare;
extern const Event FailedPull;
extern const Event SuccessfulQuery;
} // namespace memgraph::metrics
namespace memgraph::query {
@@ -99,7 +95,7 @@ class ReplicationQueryHandler {
const std::chrono::seconds replica_check_frequency) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void DropReplica(std::string_view replica_name) = 0;
virtual void DropReplica(const std::string &replica_name) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual std::vector<Replica> ShowReplicas() const = 0;
@@ -443,18 +439,12 @@ std::map<std::string, TypedValue> Interpreter::Pull(TStream *result_stream, std:
query_execution.reset(nullptr);
throw;
} catch (const utils::BasicException &) {
// Trigger first failed query
metrics::FirstFailedQuery();
memgraph::metrics::IncrementCounter(memgraph::metrics::FailedQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::FailedPull);
AbortCommand(&query_execution);
throw;
}
if (maybe_summary) {
// Toggle first successfully completed query
metrics::FirstSuccessfulQuery();
memgraph::metrics::IncrementCounter(memgraph::metrics::SuccessfulQuery);
// return the execution summary
maybe_summary->insert_or_assign("has_more", false);
return std::move(*maybe_summary);

View File

@@ -891,12 +891,15 @@ bool Expand::ExpandCursor::InitEdges(Frame &frame, ExecutionContext &context) {
if (self_.common_.existing_node) {
if (expansion_info_.existing_node) {
auto existing_node = *expansion_info_.existing_node;
context.db_accessor->PrefetchInEdges(vertex);
auto edges_result = UnwrapEdgesResult(vertex.InEdges(self_.view_, self_.common_.edge_types, existing_node));
in_edges_.emplace(edges_result.edges);
num_expanded_first = edges_result.expanded_count;
}
} else {
context.db_accessor->PrefetchInEdges(vertex);
auto edges_result = UnwrapEdgesResult(vertex.InEdges(self_.view_, self_.common_.edge_types));
in_edges_.emplace(edges_result.edges);
num_expanded_first = edges_result.expanded_count;
@@ -911,11 +914,15 @@ bool Expand::ExpandCursor::InitEdges(Frame &frame, ExecutionContext &context) {
if (self_.common_.existing_node) {
if (expansion_info_.existing_node) {
auto existing_node = *expansion_info_.existing_node;
context.db_accessor->PrefetchOutEdges(vertex);
auto edges_result = UnwrapEdgesResult(vertex.OutEdges(self_.view_, self_.common_.edge_types, existing_node));
out_edges_.emplace(edges_result.edges);
num_expanded_second = edges_result.expanded_count;
}
} else {
context.db_accessor->PrefetchOutEdges(vertex);
auto edges_result = UnwrapEdgesResult(vertex.OutEdges(self_.view_, self_.common_.edge_types));
out_edges_.emplace(edges_result.edges);
num_expanded_second = edges_result.expanded_count;
@@ -1002,6 +1009,7 @@ auto ExpandFromVertex(const VertexAccessor &vertex, EdgeAtom::Direction directio
memory);
if (direction != EdgeAtom::Direction::OUT) {
db_accessor->PrefetchInEdges(vertex);
auto edges = UnwrapEdgesResult(vertex.InEdges(view, edge_types)).edges;
if (edges.begin() != edges.end()) {
chain_elements.emplace_back(wrapper(EdgeAtom::Direction::IN, std::move(edges)));
@@ -1009,6 +1017,7 @@ auto ExpandFromVertex(const VertexAccessor &vertex, EdgeAtom::Direction directio
}
if (direction != EdgeAtom::Direction::IN) {
db_accessor->PrefetchOutEdges(vertex);
auto edges = UnwrapEdgesResult(vertex.OutEdges(view, edge_types)).edges;
if (edges.begin() != edges.end()) {
chain_elements.emplace_back(wrapper(EdgeAtom::Direction::OUT, std::move(edges)));
@@ -1368,6 +1377,7 @@ class STShortestPathCursor : public query::plan::Cursor {
for (const auto &vertex : source_frontier) {
if (self_.common_.direction != EdgeAtom::Direction::IN) {
context.db_accessor->PrefetchOutEdges(vertex);
auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : out_edges) {
#ifdef MG_ENTERPRISE
@@ -1394,6 +1404,7 @@ class STShortestPathCursor : public query::plan::Cursor {
}
}
if (self_.common_.direction != EdgeAtom::Direction::OUT) {
dba.PrefetchInEdges(vertex);
auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : in_edges) {
#ifdef MG_ENTERPRISE
@@ -1434,6 +1445,7 @@ class STShortestPathCursor : public query::plan::Cursor {
// reversed.
for (const auto &vertex : sink_frontier) {
if (self_.common_.direction != EdgeAtom::Direction::OUT) {
context.db_accessor->PrefetchOutEdges(vertex);
auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : out_edges) {
#ifdef MG_ENTERPRISE
@@ -1459,6 +1471,7 @@ class STShortestPathCursor : public query::plan::Cursor {
}
}
if (self_.common_.direction != EdgeAtom::Direction::IN) {
dba.PrefetchInEdges(vertex);
auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : in_edges) {
#ifdef MG_ENTERPRISE
@@ -1548,12 +1561,14 @@ class SingleSourceShortestPathCursor : public query::plan::Cursor {
// populates the to_visit_next_ structure with expansions
// from the given vertex. skips expansions that don't satisfy
// the "where" condition.
auto expand_from_vertex = [this, &expand_pair](const auto &vertex) {
auto expand_from_vertex = [this, &expand_pair, &context](const auto &vertex) {
if (self_.common_.direction != EdgeAtom::Direction::IN) {
context.db_accessor->PrefetchOutEdges(vertex);
auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : out_edges) expand_pair(edge, edge.To());
}
if (self_.common_.direction != EdgeAtom::Direction::OUT) {
context.db_accessor->PrefetchInEdges(vertex);
auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : in_edges) expand_pair(edge, edge.From());
}
@@ -1748,15 +1763,17 @@ class ExpandWeightedShortestPathCursor : public query::plan::Cursor {
// Populates the priority queue structure with expansions
// from the given vertex. skips expansions that don't satisfy
// the "where" condition.
auto expand_from_vertex = [this, &expand_pair](const VertexAccessor &vertex, const TypedValue &weight,
int64_t depth) {
auto expand_from_vertex = [this, &expand_pair, &context](const VertexAccessor &vertex, const TypedValue &weight,
int64_t depth) {
if (self_.common_.direction != EdgeAtom::Direction::IN) {
context.db_accessor->PrefetchOutEdges(vertex);
auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : out_edges) {
expand_pair(edge, edge.To(), weight, depth);
}
}
if (self_.common_.direction != EdgeAtom::Direction::OUT) {
context.db_accessor->PrefetchInEdges(vertex);
auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : in_edges) {
expand_pair(edge, edge.From(), weight, depth);
@@ -2015,6 +2032,7 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor {
auto expand_from_vertex = [this, &expand_vertex, &context](const VertexAccessor &vertex, const TypedValue &weight,
int64_t depth) {
if (self_.common_.direction != EdgeAtom::Direction::IN) {
context.db_accessor->PrefetchOutEdges(vertex);
auto out_edges = UnwrapEdgesResult(vertex.OutEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : out_edges) {
#ifdef MG_ENTERPRISE
@@ -2029,6 +2047,7 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor {
}
}
if (self_.common_.direction != EdgeAtom::Direction::OUT) {
context.db_accessor->PrefetchInEdges(vertex);
auto in_edges = UnwrapEdgesResult(vertex.InEdges(storage::View::OLD, self_.common_.edge_types)).edges;
for (const auto &edge : in_edges) {
#ifdef MG_ENTERPRISE

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -488,10 +488,6 @@ class RuleBasedPlanner {
last_op = std::make_unique<ScanAll>(std::move(last_op), node1_symbol, view);
new_symbols.emplace_back(node1_symbol);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
} else if (named_paths.size() == 1U) {
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);
last_op = impl::GenNamedPaths(std::move(last_op), bound_symbols, named_paths);
last_op = GenFilters(std::move(last_op), bound_symbols, filters, storage, symbol_table);

View File

@@ -104,37 +104,30 @@ void MgpFreeImpl(memgraph::utils::MemoryResource &memory, void *const p) noexcep
}
struct DeletedObjectException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(DeletedObjectException)
};
struct KeyAlreadyExistsException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(KeyAlreadyExistsException)
};
struct InsufficientBufferException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(InsufficientBufferException)
};
struct ImmutableObjectException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(ImmutableObjectException)
};
struct ValueConversionException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(ValueConversionException)
};
struct SerializationException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(SerializationException)
};
struct AuthorizationException : public memgraph::utils::BasicException {
using memgraph::utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(AuthorizationException)
};
template <typename TFunc, typename TReturn>
@@ -1947,7 +1940,7 @@ void mgp_vertex_destroy(mgp_vertex *v) { DeleteRawMgpObject(v); }
mgp_error mgp_vertex_equal(mgp_vertex *v1, mgp_vertex *v2, int *result) {
// NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression)
static_assert(noexcept(*v1 == *v2));
static_assert(noexcept(*result = *v1 == *v2 ? 1 : 0));
*result = *v1 == *v2 ? 1 : 0;
return mgp_error::MGP_ERROR_NO_ERROR;
}
@@ -2121,6 +2114,14 @@ void NextPermittedEdge(mgp_edges_iterator &it, const bool for_in) {
mgp_error mgp_vertex_iter_in_edges(mgp_vertex *v, mgp_memory *memory, mgp_edges_iterator **result) {
return WrapExceptions(
[v, memory] {
auto dbAccessor = v->graph->impl;
if (std::holds_alternative<memgraph::query::DbAccessor *>(dbAccessor)) {
std::get<memgraph::query::DbAccessor *>(dbAccessor)
->PrefetchInEdges(std::get<memgraph::query::VertexAccessor>(v->impl));
} else {
std::get<memgraph::query::SubgraphDbAccessor *>(dbAccessor)
->PrefetchInEdges(std::get<memgraph::query::SubgraphVertexAccessor>(v->impl));
}
auto it = NewMgpObject<mgp_edges_iterator>(memory, *v);
MG_ASSERT(it != nullptr);
@@ -2172,6 +2173,14 @@ mgp_error mgp_vertex_iter_in_edges(mgp_vertex *v, mgp_memory *memory, mgp_edges_
mgp_error mgp_vertex_iter_out_edges(mgp_vertex *v, mgp_memory *memory, mgp_edges_iterator **result) {
return WrapExceptions(
[v, memory] {
auto dbAccessor = v->graph->impl;
if (std::holds_alternative<memgraph::query::DbAccessor *>(dbAccessor)) {
std::get<memgraph::query::DbAccessor *>(dbAccessor)
->PrefetchOutEdges(std::get<memgraph::query::VertexAccessor>(v->impl));
} else {
std::get<memgraph::query::SubgraphDbAccessor *>(dbAccessor)
->PrefetchOutEdges(std::get<memgraph::query::SubgraphVertexAccessor>(v->impl));
}
auto it = NewMgpObject<mgp_edges_iterator>(memory, *v);
MG_ASSERT(it != nullptr);
auto maybe_edges = std::visit([v](auto &impl) { return impl.OutEdges(v->graph->view); }, v->impl);
@@ -2304,7 +2313,7 @@ void mgp_edge_destroy(mgp_edge *e) { DeleteRawMgpObject(e); }
mgp_error mgp_edge_equal(mgp_edge *e1, mgp_edge *e2, int *result) {
// NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression)
static_assert(noexcept(*e1 == *e2));
static_assert(noexcept(*result = *e1 == *e2 ? 1 : 0));
*result = *e1 == *e2 ? 1 : 0;
return mgp_error::MGP_ERROR_NO_ERROR;
}
@@ -3531,3 +3540,28 @@ mgp_error mgp_log(const mgp_log_level log_level, const char *output) {
throw std::invalid_argument{fmt::format("Invalid log level: {}", log_level)};
});
}
mgp_error mgp_track_thread_allocations(mgp_graph *graph, const char *thread_id) {
return WrapExceptions([&]() {
std::visit([thread_id](auto *db_accessor) -> void { db_accessor->TrackThreadAllocations(thread_id); }, graph->impl);
});
}
mgp_error mgp_track_current_thread_allocations(mgp_graph *graph) {
return WrapExceptions([&]() {
std::visit([](auto *db_accessor) -> void { db_accessor->TrackCurrentThreadAllocations(); }, graph->impl);
});
}
mgp_error mgp_untrack_thread_allocations(mgp_graph *graph, const char *thread_id) {
return WrapExceptions([&]() {
std::visit([thread_id](auto *db_accessor) -> void { db_accessor->UntrackThreadAllocations(thread_id); },
graph->impl);
});
}
mgp_error mgp_untrack_current_thread_allocations(mgp_graph *graph) {
return WrapExceptions([&]() {
std::visit([](auto *db_accessor) -> void { db_accessor->UntrackCurrentThreadAllocations(); }, graph->impl);
});
}

View File

@@ -41,7 +41,6 @@ namespace stream {
class StreamsException : public utils::BasicException {
public:
using BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(StreamsException)
};
template <typename T>

View File

@@ -302,6 +302,7 @@ void TriggerContext::AdaptForAccessor(DbAccessor *accessor) {
if (!maybe_from_vertex) {
continue;
}
accessor->PrefetchOutEdges(*maybe_from_vertex);
auto maybe_out_edges = maybe_from_vertex->OutEdges(storage::View::OLD);
MG_ASSERT(maybe_out_edges.HasValue());
const auto edge_gid = created_edge.object.Gid();
@@ -323,6 +324,7 @@ void TriggerContext::AdaptForAccessor(DbAccessor *accessor) {
auto it = values->begin();
for (const auto &value : *values) {
if (auto maybe_vertex = accessor->FindVertex(value.object.From().Gid(), storage::View::OLD); maybe_vertex) {
accessor->PrefetchOutEdges(*maybe_vertex);
auto maybe_out_edges = maybe_vertex->OutEdges(storage::View::OLD);
MG_ASSERT(maybe_out_edges.HasValue());
for (const auto &edge : maybe_out_edges->edges) {

View File

@@ -566,7 +566,6 @@ class TypedValue {
class TypedValueException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
SPECIALIZE_GET_EXCEPTION_NAME(TypedValueException)
};
// binary bool operators

View File

@@ -1,24 +0,0 @@
add_library(mg-replication STATIC)
add_library(mg::replication ALIAS mg-replication)
target_sources(mg-replication
PUBLIC
include/replication/state.hpp
include/replication/epoch.hpp
include/replication/config.hpp
include/replication/mode.hpp
include/replication/role.hpp
include/replication/status.hpp
PRIVATE
state.cpp
epoch.cpp
config.cpp
status.cpp
)
target_include_directories(mg-replication PUBLIC include)
find_package(fmt REQUIRED)
target_link_libraries(mg-replication
PUBLIC mg::utils mg::kvstore lib::json
PRIVATE fmt::fmt
)

View File

@@ -1,11 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "replication/config.hpp"

View File

@@ -1,12 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "replication/epoch.hpp"

View File

@@ -1,49 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <string>
#include <utility>
#include "utils/uuid.hpp"
namespace memgraph::replication {
struct ReplicationEpoch {
ReplicationEpoch() : id_(memgraph::utils::GenerateUUID()) {}
ReplicationEpoch(ReplicationEpoch const &) = delete;
ReplicationEpoch(ReplicationEpoch &&) = delete;
ReplicationEpoch &operator=(ReplicationEpoch const &) = delete;
ReplicationEpoch &operator=(ReplicationEpoch &&) = delete;
auto id() const -> std::string_view { return id_; }
auto NewEpoch() -> std::string { return std::exchange(id_, memgraph::utils::GenerateUUID()); }
auto SetEpoch(std::string new_epoch) -> std::string { return std::exchange(id_, std::move(new_epoch)); }
private:
// UUID to distinguish different main instance runs for replication process
// on SAME storage.
// Multiple instances can have same storage UUID and be MAIN at the same time.
// We cannot compare commit timestamps of those instances if one of them
// becomes the replica of the other so we use epoch_id_ as additional
// discriminating property.
// Example of this:
// We have 2 instances of the same storage, S1 and S2.
// S1 and S2 are MAIN and accept their own commits and write them to the WAL.
// At the moment when S1 commited a transaction with timestamp 20, and S2
// a different transaction with timestamp 15, we change S2's role to REPLICA
// and register it on S1.
// Without using the epoch_id, we don't know that S1 and S2 have completely
// different transactions, we think that the S2 is behind only by 5 commits.
std::string id_;
};
} // namespace memgraph::replication

Some files were not shown because too many files have changed in this diff Show More