Compare commits

..

2 Commits

Author SHA1 Message Date
Kostas
ebe54d759e Fix websocket headers 2022-01-24 12:07:35 +02:00
Antonio Andelic
492a13d5e7 Bolt over websocket initial 2022-01-03 16:39:44 +01:00
156 changed files with 1280 additions and 5491 deletions

View File

@@ -1,9 +1,7 @@
---
Checks: '*,
-abseil-string-find-str-contains,
-altera-id-dependent-backward-branch,
-altera-struct-pack-align,
-altera-unroll-loops,
-android-*,
-cert-err58-cpp,
-cppcoreguidelines-avoid-c-arrays,

View File

@@ -32,7 +32,7 @@ for file in $modified_files; do
fi
echo "Running header checker..."
$project_folder/tools/header-checker.py $tmpdir/$file $file --amend-year
$project_folder/tools/header-checker.py $tmpdir/$file
code=$?
if [ $code -ne 0 ]; then

View File

@@ -25,7 +25,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init

View File

@@ -32,7 +32,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -45,7 +45,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -70,7 +70,7 @@ jobs:
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -82,7 +82,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests. It is restricted to 2 threads intentionally, because higher concurrency makes the timing related tests unstable.
cd build
@@ -91,7 +91,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Compute code coverage.
cd tools/github
@@ -109,7 +109,7 @@ jobs:
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
@@ -136,7 +136,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -149,7 +149,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -181,7 +181,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -212,7 +212,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -238,7 +238,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -272,7 +272,7 @@ jobs:
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
cd build
@@ -319,7 +319,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -360,7 +360,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init

View File

@@ -23,7 +23,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -36,10 +36,10 @@ jobs:
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# The results are also written to standard output in order to retain them in the logs
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v4/bin/clang-tidy "$PWD/src/*" |
./tools/github/clang-tidy/run-clang-tidy.py -p build -j $THREADS -clang-tidy-binary=/opt/toolchain-v3/bin/clang-tidy "$PWD/src/*" |
tee ./build/full_clang_tidy_output.txt
- name: Summarize clang-tidy results

View File

@@ -39,6 +39,23 @@ jobs:
name: centos-8
path: build/output/centos-8/memgraph*.rpm
debian-9:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-9
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-9
path: build/output/debian-9/memgraph*.deb
debian-10:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60

View File

@@ -26,7 +26,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -39,7 +39,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -64,7 +64,7 @@ jobs:
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -77,7 +77,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -86,7 +86,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Compute code coverage.
cd tools/github
@@ -121,7 +121,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -134,7 +134,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -166,7 +166,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -198,7 +198,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -211,7 +211,7 @@ jobs:
- name: Create enterprise RPM package
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
cd build
@@ -233,7 +233,7 @@ jobs:
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run micro benchmark tests.
cd build
@@ -272,7 +272,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build

View File

@@ -26,7 +26,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -39,7 +39,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -64,7 +64,7 @@ jobs:
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -77,7 +77,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -86,7 +86,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Compute code coverage.
cd tools/github
@@ -121,7 +121,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -134,7 +134,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -166,7 +166,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -198,7 +198,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -211,7 +211,7 @@ jobs:
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
cd build
@@ -232,7 +232,7 @@ jobs:
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run micro benchmark tests.
cd build
@@ -271,7 +271,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -333,7 +333,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init

View File

@@ -26,7 +26,7 @@ jobs:
- name: Build community binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -39,7 +39,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -64,7 +64,7 @@ jobs:
- name: Build coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -77,7 +77,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build
@@ -86,7 +86,7 @@ jobs:
- name: Compute code coverage
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Compute code coverage.
cd tools/github
@@ -121,7 +121,7 @@ jobs:
- name: Build debug binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -134,7 +134,7 @@ jobs:
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
@@ -166,7 +166,7 @@ jobs:
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run cppcheck and clang-format.
cd tools/github
@@ -198,7 +198,7 @@ jobs:
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Initialize dependencies.
./init
@@ -211,7 +211,7 @@ jobs:
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
cd build
@@ -232,7 +232,7 @@ jobs:
- name: Run micro benchmark tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run micro benchmark tests.
cd build
@@ -271,7 +271,7 @@ jobs:
- name: Run unit tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
source /opt/toolchain-v3/activate
# Run unit tests.
cd build

View File

@@ -204,8 +204,6 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=gold")
# release flags
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
#debug flags
set(PREFERRED_DEBUGGER "gdb" CACHE STRING
"Tunes the debug output for your preferred debugger (gdb or lldb).")
@@ -230,8 +228,6 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
# setup external dependencies -------------------------------------------------
# threading
@@ -326,8 +322,7 @@ if (UBSAN)
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
endif()
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact Python version used by the query modules")
set(MG_PYTHON_PATH "" CACHE STRING "Specify the exact Python path used by the query modules")
set(MG_PYTHON_VERSION "" CACHE STRING "Specify the exact python version used by the query modules")
# Add subprojects
include_directories(src)

View File

@@ -1,55 +0,0 @@
# Try to find jemalloc library
#
# Use this module as:
# find_package(Jemalloc)
#
# or:
# find_package(Jemalloc REQUIRED)
#
# This will define the following variables:
#
# Jemalloc_FOUND True if the system has the jemalloc library.
# Jemalloc_INCLUDE_DIRS Include directories needed to use jemalloc.
# Jemalloc_LIBRARIES Libraries needed to link to jemalloc.
#
# The following cache variables may also be set:
#
# Jemalloc_INCLUDE_DIR The directory containing jemalloc/jemalloc.h.
# Jemalloc_LIBRARY The path to the jemalloc static library.
find_path(Jemalloc_INCLUDE_DIR NAMES jemalloc/jemalloc.h PATH_SUFFIXES include)
find_library(Jemalloc_LIBRARY NAMES libjemalloc.a PATH_SUFFIXES lib)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Jemalloc
FOUND_VAR Jemalloc_FOUND
REQUIRED_VARS
Jemalloc_LIBRARY
Jemalloc_INCLUDE_DIR
)
if(Jemalloc_FOUND)
set(Jemalloc_LIBRARIES ${Jemalloc_LIBRARY})
set(Jemalloc_INCLUDE_DIRS ${Jemalloc_INCLUDE_DIR})
else()
if(Jemalloc_FIND_REQUIRED)
message(FATAL_ERROR "Cannot find jemalloc!")
else()
message(WARNING "jemalloc is not found!")
endif()
endif()
if(Jemalloc_FOUND AND NOT TARGET Jemalloc::Jemalloc)
add_library(Jemalloc::Jemalloc UNKNOWN IMPORTED)
set_target_properties(Jemalloc::Jemalloc
PROPERTIES
IMPORTED_LOCATION "${Jemalloc_LIBRARY}"
INTERFACE_INCLUDE_DIRECTORIES "${Jemalloc_INCLUDE_DIR}"
)
endif()
mark_as_advanced(
Jemalloc_INCLUDE_DIR
Jemalloc_LIBRARY
)

View File

@@ -11,18 +11,12 @@ TOOLCHAIN_BUILD_DEPS=(
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python3-devel # gdb
expat-devel libipt-devel libbabeltrace-devel xz-devel python3-devel # gdb
texinfo # gdb
libcurl-devel # cmake
curl # snappy
readline-devel # cmake and llvm
libffi-devel libxml2-devel perl-Digest-MD5 # llvm
libedit-devel pcre-devel automake bison # swig
file
openssl-devel
gmp-devel
gperf
patch
)
TOOLCHAIN_RUN_DEPS=(
@@ -32,10 +26,10 @@ TOOLCHAIN_RUN_DEPS=(
expat libipt libbabeltrace xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconfig # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
@@ -101,12 +95,8 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
yum install -y epel-release
yum remove -y ius-release
yum install -y \
https://repo.ius.io/ius-release-el7.rpm
yum update -y
yum install -y wget python3 python3-pip
yum install -y git224
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then

View File

@@ -6,22 +6,16 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
expat-devel libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
patch
)
TOOLCHAIN_RUN_DEPS=(
@@ -31,7 +25,6 @@ TOOLCHAIN_RUN_DEPS=(
expat libipt libbabeltrace xz-libs python36 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
@@ -117,10 +110,9 @@ install() {
# https://centos.pkgs.org
# Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
# within GDB yet (an assumption).
# http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working
if [ "$pkg" == libbabeltrace-devel ]; then
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
dnf install -y http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
fi
continue
fi

View File

@@ -1,139 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it manually using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "python3-virtualenv" ]; then
continue
fi
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
pushd /tmp/sbcl-1.4.2-arm64-linux
INSTALL_ROOT=/usr/local sh install.sh
popd
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
if [ "$pkg" == python3-virtualenv ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user virtualenv
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
fi
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -15,12 +15,6 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file # for libunwind
libssl-dev # for libevent
libgmp-dev # for gdb
gperf # for proxygen
git # for fbthrift
libedit-dev libpcre3-dev automake bison # for swig
)
@@ -32,7 +26,6 @@ TOOLCHAIN_RUN_DEPS=(
libcurl4 # for cmake
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(

View File

@@ -16,12 +16,6 @@ TOOLCHAIN_BUILD_DEPS=(
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
curl # snappy
file # for libunwind
libssl-dev # for libevent
libgmp-dev
gperf # for proxygen
git # for fbthrift
)
TOOLCHAIN_RUN_DEPS=(
@@ -33,7 +27,6 @@ TOOLCHAIN_RUN_DEPS=(
file # for CPack
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(

63
environment/os/debian-9.sh Executable file
View File

@@ -0,0 +1,63 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev libipt-dev libbabeltrace-dev libbabeltrace-ctf-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 libipt1 libbabeltrace1 libbabeltrace-ctf1 liblzma5 python3 # for gdb
libcurl3 # for cmake
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
python3 python-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
install_all_apt "$1"
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -16,12 +16,6 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # cmake
libreadline-dev # cmake and llvm
libffi-dev libxml2-dev # llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # swig
)
@@ -33,7 +27,6 @@ TOOLCHAIN_RUN_DEPS=(
libcurl4 # for cmake
libreadline7 # for cmake and llvm
libffi6 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(

View File

@@ -15,12 +15,6 @@ TOOLCHAIN_BUILD_DEPS=(
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # for swig
)
@@ -32,7 +26,6 @@ TOOLCHAIN_RUN_DEPS=(
libcurl4 # for cmake
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(

View File

@@ -1,41 +0,0 @@
diff -ur a/folly/CMakeLists.txt b/folly/CMakeLists.txt
--- a/folly/CMakeLists.txt 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/CMakeLists.txt 2022-02-03 15:19:41.349693134 +0100
@@ -28,7 +28,6 @@
)
add_subdirectory(experimental/exception_tracer)
-add_subdirectory(logging/example)
if (PYTHON_EXTENSIONS)
# Create tree of symbolic links in structure required for successful
diff -ur a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp
--- a/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/experimental/exception_tracer/ExceptionTracerLib.cpp 2022-02-03 15:19:11.003368891 +0100
@@ -96,6 +96,7 @@
#define __builtin_unreachable()
#endif
+#if 0
namespace __cxxabiv1 {
void __cxa_throw(
@@ -154,5 +155,5 @@
}
} // namespace std
-
+#endif
#endif // defined(__GLIBCXX__)
diff -ur a/folly/Portability.h b/folly/Portability.h
--- a/folly/Portability.h 2021-12-12 23:10:42.000000000 +0100
+++ b/folly/Portability.h 2022-02-03 15:19:11.003368891 +0100
@@ -566,7 +566,7 @@
#define FOLLY_HAS_COROUTINES 0
#elif (__cpp_coroutines >= 201703L || __cpp_impl_coroutine >= 201902L) && \
(__has_include(<coroutine>) || __has_include(<experimental/coroutine>))
-#define FOLLY_HAS_COROUTINES 1
+#define FOLLY_HAS_COROUTINES 0
// This is mainly to workaround bugs triggered by LTO, when stack allocated
// variables in await_suspend end up on a coroutine frame.
#define FOLLY_CORO_AWAIT_SUSPEND_NONTRIVIAL_ATTRIBUTES FOLLY_NOINLINE

View File

@@ -1,11 +0,0 @@
diff -ur a/cmake/proxygen-config.cmake.in b/cmake/proxygen-config.cmake.in
--- a/cmake/proxygen-config.cmake.in 2021-12-13 02:37:05.000000000 +0100
+++ b/cmake/proxygen-config.cmake.in 2022-01-27 17:14:28.284810621 +0100
@@ -21,7 +21,6 @@
find_dependency(folly)
find_dependency(wangle)
find_dependency(Fizz)
-find_dependency(mvfst)
# For now, anything that depends on Proxygen has to copy its FindZstd.cmake
# and issue a `find_package(Zstd)`. Uncommenting this won't work because
# this Zstd module exposes a library called `zstd`. The right fix is

View File

@@ -1,29 +0,0 @@
diff -ur a/CMakeLists.txt b/CMakeLists.txt
--- a/CMakeLists.txt 2021-05-05 00:53:34.000000000 +0200
+++ b/CMakeLists.txt 2022-01-27 17:18:34.758302398 +0100
@@ -52,9 +52,9 @@
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHs-c-")
add_definitions(-D_HAS_EXCEPTIONS=0)
- # Disable RTTI.
- string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "/GR" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /GR-")
else(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# Use -Wall for clang and gcc.
if(NOT CMAKE_CXX_FLAGS MATCHES "-Wall")
@@ -77,9 +77,9 @@
string(REGEX REPLACE "-fexceptions" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
- # Disable RTTI.
- string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
+ # # Disable RTTI.
+ # string(REGEX REPLACE "-frtti" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
endif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
# BUILD_SHARED_LIBS is a standard CMake variable, but we declare it here to make

File diff suppressed because it is too large Load Diff

View File

@@ -1,75 +0,0 @@
-----BEGIN PGP PUBLIC KEY BLOCK-----
mQINBEzEOZIBEACxg/IuXERlDB48JBWmF4NxNUuuup1IhJAJyFGFSKh3OGAO2Ard
sNuRLjANsFXA7m7P5eTFcG+BoHHuAVYmKnI3PPZtHVLnUt4pGItPczQZ2BE1WpcI
ayjGTBJeKItX3Npqg9D/odO9WWS1i3FQPVdrLn0YH37/BA66jeMQCRo7g7GLpaNf
IrvYGsqTbxCwsmA37rpE7oyU4Yrf74HT091WBsRIoq/MelhbxTDMR8eu/dUGZQVc
Kj3lN55RepwWwUUKyqarY0zMt4HkFJ7v7yRL+Cvzy92Ouv4Wf2FlhNtEs5LE4Tax
W0PO5AEmUoKjX87SezQK0f652018b4u6Ex52cY7p+n5TII/UyoowH6+tY8UHo9yb
fStrqgNE/mY2bhA6+AwCaOUGsFzVVPTbjtxL3HacUP/jlA1h78V8VTvTs5d55iG7
jSqR9o05wje8rwNiXXK0xtiJahyNzL97Kn/DgPSqPIi45G+8nxWSPFM5eunBKRl9
vAnsvwrdPRsR6YR3uMHTuVhQX9/CY891MHkaZJ6wydWtKt3yQwJLYqwo5d4DwnUX
CduUwSKv+6RmtWI5ZmTQYOcBRcZyGKml9X9Q8iSbm6cnpFXmLrNQwCJN+D3SiYGc
MtbltZo0ysPMa6Xj5xFaYqWk/BI4iLb2Gs+ByGo/+a0Eq4XYBMOpitNniQARAQAB
tCdMYXNzZSBDb2xsaW4gPGxhc3NlLmNvbGxpbkB0dWthYW5pLm9yZz6JAlEEEwEK
ADsCGwMCHgECF4AECwkIBwMVCggFFgIDAQAWIQQ2kMJAzlG0Zw0wrRw47nV9aRhG
IAUCYEt9dQUJFxeR4wAKCRA47nV9aRhGIBNDEACxD6vJ+enZwe3IgkJh5JtLsC9b
MWCQRlPW1EVMsg96Cb5Rtron1eN1pp1TlzENJu1/C7C/VEsr9WwOPg26Men7fNf/
O21QM9IBWd/uB0Pu333WqKh92ESS5x9ST9DrG39nVGSPkQQBMuia72VrA+crPnwT
/h/u1IN6/sff5VDIU24rUiqW2Npy733dANruj7Ny0scRXVPltnVdhqwPHt6qNjC1
t+/cCnwHgW1BR1RYXBPpB42z/m29dL9rPrG0YPGWs2Bc+EATUICfEE6eIvwfciue
IJTjKT9Y9DrogJC2AYFhjC7N04OKdCB2hFs4BjexJwr4X0GJO7LhFl03c951AsIE
GHwrucRPB5bo2vmvQ8IvZn7CmtdUJzXv9JlyU6p+MIK1pz7TK6GgSOSffQIXZn6e
nUPtm9mEwuncOfmW8/ODYPs1gCWYgyiFJx8h7eEu+M4MxHSFBs7MwXf/Ae2fSp+M
P/p198qB8fC5oVBnF95qb0Qi0uc1D+Gb+gpBF+ymMb+s/VBOR3QWiym7AzBrJ62g
UnbC9jMLGnSRI+7p7raUfMTgXr5/oQoBw7ExJVltSSRrim2YH/t4CV47mO6dR9J3
1RtsTFIRNhz+07XPsETcuCV/dgqeC8fOFLt9MY17Sufhb1DcGy4urZBOIhXcpTV7
vHVj5IYH5nYOT49NRYkCOAQTAQIAIgUCTMQ5kgIbAwYLCQgHAwIGFQgCCQoLBBYC
AwECHgECF4AACgkQOO51fWkYRiAg4A/7BXKwoRaXrMbMPOW7vuVF7c2IKB2Yqzn1
vLBCwuEHkqY237lDcXY4/5LR+1gcZ3Duw1n/BRSm0FBdvyX/JTWiWNSDUkKAO/0l
T2Tg44YLrDT3bzwu8dbU9xQt6kH+SCOHvv5Oe4k79l5mro6fF3H1M0bN63x/YoFY
ojy09D7/JptY82oR4f/VdKnfZLJcCViCb0wp8SD2NkDAudKg+K+7PD8HlTWklQQg
TZdRXxVZKIJeU42aJDqnRbAhJd64YHyClhqut9F5LUmiP5qfLfNhkKDhNOwk2Blr
BGBJkSd7wPyzcX4Mun/L6YspHjbeVMt9TD7HQlo+OOd2OjAHCx6pqwkXnzeLPEaE
cPdQ1SHgrBViAxX3DNPubLP0Knw8XwFu96EuhHZgexE1W7bB4LFsJyXAc5k1PqPD
CLsAauxmvI2OfI7opG/8wyxDvNgoPjG8fZNAgY0REqPC0JnTXChH31IxUmhNotH8
tD3DDTZOHw05n5MwwUrEE9xiETVDfFQcMLfxZ9KLz+BC2g1t5LYublRgnCMNJzFg
sNUMM02CphABzl/LCLnumr0eyQQ/weV4twEhLwSDmqLYHL0EdYW0Y3CnnU9vmYxQ
cXKbstS71sEJJYBBmSBbf9GxkOY8BRNtwVwY0kPgxv1WqdVBiAFvfB+pyAsrax9B
3UeB7ZSwRD6JAhwEEAEKAAYFAlS25GwACgkQlbYYGy0z6ew92Q//ZA9/6piQtoW4
PwP/1DtWGyKU8hwR+9FG669iPk/dAG+yoEJtFMOUpg/FUFmCX8Bc4oEHsCVyLxKt
DcCVUIRcYNSFi5hTZaBEbwsOlDT37gtlfIIu34hhHRccKaLnN/N9gNMNw8wGh9xg
Q/KtxZwcbk/bZIlDkKTJkFBRAekdEGAFDWb/AZOy+LQxS8ZAh1eWkfV0i8opmK9k
gPXtLE0WSsqtYyGs58z+BFE9NH3tEUwK6jSvtuLwQl4UrICNbKthcpb8WwH6UXzb
q3QNSYVOpf/cqRdBJA6bvb/ku/xyKVL08lGmxD9v1b137R7mafDAFPTsvH2Mt/0V
YuhtWav3r1Bl9QksDxt2DTS8wiWDUBetGqOVdcw7vBrXPEWDNBmxeJXsiJ7zJlR+
9wrJOm6RV2+l1IPxu96EaPS+kTNBijKrhxb67bww8BTEWTd0wcdJmgWRkM8SIstp
IKqd0L2TFYph2/NtrBhRg+DIEPJPpSTGsUMcCEXCZPQ+cIdlQKsWpk0tZ62DlvEl
r7E+wgUSQolRfx5KrpZifiS2zQlhzdXv28CJhsVbLyw5fUAWUKIH/dCo5NKsNLk2
Lc5DH9VWnFgxAAtW290FqeK/4ulMq7Vs1dQSwyHM2Ni3QqqeaiOrh8gbSY5CMLFN
Y3HYRwuTYPa3AobsozCzBj0Zdf/6AFe5Ag0ETMQ5kgEQAL/FwKdjxgPxtSpgq1SM
zgZtTTyLqhgGD3NZfadHWHYRIL38NDV3JeTA79Y2zj2dj7KQPDT+0aqeizTV2E3j
P3iCQ53VOT4consBaQAgKexpptnS+T1DobtICFJ0GGzf0HRj6KO2zSOuOitWPWlU
wbvX7M0LLI2+hqlx0jTPqbJFZ/Za6KTtbS6xdCPVUpUqYZQpokEZcwQmUp8Q+lGo
JD2sNYCZyap63X/aAOgCGr2RXYddOH5e8vGzGW+mwtCv+WQ9Ay35mGqI5MqkbZd1
Qbuv2b1647E/QEEucfRHVbJVKGGPpFMUJtcItyyIt5jo+r9CCL4Cs47dF/9/RNwu
NvpvHXUyqMBQdWNZRMx4k/NGD/WviPi9m6mIMui6rOQsSOaqYdcUX4Nq2Orr3Oaz
2JPQdUfeI23iot1vK8hxvUCQTV3HfJghizN6spVl0yQOKBiE8miJRgrjHilH3hTb
xoo42xDkNAq+CQo3QAm1ibDxKCDq0RcWPjcCRAN/Q5MmpcodpdKkzV0yGIS4g7s5
frVrgV/kox2r4/Yxsr8K909+4H82AjTKGX/BmsQFCTAqBk6p7I0zxjIqJ/w33TZB
Q0Pn4r3WIlUPafzY6a9/LAvN1fHRxf9SpCByJsszD03Qu5f5TB8gthsdnVmTo7jj
iordEKMtw2aEMLzdWWTQ/TNVABEBAAGJAjwEGAEKACYCGwwWIQQ2kMJAzlG0Zw0w
rRw47nV9aRhGIAUCYEt9YAUJFxeRzgAKCRA47nV9aRhGIMLtD/9HuKM4pngImcuz
YwzQmdv4j26YYyh4jVsKEmVWTiRcehEgUIlrWkCu3qzd5NK+RetS7kJ8MPnzEUfj
YbpdC6yrF6n1mSrZZ4VJMkV2ev37bIgXM+Wp1mCAGbjNxQnjn9RabT/gjIqmGuRn
AP7RsSeOSuO/gO9h2Pteciz23ussTilB+8cTooQEQQZe6Kv/zukvL+ccSehLHsZ7
qVfRUAmtt8nFkXXE+s8jfLfhqstaI2/RJu5witaPcXM8Mnz2E95aASAbZy0eQot9
0Pvf07n9yuC3tueTvzvlXx3h5U3yT44tIOmzANIQjay1TGdm+RBJ2ZYyhyLawlZ2
NVUXXSp4QZZXPA0UWbF+pb7Q9cdKDNFVuvGBljuea0Yd0T2o+ibDq43HziX9ll+l
SXk9mqvW1UcDOaxWrSsm1Gc1O9g3wqH5xHAhtY8GPh/7VgAawskPkmnlkMW6pYPy
zibbeISJL1gd1jIT63y6aoVrtNoo+wYJm280ROflh4+5QOo6QJ+jm70fkXSG/qJ5
a8/qCPTHkJc/rpkL6/TDQAJURi9RhDAC0gb40HtusbN1LZEA+i0cWTmYXap+DB4Y
R4pApilpaG87M+VUokR4xpnx7vTb2MPa7Mdenvi9FEGnKXadmT8038vlfzz5GGUT
MlVin9BQPTpdA+PpRiJvKJgVDeAFOg==
=asTC
-----END PGP PUBLIC KEY BLOCK-----

59
init
View File

@@ -10,8 +10,6 @@ function print_help () {
echo -e "Check for missing packages and setup the project.\n"
echo "Optional arguments:"
echo -e " -h\tdisplay this help and exit"
echo -e " --without-libs-setup\tskip the step for setting up libs"
echo -e " --wsl-quicklisp-proxy \"host:port\"\tquicklist HTTP proxy (this flag + HTTP proxy are required on WSL)"
}
function setup_virtualenv () {
@@ -32,36 +30,21 @@ function setup_virtualenv () {
popd > /dev/null
}
wsl_quicklisp_proxy=""
setup_libs=true
if [[ $# -eq 1 && "$1" == "-h" ]]; then
if [[ $# -gt 1 ]]; then
print_help
exit 0
else
while(($#)); do
case "$1" in
--wsl-quicklisp-proxy)
shift
if [[ $# -eq 0 ]]; then
echo "Missing proxy URL"
print_help
exit 1
fi
wsl_quicklisp_proxy=":proxy \"http://$1/\""
shift
;;
--without-libs-setup)
shift
setup_libs=false
;;
*)
# unknown option
echo "Invalid argument provided: $1"
print_help
exit 1
;;
esac
done
exit 1
elif [[ $# -eq 1 ]]; then
case "$1" in
-h)
print_help
exit 0
;;
*)
# unknown option
print_help
exit 1
;;
esac
fi
DISTRO=$(operating_system)
@@ -83,7 +66,7 @@ if [[ ! -f "${quicklisp_install_dir}/setup.lisp" ]]; then
echo \
"
(load \"${DIR}/quicklisp.lisp\")
(quicklisp-quickstart:install $wsl_quicklisp_proxy :path \"${quicklisp_install_dir}\")
(quicklisp-quickstart:install :path \"${quicklisp_install_dir}\")
" | sbcl --script || exit 1
rm -rf quicklisp.lisp || exit 1
fi
@@ -97,13 +80,11 @@ echo \
(ql:quickload '(:lcp :lcp/test) :silent t)
" | sbcl --script
if [[ "$setup_libs" == "true" ]]; then
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
fi
# Setup libs (download).
cd libs
./cleanup.sh
./setup.sh
cd ..
# setup gql_behave dependencies
setup_virtualenv tests/gql_behave

View File

@@ -8,15 +8,6 @@ if (NPROC EQUAL 0)
set(NPROC 1)
endif()
find_package(Boost 1.78 REQUIRED)
find_package(BZip2 1.0.6 REQUIRED)
find_package(Threads REQUIRED)
set(GFLAGS_NOTHREADS OFF)
find_package(gflags REQUIRED)
find_package(fmt 8.0.1)
find_package(Jemalloc REQUIRED)
find_package(ZLIB 1.2.11 REQUIRED)
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})
# convenience functions
@@ -111,13 +102,19 @@ import_external_library(antlr4 STATIC
# Setup google benchmark.
import_external_library(benchmark STATIC
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/${CMAKE_INSTALL_LIBDIR}/libbenchmark.a
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/lib/libbenchmark.a
${CMAKE_CURRENT_SOURCE_DIR}/benchmark/include
# Skip testing. The tests don't compile with Clang 8.
CMAKE_ARGS -DBENCHMARK_ENABLE_TESTING=OFF)
include(FetchContent)
# setup fmt format
FetchContent_Declare(fmt
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fmt)
FetchContent_MakeAvailable(fmt)
# setup rapidcheck (it cannot be external, since it doesn't have install
# target)
set(RC_ENABLE_GTEST ON CACHE BOOL "Build Google Test integration" FORCE)
@@ -143,19 +140,58 @@ import_library(gtest_main STATIC ${GTEST_MAIN_LIBRARY} ${GTEST_INCLUDE_DIR} gtes
import_library(gmock STATIC ${GMOCK_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj)
import_library(gmock_main STATIC ${GMOCK_MAIN_LIBRARY} ${GTEST_INCLUDE_DIR} gtest-proj)
# setup google flags
set(GFLAGS_NO_FILENAMES "0")
if ("${CMAKE_BUILD_TYPE}" MATCHES "^(R|r)(E|e)(L|l).+")
set(GFLAGS_NO_FILENAMES "1")
endif()
# setup google flags
import_external_library(gflags STATIC
${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/libgflags.a
${CMAKE_CURRENT_SOURCE_DIR}/gflags/include
# Not needed, since gflags is C++ only.
NO_C_COMPILER
# Don't register installation in ~/.cmake
CMAKE_ARGS -DREGISTER_INSTALL_PREFIX=OFF
-DBUILD_gflags_nothreads_LIB=OFF
-DGFLAGS_NO_FILENAMES=${GFLAGS_NO_FILENAMES})
# Setup cppitertools
import_header_library(cppitertools ${CMAKE_CURRENT_SOURCE_DIR})
# Setup json
import_header_library(json ${CMAKE_CURRENT_SOURCE_DIR})
# Setup bzip2
import_external_library(bzip2 STATIC
${CMAKE_CURRENT_SOURCE_DIR}/bzip2/libbz2.a
${CMAKE_CURRENT_SOURCE_DIR}/bzip2
# bzip2's Makefile has -g CFLAG which is redundant
CONFIGURE_COMMAND sed -i "s/-Wall -Winline -O2 -g/-Wall -Winline -O2/g" ${CMAKE_CURRENT_SOURCE_DIR}/bzip2/Makefile
BUILD_COMMAND make -C ${CMAKE_CURRENT_SOURCE_DIR}/bzip2
CC=${CMAKE_C_COMPILER}
CXX=${CMAKE_CXX_COMPILER}
INSTALL_COMMAND true)
# Setup zlib
set(ZLIB_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/zlib)
set(ZLIB_LIBRARIES ${ZLIB_ROOT}/lib/libz.a)
set(ZLIB_INCLUDE_DIRS ${ZLIB_ROOT}/include)
import_external_library(zlib STATIC
${ZLIB_LIBRARIES}
${ZLIB_INCLUDE_DIRS}
CMAKE_ARGS -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
BUILD_COMMAND $(MAKE) zlibstatic)
# Setup RocksDB
import_external_library(rocksdb STATIC
${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/lib/librocksdb.a
${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include
CMAKE_ARGS -DUSE_RTTI=ON
DEPENDS gflags-proj
CMAKE_ARGS -Dgflags_DIR=${CMAKE_CURRENT_SOURCE_DIR}/gflags/lib/cmake/gflags
-DUSE_RTTI=ON
-DWITH_TESTS=OFF
-DGFLAGS_NOTHREADS=OFF
-DCMAKE_INSTALL_LIBDIR=lib
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
BUILD_COMMAND $(MAKE) rocksdb)
@@ -196,6 +232,8 @@ FetchContent_Declare(spdlog
FetchContent_MakeAvailable(spdlog)
include(jemalloc.cmake)
# Setup librdkafka.
import_external_library(librdkafka STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka.a
@@ -203,13 +241,12 @@ import_external_library(librdkafka STATIC
CMAKE_ARGS -DRDKAFKA_BUILD_STATIC=ON
-DRDKAFKA_BUILD_EXAMPLES=OFF
-DRDKAFKA_BUILD_TESTS=OFF
-DWITH_ZSTD=OFF
-DENABLE_LZ4_EXT=OFF
-DCMAKE_INSTALL_LIBDIR=lib
-DWITH_SSL=ON
# If we want SASL, we need to install it on build machines
-DWITH_SASL=OFF)
target_link_libraries(librdkafka INTERFACE ${OPENSSL_LIBRARIES} ZLIB::ZLIB)
target_link_libraries(librdkafka INTERFACE ${OPENSSL_LIBRARIES} zlib)
import_library(librdkafka++ STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdkafka/lib/librdkafka++.a
@@ -224,6 +261,9 @@ import_external_library(protobuf STATIC
BUILD_IN_SOURCE 1
CONFIGURE_COMMAND true)
set(BOOST_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/boost/lib)
set(BOOST_ROOT ${BOOST_ROOT} PARENT_SCOPE)
import_external_library(pulsar STATIC
${CMAKE_CURRENT_SOURCE_DIR}/pulsar/pulsar-client-cpp/lib/libpulsarwithdeps.a
${CMAKE_CURRENT_SOURCE_DIR}/pulsar/install/include
@@ -240,18 +280,10 @@ import_external_library(pulsar STATIC
-DBOOST_ROOT=${BOOST_ROOT}
-DCMAKE_PREFIX_PATH=${PROTOBUF_ROOT}
-DProtobuf_INCLUDE_DIRS=${PROTOBUF_ROOT}/include
-DZLIB_LIBRARIES=${ZLIB_LIBRARIES}
-DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIRS}
-DBUILD_PYTHON_WRAPPER=OFF
-DBUILD_PERF_TOOLS=OFF
-DUSE_LOG4CXX=OFF
BUILD_COMMAND $(MAKE) pulsarStaticWithDeps)
add_dependencies(pulsar-proj protobuf)
if (${MG_ARCH} STREQUAL "ARM64")
set(MG_LIBRDTSC_CMAKE_ARGS -DLIBRDTSC_ARCH_x86=OFF -DLIBRDTSC_ARCH_ARM64=ON)
endif()
import_external_library(librdtsc STATIC
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/lib/librdtsc.a
${CMAKE_CURRENT_SOURCE_DIR}/librdtsc/include
CMAKE_ARGS ${MG_LIBRDTSC_CMAKE_ARGS}
BUILD_COMMAND $(MAKE) rdtsc)
add_dependencies(pulsar-proj protobuf zlib)

View File

@@ -1,43 +0,0 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index a8503bb..11362cf 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
ExternalProject_Add(
utfcpp
- GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
- GIT_TAG "v3.1.1"
+ GIT_REPOSITORY "https://github.com/nemtrif/utfcpp"
+ GIT_TAG "v3.2.1"
SOURCE_DIR ${UTFCPP_DIR}
UPDATE_DISCONNECTED 1
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
DESTINATION lib
EXPORT antlr4-targets)
install(TARGETS antlr4_static
diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
index 468db98..65a473b 100644
--- a/runtime/Cpp/runtime/src/support/Any.h
+++ b/runtime/Cpp/runtime/src/support/Any.h
@@ -122,12 +122,12 @@ private:
}
private:
- template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return new Derived<T>(value);
}
- template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return nullptr;
}

View File

@@ -1,29 +0,0 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ee9b58c..31359a9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU "Enables PMU usage on ARM platforms" OFF)
# | Library Build and Install Properties |
# +--------------------------------------------------------+
-add_library(rdtsc SHARED
+add_library(rdtsc
src/cycles.c
src/common_timer.c
src/timer.c
@@ -72,15 +72,6 @@ target_include_directories(rdtsc
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
)
-# Install directory changes depending on build mode
-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug")
- # During debug, the library will be installed into a local directory
- set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/_install CACHE PATH "" FORCE)
-else ()
- # This will install in /usr/lib and /usr/include
- set(CMAKE_INSTALL_PREFIX /usr CACHE PATH "" FORCE)
-endif ()
-
# Specifying what to export when installing (GNUInstallDirs required)
install(TARGETS rdtsc
EXPORT librstsc-config

View File

@@ -1,21 +0,0 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6761929..6a369af 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -220,6 +220,7 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
endif()
endif()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-copy -Wno-unused-but-set-variable")
endif()
include(CheckCCompilerFlag)
@@ -997,7 +998,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(ROCKSDB_BUILD_SHARED)
install(
- TARGETS ${ROCKSDB_SHARED_LIB}
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
EXPORT RocksDBTargets
COMPONENT runtime
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"

View File

@@ -107,21 +107,26 @@ declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["fmt"]="http://$local_cache_host/git/fmt.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
["gtest"]="http://$local_cache_host/git/googletest.git"
["gflags"]="http://$local_cache_host/git/gflags.git"
["libbcrypt"]="http://$local_cache_host/git/libbcrypt.git"
["bzip2"]="http://$local_cache_host/git/bzip2.git"
["zlib"]="http://$local_cache_host/git/zlib.git"
["rocksdb"]="http://$local_cache_host/git/rocksdb.git"
["mgclient"]="http://$local_cache_host/git/mgclient.git"
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["jemalloc"]="http://$local_cache_host/git/jemalloc.git"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
["protobuf"]="http://$local_cache_host/git/protobuf.git"
["boost"]="http://$local_cache_host/file/boost_1_77_0.tar.gz"
["pulsar"]="http://$local_cache_host/git/pulsar.git"
["librdtsc"]="http://$local_cache_host/git/librdtsc.git"
)
# The goal of secondary urls is to have links to the "source of truth" of
@@ -132,21 +137,26 @@ declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["fmt"]="https://github.com/fmtlib/fmt.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
["gtest"]="https://github.com/google/googletest.git"
["gflags"]="https://github.com/memgraph/gflags.git"
["libbcrypt"]="https://github.com/rg3/libbcrypt"
["bzip2"]="https://github.com/VFR-maniac/bzip2"
["zlib"]="https://github.com/madler/zlib.git"
["rocksdb"]="https://github.com/facebook/rocksdb.git"
["mgclient"]="https://github.com/memgraph/mgclient.git"
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
["spdlog"]="https://github.com/gabime/spdlog"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["jemalloc"]="https://github.com/jemalloc/jemalloc.git"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/b3e5cb7f20dcc5c806e418df34324eca60d17d4e/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
["boost"]="https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.gz"
["pulsar"]="https://github.com/apache/pulsar.git"
["librdtsc"]="https://github.com/gabrieleara/librdtsc.git"
)
# antlr
@@ -154,26 +164,39 @@ file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4
antlr4_tag="4.9.2" # v4.9.2
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
pushd antlr4
git apply ../antlr4.patch
popd
# remove shared library from install dependencies
sed -i 's/install(TARGETS antlr4_shared/install(TARGETS antlr4_shared OPTIONAL/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
# fix issue https://github.com/antlr/antlr4/issues/3194 - should update Antlr commit once the PR related to the issue gets merged
sed -i 's/std::is_nothrow_copy_constructible/std::is_copy_constructible/' antlr4/runtime/Cpp/runtime/src/support/Any.h
# replace the utf8cpp version which is used because the older one uses gtest that doesn't
# compile with the newer compilers because of uninitialized variable
# the newer utf8cpp switched to ftest
sed -i 's/v3.1.1/v3.2.1/' antlr4/runtime/Cpp/runtime/CMakeLists.txt
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
# fmt
fmt_tag="8.0.1" # (2021-07-03)
repo_clone_try_double "${primary_urls[fmt]}" "${secondary_urls[fmt]}" "fmt" "$fmt_tag" true
# rapidcheck
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
# google benchmark
benchmark_tag="v1.6.0"
benchmark_tag="v1.1.0"
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
# google test
googletest_tag="release-1.8.0"
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
# google flags
gflags_tag="b37ceb03a0e56c9f15ce80409438a555f8a67b7c" # custom version (May 6, 2017)
repo_clone_try_double "${primary_urls[gflags]}" "${secondary_urls[gflags]}" "gflags" "$gflags_tag"
# libbcrypt
libbcrypt_tag="8aa32ad94ebe06b76853b0767c910c9fbf7ccef4" # custom version (Dec 16, 2016)
repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}" "libbcrypt" "$libbcrypt_tag"
@@ -192,15 +215,22 @@ cd json
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
bzip2_tag="0405487e2b1de738e7f1c8afb50d19cf44e8d580" # v1.0.6 (May 26, 2011)
repo_clone_try_double "${primary_urls[bzip2]}" "${secondary_urls[bzip2]}" "bzip2" "$bzip2_tag"
zlib_tag="v1.2.11" # v1.2.11.
repo_clone_try_double "${primary_urls[zlib]}" "${secondary_urls[zlib]}" "zlib" "$zlib_tag" true
# remove shared library from install dependencies
sed -i 's/install(TARGETS zlib zlibstatic/install(TARGETS zlibstatic/g' zlib/CMakeLists.txt
rocksdb_tag="v6.14.6" # (2020-10-14)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
pushd rocksdb
git apply ../rocksdb.patch
popd
# remove shared library from install dependencies
sed -i 's/TARGETS ${ROCKSDB_SHARED_LIB}/TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL/' rocksdb/CMakeLists.txt
# mgclient
mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
mgclient_tag="v1.3.0" # (2021-09-23)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag" true
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt
# pymgclient
@@ -214,6 +244,25 @@ repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}
spdlog_tag="v1.9.2" # (2021-08-12)
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag" true
jemalloc_tag="ea6b3e973b477b8061e0076bb257dbd7f3faa756" # (2021-02-11)
repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}" "jemalloc" "$jemalloc_tag"
pushd jemalloc
# ThreadPool select job randomly, and there can be some threads that had been
# performed some memory heavy task before and will be inactive for some time,
# but until it will became active again, the memory will not be freed since by
# default each thread has it's own arena, but there should be not more then
# 4*CPU arenas (see opt.nareans description).
#
# By enabling percpu_arena number of arenas limited to number of CPUs and hence
# this problem should go away.
#
# muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
./autogen.sh --with-malloc-conf="percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000"
popd
# librdkafka
librdkafka_tag="v1.7.0" # (2021-05-06)
repo_clone_try_double "${primary_urls[librdkafka]}" "${secondary_urls[librdkafka]}" "librdkafka" "$librdkafka_tag" true
@@ -225,16 +274,18 @@ pushd protobuf
./autogen.sh && ./configure CC=clang CXX=clang++ --prefix=$(pwd)/lib
popd
# boost
file_get_try_double "${primary_urls[boost]}" "${secondary_urls[boost]}"
tar -xzf boost_1_77_0.tar.gz
mv boost_1_77_0 boost
pushd boost
./bootstrap.sh --prefix=$(pwd)/lib --with-libraries="system,regex" --with-toolset=clang
./b2 toolset=clang -j$(nproc) install variant=release
popd
#pulsar
pulsar_tag="v2.8.1"
repo_clone_try_double "${primary_urls[pulsar]}" "${secondary_urls[pulsar]}" "pulsar" "$pulsar_tag" true
pushd pulsar
git apply ../pulsar.patch
popd
#librdtsc
librdtsc_tag="v0.3"
repo_clone_try_double "${primary_urls[librdtsc]}" "${secondary_urls[librdtsc]}" "librdtsc" "$librdtsc_tag" true
pushd librdtsc
git apply ../librdtsc.patch
popd

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2026-18-02
CHANGE DATE: 2025-12-08
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -36,18 +36,9 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
# RPM specific
set(MG_ARCH_EXTENSION "noarch")
if (${MG_ARCH} STREQUAL "x86_64")
set(MG_ARCH_EXTENSION "x86_64")
elseif (${MG_ARCH} STREQUAL "ARM64")
set(MG_ARCH_EXTENSION "aarch64")
endif()
set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.x86_64.rpm")
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
/var /var/lib /var/log /etc/logrotate.d
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)

View File

@@ -1,19 +0,0 @@
FROM dokken/centos-stream-9
ARG env_folder
ARG toolchain_version
COPY ${env_folder} /env_folder
RUN yum update && yum install -y curl git
RUN /${env_folder}/os/centos-9.sh install MEMGRAPH_BUILD_DEPS
RUN /${env_folder}/os/centos-9.sh install TOOLCHAIN_RUN_DEPS
RUN rm -rf /env_folder
RUN yum clean all
RUN curl https://s3.eu-west-1.amazonaws.com/deps.memgraph.io/${toolchain_version}/${toolchain_version}-binaries-centos-9-arm64.tar.gz -o /tmp/toolchain.tar.gz \
&& tar xvzf /tmp/toolchain.tar.gz -C /opt \
&& rm /tmp/toolchain.tar.gz

View File

@@ -1,5 +0,0 @@
#!/bin/bash
cp -r ../../environment env_folder
docker build -f build_env.dockerfile --build-arg env_folder=env_folder --build-arg toolchain_version=toolchain-v4 -t mg_build_env .
rm -rf env_folder

View File

@@ -1,7 +1,7 @@
FROM debian:bullseye
# NOTE: If you change the base distro update release/package as well.
ARG release
ARG deb_release
RUN apt-get update && apt-get install -y \
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
@@ -10,10 +10,10 @@ RUN apt-get update && apt-get install -y \
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${release} /
COPY ${deb_release} /
# Install memgraph package
RUN dpkg -i ${release}
RUN dpkg -i ${deb_release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687

View File

@@ -1,31 +0,0 @@
FROM dokken/centos-stream-9
# NOTE: If you change the base distro update release/package as well.
ARG release
RUN yum update && yum install -y \
openssl libcurl libseccomp python3 python3-pip \
--nobest --allowerasing \
&& rm -rf /tmp/* \
&& yum clean all
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${release} /
# Install memgraph package
RUN rpm -i ${release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687
# Snapshots and logging volumes
VOLUME /var/log/memgraph
VOLUME /var/lib/memgraph
# Configuration volume
VOLUME /etc/memgraph
USER memgraph
WORKDIR /usr/lib/memgraph
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
CMD [""]

View File

@@ -0,0 +1,46 @@
#!/bin/bash -e
# Build and package Docker image of Memgraph.
function print_help () {
echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.deb"
echo "Optional arguments:"
echo -e "\t-h|--help\t\tPrint help."
echo -e "\t--latest\t\tTag image as latest version."
}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
deb_path="$1"
if [[ ! -f "$deb_path" ]]; then
echo "File '$deb_path' does not exist!"
exit 1
fi
# Copy the .deb to working directory.
cp "$deb_path" "${working_dir}/"
cd ${working_dir}
# Extract version and offering from deb name.
deb_name=`echo $(basename "$deb_path") | sed 's/.deb$//'`
version=`echo ${deb_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
dockerfile_path="${working_dir}/memgraph.dockerfile"
image_name="memgraph:${version}"
package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg deb_release=${deb_name}.deb .
docker save ${image_name} ${latest_image} > ${package_name}
rm "${deb_name}.deb"
echo "Built Docker image at '${working_dir}/${package_name}'"

View File

@@ -1,61 +0,0 @@
#!/bin/bash -e
# Build and package Docker image of Memgraph.
function print_help () {
echo "Usage: $0 [--latest] MEMGRAPH_PACKAGE.(deb|rpm)"
echo "Optional arguments:"
echo -e "\t-h|--help\t\tPrint help."
echo -e "\t--latest\t\tTag image as latest version."
}
working_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
latest_image=""
tag_latest=""
if [[ $# -eq 2 && "$1" == "--latest" ]]; then
latest_image="memgraph:latest"
tag_latest="-t memgraph:latest"
shift
elif [[ $# -ne 1 || "$1" == "-h" || "$1" == "--help" ]]; then
print_help
exit 1
fi
package_path="$1"
if [[ ! -f "$package_path" ]]; then
echo "File '$package_path' does not exist!"
exit 1
fi
# Copy the .deb to working directory.
cp "$package_path" "${working_dir}/"
cd ${working_dir}
extension="${package_path##*.}"
if [[ "$extension" == "deb" ]]; then
# Extract version and offering from deb name.
package_name=`echo $(basename "$package_path") | sed 's/.deb$//'`
version=`echo ${package_name} | cut -d '_' -f 2 | rev | cut -d '-' -f 2- | rev | tr '+~' '__'`
dockerfile_path="${working_dir}/memgraph_deb.dockerfile"
elif [[ "$extension" == "rpm" ]]; then
# Extract version and offering from deb name.
package_name=`echo $(basename "$package_path") | sed 's/.rpm$//'`
version=`echo ${package_name} | cut -d '-' -f 2 | rev | cut -d '-' -f 2- | rev`
version=${version%_1}
dockerfile_path="${working_dir}/memgraph_rpm.dockerfile"
else
echo "Invalid file sent as the package"
print_help
exit 1
fi
image_name="memgraph:${version}"
image_package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
docker save ${image_name} ${latest_image} | gzip > ${image_package_name}
rm "${package_name}.${extension}"
echo "Built Docker image at '${working_dir}/${image_package_name}'"

View File

@@ -0,0 +1,17 @@
FROM debian:9
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-9.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-9.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-9.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -8,6 +8,10 @@ services:
build:
context: centos-8
container_name: "mgbuild_centos-8"
mgbuild_debian-9:
build:
context: debian-9
container_name: "mgbuild_debian-9"
mgbuild_debian-10:
build:
context: debian-10

View File

@@ -3,9 +3,9 @@
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
SUPPORTED_OS=(centos-7 centos-8 debian-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v4"
TOOLCHAIN_VERSION="toolchain-v3"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
@@ -106,7 +106,7 @@ case "$1" in
last_package_name=$(cd "$HOST_OUTPUT_DIR/$based_on_os" && ls -t memgraph* | head -1)
docker_build_folder="$PROJECT_ROOT/release/docker"
cd "$docker_build_folder"
./package_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
./package_deb_docker --latest "$HOST_OUTPUT_DIR/$based_on_os/$last_package_name"
# shellcheck disable=SC2012
docker_image_name=$(cd "$docker_build_folder" && ls -t memgraph* | head -1)
docker_host_folder="$HOST_OUTPUT_DIR/docker"

View File

@@ -1,8 +1,5 @@
set(audit_src_files log.cpp)
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
add_library(mg-audit STATIC ${audit_src_files})
target_link_libraries(mg-audit json gflags fmt::fmt)
target_link_libraries(mg-audit mg-utils mg-storage-v2)

View File

@@ -5,13 +5,10 @@ set(auth_src_files
module.cpp)
find_package(Seccomp REQUIRED)
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
add_library(mg-auth STATIC ${auth_src_files})
target_link_libraries(mg-auth json libbcrypt gflags fmt::fmt)
target_link_libraries(mg-auth mg-utils mg-kvstore mg-license )
target_link_libraries(mg-auth mg-utils mg-kvstore)
target_link_libraries(mg-auth ${Seccomp_LIBRARIES})
target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS})

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -30,17 +30,6 @@ DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
namespace auth {
namespace {
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {
Permission::MATCH, Permission::CREATE, Permission::MERGE, Permission::DELETE,
Permission::SET, Permission::REMOVE, Permission::INDEX, Permission::STATS,
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET};
} // namespace
std::string PermissionToString(Permission permission) {
switch (permission) {
case Permission::MATCH:
@@ -79,12 +68,6 @@ std::string PermissionToString(Permission permission) {
return "AUTH";
case Permission::STREAM:
return "STREAM";
case Permission::MODULE_READ:
return "MODULE_READ";
case Permission::MODULE_WRITE:
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
}
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -19,30 +19,35 @@ namespace auth {
// bitmask.
// clang-format off
enum class Permission : uint64_t {
MATCH = 1,
CREATE = 1U << 1U,
MERGE = 1U << 2U,
DELETE = 1U << 3U,
SET = 1U << 4U,
REMOVE = 1U << 5U,
INDEX = 1U << 6U,
STATS = 1U << 7U,
CONSTRAINT = 1U << 8U,
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
DURABILITY = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
TRIGGER = 1U << 14U,
CONFIG = 1U << 15U,
AUTH = 1U << 16U,
STREAM = 1U << 17U,
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U
MATCH = 1,
CREATE = 1U << 1U,
MERGE = 1U << 2U,
DELETE = 1U << 3U,
SET = 1U << 4U,
REMOVE = 1U << 5U,
INDEX = 1U << 6U,
STATS = 1U << 7U,
CONSTRAINT = 1U << 8U,
DUMP = 1U << 9U,
REPLICATION = 1U << 10U,
DURABILITY = 1U << 11U,
READ_FILE = 1U << 12U,
FREE_MEMORY = 1U << 13U,
TRIGGER = 1U << 14U,
CONFIG = 1U << 15U,
AUTH = 1U << 16U,
STREAM = 1U << 17U
};
// clang-format on
// Constant list of all available permissions.
const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permission::CREATE, Permission::MERGE,
Permission::DELETE, Permission::SET, Permission::REMOVE,
Permission::INDEX, Permission::STATS, Permission::CONSTRAINT,
Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY,
Permission::TRIGGER, Permission::CONFIG, Permission::STREAM};
// Function that converts a permission to its string representation.
std::string PermissionToString(Permission permission);

View File

@@ -1,11 +1,4 @@
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
set(communication_src_files
websocket/auth.cpp
websocket/server.cpp
websocket/listener.cpp
websocket/session.cpp
bolt/v1/value.cpp
buffer.cpp
client.cpp
@@ -16,7 +9,7 @@ set(communication_src_files
find_package(Boost REQUIRED)
add_library(mg-communication STATIC ${communication_src_files})
target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io mg-auth fmt::fmt gflags)
target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils mg-io fmt::fmt gflags)
find_package(OpenSSL REQUIRED)
target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -73,9 +73,11 @@ SSL_CTX *ClientContext::context() { return ctx_; }
bool ClientContext::use_ssl() { return use_ssl_; }
ServerContext::ServerContext() : use_ssl_(false), ctx_(nullptr) {}
ServerContext::ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file,
bool verify_peer)
:
: use_ssl_(true),
#if OPENSSL_VERSION_NUMBER < 0x10100000L
ctx_(SSL_CTX_new(SSLv23_server_method()))
#else
@@ -108,47 +110,43 @@ ServerContext::ServerContext(const std::string &key_file, const std::string &cer
SSL_CTX_set_client_CA_list(ctx_, ca_names);
// Enable verification of the client certificate.
// NOLINTNEXTLINE(hicpp-signed-bitwise)
SSL_CTX_set_verify(ctx_, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
}
}
}
ServerContext::ServerContext(ServerContext &&other) noexcept { std::swap(ctx_, other.ctx_); }
ServerContext::ServerContext(ServerContext &&other) noexcept : use_ssl_(other.use_ssl_), ctx_(other.ctx_) {
other.use_ssl_ = false;
other.ctx_ = nullptr;
}
ServerContext &ServerContext::operator=(ServerContext &&other) noexcept {
if (this == &other) return *this;
// destroy my objects
if (ctx_) {
if (use_ssl_) {
SSL_CTX_free(ctx_);
}
// move other objects to self
use_ssl_ = other.use_ssl_;
ctx_ = other.ctx_;
// reset other objects
other.use_ssl_ = false;
other.ctx_ = nullptr;
return *this;
}
ServerContext::~ServerContext() {
if (ctx_) {
if (use_ssl_) {
SSL_CTX_free(ctx_);
}
}
SSL_CTX *ServerContext::context() {
MG_ASSERT(ctx_);
return ctx_;
}
SSL_CTX *ServerContext::context_clone() {
MG_ASSERT(ctx_);
SSL_CTX_up_ref(ctx_);
return ctx_;
}
SSL_CTX *ServerContext::context() { return ctx_; }
bool ServerContext::use_ssl() const { return ctx_ != nullptr; }
bool ServerContext::use_ssl() { return use_ssl_; }
} // namespace communication

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -69,7 +69,11 @@ class ClientContext final {
*/
class ServerContext final {
public:
ServerContext() = default;
/**
* This constructor constructs a ServerContext that doesn't use SSL.
*/
ServerContext();
/**
* This constructor constructs a ServerContext that uses SSL. The parameters
* `key_file` and `cert_file` can't be "" because when setting up a server it
@@ -91,15 +95,16 @@ class ServerContext final {
ServerContext(ServerContext &&other) noexcept;
ServerContext &operator=(ServerContext &&other) noexcept;
// Destructor that handles ownership of the SSL object.
~ServerContext();
SSL_CTX *context();
SSL_CTX *context_clone();
bool use_ssl() const;
bool use_ssl();
private:
SSL_CTX *ctx_{nullptr};
bool use_ssl_;
SSL_CTX *ctx_;
};
} // namespace communication

View File

@@ -89,8 +89,7 @@ class Server final {
alive_.store(true);
if (!socket_.Bind(endpoint_)) {
spdlog::error(
utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint_, "https://memgr.ph/socket"));
spdlog::error(utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint_, "https://memgr.ph/socket"));
alive_.store(false);
return false;
}
@@ -103,8 +102,9 @@ class Server final {
listener_.Start();
thread_ = std::thread([this]() {
utils::ThreadSetName(fmt::format("{} server", service_name_));
std::string service_name(service_name_);
thread_ = std::thread([this, service_name]() {
utils::ThreadSetName(fmt::format("{} server", service_name));
spdlog::info("{} server is fully armed and operational", service_name_);
spdlog::info("{} listening on {}", service_name_, socket_.endpoint());

View File

@@ -1,30 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "communication/websocket/auth.hpp"
#include <string>
namespace communication::websocket {
bool SafeAuth::Authenticate(const std::string &username, const std::string &password) const {
return auth_->Lock()->Authenticate(username, password).has_value();
}
bool SafeAuth::HasUserPermission(const std::string &username, const auth::Permission permission) const {
if (const auto user = auth_->ReadLock()->GetUser(username); user) {
return user->GetPermissions().Has(permission) == auth::PermissionLevel::GRANT;
}
return false;
}
bool SafeAuth::HasAnyUsers() const { return auth_->ReadLock()->HasUsers(); }
} // namespace communication::websocket

View File

@@ -1,44 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <string>
#include "auth/auth.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
namespace communication::websocket {
class AuthenticationInterface {
public:
virtual bool Authenticate(const std::string &username, const std::string &password) const = 0;
virtual bool HasUserPermission(const std::string &username, auth::Permission permission) const = 0;
virtual bool HasAnyUsers() const = 0;
};
class SafeAuth : public AuthenticationInterface {
public:
explicit SafeAuth(utils::Synchronized<auth::Auth, utils::WritePrioritizedRWLock> *auth) : auth_{auth} {}
bool Authenticate(const std::string &username, const std::string &password) const override;
bool HasUserPermission(const std::string &username, auth::Permission permission) const override;
bool HasAnyUsers() const override;
private:
utils::Synchronized<auth::Auth, utils::WritePrioritizedRWLock> *auth_;
};
} // namespace communication::websocket

View File

@@ -1,90 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "communication/websocket/listener.hpp"
namespace communication::websocket {
namespace {
void LogError(boost::beast::error_code ec, const std::string_view what) {
spdlog::warn("Websocket listener failed on {}: {}", what, ec.message());
}
} // namespace
void Listener::Run() { DoAccept(); }
void Listener::WriteToAll(std::shared_ptr<std::string> message) {
auto sessions_ptr = sessions_.Lock();
for (auto &session : *sessions_ptr) {
session->Write(message);
}
}
boost::asio::ip::tcp::endpoint Listener::GetEndpoint() const { return acceptor_.local_endpoint(); };
Listener::Listener(boost::asio::io_context &ioc, ServerContext *context, tcp::endpoint endpoint,
AuthenticationInterface &auth)
: ioc_(ioc), context_(context), acceptor_(ioc), auth_(auth) {
boost::beast::error_code ec;
// Open the acceptor
acceptor_.open(endpoint.protocol(), ec);
if (ec) {
LogError(ec, "open");
return;
}
// Allow address reuse
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
if (ec) {
LogError(ec, "set_option");
return;
}
// Bind to the server address
acceptor_.bind(endpoint, ec);
if (ec) {
LogError(ec, "bind");
return;
}
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
if (ec) {
LogError(ec, "listen");
return;
}
spdlog::info("WebSocket server is listening on {}:{}", endpoint.address(), endpoint.port());
}
void Listener::DoAccept() {
acceptor_.async_accept(
ioc_, [shared_this = shared_from_this()](auto ec, auto socket) { shared_this->OnAccept(ec, std::move(socket)); });
}
void Listener::OnAccept(boost::beast::error_code ec, tcp::socket socket) {
if (ec) {
return LogError(ec, "accept");
}
auto session = Session::Create(std::move(socket), *context_, auth_);
if (session->Run()) {
auto sessions_ptr = sessions_.Lock();
// Clean disconnected clients
std::erase_if(*sessions_ptr, [](const auto &elem) { return !elem->IsConnected(); });
sessions_ptr->emplace_back(std::move(session));
}
DoAccept();
}
} // namespace communication::websocket

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,8 +11,6 @@
#pragma once
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
#include <list>
#include <memory>
@@ -22,13 +20,13 @@
#include <boost/asio/strand.hpp>
#include <boost/beast/core.hpp>
#include "communication/context.hpp"
#include "communication/websocket/session.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
namespace communication::websocket {
class Listener : public std::enable_shared_from_this<Listener> {
template <typename TSession, typename TSessionData>
class Listener : public std::enable_shared_from_this<Listener<TSession, TSessionData>> {
using tcp = boost::asio::ip::tcp;
public:
@@ -38,20 +36,59 @@ class Listener : public std::enable_shared_from_this<Listener> {
}
// Start accepting incoming connections
void Run();
void WriteToAll(std::shared_ptr<std::string> message);
tcp::endpoint GetEndpoint() const;
void Run() {
// The new connection gets its own strand
acceptor_.async_accept(ioc_, [shared_this = this->shared_from_this()](auto ec, auto socket) {
shared_this->OnAccept(ec, std::move(socket));
});
}
private:
Listener(boost::asio::io_context &ioc, ServerContext *context, tcp::endpoint endpoint, AuthenticationInterface &auth);
Listener(boost::asio::io_context &ioc, tcp::endpoint endpoint, TSessionData *data)
: data_{data}, ioc_(ioc), acceptor_(ioc) {
boost::beast::error_code ec;
void DoAccept();
void OnAccept(boost::beast::error_code ec, tcp::socket socket);
// Open the acceptor
acceptor_.open(endpoint.protocol(), ec);
if (ec) {
LogError(ec, "open");
return;
}
// Allow address reuse
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
if (ec) {
LogError(ec, "set_option");
return;
}
// Bind to the server address
acceptor_.bind(endpoint, ec);
if (ec) {
LogError(ec, "bind");
return;
}
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
if (ec) {
LogError(ec, "listen");
return;
}
spdlog::info("WebSocket server is listening on {}:{}", endpoint.address(), endpoint.port());
}
void OnAccept(boost::beast::error_code ec, tcp::socket socket) {
if (ec) {
return LogError(ec, "accept");
}
Session<TSession, TSessionData>::Create(std::move(socket), data_)->Run();
Run();
}
TSessionData *data_;
boost::asio::io_context &ioc_;
ServerContext *context_;
tcp::acceptor acceptor_;
utils::Synchronized<std::list<std::shared_ptr<Session>>, utils::SpinLock> sessions_;
AuthenticationInterface &auth_;
};
} // namespace communication::websocket

View File

@@ -1,84 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "communication/websocket/server.hpp"
#include <spdlog/pattern_formatter.h>
namespace communication::websocket {
Server::~Server() {
MG_ASSERT(!background_thread_ || (ioc_.stopped() && !background_thread_->joinable()),
"Server wasn't shutdown properly");
}
void Server::Start() {
MG_ASSERT(!background_thread_, "The server was already started!");
listener_->Run();
background_thread_.emplace([this] { ioc_.run(); });
}
void Server::Shutdown() { ioc_.stop(); }
void Server::AwaitShutdown() {
if (background_thread_ && background_thread_->joinable()) {
background_thread_->join();
}
}
bool Server::IsRunning() const { return background_thread_ && !ioc_.stopped(); }
boost::asio::ip::tcp::endpoint Server::GetEndpoint() const { return listener_->GetEndpoint(); };
namespace {
class QuoteEscapeFormatter : public spdlog::custom_flag_formatter {
public:
void format(const spdlog::details::log_msg &msg, const std::tm & /*time*/, spdlog::memory_buf_t &dest) override {
for (const auto c : msg.payload) {
if (c == '"') {
constexpr std::string_view escaped_quote = "\\\"";
dest.append(escaped_quote.data(), escaped_quote.data() + escaped_quote.size());
} else if (c == '\n') {
constexpr std::string_view escaped_newline = "\\n";
dest.append(escaped_newline.data(), escaped_newline.data() + escaped_newline.size());
} else {
dest.push_back(c);
}
}
}
std::unique_ptr<custom_flag_formatter> clone() const override {
return spdlog::details::make_unique<QuoteEscapeFormatter>();
}
};
}; // namespace
void Server::LoggingSink::sink_it_(const spdlog::details::log_msg &msg) {
const auto listener = listener_.lock();
if (!listener) {
return;
}
using memory_buf_t = fmt::basic_memory_buffer<char, 250>;
memory_buf_t formatted;
base_sink<std::mutex>::formatter_->format(msg, formatted);
listener->WriteToAll(std::make_shared<std::string>(formatted.data(), formatted.size()));
}
std::shared_ptr<Server::LoggingSink> Server::GetLoggingSink() {
auto formatter = std::make_unique<spdlog::pattern_formatter>();
formatter->add_flag<QuoteEscapeFormatter>('*').set_pattern(
R"json({"event": "log", "level": "%l", "message": "%*"})json");
auto sink = std::make_shared<LoggingSink>(listener_);
sink->set_formatter(std::move(formatter));
return sink;
}
} // namespace communication::websocket

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,45 +24,46 @@
namespace communication::websocket {
template <typename TSession, typename TSessionData>
class Server final {
using tcp = boost::asio::ip::tcp;
public:
explicit Server(io::network::Endpoint endpoint, ServerContext *context, AuthenticationInterface &auth)
: listener_{Listener::Create(
ioc_, context, tcp::endpoint{boost::asio::ip::make_address(endpoint.address), endpoint.port}, auth)} {}
explicit Server(io::network::Endpoint endpoint, TSessionData *data)
: ioc_{},
listener_{Listener<TSession, TSessionData>::Create(
ioc_, tcp::endpoint{boost::asio::ip::make_address(endpoint.address), endpoint.port}, data)} {}
Server(const Server &) = delete;
Server(Server &&) = delete;
Server &operator=(const Server &) = delete;
Server &operator=(Server &&) = delete;
~Server();
~Server() {
MG_ASSERT(!background_thread_ || (ioc_.stopped() && !background_thread_->joinable()),
"Server wasn't shutdown properly");
}
void Start();
void Shutdown();
void AwaitShutdown();
bool IsRunning() const;
tcp::endpoint GetEndpoint() const;
void Start() {
MG_ASSERT(!background_thread_, "The server was already started!");
listener_->Run();
background_thread_.emplace([this] { ioc_.run(); });
}
class LoggingSink : public spdlog::sinks::base_sink<std::mutex> {
public:
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(listener) {}
void Shutdown() { ioc_.stop(); }
private:
void sink_it_(const spdlog::details::log_msg &msg) override;
void AwaitShutdown() {
if (background_thread_ && background_thread_->joinable()) {
background_thread_->join();
}
}
void flush_() override {}
std::weak_ptr<Listener> listener_;
};
std::shared_ptr<LoggingSink> GetLoggingSink();
bool IsRunning() const { return background_thread_ && !ioc_.stopped(); }
private:
boost::asio::io_context ioc_;
std::shared_ptr<Listener> listener_;
std::shared_ptr<Listener<TSession, TSessionData>> listener_;
std::optional<std::thread> background_thread_;
};
} // namespace communication::websocket

View File

@@ -1,235 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "communication/websocket/session.hpp"
#include <functional>
#include <memory>
#include <string>
#include <fmt/format.h>
#include <spdlog/spdlog.h>
#include <boost/asio/bind_executor.hpp>
#include <boost/beast/core/buffers_to_string.hpp>
#include <boost/beast/core/stream_traits.hpp>
#include <json/json.hpp>
#include "communication/context.hpp"
#include "communication/websocket/auth.hpp"
#include "utils/logging.hpp"
namespace communication::websocket {
namespace {
void LogError(const boost::beast::error_code ec, const std::string_view what) {
spdlog::warn("Websocket session failed on {}: {}", what, ec.message());
}
} // namespace
std::variant<Session::PlainWebSocket, Session::SSLWebSocket> Session::CreateWebSocket(tcp::socket &&socket,
ServerContext &context) {
if (context.use_ssl()) {
ssl_context_.emplace(context.context_clone());
return Session::SSLWebSocket{std::move(socket), *ssl_context_};
}
return Session::PlainWebSocket{std::move(socket)};
}
Session::Session(tcp::socket &&socket, ServerContext &context, AuthenticationInterface &auth)
: ws_(CreateWebSocket(std::move(socket), context)), strand_{boost::asio::make_strand(GetExecutor())}, auth_{auth} {}
bool Session::Run() {
ExecuteForWebsocket([](auto &&ws) {
ws.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
ws.set_option(boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::response_type &res) {
res.set(boost::beast::http::field::server, "Memgraph WS");
}));
});
if (auto *ssl_ws = std::get_if<SSLWebSocket>(&ws_); ssl_ws != nullptr) {
try {
boost::beast::get_lowest_layer(*ssl_ws).expires_after(std::chrono::seconds(30));
ssl_ws->next_layer().handshake(boost::asio::ssl::stream_base::server);
} catch (const boost::system::system_error &e) {
spdlog::warn("Failed on SSL handshake: {}", e.what());
return false;
}
}
auto result = ExecuteForWebsocket([](auto &&ws) -> bool {
// Accept the websocket handshake
boost::beast::error_code ec;
ws.accept(ec);
if (ec) {
LogError(ec, "accept");
return false;
}
return true;
});
if (!result) {
return false;
}
authenticated_ = !auth_.HasAnyUsers();
connected_.store(true, std::memory_order_relaxed);
// run on the strand
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
return true;
}
void Session::Write(std::shared_ptr<std::string> message) {
boost::asio::dispatch(strand_, [message = std::move(message), shared_this = shared_from_this()]() mutable {
if (!shared_this->connected_.load(std::memory_order_relaxed)) {
return;
}
if (!shared_this->IsAuthenticated()) {
return;
}
shared_this->messages_.push_back(std::move(message));
if (shared_this->messages_.size() > 1) {
return;
}
shared_this->DoWrite();
});
}
bool Session::IsConnected() const { return connected_.load(std::memory_order_relaxed); }
void Session::DoWrite() {
ExecuteForWebsocket([this](auto &&ws) {
auto next_message = messages_.front();
ws.async_write(boost::asio::buffer(*next_message),
boost::asio::bind_executor(
strand_, [message_string = std::move(next_message), shared_this = shared_from_this()](
boost::beast::error_code ec, const size_t bytes_transferred) {
shared_this->OnWrite(ec, bytes_transferred);
}));
});
}
void Session::OnWrite(boost::beast::error_code ec, size_t /*bytes_transferred*/) {
messages_.pop_front();
if (close_) {
DoShutdown();
return;
}
if (ec) {
close_ = true;
return LogError(ec, "write");
}
if (!messages_.empty()) {
DoWrite();
}
}
void Session::DoRead() {
ExecuteForWebsocket([this](auto &&ws) {
ws.async_read(buffer_, boost::asio::bind_executor(strand_, std::bind_front(&Session::OnRead, shared_from_this())));
});
;
}
void Session::DoClose() {
ExecuteForWebsocket([this](auto &&ws) mutable {
ws.async_close(boost::beast::websocket::close_code::normal,
boost::asio::bind_executor(strand_, [shared_this = shared_from_this()](boost::beast::error_code ec) {
shared_this->OnClose(ec);
}));
});
}
void Session::OnClose(boost::beast::error_code ec) {
connected_.store(false, std::memory_order_relaxed);
if (ec) {
return LogError(ec, "close");
}
}
utils::BasicResult<std::string> Session::Authorize(const nlohmann::json &creds) {
if (!auth_.Authenticate(creds.at("username").get<std::string>(), creds.at("password").get<std::string>())) {
return {"Authentication failed!"};
}
#ifdef MG_ENTERPRISE
if (!auth_.HasUserPermission(creds.at("username").get<std::string>(), auth::Permission::WEBSOCKET)) {
return {"Authorization failed!"};
}
#endif
return {};
}
void Session::OnRead(const boost::beast::error_code ec, const size_t /*bytes_transferred*/) {
if (ec == boost::beast::websocket::error::closed) {
DoShutdown();
return;
}
if (!IsAuthenticated()) {
auto response = nlohmann::json();
auto auth_failed = [this, &response](const std::string &message) {
response["success"] = false;
response["message"] = message;
MG_ASSERT(messages_.empty());
messages_.push_back(std::make_shared<std::string>(response.dump()));
close_ = true;
DoWrite();
};
try {
const auto creds = nlohmann::json::parse(boost::beast::buffers_to_string(buffer_.data()));
buffer_.consume(buffer_.size());
if (const auto result = Authorize(creds); result.HasError()) {
std::invoke(auth_failed, result.GetError());
return;
}
response["success"] = true;
response["message"] = "User has been successfully authenticated!";
MG_ASSERT(messages_.empty());
authenticated_ = true;
messages_.push_back(std::make_shared<std::string>(response.dump()));
DoWrite();
} catch (const nlohmann::json::out_of_range &out_of_range) {
const auto err_msg = fmt::format("Invalid JSON for authentication received: {}!", out_of_range.what());
spdlog::error(err_msg);
std::invoke(auth_failed, err_msg);
return;
} catch (const nlohmann::json::parse_error &parse_error) {
const auto err_msg = fmt::format("Cannot parse JSON for WebSocket authentication: {}!", parse_error.what());
spdlog::error(err_msg);
std::invoke(auth_failed, err_msg);
return;
}
}
DoRead();
}
bool Session::IsAuthenticated() const { return authenticated_; }
void Session::DoShutdown() {
std::visit(utils::Overloaded{[this](SSLWebSocket &ssl_ws) {
boost::beast::get_lowest_layer(ssl_ws).expires_after(std::chrono::seconds(30));
ssl_ws.next_layer().async_shutdown(
[shared_this = shared_from_this()](boost::beast::error_code ec) {
if (ec) {
LogError(ec, "shutdown");
}
shared_this->DoClose();
});
},
[this](auto && /* plain_ws */) { DoClose(); }},
ws_);
}
} // namespace communication::websocket

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,29 +11,27 @@
#pragma once
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
#include <deque>
#include <memory>
#include <optional>
#include <variant>
#include <span>
#include <boost/asio/dispatch.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core/tcp_stream.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/websocket.hpp>
#include <json/json.hpp>
#include "communication/context.hpp"
#include "communication/websocket/auth.hpp"
#include "utils/result.hpp"
#include "utils/synchronized.hpp"
#include "utils/variant_helpers.hpp"
#include "communication/buffer.hpp"
#include "communication/session.hpp"
namespace communication::websocket {
class Session : public std::enable_shared_from_this<Session> {
void LogError(boost::beast::error_code ec, const std::string_view what) {
spdlog::warn("Websocket session failed on {}: {}", what, ec.message());
}
template <typename TSession, typename TSessionData>
class Session : public std::enable_shared_from_this<Session<TSession, TSessionData>> {
using tcp = boost::asio::ip::tcp;
public:
@@ -42,50 +40,118 @@ class Session : public std::enable_shared_from_this<Session> {
return std::shared_ptr<Session>{new Session{std::forward<Args>(args)...}};
}
bool Run();
void Write(std::shared_ptr<std::string> message);
bool IsConnected() const;
void Run() {
boost::asio::dispatch(strand_, [shared_this = this->shared_from_this()] { shared_this->OnRun(); });
}
private:
using PlainWebSocket = boost::beast::websocket::stream<boost::beast::tcp_stream>;
using SSLWebSocket = boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>;
explicit Session(tcp::socket &&socket, TSessionData *data)
: endpoint_(socket.local_endpoint()),
ws_(std::move(socket)),
strand_{boost::asio::make_strand(ws_.get_executor())},
data_{data} {}
explicit Session(tcp::socket &&socket, ServerContext &context, AuthenticationInterface &auth);
void OnRun() {
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
boost::asio::socket_base::keep_alive option(true);
ws_.set_option(boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::response_type &res) {
res.set(boost::beast::http::field::server, "Memgraph WS");
res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
}));
void DoWrite();
void OnWrite(boost::beast::error_code ec, size_t bytes_transferred);
ws_.binary(true);
// This buffer will hold the HTTP request as raw characters
void DoRead();
void OnRead(boost::beast::error_code ec, size_t bytes_transferred);
// This buffer is required for reading HTTP messages
// flat_buffer buffer;
void DoClose();
void OnClose(boost::beast::error_code ec);
// Read the HTTP request ourselves
// boost::beast::http::request<http::string_body> req;
// http::read(sock, buffer, req);
// std::cout << "bu
// Read into our buffer until we reach the end of the HTTP request.
// No parsing takes place here, we are just accumulating data.
bool IsAuthenticated() const;
// boost::beast::net::read_until(sock, net::dynamic_buffer(s), "\r\n\r\n");
utils::BasicResult<std::string> Authorize(const nlohmann::json &creds);
void DoShutdown();
auto GetExecutor() {
return std::visit(utils::Overloaded{[](auto &&ws) { return ws.get_executor(); }}, ws_);
// Now accept the connection, using the buffered data.
// ws_.accept(net::buffer(s));
ws_.async_accept(boost::asio::bind_executor(
strand_, [shared_this = this->shared_from_this()](auto ec) { shared_this->OnAccept(ec); }));
}
template <typename F>
decltype(auto) ExecuteForWebsocket(F &&fn) {
return std::visit(utils::Overloaded{std::forward<F>(fn)}, ws_);
void OnAccept(boost::beast::error_code ec) {
if (ec) {
return LogError(ec, "accept");
}
session_data_.emplace(this->shared_from_this());
// run on the strand
boost::asio::dispatch(strand_, [shared_this = this->shared_from_this()] { shared_this->DoRead(); });
}
std::variant<PlainWebSocket, SSLWebSocket> CreateWebSocket(tcp::socket &&socket, ServerContext &context);
void DoWrite(const uint8_t *data, size_t len, bool have_more = false) {
boost::beast::error_code ec;
ws_.write(boost::asio::const_buffer{data, len}, ec);
if (ec) {
return LogError(ec, "write");
}
}
std::optional<boost::asio::ssl::context> ssl_context_;
std::variant<PlainWebSocket, SSLWebSocket> ws_;
void DoRead() {
auto buf = session_data_->input_buffer_.write_end()->Allocate();
auto mutable_buffer = std::make_unique<boost::asio::mutable_buffer>(buf.data, buf.len);
ws_.async_read_some(
*mutable_buffer,
boost::asio::bind_executor(
strand_, [mutable_buffer = std::move(mutable_buffer), shared_this = this->shared_from_this()](
auto ec, auto bytes_transferred) { shared_this->OnRead(ec, bytes_transferred); }));
}
void OnRead(boost::beast::error_code ec, size_t bytes_transferred) {
if (ec == boost::beast::websocket::error::closed) {
return;
}
if (ec) {
return LogError(ec, "read");
}
session_data_->input_buffer_.write_end()->Written(bytes_transferred);
try {
session_data_->session_.Execute();
} catch (const SessionClosedException &e) {
spdlog::info("Closed connection");
return;
}
DoRead();
}
boost::asio::ip::tcp::endpoint endpoint_;
boost::beast::websocket::stream<boost::beast::tcp_stream> ws_;
boost::beast::flat_buffer buffer_;
std::deque<std::shared_ptr<std::string>> messages_;
boost::asio::strand<PlainWebSocket::executor_type> strand_;
std::atomic<bool> connected_{false};
bool authenticated_{false};
bool close_{false};
AuthenticationInterface &auth_;
boost::asio::strand<decltype(ws_)::executor_type> strand_;
TSessionData *data_;
struct SessionData {
explicit SessionData(std::shared_ptr<Session<TSession, TSessionData>> session)
: output_stream_([session](const uint8_t *data, size_t len, bool have_more) {
session->DoWrite(data, len, have_more);
return true;
}),
session_(session->data_,
io::network::Endpoint{session->endpoint_.address().to_string(), session->endpoint_.port()},
input_buffer_.read_end(), &output_stream_) {}
Buffer input_buffer_;
communication::OutputStream output_stream_;
TSession session_;
};
std::optional<SessionData> session_data_;
};
} // namespace communication::websocket

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,12 +51,6 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::AUTH;
case query::AuthQuery::Privilege::STREAM:
return auth::Permission::STREAM;
case query::AuthQuery::Privilege::MODULE_READ:
return auth::Permission::MODULE_READ;
case query::AuthQuery::Privilege::MODULE_WRITE:
return auth::Permission::MODULE_WRITE;
case query::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
}
}
} // namespace glue

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,14 +10,11 @@
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <string>
namespace integrations {
constexpr int64_t kDefaultCheckBatchLimit{1};
constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
constexpr std::chrono::milliseconds kMinimumInterval{1};
constexpr int64_t kMinimumSize{1};
const std::string kReducted{"<REDUCTED>"};
} // namespace integrations

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -126,18 +126,6 @@ Consumer::Consumer(ConsumerInfo info, ConsumerFunction consumer_function)
std::string error;
for (const auto &[key, value] : info_.public_configs) {
if (conf->set(key, value, error) != RdKafka::Conf::CONF_OK) {
throw SettingCustomConfigFailed(info_.consumer_name, error, key, value);
}
}
for (const auto &[key, value] : info_.private_configs) {
if (conf->set(key, value, error) != RdKafka::Conf::CONF_OK) {
throw SettingCustomConfigFailed(info_.consumer_name, error, key, kReducted);
}
}
if (conf->set("event_cb", this, error) != RdKafka::Conf::CONF_OK) {
throw ConsumerFailedToInitializeException(info_.consumer_name, error);
}
@@ -440,11 +428,11 @@ void Consumer::ConsumerRebalanceCb::rebalance_cb(RdKafka::KafkaConsumer *consume
}
auto maybe_error = consumer->assign(partitions);
if (maybe_error != RdKafka::ErrorCode::ERR_NO_ERROR) {
spdlog::warn("Assigning offset of consumer {} failed: {}", consumer_name_, RdKafka::err2str(maybe_error));
spdlog::warn("Assigning offset of consumer {} failed: {}", consumer_name_, RdKafka::err2str(err));
}
maybe_error = consumer->commitSync(partitions);
if (maybe_error != RdKafka::ErrorCode::ERR_NO_ERROR) {
spdlog::warn("Commiting offsets of consumer {} failed: {}", consumer_name_, RdKafka::err2str(maybe_error));
spdlog::warn("Commiting offsets of consumer {} failed: {}", consumer_name_, RdKafka::err2str(err));
}
}
void Consumer::ConsumerRebalanceCb::set_offset(int64_t offset) { offset_ = offset; }

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -86,8 +86,6 @@ struct ConsumerInfo {
std::string bootstrap_servers;
std::chrono::milliseconds batch_interval;
int64_t batch_size;
std::unordered_map<std::string, std::string> public_configs;
std::unordered_map<std::string, std::string> private_configs;
};
/// Memgraphs Kafka consumer wrapper.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,7 +11,7 @@
#pragma once
#include <string_view>
#include <string>
#include "utils/exceptions.hpp"
@@ -22,46 +22,37 @@ class KafkaStreamException : public utils::BasicException {
class ConsumerFailedToInitializeException : public KafkaStreamException {
public:
ConsumerFailedToInitializeException(const std::string_view consumer_name, const std::string_view error)
ConsumerFailedToInitializeException(const std::string &consumer_name, const std::string &error)
: KafkaStreamException("Failed to initialize Kafka consumer {} : {}", consumer_name, error) {}
};
class SettingCustomConfigFailed : public ConsumerFailedToInitializeException {
public:
SettingCustomConfigFailed(const std::string_view consumer_name, const std::string_view error,
const std::string_view key, const std::string_view value)
: ConsumerFailedToInitializeException(
consumer_name,
fmt::format(R"(failed to set custom config ("{}": "{}"), because of error {})", key, value, error)) {}
};
class ConsumerRunningException : public KafkaStreamException {
public:
explicit ConsumerRunningException(const std::string_view consumer_name)
explicit ConsumerRunningException(const std::string &consumer_name)
: KafkaStreamException("Kafka consumer {} is already running", consumer_name) {}
};
class ConsumerStoppedException : public KafkaStreamException {
public:
explicit ConsumerStoppedException(const std::string_view consumer_name)
explicit ConsumerStoppedException(const std::string &consumer_name)
: KafkaStreamException("Kafka consumer {} is already stopped", consumer_name) {}
};
class ConsumerCheckFailedException : public KafkaStreamException {
public:
explicit ConsumerCheckFailedException(const std::string_view consumer_name, const std::string_view error)
explicit ConsumerCheckFailedException(const std::string &consumer_name, const std::string &error)
: KafkaStreamException("Kafka consumer {} check failed: {}", consumer_name, error) {}
};
class ConsumerStartFailedException : public KafkaStreamException {
public:
explicit ConsumerStartFailedException(const std::string_view consumer_name, const std::string_view error)
explicit ConsumerStartFailedException(const std::string &consumer_name, const std::string &error)
: KafkaStreamException("Starting Kafka consumer {} failed: {}", consumer_name, error) {}
};
class TopicNotFoundException : public KafkaStreamException {
public:
TopicNotFoundException(const std::string_view consumer_name, const std::string_view topic_name)
TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
: KafkaStreamException("Kafka consumer {} cannot find topic {}", consumer_name, topic_name) {}
};
} // namespace integrations::kafka

View File

@@ -4,8 +4,5 @@ set(io_src_files
network/socket.cpp
network/utils.cpp)
find_package(fmt REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-io STATIC ${io_src_files})
target_link_libraries(mg-io stdc++fs Threads::Threads fmt::fmt mg-utils)

View File

@@ -1,10 +1,6 @@
find_package(gflags REQUIRED)
find_package(BZip2 REQUIRED)
find_package(ZLIB REQUIRED)
# STATIC library used to store key-value pairs
add_library(mg-kvstore STATIC kvstore.cpp)
target_link_libraries(mg-kvstore stdc++fs mg-utils rocksdb BZip2::BZip2 ZLIB::ZLIB gflags)
target_link_libraries(mg-kvstore stdc++fs mg-utils rocksdb bzip2 zlib gflags)
# STATIC library for dummy key-value storage
# add_library(mg-kvstore-dummy STATIC kvstore_dummy.cpp)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -29,11 +29,9 @@
#include <gflags/gflags.h>
#include <spdlog/common.h>
#include <spdlog/sinks/daily_file_sink.h>
#include <spdlog/sinks/dist_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include "communication/bolt/v1/constants.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "helpers.hpp"
#include "py/py.hpp"
@@ -130,15 +128,8 @@ std::optional<Enum> StringToEnum(const auto &value, const auto &mappings) {
// Bolt server flags.
DEFINE_string(bolt_address, "0.0.0.0", "IP address on which the Bolt server should listen.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(monitoring_address, "0.0.0.0",
"IP address on which the websocket server for Memgraph monitoring should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687, "Port on which the Bolt server should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(monitoring_port, 7444,
"Port on which the websocket server for Memgraph monitoring should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
DEFINE_VALIDATED_int32(bolt_num_workers, std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers used by the Bolt server. By default, this will be the "
"number of processing units available on the machine.",
@@ -354,49 +345,39 @@ DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
});
namespace {
spdlog::level::level_enum ParseLogLevel() {
void ParseLogLevel() {
const auto log_level = StringToEnum<spdlog::level::level_enum>(FLAGS_log_level, log_level_mappings);
MG_ASSERT(log_level, "Invalid log level");
return *log_level;
spdlog::set_level(*log_level);
}
// 5 weeks * 7 days
constexpr auto log_retention_count = 35;
void CreateLoggerFromSink(const auto &sinks, const auto log_level) {
auto logger = std::make_shared<spdlog::logger>("memgraph_log", sinks.begin(), sinks.end());
logger->set_level(log_level);
logger->flush_on(spdlog::level::trace);
spdlog::set_default_logger(std::move(logger));
}
void InitializeLogger() {
std::vector<spdlog::sink_ptr> sinks;
void ConfigureLogging() {
std::vector<spdlog::sink_ptr> loggers;
if (FLAGS_also_log_to_stderr) {
sinks.emplace_back(std::make_shared<spdlog::sinks::stderr_color_sink_mt>());
loggers.emplace_back(std::make_shared<spdlog::sinks::stderr_color_sink_mt>());
}
if (!FLAGS_log_file.empty()) {
// get local time
time_t current_time{0};
time_t current_time;
struct tm *local_time{nullptr};
time(&current_time);
local_time = localtime(&current_time);
sinks.emplace_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(
loggers.emplace_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(
FLAGS_log_file, local_time->tm_hour, local_time->tm_min, false, log_retention_count));
}
CreateLoggerFromSink(sinks, ParseLogLevel());
}
void AddLoggerSink(spdlog::sink_ptr new_sink) {
auto default_logger = spdlog::default_logger();
auto sinks = default_logger->sinks();
sinks.push_back(new_sink);
CreateLoggerFromSink(sinks, default_logger->level());
}
spdlog::set_default_logger(std::make_shared<spdlog::logger>("memgraph_log", loggers.begin(), loggers.end()));
spdlog::flush_on(spdlog::level::trace);
ParseLogLevel();
}
} // namespace
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
@@ -1007,7 +988,7 @@ int main(int argc, char **argv) {
LoadConfig("memgraph");
gflags::ParseCommandLineFlags(&argc, &argv, true);
InitializeLogger();
ConfigureLogging();
// Unhandled exception handler init.
std::set_terminate(&utils::TerminateHandler);
@@ -1166,7 +1147,7 @@ int main(int argc, char **argv) {
SessionData session_data{&db, &interpreter_context, &auth};
#endif
query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories, FLAGS_data_directory);
query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories);
query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectories();
AuthQueryHandler auth_handler(&auth, FLAGS_auth_user_or_role_name_regex);
@@ -1197,8 +1178,8 @@ int main(int argc, char **argv) {
spdlog::warn(utils::MessageWithLink("Using non-secure Bolt connection (without SSL).", "https://memgr.ph/ssl"));
}
ServerT server({FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)}, &session_data, &context,
FLAGS_bolt_session_inactivity_timeout, service_name, FLAGS_bolt_num_workers);
auto server = communication::websocket::Server<BoltSession, SessionData>{
{FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)}, &session_data};
// Setup telemetry
std::optional<telemetry::Telemetry> telemetry;
@@ -1220,13 +1201,8 @@ int main(int argc, char **argv) {
[]() -> nlohmann::json { return query::plan::CallProcedure::GetAndResetCounters(); });
}
communication::websocket::SafeAuth websocket_auth{&auth};
communication::websocket::Server websocket_server{
{FLAGS_monitoring_address, static_cast<uint16_t>(FLAGS_monitoring_port)}, &context, websocket_auth};
AddLoggerSink(websocket_server.GetLoggingSink());
// Handler for regular termination signals
auto shutdown = [&websocket_server, &server, &interpreter_context] {
auto shutdown = [&server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
// a race condition when a transaction is accepted during server shutdown.
server.Shutdown();
@@ -1234,17 +1210,11 @@ int main(int argc, char **argv) {
// connections we tell the execution engine to stop processing all pending
// queries.
query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
};
InitSignalHandlers(shutdown);
MG_ASSERT(server.Start(), "Couldn't start the Bolt server!");
websocket_server.Start();
server.Start();
server.AwaitShutdown();
websocket_server.AwaitShutdown();
query::procedure::gModuleRegistry.UnloadAllModules();
Py_END_ALLOW_THREADS;

View File

@@ -2,11 +2,9 @@ set(memory_src_files
new_delete.cpp
memory_control.cpp)
find_package(Jemalloc REQUIRED)
add_library(mg-memory STATIC ${memory_src_files})
target_link_libraries(mg-memory mg-utils fmt)
if (ENABLE_JEMALLOC)
target_link_libraries(mg-memory Jemalloc::Jemalloc)
target_link_libraries(mg-memory jemalloc)
endif()

View File

@@ -30,7 +30,6 @@ set(mg_query_sources
plan/rule_based_planner.cpp
plan/variable_start_planner.cpp
procedure/mg_procedure_impl.cpp
procedure/mg_procedure_helpers.cpp
procedure/module.cpp
procedure/py_module.cpp
serialization/property_value.cpp
@@ -48,9 +47,6 @@ add_dependencies(mg-query generate_lcp_query)
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query dl cppitertools Boost::headers)
target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,13 +11,23 @@
#pragma once
//////////////////////////////////////////////////////
// THIS INCLUDE SHOULD ALWAYS COME BEFORE THE
// "cypher_main_visitor.hpp"
// "planner.hpp" includes json.hpp which uses libc's
// EOF macro while "cypher_main_visitor.hpp" includes
// "antlr4-runtime.h" which contains a static variable
// of the same name, EOF.
// This hides the definition of the macro which causes
// the compilation to fail.
#include "query/plan/planner.hpp"
//////////////////////////////////////////////////////
#include "query/config.hpp"
#include "query/frontend/ast/cypher_main_visitor.hpp"
#include "query/frontend/opencypher/parser.hpp"
#include "query/frontend/semantic/required_privileges.hpp"
#include "query/frontend/semantic/symbol_generator.hpp"
#include "query/frontend/stripped.hpp"
#include "query/plan/planner.hpp"
#include "utils/flag_validation.hpp"
#include "utils/timer.hpp"

View File

@@ -217,11 +217,4 @@ class SettingConfigInMulticommandTxException final : public QueryException {
SettingConfigInMulticommandTxException()
: QueryException("Settings cannot be changed or fetched in multicommand transactions.") {}
};
class VersionInfoInMulticommandTxException : public QueryException {
public:
VersionInfoInMulticommandTxException()
: QueryException("Version info query not allowed in multicommand transactions.") {}
};
} // namespace query

View File

@@ -54,7 +54,9 @@ cpp<#
size_t size = 0;
slk::Load(&size, reader);
self->${member}.resize(size);
for (size_t i = 0; i < size; ++i) {
for (size_t i = 0;
i < size;
++i) {
self->${member}[i] = query::LoadAstPointer<query::${type}>(storage, reader);
}
cpp<#))
@@ -73,7 +75,9 @@ cpp<#
#>cpp
size_t size = 0;
slk::Load(&size, reader);
for (size_t i = 0; i < size; ++i) {
for (size_t i = 0;
i < size;
++i) {
query::PropertyIx key;
slk::Load(&key, reader, storage);
auto *value = query::LoadAstPointer<query::Expression>(storage, reader);
@@ -89,34 +93,6 @@ cpp<#
}
cpp<#)
(defun slk-save-expression-map (member)
#>cpp
size_t size = self.${member}.size();
slk::Save(size, builder);
for (const auto &entry : self.${member}) {
query::SaveAstPointer(entry.first, builder);
query::SaveAstPointer(entry.second, builder);
}
cpp<#)
(defun slk-load-expression-map (member)
#>cpp
size_t size = 0;
slk::Load(&size, reader);
for (size_t i = 0; i < size; ++i) {
auto *key = query::LoadAstPointer<query::Expression>(storage, reader);
auto *value = query::LoadAstPointer<query::Expression>(storage, reader);
self->${member}.emplace(key, value);
}
cpp<#)
(defun clone-expression-map (source dest)
#>cpp
for (const auto &[key, value] : ${source}) {
${dest}[key->Clone(storage)] = value->Clone(storage);
}
cpp<#)
(defun slk-load-name-ix (name-type)
(lambda (member)
#>cpp
@@ -1843,7 +1819,9 @@ cpp<#
size_t size = 0;
slk::Load(&size, reader);
self->${member}.resize(size);
for (size_t i = 0; i < size; ++i) {
for (size_t i = 0;
i < size;
++i) {
slk::Load(&self->${member}[i], reader, storage);
}
cpp<#)
@@ -2247,8 +2225,7 @@ cpp<#
(:serialize))
(lcp:define-enum privilege
(create delete match merge set remove index stats auth constraint
dump replication durability read_file free_memory trigger config stream module_read module_write
websocket)
dump replication durability read_file free_memory trigger config stream)
(:serialize))
#>cpp
AuthQuery() = default;
@@ -2288,9 +2265,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
AuthQuery::Privilege::READ_FILE,
AuthQuery::Privilege::DURABILITY,
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM,
AuthQuery::Privilege::MODULE_READ, AuthQuery::Privilege::MODULE_WRITE,
AuthQuery::Privilege::WEBSOCKET};
AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM};
cpp<#
(lcp:define-class info-query (query)
@@ -2556,7 +2531,7 @@ cpp<#
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(topic_names "std::variant<Expression*, std::vector<std::string>>" :initval "nullptr"
(topic_names "std::variant<Expression*, std::vector<std::string>>" :initval "nullptr"
:clone #'clone-variant-topic-names
:scope :public)
(consumer_group "std::string" :scope :public)
@@ -2566,17 +2541,7 @@ cpp<#
(service_url "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(configs "std::unordered_map<Expression *, Expression *>" :scope :public
:slk-save #'slk-save-expression-map
:slk-load #'slk-load-expression-map
:clone #'clone-expression-map)
(credentials "std::unordered_map<Expression *, Expression *>" :scope :public
:slk-save #'slk-save-expression-map
:slk-load #'slk-load-expression-map
:clone #'clone-expression-map))
:slk-load (slk-load-ast-pointer "Expression")))
(:public
(lcp:define-enum action
@@ -2618,12 +2583,4 @@ cpp<#
(:serialize (:slk))
(:clone))
(lcp:define-class version-query (query) ()
(:public
#>cpp
DEFVISITABLE(QueryVisitor<void>);
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:pop-namespace) ;; namespace query

View File

@@ -92,7 +92,6 @@ class IsolationLevelQuery;
class CreateSnapshotQuery;
class StreamQuery;
class SettingQuery;
class VersionQuery;
using TreeCompositeVisitor = ::utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@@ -124,9 +123,9 @@ class ExpressionVisitor
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch> {};
template <class TResult>
class QueryVisitor : public ::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery,
StreamQuery, SettingQuery, VersionQuery> {};
class QueryVisitor
: public ::utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery, InfoQuery,
ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery, FreeMemoryQuery,
TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery, StreamQuery, SettingQuery> {};
} // namespace query

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -9,6 +9,19 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
//////////////////////////////////////////////////////
// THIS INCLUDE SHOULD ALWAYS COME BEFORE THE
// "cypher_main_visitor.hpp"
// "module.hpp" includes json.hpp which uses libc's
// EOF macro while "cypher_main_visitor.hpp" includes
// "antlr4-runtime.h" which contains a static variable
// of the same name, EOF.
// This hides the definition of the macro which causes
// the compilation to fail.
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
#include "query/procedure/module.hpp"
//////////////////////////////////////////////////////
#include "query/frontend/ast/cypher_main_visitor.hpp"
#include <algorithm>
@@ -28,11 +41,8 @@
#include <boost/preprocessor/cat.hpp>
#include "query/exceptions.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
#include "query/frontend/parsing.hpp"
#include "query/interpret/awesome_memgraph_functions.hpp"
#include "query/procedure/module.hpp"
#include "query/stream/common.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
@@ -554,7 +564,7 @@ std::string_view ToString(const CommonStreamConfigKey key) {
__VA_ARGS__ \
};
GENERATE_STREAM_CONFIG_KEY_ENUM(Kafka, TOPICS, CONSUMER_GROUP, BOOTSTRAP_SERVERS, CONFIGS, CREDENTIALS);
GENERATE_STREAM_CONFIG_KEY_ENUM(Kafka, TOPICS, CONSUMER_GROUP, BOOTSTRAP_SERVERS);
std::string_view ToString(const KafkaConfigKey key) {
switch (key) {
@@ -564,10 +574,6 @@ std::string_view ToString(const KafkaConfigKey key) {
return "CONSUMER_GROUP";
case KafkaConfigKey::BOOTSTRAP_SERVERS:
return "BOOTSTRAP_SERVERS";
case KafkaConfigKey::CONFIGS:
return "CONFIGS";
case KafkaConfigKey::CREDENTIALS:
return "CREDENTIALS";
}
}
@@ -578,21 +584,6 @@ void MapCommonStreamConfigs(auto &memory, StreamQuery &stream_query) {
}
} // namespace
antlrcpp::Any CypherMainVisitor::visitConfigKeyValuePair(MemgraphCypher::ConfigKeyValuePairContext *ctx) {
MG_ASSERT(ctx->literal().size() == 2);
return std::pair{ctx->literal(0)->accept(this).as<Expression *>(), ctx->literal(1)->accept(this).as<Expression *>()};
}
antlrcpp::Any CypherMainVisitor::visitConfigMap(MemgraphCypher::ConfigMapContext *ctx) {
std::unordered_map<Expression *, Expression *> map;
for (auto *key_value_pair : ctx->configKeyValuePair()) {
// If the queries are cached, then only the stripped query is parsed, so the actual keys cannot be determined
// here. That means duplicates cannot be checked.
map.insert(key_value_pair->accept(this).as<std::pair<Expression *, Expression *>>());
}
return map;
}
antlrcpp::Any CypherMainVisitor::visitKafkaCreateStream(MemgraphCypher::KafkaCreateStreamContext *ctx) {
auto *stream_query = storage_->Create<StreamQuery>();
stream_query->action_ = StreamQuery::Action::CREATE_STREAM;
@@ -606,10 +597,6 @@ antlrcpp::Any CypherMainVisitor::visitKafkaCreateStream(MemgraphCypher::KafkaCre
MapConfig<true, std::vector<std::string>, Expression *>(memory_, KafkaConfigKey::TOPICS, stream_query->topic_names_);
MapConfig<false, std::string>(memory_, KafkaConfigKey::CONSUMER_GROUP, stream_query->consumer_group_);
MapConfig<false, Expression *>(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS, stream_query->bootstrap_servers_);
MapConfig<false, std::unordered_map<Expression *, Expression *>>(memory_, KafkaConfigKey::CONFIGS,
stream_query->configs_);
MapConfig<false, std::unordered_map<Expression *, Expression *>>(memory_, KafkaConfigKey::CREDENTIALS,
stream_query->credentials_);
MapCommonStreamConfigs(memory_, *stream_query);
@@ -645,33 +632,18 @@ antlrcpp::Any CypherMainVisitor::visitKafkaCreateStreamConfig(MemgraphCypher::Ka
if (ctx->TOPICS()) {
ThrowIfExists(memory_, KafkaConfigKey::TOPICS);
constexpr auto topics_key = static_cast<uint8_t>(KafkaConfigKey::TOPICS);
const auto topics_key = static_cast<uint8_t>(KafkaConfigKey::TOPICS);
GetTopicNames(memory_[topics_key], ctx->topicNames(), *this);
return {};
}
if (ctx->CONSUMER_GROUP()) {
ThrowIfExists(memory_, KafkaConfigKey::CONSUMER_GROUP);
constexpr auto consumer_group_key = static_cast<uint8_t>(KafkaConfigKey::CONSUMER_GROUP);
const auto consumer_group_key = static_cast<uint8_t>(KafkaConfigKey::CONSUMER_GROUP);
memory_[consumer_group_key] = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup);
return {};
}
if (ctx->CONFIGS()) {
ThrowIfExists(memory_, KafkaConfigKey::CONFIGS);
constexpr auto configs_key = static_cast<uint8_t>(KafkaConfigKey::CONFIGS);
memory_.emplace(configs_key, ctx->configsMap->accept(this).as<std::unordered_map<Expression *, Expression *>>());
return {};
}
if (ctx->CREDENTIALS()) {
ThrowIfExists(memory_, KafkaConfigKey::CREDENTIALS);
constexpr auto credentials_key = static_cast<uint8_t>(KafkaConfigKey::CREDENTIALS);
memory_.emplace(credentials_key,
ctx->credentialsMap->accept(this).as<std::unordered_map<Expression *, Expression *>>());
return {};
}
MG_ASSERT(ctx->BOOTSTRAP_SERVERS());
ThrowIfExists(memory_, KafkaConfigKey::BOOTSTRAP_SERVERS);
if (!ctx->bootstrapServers->StringLiteral()) {
@@ -870,12 +842,6 @@ antlrcpp::Any CypherMainVisitor::visitShowSettings(MemgraphCypher::ShowSettingsC
return setting_query;
}
antlrcpp::Any CypherMainVisitor::visitVersionQuery(MemgraphCypher::VersionQueryContext * /*ctx*/) {
auto *version_query = storage_->Create<VersionQuery>();
query_ = version_query;
return version_query;
}
antlrcpp::Any CypherMainVisitor::visitCypherUnion(MemgraphCypher::CypherUnionContext *ctx) {
bool distinct = !ctx->ALL();
auto *cypher_union = storage_->Create<CypherUnion>(distinct);
@@ -1091,7 +1057,7 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
if (!maybe_found) {
throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
}
call_proc->is_write_ = maybe_found->second->info.is_write;
call_proc->is_write_ = maybe_found->second->is_write_procedure;
auto *yield_ctx = ctx->yieldProcedureResults();
if (!yield_ctx) {
@@ -1330,9 +1296,6 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
if (ctx->CONFIG()) return AuthQuery::Privilege::CONFIG;
if (ctx->DURABILITY()) return AuthQuery::Privilege::DURABILITY;
if (ctx->STREAM()) return AuthQuery::Privilege::STREAM;
if (ctx->MODULE_READ()) return AuthQuery::Privilege::MODULE_READ;
if (ctx->MODULE_WRITE()) return AuthQuery::Privilege::MODULE_WRITE;
if (ctx->WEBSOCKET()) return AuthQuery::Privilege::WEBSOCKET;
LOG_FATAL("Should not get here - unknown privilege!");
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -269,16 +269,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitCreateStream(MemgraphCypher::CreateStreamContext *ctx) override;
/**
* @return StreamQuery*
*/
antlrcpp::Any visitConfigKeyValuePair(MemgraphCypher::ConfigKeyValuePairContext *ctx) override;
/**
* @return StreamQuery*
*/
antlrcpp::Any visitConfigMap(MemgraphCypher::ConfigMapContext *ctx) override;
/**
* @return StreamQuery*
*/
@@ -359,11 +349,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitShowSettings(MemgraphCypher::ShowSettingsContext *ctx) override;
/**
* @return VersionQuery*
*/
antlrcpp::Any visitVersionQuery(MemgraphCypher::VersionQueryContext *ctx) override;
/**
* @return CypherUnion*
*/
@@ -864,9 +849,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
ParsingContext context_;
AstStorage *storage_;
std::unordered_map<uint8_t, std::variant<Expression *, std::string, std::vector<std::string>,
std::unordered_map<Expression *, Expression *>>>
memory_;
std::unordered_map<uint8_t, std::variant<Expression *, std::string, std::vector<std::string>>> memory_;
// Set of identifiers from queries.
std::unordered_set<std::string> users_identifiers;
// Identifiers that user didn't name.

View File

@@ -35,9 +35,7 @@ memgraphCypherKeyword : cypherKeyword
| COMMIT
| COMMITTED
| CONFIG
| CONFIGS
| CONSUMER_GROUP
| CREDENTIALS
| CSV
| DATA
| DELIMITER
@@ -54,7 +52,6 @@ memgraphCypherKeyword : cypherKeyword
| HEADER
| IDENTIFIED
| ISOLATION
| KAFKA
| LEVEL
| LOAD
| LOCK
@@ -63,7 +60,6 @@ memgraphCypherKeyword : cypherKeyword
| NEXT
| NO
| PASSWORD
| PULSAR
| PORT
| PRIVILEGES
| READ
@@ -96,7 +92,6 @@ memgraphCypherKeyword : cypherKeyword
| UPDATE
| USER
| USERS
| VERSION
;
symbolicName : UnescapedSymbolicName
@@ -120,7 +115,6 @@ query : cypherQuery
| createSnapshotQuery
| streamQuery
| settingQuery
| versionQuery
;
authQuery : createRole
@@ -239,9 +233,6 @@ privilege : CREATE
| CONFIG
| DURABILITY
| STREAM
| MODULE_READ
| MODULE_WRITE
| WEBSOCKET
;
privilegeList : privilege ( ',' privilege )* ;
@@ -315,15 +306,9 @@ commonCreateStreamConfig : TRANSFORM transformationName=procedureName
createStream : kafkaCreateStream | pulsarCreateStream ;
configKeyValuePair : literal ':' literal ;
configMap : '{' ( configKeyValuePair ( ',' configKeyValuePair )* )? '}' ;
kafkaCreateStreamConfig : TOPICS topicNames
| CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus
| BOOTSTRAP_SERVERS bootstrapServers=literal
| CONFIGS configsMap=configMap
| CREDENTIALS credentialsMap=configMap
| commonCreateStreamConfig
;
@@ -360,5 +345,3 @@ setSetting : SET DATABASE SETTING settingName TO settingValue ;
showSetting : SHOW DATABASE SETTING settingName ;
showSettings : SHOW DATABASE SETTINGS ;
versionQuery : SHOW VERSION ;

View File

@@ -40,9 +40,7 @@ CLEAR : C L E A R ;
COMMIT : C O M M I T ;
COMMITTED : C O M M I T T E D ;
CONFIG : C O N F I G ;
CONFIGS : C O N F I G S;
CONSUMER_GROUP : C O N S U M E R UNDERSCORE G R O U P ;
CREDENTIALS : C R E D E N T I A L S ;
CSV : C S V ;
DATA : D A T A ;
DELIMITER : D E L I M I T E R ;
@@ -70,8 +68,6 @@ LOAD : L O A D ;
LOCK : L O C K ;
MAIN : M A I N ;
MODE : M O D E ;
MODULE_READ : M O D U L E UNDERSCORE R E A D ;
MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ;
NEXT : N E X T ;
NO : N O ;
PASSWORD : P A S S W O R D ;
@@ -111,5 +107,3 @@ UNLOCK : U N L O C K ;
UPDATE : U P D A T E ;
USER : U S E R ;
USERS : U S E R S ;
VERSION : V E R S I O N ;
WEBSOCKET : W E B S O C K E T ;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,8 +11,6 @@
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
#include "query/procedure/module.hpp"
#include "utils/memory.hpp"
namespace query {
@@ -78,18 +76,12 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
void Visit(SettingQuery & /*setting_query*/) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
void Visit(VersionQuery & /*version_query*/) override { AddPrivilege(AuthQuery::Privilege::STATS); }
bool PreVisit(Create & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::CREATE);
return false;
}
bool PreVisit(CallProcedure &procedure) override {
const auto maybe_proc =
procedure::FindProcedure(procedure::gModuleRegistry, procedure.procedure_name_, utils::NewDeleteResource());
if (maybe_proc && maybe_proc->second->info.required_privilege) {
AddPrivilege(*maybe_proc->second->info.required_privilege);
}
bool PreVisit(CallProcedure & /*unused*/) override {
// TODO: Corresponding privilege
return false;
}
bool PreVisit(Delete & /*unused*/) override {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -202,9 +202,7 @@ const trie::Trie kKeywords = {"union",
"bootstrap_servers",
"kafka",
"pulsar",
"service_url",
"version",
"websocket"};
"service_url"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -1080,7 +1080,7 @@ void MapNumericParameters(auto &parameter_mappings, const auto &input_parameters
TypedValue Date(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
FType<Optional<Or<String, Map>>>("date", args, nargs);
if (nargs == 0) {
return TypedValue(utils::LocalDateTime(ctx.timestamp).date, ctx.memory);
return TypedValue(utils::UtcToday(), ctx.memory);
}
if (args[0].IsString()) {
@@ -1103,7 +1103,7 @@ TypedValue LocalTime(const TypedValue *args, int64_t nargs, const FunctionContex
FType<Optional<Or<String, Map>>>("localtime", args, nargs);
if (nargs == 0) {
return TypedValue(utils::LocalDateTime(ctx.timestamp).local_time, ctx.memory);
return TypedValue(utils::UtcLocalTime(), ctx.memory);
}
if (args[0].IsString()) {
@@ -1130,7 +1130,7 @@ TypedValue LocalDateTime(const TypedValue *args, int64_t nargs, const FunctionCo
FType<Optional<Or<String, Map>>>("localdatetime", args, nargs);
if (nargs == 0) {
return TypedValue(utils::LocalDateTime(ctx.timestamp), ctx.memory);
return TypedValue(utils::UtcLocalDateTime(), ctx.memory);
}
if (args[0].IsString()) {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -564,26 +564,10 @@ Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, Exp
}
auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator);
const auto get_config_map = [&evaluator](std::unordered_map<Expression *, Expression *> map,
std::string_view map_name) -> std::unordered_map<std::string, std::string> {
std::unordered_map<std::string, std::string> config_map;
for (const auto [key_expr, value_expr] : map) {
const auto key = key_expr->Accept(evaluator);
const auto value = value_expr->Accept(evaluator);
if (!key.IsString() || !value.IsString()) {
throw SemanticException("{} must contain only string keys and values!", map_name);
}
config_map.emplace(key.ValueString(), value.ValueString());
}
return config_map;
};
return [interpreter_context, stream_name = stream_query->stream_name_,
topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
consumer_group = std::move(consumer_group), common_stream_info = std::move(common_stream_info),
bootstrap_servers = std::move(bootstrap), owner = StringPointerToOptional(username),
configs = get_config_map(stream_query->configs_, "Configs"),
credentials = get_config_map(stream_query->credentials_, "Credentials")]() mutable {
bootstrap_servers = std::move(bootstrap), owner = StringPointerToOptional(username)]() mutable {
std::string bootstrap = bootstrap_servers
? std::move(*bootstrap_servers)
: std::string{interpreter_context->config.default_kafka_bootstrap_servers};
@@ -591,9 +575,7 @@ Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, Exp
{.common_info = std::move(common_stream_info),
.topics = std::move(topic_names),
.consumer_group = std::move(consumer_group),
.bootstrap_servers = std::move(bootstrap),
.configs = std::move(configs),
.credentials = std::move(credentials)},
.bootstrap_servers = std::move(bootstrap)},
std::move(owner));
return std::vector<std::vector<TypedValue>>{};
@@ -1725,24 +1707,6 @@ PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explic
// NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks)
}
PreparedQuery PrepareVersionQuery(ParsedQuery parsed_query, const bool in_explicit_transaction) {
if (in_explicit_transaction) {
throw VersionInfoInMulticommandTxException();
}
return PreparedQuery{{"version"},
std::move(parsed_query.required_privileges),
[](AnyStream *stream, std::optional<int> /*n*/) {
std::vector<TypedValue> version_value;
version_value.reserve(1);
version_value.emplace_back(gflags::VersionString());
stream->Result(version_value);
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
}
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
storage::Storage *db, utils::MemoryResource *execution_memory) {
@@ -2146,8 +2110,6 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
} else if (utils::Downcast<SettingQuery>(parsed_query.query)) {
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_);
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
} else {
LOG_FATAL("Should not get here -- unknown query type!");
}
@@ -2314,7 +2276,7 @@ void Interpreter::Commit() {
reset_necessary_members();
SPDLOG_DEBUG("Finished committing the transaction");
SPDLOG_DEBUG("Finished comitting the transaction");
}
void Interpreter::AdvanceCommand() {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -2107,7 +2107,7 @@ template <typename T>
concept AccessorWithProperties = requires(T value, storage::PropertyId property_id,
storage::PropertyValue property_value) {
{ value.ClearProperties() } -> std::same_as<storage::Result<std::map<storage::PropertyId, storage::PropertyValue>>>;
{value.SetProperty(property_id, property_value)};
{ value.SetProperty(property_id, property_value) };
};
/// Helper function that sets the given values on either a Vertex or an Edge.
@@ -2599,8 +2599,8 @@ namespace {
TypedValue DefaultAggregationOpValue(const Aggregate::Element &element, utils::MemoryResource *memory) {
switch (element.op) {
case Aggregation::Op::COUNT:
case Aggregation::Op::SUM:
return TypedValue(0, memory);
case Aggregation::Op::SUM:
case Aggregation::Op::MIN:
case Aggregation::Op::MAX:
case Aggregation::Op::AVG:
@@ -3813,13 +3813,13 @@ class CallProcedureCursor : public Cursor {
throw QueryRuntimeException("There is no procedure named '{}'.", self_->procedure_name_);
}
const auto &[module, proc] = *maybe_found;
if (proc->info.is_write != self_->is_write_) {
if (proc->is_write_procedure != self_->is_write_) {
auto get_proc_type_str = [](bool is_write) { return is_write ? "write" : "read"; };
throw QueryRuntimeException("The procedure named '{}' was a {} procedure, but changed to be a {} procedure.",
self_->procedure_name_, get_proc_type_str(self_->is_write_),
get_proc_type_str(proc->info.is_write));
get_proc_type_str(proc->is_write_procedure));
}
const auto graph_view = proc->info.is_write ? storage::View::NEW : storage::View::OLD;
const auto graph_view = proc->is_write_procedure ? storage::View::NEW : storage::View::OLD;
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor,
graph_view);

View File

@@ -1,36 +0,0 @@
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/procedure/mg_procedure_helpers.hpp"
namespace query::procedure {
MgpUniquePtr<mgp_value> GetStringValueOrSetError(const char *string, mgp_memory *memory, mgp_result *result) {
procedure::MgpUniquePtr<mgp_value> value{nullptr, mgp_value_destroy};
const auto success =
TryOrSetError([&] { return procedure::CreateMgpObject(value, mgp_value_make_string, string, memory); }, result);
if (!success) {
value.reset();
}
return value;
}
bool InsertResultOrSetError(mgp_result *result, mgp_result_record *record, const char *result_name, mgp_value *value) {
if (const auto err = mgp_result_record_insert(record, result_name, value); err != MGP_ERROR_NO_ERROR) {
const auto error_msg = fmt::format("Unable to set the result for {}, error = {}", result_name, err);
static_cast<void>(mgp_result_set_error_msg(result, error_msg.c_str()));
return false;
}
return true;
}
} // namespace query::procedure

View File

@@ -15,8 +15,6 @@
#include <type_traits>
#include <utility>
#include <fmt/format.h>
#include "mg_procedure.h"
namespace query::procedure {
@@ -47,23 +45,4 @@ mgp_error CreateMgpObject(MgpUniquePtr<TObj> &obj, TFunc func, TArgs &&...args)
obj.reset(raw_obj);
return err;
}
template <typename Fun>
[[nodiscard]] bool TryOrSetError(Fun &&func, mgp_result *result) {
if (const auto err = func(); err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return false;
} else if (err != MGP_ERROR_NO_ERROR) {
const auto error_msg = fmt::format("Unexpected error ({})!", err);
static_cast<void>(mgp_result_set_error_msg(result, error_msg.c_str()));
return false;
}
return true;
}
[[nodiscard]] MgpUniquePtr<mgp_value> GetStringValueOrSetError(const char *string, mgp_memory *memory,
mgp_result *result);
[[nodiscard]] bool InsertResultOrSetError(mgp_result *result, mgp_result_record *record, const char *result_name,
mgp_value *value);
} // namespace query::procedure

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -1178,7 +1178,7 @@ mgp_error mgp_date_timestamp(mgp_date *date, int64_t *timestamp) {
}
mgp_error mgp_date_now(mgp_memory *memory, mgp_date **date) {
return WrapExceptions([memory] { return NewRawMgpObject<mgp_date>(memory, utils::CurrentDate()); }, date);
return WrapExceptions([memory] { return NewRawMgpObject<mgp_date>(memory, utils::UtcToday()); }, date);
}
mgp_error mgp_date_add_duration(mgp_date *date, mgp_duration *dur, mgp_memory *memory, mgp_date **result) {
@@ -1241,7 +1241,7 @@ mgp_error mgp_local_time_timestamp(mgp_local_time *local_time, int64_t *timestam
}
mgp_error mgp_local_time_now(mgp_memory *memory, mgp_local_time **local_time) {
return WrapExceptions([memory] { return NewRawMgpObject<mgp_local_time>(memory, utils::CurrentLocalTime()); },
return WrapExceptions([memory] { return NewRawMgpObject<mgp_local_time>(memory, utils::UtcLocalTime()); },
local_time);
}
@@ -1334,9 +1334,8 @@ mgp_error mgp_local_date_time_timestamp(mgp_local_date_time *local_date_time, in
}
mgp_error mgp_local_date_time_now(mgp_memory *memory, mgp_local_date_time **local_date_time) {
return WrapExceptions(
[memory] { return NewRawMgpObject<mgp_local_date_time>(memory, utils::CurrentLocalDateTime()); },
local_date_time);
return WrapExceptions([memory] { return NewRawMgpObject<mgp_local_date_time>(memory, utils::UtcLocalDateTime()); },
local_date_time);
}
mgp_error mgp_local_date_time_add_duration(mgp_local_date_time *local_date_time, mgp_duration *dur, mgp_memory *memory,
@@ -2353,8 +2352,7 @@ mgp_error mgp_type_nullable(mgp_type *type, mgp_type **result) {
}
namespace {
mgp_proc *mgp_module_add_procedure(mgp_module *module, const char *name, mgp_proc_cb cb,
const ProcedureInfo &procedure_info) {
mgp_proc *mgp_module_add_procedure(mgp_module *module, const char *name, mgp_proc_cb cb, bool is_write_procedure) {
if (!IsValidIdentifierName(name)) {
throw std::invalid_argument{fmt::format("Invalid procedure name: {}", name)};
}
@@ -2364,16 +2362,16 @@ mgp_proc *mgp_module_add_procedure(mgp_module *module, const char *name, mgp_pro
auto *memory = module->procedures.get_allocator().GetMemoryResource();
// May throw std::bad_alloc, std::length_error
return &module->procedures.emplace(name, mgp_proc(name, cb, memory, procedure_info)).first->second;
return &module->procedures.emplace(name, mgp_proc(name, cb, memory, is_write_procedure)).first->second;
}
} // namespace
mgp_error mgp_module_add_read_procedure(mgp_module *module, const char *name, mgp_proc_cb cb, mgp_proc **result) {
return WrapExceptions([=] { return mgp_module_add_procedure(module, name, cb, {.is_write = false}); }, result);
return WrapExceptions([=] { return mgp_module_add_procedure(module, name, cb, false); }, result);
}
mgp_error mgp_module_add_write_procedure(mgp_module *module, const char *name, mgp_proc_cb cb, mgp_proc **result) {
return WrapExceptions([=] { return mgp_module_add_procedure(module, name, cb, {.is_write = true}); }, result);
return WrapExceptions([=] { return mgp_module_add_procedure(module, name, cb, true); }, result);
}
mgp_error mgp_proc_add_arg(mgp_proc *proc, const char *name, mgp_type *type) {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -23,7 +23,6 @@
#include "integrations/pulsar/consumer.hpp"
#include "query/context.hpp"
#include "query/db_accessor.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/procedure/cypher_type_ptr.hpp"
#include "query/typed_value.hpp"
#include "storage/v2/view.hpp"
@@ -654,29 +653,40 @@ struct mgp_type {
query::procedure::CypherTypePtr impl;
};
struct ProcedureInfo {
bool is_write = false;
std::optional<query::AuthQuery::Privilege> required_privilege = std::nullopt;
};
struct mgp_proc {
using allocator_type = utils::Allocator<mgp_proc>;
/// @throw std::bad_alloc
/// @throw std::length_error
mgp_proc(const char *name, mgp_proc_cb cb, utils::MemoryResource *memory, const ProcedureInfo &info = {})
: name(name, memory), cb(cb), args(memory), opt_args(memory), results(memory), info(info) {}
mgp_proc(const char *name, mgp_proc_cb cb, utils::MemoryResource *memory, bool is_write_procedure)
: name(name, memory),
cb(cb),
args(memory),
opt_args(memory),
results(memory),
is_write_procedure(is_write_procedure) {}
/// @throw std::bad_alloc
/// @throw std::length_error
mgp_proc(const char *name, std::function<void(mgp_list *, mgp_graph *, mgp_result *, mgp_memory *)> cb,
utils::MemoryResource *memory, const ProcedureInfo &info = {})
: name(name, memory), cb(cb), args(memory), opt_args(memory), results(memory), info(info) {}
utils::MemoryResource *memory, bool is_write_procedure)
: name(name, memory),
cb(cb),
args(memory),
opt_args(memory),
results(memory),
is_write_procedure(is_write_procedure) {}
/// @throw std::bad_alloc
/// @throw std::length_error
mgp_proc(const std::string_view name, std::function<void(mgp_list *, mgp_graph *, mgp_result *, mgp_memory *)> cb,
utils::MemoryResource *memory, const ProcedureInfo &info = {})
: name(name, memory), cb(cb), args(memory), opt_args(memory), results(memory), info(info) {}
utils::MemoryResource *memory, bool is_write_procedure)
: name(name, memory),
cb(cb),
args(memory),
opt_args(memory),
results(memory),
is_write_procedure(is_write_procedure) {}
/// @throw std::bad_alloc
/// @throw std::length_error
@@ -686,7 +696,7 @@ struct mgp_proc {
args(other.args, memory),
opt_args(other.opt_args, memory),
results(other.results, memory),
info(other.info) {}
is_write_procedure(other.is_write_procedure) {}
mgp_proc(mgp_proc &&other, utils::MemoryResource *memory)
: name(std::move(other.name), memory),
@@ -694,7 +704,7 @@ struct mgp_proc {
args(std::move(other.args), memory),
opt_args(std::move(other.opt_args), memory),
results(std::move(other.results), memory),
info(other.info) {}
is_write_procedure(other.is_write_procedure) {}
mgp_proc(const mgp_proc &other) = default;
mgp_proc(mgp_proc &&other) = default;
@@ -714,7 +724,7 @@ struct mgp_proc {
utils::pmr::vector<std::tuple<utils::pmr::string, const query::procedure::CypherType *, query::TypedValue>> opt_args;
/// Fields this procedure returns, as a (name -> (type, is_deprecated)) map.
utils::pmr::map<utils::pmr::string, std::pair<const query::procedure::CypherType *, bool>> results;
ProcedureInfo info;
bool is_write_procedure{false};
};
struct mgp_trans {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,23 +10,20 @@
// licenses/APL.txt.
#include "query/procedure/module.hpp"
#include <filesystem>
#include <optional>
#include "utils/memory.hpp"
extern "C" {
#include <dlfcn.h>
}
#include <fmt/format.h>
#include <unistd.h>
#include <optional>
#include "fmt/format.h"
#include "py/py.hpp"
#include "query/procedure/mg_procedure_helpers.hpp"
#include "query/procedure/py_module.hpp"
#include "utils/file.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/message.hpp"
#include "utils/pmr/vector.hpp"
#include "utils/string.hpp"
@@ -56,8 +53,6 @@ class BuiltinModule final : public Module {
void AddTransformation(std::string_view name, mgp_trans trans);
std::optional<std::filesystem::path> Path() const override { return std::nullopt; }
private:
/// Registered procedures
std::map<std::string, mgp_proc, std::less<>> procedures_;
@@ -84,12 +79,6 @@ void BuiltinModule::AddTransformation(std::string_view name, mgp_trans trans) {
namespace {
auto WithUpgradedLock(auto *lock, const auto &function) {
lock->unlock_shared();
utils::OnScopeExit shared_lock{[&] { lock->lock_shared(); }};
function();
};
void RegisterMgLoad(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) {
// Loading relies on the fact that regular procedure invocation through
// CallProcedureCursor::Pull takes ModuleRegistry::lock_ with READ access. To
@@ -103,19 +92,31 @@ void RegisterMgLoad(ModuleRegistry *module_registry, utils::RWLock *lock, Builti
// single thread may only take either a READ or a WRITE lock, it's not
// possible for a thread to hold both. If a thread tries to do that, it will
// deadlock immediately (no other thread needs to do anything).
auto load_all_cb = [module_registry, lock](mgp_list * /*args*/, mgp_graph * /*graph*/, mgp_result * /*result*/,
mgp_memory * /*memory*/) {
WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); });
auto with_unlock_shared = [lock](const auto &load_function) {
lock->unlock_shared();
try {
load_function();
// There's no finally in C++, but we have to return our original READ lock
// state in any possible case.
} catch (...) {
lock->lock_shared();
throw;
}
lock->lock_shared();
};
mgp_proc load_all("load_all", load_all_cb, utils::NewDeleteResource());
auto load_all_cb = [module_registry, with_unlock_shared](mgp_list * /*args*/, mgp_graph * /*graph*/,
mgp_result * /*result*/, mgp_memory * /*memory*/) {
with_unlock_shared([&]() { module_registry->UnloadAndLoadModulesFromDirectories(); });
};
mgp_proc load_all("load_all", load_all_cb, utils::NewDeleteResource(), false);
module->AddProcedure("load_all", std::move(load_all));
auto load_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result,
mgp_memory * /*memory*/) {
auto load_cb = [module_registry, with_unlock_shared](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result,
mgp_memory * /*memory*/) {
MG_ASSERT(Call<size_t>(mgp_list_size, args) == 1U, "Should have been type checked already");
auto *arg = Call<mgp_value *>(mgp_list_at, args, 0);
MG_ASSERT(CallBool(mgp_value_is_string, arg), "Should have been type checked already");
bool succ = false;
WithUpgradedLock(lock, [&]() {
with_unlock_shared([&]() {
const char *arg_as_string{nullptr};
if (const auto err = mgp_value_get_string(arg, &arg_as_string); err != MGP_ERROR_NO_ERROR) {
succ = false;
@@ -127,25 +128,11 @@ void RegisterMgLoad(ModuleRegistry *module_registry, utils::RWLock *lock, Builti
MG_ASSERT(mgp_result_set_error_msg(result, "Failed to (re)load the module.") == MGP_ERROR_NO_ERROR);
}
};
mgp_proc load("load", load_cb, utils::NewDeleteResource());
mgp_proc load("load", load_cb, utils::NewDeleteResource(), false);
MG_ASSERT(mgp_proc_add_arg(&load, "module_name", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("load", std::move(load));
}
namespace {
[[nodiscard]] bool IsFileEditable(const std::optional<std::filesystem::path> &path) {
return path && access(path->c_str(), W_OK) == 0;
}
std::string GetPathString(const std::optional<std::filesystem::path> &path) {
if (!path) {
return "builtin";
}
return std::filesystem::canonical(*path).generic_string();
}
} // namespace
void RegisterMgProcedures(
// We expect modules to be sorted by name.
const std::map<std::string, std::unique_ptr<Module>, std::less<>> *all_modules, BuiltinModule *module) {
@@ -160,82 +147,66 @@ void RegisterMgProcedures(
static_assert(
std::is_same_v<decltype(module->Procedures()), const std::map<std::string, mgp_proc, std::less<>> *>,
"Expected module procedures to be sorted by name");
const auto path = module->Path();
const auto path_string = GetPathString(path);
const auto is_editable = IsFileEditable(path);
for (const auto &[proc_name, proc] : *module->Procedures()) {
mgp_result_record *record{nullptr};
if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
if (const auto err = mgp_result_new_record(result, &record); err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return;
}
const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result);
if (!path_value) {
return;
}
MgpUniquePtr<mgp_value> is_editable_value{nullptr, mgp_value_destroy};
if (!TryOrSetError([&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); },
result)) {
} else if (err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unexpected error"));
return;
}
utils::pmr::string full_name(module_name, memory->impl);
full_name.append(1, '.');
full_name.append(proc_name);
const auto name_value = GetStringValueOrSetError(full_name.c_str(), memory, result);
if (!name_value) {
MgpUniquePtr<mgp_value> name_value{nullptr, mgp_value_destroy};
if (const auto err = CreateMgpObject(name_value, mgp_value_make_string, full_name.c_str(), memory);
err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return;
} else if (err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unexpected error"));
return;
}
std::stringstream ss;
ss << module_name << ".";
PrintProcSignature(proc, &ss);
const auto signature = ss.str();
const auto signature_value = GetStringValueOrSetError(signature.c_str(), memory, result);
if (!signature_value) {
MgpUniquePtr<mgp_value> signature_value{nullptr, mgp_value_destroy};
if (const auto err = CreateMgpObject(signature_value, mgp_value_make_string, signature.c_str(), memory);
err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return;
} else if (err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unexpected error"));
return;
}
MgpUniquePtr<mgp_value> is_write_value{nullptr, mgp_value_destroy};
if (!TryOrSetError(
[&, &proc = proc] {
return CreateMgpObject(is_write_value, mgp_value_make_bool, proc.info.is_write ? 1 : 0, memory);
},
result)) {
if (const auto err =
CreateMgpObject(is_write_value, mgp_value_make_bool, proc.is_write_procedure ? 1 : 0, memory);
err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return;
} else if (err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unexpected error"));
return;
}
if (!InsertResultOrSetError(result, record, "name", name_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "signature", signature_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "is_write", is_write_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "path", path_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) {
const auto err1 = mgp_result_record_insert(record, "name", name_value.get());
const auto err2 = mgp_result_record_insert(record, "signature", signature_value.get());
const auto err3 = mgp_result_record_insert(record, "is_write", is_write_value.get());
if (err1 != MGP_ERROR_NO_ERROR || err2 != MGP_ERROR_NO_ERROR || err3 != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unable to set the result!"));
return;
}
}
}
};
mgp_proc procedures("procedures", procedures_cb, utils::NewDeleteResource());
mgp_proc procedures("procedures", procedures_cb, utils::NewDeleteResource(), false);
MG_ASSERT(mgp_proc_add_result(&procedures, "name", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&procedures, "signature", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&procedures, "is_write", Call<mgp_type *>(mgp_type_bool)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&procedures, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&procedures, "is_editable", Call<mgp_type *>(mgp_type_bool)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("procedures", std::move(procedures));
}
@@ -248,25 +219,13 @@ void RegisterMgTransformations(const std::map<std::string, std::unique_ptr<Modul
static_assert(
std::is_same_v<decltype(module->Transformations()), const std::map<std::string, mgp_trans, std::less<>> *>,
"Expected module transformations to be sorted by name");
const auto path = module->Path();
const auto path_string = GetPathString(path);
const auto is_editable = IsFileEditable(path);
for (const auto &[trans_name, proc] : *module->Transformations()) {
mgp_result_record *record{nullptr};
if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
if (const auto err = mgp_result_new_record(result, &record); err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return;
}
const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result);
if (!path_value) {
return;
}
MgpUniquePtr<mgp_value> is_editable_value{nullptr, mgp_value_destroy};
if (!TryOrSetError([&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); },
result)) {
} else if (err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unexpected error"));
return;
}
@@ -274,377 +233,28 @@ void RegisterMgTransformations(const std::map<std::string, std::unique_ptr<Modul
full_name.append(1, '.');
full_name.append(trans_name);
const auto name_value = GetStringValueOrSetError(full_name.c_str(), memory, result);
if (!name_value) {
MgpUniquePtr<mgp_value> name_value{nullptr, mgp_value_destroy};
if (const auto err = CreateMgpObject(name_value, mgp_value_make_string, full_name.c_str(), memory);
err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return;
} else if (err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unexpected error"));
return;
}
if (!InsertResultOrSetError(result, record, "name", name_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "path", path_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) {
if (const auto err = mgp_result_record_insert(record, "name", name_value.get()); err != MGP_ERROR_NO_ERROR) {
static_cast<void>(mgp_result_set_error_msg(result, "Unable to set the result!"));
return;
}
}
}
};
mgp_proc procedures("transformations", transformations_cb, utils::NewDeleteResource());
mgp_proc procedures("transformations", transformations_cb, utils::NewDeleteResource(), false);
MG_ASSERT(mgp_proc_add_result(&procedures, "name", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&procedures, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&procedures, "is_editable", Call<mgp_type *>(mgp_type_bool)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("transformations", std::move(procedures));
}
namespace {
bool IsAllowedExtension(const auto &extension) {
constexpr std::array<std::string_view, 1> allowed_extensions{".py"};
return std::any_of(allowed_extensions.begin(), allowed_extensions.end(),
[&](const auto allowed_extension) { return allowed_extension == extension; });
}
bool IsSubPath(const auto &base, const auto &destination) {
const auto relative = std::filesystem::relative(destination, base);
return !relative.empty() && *relative.begin() != "..";
}
std::optional<std::string> ReadFile(const auto &path) {
std::ifstream file(path);
if (!file.is_open()) {
return std::nullopt;
}
const auto size = std::filesystem::file_size(path);
std::string content(size, '\0');
file.read(content.data(), static_cast<std::streamsize>(size));
return std::move(content);
}
// Return the module directory that contains the `path`
utils::BasicResult<const char *, std::filesystem::path> ParentModuleDirectory(const ModuleRegistry &module_registry,
const std::filesystem::path &path) {
const auto &module_directories = module_registry.GetModulesDirectory();
auto longest_parent_directory = module_directories.end();
auto max_length = std::numeric_limits<uint64_t>::min();
for (auto it = module_directories.begin(); it != module_directories.end(); ++it) {
if (IsSubPath(*it, path)) {
const auto length = std::filesystem::canonical(*it).string().size();
if (length > max_length) {
longest_parent_directory = it;
max_length = length;
}
}
}
if (longest_parent_directory == module_directories.end()) {
return "The specified file isn't contained in any of the module directories.";
}
return *longest_parent_directory;
}
} // namespace
void RegisterMgGetModuleFiles(ModuleRegistry *module_registry, BuiltinModule *module) {
auto get_module_files_cb = [module_registry](mgp_list * /*args*/, mgp_graph * /*unused*/, mgp_result *result,
mgp_memory *memory) {
for (const auto &module_directory : module_registry->GetModulesDirectory()) {
for (const auto &dir_entry : std::filesystem::recursive_directory_iterator(module_directory)) {
if (dir_entry.is_regular_file() && IsAllowedExtension(dir_entry.path().extension())) {
mgp_result_record *record{nullptr};
if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
return;
}
const auto path_string = GetPathString(dir_entry);
const auto is_editable = IsFileEditable(dir_entry);
const auto path_value = GetStringValueOrSetError(path_string.c_str(), memory, result);
if (!path_value) {
return;
}
MgpUniquePtr<mgp_value> is_editable_value{nullptr, mgp_value_destroy};
if (!TryOrSetError(
[&] { return CreateMgpObject(is_editable_value, mgp_value_make_bool, is_editable, memory); },
result)) {
return;
}
if (!InsertResultOrSetError(result, record, "path", path_value.get())) {
return;
}
if (!InsertResultOrSetError(result, record, "is_editable", is_editable_value.get())) {
return;
}
}
}
}
};
mgp_proc get_module_files("get_module_files", get_module_files_cb, utils::NewDeleteResource(),
{.required_privilege = AuthQuery::Privilege::MODULE_READ});
MG_ASSERT(mgp_proc_add_result(&get_module_files, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&get_module_files, "is_editable", Call<mgp_type *>(mgp_type_bool)) ==
MGP_ERROR_NO_ERROR);
module->AddProcedure("get_module_files", std::move(get_module_files));
}
void RegisterMgGetModuleFile(ModuleRegistry *module_registry, BuiltinModule *module) {
auto get_module_file_cb = [module_registry](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result,
mgp_memory *memory) {
MG_ASSERT(Call<size_t>(mgp_list_size, args) == 1U, "Should have been type checked already");
auto *arg = Call<mgp_value *>(mgp_list_at, args, 0);
MG_ASSERT(CallBool(mgp_value_is_string, arg), "Should have been type checked already");
const char *path_str{nullptr};
if (!TryOrSetError([&] { return mgp_value_get_string(arg, &path_str); }, result)) {
return;
}
const std::filesystem::path path{path_str};
if (!path.is_absolute()) {
static_cast<void>(mgp_result_set_error_msg(result, "The path should be an absolute path."));
return;
}
if (!IsAllowedExtension(path.extension())) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file isn't in the supported format."));
return;
}
if (!std::filesystem::exists(path)) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file doesn't exist."));
return;
}
if (auto maybe_error_msg = ParentModuleDirectory(*module_registry, path); maybe_error_msg.HasError()) {
static_cast<void>(mgp_result_set_error_msg(result, maybe_error_msg.GetError()));
return;
}
const auto maybe_content = ReadFile(path);
if (!maybe_content) {
static_cast<void>(mgp_result_set_error_msg(result, "Couldn't read the content of the file."));
return;
}
mgp_result_record *record{nullptr};
if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
return;
}
const auto content_value = GetStringValueOrSetError(maybe_content->c_str(), memory, result);
if (!content_value) {
return;
}
if (!InsertResultOrSetError(result, record, "content", content_value.get())) {
return;
}
};
mgp_proc get_module_file("get_module_file", std::move(get_module_file_cb), utils::NewDeleteResource(),
{.required_privilege = AuthQuery::Privilege::MODULE_READ});
MG_ASSERT(mgp_proc_add_arg(&get_module_file, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&get_module_file, "content", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("get_module_file", std::move(get_module_file));
}
namespace {
utils::BasicResult<std::string> WriteToFile(const std::filesystem::path &file, const std::string_view content) {
std::ofstream output_file{file};
if (!output_file.is_open()) {
return fmt::format("Failed to open the file at location {}", file);
}
output_file.write(content.data(), static_cast<std::streamsize>(content.size()));
output_file.flush();
return {};
}
} // namespace
void RegisterMgCreateModuleFile(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) {
auto create_module_file_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result,
mgp_memory *memory) {
MG_ASSERT(Call<size_t>(mgp_list_size, args) == 2U, "Should have been type checked already");
auto *filename_arg = Call<mgp_value *>(mgp_list_at, args, 0);
MG_ASSERT(CallBool(mgp_value_is_string, filename_arg), "Should have been type checked already");
const char *filename_str{nullptr};
if (!TryOrSetError([&] { return mgp_value_get_string(filename_arg, &filename_str); }, result)) {
return;
}
const auto file_path = module_registry->InternalModuleDir() / filename_str;
if (!IsSubPath(module_registry->InternalModuleDir(), file_path)) {
static_cast<void>(mgp_result_set_error_msg(
result,
"Invalid relative path defined. The module file cannot be define outside the internal modules directory."));
return;
}
if (!IsAllowedExtension(file_path.extension())) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file isn't in the supported format."));
return;
}
if (std::filesystem::exists(file_path)) {
static_cast<void>(mgp_result_set_error_msg(result, "File with the same name already exists!"));
return;
}
utils::EnsureDir(file_path.parent_path());
auto *content_arg = Call<mgp_value *>(mgp_list_at, args, 1);
MG_ASSERT(CallBool(mgp_value_is_string, content_arg), "Should have been type checked already");
const char *content_str{nullptr};
if (!TryOrSetError([&] { return mgp_value_get_string(content_arg, &content_str); }, result)) {
return;
}
if (auto maybe_error = WriteToFile(file_path, {content_str, std::strlen(content_str)}); maybe_error.HasError()) {
static_cast<void>(mgp_result_set_error_msg(result, maybe_error.GetError().c_str()));
return;
}
mgp_result_record *record{nullptr};
if (!TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
return;
}
const auto path_value = GetStringValueOrSetError(std::filesystem::canonical(file_path).c_str(), memory, result);
if (!path_value) {
return;
}
if (!InsertResultOrSetError(result, record, "path", path_value.get())) {
return;
}
WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); });
};
mgp_proc create_module_file("create_module_file", std::move(create_module_file_cb), utils::NewDeleteResource(),
{.required_privilege = AuthQuery::Privilege::MODULE_WRITE});
MG_ASSERT(mgp_proc_add_arg(&create_module_file, "filename", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_arg(&create_module_file, "content", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&create_module_file, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("create_module_file", std::move(create_module_file));
}
void RegisterMgUpdateModuleFile(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) {
auto update_module_file_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result,
mgp_memory * /*memory*/) {
MG_ASSERT(Call<size_t>(mgp_list_size, args) == 2U, "Should have been type checked already");
auto *path_arg = Call<mgp_value *>(mgp_list_at, args, 0);
MG_ASSERT(CallBool(mgp_value_is_string, path_arg), "Should have been type checked already");
const char *path_str{nullptr};
if (!TryOrSetError([&] { return mgp_value_get_string(path_arg, &path_str); }, result)) {
return;
}
const std::filesystem::path path{path_str};
if (!path.is_absolute()) {
static_cast<void>(mgp_result_set_error_msg(result, "The path should be an absolute path."));
return;
}
if (!IsAllowedExtension(path.extension())) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file isn't in the supported format."));
return;
}
if (!std::filesystem::exists(path)) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file doesn't exist."));
return;
}
if (auto maybe_error_msg = ParentModuleDirectory(*module_registry, path); maybe_error_msg.HasError()) {
static_cast<void>(mgp_result_set_error_msg(result, maybe_error_msg.GetError()));
return;
}
auto *content_arg = Call<mgp_value *>(mgp_list_at, args, 1);
MG_ASSERT(CallBool(mgp_value_is_string, content_arg), "Should have been type checked already");
const char *content_str{nullptr};
if (!TryOrSetError([&] { return mgp_value_get_string(content_arg, &content_str); }, result)) {
return;
}
if (auto maybe_error = WriteToFile(path, {content_str, std::strlen(content_str)}); maybe_error.HasError()) {
static_cast<void>(mgp_result_set_error_msg(result, maybe_error.GetError().c_str()));
return;
}
WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); });
};
mgp_proc update_module_file("update_module_file", std::move(update_module_file_cb), utils::NewDeleteResource(),
{.required_privilege = AuthQuery::Privilege::MODULE_WRITE});
MG_ASSERT(mgp_proc_add_arg(&update_module_file, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_arg(&update_module_file, "content", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("update_module_file", std::move(update_module_file));
}
void RegisterMgDeleteModuleFile(ModuleRegistry *module_registry, utils::RWLock *lock, BuiltinModule *module) {
auto delete_module_file_cb = [module_registry, lock](mgp_list *args, mgp_graph * /*unused*/, mgp_result *result,
mgp_memory * /*memory*/) {
MG_ASSERT(Call<size_t>(mgp_list_size, args) == 1U, "Should have been type checked already");
auto *path_arg = Call<mgp_value *>(mgp_list_at, args, 0);
MG_ASSERT(CallBool(mgp_value_is_string, path_arg), "Should have been type checked already");
const char *path_str{nullptr};
if (!TryOrSetError([&] { return mgp_value_get_string(path_arg, &path_str); }, result)) {
return;
}
const std::filesystem::path path{path_str};
if (!path.is_absolute()) {
static_cast<void>(mgp_result_set_error_msg(result, "The path should be an absolute path."));
return;
}
if (!IsAllowedExtension(path.extension())) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file isn't in the supported format."));
return;
}
if (!std::filesystem::exists(path)) {
static_cast<void>(mgp_result_set_error_msg(result, "The specified file doesn't exist."));
return;
}
const auto parent_module_directory = ParentModuleDirectory(*module_registry, path);
if (parent_module_directory.HasError()) {
static_cast<void>(mgp_result_set_error_msg(result, parent_module_directory.GetError()));
return;
}
std::error_code ec;
if (!std::filesystem::remove(path, ec)) {
static_cast<void>(
mgp_result_set_error_msg(result, fmt::format("Failed to delete the module: {}", ec.message()).c_str()));
return;
}
auto parent_path = path.parent_path();
while (!std::filesystem::is_symlink(parent_path) && std::filesystem::is_empty(parent_path) &&
!std::filesystem::equivalent(*parent_module_directory, parent_path)) {
std::filesystem::remove(parent_path);
parent_path = parent_path.parent_path();
}
WithUpgradedLock(lock, [&]() { module_registry->UnloadAndLoadModulesFromDirectories(); });
};
mgp_proc delete_module_file("delete_module_file", std::move(delete_module_file_cb), utils::NewDeleteResource(),
{.required_privilege = AuthQuery::Privilege::MODULE_WRITE});
MG_ASSERT(mgp_proc_add_arg(&delete_module_file, "path", Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
module->AddProcedure("delete_module_file", std::move(delete_module_file));
}
// Run `fun` with `mgp_module *` and `mgp_memory *` arguments. If `fun` returned
// a `true` value, store the `mgp_module::procedures` and
// `mgp_module::transformations into `proc_map`. The return value of WithModuleRegistration
@@ -687,8 +297,6 @@ class SharedLibraryModule final : public Module {
const std::map<std::string, mgp_trans, std::less<>> *Transformations() const override;
std::optional<std::filesystem::path> Path() const override { return file_path_; }
private:
/// Path as requested for loading the module from a library.
std::filesystem::path file_path_;
@@ -816,7 +424,6 @@ class PythonModule final : public Module {
const std::map<std::string, mgp_proc, std::less<>> *Procedures() const override;
const std::map<std::string, mgp_trans, std::less<>> *Transformations() const override;
std::optional<std::filesystem::path> Path() const override { return file_path_; }
private:
std::filesystem::path file_path_;
@@ -885,9 +492,6 @@ bool PythonModule::Close() {
py_module_ = py::Object(nullptr);
return false;
}
// Remove the cached bytecode if it's present
std::filesystem::remove_all(file_path_.parent_path() / "__pycache__");
py_module_ = py::Object(nullptr);
spdlog::info("Closed module {}", file_path_);
return true;
@@ -955,24 +559,13 @@ ModuleRegistry::ModuleRegistry() {
RegisterMgProcedures(&modules_, module.get());
RegisterMgTransformations(&modules_, module.get());
RegisterMgLoad(this, &lock_, module.get());
RegisterMgGetModuleFiles(this, module.get());
RegisterMgGetModuleFile(this, module.get());
RegisterMgCreateModuleFile(this, &lock_, module.get());
RegisterMgUpdateModuleFile(this, &lock_, module.get());
RegisterMgDeleteModuleFile(this, &lock_, module.get());
modules_.emplace("mg", std::move(module));
}
void ModuleRegistry::SetModulesDirectory(std::vector<std::filesystem::path> modules_dirs,
const std::filesystem::path &data_directory) {
internal_module_dir_ = data_directory / "internal_modules";
utils::EnsureDirOrDie(internal_module_dir_);
void ModuleRegistry::SetModulesDirectory(std::vector<std::filesystem::path> modules_dirs) {
modules_dirs_ = std::move(modules_dirs);
modules_dirs_.push_back(internal_module_dir_);
}
const std::vector<std::filesystem::path> &ModuleRegistry::GetModulesDirectory() const { return modules_dirs_; }
bool ModuleRegistry::LoadModuleIfFound(const std::filesystem::path &modules_dir, const std::string_view name) {
if (!utils::DirExists(modules_dir)) {
spdlog::error(
@@ -1061,8 +654,6 @@ bool ModuleRegistry::RegisterMgProcedure(const std::string_view name, mgp_proc p
return false;
}
const std::filesystem::path &ModuleRegistry::InternalModuleDir() const noexcept { return internal_module_dir_; }
namespace {
/// This function returns a pair of either
@@ -1086,9 +677,9 @@ template <typename T>
concept ModuleProperties = utils::SameAsAnyOf<T, mgp_proc, mgp_trans>;
template <ModuleProperties T>
std::optional<std::pair<ModulePtr, const T *>> MakePairIfPropFound(const ModuleRegistry &module_registry,
std::string_view fully_qualified_name,
utils::MemoryResource *memory) {
std::optional<std::pair<procedure::ModulePtr, const T *>> MakePairIfPropFound(const ModuleRegistry &module_registry,
std::string_view fully_qualified_name,
utils::MemoryResource *memory) {
auto prop_fun = [](auto &module) {
if constexpr (std::is_same_v<T, mgp_proc>) {
return module->Procedures();
@@ -1109,13 +700,13 @@ std::optional<std::pair<ModulePtr, const T *>> MakePairIfPropFound(const ModuleR
} // namespace
std::optional<std::pair<ModulePtr, const mgp_proc *>> FindProcedure(const ModuleRegistry &module_registry,
std::string_view fully_qualified_procedure_name,
utils::MemoryResource *memory) {
std::optional<std::pair<procedure::ModulePtr, const mgp_proc *>> FindProcedure(
const ModuleRegistry &module_registry, std::string_view fully_qualified_procedure_name,
utils::MemoryResource *memory) {
return MakePairIfPropFound<mgp_proc>(module_registry, fully_qualified_procedure_name, memory);
}
std::optional<std::pair<ModulePtr, const mgp_trans *>> FindTransformation(
std::optional<std::pair<procedure::ModulePtr, const mgp_trans *>> FindTransformation(
const ModuleRegistry &module_registry, std::string_view fully_qualified_transformation_name,
utils::MemoryResource *memory) {
return MakePairIfPropFound<mgp_trans>(module_registry, fully_qualified_transformation_name, memory);

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -45,8 +45,6 @@ class Module {
virtual const std::map<std::string, mgp_proc, std::less<>> *Procedures() const = 0;
/// Returns registered transformations of this module
virtual const std::map<std::string, mgp_trans, std::less<>> *Transformations() const = 0;
virtual std::optional<std::filesystem::path> Path() const = 0;
};
/// Proxy for a registered Module, acquires a read lock from ModuleRegistry.
@@ -87,8 +85,7 @@ class ModuleRegistry final {
ModuleRegistry();
/// Set the modules directories that will be used when (re)loading modules.
void SetModulesDirectory(std::vector<std::filesystem::path> modules_dir, const std::filesystem::path &data_directory);
const std::vector<std::filesystem::path> &GetModulesDirectory() const;
void SetModulesDirectory(std::vector<std::filesystem::path> modules_dir);
/// Atomically load or reload a module with a particular name from the given
/// directory.
@@ -122,11 +119,8 @@ class ModuleRegistry final {
bool RegisterMgProcedure(std::string_view name, mgp_proc proc);
const std::filesystem::path &InternalModuleDir() const noexcept;
private:
std::vector<std::filesystem::path> modules_dirs_;
std::filesystem::path internal_module_dir_;
};
/// Single, global module registry.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -181,7 +181,6 @@ PyObject *PyVerticesIteratorGet(PyVerticesIterator *self, PyObject *Py_UNUSED(ig
return nullptr;
}
if (vertex == nullptr) {
Py_INCREF(Py_None);
return Py_None;
}
return MakePyVertex(*vertex, self->py_graph);
@@ -196,7 +195,6 @@ PyObject *PyVerticesIteratorNext(PyVerticesIterator *self, PyObject *Py_UNUSED(i
return nullptr;
}
if (vertex == nullptr) {
Py_INCREF(Py_None);
return Py_None;
}
return MakePyVertex(*vertex, self->py_graph);
@@ -253,7 +251,6 @@ PyObject *PyEdgesIteratorGet(PyEdgesIterator *self, PyObject *Py_UNUSED(ignored)
return nullptr;
}
if (edge == nullptr) {
Py_INCREF(Py_None);
return Py_None;
}
return MakePyEdge(*edge, self->py_graph);
@@ -268,7 +265,6 @@ PyObject *PyEdgesIteratorNext(PyEdgesIterator *self, PyObject *Py_UNUSED(ignored
return nullptr;
}
if (edge == nullptr) {
Py_INCREF(Py_None);
return Py_None;
}
return MakePyEdge(*edge, self->py_graph);
@@ -1046,11 +1042,12 @@ PyObject *PyQueryModuleAddProcedure(PyQueryModule *self, PyObject *cb, bool is_w
return nullptr;
}
auto *memory = self->module->procedures.get_allocator().GetMemoryResource();
mgp_proc proc(name,
[py_cb](mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memory *memory) {
CallPythonProcedure(py_cb, args, graph, result, memory);
},
memory, {.is_write = is_write_procedure});
mgp_proc proc(
name,
[py_cb](mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memory *memory) {
CallPythonProcedure(py_cb, args, graph, result, memory);
},
memory, is_write_procedure);
const auto &[proc_it, did_insert] = self->module->procedures.emplace(name, std::move(proc));
if (!did_insert) {
PyErr_SetString(PyExc_ValueError, "Already registered a procedure with the same name.");
@@ -1243,7 +1240,6 @@ PyObject *PyPropertiesIteratorGet(PyPropertiesIterator *self, PyObject *Py_UNUSE
return nullptr;
}
if (property == nullptr) {
Py_INCREF(Py_None);
return Py_None;
}
py::Object py_name(PyUnicode_FromString(property->name));
@@ -1262,7 +1258,6 @@ PyObject *PyPropertiesIteratorNext(PyPropertiesIterator *self, PyObject *Py_UNUS
return nullptr;
}
if (property == nullptr) {
Py_INCREF(Py_None);
return Py_None;
}
py::Object py_name(PyUnicode_FromString(property->name));

View File

@@ -58,8 +58,7 @@ concept Stream = requires(TStream stream) {
stream.Check(std::optional<std::chrono::milliseconds>{}, std::optional<int64_t>{},
ConsumerFunction<typename TStream::Message>{})
} -> std::same_as<void>;
requires std::same_as<std::decay_t<decltype(std::declval<typename TStream::StreamInfo>().common_info)>,
CommonStreamInfo>;
{ typename TStream::StreamInfo{}.common_info } -> std::same_as<CommonStreamInfo>;
requires ConvertableToMgpMessage<typename TStream::Message>;
requires ConvertableToJson<typename TStream::StreamInfo>;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,8 +13,6 @@
#include <json/json.hpp>
#include "integrations/constants.hpp"
namespace query::stream {
KafkaStream::KafkaStream(std::string stream_name, StreamInfo stream_info,
ConsumerFunction<integrations::kafka::Message> consumer_function) {
@@ -25,8 +23,6 @@ KafkaStream::KafkaStream(std::string stream_name, StreamInfo stream_info,
.bootstrap_servers = std::move(stream_info.bootstrap_servers),
.batch_interval = stream_info.common_info.batch_interval,
.batch_size = stream_info.common_info.batch_size,
.public_configs = std::move(stream_info.configs),
.private_configs = std::move(stream_info.credentials),
};
consumer_.emplace(std::move(consumer_info), std::move(consumer_function));
};
@@ -38,9 +34,7 @@ KafkaStream::StreamInfo KafkaStream::Info(std::string transformation_name) const
.transformation_name = std::move(transformation_name)},
.topics = info.topics,
.consumer_group = info.consumer_group,
.bootstrap_servers = info.bootstrap_servers,
.configs = info.public_configs,
.credentials = info.private_configs};
.bootstrap_servers = info.bootstrap_servers};
}
void KafkaStream::Start() { consumer_->Start(); }
@@ -60,10 +54,6 @@ namespace {
const std::string kTopicsKey{"topics"};
const std::string kConsumerGroupKey{"consumer_group"};
const std::string kBoostrapServers{"bootstrap_servers"};
const std::string kConfigs{"configs"};
const std::string kCredentials{"credentials"};
const std::unordered_map<std::string, std::string> kDefaultConfigsMap;
} // namespace
void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info) {
@@ -71,8 +61,6 @@ void to_json(nlohmann::json &data, KafkaStream::StreamInfo &&info) {
data[kTopicsKey] = std::move(info.topics);
data[kConsumerGroupKey] = info.consumer_group;
data[kBoostrapServers] = std::move(info.bootstrap_servers);
data[kConfigs] = std::move(info.configs);
data[kCredentials] = std::move(info.credentials);
}
void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info) {
@@ -80,9 +68,6 @@ void from_json(const nlohmann::json &data, KafkaStream::StreamInfo &info) {
data.at(kTopicsKey).get_to(info.topics);
data.at(kConsumerGroupKey).get_to(info.consumer_group);
data.at(kBoostrapServers).get_to(info.bootstrap_servers);
// These values might not be present in the persisted JSON object
info.configs = data.value(kConfigs, kDefaultConfigsMap);
info.credentials = data.value(kCredentials, kDefaultConfigsMap);
}
PulsarStream::PulsarStream(std::string stream_name, StreamInfo stream_info,

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,8 +24,6 @@ struct KafkaStream {
std::vector<std::string> topics;
std::string consumer_group;
std::string bootstrap_servers;
std::unordered_map<std::string, std::string> configs;
std::unordered_map<std::string, std::string> credentials;
};
using Message = integrations::kafka::Message;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,6 @@
#include <spdlog/spdlog.h>
#include <json/json.hpp>
#include "integrations/constants.hpp"
#include "mg_procedure.h"
#include "query/db_accessor.hpp"
#include "query/discard_value_stream.hpp"
@@ -30,7 +29,6 @@
#include "query/stream/sources.hpp"
#include "query/typed_value.hpp"
#include "utils/event_counter.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/pmr/string.hpp"
@@ -56,7 +54,7 @@ auto GetStream(auto &map, const std::string &stream_name) {
}
std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformationResult(
const utils::pmr::map<utils::pmr::string, TypedValue> &values, const std::string_view transformation_name,
utils::pmr::map<utils::pmr::string, TypedValue> &&values, const std::string_view transformation_name,
const std::string_view stream_name) {
if (values.size() != kExpectedTransformationResultSize) {
throw StreamsException(
@@ -64,7 +62,7 @@ std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformation
transformation_name, stream_name);
}
auto get_value = [&](const utils::pmr::string &field_name) mutable -> const TypedValue & {
auto get_value = [&](const utils::pmr::string &field_name) mutable -> TypedValue & {
auto it = values.find(field_name);
if (it == values.end()) {
throw StreamsException{"Transformation '{}' in stream '{}' did not yield a record with '{}' field.",
@@ -73,11 +71,11 @@ std::pair<TypedValue /*query*/, TypedValue /*parameters*/> ExtractTransformation
return it->second;
};
const auto &query_value = get_value(query_param_name);
auto &query_value = get_value(query_param_name);
MG_ASSERT(query_value.IsString());
const auto &params_value = get_value(params_param_name);
auto &params_value = get_value(params_param_name);
MG_ASSERT(params_value.IsNull() || params_value.IsMap());
return {query_value, params_value};
return {std::move(query_value), std::move(params_value)};
}
template <typename TMessage>
@@ -160,6 +158,43 @@ void from_json(const nlohmann::json &data, StreamStatus<TStream> &status) {
from_json(data, status.info);
}
namespace {
template <typename Fun>
[[nodiscard]] bool TryOrSetError(Fun &&func, mgp_result *result) {
if (const auto err = func(); err == MGP_ERROR_UNABLE_TO_ALLOCATE) {
static_cast<void>(mgp_result_set_error_msg(result, "Not enough memory!"));
return false;
} else if (err != MGP_ERROR_NO_ERROR) {
const auto error_msg = fmt::format("Unexpected error ({})!", err);
static_cast<void>(mgp_result_set_error_msg(result, error_msg.c_str()));
return false;
}
return true;
}
[[nodiscard]] auto GetStringValueOrSetError(const char *string, mgp_memory *memory, mgp_result *result) {
procedure::MgpUniquePtr<mgp_value> value{nullptr, mgp_value_destroy};
const auto success =
TryOrSetError([&] { return procedure::CreateMgpObject(value, mgp_value_make_string, string, memory); }, result);
if (!success) {
value.reset();
}
return value;
}
[[nodiscard]] bool InsertResultOrSetError(mgp_result *result, mgp_result_record *record, const auto *result_name,
mgp_value *value) {
if (const auto err = mgp_result_record_insert(record, result_name, value); err != MGP_ERROR_NO_ERROR) {
const auto error_msg = fmt::format("Unable to set the result for {}, error = {}", result_name, err);
static_cast<void>(mgp_result_set_error_msg(result, error_msg.c_str()));
return false;
}
return true;
}
} // namespace
Streams::Streams(InterpreterContext *interpreter_context, std::filesystem::path directory)
: interpreter_context_(interpreter_context), storage_(std::move(directory)) {
RegisterProcedures();
@@ -196,7 +231,7 @@ void Streams::RegisterKafkaProcedures() {
it->second);
};
mgp_proc proc(proc_name, set_stream_offset, utils::NewDeleteResource());
mgp_proc proc(proc_name, set_stream_offset, utils::NewDeleteResource(), false);
MG_ASSERT(mgp_proc_add_arg(&proc, "stream_name", procedure::Call<mgp_type *>(mgp_type_string)) ==
MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_arg(&proc, "offset", procedure::Call<mgp_type *>(mgp_type_int)) == MGP_ERROR_NO_ERROR);
@@ -210,12 +245,10 @@ void Streams::RegisterKafkaProcedures() {
constexpr std::string_view consumer_group_result_name = "consumer_group";
constexpr std::string_view topics_result_name = "topics";
constexpr std::string_view bootstrap_servers_result_name = "bootstrap_servers";
constexpr std::string_view configs_result_name = "configs";
constexpr std::string_view credentials_result_name = "credentials";
auto get_stream_info = [this, proc_name, consumer_group_result_name, topics_result_name,
bootstrap_servers_result_name, configs_result_name, credentials_result_name](
mgp_list *args, mgp_graph * /*graph*/, mgp_result *result, mgp_memory *memory) {
bootstrap_servers_result_name](mgp_list *args, mgp_graph * /*graph*/, mgp_result *result,
mgp_memory *memory) {
auto *arg_stream_name = procedure::Call<mgp_value *>(mgp_list_at, args, 0);
const auto *stream_name = procedure::Call<const char *>(mgp_value_get_string, arg_stream_name);
auto lock_ptr = streams_.Lock();
@@ -226,28 +259,32 @@ void Streams::RegisterKafkaProcedures() {
auto stream_source_ptr = kafka_stream.stream_source->Lock();
const auto info = stream_source_ptr->Info(kafka_stream.transformation_name);
mgp_result_record *record{nullptr};
if (!procedure::TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
return;
{
const auto success = TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result);
if (!success) {
return;
}
}
const auto consumer_group_value =
procedure::GetStringValueOrSetError(info.consumer_group.c_str(), memory, result);
const auto consumer_group_value = GetStringValueOrSetError(info.consumer_group.c_str(), memory, result);
if (!consumer_group_value) {
return;
}
procedure::MgpUniquePtr<mgp_list> topic_names{nullptr, mgp_list_destroy};
if (!procedure::TryOrSetError(
[&] {
return procedure::CreateMgpObject(topic_names, mgp_list_make_empty, info.topics.size(),
memory);
},
result)) {
return;
{
const auto success = TryOrSetError(
[&] {
return procedure::CreateMgpObject(topic_names, mgp_list_make_empty, info.topics.size(), memory);
},
result);
if (!success) {
return;
}
}
for (const auto &topic : info.topics) {
auto topic_value = procedure::GetStringValueOrSetError(topic.c_str(), memory, result);
auto topic_value = GetStringValueOrSetError(topic.c_str(), memory, result);
if (!topic_value) {
return;
}
@@ -255,87 +292,34 @@ void Streams::RegisterKafkaProcedures() {
}
procedure::MgpUniquePtr<mgp_value> topics_value{nullptr, mgp_value_destroy};
if (!procedure::TryOrSetError(
[&] {
return procedure::CreateMgpObject(topics_value, mgp_value_make_list, topic_names.get());
},
result)) {
return;
{
const auto success = TryOrSetError(
[&] {
return procedure::CreateMgpObject(topics_value, mgp_value_make_list, topic_names.release());
},
result);
if (!success) {
return;
}
}
static_cast<void>(topic_names.release());
const auto bootstrap_servers_value =
procedure::GetStringValueOrSetError(info.bootstrap_servers.c_str(), memory, result);
GetStringValueOrSetError(info.bootstrap_servers.c_str(), memory, result);
if (!bootstrap_servers_value) {
return;
}
const auto convert_config_map =
[result, memory](const std::unordered_map<std::string, std::string> &configs_to_convert)
-> procedure::MgpUniquePtr<mgp_value> {
procedure::MgpUniquePtr<mgp_value> configs_value{nullptr, mgp_value_destroy};
procedure::MgpUniquePtr<mgp_map> configs{nullptr, mgp_map_destroy};
if (!procedure::TryOrSetError(
[&] { return procedure::CreateMgpObject(configs, mgp_map_make_empty, memory); }, result)) {
return configs_value;
}
for (const auto &[key, value] : configs_to_convert) {
auto value_value = procedure::GetStringValueOrSetError(value.c_str(), memory, result);
if (!value_value) {
return configs_value;
}
configs->items.emplace(key, std::move(*value_value));
}
if (!procedure::TryOrSetError(
[&] { return procedure::CreateMgpObject(configs_value, mgp_value_make_map, configs.get()); },
result)) {
return configs_value;
}
static_cast<void>(configs.release());
return configs_value;
};
const auto configs_value = convert_config_map(info.configs);
if (configs_value == nullptr) {
if (!InsertResultOrSetError(result, record, consumer_group_result_name.data(),
consumer_group_value.get())) {
return;
}
using CredentialsType = decltype(KafkaStream::StreamInfo::credentials);
CredentialsType reducted_credentials;
std::transform(info.credentials.begin(), info.credentials.end(),
std::inserter(reducted_credentials, reducted_credentials.end()),
[](const auto &pair) -> CredentialsType::value_type {
return {pair.first, integrations::kReducted};
});
const auto credentials_value = convert_config_map(reducted_credentials);
if (credentials_value == nullptr) {
if (!InsertResultOrSetError(result, record, topics_result_name.data(), topics_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, consumer_group_result_name.data(),
consumer_group_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, topics_result_name.data(), topics_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, bootstrap_servers_result_name.data(),
bootstrap_servers_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, configs_result_name.data(),
configs_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, credentials_result_name.data(),
credentials_value.get())) {
if (!InsertResultOrSetError(result, record, bootstrap_servers_result_name.data(),
bootstrap_servers_value.get())) {
return;
}
},
@@ -345,7 +329,7 @@ void Streams::RegisterKafkaProcedures() {
it->second);
};
mgp_proc proc(proc_name, get_stream_info, utils::NewDeleteResource());
mgp_proc proc(proc_name, get_stream_info, utils::NewDeleteResource(), false);
MG_ASSERT(mgp_proc_add_arg(&proc, "stream_name", procedure::Call<mgp_type *>(mgp_type_string)) ==
MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&proc, consumer_group_result_name.data(),
@@ -356,10 +340,6 @@ void Streams::RegisterKafkaProcedures() {
MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&proc, bootstrap_servers_result_name.data(),
procedure::Call<mgp_type *>(mgp_type_string)) == MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&proc, configs_result_name.data(), procedure::Call<mgp_type *>(mgp_type_map)) ==
MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&proc, credentials_result_name.data(), procedure::Call<mgp_type *>(mgp_type_map)) ==
MGP_ERROR_NO_ERROR);
procedure::gModuleRegistry.RegisterMgProcedure(proc_name, std::move(proc));
}
@@ -382,27 +362,32 @@ void Streams::RegisterPulsarProcedures() {
auto stream_source_ptr = pulsar_stream.stream_source->Lock();
const auto info = stream_source_ptr->Info(pulsar_stream.transformation_name);
mgp_result_record *record{nullptr};
if (!procedure::TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result)) {
return;
{
const auto success = TryOrSetError([&] { return mgp_result_new_record(result, &record); }, result);
if (!success) {
return;
}
}
auto service_url_value = procedure::GetStringValueOrSetError(info.service_url.c_str(), memory, result);
auto service_url_value = GetStringValueOrSetError(info.service_url.c_str(), memory, result);
if (!service_url_value) {
return;
}
procedure::MgpUniquePtr<mgp_list> topic_names{nullptr, mgp_list_destroy};
if (!procedure::TryOrSetError(
[&] {
return procedure::CreateMgpObject(topic_names, mgp_list_make_empty, info.topics.size(),
memory);
},
result)) {
return;
{
const auto success = TryOrSetError(
[&] {
return procedure::CreateMgpObject(topic_names, mgp_list_make_empty, info.topics.size(), memory);
},
result);
if (!success) {
return;
}
}
for (const auto &topic : info.topics) {
auto topic_value = procedure::GetStringValueOrSetError(topic.c_str(), memory, result);
auto topic_value = GetStringValueOrSetError(topic.c_str(), memory, result);
if (!topic_value) {
return;
}
@@ -410,20 +395,22 @@ void Streams::RegisterPulsarProcedures() {
}
procedure::MgpUniquePtr<mgp_value> topics_value{nullptr, mgp_value_destroy};
if (!procedure::TryOrSetError(
[&] {
return procedure::CreateMgpObject(topics_value, mgp_value_make_list, topic_names.release());
},
result)) {
{
const auto success = TryOrSetError(
[&] {
return procedure::CreateMgpObject(topics_value, mgp_value_make_list, topic_names.release());
},
result);
if (!success) {
return;
}
}
if (!InsertResultOrSetError(result, record, topics_result_name.data(), topics_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, topics_result_name.data(), topics_value.get())) {
return;
}
if (!procedure::InsertResultOrSetError(result, record, service_url_result_name.data(),
service_url_value.get())) {
if (!InsertResultOrSetError(result, record, service_url_result_name.data(), service_url_value.get())) {
return;
}
},
@@ -433,7 +420,7 @@ void Streams::RegisterPulsarProcedures() {
it->second);
};
mgp_proc proc(proc_name, get_stream_info, utils::NewDeleteResource());
mgp_proc proc(proc_name, get_stream_info, utils::NewDeleteResource(), false);
MG_ASSERT(mgp_proc_add_arg(&proc, "stream_name", procedure::Call<mgp_type *>(mgp_type_string)) ==
MGP_ERROR_NO_ERROR);
MG_ASSERT(mgp_proc_add_result(&proc, service_url_result_name.data(),
@@ -508,7 +495,8 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std
interpreter->BeginTransaction();
for (auto &row : result.rows) {
spdlog::trace("Processing row in stream '{}'", stream_name);
auto [query_value, params_value] = ExtractTransformationResult(row.values, transformation_name, stream_name);
auto [query_value, params_value] =
ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
storage::PropertyValue params_prop{params_value};
std::string query{query_value.ValueString()};
@@ -529,7 +517,6 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std
result.rows.clear();
break;
} catch (const query::TransactionSerializationException &e) {
interpreter->Abort();
if (i == total_retries) {
throw;
}
@@ -726,7 +713,8 @@ TransformationResult Streams::Check(const std::string &stream_name, std::optiona
CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
for (auto &row : result.rows) {
auto [query, parameters] = ExtractTransformationResult(row.values, transformation_name, stream_name);
auto [query, parameters] =
ExtractTransformationResult(std::move(row.values), transformation_name, stream_name);
std::vector<TypedValue> result_row;
result_row.reserve(kExpectedTransformationResultSize);
result_row.push_back(std::move(query));

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -31,7 +31,6 @@
#include "utils/rw_lock.hpp"
#include "utils/synchronized.hpp"
class StreamsTest;
namespace query {
struct InterpreterContext;
@@ -74,8 +73,6 @@ using TransformationResult = std::vector<std::vector<TypedValue>>;
///
/// This class is responsible for all query supported actions to happen.
class Streams final {
friend StreamsTest;
public:
/// Initializes the streams.
///
@@ -153,6 +150,9 @@ class Streams final {
std::optional<std::chrono::milliseconds> timeout = std::nullopt,
std::optional<int64_t> batch_limit = std::nullopt) const;
/// Return the configuration value passed to memgraph.
std::string_view BootstrapServers() const;
private:
template <Stream TStream>
using SynchronizedStreamSource = utils::Synchronized<TStream, utils::WritePrioritizedRWLock>;

View File

@@ -2,9 +2,6 @@ set(requests_src_files
requests.cpp)
find_package(CURL REQUIRED)
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
add_library(mg-requests STATIC ${requests_src_files})
target_link_libraries(mg-requests mg-utils spdlog::spdlog fmt::fmt gflags json ${CURL_LIBRARIES})

View File

@@ -3,9 +3,6 @@ set(rpc_src_files
protocol.cpp
server.cpp)
find_package(fmt REQUIRED)
find_package(gflags REQUIRED)
add_library(mg-rpc STATIC ${rpc_src_files})
target_link_libraries(mg-rpc Threads::Threads mg-communication mg-utils mg-io fmt::fmt gflags)
target_link_libraries(mg-rpc mg-slk)

View File

@@ -1,8 +1,6 @@
set(slk_src_files
streams.cpp)
find_package(gflags REQUIRED)
add_library(mg-slk STATIC ${slk_src_files})
target_link_libraries(mg-slk gflags)
target_link_libraries(mg-slk mg-utils)

View File

@@ -30,9 +30,6 @@ set(storage_v2_src_files
#######################
find_package(gflags REQUIRED)
find_package(Threads REQUIRED)
add_library(mg-storage-v2 STATIC ${storage_v2_src_files})
target_link_libraries(mg-storage-v2 Threads::Threads mg-utils gflags)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -43,19 +43,17 @@ class PreviousPtr {
public:
enum class Type {
NULLPTR,
DELTA,
VERTEX,
EDGE,
};
struct Pointer {
Pointer() = default;
explicit Pointer(Delta *delta) : type(Type::DELTA), delta(delta) {}
explicit Pointer(Vertex *vertex) : type(Type::VERTEX), vertex(vertex) {}
explicit Pointer(Edge *edge) : type(Type::EDGE), edge(edge) {}
Type type{Type::NULLPTR};
Type type;
Delta *delta{nullptr};
Vertex *vertex{nullptr};
Edge *edge{nullptr};
@@ -67,9 +65,6 @@ class PreviousPtr {
Pointer Get() const {
uintptr_t value = storage_.load(std::memory_order_acquire);
if (value == 0) {
return {};
}
uintptr_t type = value & kMask;
if (type == kDelta) {
return Pointer{reinterpret_cast<Delta *>(value & ~kMask)};
@@ -113,8 +108,6 @@ inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer
return a.edge == b.edge;
case PreviousPtr::Type::DELTA:
return a.delta == b.delta;
case PreviousPtr::Type::NULLPTR:
return b.type == PreviousPtr::Type::NULLPTR;
}
}

View File

@@ -903,6 +903,9 @@ uint64_t ToPowerOf8(uint64_t size) {
// of the two sets of data is currently active. Because the first byte of the
// buffer is used to distinguish which of the two sets of data is used, we can
// only use the leftover 15 bytes for raw data storage.
#ifndef __x86_64__
#error The PropertyStore only supports x86_64
#endif
const uint8_t kUseLocalBuffer = 0x01;

View File

@@ -19,7 +19,6 @@
namespace storage {
class PropertyStore {
static_assert(std::endian::native == std::endian::little, "PropertyStore supports only architectures using little-endian.");
public:
PropertyStore();

Some files were not shown because too many files have changed in this diff Show More