Compare commits

..

47 Commits

Author SHA1 Message Date
jbajic
190fe3c373 Update markers 2022-07-05 12:20:31 +02:00
jbajic
81a1bd7957 Fix replication errors 2022-07-04 16:27:17 +02:00
jbajic
c18555bf68 Remove serialization logic 2022-07-04 16:26:52 +02:00
jbajic
73d7f6cabc Add encoding to wal 2022-07-01 16:23:30 +02:00
jbajic
9e0540f8ec Add serialization for uint8 2022-07-01 16:21:01 +02:00
jbajic
534d5ee954 Update storage interaface 2022-07-01 16:19:39 +02:00
jbajic
e65805abe9 Add interface for serialization 2022-07-01 11:36:35 +02:00
jbajic
2279575d2c Fix GetSchema 2022-07-01 11:35:15 +02:00
jbajic
201f9f1023 Fix clang tidy errors 2022-06-28 18:02:01 +02:00
jbajic
eb922babb8 Fix tests 2022-06-28 17:15:16 +02:00
jbajic
21cdd07103 Merge branch 'E112-MG-implement-partial-schema' into T0849-MG-create-schema-ddl-expressions 2022-06-28 16:37:15 +02:00
jbajic
beb7c41497 Finish interpreter tests 2022-06-28 15:22:56 +02:00
jbajic
cbbecebb74 Drop float support 2022-06-28 12:27:01 +02:00
jbajic
0e43cf3145 Add notifications 2022-06-28 12:25:14 +02:00
jbajic
44910def21 Fix conflicting keywords 2022-06-28 12:17:11 +02:00
jbajic
b90e856619 Catch repeating type name in schema definition 2022-06-27 14:28:20 +02:00
jbajic
e9053368d7 Add privilege and cypher visitor tests 2022-06-24 17:28:02 +02:00
jbajic
5d22af7c36 Fix grammar with schemaTypeMap 2022-06-24 17:27:32 +02:00
jbajic
ff34e26878 Remove list and map from lexer 2022-06-24 15:51:40 +02:00
jbajic
ed6f9a91a2 Rename DeleteSchema to DropSchema 2022-06-24 14:08:45 +02:00
jbajic
ed814f3188 Add verification on creation and deletion 2022-06-24 13:39:22 +02:00
Jure Bajic
6666eea897 Add initial schema implementation
* Add initial schema implementation

* Add index to schema

* List schemas and enable multiple properties

* Implement SchemaTypes

* Apply suggestions from code review

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

* Address review comments

* Remove Map and List

* Apply suggestions from code review

Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>

Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
Co-authored-by: Kostas Kyrimis  <kostaskyrim@gmail.com>
2022-06-23 14:04:44 +02:00
jbajic
21ab15f8e3 Override visitPropertyType 2022-06-15 19:12:57 +02:00
jbajic
88d54d4e2f Enable Create schema ddl 2022-06-15 18:23:30 +02:00
jbajic
cb0e072c7d Rename schemaproperty to schemapropertytype 2022-06-15 09:03:20 +02:00
jbajic
7f81fda1c6 Fix naming for schemaproperty type to schema type 2022-06-15 09:01:55 +02:00
jbajic
8eb136bc82 Temporary create ddl logic 2022-06-14 16:44:48 +02:00
jbajic
33f8b27a2a Fix grammar 2022-06-14 16:44:25 +02:00
jbajic
a2aabdb5ae Add common schema type 2022-06-14 16:08:00 +02:00
jbajic
eebb48d094 Fix schema visitor 2022-06-13 14:00:39 +02:00
jbajic
c95c428355 Add show schema query 2022-06-10 17:42:32 +02:00
jbajic
1afe86d4ce Implement show schemas 2022-06-10 14:14:50 +02:00
jbajic
69a5db6875 Add PrepareSchemaQuery function 2022-06-10 14:14:50 +02:00
jbajic
904e3df3a1 Update metadata 2022-06-10 14:14:50 +02:00
jbajic
9922f90e81 Add schema visitors 2022-06-10 14:14:50 +02:00
jbajic
c4863b36a5 Add drop schema query 2022-06-10 14:14:50 +02:00
jbajic
0cf677eef6 Add missing keywords into lexer 2022-06-10 14:14:50 +02:00
jbajic
300eae544d Add privileges for schema 2022-06-10 14:14:50 +02:00
jbajic
382f85cd4d Add create and show schema queries 2022-06-10 14:14:50 +02:00
jbajic
88b98a97cb Add schema operations in storage 2022-06-10 14:14:49 +02:00
jbajic
27f688d7ca Remove Map and List 2022-06-10 14:12:52 +02:00
jbajic
53a5b77f14 Address review comments 2022-05-27 11:40:06 +02:00
Jure Bajic
8cb56a4970 Apply suggestions from code review
Co-authored-by: Jeremy B <97525434+42jeremy@users.noreply.github.com>
Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-05-26 17:22:54 +02:00
jbajic
eb4d23c504 Implement SchemaTypes 2022-05-25 10:41:43 +02:00
jbajic
65b8e055a6 List schemas and enable multiple properties 2022-05-25 10:41:18 +02:00
jbajic
f8e1d8c3f6 Add index to schema 2022-05-19 16:56:48 +02:00
jbajic
01618fb0f2 Add initial schema implementation 2022-05-19 14:42:01 +02:00
170 changed files with 3025 additions and 5792 deletions

View File

@@ -61,9 +61,7 @@ Checks: '*,
-readability-magic-numbers,
-readability-named-parameter,
-misc-no-recursion,
-concurrency-mt-unsafe,
-bugprone-easily-swappable-parameters'
-concurrency-mt-unsafe'
WarningsAsErrors: ''
HeaderFilterRegex: 'src/.*'
AnalyzeTemporaryDtors: false
@@ -90,3 +88,4 @@ CheckOptions:
- key: modernize-use-nullptr.NullMacros
value: 'NULL'
...

View File

@@ -3,9 +3,7 @@
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Write E2E tests
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
- [ ] Provide the full content or a guide for the final git message
[master < Task] PR
- [ ] Check, and update documentation if necessary
- [ ] Update [changelog](https://docs.memgraph.com/memgraph/changelog)
- [ ] Provide the full content or a guide for the final git message

View File

@@ -1,7 +1,4 @@
name: Diff
concurrency:
group: ${{ github.head_ref || github.sha }}
cancel-in-progress: true
on:
push:
@@ -70,11 +67,6 @@ jobs:
# branches and tags. (default: 1)
fetch-depth: 0
# This is also needed if we want do to comparison against other branches
# See https://github.community/t/checkout-code-fails-when-it-runs-lerna-run-test-since-master/17920
- name: Fetch all history for all tags and branches
run: git fetch
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
@@ -115,22 +107,12 @@ jobs:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Run clang-tidy
run: |
source /opt/toolchain-v4/activate
# Restrict clang-tidy results only to the modified parts
git diff -U0 ${{ env.BASE_BRANCH }}... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build -regex ".+\.cpp" | tee ./build/clang_tidy_output.txt
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
# Fail if any warning is reported
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null

View File

@@ -6,11 +6,11 @@ on: workflow_dispatch
jobs:
centos-7:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -22,29 +22,29 @@ jobs:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
centos-9:
runs-on: [self-hosted, DockerMgBuild, X64]
centos-8:
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package centos-9
./release/package/run.sh package centos-8
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
name: centos-8
path: build/output/centos-8/memgraph*.rpm
debian-10:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -57,11 +57,11 @@ jobs:
path: build/output/debian-10/memgraph*.deb
debian-11:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -74,11 +74,11 @@ jobs:
path: build/output/debian-11/memgraph*.deb
docker:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -93,11 +93,11 @@ jobs:
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -110,11 +110,11 @@ jobs:
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -126,29 +126,12 @@ jobs:
name: ubuntu-2004
path: build/output/ubuntu-20.04/memgraph*.deb
ubuntu-2204:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: ubuntu-2204
path: build/output/ubuntu-22.04/memgraph*.deb
debian-11-platform:
runs-on: [self-hosted, DockerMgBuild, X64]
runs-on: [self-hosted, DockerMgBuild]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
@@ -159,20 +142,3 @@ jobs:
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
debian-11-arm:
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm
- name: "Upload package"
uses: actions/upload-artifact@v2
with:
name: debian-11-arm
path: build/output/debian-11-arm/memgraph*.deb

View File

@@ -4,12 +4,8 @@ on:
workflow_dispatch:
inputs:
version:
description: "Memgraph binary version to publish on DockerHub."
description: "Memgraph binary version to publish on Dockerhub."
required: true
force_release:
type: boolean
required: false
default: false
jobs:
docker_publish:
@@ -40,22 +36,6 @@ jobs:
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
- name: Check if specified version is already pushed
run: |
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
echo $EXISTS
if [[ ${EXISTS} -eq 0 ]]; then
echo 'The specified version has been already released to DockerHub.'
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
echo 'Forcing the release!'
else
echo 'Stopping the release!'
exit 1
fi
else
echo 'All good the specified version has not been release to DockerHub.'
fi
- name: Build & push docker images
run: |
cd release/docker

1
CODEOWNERS Normal file
View File

@@ -0,0 +1 @@
* @antaljanosbenjamin @kostasrim

View File

@@ -5,9 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "centos-7"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download

161
environment/os/centos-8.sh Executable file
View File

@@ -0,0 +1,161 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
patch
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat libipt libbabeltrace xz-libs python36 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python36-devel # for query modules
openssl-devel
libseccomp-devel
python36 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it manually using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf install -y epel-release
dnf install -y 'dnf-command(config-manager)'
dnf config-manager --set-enabled powertools # Required to install texinfo.
dnf update -y
dnf install -y wget git python36 python3-pip
for pkg in $1; do
if [ "$pkg" == libipt ]; then
if ! dnf list installed libipt >/dev/null 2>/dev/null; then
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libipt-devel ]; then
if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
# Install GDB dependencies not present in the standard repos.
# https://bugs.centos.org/view.php?id=17068
# https://centos.pkgs.org
# Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
# within GDB yet (an assumption).
# http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working
if [ "$pkg" == libbabeltrace-devel ]; then
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == sbcl ]; then
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
fi
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
fi
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dnf list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
wget -nv https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm
dnf update -y
dnf install -y dotnet-sdk-3.1
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -5,16 +5,15 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "centos-9"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
coreutils gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
expat-devel xz-devel python3-devel texinfo # for gdb
libcurl-devel # for cmake
curl # snappy
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel automake bison # for swig
@@ -22,9 +21,6 @@ TOOLCHAIN_BUILD_DEPS=(
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
)
TOOLCHAIN_RUN_DEPS=(
@@ -35,19 +31,18 @@ TOOLCHAIN_RUN_DEPS=(
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
perl # for openssl
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
wget # for downloading libs
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it manually using `pip3 install PyYAML`
@@ -78,6 +73,12 @@ check() {
if [ "$pkg" == "python3-virtualenv" ]; then
continue
fi
if [ "$pkg" == sbcl ]; then
if ! sbcl --version &> /dev/null; then
missing="$pkg $missing"
fi
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
@@ -104,37 +105,13 @@ install() {
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
# Since there is no support for libipt-devel for CentOS 9 we install
# Fedoras version of same libs, they are the same version but released
# for different OS
# TODO Update when libipt-devel releases for CentOS 9
if [ "$pkg" == libipt ]; then
if ! dnf list installed libipt >/dev/null 2>/dev/null; then
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libipt-devel ]; then
if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libbabeltrace-devel ]; then
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
dnf install -y http://mirror.stream.centos.org/9-stream/CRB/x86_64/os/Packages/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
fi
continue
fi
if [ "$pkg" == sbcl ]; then
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
fi
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
fi
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
if ! sbcl --version &> /dev/null; then
curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
pushd /tmp/sbcl-1.4.2-arm64-linux
INSTALL_ROOT=/usr/local sh install.sh
popd
fi
continue
fi
@@ -148,11 +125,9 @@ install() {
fi
if [ "$pkg" == python3-virtualenv ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install virtualenv
pip3 install virtualenvwrapper
pip3 install --user virtualenv
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
fi
continue
fi

View File

@@ -5,9 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-10"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -5,9 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "arm64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
@@ -90,6 +87,15 @@ EOF
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb
apt-get update
apt-get install -y apt-transport-https dotnet-sdk-3.1
fi
continue
fi
apt install -y "$pkg"
done
}

View File

@@ -5,9 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-11"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
@@ -90,7 +87,7 @@ EOF
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb

View File

@@ -5,8 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "todo-os-name"
TOOLCHAIN_BUILD_DEPS=(
pkg
)

View File

@@ -5,9 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-18.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # archive download

View File

@@ -5,9 +5,6 @@ set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-20.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download

View File

@@ -1,96 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "ubuntu-22.04"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
curl # snappy
file
git # for thrift
libgmp-dev # for gdb
gperf # for proxygen
libssl-dev
libedit-dev libpcre3-dev automake bison # for swig
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
libcurl4 # for cmake
libreadline8 # for cmake and llvm
libffi7 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
netcat # tests are using nc to wait for memgraph
python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-6.0 golang nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
list() {
echo "$1"
}
check() {
check_all_dpkg "$1"
}
install() {
cd "$DIR"
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb
apt-get update
apt-get install -y apt-transport-https dotnet-sdk-6.0
fi
continue
fi
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -675,7 +675,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
SNAPPY_VERSION=1.1.9
XZ_VERSION=5.2.5 # for LZMA
ZLIB_VERSION=1.2.12
ZLIB_VERSION=1.2.11
ZSTD_VERSION=1.5.0
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
@@ -1178,21 +1178,12 @@ popd
# create toolchain archive
if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
DISTRO_FULL_NAME=${DISTRO}
if [[ "${DISTRO}" == centos* ]]; then
if [[ "$for_arm" = "true" ]]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
else
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
fi
DISTRO_FULL_NAME=$DISTRO
if [ "$for_arm" = true ]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
else
if [[ "$for_arm" = "true" ]]; then
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-arm64"
else
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
fi
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
fi
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
fi

View File

@@ -5,28 +5,10 @@ operating_system() {
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
}
check_operating_system() {
if [ "$(operating_system)" != "$1" ]; then
echo "Not the right operating system!"
exit 1
else
echo "The right operating system."
fi
}
architecture() {
uname -m
}
check_architecture() {
if [ "$(architecture)" != "$1" ]; then
echo "Not the right architecture!"
exit 1
else
echo "The right architecture."
fi
}
check_all_yum() {
local missing=""
for pkg in $1; do

File diff suppressed because it is too large Load Diff

7
init
View File

@@ -24,7 +24,7 @@ function setup_virtualenv () {
fi
# create new virtualenv
python3 -m virtualenv -p python3 ve3 || exit 1
virtualenv -p python3 ve3 || exit 1
source ve3/bin/activate
pip --timeout 1000 install -r requirements.txt || exit 1
deactivate
@@ -66,7 +66,7 @@ fi
DISTRO=$(operating_system)
ARCHITECTURE=$(architecture)
if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
if [ "${ARCHITECTURE}" = "arm64" ]; then
OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
else
OS_SCRIPT=$DIR/environment/os/$DISTRO.sh
@@ -139,6 +139,3 @@ done;
# Install precommit hook
python3 -m pip install pre-commit
python3 -m pre_commit install
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

2
libs/.gitignore vendored
View File

@@ -4,5 +4,5 @@
!cleanup.sh
!CMakeLists.txt
!__main.cpp
!jemalloc.cmake
!pulsar.patch
!antlr4.10.1.patch

View File

@@ -106,7 +106,6 @@ import_external_library(antlr4 STATIC
-DWITH_LIBCXX=OFF # because of debian bug
-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=true
-DCMAKE_CXX_STANDARD=20
-DANTLR_BUILD_CPP_TESTS=OFF
BUILD_COMMAND $(MAKE) antlr4_static
INSTALL_COMMAND $(MAKE) install)

View File

@@ -1,13 +0,0 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index baf46cac9..2e7756de8 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -134,7 +134,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
EXPORT antlr4-targets
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

43
libs/antlr4.patch Normal file
View File

@@ -0,0 +1,43 @@
diff --git a/runtime/Cpp/runtime/CMakeLists.txt b/runtime/Cpp/runtime/CMakeLists.txt
index a8503bb..11362cf 100644
--- a/runtime/Cpp/runtime/CMakeLists.txt
+++ b/runtime/Cpp/runtime/CMakeLists.txt
@@ -5,8 +5,8 @@ set(THIRDPARTY_DIR ${CMAKE_BINARY_DIR}/runtime/thirdparty)
set(UTFCPP_DIR ${THIRDPARTY_DIR}/utfcpp)
ExternalProject_Add(
utfcpp
- GIT_REPOSITORY "git://github.com/nemtrif/utfcpp"
- GIT_TAG "v3.1.1"
+ GIT_REPOSITORY "https://github.com/nemtrif/utfcpp"
+ GIT_TAG "v3.2.1"
SOURCE_DIR ${UTFCPP_DIR}
UPDATE_DISCONNECTED 1
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${UTFCPP_DIR}/install -Dgtest_force_shared_crt=ON
@@ -118,7 +118,7 @@ set_target_properties(antlr4_static
ARCHIVE_OUTPUT_DIRECTORY ${LIB_OUTPUT_DIR}
COMPILE_FLAGS "${disabled_compile_warnings} ${extra_static_compile_flags}")
-install(TARGETS antlr4_shared
+install(TARGETS antlr4_shared OPTIONAL
DESTINATION lib
EXPORT antlr4-targets)
install(TARGETS antlr4_static
diff --git a/runtime/Cpp/runtime/src/support/Any.h b/runtime/Cpp/runtime/src/support/Any.h
index 468db98..65a473b 100644
--- a/runtime/Cpp/runtime/src/support/Any.h
+++ b/runtime/Cpp/runtime/src/support/Any.h
@@ -122,12 +122,12 @@ private:
}
private:
- template<int N = 0, typename std::enable_if<N == N && std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return new Derived<T>(value);
}
- template<int N = 0, typename std::enable_if<N == N && !std::is_nothrow_copy_constructible<T>::value, int>::type = 0>
+ template<int N = 0, typename std::enable_if<N == N && !std::is_copy_constructible<T>::value, int>::type = 0>
Base* clone() const {
return nullptr;
}

55
libs/jemalloc.cmake Normal file
View File

@@ -0,0 +1,55 @@
set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
set(JEMALLOC_SRCS
${JEMALLOC_DIR}/src/arena.c
${JEMALLOC_DIR}/src/background_thread.c
${JEMALLOC_DIR}/src/base.c
${JEMALLOC_DIR}/src/bin.c
${JEMALLOC_DIR}/src/bitmap.c
${JEMALLOC_DIR}/src/ckh.c
${JEMALLOC_DIR}/src/ctl.c
${JEMALLOC_DIR}/src/div.c
${JEMALLOC_DIR}/src/extent.c
${JEMALLOC_DIR}/src/extent_dss.c
${JEMALLOC_DIR}/src/extent_mmap.c
${JEMALLOC_DIR}/src/hash.c
${JEMALLOC_DIR}/src/hook.c
${JEMALLOC_DIR}/src/jemalloc.c
${JEMALLOC_DIR}/src/large.c
${JEMALLOC_DIR}/src/log.c
${JEMALLOC_DIR}/src/malloc_io.c
${JEMALLOC_DIR}/src/mutex.c
${JEMALLOC_DIR}/src/mutex_pool.c
${JEMALLOC_DIR}/src/nstime.c
${JEMALLOC_DIR}/src/pages.c
${JEMALLOC_DIR}/src/prng.c
${JEMALLOC_DIR}/src/prof.c
${JEMALLOC_DIR}/src/rtree.c
${JEMALLOC_DIR}/src/sc.c
${JEMALLOC_DIR}/src/stats.c
${JEMALLOC_DIR}/src/sz.c
${JEMALLOC_DIR}/src/tcache.c
${JEMALLOC_DIR}/src/test_hooks.c
${JEMALLOC_DIR}/src/ticker.c
${JEMALLOC_DIR}/src/tsd.c
${JEMALLOC_DIR}/src/witness.c
${JEMALLOC_DIR}/src/safety_check.c
)
add_library(jemalloc ${JEMALLOC_SRCS})
target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
find_package(Threads REQUIRED)
target_link_libraries(jemalloc PUBLIC Threads::Threads)
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
endif()
target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
# for RTLD_NEXT
target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)

View File

@@ -105,7 +105,7 @@ repo_clone_try_double () {
# Download from primary_urls might fail because the cache is not installed.
declare -A primary_urls=(
["antlr4-code"]="http://$local_cache_host/git/antlr4.git"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.10.1-complete.jar"
["antlr4-generator"]="http://$local_cache_host/file/antlr-4.9.2-complete.jar"
["cppitertools"]="http://$local_cache_host/git/cppitertools.git"
["rapidcheck"]="http://$local_cache_host/git/rapidcheck.git"
["gbenchmark"]="http://$local_cache_host/git/benchmark.git"
@@ -130,7 +130,7 @@ declare -A primary_urls=(
# should fail.
declare -A secondary_urls=(
["antlr4-code"]="https://github.com/antlr/antlr4.git"
["antlr4-generator"]="https://www.antlr.org/download/antlr-4.10.1-complete.jar"
["antlr4-generator"]="http://www.antlr.org/download/antlr-4.9.2-complete.jar"
["cppitertools"]="https://github.com/ryanhaining/cppitertools.git"
["rapidcheck"]="https://github.com/emil-e/rapidcheck.git"
["gbenchmark"]="https://github.com/google/benchmark.git"
@@ -152,10 +152,10 @@ declare -A secondary_urls=(
# antlr
file_get_try_double "${primary_urls[antlr4-generator]}" "${secondary_urls[antlr4-generator]}"
antlr4_tag="4.10.1" # v4.10.1
antlr4_tag="4.9.2" # v4.9.2
repo_clone_try_double "${primary_urls[antlr4-code]}" "${secondary_urls[antlr4-code]}" "antlr4" "$antlr4_tag" true
pushd antlr4
git apply ../antlr4.10.1.patch
git apply ../antlr4.patch
popd
# cppitertools v2.0 2019-12-23
@@ -199,7 +199,7 @@ git apply ../rocksdb.patch
popd
# mgclient
mgclient_tag="v1.4.0" # (2022-06-14)
mgclient_tag="96e95c6845463cbe88948392be58d26da0d5ffd3" # (2022-02-08)
repo_clone_try_double "${primary_urls[mgclient]}" "${secondary_urls[mgclient]}" "mgclient" "$mgclient_tag"
sed -i 's/\${CMAKE_INSTALL_LIBDIR}/lib/' mgclient/src/CMakeLists.txt

View File

@@ -67,7 +67,7 @@ It aims to deliver developers the speed, simplicity and scale required to build
the next generation of applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate")
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6")
# All variables must be set before including.
include(CPack)

View File

@@ -0,0 +1,31 @@
FROM dokken/centos-stream-9
# NOTE: If you change the base distro update release/package as well.
ARG release
RUN yum update && yum install -y \
openssl libcurl libseccomp python3 python3-pip \
--nobest --allowerasing \
&& rm -rf /tmp/* \
&& yum clean all
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
COPY ${release} /
# Install memgraph package
RUN rpm -i ${release}
# Memgraph listens for Bolt Protocol on this port by default.
EXPOSE 7687
# Snapshots and logging volumes
VOLUME /var/log/memgraph
VOLUME /var/lib/memgraph
# Configuration volume
VOLUME /etc/memgraph
USER memgraph
WORKDIR /usr/lib/memgraph
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
CMD [""]

View File

@@ -1,3 +0,0 @@
.venv
dist
mgp.py

View File

@@ -1,201 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@@ -1,4 +0,0 @@
# mgp
PyPi package used for type hinting when creating MAGE modules. The get started
using MAGE repository checkout the repository here: https://github.com/memgraph/mage.

View File

@@ -1,255 +0,0 @@
from typing import Any
class MgpIterable:
def get() -> Any:
pass
def next() -> Any:
pass
class Vertex:
def is_valid() -> bool: # type: ignore
pass
def underlying_graph_is_mutable() -> bool: # type: ignore
pass
def iter_properties() -> MgpIterable: # type: ignore
pass
def get_property(self, property_name: str) -> "Property": # type: ignore
pass
def set_property(self, property_name: str, value: Any) -> "Property": # type: ignore
pass
def get_id() -> "VertexId": # type: ignore
pass
def label_at(self, index: int) -> "Label": # type: ignore
pass
def labels_count() -> int: # type: ignore
pass
def add_label(self, label: Any):
pass
def remove_label(self, label: Any):
pass
def iter_in_edges() -> MgpIterable: # type: ignore
pass
def iter_out_edges() -> MgpIterable: # type: ignore
pass
class Edge:
def is_valid() -> bool: # type: ignore
pass
def underlying_graph_is_mutable() -> bool: # type: ignore
pass
def iter_properties() -> MgpIterable: # type: ignore
pass
def get_property(self, property_name: str) -> "Property": # type: ignore
pass
def set_property(self, property_name: str, valuse: Any) -> "Property": # type: ignore
pass
def get_type_name() -> str: # type: ignore
pass
def get_id() -> "EdgeId": # type: ignore
pass
def from_vertex() -> Vertex: # type: ignore
pass
def to_vertex() -> Vertex: # type: ignore
pass
class Path:
def is_valid() -> bool: # type: ignore
pass
@staticmethod
def make_with_start(vertex: Vertex) -> "Path": # type: ignore
pass
class Graph:
def is_valid() -> bool: # type: ignore
pass
class CypherType:
pass
class Message:
def is_valid() -> bool: # type: ignore
pass
def source_type() -> str: # type: ignore
pass
def topic_name() -> str: # type: ignore
pass
def key() -> bytes: # type: ignore
pass
def timestamp() -> int: # type: ignore
pass
def offset() -> int: # type: ignore
pass
def payload() -> bytes: # type: ignore
pass
class Messages:
def is_valid() -> bool: # type: ignore
pass
def message_at(self, id: int) -> Message: # type: ignore
pass
def total_messages() -> int: # type: ignore
pass
class UnknownError(Exception):
pass
class UnableToAllocateError(Exception):
pass
class InsufficientBufferError(Exception):
pass
class OutOfRangeError(Exception):
pass
class LogicErrorError(Exception):
pass
class DeletedObjectError(Exception):
pass
class InvalidArgumentError(Exception):
pass
class KeyAlreadyExistsError(Exception):
pass
class ImmutableObjectError(Exception):
pass
class ValueConversionError(Exception):
pass
class SerializationError(Exception):
pass
def type_nullable(elem: Any):
pass
def type_list(elem: Any):
pass
def type_bool():
pass
def type_string():
pass
def type_int():
pass
def type_float():
pass
def type_number():
pass
def type_map():
pass
def type_node():
pass
def type_relationship():
pass
def type_path():
pass
def type_date():
pass
def type_local_time():
pass
def type_local_date_time():
pass
def type_duration():
pass
def type_any():
pass
class _MODULE:
@staticmethod
def add_read_procedure(wrapper):
pass
@staticmethod
def add_write_procedure(wrapper):
pass
@staticmethod
def add_transformation(wrapper):
pass
@staticmethod
def add_function(wrapper):
pass

View File

@@ -1,22 +0,0 @@
# How to publish new versions
## Prerequisites
1. Installed poetry
```
pip install poetry
```
2. Set up [API tokens](https://pypi.org/help/#apitoken)
3. Be a collaborator on [pypi](https://pypi.org/project/mgp/)
## Making changes
1. Make changes to the package
2. Bump version in `pyproject.tml`
3. `poetry build`
4. `poetry publish`
## Why is this not automatized?
Because someone always has to manually bump up the version in `pyproject.toml`
## Why does `_mgp.py` exists?
Because we are mocking here all the types that are created by Memgraph
in order to fix typing errors in `mgp.py`.

View File

@@ -1,23 +0,0 @@
[tool.poetry]
name = "mgp"
version = "1.0.0"
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
authors = [
"MasterMedo <mislav.vuletic@gmail.com>",
"jbajic <jure.bajic@memgraph.io>",
"katarinasupe <katarina.supe@memgraph.io>",
"antejavor <ante.javor@memgraph.io>",
"antaljanosbenjamin <benjamin.antal@memgraph.io>",
]
license = "Apache-2.0"
readme = "README.md"
include = ["mgp.py", "_mgp.py"]
[tool.poetry.dependencies]
python = "^3.7"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@@ -7,8 +7,8 @@ RUN yum -y update \
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -0,0 +1,14 @@
FROM centos:8
ARG TOOLCHAIN_VERSION
RUN dnf -y update \
&& dnf install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,14 +0,0 @@
FROM quay.io/centos/centos:stream9
ARG TOOLCHAIN_VERSION
RUN yum -y update \
&& yum install -y wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,17 +0,0 @@
FROM debian:11
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -4,10 +4,10 @@ services:
build:
context: centos-7
container_name: "mgbuild_centos-7"
mgbuild_centos-9:
mgbuild_centos-8:
build:
context: centos-9
container_name: "mgbuild_centos-9"
context: centos-8
container_name: "mgbuild_centos-8"
mgbuild_debian-10:
build:
context: debian-10
@@ -24,7 +24,3 @@ services:
build:
context: ubuntu-20.04
container_name: "mgbuild_ubuntu-20.04"
mgbuild_ubuntu-22.04:
build:
context: ubuntu-22.04
container_name: "mgbuild_ubuntu-22.04"

View File

@@ -3,7 +3,7 @@
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm)
SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
PROJECT_ROOT="$SCRIPT_DIR/../.."
TOOLCHAIN_VERSION="toolchain-v4"
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
@@ -67,18 +67,14 @@ make_package () {
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
echo "Installing dependencies..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
echo "Building targeted package..."
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
if [[ "$os" == "debian-11-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
fi
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
# ' is used instead of " because we need to run make within the allowed
# container resources.
# shellcheck disable=SC2016

View File

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -1,17 +0,0 @@
FROM ubuntu:22.04
ARG TOOLCHAIN_VERSION
# Stops tzdata interactive configuration.
ENV DEBIAN_FRONTEND=noninteractive
RUN apt update && apt install -y \
ca-certificates wget git
# Do NOT be smart here and clean the cache because the container is used in the
# stateful context.
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt
ENTRYPOINT ["sleep", "infinity"]

View File

@@ -37,7 +37,7 @@ const std::vector<Permission> kPermissionsAll = {
Permission::CONSTRAINT, Permission::DUMP, Permission::AUTH, Permission::REPLICATION,
Permission::DURABILITY, Permission::READ_FILE, Permission::FREE_MEMORY, Permission::TRIGGER,
Permission::CONFIG, Permission::STREAM, Permission::MODULE_READ, Permission::MODULE_WRITE,
Permission::WEBSOCKET};
Permission::WEBSOCKET, Permission::SCHEMA};
} // namespace
std::string PermissionToString(Permission permission) {
@@ -84,6 +84,8 @@ std::string PermissionToString(Permission permission) {
return "MODULE_WRITE";
case Permission::WEBSOCKET:
return "WEBSOCKET";
case Permission::SCHEMA:
return "SCHEMA";
}
}
@@ -183,137 +185,19 @@ bool operator==(const Permissions &first, const Permissions &second) {
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
const std::string ASTERISK = "*";
FineGrainedAccessPermissions::FineGrainedAccessPermissions(const std::unordered_set<std::string> &grants,
const std::unordered_set<std::string> &denies)
: grants_(grants), denies_(denies) {}
PermissionLevel FineGrainedAccessPermissions::Has(const std::string &permission) const {
if ((denies_.size() == 1 && denies_.find(ASTERISK) != denies_.end()) || denies_.find(permission) != denies_.end()) {
return PermissionLevel::DENY;
}
if ((grants_.size() == 1 && grants_.find(ASTERISK) != grants_.end()) || grants_.find(permission) != denies_.end()) {
return PermissionLevel::GRANT;
}
return PermissionLevel::NEUTRAL;
}
void FineGrainedAccessPermissions::Grant(const std::string &permission) {
if (permission == ASTERISK) {
grants_.clear();
grants_.insert(permission);
return;
}
auto deniedPermissionIter = denies_.find(permission);
if (deniedPermissionIter != denies_.end()) {
denies_.erase(deniedPermissionIter);
}
if (grants_.size() == 1 && grants_.find(ASTERISK) != grants_.end()) {
grants_.erase(ASTERISK);
}
if (grants_.find(permission) == grants_.end()) {
grants_.insert(permission);
}
}
void FineGrainedAccessPermissions::Revoke(const std::string &permission) {
if (permission == ASTERISK) {
grants_.clear();
denies_.clear();
return;
}
auto deniedPermissionIter = denies_.find(permission);
auto grantedPermissionIter = grants_.find(permission);
if (deniedPermissionIter != denies_.end()) {
denies_.erase(deniedPermissionIter);
}
if (grantedPermissionIter != grants_.end()) {
grants_.erase(grantedPermissionIter);
}
}
void FineGrainedAccessPermissions::Deny(const std::string &permission) {
if (permission == ASTERISK) {
denies_.clear();
denies_.insert(permission);
return;
}
auto grantedPermissionIter = grants_.find(permission);
if (grantedPermissionIter != grants_.end()) {
grants_.erase(grantedPermissionIter);
}
if (denies_.size() == 1 && denies_.find(ASTERISK) != denies_.end()) {
denies_.erase(ASTERISK);
}
if (denies_.find(permission) == denies_.end()) {
denies_.insert(permission);
}
}
nlohmann::json FineGrainedAccessPermissions::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["grants"] = grants_;
data["denies"] = denies_;
return data;
}
FineGrainedAccessPermissions FineGrainedAccessPermissions::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load permissions data!");
}
return FineGrainedAccessPermissions(data["grants"], data["denies"]);
}
const std::unordered_set<std::string> &FineGrainedAccessPermissions::grants() const { return grants_; }
const std::unordered_set<std::string> &FineGrainedAccessPermissions::denies() const { return denies_; }
bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return first.grants() == second.grants() && first.denies() == second.denies();
}
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second) {
return !(first == second);
}
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
Role::Role(const std::string &rolename, const Permissions &permissions,
const FineGrainedAccessPermissions &fine_grained_access_permissions)
: rolename_(utils::ToLowerCase(rolename)),
permissions_(permissions),
fine_grained_access_permissions_(fine_grained_access_permissions) {}
Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
const std::string &Role::rolename() const { return rolename_; }
const Permissions &Role::permissions() const { return permissions_; }
Permissions &Role::permissions() { return permissions_; }
const FineGrainedAccessPermissions &Role::fine_grained_access_permissions() const {
return fine_grained_access_permissions_;
}
FineGrainedAccessPermissions &Role::fine_grained_access_permissions() { return fine_grained_access_permissions_; }
nlohmann::json Role::Serialize() const {
nlohmann::json data = nlohmann::json::object();
data["rolename"] = rolename_;
data["permissions"] = permissions_.Serialize();
data["fine_grained_access_permissions"] = fine_grained_access_permissions_.Serialize();
return data;
}
@@ -321,14 +205,11 @@ Role Role::Deserialize(const nlohmann::json &data) {
if (!data.is_object()) {
throw AuthException("Couldn't load role data!");
}
if (!data["rolename"].is_string() || !data["permissions"].is_object() ||
!data["fine_grained_access_permissions"].is_object()) {
if (!data["rolename"].is_string() || !data["permissions"].is_object()) {
throw AuthException("Couldn't load role data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
auto fine_grained_access_permissions =
FineGrainedAccessPermissions::Deserialize(data["fine_grained_access_permissions"]);
return {data["rolename"], permissions, fine_grained_access_permissions};
return {data["rolename"], permissions};
}
bool operator==(const Role &first, const Role &second) {
@@ -337,12 +218,8 @@ bool operator==(const Role &first, const Role &second) {
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
const FineGrainedAccessPermissions &fine_grained_access_permissions)
: username_(utils::ToLowerCase(username)),
password_hash_(password_hash),
permissions_(permissions),
fine_grained_access_permissions_(fine_grained_access_permissions) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
bool User::CheckPassword(const std::string &password) {
if (password_hash_.empty()) return true;
@@ -391,35 +268,10 @@ Permissions User::GetPermissions() const {
return permissions_;
}
FineGrainedAccessPermissions User::GetFineGrainedAccessPermissions() const {
if (role_) {
std::unordered_set<std::string> resultGrants;
std::set_union(fine_grained_access_permissions_.grants().begin(), fine_grained_access_permissions_.grants().end(),
role_->fine_grained_access_permissions().grants().begin(),
role_->fine_grained_access_permissions().grants().end(),
std::inserter(resultGrants, resultGrants.begin()));
std::unordered_set<std::string> resultDenies;
std::set_union(fine_grained_access_permissions_.denies().begin(), fine_grained_access_permissions_.denies().end(),
role_->fine_grained_access_permissions().denies().begin(),
role_->fine_grained_access_permissions().denies().end(),
std::inserter(resultDenies, resultDenies.begin()));
return FineGrainedAccessPermissions(resultGrants, resultDenies);
}
return fine_grained_access_permissions_;
}
const std::string &User::username() const { return username_; }
const Permissions &User::permissions() const { return permissions_; }
Permissions &User::permissions() { return permissions_; }
const FineGrainedAccessPermissions &User::fine_grained_access_permissions() const {
return fine_grained_access_permissions_;
}
FineGrainedAccessPermissions &User::fine_grained_access_permissions() { return fine_grained_access_permissions_; }
const Role *User::role() const {
if (role_.has_value()) {
@@ -433,7 +285,6 @@ nlohmann::json User::Serialize() const {
data["username"] = username_;
data["password_hash"] = password_hash_;
data["permissions"] = permissions_.Serialize();
data["fine_grained_access_permissions"] = fine_grained_access_permissions_.Serialize();
// The role shouldn't be serialized here, it is stored as a foreign key.
return data;
}
@@ -446,9 +297,7 @@ User User::Deserialize(const nlohmann::json &data) {
throw AuthException("Couldn't load user data!");
}
auto permissions = Permissions::Deserialize(data["permissions"]);
auto fine_grained_access_permissions =
FineGrainedAccessPermissions::Deserialize(data["fine_grained_access_permissions"]);
return {data["username"], data["password_hash"], permissions, fine_grained_access_permissions};
return {data["username"], data["password_hash"], permissions};
}
bool operator==(const User &first, const User &second) {

View File

@@ -10,7 +10,6 @@
#include <optional>
#include <string>
#include <unordered_set>
#include <json/json.hpp>
@@ -39,7 +38,8 @@ enum class Permission : uint64_t {
STREAM = 1U << 17U,
MODULE_READ = 1U << 18U,
MODULE_WRITE = 1U << 19U,
WEBSOCKET = 1U << 20U
WEBSOCKET = 1U << 20U,
SCHEMA = 1U << 21U
};
// clang-format on
@@ -89,48 +89,15 @@ bool operator==(const Permissions &first, const Permissions &second);
bool operator!=(const Permissions &first, const Permissions &second);
class FineGrainedAccessPermissions final {
public:
explicit FineGrainedAccessPermissions(const std::unordered_set<std::string> &grants = {},
const std::unordered_set<std::string> &denies = {});
PermissionLevel Has(const std::string &permission) const;
void Grant(const std::string &permission);
void Revoke(const std::string &permission);
void Deny(const std::string &permission);
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static FineGrainedAccessPermissions Deserialize(const nlohmann::json &data);
const std::unordered_set<std::string> &grants() const;
const std::unordered_set<std::string> &denies() const;
private:
std::unordered_set<std::string> grants_{};
std::unordered_set<std::string> denies_{};
};
bool operator==(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
bool operator!=(const FineGrainedAccessPermissions &first, const FineGrainedAccessPermissions &second);
class Role final {
public:
Role(const std::string &rolename);
Role(const std::string &rolename, const Permissions &permissions,
const FineGrainedAccessPermissions &fine_grained_access_permissions);
Role(const std::string &rolename, const Permissions &permissions);
const std::string &rolename() const;
const Permissions &permissions() const;
Permissions &permissions();
const FineGrainedAccessPermissions &fine_grained_access_permissions() const;
FineGrainedAccessPermissions &fine_grained_access_permissions();
nlohmann::json Serialize() const;
@@ -142,7 +109,6 @@ class Role final {
private:
std::string rolename_;
Permissions permissions_;
FineGrainedAccessPermissions fine_grained_access_permissions_;
};
bool operator==(const Role &first, const Role &second);
@@ -152,8 +118,7 @@ class User final {
public:
User(const std::string &username);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
const FineGrainedAccessPermissions &fine_grained_access_permissions);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
/// @throw AuthException if unable to verify the password.
bool CheckPassword(const std::string &password);
@@ -166,14 +131,11 @@ class User final {
void ClearRole();
Permissions GetPermissions() const;
FineGrainedAccessPermissions GetFineGrainedAccessPermissions() const;
const std::string &username() const;
const Permissions &permissions() const;
Permissions &permissions();
const FineGrainedAccessPermissions &fine_grained_access_permissions() const;
FineGrainedAccessPermissions &fine_grained_access_permissions();
const Role *role() const;
@@ -188,7 +150,6 @@ class User final {
std::string username_;
std::string password_hash_;
Permissions permissions_;
FineGrainedAccessPermissions fine_grained_access_permissions_;
std::optional<Role> role_;
};

View File

@@ -11,14 +11,9 @@
#pragma once
#include "auth/models.hpp"
#include "query/frontend/ast/ast.hpp"
#include "storage/v2/id_types.hpp"
#include <cstdint>
namespace memgraph::query {
class FineGrainedAccessChecker {
public:
virtual bool IsUserAuthorizedLabels(const std::vector<memgraph::storage::LabelId> &label,
memgraph::query::DbAccessor *dba) const = 0;
};
} // namespace memgraph::query
namespace memgraph::common {
enum class SchemaType : uint8_t { BOOL, INT, STRING, DATE, LOCALTIME, LOCALDATETIME, DURATION };
} // namespace memgraph::common

View File

@@ -105,16 +105,9 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
boost::asio::socket_base::keep_alive option(true);
// Set a decorator to change the Server of the handshake
ws_.set_option(boost::beast::websocket::stream_base::decorator([&req](boost::beast::websocket::response_type &res) {
ws_.set_option(boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::response_type &res) {
res.set(boost::beast::http::field::server, std::string("Memgraph Bolt WS"));
// We need to do this to support WASM clients, which explicitly send this flag
// in their upgrade request
// Neo4j client breaks when this flag is sent
if (const auto secondary_protocol = req.base().find(boost::beast::http::field::sec_websocket_protocol);
secondary_protocol != res.base().end() && secondary_protocol->value() == "binary") {
res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
}
res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
}));
ws_.binary(true);
@@ -169,7 +162,7 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnRead, shared_from_this())));
}
void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
void OnRead(const boost::system::error_code &ec, [[maybe_unused]] const size_t bytes_transferred) {
// This indicates that the WebsocketSession was closed
if (ec == boost::beast::websocket::error::closed) {
return;
@@ -329,8 +322,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
socket.lowest_layer().non_blocking(false);
});
timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
spdlog::info("Accepted a connection from {}: {}:{}", service_name_, remote_endpoint_.address(),
remote_endpoint_.port());
spdlog::info("Accepted a connection from {}:", service_name_, remote_endpoint_.address(), remote_endpoint_.port());
}
void DoRead() {

View File

@@ -57,6 +57,8 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
return auth::Permission::MODULE_WRITE;
case query::AuthQuery::Privilege::WEBSOCKET:
return auth::Permission::WEBSOCKET;
case query::AuthQuery::Privilege::SCHEMA:
return auth::Permission::SCHEMA;
}
}
} // namespace memgraph::glue

View File

@@ -17,7 +17,6 @@
namespace memgraph::integrations {
inline constexpr int64_t kDefaultCheckBatchLimit{1};
inline constexpr int64_t kMinimumStartBatchLimit{1};
inline constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
inline constexpr std::chrono::milliseconds kMinimumInterval{1};
inline constexpr int64_t kMinimumSize{1};

View File

@@ -74,36 +74,6 @@ utils::BasicResult<std::string, std::vector<Message>> GetBatch(RdKafka::KafkaCon
return std::move(batch);
}
void CheckAndDestroyLastAssignmentIfNeeded(RdKafka::KafkaConsumer &consumer, const ConsumerInfo &info,
std::vector<RdKafka::TopicPartition *> &last_assignment) {
if (!last_assignment.empty()) {
if (const auto err = consumer.assign(last_assignment); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerStartFailedException(info.consumer_name,
fmt::format("Couldn't restore commited offsets: '{}'", RdKafka::err2str(err)));
}
RdKafka::TopicPartition::destroy(last_assignment);
}
}
void TryToConsumeBatch(RdKafka::KafkaConsumer &consumer, const ConsumerInfo &info,
const ConsumerFunction &consumer_function, const std::vector<Message> &batch) {
consumer_function(batch);
std::vector<RdKafka::TopicPartition *> partitions;
utils::OnScopeExit clear_partitions([&]() { RdKafka::TopicPartition::destroy(partitions); });
if (const auto err = consumer.assignment(partitions); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerCommitFailedException(
info.consumer_name, fmt::format("Couldn't get assignment to commit offsets: {}", RdKafka::err2str(err)));
}
if (const auto err = consumer.position(partitions); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerCommitFailedException(info.consumer_name,
fmt::format("Couldn't get offsets from librdkafka {}", RdKafka::err2str(err)));
}
if (const auto err = consumer.commitSync(partitions); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerCommitFailedException(info.consumer_name, RdKafka::err2str(err));
}
}
} // namespace
Message::Message(std::unique_ptr<RdKafka::Message> &&message) : message_{std::move(message)} {
@@ -251,21 +221,10 @@ void Consumer::Start() {
StartConsuming();
}
void Consumer::StartWithLimit(const uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
if (is_running_) {
throw ConsumerRunningException(info_.consumer_name);
void Consumer::StartIfStopped() {
if (!is_running_) {
StartConsuming();
}
if (limit_batches < kMinimumStartBatchLimit) {
throw ConsumerStartFailedException(
info_.consumer_name, fmt::format("Batch limit has to be greater than or equal to {}", kMinimumStartBatchLimit));
}
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
throw ConsumerStartFailedException(
info_.consumer_name,
fmt::format("Timeout has to be greater than or equal to {} milliseconds", kMinimumInterval.count()));
}
StartConsumingWithLimit(limit_batches, timeout);
}
void Consumer::Stop() {
@@ -285,7 +244,7 @@ void Consumer::StopIfRunning() {
}
}
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
const ConsumerFunction &check_consumer_function) const {
// NOLINTNEXTLINE (modernize-use-nullptr)
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
@@ -385,7 +344,13 @@ void Consumer::StartConsuming() {
is_running_.store(true);
CheckAndDestroyLastAssignmentIfNeeded(*consumer_, info_, last_assignment_);
if (!last_assignment_.empty()) {
if (const auto err = consumer_->assign(last_assignment_); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerStartFailedException(info_.consumer_name,
fmt::format("Couldn't restore commited offsets: '{}'", RdKafka::err2str(err)));
}
RdKafka::TopicPartition::destroy(last_assignment_);
}
thread_ = std::thread([this] {
static constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
@@ -396,18 +361,33 @@ void Consumer::StartConsuming() {
while (is_running_) {
auto maybe_batch = GetBatch(*consumer_, info_, is_running_);
if (maybe_batch.HasError()) {
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
maybe_batch.GetError());
break;
}
const auto &batch = maybe_batch.GetValue();
if (batch.empty()) {
continue;
}
if (batch.empty()) continue;
spdlog::info("Kafka consumer {} is processing a batch", info_.consumer_name);
try {
TryToConsumeBatch(*consumer_, info_, consumer_function_, batch);
consumer_function_(batch);
std::vector<RdKafka::TopicPartition *> partitions;
utils::OnScopeExit clear_partitions([&]() { RdKafka::TopicPartition::destroy(partitions); });
if (const auto err = consumer_->assignment(partitions); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerCheckFailedException(
info_.consumer_name, fmt::format("Couldn't get assignment to commit offsets: {}", RdKafka::err2str(err)));
}
if (const auto err = consumer_->position(partitions); err != RdKafka::ERR_NO_ERROR) {
throw ConsumerCheckFailedException(
info_.consumer_name, fmt::format("Couldn't get offsets from librdkafka {}", RdKafka::err2str(err)));
}
if (const auto err = consumer_->commitSync(partitions); err != RdKafka::ERR_NO_ERROR) {
spdlog::warn("Committing offset of consumer {} failed: {}", info_.consumer_name, RdKafka::err2str(err));
break;
}
} catch (const std::exception &e) {
spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
break;
@@ -418,44 +398,6 @@ void Consumer::StartConsuming() {
});
}
void Consumer::StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
MG_ASSERT(!is_running_, "Cannot start already running consumer!");
if (is_running_.exchange(true)) {
throw ConsumerRunningException(info_.consumer_name);
}
utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
CheckAndDestroyLastAssignmentIfNeeded(*consumer_, info_, last_assignment_);
const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
const auto start = std::chrono::steady_clock::now();
for (uint64_t batch_count = 0; batch_count < limit_batches;) {
const auto now = std::chrono::steady_clock::now();
if (now - start >= timeout_to_use) {
throw ConsumerStartFailedException(info_.consumer_name, "Timeout reached");
}
const auto maybe_batch = GetBatch(*consumer_, info_, is_running_);
if (maybe_batch.HasError()) {
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
}
const auto &batch = maybe_batch.GetValue();
if (batch.empty()) {
continue;
}
++batch_count;
spdlog::info("Kafka consumer {} is processing a batch", info_.consumer_name);
TryToConsumeBatch(*consumer_, info_, consumer_function_, batch);
spdlog::info("Kafka consumer {} finished processing", info_.consumer_name);
}
}
void Consumer::StopConsuming() {
is_running_.store(false);
if (thread_.joinable()) thread_.join();

View File

@@ -113,19 +113,11 @@ class Consumer final : public RdKafka::EventCb {
/// This method will start a new thread which will poll all the topics for messages.
///
/// @throws ConsumerRunningException if the consumer is already running
/// @throws ConsumerStartFailedException if the commited offsets cannot be restored
void Start();
/// Starts consuming messages.
/// Starts consuming messages if it is not started already.
///
/// This method will start a new thread which will poll all the topics for messages.
///
/// @param limit_batches the consumer will only consume the given number of batches.
/// @param timeout the maximum duration during which the command should run.
///
/// @throws ConsumerRunningException if the consumer is already running
/// @throws ConsumerStartFailedException if the commited offsets cannot be restored
void StartWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
void StartIfStopped();
/// Stops consuming messages.
///
@@ -144,9 +136,9 @@ class Consumer final : public RdKafka::EventCb {
/// used.
/// @param check_consumer_function a function to feed the received messages in, only used during this dry-run.
///
/// @throws ConsumerRunningException if the consumer is already running.
/// @throws ConsumerRunningException if the consumer is alredy running.
/// @throws ConsumerCheckFailedException if check isn't successful.
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
const ConsumerFunction &check_consumer_function) const;
/// Returns true if the consumer is actively consuming messages.
@@ -165,7 +157,6 @@ class Consumer final : public RdKafka::EventCb {
void event_cb(RdKafka::Event &event) override;
void StartConsuming();
void StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
void StopConsuming();
@@ -187,6 +178,7 @@ class Consumer final : public RdKafka::EventCb {
ConsumerFunction consumer_function_;
mutable std::atomic<bool> is_running_{false};
mutable std::vector<RdKafka::TopicPartition *> last_assignment_; // Protected by is_running_
std::optional<int64_t> limit_batches_{std::nullopt};
std::unique_ptr<RdKafka::KafkaConsumer, std::function<void(RdKafka::KafkaConsumer *)>> consumer_;
std::thread thread_;
ConsumerRebalanceCb cb_;

View File

@@ -64,16 +64,4 @@ class TopicNotFoundException : public KafkaStreamException {
TopicNotFoundException(const std::string_view consumer_name, const std::string_view topic_name)
: KafkaStreamException("Kafka consumer {} cannot find topic {}", consumer_name, topic_name) {}
};
class ConsumerCommitFailedException : public KafkaStreamException {
public:
ConsumerCommitFailedException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Committing offset of consumer {} failed: {}", consumer_name, error) {}
};
class ConsumerReadMessagesFailedException : public KafkaStreamException {
public:
ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
: KafkaStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
};
} // namespace memgraph::integrations::kafka

View File

@@ -11,14 +11,13 @@
#include "integrations/pulsar/consumer.hpp"
#include <algorithm>
#include <chrono>
#include <thread>
#include <fmt/format.h>
#include <pulsar/Client.h>
#include <pulsar/InitialPosition.h>
#include <chrono>
#include <thread>
#include "integrations/constants.hpp"
#include "integrations/pulsar/exceptions.hpp"
#include "utils/concepts.hpp"
@@ -34,10 +33,6 @@ namespace {
template <typename T>
concept PulsarConsumer = utils::SameAsAnyOf<T, pulsar_client::Consumer, pulsar_client::Reader>;
template <typename TFunc>
concept PulsarMessageGetter =
std::same_as<const pulsar_client::Message &, std::invoke_result_t<TFunc, const Message &>>;
pulsar_client::Result ConsumeMessage(pulsar_client::Consumer &consumer, pulsar_client::Message &message,
int remaining_timeout_in_ms) {
return consumer.receive(message, remaining_timeout_in_ms);
@@ -102,26 +97,6 @@ pulsar_client::Client CreateClient(const std::string &service_url) {
conf.setLogger(new SpdlogLoggerFactory);
return {service_url, conf};
}
template <PulsarConsumer TConsumer, PulsarMessageGetter TPulsarMessageGetter>
void TryToConsumeBatch(TConsumer &consumer, const ConsumerInfo &info, const ConsumerFunction &consumer_function,
pulsar_client::MessageId &last_message_id, const std::vector<Message> &batch,
const TPulsarMessageGetter &message_getter) {
consumer_function(batch);
auto has_message_failed = [&consumer, &info, &last_message_id, &message_getter](const auto &message) {
if (const auto result = consumer.acknowledge(message_getter(message)); result != pulsar_client::ResultOk) {
spdlog::warn("Acknowledging a message of consumer {} failed: {}", info.consumer_name, result);
return true;
}
last_message_id = message_getter(message).getMessageId();
return false;
};
if (std::ranges::any_of(batch, has_message_failed)) {
throw ConsumerAcknowledgeMessagesFailedException(info.consumer_name);
}
}
} // namespace
Message::Message(pulsar_client::Message &&message) : message_{std::move(message)} {}
@@ -162,24 +137,6 @@ void Consumer::Start() {
StartConsuming();
}
void Consumer::StartWithLimit(const uint64_t limit_batches,
const std::optional<std::chrono::milliseconds> timeout) const {
if (is_running_) {
throw ConsumerRunningException(info_.consumer_name);
}
if (limit_batches < kMinimumStartBatchLimit) {
throw ConsumerStartFailedException(
info_.consumer_name, fmt::format("Batch limit has to be greater than or equal to {}", kMinimumStartBatchLimit));
}
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
throw ConsumerStartFailedException(
info_.consumer_name,
fmt::format("Timeout has to be greater than or equal to {} milliseconds", kMinimumInterval.count()));
}
StartConsumingWithLimit(limit_batches, timeout);
}
void Consumer::Stop() {
if (!is_running_) {
throw ConsumerStoppedException(info_.consumer_name);
@@ -197,7 +154,7 @@ void Consumer::StopIfRunning() {
}
}
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
const ConsumerFunction &check_consumer_function) const {
// NOLINTNEXTLINE (modernize-use-nullptr)
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
@@ -283,7 +240,9 @@ void Consumer::StartConsuming() {
auto maybe_batch = GetBatch(consumer_, info_, is_running_, last_message_id_);
if (maybe_batch.HasError()) {
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
maybe_batch.GetError());
break;
}
const auto &batch = maybe_batch.GetValue();
@@ -295,8 +254,18 @@ void Consumer::StartConsuming() {
spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
try {
TryToConsumeBatch(consumer_, info_, consumer_function_, last_message_id_, batch,
[&](const Message &message) -> const pulsar_client::Message & { return message.message_; });
consumer_function_(batch);
if (std::any_of(batch.begin(), batch.end(), [&](const auto &message) {
if (const auto result = consumer_.acknowledge(message.message_); result != pulsar_client::ResultOk) {
spdlog::warn("Acknowledging a message of consumer {} failed: {}", info_.consumer_name, result);
return true;
}
last_message_id_ = message.message_.getMessageId();
return false;
})) {
break;
}
} catch (const std::exception &e) {
spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
break;
@@ -308,43 +277,6 @@ void Consumer::StartConsuming() {
});
}
void Consumer::StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
if (is_running_.exchange(true)) {
throw ConsumerRunningException(info_.consumer_name);
}
utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
const auto start = std::chrono::steady_clock::now();
for (uint64_t batch_count = 0; batch_count < limit_batches;) {
const auto now = std::chrono::steady_clock::now();
if (now - start >= timeout_to_use) {
throw ConsumerCheckFailedException(info_.consumer_name, "Timeout reached");
}
const auto maybe_batch = GetBatch(consumer_, info_, is_running_, last_message_id_);
if (maybe_batch.HasError()) {
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
}
const auto &batch = maybe_batch.GetValue();
if (batch.empty()) {
continue;
}
++batch_count;
spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
TryToConsumeBatch(consumer_, info_, consumer_function_, last_message_id_, batch,
[](const Message &message) -> const pulsar_client::Message & { return message.message_; });
spdlog::info("Pulsar consumer {} finished processing", info_.consumer_name);
}
}
void Consumer::StopConsuming() {
is_running_.store(false);
if (thread_.joinable()) {

View File

@@ -58,27 +58,25 @@ class Consumer final {
bool IsRunning() const;
void Start();
void StartWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
void Stop();
void StopIfRunning();
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
const ConsumerFunction &check_consumer_function) const;
const ConsumerInfo &Info() const;
private:
void StartConsuming();
void StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
void StopConsuming();
ConsumerInfo info_;
mutable pulsar_client::Client client_;
mutable pulsar_client::Consumer consumer_;
pulsar_client::Consumer consumer_;
ConsumerFunction consumer_function_;
mutable std::atomic<bool> is_running_{false};
mutable pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()}; // Protected by is_running_
pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()};
std::thread thread_;
};
} // namespace memgraph::integrations::pulsar

View File

@@ -55,16 +55,4 @@ class TopicNotFoundException : public PulsarStreamException {
TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
: PulsarStreamException("Pulsar consumer {} cannot find topic {}", consumer_name, topic_name) {}
};
class ConsumerReadMessagesFailedException : public PulsarStreamException {
public:
ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
: PulsarStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
};
class ConsumerAcknowledgeMessagesFailedException : public PulsarStreamException {
public:
explicit ConsumerAcknowledgeMessagesFailedException(const std::string_view consumer_name)
: PulsarStreamException("Acknowledging a message of consumer {} has failed!", consumer_name) {}
};
} // namespace memgraph::integrations::pulsar

View File

@@ -216,11 +216,6 @@ DEFINE_bool(telemetry_enabled, false,
"the database runtime (vertex and edge counts and resource usage) "
"to allow for easier improvement of the product.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_restore_replicas_on_startup, true,
"Controls replicas should be restored automatically."); // TODO(42jeremy) this must be removed once T0835
// is implemented.
// Streams flags
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(
@@ -257,11 +252,6 @@ DEFINE_double(query_execution_timeout_sec, 600,
"Maximum allowed query execution time. Queries exceeding this "
"limit will be aborted. Value of 0 means no limit.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(replication_replica_check_frequency_sec, 1,
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
"The MAIN instance allocates a new thread for each REPLICA.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(
memory_limit, 0,
@@ -506,7 +496,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
if (first_user) {
spdlog::info("{} is first created user. Granting all privileges.", username);
GrantPrivilege(username, memgraph::query::kPrivilegesAll, {"*"});
GrantPrivilege(username, memgraph::query::kPrivilegesAll);
}
return user_added;
@@ -751,28 +741,9 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
}
}
memgraph::auth::User *GetUser(const std::string &username) override {
if (!std::regex_match(username, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user name.");
}
try {
auto locked_auth = auth_->Lock();
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
return new memgraph::auth::User(*user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void GrantPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) override {
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
@@ -781,9 +752,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
}
void DenyPrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) override {
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
@@ -792,9 +762,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
}
void RevokePrivilege(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) override {
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
// TODO (mferencevic): should we first check that the
// privilege is granted/denied/revoked before
// unconditionally granting/denying/revoking it?
@@ -805,8 +774,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
private:
template <class TEditFun>
void EditPermissions(const std::string &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels, const TEditFun &edit_fun) {
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges, const TEditFun &edit_fun) {
if (!std::regex_match(user_or_role, name_regex_)) {
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
}
@@ -826,18 +794,11 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
for (const auto &permission : permissions) {
edit_fun(&user->permissions(), permission);
}
for (const auto &label : labels) {
edit_fun(&user->fine_grained_access_permissions(), label);
}
locked_auth->SaveUser(*user);
} else {
for (const auto &permission : permissions) {
edit_fun(&role->permissions(), permission);
}
for (const auto &label : labels) {
edit_fun(&user->fine_grained_access_permissions(), label);
}
locked_auth->SaveRole(*role);
}
} catch (const memgraph::auth::AuthException &e) {
@@ -1229,8 +1190,7 @@ int main(int argc, char **argv) {
.snapshot_retention_count = FLAGS_storage_snapshot_retention_count,
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit,
.restore_replicas_on_startup = FLAGS_storage_restore_replicas_on_startup},
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit},
.transaction = {.isolation_level = ParseIsolationLevel()}};
if (FLAGS_storage_snapshot_interval_sec == 0) {
if (FLAGS_storage_wal_enabled) {
@@ -1256,7 +1216,6 @@ int main(int argc, char **argv) {
&db,
{.query = {.allow_load_csv = FLAGS_allow_load_csv},
.execution_timeout_sec = FLAGS_query_execution_timeout_sec,
.replication_replica_check_frequency = std::chrono::seconds(FLAGS_replication_replica_check_frequency_sec),
.default_kafka_bootstrap_servers = FLAGS_kafka_bootstrap_servers,
.default_pulsar_service_url = FLAGS_pulsar_service_url,
.stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries,
@@ -1281,8 +1240,9 @@ int main(int argc, char **argv) {
// the triggers
auto storage_accessor = interpreter_context.db->Access();
auto dba = memgraph::query::DbAccessor{&storage_accessor};
interpreter_context.trigger_store.RestoreTriggers(
&interpreter_context.ast_cache, &dba, interpreter_context.config.query, interpreter_context.auth_checker);
interpreter_context.trigger_store.RestoreTriggers(&interpreter_context.ast_cache, &dba,
&interpreter_context.antlr_lock, interpreter_context.config.query,
interpreter_context.auth_checker);
}
// As the Stream transformations are using modules, they have to be restored after the query modules are loaded.

View File

@@ -9,5 +9,4 @@ target_link_libraries(mg-memory mg-utils fmt)
if (ENABLE_JEMALLOC)
target_link_libraries(mg-memory Jemalloc::Jemalloc)
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1)
endif()

View File

@@ -170,7 +170,7 @@ enum class CsvParserState {
EXPECT_DELIMITER,
};
bool SubstringStartsWith(const std::string_view str, size_t pos, const std::string_view what) {
bool SubstringStartsWith(const std::string_view &str, size_t pos, const std::string_view &what) {
return memgraph::utils::StartsWith(memgraph::utils::Substr(str, pos), what);
}

View File

@@ -82,7 +82,7 @@ add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.9.2-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}

View File

@@ -21,8 +21,6 @@ struct InterpreterConfig {
// The default execution timeout is 10 minutes.
double execution_timeout_sec{600.0};
// The same as \ref memgraph::storage::replication::ReplicationClientConfig
std::chrono::seconds replication_replica_check_frequency{1};
std::string default_kafka_bootstrap_servers;
std::string default_pulsar_service_url;

View File

@@ -14,7 +14,6 @@
#include <type_traits>
#include "query/common.hpp"
#include "query/fine_grained_access_checker.hpp"
#include "query/frontend/semantic/symbol_table.hpp"
#include "query/metadata.hpp"
#include "query/parameters.hpp"
@@ -73,7 +72,6 @@ struct ExecutionContext {
ExecutionStats execution_stats;
TriggerContextCollector *trigger_context_collector{nullptr};
utils::AsyncTimer timer;
FineGrainedAccessChecker *fine_grained_access_checker{nullptr};
};
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");

View File

@@ -21,7 +21,8 @@ namespace memgraph::query {
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config) {
utils::SkipList<QueryCacheEntry> *cache, utils::SpinLock *antlr_lock,
const InterpreterConfig::Query &query_config) {
// Strip the query for caching purposes. The process of stripping a query
// "normalizes" it by replacing any literals with new parameters. This
// results in just the *structure* of the query being taken into account for
@@ -62,16 +63,20 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
};
if (it == accessor.end()) {
try {
parser = std::make_unique<frontend::opencypher::Parser>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<frontend::opencypher::Parser>(query_string);
{
std::unique_lock<utils::SpinLock> guard(*antlr_lock);
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
try {
parser = std::make_unique<frontend::opencypher::Parser>(stripped_query.query());
} catch (const SyntaxException &e) {
// There is a syntax exception in the stripped query. Re-run the parser
// on the original query to get an appropriate error messsage.
parser = std::make_unique<frontend::opencypher::Parser>(query_string);
// If an exception was not thrown here, the stripper messed something
// up.
LOG_FATAL("The stripped query can't be parsed, but the original can.");
}
}
// Convert the ANTLR4 parse tree into an AST.

View File

@@ -111,7 +111,8 @@ struct ParsedQuery {
};
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config);
utils::SkipList<QueryCacheEntry> *cache, utils::SpinLock *antlr_lock,
const InterpreterConfig::Query &query_config);
class SingleNodeLogicalPlan final : public LogicalPlan {
public:

View File

@@ -310,11 +310,11 @@ class DbAccessor final {
return std::make_optional<VertexAccessor>(*value);
}
storage::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
storage::PropertyId NameToProperty(const std::string_view &name) { return accessor_->NameToProperty(name); }
storage::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
storage::LabelId NameToLabel(const std::string_view &name) { return accessor_->NameToLabel(name); }
storage::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
storage::EdgeTypeId NameToEdgeType(const std::string_view &name) { return accessor_->NameToEdgeType(name); }
const std::string &PropertyToName(storage::PropertyId prop) const { return accessor_->PropertyToName(prop); }

View File

@@ -46,7 +46,7 @@ const char *kInternalPropertyId = "__mg_id__";
const char *kInternalVertexLabel = "__mg_vertex__";
/// A helper function that escapes label, edge type and property names.
std::string EscapeName(const std::string_view value) {
std::string EscapeName(const std::string_view &value) {
std::string out;
out.reserve(value.size() + 2);
out.append(1, '`');

View File

@@ -17,6 +17,7 @@
#include <variant>
#include <vector>
#include "common/types.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
#include "query/frontend/semantic/symbol.hpp"
#include "query/interpret/awesome_memgraph_functions.hpp"
@@ -2234,7 +2235,6 @@ cpp<#
(:serialize (:slk))
(:clone))
(lcp:define-class auth-query (query)
((action "Action" :scope :public)
(user "std::string" :scope :public)
@@ -2243,8 +2243,7 @@ cpp<#
(password "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(privileges "std::vector<Privilege>" :scope :public)
(labels "std::vector<std::string>" :scope :public))
(privileges "std::vector<Privilege>" :scope :public))
(:public
(lcp:define-enum action
(create-role drop-role show-roles create-user set-password drop-user
@@ -2255,7 +2254,7 @@ cpp<#
(lcp:define-enum privilege
(create delete match merge set remove index stats auth constraint
dump replication durability read_file free_memory trigger config stream module_read module_write
websocket)
websocket schema)
(:serialize))
#>cpp
AuthQuery() = default;
@@ -2266,14 +2265,13 @@ cpp<#
#>cpp
AuthQuery(Action action, std::string user, std::string role,
std::string user_or_role, Expression *password,
std::vector<Privilege> privileges, std::vector<std::string> labels)
std::vector<Privilege> privileges)
: action_(action),
user_(user),
role_(role),
user_or_role_(user_or_role),
password_(password),
privileges_(privileges),
labels_(labels) {}
privileges_(privileges) {}
cpp<#)
(:private
#>cpp
@@ -2298,7 +2296,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM,
AuthQuery::Privilege::MODULE_READ, AuthQuery::Privilege::MODULE_WRITE,
AuthQuery::Privilege::WEBSOCKET};
AuthQuery::Privilege::WEBSOCKET, AuthQuery::Privilege::SCHEMA};
cpp<#
(lcp:define-class info-query (query)
@@ -2378,7 +2376,10 @@ cpp<#
(port "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression"))
(sync_mode "SyncMode" :scope :public))
(sync_mode "SyncMode" :scope :public)
(timeout "Expression *" :initval "nullptr" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression")))
(:public
(lcp:define-enum action
@@ -2391,9 +2392,6 @@ cpp<#
(lcp:define-enum sync-mode
(sync async)
(:serialize))
(lcp:define-enum replica-state
(ready replicating recovery invalid)
(:serialize))
#>cpp
ReplicationQuery() = default;
@@ -2668,5 +2666,37 @@ cpp<#
(:serialize (:slk))
(:clone))
(lcp:define-class schema-query (query)
((action "Action" :scope :public)
(label "LabelIx" :scope :public
:slk-load (lambda (member)
#>cpp
slk::Load(&self->${member}, reader, storage);
cpp<#)
:clone (lambda (source dest)
#>cpp
${dest} = storage->GetLabelIx(${source}.name);
cpp<#))
(schema_type_map "std::unordered_map<PropertyIx, common::SchemaType>"
:slk-save #'slk-save-property-map
:slk-load #'slk-load-property-map
:scope :public))
(:public
(lcp:define-enum action
(create-schema drop-schema show-schema show-schemas)
(:serialize))
#>cpp
SchemaQuery() = default;
DEFVISITABLE(QueryVisitor<void>);
cpp<#)
(:private
#>cpp
friend class AstStorage;
cpp<#)
(:serialize (:slk))
(:clone))
(lcp:pop-namespace) ;; namespace query
(lcp:pop-namespace) ;; namespace memgraph

View File

@@ -94,6 +94,7 @@ class StreamQuery;
class SettingQuery;
class VersionQuery;
class Foreach;
class SchemaQuery;
using TreeCompositeVisitor = utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@@ -125,9 +126,9 @@ class ExpressionVisitor
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch> {};
template <class TResult>
class QueryVisitor
: public utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery, InfoQuery,
ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery, FreeMemoryQuery, TriggerQuery,
IsolationLevelQuery, CreateSnapshotQuery, StreamQuery, SettingQuery, VersionQuery> {};
class QueryVisitor : public utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery,
InfoQuery, ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery,
FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery,
StreamQuery, SettingQuery, VersionQuery, SchemaQuery> {};
} // namespace memgraph::query

File diff suppressed because it is too large Load Diff

View File

@@ -115,7 +115,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
auto operators = ExtractOperators(all_children, allowed_operators);
for (auto *expression : _expressions) {
expressions.push_back(std::any_cast<Expression *>(expression->accept(this)));
expressions.push_back(expression->accept(this));
}
Expression *first_operand = expressions[0];
@@ -131,7 +131,7 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
DMG_ASSERT(_expression, "can't happen");
auto operators = ExtractOperators(all_children, allowed_operators);
Expression *expression = std::any_cast<Expression *>(_expression->accept(this));
Expression *expression = _expression->accept(this);
for (int i = (int)operators.size() - 1; i >= 0; --i) {
expression = CreateUnaryOperatorByToken(operators[i], expression);
}
@@ -478,11 +478,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitShowPrivileges(MemgraphCypher::ShowPrivilegesContext *ctx) override;
/**
* @return AuthQuery::LabelList
*/
antlrcpp::Any visitLabelList(MemgraphCypher::LabelListContext *ctx) override;
/**
* @return AuthQuery*
*/
@@ -854,6 +849,41 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitForeach(MemgraphCypher::ForeachContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitPropertyType(MemgraphCypher::PropertyTypeContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitSchemaTypeMap(MemgraphCypher::SchemaTypeMapContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitSchemaQuery(MemgraphCypher::SchemaQueryContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitShowSchema(MemgraphCypher::ShowSchemaContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitShowSchemas(MemgraphCypher::ShowSchemasContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitCreateSchema(MemgraphCypher::CreateSchemaContext *ctx) override;
/**
* @return Schema*
*/
antlrcpp::Any visitDropSchema(MemgraphCypher::DropSchemaContext *ctx) override;
public:
Query *query() { return query_; }
const static std::string kAnonPrefix;

View File

@@ -46,17 +46,16 @@ memgraphCypherKeyword : cypherKeyword
| DROP
| DUMP
| EXECUTE
| FOR
| FOREACH
| FREE
| FROM
| FOR
| FOREACH
| GLOBAL
| GRANT
| HEADER
| IDENTIFIED
| ISOLATION
| KAFKA
| LABELS
| LEVEL
| LOAD
| LOCK
@@ -77,6 +76,8 @@ memgraphCypherKeyword : cypherKeyword
| ROLE
| ROLES
| QUOTE
| SCHEMA
| SCHEMAS
| SESSION
| SETTING
| SETTINGS
@@ -123,6 +124,7 @@ query : cypherQuery
| streamQuery
| settingQuery
| versionQuery
| schemaQuery
;
authQuery : createRole
@@ -193,6 +195,12 @@ settingQuery : setSetting
| showSettings
;
schemaQuery : showSchema
| showSchemas
| createSchema
| dropSchema
;
loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
( IGNORE BAD ) ?
( DELIMITER delimiter ) ?
@@ -255,17 +263,11 @@ privilege : CREATE
| MODULE_READ
| MODULE_WRITE
| WEBSOCKET
| LABELS labels=labelList
| SCHEMA
;
privilegeList : privilege ( ',' privilege )* ;
labelList : '*' | listOfLabels ;
listOfLabels : label ( ',' label )* ;
label : COLON symbolicName ;
showPrivileges : SHOW PRIVILEGES FOR userOrRole=userOrRoleName ;
showRoleForUser : SHOW ROLE FOR user=userOrRoleName ;
@@ -284,6 +286,7 @@ replicaName : symbolicName ;
socketAddress : literal ;
registerReplica : REGISTER REPLICA replicaName ( SYNC | ASYNC )
( WITH TIMEOUT timeout=literal ) ?
TO socketAddress ;
dropReplica : DROP REPLICA replicaName ;
@@ -358,7 +361,7 @@ pulsarCreateStream : CREATE PULSAR STREAM streamName ( pulsarCreateStreamConfig
dropStream : DROP STREAM streamName ;
startStream : START STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ;
startStream : START STREAM streamName ;
startAllStreams : START ALL STREAMS ;
@@ -381,3 +384,17 @@ showSetting : SHOW DATABASE SETTING settingName ;
showSettings : SHOW DATABASE SETTINGS ;
versionQuery : SHOW VERSION ;
showSchema : SHOW SCHEMA ON ':' labelName ;
showSchemas : SHOW SCHEMAS ;
propertyType : symbolicName ;
propertyKeyTypePair : propertyKeyName propertyType ;
schemaTypeMap : '(' propertyKeyTypePair ( ',' propertyKeyTypePair )* ')' ;
createSchema : CREATE SCHEMA ON ':' labelName schemaTypeMap ;
dropSchema : DROP SCHEMA ON ':' labelName ;

View File

@@ -66,7 +66,6 @@ IDENTIFIED : I D E N T I F I E D ;
IGNORE : I G N O R E ;
ISOLATION : I S O L A T I O N ;
KAFKA : K A F K A ;
LABELS : L A B E L S ;
LEVEL : L E V E L ;
LOAD : L O A D ;
LOCK : L O C K ;
@@ -90,6 +89,8 @@ REVOKE : R E V O K E ;
ROLE : R O L E ;
ROLES : R O L E S ;
QUOTE : Q U O T E ;
SCHEMA : S C H E M A ;
SCHEMAS : S C H E M A S ;
SERVICE_URL : S E R V I C E UNDERSCORE U R L ;
SESSION : S E S S I O N ;
SETTING : S E T T I N G ;

View File

@@ -80,6 +80,8 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
void Visit(VersionQuery & /*version_query*/) override { AddPrivilege(AuthQuery::Privilege::STATS); }
void Visit(SchemaQuery & /*schema_query*/) override { AddPrivilege(AuthQuery::Privilege::SCHEMA); }
bool PreVisit(Create & /*unused*/) override {
AddPrivilege(AuthQuery::Privilege::CREATE);
return false;

View File

@@ -206,7 +206,7 @@ const trie::Trie kKeywords = {"union",
"version",
"websocket",
"foreach",
"labels"};
"schema"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

View File

@@ -882,36 +882,21 @@ TypedValue Id(const TypedValue *args, int64_t nargs, const FunctionContext &ctx)
}
TypedValue ToString(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
FType<Or<Null, String, Number, Date, LocalTime, LocalDateTime, Duration, Bool>>("toString", args, nargs);
FType<Or<Null, String, Number, Bool>>("toString", args, nargs);
const auto &arg = args[0];
if (arg.IsNull()) {
return TypedValue(ctx.memory);
}
if (arg.IsString()) {
} else if (arg.IsString()) {
return TypedValue(arg, ctx.memory);
}
if (arg.IsInt()) {
} else if (arg.IsInt()) {
// TODO: This is making a pointless copy of std::string, we may want to
// use a different conversion to string
return TypedValue(std::to_string(arg.ValueInt()), ctx.memory);
}
if (arg.IsDouble()) {
} else if (arg.IsDouble()) {
return TypedValue(std::to_string(arg.ValueDouble()), ctx.memory);
} else {
return TypedValue(arg.ValueBool() ? "true" : "false", ctx.memory);
}
if (arg.IsDate()) {
return TypedValue(arg.ValueDate().ToString(), ctx.memory);
}
if (arg.IsLocalTime()) {
return TypedValue(arg.ValueLocalTime().ToString(), ctx.memory);
}
if (arg.IsLocalDateTime()) {
return TypedValue(arg.ValueLocalDateTime().ToString(), ctx.memory);
}
if (arg.IsDuration()) {
return TypedValue(arg.ValueDuration().ToString(), ctx.memory);
}
return TypedValue(arg.ValueBool() ? "true" : "false", ctx.memory);
}
TypedValue Timestamp(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {

View File

@@ -111,7 +111,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
TypedValue Visit(IfOperator &if_operator) override {
auto condition = if_operator.condition_->Accept(*this);
if (condition.IsNull()) {
return if_operator.else_expression_->Accept(*this);
return if_operator.then_expression_->Accept(*this);
}
if (condition.type() != TypedValue::Type::Bool) {
// At the moment IfOperator is used only in CASE construct.
@@ -716,7 +716,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
template <class TRecordAccessor>
storage::PropertyValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
storage::PropertyValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view &name) {
auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
if (maybe_prop.HasError() && maybe_prop.GetError() == storage::Error::NONEXISTENT_OBJECT) {
// This is a very nasty and temporary hack in order to make MERGE work.

View File

@@ -29,7 +29,6 @@
#include "query/db_accessor.hpp"
#include "query/dump.hpp"
#include "query/exceptions.hpp"
#include "query/fine_grained_access_checker.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/ast/ast_visitor.hpp"
#include "query/frontend/ast/cypher_main_visitor.hpp"
@@ -45,7 +44,7 @@
#include "query/trigger.hpp"
#include "query/typed_value.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/replication/enums.hpp"
#include "storage/v2/schemas.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
#include "utils/event_counter.hpp"
@@ -162,8 +161,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
void RegisterReplica(const std::string &name, const std::string &socket_address,
const ReplicationQuery::SyncMode sync_mode,
const std::chrono::seconds replica_check_frequency) override {
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout) override {
if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
// replica can't register another replica
throw QueryRuntimeException("Replica can't register another replica!");
@@ -185,9 +183,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::kDefaultReplicationPort);
if (maybe_ip_and_port) {
auto [ip, port] = *maybe_ip_and_port;
auto ret = db_->RegisterReplica(name, {std::move(ip), port}, repl_mode,
storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID,
{.replica_check_frequency = replica_check_frequency, .ssl = std::nullopt});
auto ret =
db_->RegisterReplica(name, {std::move(ip), port}, repl_mode, {.timeout = timeout, .ssl = std::nullopt});
if (ret.HasError()) {
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
}
@@ -230,24 +227,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
replica.sync_mode = ReplicationQuery::SyncMode::ASYNC;
break;
}
replica.current_timestamp_of_replica = repl_info.timestamp_info.current_timestamp_of_replica;
replica.current_number_of_timestamp_behind_master =
repl_info.timestamp_info.current_number_of_timestamp_behind_master;
switch (repl_info.state) {
case storage::replication::ReplicaState::READY:
replica.state = ReplicationQuery::ReplicaState::READY;
break;
case storage::replication::ReplicaState::REPLICATING:
replica.state = ReplicationQuery::ReplicaState::REPLICATING;
break;
case storage::replication::ReplicaState::RECOVERY:
replica.state = ReplicationQuery::ReplicaState::RECOVERY;
break;
case storage::replication::ReplicaState::INVALID:
replica.state = ReplicationQuery::ReplicaState::INVALID;
break;
if (repl_info.timeout) {
replica.timeout = *repl_info.timeout;
}
return replica;
@@ -260,24 +241,6 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
private:
storage::Storage *db_;
};
class FineGrainedAccessChecker final : public memgraph::query::FineGrainedAccessChecker {
public:
explicit FineGrainedAccessChecker(memgraph::auth::User *user) : user_{user} {}
bool IsUserAuthorizedLabels(const std::vector<memgraph::storage::LabelId> &labels,
memgraph::query::DbAccessor *dba) const final {
auto labelPermissions = user_->GetFineGrainedAccessPermissions();
return std::any_of(labels.begin(), labels.end(), [&labelPermissions, &dba](const auto label) {
return labelPermissions.Has(dba->LabelToName(label)) == memgraph::auth::PermissionLevel::GRANT;
});
}
private:
memgraph::auth::User *user_;
};
/// returns false if the replication role can't be set
/// @throw QueryRuntimeException if an error ocurred.
@@ -299,7 +262,6 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
std::string rolename = auth_query->role_;
std::string user_or_role = auth_query->user_or_role_;
std::vector<AuthQuery::Privilege> privileges = auth_query->privileges_;
std::vector<std::string> labels = auth_query->labels_;
auto password = EvaluateOptionalExpression(auth_query->password_, &evaluator);
Callback callback;
@@ -329,7 +291,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
// If the license is not valid we create users with admin access
if (!valid_enterprise_license) {
spdlog::warn("Granting all the privileges to {}.", username);
auth->GrantPrivilege(username, kPrivilegesAll, {"*"});
auth->GrantPrivilege(username, kPrivilegesAll);
}
return std::vector<std::vector<TypedValue>>();
@@ -404,20 +366,20 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
};
return callback;
case AuthQuery::Action::GRANT_PRIVILEGE:
callback.fn = [auth, user_or_role, privileges, labels] {
auth->GrantPrivilege(user_or_role, privileges, labels);
callback.fn = [auth, user_or_role, privileges] {
auth->GrantPrivilege(user_or_role, privileges);
return std::vector<std::vector<TypedValue>>();
};
return callback;
case AuthQuery::Action::DENY_PRIVILEGE:
callback.fn = [auth, user_or_role, privileges, labels] {
auth->DenyPrivilege(user_or_role, privileges, labels);
callback.fn = [auth, user_or_role, privileges] {
auth->DenyPrivilege(user_or_role, privileges);
return std::vector<std::vector<TypedValue>>();
};
return callback;
case AuthQuery::Action::REVOKE_PRIVILEGE: {
callback.fn = [auth, user_or_role, privileges, labels] {
auth->RevokePrivilege(user_or_role, privileges, labels);
callback.fn = [auth, user_or_role, privileges] {
auth->RevokePrivilege(user_or_role, privileges);
return std::vector<std::vector<TypedValue>>();
};
return callback;
@@ -487,7 +449,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
return callback;
}
case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: {
callback.header = {"replication role"};
callback.header = {"replication mode"};
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}] {
auto mode = handler.ShowReplicationRole();
switch (mode) {
@@ -505,18 +467,22 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
const auto &name = repl_query->replica_name_;
const auto &sync_mode = repl_query->sync_mode_;
auto socket_address = repl_query->socket_address_->Accept(evaluator);
const auto replica_check_frequency = interpreter_context->config.replication_replica_check_frequency;
auto timeout = EvaluateOptionalExpression(repl_query->timeout_, &evaluator);
std::optional<double> maybe_timeout;
if (timeout.IsDouble()) {
maybe_timeout = timeout.ValueDouble();
} else if (timeout.IsInt()) {
maybe_timeout = static_cast<double>(timeout.ValueInt());
}
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name, socket_address, sync_mode,
replica_check_frequency]() mutable {
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, replica_check_frequency);
maybe_timeout]() mutable {
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, maybe_timeout);
return std::vector<std::vector<TypedValue>>();
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::REGISTER_REPLICA,
fmt::format("Replica {} is registered.", repl_query->replica_name_));
return callback;
}
case ReplicationQuery::Action::DROP_REPLICA: {
const auto &name = repl_query->replica_name_;
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name]() mutable {
@@ -527,11 +493,8 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
fmt::format("Replica {} is dropped.", repl_query->replica_name_));
return callback;
}
case ReplicationQuery::Action::SHOW_REPLICAS: {
callback.header = {
"name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master",
"state"};
callback.header = {"name", "socket_address", "sync_mode", "timeout"};
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, replica_nfields = callback.header.size()] {
const auto &replicas = handler.ShowReplicas();
auto typed_replicas = std::vector<std::vector<TypedValue>>{};
@@ -542,7 +505,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
typed_replica.emplace_back(TypedValue(replica.name));
typed_replica.emplace_back(TypedValue(replica.socket_address));
switch (replica.sync_mode) {
case ReplicationQuery::SyncMode::SYNC:
typed_replica.emplace_back(TypedValue("sync"));
@@ -551,24 +513,11 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
typed_replica.emplace_back(TypedValue("async"));
break;
}
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.current_timestamp_of_replica)));
typed_replica.emplace_back(
TypedValue(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master)));
switch (replica.state) {
case ReplicationQuery::ReplicaState::READY:
typed_replica.emplace_back(TypedValue("ready"));
break;
case ReplicationQuery::ReplicaState::REPLICATING:
typed_replica.emplace_back(TypedValue("replicating"));
break;
case ReplicationQuery::ReplicaState::RECOVERY:
typed_replica.emplace_back(TypedValue("recovery"));
break;
case ReplicationQuery::ReplicaState::INVALID:
typed_replica.emplace_back(TypedValue("invalid"));
break;
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.sync_mode)));
if (replica.timeout) {
typed_replica.emplace_back(TypedValue(*replica.timeout));
} else {
typed_replica.emplace_back(TypedValue());
}
typed_replicas.emplace_back(std::move(typed_replica));
@@ -704,26 +653,12 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
return callback;
}
case StreamQuery::Action::START_STREAM: {
const auto batch_limit = GetOptionalValue<int64_t>(stream_query->batch_limit_, evaluator);
const auto timeout = GetOptionalValue<std::chrono::milliseconds>(stream_query->timeout_, evaluator);
if (batch_limit.has_value()) {
if (batch_limit.value() < 0) {
throw utils::BasicException("Parameter BATCH_LIMIT cannot hold negative value");
}
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_, batch_limit, timeout]() {
interpreter_context->streams.StartWithLimit(stream_name, static_cast<uint64_t>(batch_limit.value()), timeout);
return std::vector<std::vector<TypedValue>>{};
};
} else {
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() {
interpreter_context->streams.Start(stream_name);
return std::vector<std::vector<TypedValue>>{};
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::START_STREAM,
fmt::format("Started stream {}.", stream_query->stream_name_));
}
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() {
interpreter_context->streams.Start(stream_name);
return std::vector<std::vector<TypedValue>>{};
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::START_STREAM,
fmt::format("Started stream {}.", stream_query->stream_name_));
return callback;
}
case StreamQuery::Action::START_ALL_STREAMS: {
@@ -792,16 +727,10 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
return callback;
}
case StreamQuery::Action::CHECK_STREAM: {
callback.header = {"queries", "raw messages"};
const auto batch_limit = GetOptionalValue<int64_t>(stream_query->batch_limit_, evaluator);
if (batch_limit.has_value() && batch_limit.value() < 0) {
throw utils::BasicException("Parameter BATCH_LIMIT cannot hold negative value");
}
callback.header = {"query", "parameters"};
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_,
timeout = GetOptionalValue<std::chrono::milliseconds>(stream_query->timeout_, evaluator),
batch_limit]() mutable {
batch_limit = GetOptionalValue<int64_t>(stream_query->batch_limit_, evaluator)]() mutable {
return interpreter_context->streams.Check(stream_name, timeout, batch_limit);
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::CHECK_STREAM,
@@ -892,6 +821,108 @@ Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &param
}
}
Callback HandleSchemaQuery(SchemaQuery *schema_query, InterpreterContext *interpreter_context,
std::vector<Notification> *notifications) {
Callback callback;
switch (schema_query->action_) {
case SchemaQuery::Action::SHOW_SCHEMAS: {
callback.header = {"label", "primary_key", "primary_key_type"};
callback.fn = [interpreter_context]() {
auto *db = interpreter_context->db;
auto schemas_info = db->ListAllSchemas();
std::vector<std::vector<TypedValue>> results;
results.reserve(schemas_info.schemas.size());
for (const auto &[label_id, schema_types] : schemas_info.schemas) {
std::vector<TypedValue> schema_info_row;
schema_info_row.reserve(3);
schema_info_row.emplace_back(db->LabelToName(label_id));
std::vector<std::string> primary_key_properties;
primary_key_properties.reserve(schema_types.size());
std::transform(schema_types.begin(), schema_types.end(), std::back_inserter(primary_key_properties),
[&db](const auto &schema_type) {
return db->PropertyToName(schema_type.property_id) +
"::" + storage::SchemaTypeToString(schema_type.type);
});
schema_info_row.emplace_back(utils::Join(primary_key_properties, ", "));
schema_info_row.emplace_back(schema_types.size() == 1 ? "Single" : "Composite");
results.push_back(std::move(schema_info_row));
}
return results;
};
return callback;
}
case SchemaQuery::Action::SHOW_SCHEMA: {
callback.header = {"property_name", "property_type"};
callback.fn = [interpreter_context, primary_label = schema_query->label_]() {
auto *db = interpreter_context->db;
const auto label = db->NameToLabel(primary_label.name);
const auto schemas_info = db->GetSchema(label);
MG_ASSERT(schemas_info.schemas.size() < 2, "There can be only one schema under single label!");
std::vector<std::vector<TypedValue>> results;
if (!schemas_info.schemas.empty()) {
const auto schema = schemas_info.schemas[0];
for (const auto &schema_property : schema.second) {
std::vector<TypedValue> schema_info_row;
schema_info_row.reserve(2);
schema_info_row.emplace_back(db->PropertyToName(schema_property.property_id));
schema_info_row.emplace_back(storage::SchemaTypeToString(schema_property.type));
results.push_back(std::move(schema_info_row));
}
}
return results;
};
return callback;
}
case SchemaQuery::Action::CREATE_SCHEMA: {
auto schema_type_map = schema_query->schema_type_map_;
if (schema_query->schema_type_map_.empty()) {
throw SyntaxException("One or more types have to be defined in schema definition.");
}
callback.fn = [interpreter_context, primary_label = schema_query->label_,
schema_type_map = std::move(schema_type_map)]() {
auto *db = interpreter_context->db;
const auto label = db->NameToLabel(primary_label.name);
std::vector<storage::SchemaPropertyType> schemas_types;
schemas_types.reserve(schema_type_map.size());
for (const auto &schema_type : schema_type_map) {
auto property_id = db->NameToProperty(schema_type.first.name);
schemas_types.push_back({schema_type.second, property_id});
}
if (!db->CreateSchema(label, schemas_types)) {
throw QueryException(fmt::format("Schema on label :{} already exists!", primary_label.name));
}
return std::vector<std::vector<TypedValue>>{};
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::CREATE_SCHEMA,
fmt::format("Create schema on label :{}", schema_query->label_.name));
return callback;
}
case SchemaQuery::Action::DROP_SCHEMA: {
callback.fn = [interpreter_context, primary_label = schema_query->label_]() {
auto *db = interpreter_context->db;
const auto label = db->NameToLabel(primary_label.name);
if (!db->DropSchema(label)) {
throw QueryException(fmt::format("Schema on label :{} does not exist!", primary_label.name));
}
return std::vector<std::vector<TypedValue>>{};
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::DROP_SCHEMA,
fmt::format("Dropped schema on label :{}", schema_query->label_.name));
return callback;
}
}
return callback;
}
// Struct for lazy pulling from a vector
struct PullPlanVector {
explicit PullPlanVector(std::vector<std::vector<TypedValue>> values) : values_(std::move(values)) {}
@@ -917,7 +948,7 @@ struct PullPlanVector {
struct PullPlan {
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<std::string> username, TriggerContextCollector *trigger_context_collector = nullptr,
TriggerContextCollector *trigger_context_collector = nullptr,
std::optional<size_t> memory_limit = {});
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
@@ -946,8 +977,7 @@ struct PullPlan {
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<std::string> username, TriggerContextCollector *trigger_context_collector,
const std::optional<size_t> memory_limit)
TriggerContextCollector *trigger_context_collector, const std::optional<size_t> memory_limit)
: plan_(plan),
cursor_(plan->plan().MakeCursor(execution_memory)),
frame_(plan->symbol_table().max_position(), execution_memory),
@@ -958,12 +988,6 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
ctx_.evaluation_context.parameters = parameters;
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, dba);
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, dba);
#ifdef MG_ENTERPRISE
if (username.has_value()) {
memgraph::auth::User *user = interpreter_context->auth->GetUser(*username);
ctx_.fine_grained_access_checker = new FineGrainedAccessChecker{user};
}
#endif
if (interpreter_context->config.execution_timeout_sec > 0) {
ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec};
}
@@ -1137,7 +1161,6 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
const std::string *username,
TriggerContextCollector *trigger_context_collector = nullptr) {
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
@@ -1181,9 +1204,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
header.push_back(
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
}
auto pull_plan =
std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory,
StringPointerToOptional(username), trigger_context_collector, memory_limit);
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, trigger_context_collector, memory_limit);
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
@@ -1210,7 +1232,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
// full query string) when given just the inner query to execute.
ParsedQuery parsed_inner_query =
ParseQuery(parsed_query.query_string.substr(kExplainQueryStart.size()), parsed_query.user_parameters,
&interpreter_context->ast_cache, interpreter_context->config.query);
&interpreter_context->ast_cache, &interpreter_context->antlr_lock, interpreter_context->config.query);
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
@@ -1243,8 +1265,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory,
const std::string *username) {
DbAccessor *dba, utils::MemoryResource *execution_memory) {
const std::string kProfileQueryStart = "profile ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -1278,7 +1299,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
// full query string) when given just the inner query to execute.
ParsedQuery parsed_inner_query =
ParseQuery(parsed_query.query_string.substr(kProfileQueryStart.size()), parsed_query.user_parameters,
&interpreter_context->ast_cache, interpreter_context->config.query);
&interpreter_context->ast_cache, &interpreter_context->antlr_lock, interpreter_context->config.query);
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
@@ -1294,14 +1315,12 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
auto rw_type_checker = plan::ReadWriteTypeChecker();
auto optional_username = StringPointerToOptional(username);
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
summary, dba, interpreter_context, execution_memory, memory_limit, optional_username,
summary, dba, interpreter_context, execution_memory, memory_limit,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
@@ -1310,7 +1329,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
// No output symbols are given so that nothing is streamed.
if (!stats_and_total_time) {
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
execution_memory, optional_username, nullptr, memory_limit)
execution_memory, nullptr, memory_limit)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
}
@@ -1445,7 +1464,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory, const std::string *username) {
DbAccessor *dba, utils::MemoryResource *execution_memory) {
if (in_explicit_transaction) {
throw UserModificationInMulticommandTxException();
}
@@ -1465,8 +1484,8 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
[fn = callback.fn](Frame *, ExecutionContext *) { return fn(); }),
0.0, AstStorage{}, symbol_table));
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, StringPointerToOptional(username));
auto pull_plan =
std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory);
return PreparedQuery{
callback.header, std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), callback = std::move(callback), output_symbols = std::move(output_symbols),
@@ -1598,7 +1617,8 @@ Callback CreateTrigger(TriggerQuery *trigger_query,
interpreter_context->trigger_store.AddTrigger(
std::move(trigger_name), trigger_statement, user_parameters, ToTriggerEventType(event_type),
before_commit ? TriggerPhase::BEFORE_COMMIT : TriggerPhase::AFTER_COMMIT, &interpreter_context->ast_cache,
dba, interpreter_context->config.query, std::move(owner), interpreter_context->auth_checker);
dba, &interpreter_context->antlr_lock, interpreter_context->config.query, std::move(owner),
interpreter_context->auth_checker);
return {};
}};
}
@@ -2098,6 +2118,32 @@ PreparedQuery PrepareConstraintQuery(ParsedQuery parsed_query, bool in_explicit_
RWType::NONE};
}
PreparedQuery PrepareSchemaQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
InterpreterContext *interpreter_context, std::vector<Notification> *notifications) {
if (in_explicit_transaction) {
throw ConstraintInMulticommandTxException();
}
auto *schema_query = utils::Downcast<SchemaQuery>(parsed_query.query);
MG_ASSERT(schema_query);
auto callback = HandleSchemaQuery(schema_query, interpreter_context, notifications);
return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges),
[handler = std::move(callback.fn), action = QueryHandlerResult::NOTHING,
pull_plan = std::shared_ptr<PullPlanVector>(nullptr)](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
if (!pull_plan) {
auto results = handler();
pull_plan = std::make_shared<PullPlanVector>(std::move(results));
}
if (pull_plan->Pull(stream, n)) {
return action;
}
return std::nullopt;
},
RWType::NONE};
}
void Interpreter::BeginTransaction() {
const auto prepared_query = PrepareTransactionQuery("BEGIN");
prepared_query.query_handler(nullptr, {});
@@ -2154,8 +2200,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
query_execution->summary["cost_estimate"] = 0.0;
utils::Timer parsing_timer;
ParsedQuery parsed_query =
ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query);
ParsedQuery parsed_query = ParseQuery(query_string, params, &interpreter_context_->ast_cache,
&interpreter_context_->antlr_lock, interpreter_context_->config.query);
query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count();
// Some queries require an active transaction in order to be prepared.
@@ -2178,7 +2224,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory,
&query_execution->notifications, username,
&query_execution->notifications,
trigger_context_collector_ ? &*trigger_context_collector_ : nullptr);
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
@@ -2186,7 +2232,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, username);
&query_execution->execution_memory_with_exception);
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
&query_execution->execution_memory);
@@ -2196,7 +2242,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<AuthQuery>(parsed_query.query)) {
prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, username);
&query_execution->execution_memory_with_exception);
} else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, interpreter_context_->db,
@@ -2231,6 +2277,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_);
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
} else if (utils::Downcast<SchemaQuery>(parsed_query.query)) {
prepared_query = PrepareSchemaQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_,
&query_execution->notifications);
} else {
LOG_FATAL("Should not get here -- unknown query type!");
}

View File

@@ -13,7 +13,6 @@
#include <gflags/gflags.h>
#include "auth/models.hpp"
#include "query/auth_checker.hpp"
#include "query/config.hpp"
#include "query/context.hpp"
@@ -100,19 +99,14 @@ class AuthQueryHandler {
virtual std::vector<std::vector<TypedValue>> GetPrivileges(const std::string &user_or_role) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual memgraph::auth::User *GetUser(const std::string &username) = 0;
virtual void GrantPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void GrantPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) = 0;
virtual void DenyPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void DenyPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void RevokePrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
const std::vector<std::string> &labels) = 0;
virtual void RevokePrivilege(const std::string &user_or_role,
const std::vector<AuthQuery::Privilege> &privileges) = 0;
};
enum class QueryHandlerResult { COMMIT, ABORT, NOTHING };
@@ -133,9 +127,6 @@ class ReplicationQueryHandler {
std::string socket_address;
ReplicationQuery::SyncMode sync_mode;
std::optional<double> timeout;
uint64_t current_timestamp_of_replica;
uint64_t current_number_of_timestamp_behind_master;
ReplicationQuery::ReplicaState state;
};
/// @throw QueryRuntimeException if an error ocurred.
@@ -146,8 +137,7 @@ class ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
virtual void RegisterReplica(const std::string &name, const std::string &socket_address,
ReplicationQuery::SyncMode sync_mode,
const std::chrono::seconds replica_check_frequency) = 0;
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout) = 0;
/// @throw QueryRuntimeException if an error ocurred.
virtual void DropReplica(const std::string &replica_name) = 0;
@@ -179,6 +169,13 @@ struct InterpreterContext {
storage::Storage *db;
// ANTLR has singleton instance that is shared between threads. It is
// protected by locks inside of ANTLR. Unfortunately, they are not protected
// in a very good way. Once we have ANTLR version without race conditions we
// can remove this lock. This will probably never happen since ANTLR
// developers introduce more bugs in each version. Fortunately, we have
// cache so this lock probably won't impact performance much...
utils::SpinLock antlr_lock;
std::optional<double> tsc_frequency{utils::GetTSCFrequency()};
std::atomic<bool> is_shutting_down{false};

View File

@@ -38,6 +38,8 @@ constexpr std::string_view GetCodeString(const NotificationCode code) {
return "CreateIndex"sv;
case NotificationCode::CREATE_STREAM:
return "CreateStream"sv;
case NotificationCode::CREATE_SCHEMA:
return "CreateSchema"sv;
case NotificationCode::CHECK_STREAM:
return "CheckStream"sv;
case NotificationCode::CREATE_TRIGGER:
@@ -48,6 +50,8 @@ constexpr std::string_view GetCodeString(const NotificationCode code) {
return "DropReplica"sv;
case NotificationCode::DROP_INDEX:
return "DropIndex"sv;
case NotificationCode::DROP_SCHEMA:
return "DropSchema"sv;
case NotificationCode::DROP_STREAM:
return "DropStream"sv;
case NotificationCode::DROP_TRIGGER:
@@ -68,6 +72,10 @@ constexpr std::string_view GetCodeString(const NotificationCode code) {
return "ReplicaPortWarning"sv;
case NotificationCode::SET_REPLICA:
return "SetReplica"sv;
case NotificationCode::SHOW_SCHEMA:
return "ShowSchema"sv;
case NotificationCode::SHOW_SCHEMAS:
return "ShowSchemas"sv;
case NotificationCode::START_STREAM:
return "StartStream"sv;
case NotificationCode::START_ALL_STREAMS:
@@ -114,4 +122,4 @@ std::string ExecutionStatsKeyToString(const ExecutionStats::Key key) {
}
}
} // namespace memgraph::query
} // namespace memgraph::query

View File

@@ -26,12 +26,14 @@ enum class SeverityLevel : uint8_t { INFO, WARNING };
enum class NotificationCode : uint8_t {
CREATE_CONSTRAINT,
CREATE_INDEX,
CREATE_SCHEMA,
CHECK_STREAM,
CREATE_STREAM,
CREATE_TRIGGER,
DROP_CONSTRAINT,
DROP_INDEX,
DROP_REPLICA,
DROP_SCHEMA,
DROP_STREAM,
DROP_TRIGGER,
EXISTANT_INDEX,
@@ -42,6 +44,8 @@ enum class NotificationCode : uint8_t {
REPLICA_PORT_WARNING,
REGISTER_REPLICA,
SET_REPLICA,
SHOW_SCHEMA,
SHOW_SCHEMAS,
START_STREAM,
START_ALL_STREAMS,
STOP_STREAM,

View File

@@ -29,7 +29,6 @@
#include "query/context.hpp"
#include "query/db_accessor.hpp"
#include "query/exceptions.hpp"
#include "query/fine_grained_access_checker.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/semantic/symbol_table.hpp"
#include "query/interpret/eval.hpp"
@@ -406,29 +405,11 @@ class ScanAllCursor : public Cursor {
vertices_it_.emplace(vertices_.value().begin());
}
#ifdef MG_ENTERPRISE
FilterNodes(context.fine_grained_access_checker, context.db_accessor);
if (vertices_it_.value() == vertices_.value().end()) return false;
#endif
frame[output_symbol_] = *vertices_it_.value();
++vertices_it_.value();
return true;
}
void FilterNodes(const FineGrainedAccessChecker *fine_grained_access_checker, DbAccessor *db_accessor) {
if (!fine_grained_access_checker) return;
while (vertices_it_.value() != vertices_.value().end()) {
VertexAccessor vertex = *vertices_it_.value();
auto vertex_labels = vertex.Labels(memgraph::storage::View::NEW).GetValue();
if (fine_grained_access_checker->IsUserAuthorizedLabels(vertex_labels, db_accessor)) {
break;
}
++vertices_it_.value();
}
}
void Shutdown() override { input_cursor_->Shutdown(); }
void Reset() override {
@@ -2619,13 +2600,13 @@ namespace {
* when there are */
TypedValue DefaultAggregationOpValue(const Aggregate::Element &element, utils::MemoryResource *memory) {
switch (element.op) {
case Aggregation::Op::COUNT:
return TypedValue(0, memory);
case Aggregation::Op::SUM:
case Aggregation::Op::MIN:
case Aggregation::Op::MAX:
case Aggregation::Op::AVG:
return TypedValue(memory);
case Aggregation::Op::COUNT:
case Aggregation::Op::SUM:
return TypedValue(0, memory);
case Aggregation::Op::COLLECT_LIST:
return TypedValue(TypedValue::TVector(memory));
case Aggregation::Op::COLLECT_MAP:
@@ -2647,7 +2628,9 @@ class AggregateCursor : public Cursor {
pulled_all_input_ = true;
aggregation_it_ = aggregation_.begin();
if (aggregation_.empty()) {
// in case there is no input and no group_bys we need to return true
// just this once
if (aggregation_.empty() && self_.group_by_.empty()) {
auto *pull_memory = context.evaluation_context.memory;
// place default aggregation values on the frame
for (const auto &elem : self_.aggregations_)
@@ -3714,7 +3697,7 @@ std::unordered_map<std::string, int64_t> CallProcedure::GetAndResetCounters() {
namespace {
void CallCustomProcedure(const std::string_view fully_qualified_procedure_name, const mgp_proc &proc,
void CallCustomProcedure(const std::string_view &fully_qualified_procedure_name, const mgp_proc &proc,
const std::vector<Expression *> &args, mgp_graph &graph, ExpressionEvaluator *evaluator,
utils::MemoryResource *memory, std::optional<size_t> memory_limit, mgp_result *result) {
static_assert(std::uses_allocator_v<mgp_value, utils::Allocator<mgp_value>>,

View File

@@ -1054,7 +1054,7 @@ std::unique_ptr<Module> LoadModuleFromFile(const std::filesystem::path &path) {
} // namespace
bool ModuleRegistry::RegisterModule(const std::string_view name, std::unique_ptr<Module> module) {
bool ModuleRegistry::RegisterModule(const std::string_view &name, std::unique_ptr<Module> module) {
MG_ASSERT(!name.empty(), "Module name cannot be empty");
MG_ASSERT(module, "Tried to register an invalid module");
if (modules_.find(name) != modules_.end()) {
@@ -1163,7 +1163,7 @@ void ModuleRegistry::UnloadAndLoadModulesFromDirectories() {
}
}
ModulePtr ModuleRegistry::GetModuleNamed(const std::string_view name) const {
ModulePtr ModuleRegistry::GetModuleNamed(const std::string_view &name) const {
std::shared_lock<utils::RWLock> guard(lock_);
auto found_it = modules_.find(name);
if (found_it == modules_.end()) return nullptr;

View File

@@ -77,7 +77,7 @@ class ModuleRegistry final {
mutable utils::RWLock lock_{utils::RWLock::Priority::WRITE};
std::unique_ptr<utils::MemoryResource> shared_{std::make_unique<utils::ResourceWithOutOfMemoryException>()};
bool RegisterModule(std::string_view name, std::unique_ptr<Module> module);
bool RegisterModule(const std::string_view &name, std::unique_ptr<Module> module);
void DoUnloadAllModules();
@@ -105,7 +105,7 @@ class ModuleRegistry final {
///
/// Return true if the module was loaded or reloaded successfully, false
/// otherwise.
bool LoadOrReloadModuleFromName(std::string_view name);
bool LoadOrReloadModuleFromName(const std::string_view name);
/// Atomically unload all modules and then load all possible modules from the
/// set directories.
@@ -115,7 +115,7 @@ class ModuleRegistry final {
/// Find a module with given name or return nullptr.
/// Takes a read lock.
ModulePtr GetModuleNamed(std::string_view name) const;
ModulePtr GetModuleNamed(const std::string_view &name) const;
/// Remove all loaded (non-builtin) modules.
/// Takes a write lock.
@@ -175,7 +175,7 @@ extern ModuleRegistry gModuleRegistry;
/// inside this function. ModulePtr must be kept alive to make sure it won't be
/// unloaded.
std::optional<std::pair<procedure::ModulePtr, const mgp_proc *>> FindProcedure(
const ModuleRegistry &module_registry, std::string_view fully_qualified_procedure_name,
const ModuleRegistry &module_registry, const std::string_view fully_qualified_procedure_name,
utils::MemoryResource *memory);
/// Return the ModulePtr and `mgp_trans *` of the found transformation after resolving
@@ -183,7 +183,7 @@ std::optional<std::pair<procedure::ModulePtr, const mgp_proc *>> FindProcedure(
/// inside this function. ModulePtr must be kept alive to make sure it won't be
/// unloaded.
std::optional<std::pair<procedure::ModulePtr, const mgp_trans *>> FindTransformation(
const ModuleRegistry &module_registry, std::string_view fully_qualified_transformation_name,
const ModuleRegistry &module_registry, const std::string_view fully_qualified_transformation_name,
utils::MemoryResource *memory);
/// Return the ModulePtr and `mgp_func *` of the found function after resolving
@@ -191,7 +191,7 @@ std::optional<std::pair<procedure::ModulePtr, const mgp_trans *>> FindTransforma
/// std::nullopt is returned. `memory` is used for temporary allocations
/// inside this function. ModulePtr must be kept alive to make sure it won't be unloaded.
std::optional<std::pair<procedure::ModulePtr, const mgp_func *>> FindFunction(
const ModuleRegistry &module_registry, std::string_view fully_qualified_function_name,
const ModuleRegistry &module_registry, const std::string_view fully_qualified_function_name,
utils::MemoryResource *memory);
template <typename T>

View File

@@ -52,11 +52,10 @@ concept Stream = requires(TStream stream) {
typename TStream::Message;
TStream{std::string{""}, typename TStream::StreamInfo{}, ConsumerFunction<typename TStream::Message>{}};
{ stream.Start() } -> std::same_as<void>;
{ stream.StartWithLimit(uint64_t{}, std::optional<std::chrono::milliseconds>{}) } -> std::same_as<void>;
{ stream.Stop() } -> std::same_as<void>;
{ stream.IsRunning() } -> std::same_as<bool>;
{
stream.Check(std::optional<std::chrono::milliseconds>{}, std::optional<uint64_t>{},
stream.Check(std::optional<std::chrono::milliseconds>{}, std::optional<int64_t>{},
ConsumerFunction<typename TStream::Message>{})
} -> std::same_as<void>;
requires std::same_as<std::decay_t<decltype(std::declval<typename TStream::StreamInfo>().common_info)>,

View File

@@ -44,13 +44,10 @@ KafkaStream::StreamInfo KafkaStream::Info(std::string transformation_name) const
}
void KafkaStream::Start() { consumer_->Start(); }
void KafkaStream::StartWithLimit(uint64_t batch_limit, std::optional<std::chrono::milliseconds> timeout) const {
consumer_->StartWithLimit(batch_limit, timeout);
}
void KafkaStream::Stop() { consumer_->Stop(); }
bool KafkaStream::IsRunning() const { return consumer_->IsRunning(); }
void KafkaStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> batch_limit,
void KafkaStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
const ConsumerFunction<integrations::kafka::Message> &consumer_function) const {
consumer_->Check(timeout, batch_limit, consumer_function);
}
@@ -109,12 +106,10 @@ PulsarStream::StreamInfo PulsarStream::Info(std::string transformation_name) con
}
void PulsarStream::Start() { consumer_->Start(); }
void PulsarStream::StartWithLimit(uint64_t batch_limit, std::optional<std::chrono::milliseconds> timeout) const {
consumer_->StartWithLimit(batch_limit, timeout);
}
void PulsarStream::Stop() { consumer_->Stop(); }
bool PulsarStream::IsRunning() const { return consumer_->IsRunning(); }
void PulsarStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> batch_limit,
void PulsarStream::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
const ConsumerFunction<Message> &consumer_function) const {
consumer_->Check(timeout, batch_limit, consumer_function);
}

View File

@@ -36,11 +36,10 @@ struct KafkaStream {
StreamInfo Info(std::string transformation_name) const;
void Start();
void StartWithLimit(uint64_t batch_limit, std::optional<std::chrono::milliseconds> timeout) const;
void Stop();
bool IsRunning() const;
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> batch_limit,
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
const ConsumerFunction<Message> &consumer_function) const;
utils::BasicResult<std::string> SetStreamOffset(int64_t offset);
@@ -72,11 +71,10 @@ struct PulsarStream {
StreamInfo Info(std::string transformation_name) const;
void Start();
void StartWithLimit(uint64_t batch_limit, std::optional<std::chrono::milliseconds> timeout) const;
void Stop();
bool IsRunning() const;
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> batch_limit,
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> batch_limit,
const ConsumerFunction<Message> &consumer_function) const;
private:

View File

@@ -43,7 +43,6 @@ extern const Event MessagesConsumed;
namespace memgraph::query::stream {
namespace {
inline constexpr auto kExpectedTransformationResultSize = 2;
inline constexpr auto kCheckStreamResultSize = 2;
const utils::pmr::string query_param_name{"query", utils::NewDeleteResource()};
const utils::pmr::string params_param_name{"parameters", utils::NewDeleteResource()};
@@ -456,7 +455,7 @@ void Streams::Create(const std::string &stream_name, typename TStream::StreamInf
try {
std::visit(
[&](const auto &stream_data) {
[&](auto &&stream_data) {
const auto stream_source_ptr = stream_data.stream_source->ReadLock();
Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
},
@@ -575,7 +574,7 @@ void Streams::RestoreStreams() {
auto it = CreateConsumer<T>(*locked_streams_map, stream_name, std::move(status.info), std::move(status.owner));
if (status.is_running) {
std::visit(
[&](const auto &stream_data) {
[&](auto &&stream_data) {
auto stream_source_ptr = stream_data.stream_source->Lock();
stream_source_ptr->Start();
},
@@ -617,12 +616,12 @@ void Streams::Drop(const std::string &stream_name) {
// function can be executing with the consumer, nothing else.
// By acquiring the write lock here for the consumer, we make sure there is
// no running Test function for this consumer, therefore it can be erased.
std::visit([&](const auto &stream_data) { stream_data.stream_source->Lock(); }, it->second);
std::visit([&](auto &&stream_data) { stream_data.stream_source->Lock(); }, it->second);
locked_streams->erase(it);
if (!storage_.Delete(stream_name)) {
throw StreamsException("Couldn't delete stream '{}' from persistent store!", stream_name);
}
locked_streams->erase(it);
// TODO(antaljanosbenjamin) Release the transformation
}
@@ -632,7 +631,7 @@ void Streams::Start(const std::string &stream_name) {
auto it = GetStream(*locked_streams, stream_name);
std::visit(
[&, this](const auto &stream_data) {
[&, this](auto &&stream_data) {
auto stream_source_ptr = stream_data.stream_source->Lock();
stream_source_ptr->Start();
Persist(CreateStatus(stream_name, stream_data.transformation_name, stream_data.owner, *stream_source_ptr));
@@ -640,27 +639,12 @@ void Streams::Start(const std::string &stream_name) {
it->second);
}
void Streams::StartWithLimit(const std::string &stream_name, uint64_t batch_limit,
std::optional<std::chrono::milliseconds> timeout) const {
std::optional locked_streams{streams_.ReadLock()};
auto it = GetStream(**locked_streams, stream_name);
std::visit(
[&](const auto &stream_data) {
const auto locked_stream_source = stream_data.stream_source->ReadLock();
locked_streams.reset();
locked_stream_source->StartWithLimit(batch_limit, timeout);
},
it->second);
}
void Streams::Stop(const std::string &stream_name) {
auto locked_streams = streams_.Lock();
auto it = GetStream(*locked_streams, stream_name);
std::visit(
[&, this](const auto &stream_data) {
[&, this](auto &&stream_data) {
auto stream_source_ptr = stream_data.stream_source->Lock();
stream_source_ptr->Stop();
@@ -672,7 +656,7 @@ void Streams::Stop(const std::string &stream_name) {
void Streams::StartAll() {
for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
std::visit(
[&stream_name = stream_name, this](const auto &stream_data) {
[&stream_name = stream_name, this](auto &&stream_data) {
auto locked_stream_source = stream_data.stream_source->Lock();
if (!locked_stream_source->IsRunning()) {
locked_stream_source->Start();
@@ -687,7 +671,7 @@ void Streams::StartAll() {
void Streams::StopAll() {
for (auto locked_streams = streams_.Lock(); auto &[stream_name, stream_data] : *locked_streams) {
std::visit(
[&stream_name = stream_name, this](const auto &stream_data) {
[&stream_name = stream_name, this](auto &&stream_data) {
auto locked_stream_source = stream_data.stream_source->Lock();
if (locked_stream_source->IsRunning()) {
locked_stream_source->Stop();
@@ -704,7 +688,7 @@ std::vector<StreamStatus<>> Streams::GetStreamInfo() const {
{
for (auto locked_streams = streams_.ReadLock(); const auto &[stream_name, stream_data] : *locked_streams) {
std::visit(
[&, &stream_name = stream_name](const auto &stream_data) {
[&, &stream_name = stream_name](auto &&stream_data) {
auto locked_stream_source = stream_data.stream_source->ReadLock();
auto info = locked_stream_source->Info(stream_data.transformation_name);
result.emplace_back(StreamStatus<>{stream_name, StreamType(*locked_stream_source),
@@ -718,12 +702,12 @@ std::vector<StreamStatus<>> Streams::GetStreamInfo() const {
}
TransformationResult Streams::Check(const std::string &stream_name, std::optional<std::chrono::milliseconds> timeout,
std::optional<uint64_t> batch_limit) const {
std::optional<int64_t> batch_limit) const {
std::optional locked_streams{streams_.ReadLock()};
auto it = GetStream(**locked_streams, stream_name);
return std::visit(
[&](const auto &stream_data) {
[&](auto &&stream_data) {
// This depends on the fact that Drop will first acquire a write lock to the consumer, and erase it only after
// that
const auto locked_stream_source = stream_data.stream_source->ReadLock();
@@ -740,27 +724,15 @@ TransformationResult Streams::Check(const std::string &stream_name, std::optiona
auto accessor = interpreter_context->db->Access();
CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
auto result_row = std::vector<TypedValue>();
result_row.reserve(kCheckStreamResultSize);
for (auto &row : result.rows) {
auto [query, parameters] = ExtractTransformationResult(row.values, transformation_name, stream_name);
std::vector<TypedValue> result_row;
result_row.reserve(kExpectedTransformationResultSize);
result_row.push_back(std::move(query));
result_row.push_back(std::move(parameters));
auto queries_and_parameters = std::vector<TypedValue>(result.rows.size());
std::transform(
result.rows.cbegin(), result.rows.cend(), queries_and_parameters.begin(), [&](const auto &row) {
auto [query, parameters] = ExtractTransformationResult(row.values, transformation_name, stream_name);
return std::map<std::string, TypedValue>{{"query", std::move(query)},
{"parameters", std::move(parameters)}};
});
result_row.emplace_back(std::move(queries_and_parameters));
auto messages_list = std::vector<TypedValue>(messages.size());
std::transform(messages.cbegin(), messages.cend(), messages_list.begin(), [](const auto &message) {
return std::string_view(message.Payload().data(), message.Payload().size());
});
result_row.emplace_back(std::move(messages_list));
test_result.emplace_back(std::move(result_row));
test_result.push_back(std::move(result_row));
}
};
locked_stream_source->Check(timeout, batch_limit, consumer_function);

View File

@@ -115,17 +115,6 @@ class Streams final {
/// @throws ConsumerRunningException if the consumer is already running
void Start(const std::string &stream_name);
/// Start consuming from a stream.
///
/// @param stream_name name of the stream that needs to be started
/// @param batch_limit number of batches we want to consume before stopping
/// @param timeout the maximum duration during which the command should run.
///
/// @throws StreamsException if the stream doesn't exist
/// @throws ConsumerRunningException if the consumer is already running
void StartWithLimit(const std::string &stream_name, uint64_t batch_limit,
std::optional<std::chrono::milliseconds> timeout) const;
/// Stop consuming from a stream.
///
/// @param stream_name name of the stream that needs to be stopped
@@ -153,7 +142,6 @@ class Streams final {
///
/// @param stream_name name of the stream we want to test
/// @param batch_limit number of batches we want to test before stopping
/// @param timeout the maximum duration during which the command should run.
///
/// @returns A vector of vectors of TypedValue. Each subvector contains two elements, the query string and the
/// nullable parameters map.
@@ -163,7 +151,7 @@ class Streams final {
/// @throws ConsumerCheckFailedException if the transformation function throws any std::exception during processing
TransformationResult Check(const std::string &stream_name,
std::optional<std::chrono::milliseconds> timeout = std::nullopt,
std::optional<uint64_t> batch_limit = std::nullopt) const;
std::optional<int64_t> batch_limit = std::nullopt) const;
private:
template <Stream TStream>
@@ -188,7 +176,7 @@ class Streams final {
void Persist(StreamStatus<TStream> &&status) {
const std::string stream_name = status.name;
if (!storage_.Put(stream_name, nlohmann::json(std::move(status)).dump())) {
throw StreamsException{"Couldn't persist stream data for stream '{}'", stream_name};
throw StreamsException{"Couldn't persist steam data for stream '{}'", stream_name};
}
}

View File

@@ -153,10 +153,10 @@ std::vector<std::pair<Identifier, TriggerIdentifierTag>> GetPredefinedIdentifier
Trigger::Trigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters,
const TriggerEventType event_type, utils::SkipList<QueryCacheEntry> *query_cache,
DbAccessor *db_accessor, const InterpreterConfig::Query &query_config,
DbAccessor *db_accessor, utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
std::optional<std::string> owner, const query::AuthChecker *auth_checker)
: name_{std::move(name)},
parsed_statements_{ParseQuery(query, user_parameters, query_cache, query_config)},
parsed_statements_{ParseQuery(query, user_parameters, query_cache, antlr_lock, query_config)},
event_type_{event_type},
owner_{std::move(owner)} {
// We check immediately if the query is valid by trying to create a plan.
@@ -257,7 +257,7 @@ inline constexpr uint64_t kVersion{2};
TriggerStore::TriggerStore(std::filesystem::path directory) : storage_{std::move(directory)} {}
void TriggerStore::RestoreTriggers(utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
const InterpreterConfig::Query &query_config,
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
const query::AuthChecker *auth_checker) {
MG_ASSERT(before_commit_triggers_.size() == 0 && after_commit_triggers_.size() == 0,
"Cannot restore trigger when some triggers already exist!");
@@ -317,8 +317,8 @@ void TriggerStore::RestoreTriggers(utils::SkipList<QueryCacheEntry> *query_cache
std::optional<Trigger> trigger;
try {
trigger.emplace(trigger_name, statement, user_parameters, event_type, query_cache, db_accessor, query_config,
std::move(owner), auth_checker);
trigger.emplace(trigger_name, statement, user_parameters, event_type, query_cache, db_accessor, antlr_lock,
query_config, std::move(owner), auth_checker);
} catch (const utils::BasicException &e) {
spdlog::warn("Failed to create trigger '{}' because: {}", trigger_name, e.what());
continue;
@@ -336,8 +336,8 @@ void TriggerStore::AddTrigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters,
TriggerEventType event_type, TriggerPhase phase,
utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
const InterpreterConfig::Query &query_config, std::optional<std::string> owner,
const query::AuthChecker *auth_checker) {
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
std::optional<std::string> owner, const query::AuthChecker *auth_checker) {
std::unique_lock store_guard{store_lock_};
if (storage_.Get(name)) {
throw utils::BasicException("Trigger with the same name already exists.");
@@ -345,8 +345,8 @@ void TriggerStore::AddTrigger(std::string name, const std::string &query,
std::optional<Trigger> trigger;
try {
trigger.emplace(std::move(name), query, user_parameters, event_type, query_cache, db_accessor, query_config,
std::move(owner), auth_checker);
trigger.emplace(std::move(name), query, user_parameters, event_type, query_cache, db_accessor, antlr_lock,
query_config, std::move(owner), auth_checker);
} catch (const utils::BasicException &e) {
const auto identifiers = GetPredefinedIdentifiers(event_type);
std::stringstream identifier_names_stream;

View File

@@ -34,7 +34,7 @@ namespace memgraph::query {
struct Trigger {
explicit Trigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters, TriggerEventType event_type,
utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor, utils::SpinLock *antlr_lock,
const InterpreterConfig::Query &query_config, std::optional<std::string> owner,
const query::AuthChecker *auth_checker);
@@ -81,13 +81,14 @@ struct TriggerStore {
explicit TriggerStore(std::filesystem::path directory);
void RestoreTriggers(utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
const InterpreterConfig::Query &query_config, const query::AuthChecker *auth_checker);
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
const query::AuthChecker *auth_checker);
void AddTrigger(std::string name, const std::string &query,
const std::map<std::string, storage::PropertyValue> &user_parameters, TriggerEventType event_type,
TriggerPhase phase, utils::SkipList<QueryCacheEntry> *query_cache, DbAccessor *db_accessor,
const InterpreterConfig::Query &query_config, std::optional<std::string> owner,
const query::AuthChecker *auth_checker);
utils::SpinLock *antlr_lock, const InterpreterConfig::Query &query_config,
std::optional<std::string> owner, const query::AuthChecker *auth_checker);
void DropTrigger(const std::string &name);

View File

@@ -407,7 +407,7 @@ DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(int, Int, int_v)
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(bool, Bool, bool_v)
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(int64_t, Int, int_v)
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(double, Double, double_v)
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(const std::string_view, String, string_v)
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(const std::string_view &, String, string_v)
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(const TypedValue::TVector &, List, list_v)
TypedValue &TypedValue::operator=(const std::vector<TypedValue> &other) {

View File

@@ -185,7 +185,7 @@ class TypedValue {
new (&string_v) TString(value, memory_);
}
explicit TypedValue(const std::string_view value, utils::MemoryResource *memory = utils::NewDeleteResource())
explicit TypedValue(const std::string_view &value, utils::MemoryResource *memory = utils::NewDeleteResource())
: memory_(memory), type_(Type::String) {
new (&string_v) TString(value, memory_);
}
@@ -420,7 +420,7 @@ class TypedValue {
TypedValue &operator=(bool);
TypedValue &operator=(int64_t);
TypedValue &operator=(double);
TypedValue &operator=(std::string_view);
TypedValue &operator=(const std::string_view &);
TypedValue &operator=(const TVector &);
TypedValue &operator=(const std::vector<TypedValue> &);
TypedValue &operator=(const TMap &);

View File

@@ -166,7 +166,7 @@ inline void Save(const char *obj, Builder *builder) {
builder->Save(reinterpret_cast<const uint8_t *>(obj), size);
}
inline void Save(const std::string_view obj, Builder *builder) {
inline void Save(const std::string_view &obj, Builder *builder) {
uint64_t size = obj.size();
Save(size, builder);
builder->Save(reinterpret_cast<const uint8_t *>(obj.data()), size);

View File

@@ -10,9 +10,11 @@ set(storage_v2_src_files
indices.cpp
property_store.cpp
vertex_accessor.cpp
schemas.cpp
storage.cpp)
##### Replication #####
define_add_lcp(add_lcp_storage lcp_storage_cpp_files generated_lcp_storage_files)
add_lcp_storage(replication/rpc.lcp SLK_SERIALIZE)
@@ -25,10 +27,10 @@ set(storage_v2_src_files
replication/replication_server.cpp
replication/serialization.cpp
replication/slk.cpp
replication/replication_persistence_helper.cpp
${lcp_storage_cpp_files})
#######################
find_package(gflags REQUIRED)
find_package(Threads REQUIRED)

View File

@@ -49,7 +49,7 @@ struct Config {
uint64_t wal_file_flush_every_n_tx{100000};
bool snapshot_on_exit{false};
bool restore_replicas_on_startup{false};
} durability;
struct Transaction {

View File

@@ -178,6 +178,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
RecoveryInfo recovery_info;
RecoveredIndicesAndConstraints indices_constraints;
memgraph::storage::SchemasMap recovered_schemas;
std::optional<uint64_t> snapshot_timestamp;
if (!snapshot_files.empty()) {
spdlog::info("Try recovering from snapshot directory {}.", snapshot_directory);

View File

@@ -38,6 +38,7 @@ enum class Marker : uint8_t {
SECTION_DELTA = 0x26,
SECTION_EPOCH_HISTORY = 0x27,
SECTION_OFFSETS = 0x42,
SECTION_SCHEMAS = 0x43,
DELTA_VERTEX_CREATE = 0x50,
DELTA_VERTEX_DELETE = 0x51,
@@ -56,6 +57,8 @@ enum class Marker : uint8_t {
DELTA_EXISTENCE_CONSTRAINT_DROP = 0x5e,
DELTA_UNIQUE_CONSTRAINT_CREATE = 0x5f,
DELTA_UNIQUE_CONSTRAINT_DROP = 0x60,
DELTA_SCHEMA_CREATE = 0x61,
DELTA_SCHEMA_DROP = 0x62,
VALUE_FALSE = 0x00,
VALUE_TRUE = 0xff,
@@ -63,7 +66,7 @@ enum class Marker : uint8_t {
/// List of all available markers.
/// IMPORTANT: Don't forget to update this list when you add a new Marker.
static const Marker kMarkersAll[] = {
constexpr Marker kMarkersAll[] = {
Marker::TYPE_NULL,
Marker::TYPE_BOOL,
Marker::TYPE_INT,
@@ -99,6 +102,8 @@ static const Marker kMarkersAll[] = {
Marker::DELTA_EXISTENCE_CONSTRAINT_DROP,
Marker::DELTA_UNIQUE_CONSTRAINT_CREATE,
Marker::DELTA_UNIQUE_CONSTRAINT_DROP,
Marker::DELTA_SCHEMA_CREATE,
Marker::DELTA_SCHEMA_DROP,
Marker::VALUE_FALSE,
Marker::VALUE_TRUE,
};

View File

@@ -22,7 +22,6 @@ static const std::string kSnapshotDirectory{"snapshots"};
static const std::string kWalDirectory{"wal"};
static const std::string kBackupDirectory{".backup"};
static const std::string kLockFile{".lock"};
static const std::string kReplicationDirectory{"replication"};
// This is the prefix used for Snapshot and WAL filenames. It is a timestamp
// format that equals to: YYYYmmddHHMMSSffffff

View File

@@ -27,7 +27,7 @@ void WriteSize(Encoder *encoder, uint64_t size) {
}
} // namespace
void Encoder::Initialize(const std::filesystem::path &path, const std::string_view magic, uint64_t version) {
void Encoder::Initialize(const std::filesystem::path &path, const std::string_view &magic, uint64_t version) {
file_.Open(path, utils::OutputFile::Mode::OVERWRITE_EXISTING);
Write(reinterpret_cast<const uint8_t *>(magic.data()), magic.size());
auto version_encoded = utils::HostToLittleEndian(version);
@@ -73,7 +73,7 @@ void Encoder::WriteDouble(double value) {
Write(reinterpret_cast<const uint8_t *>(&value_uint), sizeof(value_uint));
}
void Encoder::WriteString(const std::string_view value) {
void Encoder::WriteString(const std::string_view &value) {
WriteMarker(Marker::TYPE_STRING);
WriteSize(this, value.size());
Write(reinterpret_cast<const uint8_t *>(value.data()), value.size());
@@ -333,6 +333,7 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
case Marker::SECTION_DELTA:
case Marker::SECTION_EPOCH_HISTORY:
case Marker::SECTION_OFFSETS:
case Marker::SECTION_SCHEMAS:
case Marker::DELTA_VERTEX_CREATE:
case Marker::DELTA_VERTEX_DELETE:
case Marker::DELTA_VERTEX_ADD_LABEL:
@@ -350,6 +351,8 @@ std::optional<PropertyValue> Decoder::ReadPropertyValue() {
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
case Marker::DELTA_UNIQUE_CONSTRAINT_DROP:
case Marker::DELTA_SCHEMA_CREATE:
case Marker::DELTA_SCHEMA_DROP:
case Marker::VALUE_FALSE:
case Marker::VALUE_TRUE:
return std::nullopt;
@@ -432,6 +435,7 @@ bool Decoder::SkipPropertyValue() {
case Marker::SECTION_DELTA:
case Marker::SECTION_EPOCH_HISTORY:
case Marker::SECTION_OFFSETS:
case Marker::SECTION_SCHEMAS:
case Marker::DELTA_VERTEX_CREATE:
case Marker::DELTA_VERTEX_DELETE:
case Marker::DELTA_VERTEX_ADD_LABEL:
@@ -449,6 +453,8 @@ bool Decoder::SkipPropertyValue() {
case Marker::DELTA_EXISTENCE_CONSTRAINT_DROP:
case Marker::DELTA_UNIQUE_CONSTRAINT_CREATE:
case Marker::DELTA_UNIQUE_CONSTRAINT_DROP:
case Marker::DELTA_SCHEMA_CREATE:
case Marker::DELTA_SCHEMA_DROP:
case Marker::VALUE_FALSE:
case Marker::VALUE_TRUE:
return false;

Some files were not shown because too many files have changed in this diff Show More