Compare commits
3 Commits
T610-FL-Ad
...
release/2.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3bc1839df9 | ||
|
|
e07342e3ff | ||
|
|
659c0386e1 |
@@ -61,9 +61,7 @@ Checks: '*,
|
||||
-readability-magic-numbers,
|
||||
-readability-named-parameter,
|
||||
-misc-no-recursion,
|
||||
-concurrency-mt-unsafe,
|
||||
-bugprone-easily-swappable-parameters'
|
||||
|
||||
-concurrency-mt-unsafe'
|
||||
WarningsAsErrors: ''
|
||||
HeaderFilterRegex: 'src/.*'
|
||||
AnalyzeTemporaryDtors: false
|
||||
@@ -90,3 +88,4 @@ CheckOptions:
|
||||
- key: modernize-use-nullptr.NullMacros
|
||||
value: 'NULL'
|
||||
...
|
||||
|
||||
|
||||
@@ -24,6 +24,14 @@ for file in $modified_files; do
|
||||
|
||||
git checkout-index --prefix="$tmpdir/" -- $file
|
||||
|
||||
echo "Running clang-format..."
|
||||
$project_folder/tools/git-clang-format $tmpdir/$file
|
||||
CODE=$?
|
||||
|
||||
if [ $CODE -ne 0 ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Do not break header checker
|
||||
echo "Running header checker..."
|
||||
$project_folder/tools/header-checker.py $tmpdir/$file $file --amend-year
|
||||
@@ -31,6 +39,7 @@ for file in $modified_files; do
|
||||
if [ $CODE -ne 0 ]; then
|
||||
FAIL=1
|
||||
fi
|
||||
|
||||
done;
|
||||
|
||||
return ${FAIL}
|
||||
|
||||
5
.github/workflows/diff.yaml
vendored
5
.github/workflows/diff.yaml
vendored
@@ -1,7 +1,4 @@
|
||||
name: Diff
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
on:
|
||||
push:
|
||||
@@ -115,7 +112,7 @@ jobs:
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Restrict clang-tidy results only to the modified parts
|
||||
git diff -U0 master... -- src | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
|
||||
git diff -U0 master... -- src ':!*.hpp' | ./tools/github/clang-tidy/clang-tidy-diff.py -p 1 -j $THREADS -path build | tee ./build/clang_tidy_output.txt
|
||||
|
||||
# Fail if any warning is reported
|
||||
! cat ./build/clang_tidy_output.txt | ./tools/github/clang-tidy/grep_error_lines.sh > /dev/null
|
||||
|
||||
74
.github/workflows/package_all.yaml
vendored
74
.github/workflows/package_all.yaml
vendored
@@ -6,11 +6,11 @@ on: workflow_dispatch
|
||||
|
||||
jobs:
|
||||
centos-7:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -22,29 +22,29 @@ jobs:
|
||||
name: centos-7
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
centos-9:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
centos-8:
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package centos-9
|
||||
./release/package/run.sh package centos-8
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-9
|
||||
path: build/output/centos-9/memgraph*.rpm
|
||||
name: centos-8
|
||||
path: build/output/centos-8/memgraph*.rpm
|
||||
|
||||
debian-10:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -57,11 +57,11 @@ jobs:
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
debian-11:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -74,11 +74,11 @@ jobs:
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
docker:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -93,11 +93,11 @@ jobs:
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
ubuntu-1804:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -110,11 +110,11 @@ jobs:
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -126,29 +126,12 @@ jobs:
|
||||
name: ubuntu-2004
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
ubuntu-2204:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-2204
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
runs-on: [self-hosted, DockerMgBuild]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
@@ -159,20 +142,3 @@ jobs:
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
debian-11-arm:
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
49
.github/workflows/release_docker.yaml
vendored
49
.github/workflows/release_docker.yaml
vendored
@@ -1,49 +0,0 @@
|
||||
name: Publish Docker images
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Memgraph binary version to publish on Dockerhub."
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
docker_publish:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_ORGANIZATION_NAME: memgraph
|
||||
DOCKER_REPOSITORY_NAME: memgraph
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Download memgraph binary
|
||||
run: |
|
||||
cd release/docker
|
||||
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11/memgraph_${{ github.event.inputs.version }}-1_amd64.deb > memgraph-amd64.deb
|
||||
curl -L https://download.memgraph.com/memgraph/v${{ github.event.inputs.version }}/debian-11-aarch64/memgraph_${{ github.event.inputs.version }}-1_arm64.deb > memgraph-arm64.deb
|
||||
|
||||
- name: Build & push docker images
|
||||
run: |
|
||||
cd release/docker
|
||||
docker buildx build \
|
||||
--build-arg BINARY_NAME="memgraph-" \
|
||||
--build-arg EXTENSION="deb" \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
|
||||
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
|
||||
--file memgraph_deb.dockerfile \
|
||||
--push .
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -9,7 +9,6 @@
|
||||
*.swn
|
||||
*.swo
|
||||
*.swp
|
||||
|
||||
*~
|
||||
.DS_Store
|
||||
.gdb_history
|
||||
@@ -27,7 +26,6 @@ src/query/frontend/opencypher/generated/
|
||||
tags
|
||||
ve/
|
||||
ve3/
|
||||
.cache/
|
||||
perf.data*
|
||||
TAGS
|
||||
*.apollo_measurements
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v2.3.0
|
||||
hooks:
|
||||
- id: check-yaml
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.3.0
|
||||
hooks:
|
||||
- id: black
|
||||
args: # arguments to configure black
|
||||
- --line-length=120
|
||||
- --include='\.pyi?$'
|
||||
# these folders wont be formatted by black
|
||||
- --exclude="""\.git |
|
||||
\.__pycache__|
|
||||
build|
|
||||
libs|
|
||||
.cache"""
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v13.0.0
|
||||
hooks:
|
||||
- id: clang-format
|
||||
@@ -54,7 +54,7 @@ option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON)
|
||||
# Set the current version here to override the automatic version detection. The
|
||||
# version must be specified as `X.Y.Z`. Primarily used when building new patch
|
||||
# versions.
|
||||
set(MEMGRAPH_OVERRIDE_VERSION "")
|
||||
set(MEMGRAPH_OVERRIDE_VERSION "2.2.1")
|
||||
|
||||
# Custom suffix that this version should have. The suffix can be any arbitrary
|
||||
# string. Primarily used when building a version for a specific customer.
|
||||
@@ -184,8 +184,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
|
||||
-Werror=switch -Werror=switch-bool -Werror=return-type \
|
||||
-Werror=return-stack-address \
|
||||
-Wno-c99-designator \
|
||||
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
|
||||
-Wno-c99-designator")
|
||||
|
||||
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
|
||||
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
|
||||
|
||||
12
README.md
12
README.md
@@ -54,18 +54,6 @@ Memgraph is implemented in C/C++ and leverages an in-memory first architecture
|
||||
to ensure that you’re getting the best possible performance consistently and
|
||||
without surprises. It’s also ACID-compliant and highly available.
|
||||
|
||||
## :video_game: Memgraph Playground
|
||||
|
||||
You don't need to install anything to try out Memgraph. Check out
|
||||
our **[Memgraph Playground](https://playground.memgraph.com/)** sandboxes in
|
||||
your browser.
|
||||
|
||||
<p align="left">
|
||||
<a href="https://playground.memgraph.com/">
|
||||
<img width="450px" alt="Memgraph Playground" src="https://download.memgraph.com/asset/github/memgraph/memgraph-playground.png">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
## :floppy_disk: Download & Install
|
||||
|
||||
### Windows
|
||||
|
||||
@@ -47,14 +47,6 @@ modifications:
|
||||
value: ""
|
||||
override: false
|
||||
|
||||
- name: "bolt_cert_file"
|
||||
value: "/etc/memgraph/ssl/cert.pem"
|
||||
override: false
|
||||
|
||||
- name: "bolt_key_file"
|
||||
value: "/etc/memgraph/ssl/key.pem"
|
||||
override: false
|
||||
|
||||
- name: "storage_properties_on_edges"
|
||||
value: "true"
|
||||
override: true
|
||||
|
||||
@@ -18,16 +18,14 @@ WIDTH = 80
|
||||
|
||||
def wrap_text(s, initial_indent="# "):
|
||||
return "\n#\n".join(
|
||||
map(
|
||||
lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent, subsequent_indent="# "),
|
||||
s.split("\n"),
|
||||
)
|
||||
)
|
||||
map(lambda x: textwrap.fill(x, WIDTH, initial_indent=initial_indent,
|
||||
subsequent_indent="# "), s.split("\n")))
|
||||
|
||||
|
||||
def extract_flags(binary_path):
|
||||
ret = {}
|
||||
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
|
||||
data = subprocess.run([binary_path, "--help-xml"],
|
||||
stdout=subprocess.PIPE).stdout.decode("utf-8")
|
||||
root = ET.fromstring(data)
|
||||
for child in root:
|
||||
if child.tag == "usage" and child.text.lower().count("warning"):
|
||||
@@ -48,7 +46,8 @@ def apply_config_to_flags(config, flags):
|
||||
for modification in config["modifications"]:
|
||||
name = modification["name"]
|
||||
if name not in flags:
|
||||
print("WARNING: Flag '" + name + "' missing from binary!", file=sys.stderr)
|
||||
print("WARNING: Flag '" + name + "' missing from binary!",
|
||||
file=sys.stderr)
|
||||
continue
|
||||
flags[name]["default"] = modification["value"]
|
||||
flags[name]["override"] = modification["override"]
|
||||
@@ -76,9 +75,8 @@ def extract_sections(flags):
|
||||
else:
|
||||
sections.append((current_section, current_flags))
|
||||
sections.append(("other", other))
|
||||
assert set(sum(map(lambda x: x[1], sections), [])) == set(
|
||||
flags.keys()
|
||||
), "The section extraction algorithm lost some flags!"
|
||||
assert set(sum(map(lambda x: x[1], sections), [])) == set(flags.keys()), \
|
||||
"The section extraction algorithm lost some flags!"
|
||||
return sections
|
||||
|
||||
|
||||
@@ -91,7 +89,8 @@ def generate_config_file(sections, flags):
|
||||
helpstr = flag["meaning"] + " [" + flag["type"] + "]"
|
||||
ret += wrap_text(helpstr) + "\n"
|
||||
prefix = "# " if not flag["override"] else ""
|
||||
ret += prefix + "--" + flag["name"].replace("_", "-") + "=" + flag["default"] + "\n\n"
|
||||
ret += prefix + "--" + flag["name"].replace("_", "-") + \
|
||||
"=" + flag["default"] + "\n\n"
|
||||
ret += "\n"
|
||||
ret += wrap_text(config["footer"])
|
||||
return ret.strip() + "\n"
|
||||
@@ -99,16 +98,13 @@ def generate_config_file(sections, flags):
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("memgraph_binary", help="path to Memgraph binary")
|
||||
parser.add_argument(
|
||||
"output_file",
|
||||
help="path where to store the generated Memgraph " "configuration file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config-file",
|
||||
default=CONFIG_FILE,
|
||||
help="path to generator configuration file",
|
||||
)
|
||||
parser.add_argument("memgraph_binary",
|
||||
help="path to Memgraph binary")
|
||||
parser.add_argument("output_file",
|
||||
help="path where to store the generated Memgraph "
|
||||
"configuration file")
|
||||
parser.add_argument("--config-file", default=CONFIG_FILE,
|
||||
help="path to generator configuration file")
|
||||
|
||||
args = parser.parse_args()
|
||||
flags = extract_flags(args.memgraph_binary)
|
||||
|
||||
161
environment/os/centos-8.sh
Executable file
161
environment/os/centos-8.sh
Executable file
@@ -0,0 +1,161 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
expat-devel libipt libipt-devel libbabeltrace-devel xz-devel python36-devel texinfo # for gdb
|
||||
libcurl-devel # for cmake
|
||||
curl # snappy
|
||||
readline-devel # for cmake and llvm
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel automake bison # for swig
|
||||
file
|
||||
openssl-devel
|
||||
gmp-devel
|
||||
gperf
|
||||
patch
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz # used for archive unpacking
|
||||
zlib # zlib library used for all builds
|
||||
expat libipt libbabeltrace xz-libs python36 # for gdb
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
openssl-devel
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconf-pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python36-devel # for query modules
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python36 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
#
|
||||
# IMPORTANT: python3-yaml does NOT exist on CentOS
|
||||
# Install it manually using `pip3 install PyYAML`
|
||||
#
|
||||
PyYAML # Package name here does not correspond to the yum package!
|
||||
libcurl-devel # mg-requests
|
||||
rpm-build rpmlint # for RPM package building
|
||||
doxygen graphviz # source documentation generators
|
||||
which mono-complete dotnet-sdk-3.1 nodejs golang zip unzip java-11-openjdk-devel # for driver tests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == "PyYAML" ]; then
|
||||
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
done
|
||||
if [ "$missing" != "" ]; then
|
||||
echo "MISSING PACKAGES: $missing"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
echo "Please run as root."
|
||||
exit 1
|
||||
fi
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
dnf install -y epel-release
|
||||
dnf install -y 'dnf-command(config-manager)'
|
||||
dnf config-manager --set-enabled powertools # Required to install texinfo.
|
||||
dnf update -y
|
||||
dnf install -y wget git python36 python3-pip
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == libipt ]; then
|
||||
if ! dnf list installed libipt >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libipt-devel ]; then
|
||||
if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
# Install GDB dependencies not present in the standard repos.
|
||||
# https://bugs.centos.org/view.php?id=17068
|
||||
# https://centos.pkgs.org
|
||||
# Since 2020, there is Babeltrace2 (https://babeltrace.org). Not used
|
||||
# within GDB yet (an assumption).
|
||||
# http://mirror.centos.org/centos/8/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm not working
|
||||
if [ "$pkg" == libbabeltrace-devel ]; then
|
||||
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://rpmfind.net/linux/centos/8-stream/PowerTools/x86_64/os/Packages/libbabeltrace-devel-1.5.4-3.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == dotnet-sdk-3.1 ]; then
|
||||
if ! dnf list installed dotnet-sdk-3.1 >/dev/null 2>/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm -O packages-microsoft-prod.rpm
|
||||
rpm -Uvh https://packages.microsoft.com/config/centos/8/packages-microsoft-prod.rpm
|
||||
dnf update -y
|
||||
dnf install -y dotnet-sdk-3.1
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == PyYAML ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install --user PyYAML
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
dnf install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -6,12 +6,14 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils-common gcc gcc-c++ make # generic build tools
|
||||
coreutils gcc gcc-c++ make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg2 # used for archive signature verification
|
||||
tar gzip bzip2 xz unzip # used for archive unpacking
|
||||
zlib-devel # zlib library used for all builds
|
||||
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
|
||||
expat-devel xz-devel python3-devel texinfo # for gdb
|
||||
libcurl-devel # for cmake
|
||||
curl # snappy
|
||||
readline-devel # for cmake and llvm
|
||||
libffi-devel libxml2-devel # for llvm
|
||||
libedit-devel pcre-devel automake bison # for swig
|
||||
@@ -19,9 +21,6 @@ TOOLCHAIN_BUILD_DEPS=(
|
||||
openssl-devel
|
||||
gmp-devel
|
||||
gperf
|
||||
diffutils
|
||||
libipt libipt-devel # intel
|
||||
patch
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
@@ -32,19 +31,18 @@ TOOLCHAIN_RUN_DEPS=(
|
||||
readline # for cmake and llvm
|
||||
libffi libxml2 # for llvm
|
||||
openssl-devel
|
||||
perl # for openssl
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkgconf-pkg-config # build system
|
||||
wget # for downloading libs
|
||||
curl wget # for downloading libs
|
||||
libuuid-devel java-11-openjdk # required by antlr
|
||||
readline-devel # for memgraph console
|
||||
python3-devel # for query modules
|
||||
openssl-devel
|
||||
libseccomp-devel
|
||||
python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
python3 python3-virtualenv python3-pip nmap-ncat # for qa, macro_benchmark and stress tests
|
||||
#
|
||||
# IMPORTANT: python3-yaml does NOT exist on CentOS
|
||||
# Install it manually using `pip3 install PyYAML`
|
||||
@@ -75,6 +73,12 @@ check() {
|
||||
if [ "$pkg" == "python3-virtualenv" ]; then
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! sbcl --version &> /dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
|
||||
missing="$pkg $missing"
|
||||
fi
|
||||
@@ -101,37 +105,13 @@ install() {
|
||||
yum update -y
|
||||
yum install -y wget git python3 python3-pip
|
||||
for pkg in $1; do
|
||||
# Since there is no support for libipt-devel for CentOS 9 we install
|
||||
# Fedoras version of same libs, they are the same version but released
|
||||
# for different OS
|
||||
# TODO Update when libipt-devel releases for CentOS 9
|
||||
if [ "$pkg" == libipt ]; then
|
||||
if ! dnf list installed libipt >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libipt-devel ]; then
|
||||
if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == libbabeltrace-devel ]; then
|
||||
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
|
||||
dnf install -y http://mirror.stream.centos.org/9-stream/CRB/x86_64/os/Packages/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
if [ "$pkg" == sbcl ]; then
|
||||
if ! dnf list installed cl-asdf >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/cl-asdf-20101028-18.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed common-lisp-controller >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/common-lisp-controller-7.4-20.el8.noarch.rpm
|
||||
fi
|
||||
if ! dnf list installed sbcl >/dev/null 2>/dev/null; then
|
||||
dnf install -y https://pkgs.dyn.su/el8/base/x86_64/sbcl-2.0.1-4.el8.x86_64.rpm
|
||||
if ! sbcl --version &> /dev/null; then
|
||||
curl -s https://altushost-swe.dl.sourceforge.net/project/sbcl/sbcl/1.4.2/sbcl-1.4.2-arm64-linux-binary.tar.bz2 -o /tmp/sbcl-arm64.tar.bz2
|
||||
tar xvjf /tmp/sbcl-arm64.tar.bz2 -C /tmp
|
||||
pushd /tmp/sbcl-1.4.2-arm64-linux
|
||||
INSTALL_ROOT=/usr/local sh install.sh
|
||||
popd
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
@@ -145,11 +125,9 @@ install() {
|
||||
fi
|
||||
if [ "$pkg" == python3-virtualenv ]; then
|
||||
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
|
||||
pip3 install virtualenv
|
||||
pip3 install virtualenvwrapper
|
||||
pip3 install --user virtualenv
|
||||
else # Running using sudo.
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
|
||||
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user virtualenv"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg # used for archive signature verification
|
||||
tar gzip bzip2 xz-utils unzip # used for archive unpacking
|
||||
zlib1g-dev # zlib library used for all builds
|
||||
libexpat1-dev liblzma-dev python3-dev texinfo # for gdb
|
||||
libcurl4-openssl-dev # for cmake
|
||||
libreadline-dev # for cmake and llvm
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
curl # snappy
|
||||
file # for libunwind
|
||||
libssl-dev # for libevent
|
||||
libgmp-dev
|
||||
gperf # for proxygen
|
||||
git # for fbthrift
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
zlib1g # zlib library used for all builds
|
||||
libexpat1 liblzma5 python3 # for gdb
|
||||
libcurl4 # for cmake
|
||||
file # for CPack
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi7 libxml2 # for llvm
|
||||
libssl-dev # for libevent
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
uuid-dev default-jre-headless # required by antlr
|
||||
libreadline-dev # for memgraph console
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat # tests are using nc to wait for memgraph
|
||||
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
|
||||
golang nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
cat >/etc/apt/sources.list <<EOF
|
||||
deb http://deb.debian.org/debian bullseye main
|
||||
deb-src http://deb.debian.org/debian bullseye main
|
||||
|
||||
deb http://deb.debian.org/debian-security/ bullseye-security main
|
||||
deb-src http://deb.debian.org/debian-security/ bullseye-security main
|
||||
|
||||
deb http://deb.debian.org/debian bullseye-updates main
|
||||
deb-src http://deb.debian.org/debian bullseye-updates main
|
||||
EOF
|
||||
cd "$DIR"
|
||||
apt update
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
apt install -y wget
|
||||
for pkg in $1; do
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -87,7 +87,7 @@ EOF
|
||||
fi
|
||||
apt install -y wget
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == dotnet-sdk-3.1 ]; then
|
||||
if [ "$pkg" == dotnet-sdk-3.1 ]; then
|
||||
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
source "$DIR/../util.sh"
|
||||
|
||||
TOOLCHAIN_BUILD_DEPS=(
|
||||
coreutils gcc g++ build-essential make # generic build tools
|
||||
wget # used for archive download
|
||||
gnupg # used for archive signature verification
|
||||
tar gzip bzip2 xz-utils unzip # used for archive unpacking
|
||||
zlib1g-dev # zlib library used for all builds
|
||||
libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
|
||||
libcurl4-openssl-dev # for cmake
|
||||
libreadline-dev # for cmake and llvm
|
||||
libffi-dev libxml2-dev # for llvm
|
||||
curl # snappy
|
||||
file
|
||||
git # for thrift
|
||||
libgmp-dev # for gdb
|
||||
gperf # for proxygen
|
||||
libssl-dev
|
||||
libedit-dev libpcre3-dev automake bison # for swig
|
||||
)
|
||||
|
||||
TOOLCHAIN_RUN_DEPS=(
|
||||
make # generic build tools
|
||||
tar gzip bzip2 xz-utils # used for archive unpacking
|
||||
zlib1g # zlib library used for all builds
|
||||
libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
|
||||
libcurl4 # for cmake
|
||||
libreadline8 # for cmake and llvm
|
||||
libffi7 libxml2 # for llvm
|
||||
libssl-dev # for libevent
|
||||
)
|
||||
|
||||
MEMGRAPH_BUILD_DEPS=(
|
||||
git # source code control
|
||||
make pkg-config # build system
|
||||
curl wget # for downloading libs
|
||||
uuid-dev default-jre-headless # required by antlr
|
||||
libreadline-dev # for memgraph console
|
||||
libpython3-dev python3-dev # for query modules
|
||||
libssl-dev
|
||||
libseccomp-dev
|
||||
netcat # tests are using nc to wait for memgraph
|
||||
python3 python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
|
||||
python3-yaml # for the configuration generator
|
||||
libcurl4-openssl-dev # mg-requests
|
||||
sbcl # for custom Lisp C++ preprocessing
|
||||
doxygen graphviz # source documentation generators
|
||||
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
|
||||
dotnet-sdk-6.0 golang nodejs npm
|
||||
autoconf # for jemalloc code generation
|
||||
libtool # for protobuf code generation
|
||||
)
|
||||
|
||||
list() {
|
||||
echo "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
check_all_dpkg "$1"
|
||||
}
|
||||
|
||||
install() {
|
||||
cd "$DIR"
|
||||
apt update
|
||||
# If GitHub Actions runner is installed, append LANG to the environment.
|
||||
# Python related tests doesn't work the LANG export.
|
||||
if [ -d "/home/gh/actions-runner" ]; then
|
||||
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
|
||||
else
|
||||
echo "NOTE: export LANG=en_US.utf8"
|
||||
fi
|
||||
apt install -y wget
|
||||
for pkg in $1; do
|
||||
if [ "$pkg" == dotnet-sdk-6.0 ]; then
|
||||
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
|
||||
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
|
||||
dpkg -i packages-microsoft-prod.deb
|
||||
apt-get update
|
||||
apt-get install -y apt-transport-https dotnet-sdk-6.0
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
apt install -y "$pkg"
|
||||
done
|
||||
}
|
||||
|
||||
deps=$2"[*]"
|
||||
"$1" "${!deps}"
|
||||
@@ -675,7 +675,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
|
||||
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
|
||||
SNAPPY_VERSION=1.1.9
|
||||
XZ_VERSION=5.2.5 # for LZMA
|
||||
ZLIB_VERSION=1.2.12
|
||||
ZLIB_VERSION=1.2.11
|
||||
ZSTD_VERSION=1.5.0
|
||||
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
|
||||
|
||||
@@ -1178,21 +1178,12 @@ popd
|
||||
|
||||
# create toolchain archive
|
||||
if [ ! -f $NAME-binaries-$DISTRO.tar.gz ]; then
|
||||
DISTRO_FULL_NAME=${DISTRO}
|
||||
if [[ "${DISTRO}" == centos* ]]; then
|
||||
if [[ "$for_arm" = "true" ]]; then
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
|
||||
else
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
|
||||
fi
|
||||
DISTRO_FULL_NAME=$DISTRO
|
||||
if [ "$for_arm" = true ]; then
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-aarch64"
|
||||
else
|
||||
if [[ "$for_arm" = "true" ]]; then
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-arm64"
|
||||
else
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-amd64"
|
||||
fi
|
||||
DISTRO_FULL_NAME="$DISTRO_FULL_NAME-x86_64"
|
||||
fi
|
||||
|
||||
tar --owner=root --group=root -cpvzf $NAME-binaries-$DISTRO_FULL_NAME.tar.gz -C /opt $NAME
|
||||
fi
|
||||
|
||||
|
||||
@@ -5,10 +5,6 @@ operating_system() {
|
||||
sort | cut -d '=' -f 2- | sed 's/"//g' | paste -s -d '-'
|
||||
}
|
||||
|
||||
architecture() {
|
||||
uname -m
|
||||
}
|
||||
|
||||
check_all_yum() {
|
||||
local missing=""
|
||||
for pkg in $1; do
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
314
include/mgp.py
314
include/mgp.py
@@ -40,7 +40,6 @@ class InvalidContextError(Exception):
|
||||
"""
|
||||
Signals using a graph element instance outside of the registered procedure.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -48,7 +47,6 @@ class UnknownError(_mgp.UnknownError):
|
||||
"""
|
||||
Signals unspecified failure.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -56,7 +54,6 @@ class UnableToAllocateError(_mgp.UnableToAllocateError):
|
||||
"""
|
||||
Signals failed memory allocation.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -64,7 +61,6 @@ class InsufficientBufferError(_mgp.InsufficientBufferError):
|
||||
"""
|
||||
Signals that some buffer is not big enough.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -73,7 +69,6 @@ class OutOfRangeError(_mgp.OutOfRangeError):
|
||||
Signals that an index-like parameter has a value that is outside its
|
||||
possible values.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -82,7 +77,6 @@ class LogicErrorError(_mgp.LogicErrorError):
|
||||
Signals faulty logic within the program such as violating logical
|
||||
preconditions or class invariants and may be preventable.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -90,7 +84,6 @@ class DeletedObjectError(_mgp.DeletedObjectError):
|
||||
"""
|
||||
Signals accessing an already deleted object.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -98,7 +91,6 @@ class InvalidArgumentError(_mgp.InvalidArgumentError):
|
||||
"""
|
||||
Signals that some of the arguments have invalid values.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -106,7 +98,6 @@ class KeyAlreadyExistsError(_mgp.KeyAlreadyExistsError):
|
||||
"""
|
||||
Signals that a key already exists in a container-like object.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -114,7 +105,6 @@ class ImmutableObjectError(_mgp.ImmutableObjectError):
|
||||
"""
|
||||
Signals modification of an immutable object.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -122,7 +112,6 @@ class ValueConversionError(_mgp.ValueConversionError):
|
||||
"""
|
||||
Signals that the conversion failed between python and cypher values.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -131,14 +120,12 @@ class SerializationError(_mgp.SerializationError):
|
||||
Signals serialization error caused by concurrent modifications from
|
||||
different transactions.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class Label:
|
||||
"""Label of a Vertex."""
|
||||
|
||||
__slots__ = ("_name",)
|
||||
__slots__ = ('_name',)
|
||||
|
||||
def __init__(self, name: str):
|
||||
self._name = name
|
||||
@@ -158,22 +145,19 @@ class Label:
|
||||
# Named property value of a Vertex or an Edge.
|
||||
# It would be better to use typing.NamedTuple with typed fields, but that is
|
||||
# not available in Python 3.5.
|
||||
Property = namedtuple("Property", ("name", "value"))
|
||||
Property = namedtuple('Property', ('name', 'value'))
|
||||
|
||||
|
||||
class Properties:
|
||||
"""
|
||||
A collection of properties either on a Vertex or an Edge.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_vertex_or_edge",
|
||||
"_len",
|
||||
)
|
||||
__slots__ = ('_vertex_or_edge', '_len',)
|
||||
|
||||
def __init__(self, vertex_or_edge):
|
||||
if not isinstance(vertex_or_edge, (_mgp.Vertex, _mgp.Edge)):
|
||||
raise TypeError("Expected '_mgp.Vertex' or '_mgp.Edge', got {}".format(type(vertex_or_edge)))
|
||||
raise TypeError("Expected '_mgp.Vertex' or '_mgp.Edge', \
|
||||
got {}".format(type(vertex_or_edge)))
|
||||
self._len = None
|
||||
self._vertex_or_edge = vertex_or_edge
|
||||
|
||||
@@ -346,8 +330,7 @@ class Properties:
|
||||
|
||||
class EdgeType:
|
||||
"""Type of an Edge."""
|
||||
|
||||
__slots__ = ("_name",)
|
||||
__slots__ = ('_name',)
|
||||
|
||||
def __init__(self, name):
|
||||
self._name = name
|
||||
@@ -365,7 +348,7 @@ class EdgeType:
|
||||
|
||||
|
||||
if sys.version_info >= (3, 5, 2):
|
||||
EdgeId = typing.NewType("EdgeId", int)
|
||||
EdgeId = typing.NewType('EdgeId', int)
|
||||
else:
|
||||
EdgeId = int
|
||||
|
||||
@@ -377,12 +360,12 @@ class Edge:
|
||||
a query. You should not globally store an instance of an Edge. Using an
|
||||
invalid Edge instance will raise InvalidContextError.
|
||||
"""
|
||||
|
||||
__slots__ = ("_edge",)
|
||||
__slots__ = ('_edge',)
|
||||
|
||||
def __init__(self, edge):
|
||||
if not isinstance(edge, _mgp.Edge):
|
||||
raise TypeError("Expected '_mgp.Edge', got '{}'".format(type(edge)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Edge', got '{}'".format(type(edge)))
|
||||
self._edge = edge
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
@@ -425,7 +408,7 @@ class Edge:
|
||||
return EdgeType(self._edge.get_type_name())
|
||||
|
||||
@property
|
||||
def from_vertex(self) -> "Vertex":
|
||||
def from_vertex(self) -> 'Vertex':
|
||||
"""
|
||||
Get the source vertex.
|
||||
|
||||
@@ -436,7 +419,7 @@ class Edge:
|
||||
return Vertex(self._edge.from_vertex())
|
||||
|
||||
@property
|
||||
def to_vertex(self) -> "Vertex":
|
||||
def to_vertex(self) -> 'Vertex':
|
||||
"""
|
||||
Get the destination vertex.
|
||||
|
||||
@@ -470,7 +453,7 @@ class Edge:
|
||||
|
||||
|
||||
if sys.version_info >= (3, 5, 2):
|
||||
VertexId = typing.NewType("VertexId", int)
|
||||
VertexId = typing.NewType('VertexId', int)
|
||||
else:
|
||||
VertexId = int
|
||||
|
||||
@@ -482,12 +465,12 @@ class Vertex:
|
||||
in a query. You should not globally store an instance of a Vertex. Using an
|
||||
invalid Vertex instance will raise InvalidContextError.
|
||||
"""
|
||||
|
||||
__slots__ = ("_vertex",)
|
||||
__slots__ = ('_vertex',)
|
||||
|
||||
def __init__(self, vertex):
|
||||
if not isinstance(vertex, _mgp.Vertex):
|
||||
raise TypeError("Expected '_mgp.Vertex', got '{}'".format(type(vertex)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Vertex', got '{}'".format(type(vertex)))
|
||||
self._vertex = vertex
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
@@ -530,7 +513,8 @@ class Vertex:
|
||||
"""
|
||||
if not self.is_valid():
|
||||
raise InvalidContextError()
|
||||
return tuple(Label(self._vertex.label_at(i)) for i in range(self._vertex.labels_count()))
|
||||
return tuple(Label(self._vertex.label_at(i))
|
||||
for i in range(self._vertex.labels_count()))
|
||||
|
||||
def add_label(self, label: str) -> None:
|
||||
"""
|
||||
@@ -631,8 +615,7 @@ class Vertex:
|
||||
|
||||
class Path:
|
||||
"""Path containing Vertex and Edge instances."""
|
||||
|
||||
__slots__ = ("_path", "_vertices", "_edges")
|
||||
__slots__ = ('_path', '_vertices', '_edges')
|
||||
|
||||
def __init__(self, starting_vertex_or_path: typing.Union[_mgp.Path, Vertex]):
|
||||
"""Initialize with a starting Vertex.
|
||||
@@ -653,7 +636,8 @@ class Path:
|
||||
raise InvalidContextError()
|
||||
self._path = _mgp.Path.make_with_start(vertex)
|
||||
else:
|
||||
raise TypeError("Expected '_mgp.Vertex' or '_mgp.Path', got '{}'".format(type(starting_vertex_or_path)))
|
||||
raise TypeError("Expected '_mgp.Vertex' or '_mgp.Path', got '{}'"
|
||||
.format(type(starting_vertex_or_path)))
|
||||
|
||||
def __copy__(self):
|
||||
if not self.is_valid():
|
||||
@@ -694,7 +678,8 @@ class Path:
|
||||
extension.
|
||||
"""
|
||||
if not isinstance(edge, Edge):
|
||||
raise TypeError("Expected '_mgp.Edge', got '{}'".format(type(edge)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Edge', got '{}'".format(type(edge)))
|
||||
if not self.is_valid() or not edge.is_valid():
|
||||
raise InvalidContextError()
|
||||
self._path.expand(edge._edge)
|
||||
@@ -713,7 +698,8 @@ class Path:
|
||||
raise InvalidContextError()
|
||||
if self._vertices is None:
|
||||
num_vertices = self._path.size() + 1
|
||||
self._vertices = tuple(Vertex(self._path.vertex_at(i)) for i in range(num_vertices))
|
||||
self._vertices = tuple(Vertex(self._path.vertex_at(i))
|
||||
for i in range(num_vertices))
|
||||
return self._vertices
|
||||
|
||||
@property
|
||||
@@ -727,14 +713,14 @@ class Path:
|
||||
raise InvalidContextError()
|
||||
if self._edges is None:
|
||||
num_edges = self._path.size()
|
||||
self._edges = tuple(Edge(self._path.edge_at(i)) for i in range(num_edges))
|
||||
self._edges = tuple(Edge(self._path.edge_at(i))
|
||||
for i in range(num_edges))
|
||||
return self._edges
|
||||
|
||||
|
||||
class Record:
|
||||
"""Represents a record of resulting field values."""
|
||||
|
||||
__slots__ = ("fields",)
|
||||
__slots__ = ('fields',)
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize with name=value fields in kwargs."""
|
||||
@@ -743,12 +729,12 @@ class Record:
|
||||
|
||||
class Vertices:
|
||||
"""Iterable over vertices in a graph."""
|
||||
|
||||
__slots__ = ("_graph", "_len")
|
||||
__slots__ = ('_graph', '_len')
|
||||
|
||||
def __init__(self, graph):
|
||||
if not isinstance(graph, _mgp.Graph):
|
||||
raise TypeError("Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
self._graph = graph
|
||||
self._len = None
|
||||
|
||||
@@ -805,12 +791,12 @@ class Vertices:
|
||||
|
||||
class Graph:
|
||||
"""State of the graph database in current ProcCtx."""
|
||||
|
||||
__slots__ = ("_graph",)
|
||||
__slots__ = ('_graph',)
|
||||
|
||||
def __init__(self, graph):
|
||||
if not isinstance(graph, _mgp.Graph):
|
||||
raise TypeError("Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
self._graph = graph
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
@@ -899,7 +885,8 @@ class Graph:
|
||||
raise InvalidContextError()
|
||||
self._graph.detach_delete_vertex(vertex._vertex)
|
||||
|
||||
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> None:
|
||||
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex,
|
||||
edge_type: EdgeType) -> None:
|
||||
"""
|
||||
Create an edge.
|
||||
|
||||
@@ -912,7 +899,8 @@ class Graph:
|
||||
"""
|
||||
if not self.is_valid():
|
||||
raise InvalidContextError()
|
||||
return Edge(self._graph.create_edge(from_vertex._vertex, to_vertex._vertex, edge_type.name))
|
||||
return Edge(self._graph.create_edge(from_vertex._vertex,
|
||||
to_vertex._vertex, edge_type.name))
|
||||
|
||||
def delete_edge(self, edge: Edge) -> None:
|
||||
"""
|
||||
@@ -930,7 +918,6 @@ class Graph:
|
||||
|
||||
class AbortError(Exception):
|
||||
"""Signals that the procedure was asked to abort its execution."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@@ -940,12 +927,12 @@ class ProcCtx:
|
||||
Access to a ProcCtx is only valid during a single execution of a procedure
|
||||
in a query. You should not globally store a ProcCtx instance.
|
||||
"""
|
||||
|
||||
__slots__ = ("_graph",)
|
||||
__slots__ = ('_graph',)
|
||||
|
||||
def __init__(self, graph):
|
||||
if not isinstance(graph, _mgp.Graph):
|
||||
raise TypeError("Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
self._graph = Graph(graph)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
@@ -982,7 +969,8 @@ LocalDateTime = datetime.datetime
|
||||
|
||||
Duration = datetime.timedelta
|
||||
|
||||
Any = typing.Union[bool, str, Number, Map, Path, list, Date, LocalTime, LocalDateTime, Duration]
|
||||
Any = typing.Union[bool, str, Number, Map, Path,
|
||||
list, Date, LocalTime, LocalDateTime, Duration]
|
||||
|
||||
List = typing.List
|
||||
|
||||
@@ -1015,7 +1003,7 @@ def _typing_to_cypher_type(type_):
|
||||
Date: _mgp.type_date(),
|
||||
LocalTime: _mgp.type_local_time(),
|
||||
LocalDateTime: _mgp.type_local_date_time(),
|
||||
Duration: _mgp.type_duration(),
|
||||
Duration: _mgp.type_duration()
|
||||
}
|
||||
try:
|
||||
return simple_types[type_]
|
||||
@@ -1033,14 +1021,14 @@ def _typing_to_cypher_type(type_):
|
||||
if type(None) in type_args:
|
||||
types = tuple(t for t in type_args if t is not type(None)) # noqa E721
|
||||
if len(types) == 1:
|
||||
(type_arg,) = types
|
||||
type_arg, = types
|
||||
else:
|
||||
# We cannot do typing.Union[*types], so do the equivalent
|
||||
# with __getitem__ which does not even need arg unpacking.
|
||||
type_arg = typing.Union.__getitem__(types)
|
||||
return _mgp.type_nullable(_typing_to_cypher_type(type_arg))
|
||||
elif complex_type == list:
|
||||
(type_arg,) = type_args
|
||||
type_arg, = type_args
|
||||
return _mgp.type_list(_typing_to_cypher_type(type_arg))
|
||||
raise UnsupportedTypingError(type_)
|
||||
else:
|
||||
@@ -1050,17 +1038,13 @@ def _typing_to_cypher_type(type_):
|
||||
# printed the same way. `typing.List[type]` is printed as such, while
|
||||
# `typing.Optional[type]` is printed as 'typing.Union[type, NoneType]'
|
||||
def parse_type_args(type_as_str):
|
||||
return tuple(
|
||||
map(
|
||||
str.strip,
|
||||
type_as_str[type_as_str.index("[") + 1 : -1].split(","),
|
||||
)
|
||||
)
|
||||
return tuple(map(str.strip,
|
||||
type_as_str[type_as_str.index('[') + 1: -1].split(',')))
|
||||
|
||||
def fully_qualified_name(cls):
|
||||
if cls.__module__ is None or cls.__module__ == "builtins":
|
||||
if cls.__module__ is None or cls.__module__ == 'builtins':
|
||||
return cls.__name__
|
||||
return cls.__module__ + "." + cls.__name__
|
||||
return cls.__module__ + '.' + cls.__name__
|
||||
|
||||
def get_simple_type(type_as_str):
|
||||
for simple_type, cypher_type in simple_types.items():
|
||||
@@ -1076,26 +1060,28 @@ def _typing_to_cypher_type(type_):
|
||||
pass
|
||||
|
||||
def parse_typing(type_as_str):
|
||||
if type_as_str.startswith("typing.Union"):
|
||||
if type_as_str.startswith('typing.Union'):
|
||||
type_args_as_str = parse_type_args(type_as_str)
|
||||
none_type_as_str = type(None).__name__
|
||||
if none_type_as_str in type_args_as_str:
|
||||
types = tuple(t for t in type_args_as_str if t != none_type_as_str)
|
||||
types = tuple(
|
||||
t for t in type_args_as_str if t != none_type_as_str)
|
||||
if len(types) == 1:
|
||||
(type_arg_as_str,) = types
|
||||
type_arg_as_str, = types
|
||||
else:
|
||||
type_arg_as_str = "typing.Union[" + ", ".join(types) + "]"
|
||||
type_arg_as_str = 'typing.Union[' + \
|
||||
', '.join(types) + ']'
|
||||
simple_type = get_simple_type(type_arg_as_str)
|
||||
if simple_type is not None:
|
||||
return _mgp.type_nullable(simple_type)
|
||||
return _mgp.type_nullable(parse_typing(type_arg_as_str))
|
||||
elif type_as_str.startswith("typing.List"):
|
||||
elif type_as_str.startswith('typing.List'):
|
||||
type_arg_as_str = parse_type_args(type_as_str)
|
||||
|
||||
if len(type_arg_as_str) > 1:
|
||||
# Nested object could be a type consisting of a list of types (e.g. mgp.Map)
|
||||
# so we need to join the parts.
|
||||
type_arg_as_str = ", ".join(type_arg_as_str)
|
||||
type_arg_as_str = ', '.join(type_arg_as_str)
|
||||
else:
|
||||
type_arg_as_str = type_arg_as_str[0]
|
||||
|
||||
@@ -1110,11 +1096,9 @@ def _typing_to_cypher_type(type_):
|
||||
|
||||
# Procedure registration
|
||||
|
||||
|
||||
class Deprecated:
|
||||
"""Annotate a resulting Record's field as deprecated."""
|
||||
|
||||
__slots__ = ("field_type",)
|
||||
__slots__ = ('field_type',)
|
||||
|
||||
def __init__(self, type_):
|
||||
self.field_type = type_
|
||||
@@ -1122,7 +1106,8 @@ class Deprecated:
|
||||
|
||||
def raise_if_does_not_meet_requirements(func: typing.Callable[..., Record]):
|
||||
if not callable(func):
|
||||
raise TypeError("Expected a callable object, got an instance of '{}'".format(type(func)))
|
||||
raise TypeError("Expected a callable object, got an instance of '{}'"
|
||||
.format(type(func)))
|
||||
if inspect.iscoroutinefunction(func):
|
||||
raise TypeError("Callable must not be 'async def' function")
|
||||
if sys.version_info >= (3, 6):
|
||||
@@ -1132,25 +1117,24 @@ def raise_if_does_not_meet_requirements(func: typing.Callable[..., Record]):
|
||||
raise NotImplementedError("Generator functions are not supported")
|
||||
|
||||
|
||||
def _register_proc(func: typing.Callable[..., Record], is_write: bool):
|
||||
def _register_proc(func: typing.Callable[..., Record],
|
||||
is_write: bool):
|
||||
raise_if_does_not_meet_requirements(func)
|
||||
register_func = _mgp.Module.add_write_procedure if is_write else _mgp.Module.add_read_procedure
|
||||
register_func = (
|
||||
_mgp.Module.add_write_procedure if is_write
|
||||
else _mgp.Module.add_read_procedure)
|
||||
sig = inspect.signature(func)
|
||||
params = tuple(sig.parameters.values())
|
||||
if params and params[0].annotation is ProcCtx:
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(graph, args):
|
||||
return func(ProcCtx(graph), *args)
|
||||
|
||||
params = params[1:]
|
||||
mgp_proc = register_func(_mgp._MODULE, wrapper)
|
||||
else:
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(graph, args):
|
||||
return func(*args)
|
||||
|
||||
mgp_proc = register_func(_mgp._MODULE, wrapper)
|
||||
for param in params:
|
||||
name = param.name
|
||||
@@ -1165,7 +1149,8 @@ def _register_proc(func: typing.Callable[..., Record], is_write: bool):
|
||||
if sig.return_annotation is not sig.empty:
|
||||
record = sig.return_annotation
|
||||
if not isinstance(record, Record):
|
||||
raise TypeError("Expected '{}' to return 'mgp.Record', got '{}'".format(func.__name__, type(record)))
|
||||
raise TypeError("Expected '{}' to return 'mgp.Record', got '{}'"
|
||||
.format(func.__name__, type(record)))
|
||||
for name, type_ in record.fields.items():
|
||||
if isinstance(type_, Deprecated):
|
||||
cypher_type = _typing_to_cypher_type(type_.field_type)
|
||||
@@ -1179,15 +1164,16 @@ def read_proc(func: typing.Callable[..., Record]):
|
||||
"""
|
||||
Register `func` as a read-only procedure of the current module.
|
||||
|
||||
The decorator `read_proc` is meant to be used to register module procedures.
|
||||
The registered `func` needs to be a callable which optionally takes
|
||||
`ProcCtx` as its first argument. Other arguments of `func` will be bound to
|
||||
values passed in the cypherQuery. The full signature of `func` needs to be
|
||||
annotated with types. The return type must be `Record(field_name=type, ...)`
|
||||
and the procedure must produce either a complete Record or None. To mark a
|
||||
field as deprecated, use `Record(field_name=Deprecated(type), ...)`.
|
||||
Multiple records can be produced by returning an iterable of them.
|
||||
Registering generator functions is currently not supported.
|
||||
`read_proc` is meant to be used as a decorator function to register module
|
||||
procedures. The registered `func` needs to be a callable which optionally
|
||||
takes `ProcCtx` as the first argument. Other arguments of `func` will be
|
||||
bound to values passed in the cypherQuery. The full signature of `func`
|
||||
needs to be annotated with types. The return type must be
|
||||
`Record(field_name=type, ...)` and the procedure must produce either a
|
||||
complete Record or None. To mark a field as deprecated, use
|
||||
`Record(field_name=Deprecated(type), ...)`. Multiple records can be
|
||||
produced by returning an iterable of them. Registering generator functions
|
||||
is currently not supported.
|
||||
|
||||
Example usage.
|
||||
|
||||
@@ -1221,16 +1207,16 @@ def write_proc(func: typing.Callable[..., Record]):
|
||||
"""
|
||||
Register `func` as a writeable procedure of the current module.
|
||||
|
||||
The decorator `write_proc` is meant to be used to register module
|
||||
`write_proc` is meant to be used as a decorator function to register module
|
||||
procedures. The registered `func` needs to be a callable which optionally
|
||||
takes `ProcCtx` as the first argument. Other arguments of `func` will be
|
||||
bound to values passed in the cypherQuery. The full signature of `func`
|
||||
needs to be annotated with types. The return type must be
|
||||
`Record(field_name=type, ...)` and the procedure must produce either a
|
||||
complete Record or None. To mark a field as deprecated, use
|
||||
`Record(field_name=Deprecated(type), ...)`. Multiple records can be produced
|
||||
by returning an iterable of them. Registering generator functions is
|
||||
currently not supported.
|
||||
`Record(field_name=Deprecated(type), ...)`. Multiple records can be
|
||||
produced by returning an iterable of them. Registering generator functions
|
||||
is currently not supported.
|
||||
|
||||
Example usage.
|
||||
|
||||
@@ -1271,22 +1257,20 @@ class InvalidMessageError(Exception):
|
||||
"""
|
||||
Signals using a message instance outside of the registered transformation.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
SOURCE_TYPE_KAFKA = _mgp.SOURCE_TYPE_KAFKA
|
||||
SOURCE_TYPE_PULSAR = _mgp.SOURCE_TYPE_PULSAR
|
||||
|
||||
|
||||
class Message:
|
||||
"""Represents a message from a stream."""
|
||||
|
||||
__slots__ = ("_message",)
|
||||
__slots__ = ('_message',)
|
||||
|
||||
def __init__(self, message):
|
||||
if not isinstance(message, _mgp.Message):
|
||||
raise TypeError("Expected '_mgp.Message', got '{}'".format(type(message)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Message', got '{}'".format(type(message)))
|
||||
self._message = message
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
@@ -1369,18 +1353,17 @@ class Message:
|
||||
|
||||
class InvalidMessagesError(Exception):
|
||||
"""Signals using a messages instance outside of the registered transformation."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class Messages:
|
||||
"""Represents a list of messages from a stream."""
|
||||
|
||||
__slots__ = ("_messages",)
|
||||
__slots__ = ('_messages',)
|
||||
|
||||
def __init__(self, messages):
|
||||
if not isinstance(messages, _mgp.Messages):
|
||||
raise TypeError("Expected '_mgp.Messages', got '{}'".format(type(messages)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Messages', got '{}'".format(type(messages)))
|
||||
self._messages = messages
|
||||
|
||||
def __deepcopy__(self, memo):
|
||||
@@ -1412,12 +1395,12 @@ class TransCtx:
|
||||
Access to a TransCtx is only valid during a single execution of a transformation.
|
||||
You should not globally store a TransCtx instance.
|
||||
"""
|
||||
|
||||
__slots__ = "_graph"
|
||||
__slots__ = ('_graph')
|
||||
|
||||
def __init__(self, graph):
|
||||
if not isinstance(graph, _mgp.Graph):
|
||||
raise TypeError("Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
raise TypeError(
|
||||
"Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
self._graph = Graph(graph)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
@@ -1437,116 +1420,21 @@ def transformation(func: typing.Callable[..., Record]):
|
||||
params = tuple(sig.parameters.values())
|
||||
if not params or not params[0].annotation is Messages:
|
||||
if not len(params) == 2 or not params[1].annotation is Messages:
|
||||
raise NotImplementedError("Valid signatures for transformations are (TransCtx, Messages) or (Messages)")
|
||||
raise NotImplementedError(
|
||||
"Valid signatures for transformations are (TransCtx, Messages) or (Messages)")
|
||||
if params[0].annotation is TransCtx:
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(graph, messages):
|
||||
return func(TransCtx(graph), messages)
|
||||
|
||||
_mgp._MODULE.add_transformation(wrapper)
|
||||
else:
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(graph, messages):
|
||||
return func(messages)
|
||||
|
||||
_mgp._MODULE.add_transformation(wrapper)
|
||||
return func
|
||||
|
||||
|
||||
class FuncCtx:
|
||||
"""Context of a function being executed.
|
||||
|
||||
Access to a FuncCtx is only valid during a single execution of a function in
|
||||
a query. You should not globally store a FuncCtx instance. The graph object
|
||||
within the FuncCtx is not mutable.
|
||||
"""
|
||||
|
||||
__slots__ = "_graph"
|
||||
|
||||
def __init__(self, graph):
|
||||
if not isinstance(graph, _mgp.Graph):
|
||||
raise TypeError("Expected '_mgp.Graph', got '{}'".format(type(graph)))
|
||||
self._graph = Graph(graph)
|
||||
|
||||
def is_valid(self) -> bool:
|
||||
return self._graph.is_valid()
|
||||
|
||||
|
||||
def function(func: typing.Callable):
|
||||
"""
|
||||
Register `func` as a user-defined function in the current module.
|
||||
|
||||
The decorator `function` is meant to be used to register module functions.
|
||||
The registered `func` needs to be a callable which optionally takes
|
||||
`FuncCtx` as its first argument. Other arguments of `func` will be bound to
|
||||
values passed in the Cypher query. Only the funcion arguments need to be
|
||||
annotated with types. The return type doesn't need to be specified, but it
|
||||
has to be supported by `mgp.Any`. Registering generator functions is
|
||||
currently not supported.
|
||||
|
||||
Example usage.
|
||||
|
||||
```
|
||||
import mgp
|
||||
@mgp.function
|
||||
def func_example(context: mgp.FuncCtx,
|
||||
required_arg: str,
|
||||
optional_arg: mgp.Nullable[str] = None
|
||||
):
|
||||
return_args = [required_arg]
|
||||
if optional_arg is not None:
|
||||
return_args.append(optional_arg)
|
||||
# Return any kind of result supported by mgp.Any
|
||||
return return_args
|
||||
```
|
||||
|
||||
The example function above returns a list of provided arguments:
|
||||
* `required_arg` is always present and its value is the first argument of
|
||||
the function.
|
||||
* `optional_arg` is present if the second argument of the function is not
|
||||
`null`.
|
||||
Any errors can be reported by raising an Exception.
|
||||
|
||||
The function can be invoked in Cypher using the following calls:
|
||||
RETURN example.func_example("first argument", "second_argument");
|
||||
RETURN example.func_example("first argument");
|
||||
Naturally, you may pass in different arguments.
|
||||
"""
|
||||
raise_if_does_not_meet_requirements(func)
|
||||
register_func = _mgp.Module.add_function
|
||||
sig = inspect.signature(func)
|
||||
params = tuple(sig.parameters.values())
|
||||
if params and params[0].annotation is FuncCtx:
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(graph, args):
|
||||
return func(FuncCtx(graph), *args)
|
||||
|
||||
params = params[1:]
|
||||
mgp_func = register_func(_mgp._MODULE, wrapper)
|
||||
else:
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(graph, args):
|
||||
return func(*args)
|
||||
|
||||
mgp_func = register_func(_mgp._MODULE, wrapper)
|
||||
|
||||
for param in params:
|
||||
name = param.name
|
||||
type_ = param.annotation
|
||||
if type_ is param.empty:
|
||||
type_ = object
|
||||
cypher_type = _typing_to_cypher_type(type_)
|
||||
if param.default is param.empty:
|
||||
mgp_func.add_arg(name, cypher_type)
|
||||
else:
|
||||
mgp_func.add_opt_arg(name, cypher_type, param.default)
|
||||
return func
|
||||
|
||||
|
||||
def _wrap_exceptions():
|
||||
def wrap_function(func):
|
||||
@wraps(func)
|
||||
@@ -1575,7 +1463,6 @@ def _wrap_exceptions():
|
||||
raise ValueConversionError(e)
|
||||
except _mgp.SerializationError as e:
|
||||
raise SerializationError(e)
|
||||
|
||||
return wrapped_func
|
||||
|
||||
def wrap_prop_func(func):
|
||||
@@ -1586,16 +1473,11 @@ def _wrap_exceptions():
|
||||
if inspect.isfunction(obj):
|
||||
setattr(cls, name, wrap_function(obj))
|
||||
elif isinstance(obj, property):
|
||||
setattr(
|
||||
cls,
|
||||
name,
|
||||
property(
|
||||
wrap_prop_func(obj.fget),
|
||||
wrap_prop_func(obj.fset),
|
||||
wrap_prop_func(obj.fdel),
|
||||
obj.__doc__,
|
||||
),
|
||||
)
|
||||
setattr(cls, name, property(
|
||||
wrap_prop_func(obj.fget),
|
||||
wrap_prop_func(obj.fset),
|
||||
wrap_prop_func(obj.fdel),
|
||||
obj.__doc__))
|
||||
|
||||
def defined_in_this_module(obj: object):
|
||||
return getattr(obj, "__module__", "") == __name__
|
||||
|
||||
16
init
16
init
@@ -24,7 +24,7 @@ function setup_virtualenv () {
|
||||
fi
|
||||
|
||||
# create new virtualenv
|
||||
python3 -m virtualenv -p python3 ve3 || exit 1
|
||||
virtualenv -p python3 ve3 || exit 1
|
||||
source ve3/bin/activate
|
||||
pip --timeout 1000 install -r requirements.txt || exit 1
|
||||
deactivate
|
||||
@@ -65,14 +65,8 @@ else
|
||||
fi
|
||||
|
||||
DISTRO=$(operating_system)
|
||||
ARCHITECTURE=$(architecture)
|
||||
if [ "${ARCHITECTURE}" = "arm64" ] || [ "${ARCHITECTURE}" = "aarch64" ]; then
|
||||
OS_SCRIPT=$DIR/environment/os/$DISTRO-arm.sh
|
||||
else
|
||||
OS_SCRIPT=$DIR/environment/os/$DISTRO.sh
|
||||
fi
|
||||
echo "ALL BUILD PACKAGES: $($OS_SCRIPT list MEMGRAPH_BUILD_DEPS)"
|
||||
$OS_SCRIPT check MEMGRAPH_BUILD_DEPS
|
||||
echo "ALL BUILD PACKAGES: $($DIR/environment/os/$DISTRO.sh list MEMGRAPH_BUILD_DEPS)"
|
||||
$DIR/environment/os/$DISTRO.sh check MEMGRAPH_BUILD_DEPS
|
||||
echo "All packages are in-place..."
|
||||
|
||||
# create a default build directory
|
||||
@@ -135,7 +129,3 @@ for hook in $(find $DIR/.githooks -type f -printf "%f\n"); do
|
||||
ln -s -f "$DIR/.githooks/$hook" "$DIR/.git/hooks/$hook"
|
||||
echo "Added $hook hook"
|
||||
done;
|
||||
|
||||
# Install precommit hook
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pre_commit install
|
||||
|
||||
1
libs/.gitignore
vendored
1
libs/.gitignore
vendored
@@ -4,4 +4,5 @@
|
||||
!cleanup.sh
|
||||
!CMakeLists.txt
|
||||
!__main.cpp
|
||||
!jemalloc.cmake
|
||||
!pulsar.patch
|
||||
|
||||
55
libs/jemalloc.cmake
Normal file
55
libs/jemalloc.cmake
Normal file
@@ -0,0 +1,55 @@
|
||||
set(JEMALLOC_DIR "${LIB_DIR}/jemalloc")
|
||||
|
||||
set(JEMALLOC_SRCS
|
||||
${JEMALLOC_DIR}/src/arena.c
|
||||
${JEMALLOC_DIR}/src/background_thread.c
|
||||
${JEMALLOC_DIR}/src/base.c
|
||||
${JEMALLOC_DIR}/src/bin.c
|
||||
${JEMALLOC_DIR}/src/bitmap.c
|
||||
${JEMALLOC_DIR}/src/ckh.c
|
||||
${JEMALLOC_DIR}/src/ctl.c
|
||||
${JEMALLOC_DIR}/src/div.c
|
||||
${JEMALLOC_DIR}/src/extent.c
|
||||
${JEMALLOC_DIR}/src/extent_dss.c
|
||||
${JEMALLOC_DIR}/src/extent_mmap.c
|
||||
${JEMALLOC_DIR}/src/hash.c
|
||||
${JEMALLOC_DIR}/src/hook.c
|
||||
${JEMALLOC_DIR}/src/jemalloc.c
|
||||
${JEMALLOC_DIR}/src/large.c
|
||||
${JEMALLOC_DIR}/src/log.c
|
||||
${JEMALLOC_DIR}/src/malloc_io.c
|
||||
${JEMALLOC_DIR}/src/mutex.c
|
||||
${JEMALLOC_DIR}/src/mutex_pool.c
|
||||
${JEMALLOC_DIR}/src/nstime.c
|
||||
${JEMALLOC_DIR}/src/pages.c
|
||||
${JEMALLOC_DIR}/src/prng.c
|
||||
${JEMALLOC_DIR}/src/prof.c
|
||||
${JEMALLOC_DIR}/src/rtree.c
|
||||
${JEMALLOC_DIR}/src/sc.c
|
||||
${JEMALLOC_DIR}/src/stats.c
|
||||
${JEMALLOC_DIR}/src/sz.c
|
||||
${JEMALLOC_DIR}/src/tcache.c
|
||||
${JEMALLOC_DIR}/src/test_hooks.c
|
||||
${JEMALLOC_DIR}/src/ticker.c
|
||||
${JEMALLOC_DIR}/src/tsd.c
|
||||
${JEMALLOC_DIR}/src/witness.c
|
||||
${JEMALLOC_DIR}/src/safety_check.c
|
||||
)
|
||||
|
||||
add_library(jemalloc ${JEMALLOC_SRCS})
|
||||
target_include_directories(jemalloc PUBLIC "${JEMALLOC_DIR}/include")
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(jemalloc PUBLIC Threads::Threads)
|
||||
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
|
||||
|
||||
if (CMAKE_BUILD_TYPE STREQUAL "DEBUG")
|
||||
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
|
||||
endif()
|
||||
|
||||
target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
|
||||
# for RTLD_NEXT
|
||||
target_compile_definitions(jemalloc PRIVATE _GNU_SOURCE)
|
||||
|
||||
set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1)
|
||||
@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
|
||||
3. using the Licensed Work to create a work or solution
|
||||
which competes (or might reasonably be expected to
|
||||
compete) with the Licensed Work.
|
||||
CHANGE DATE: 2026-27-04
|
||||
CHANGE DATE: 2026-17-03
|
||||
CHANGE LICENSE: Apache License, Version 2.0
|
||||
|
||||
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.
|
||||
|
||||
@@ -7,11 +7,13 @@ import copy
|
||||
|
||||
|
||||
@mgp.read_proc
|
||||
def procedure(
|
||||
context: mgp.ProcCtx,
|
||||
required_arg: mgp.Nullable[mgp.Any],
|
||||
optional_arg: mgp.Nullable[mgp.Any] = None,
|
||||
) -> mgp.Record(args=list, vertex_count=int, avg_degree=mgp.Number, props=mgp.Nullable[mgp.Map]):
|
||||
def procedure(context: mgp.ProcCtx,
|
||||
required_arg: mgp.Nullable[mgp.Any],
|
||||
optional_arg: mgp.Nullable[mgp.Any] = None
|
||||
) -> mgp.Record(args=list,
|
||||
vertex_count=int,
|
||||
avg_degree=mgp.Number,
|
||||
props=mgp.Nullable[mgp.Map]):
|
||||
"""
|
||||
This example procedure returns 4 fields.
|
||||
|
||||
@@ -35,7 +37,7 @@ def procedure(
|
||||
if isinstance(required_arg, (mgp.Edge, mgp.Vertex)):
|
||||
props = dict(required_arg.properties.items())
|
||||
elif isinstance(required_arg, mgp.Path):
|
||||
(start_vertex,) = required_arg.vertices
|
||||
start_vertex, = required_arg.vertices
|
||||
props = dict(start_vertex.properties.items())
|
||||
# Count the vertices and edges in the database; this may take a while.
|
||||
vertex_count = 0
|
||||
@@ -49,13 +51,15 @@ def procedure(
|
||||
# Copy the received arguments to make it equivalent to the C example.
|
||||
args_copy = [copy.deepcopy(required_arg), copy.deepcopy(optional_arg)]
|
||||
# Multiple rows can be produced by returning an iterable of mgp.Record.
|
||||
return mgp.Record(args=args_copy, vertex_count=vertex_count, avg_degree=avg_degree, props=props)
|
||||
return mgp.Record(args=args_copy, vertex_count=vertex_count,
|
||||
avg_degree=avg_degree, props=props)
|
||||
|
||||
|
||||
@mgp.write_proc
|
||||
def write_procedure(
|
||||
context: mgp.ProcCtx, property_name: str, property_value: mgp.Nullable[mgp.Any]
|
||||
) -> mgp.Record(created_vertex=mgp.Vertex):
|
||||
def write_procedure(context: mgp.ProcCtx,
|
||||
property_name: str,
|
||||
property_value: mgp.Nullable[mgp.Any]
|
||||
) -> mgp.Record(created_vertex=mgp.Vertex):
|
||||
"""
|
||||
This example procedure creates a new vertex with the specified property
|
||||
and connects it to all existing vertex which has the same property with
|
||||
|
||||
@@ -4,17 +4,15 @@ from collections import OrderedDict
|
||||
from itertools import chain, repeat
|
||||
from inspect import cleandoc
|
||||
from typing import List, Tuple
|
||||
|
||||
try:
|
||||
import networkx as nx
|
||||
except ImportError as import_error:
|
||||
sys.stderr.write(
|
||||
(
|
||||
"\n"
|
||||
"NOTE: Please install networkx to be able to use graph_analyzer "
|
||||
"module. Using Python:\n" + sys.version + "\n"
|
||||
)
|
||||
)
|
||||
sys.stderr.write((
|
||||
'\n'
|
||||
'NOTE: Please install networkx to be able to use graph_analyzer '
|
||||
'module. Using Python:\n'
|
||||
+ sys.version +
|
||||
'\n'))
|
||||
raise import_error
|
||||
# Imported last because it also depends on networkx.
|
||||
from mgp_networkx import MemgraphMultiDiGraph # noqa E402
|
||||
@@ -25,14 +23,16 @@ _MAX_LIST_SIZE = 10
|
||||
|
||||
@mgp.read_proc
|
||||
def help() -> mgp.Record(name=str, value=str):
|
||||
"""Shows manual page for graph_analyzer."""
|
||||
'''Shows manual page for graph_analyzer.'''
|
||||
records = []
|
||||
|
||||
def make_records(name, doc):
|
||||
return (mgp.Record(name=n, value=v) for n, v in zip(chain([name], repeat("")), cleandoc(doc).splitlines()))
|
||||
return (mgp.Record(name=n, value=v) for n, v in
|
||||
zip(chain([name], repeat('')), cleandoc(doc).splitlines()))
|
||||
|
||||
for func in (help, analyze, analyze_subgraph):
|
||||
records.extend(make_records("Procedure '{}'".format(func.__name__), func.__doc__))
|
||||
records.extend(make_records("Procedure '{}'".format(func.__name__),
|
||||
func.__doc__))
|
||||
|
||||
for m, v in _get_analysis_mapping().items():
|
||||
records.extend(make_records("Analysis '{}'".format(m), v.__doc__))
|
||||
@@ -41,8 +41,10 @@ def help() -> mgp.Record(name=str, value=str):
|
||||
|
||||
|
||||
@mgp.read_proc
|
||||
def analyze(context: mgp.ProcCtx, analyses: mgp.Nullable[List[str]] = None) -> mgp.Record(name=str, value=str):
|
||||
"""
|
||||
def analyze(context: mgp.ProcCtx,
|
||||
analyses: mgp.Nullable[List[str]] = None
|
||||
) -> mgp.Record(name=str, value=str):
|
||||
'''
|
||||
Shows graph information.
|
||||
|
||||
In case of multiple results, only the first 10 will be shown.
|
||||
@@ -55,20 +57,19 @@ def analyze(context: mgp.ProcCtx, analyses: mgp.Nullable[List[str]] = None) -> m
|
||||
|
||||
Example call (with parameter):
|
||||
CALL graph_analyzer.analyze(['nodes', 'edges']) YIELD *;
|
||||
"""
|
||||
'''
|
||||
g = MemgraphMultiDiGraph(ctx=context)
|
||||
recs = _analyze_graph(context, g, analyses)
|
||||
return [mgp.Record(name=name, value=value) for name, value in recs]
|
||||
|
||||
|
||||
@mgp.read_proc
|
||||
def analyze_subgraph(
|
||||
context: mgp.ProcCtx,
|
||||
vertices: mgp.List[mgp.Vertex],
|
||||
edges: mgp.List[mgp.Edge],
|
||||
analyses: mgp.Nullable[List[str]] = None,
|
||||
) -> mgp.Record(name=str, value=str):
|
||||
"""
|
||||
def analyze_subgraph(context: mgp.ProcCtx,
|
||||
vertices: mgp.List[mgp.Vertex],
|
||||
edges: mgp.List[mgp.Edge],
|
||||
analyses: mgp.Nullable[List[str]] = None
|
||||
) -> mgp.Record(name=str, value=str):
|
||||
'''
|
||||
Shows subgraph information.
|
||||
|
||||
In case of multiple results, only the first 10 will be shown.
|
||||
@@ -90,40 +91,36 @@ def analyze_subgraph(
|
||||
CALL graph_analyzer.analyze_subgraph(nodes, edges, ['nodes', 'edges'])
|
||||
YIELD *
|
||||
RETURN name, value;
|
||||
"""
|
||||
'''
|
||||
vertices, edges = map(set, [vertices, edges])
|
||||
g = nx.subgraph_view(
|
||||
MemgraphMultiDiGraph(ctx=context),
|
||||
lambda n: n in vertices,
|
||||
lambda n1, n2, e: e in edges,
|
||||
)
|
||||
lambda n1, n2, e: e in edges)
|
||||
recs = _analyze_graph(context, g, analyses)
|
||||
return [mgp.Record(name=name, value=value) for name, value in recs]
|
||||
|
||||
|
||||
def _get_analysis_mapping():
|
||||
return OrderedDict(
|
||||
[
|
||||
("nodes", _number_of_nodes),
|
||||
("edges", _number_of_edges),
|
||||
("bridges", _bridges),
|
||||
("articulation_points", _articulation_points),
|
||||
("avg_degree", _avg_degree),
|
||||
("sorted_nodes_degree", _sorted_nodes_degree),
|
||||
("self_loops", _self_loops),
|
||||
("is_bipartite", _is_bipartite),
|
||||
("is_planar", _is_planar),
|
||||
("is_biconnected: ", _is_biconnected),
|
||||
("is_weakly_connected", _is_weakly_connected),
|
||||
("number_of_weakly_components", _weakly_components),
|
||||
("is_strongly_connected", _is_strongly_connected),
|
||||
("strongly_components", _strongly_components),
|
||||
("is_dag", _is_dag),
|
||||
("is_eulerian", _is_eulerian),
|
||||
("is_forest", _is_forest),
|
||||
("is_tree", _is_tree),
|
||||
]
|
||||
)
|
||||
return OrderedDict([
|
||||
('nodes', _number_of_nodes),
|
||||
('edges', _number_of_edges),
|
||||
('bridges', _bridges),
|
||||
('articulation_points', _articulation_points),
|
||||
('avg_degree', _avg_degree),
|
||||
('sorted_nodes_degree', _sorted_nodes_degree),
|
||||
('self_loops', _self_loops),
|
||||
('is_bipartite', _is_bipartite),
|
||||
('is_planar', _is_planar),
|
||||
('is_biconnected: ', _is_biconnected),
|
||||
('is_weakly_connected', _is_weakly_connected),
|
||||
('number_of_weakly_components', _weakly_components),
|
||||
('is_strongly_connected', _is_strongly_connected),
|
||||
('strongly_components', _strongly_components),
|
||||
('is_dag', _is_dag),
|
||||
('is_eulerian', _is_eulerian),
|
||||
('is_forest', _is_forest),
|
||||
('is_tree', _is_tree)])
|
||||
|
||||
|
||||
def _get_analysis_func(name: str):
|
||||
@@ -135,15 +132,20 @@ def _get_analysis_funcs():
|
||||
return _get_analysis_mapping().values()
|
||||
|
||||
|
||||
def _analyze_graph(context: mgp.ProcCtx, g: nx.MultiDiGraph, analyses: List[str]) -> List[Tuple[str, str]]:
|
||||
def _analyze_graph(context: mgp.ProcCtx,
|
||||
g: nx.MultiDiGraph,
|
||||
analyses: List[str]
|
||||
) -> List[Tuple[str, str]]:
|
||||
|
||||
functions = _get_analysis_funcs() if analyses is None else [_get_analysis_func(name) for name in analyses]
|
||||
functions = (_get_analysis_funcs() if analyses is None
|
||||
else [_get_analysis_func(name) for name in analyses])
|
||||
|
||||
records = []
|
||||
for index, f in enumerate(functions):
|
||||
context.check_must_abort()
|
||||
if f is None:
|
||||
raise KeyError("Graph analysis is not supported: " + analyses[index])
|
||||
raise KeyError('Graph analysis is not supported: ' +
|
||||
analyses[index])
|
||||
name, value = f(g)
|
||||
if isinstance(value, (list, set, tuple)):
|
||||
value = list(value)[:_MAX_LIST_SIZE]
|
||||
@@ -153,120 +155,126 @@ def _analyze_graph(context: mgp.ProcCtx, g: nx.MultiDiGraph, analyses: List[str]
|
||||
|
||||
|
||||
def _number_of_nodes(g: nx.MultiDiGraph) -> Tuple[str, int]:
|
||||
"""Returns number of nodes."""
|
||||
return "Number of nodes", nx.number_of_nodes(g)
|
||||
'''Returns number of nodes.'''
|
||||
return 'Number of nodes', nx.number_of_nodes(g)
|
||||
|
||||
|
||||
def _number_of_edges(g: nx.MultiDiGraph) -> Tuple[str, int]:
|
||||
"""Returns number of edges."""
|
||||
return "Number of edges", nx.number_of_edges(g)
|
||||
'''Returns number of edges.'''
|
||||
return 'Number of edges', nx.number_of_edges(g)
|
||||
|
||||
|
||||
def _avg_degree(g: nx.MultiDiGraph) -> Tuple[str, float]:
|
||||
"""Returns average degree."""
|
||||
'''Returns average degree.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
_, number_of_edges = _number_of_edges(g)
|
||||
avg_degree = 0 if number_of_nodes == 0 else number_of_edges / number_of_nodes
|
||||
return "Average degree", avg_degree
|
||||
avg_degree = (0 if number_of_nodes == 0
|
||||
else number_of_edges / number_of_nodes)
|
||||
return 'Average degree', avg_degree
|
||||
|
||||
|
||||
def _sorted_nodes_degree(g: nx.MultiDiGraph) -> Tuple[str, List[int]]:
|
||||
"""Returns list of sorted nodes degree. [(node_id, degree), ...]"""
|
||||
'''Returns list of sorted nodes degree. [(node_id, degree), ...]'''
|
||||
nodes_degree = [(n, g.degree(n)) for n in g.nodes()]
|
||||
nodes_degree.sort(key=lambda x: x[1], reverse=True)
|
||||
return "Sorted nodes degree", nodes_degree
|
||||
return 'Sorted nodes degree', nodes_degree
|
||||
|
||||
|
||||
def _self_loops(g: nx.MultiDiGraph) -> Tuple[str, int]:
|
||||
"""Returns number of self loops."""
|
||||
return "Self loops", sum((1 if e[0] == e[1] else 0 for e in g.edges()))
|
||||
'''Returns number of self loops.'''
|
||||
return 'Self loops', sum((1 if e[0] == e[1] else 0 for e in g.edges()))
|
||||
|
||||
|
||||
def _is_bipartite(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Checks if graph is bipartite."""
|
||||
'''Checks if graph is bipartite.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.algorithms.bipartite.basic.is_bipartite(g)
|
||||
return "Is bipartite", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.algorithms.bipartite.basic.is_bipartite(g))
|
||||
return 'Is bipartite', ret
|
||||
|
||||
|
||||
def _is_planar(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Checks if graph is planar."""
|
||||
'''Checks if graph is planar.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.algorithms.planarity.check_planarity(g)[0]
|
||||
return "Is planar", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.algorithms.planarity.check_planarity(g)[0])
|
||||
return 'Is planar', ret
|
||||
|
||||
|
||||
def _is_biconnected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Check if graph is biconnected."""
|
||||
'''Check if graph is biconnected.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.is_biconnected(nx.MultiDiGraph.to_undirected(g))
|
||||
return "Is biconnected", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.is_biconnected(nx.MultiDiGraph.to_undirected(g)))
|
||||
return 'Is biconnected', ret
|
||||
|
||||
|
||||
def _is_weakly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Check if graph is weakly connected."""
|
||||
'''Check if graph is weakly connected.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.is_weakly_connected(g)
|
||||
return "Is weakly connected", ret
|
||||
return 'Is weakly connected', ret
|
||||
|
||||
|
||||
def _is_strongly_connected(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Checks if graph is strongly connected."""
|
||||
'''Checks if graph is strongly connected.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.is_strongly_connected(g)
|
||||
return "Is strongly connected", ret
|
||||
return 'Is strongly connected', ret
|
||||
|
||||
|
||||
def _is_dag(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Check if graph is directed acyclic graph (DAG)"""
|
||||
'''Check if graph is directed acyclic graph (DAG)'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.algorithms.dag.is_directed_acyclic_graph(g)
|
||||
return "Is DAG", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.algorithms.dag.is_directed_acyclic_graph(g))
|
||||
return 'Is DAG', ret
|
||||
|
||||
|
||||
def _is_eulerian(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Checks if graph is Eulerian."""
|
||||
'''Checks if graph is Eulerian.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.algorithms.euler.is_eulerian(g)
|
||||
return "Is eulerian", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.algorithms.euler.is_eulerian(g))
|
||||
return 'Is eulerian', ret
|
||||
|
||||
|
||||
def _is_forest(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Checks if graph is forest, all components must be trees."""
|
||||
'''Checks if graph is forest, all components must be trees.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.algorithms.tree.recognition.is_forest(g)
|
||||
return "Is forest", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.algorithms.tree.recognition.is_forest(g))
|
||||
return 'Is forest', ret
|
||||
|
||||
|
||||
def _is_tree(g: nx.MultiDiGraph) -> Tuple[str, bool]:
|
||||
"""Checks if graph is tree."""
|
||||
'''Checks if graph is tree.'''
|
||||
_, number_of_nodes = _number_of_nodes(g)
|
||||
ret = False if number_of_nodes == 0 else nx.algorithms.tree.recognition.is_tree(g)
|
||||
return "Is tree", ret
|
||||
ret = (False if number_of_nodes == 0
|
||||
else nx.algorithms.tree.recognition.is_tree(g))
|
||||
return 'Is tree', ret
|
||||
|
||||
|
||||
def _bridges(g: nx.MultiDiGraph) -> Tuple[str, int]:
|
||||
"""Returns number of bridges, multiple edges between same nodes are
|
||||
mapped to one edge."""
|
||||
return "Number of bridges", sum(1 for _ in nx.bridges(nx.Graph(g)))
|
||||
'''Returns number of bridges, multiple edges between same nodes are
|
||||
mapped to one edge.'''
|
||||
return 'Number of bridges', sum(1 for _ in nx.bridges(nx.Graph(g)))
|
||||
|
||||
|
||||
def _articulation_points(g: nx.MultiDiGraph):
|
||||
"""Returns number of articulation points."""
|
||||
'''Returns number of articulation points.'''
|
||||
undirected = nx.MultiDiGraph.to_undirected(g)
|
||||
return (
|
||||
"Number of articulation points",
|
||||
sum(1 for _ in nx.articulation_points(undirected)),
|
||||
)
|
||||
return ('Number of articulation points',
|
||||
sum(1 for _ in nx.articulation_points(undirected)))
|
||||
|
||||
|
||||
def _weakly_components(g: nx.MultiDiGraph):
|
||||
"""Returns number of weakly components."""
|
||||
'''Returns number of weakly components.'''
|
||||
comps = nx.algorithms.components.number_weakly_connected_components(g)
|
||||
return "Number of weakly connected components", comps
|
||||
return 'Number of weakly connected components', comps
|
||||
|
||||
|
||||
def _strongly_components(g: nx.MultiDiGraph):
|
||||
"""Returns number of strongly connected components."""
|
||||
'''Returns number of strongly connected components.'''
|
||||
comps = nx.algorithms.components.number_strongly_connected_components(g)
|
||||
return "Number of strongly connected components", comps
|
||||
return 'Number of strongly connected components', comps
|
||||
|
||||
@@ -1,22 +1,20 @@
|
||||
import sys
|
||||
import mgp
|
||||
import collections
|
||||
|
||||
try:
|
||||
import networkx as nx
|
||||
except ImportError as import_error:
|
||||
sys.stderr.write(
|
||||
(
|
||||
"\n"
|
||||
"NOTE: Please install networkx to be able to use Memgraph NetworkX "
|
||||
"wrappers. Using Python:\n" + sys.version + "\n"
|
||||
)
|
||||
)
|
||||
sys.stderr.write((
|
||||
'\n'
|
||||
'NOTE: Please install networkx to be able to use Memgraph NetworkX '
|
||||
'wrappers. Using Python:\n'
|
||||
+ sys.version +
|
||||
'\n'))
|
||||
raise import_error
|
||||
|
||||
|
||||
class MemgraphAdjlistOuterDict(collections.abc.Mapping):
|
||||
__slots__ = ("_ctx", "_succ", "_multi")
|
||||
__slots__ = ('_ctx', '_succ', '_multi')
|
||||
|
||||
def __init__(self, ctx, succ=True, multi=True):
|
||||
self._ctx = ctx
|
||||
@@ -26,7 +24,8 @@ class MemgraphAdjlistOuterDict(collections.abc.Mapping):
|
||||
def __getitem__(self, key):
|
||||
if key not in self:
|
||||
raise KeyError
|
||||
return MemgraphAdjlistInnerDict(key, succ=self._succ, multi=self._multi)
|
||||
return MemgraphAdjlistInnerDict(key, succ=self._succ,
|
||||
multi=self._multi)
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._ctx.graph.vertices)
|
||||
@@ -41,7 +40,7 @@ class MemgraphAdjlistOuterDict(collections.abc.Mapping):
|
||||
|
||||
|
||||
class MemgraphAdjlistInnerDict(collections.abc.Mapping):
|
||||
__slots__ = ("_node", "_succ", "_multi", "_neighbors")
|
||||
__slots__ = ('_node', '_succ', '_multi', '_neighbors')
|
||||
|
||||
def __init__(self, node, succ=True, multi=True):
|
||||
self._node = node
|
||||
@@ -72,26 +71,31 @@ class MemgraphAdjlistInnerDict(collections.abc.Mapping):
|
||||
def _get_neighbors(self):
|
||||
if not self._neighbors:
|
||||
if self._succ:
|
||||
self._neighbors = set(e.to_vertex for e in self._node.out_edges)
|
||||
self._neighbors = set(
|
||||
e.to_vertex for e in self._node.out_edges)
|
||||
else:
|
||||
self._neighbors = set(e.from_vertex for e in self._node.in_edges)
|
||||
self._neighbors = set(
|
||||
e.from_vertex for e in self._node.in_edges)
|
||||
return self._neighbors
|
||||
|
||||
def _get_edge(self, neighbor):
|
||||
if self._succ:
|
||||
edge = list(filter(lambda e: e.to_vertex == neighbor, self._node.out_edges))
|
||||
edge = list(filter(lambda e: e.to_vertex == neighbor,
|
||||
self._node.out_edges))
|
||||
else:
|
||||
edge = list(filter(lambda e: e.from_vertex == neighbor, self._node.in_edges))
|
||||
edge = list(filter(lambda e: e.from_vertex == neighbor,
|
||||
self._node.in_edges))
|
||||
|
||||
assert len(edge) >= 1
|
||||
if len(edge) > 1:
|
||||
raise RuntimeError("Graph contains multiedges but " "is of non-multigraph type: {}".format(edge))
|
||||
raise RuntimeError('Graph contains multiedges but '
|
||||
'is of non-multigraph type: {}'.format(edge))
|
||||
|
||||
return edge[0]
|
||||
|
||||
|
||||
class MemgraphEdgeKeyDict(collections.abc.Mapping):
|
||||
__slots__ = ("_node", "_neighbor", "_succ", "_edges")
|
||||
__slots__ = ('_node', '_neighbor', '_succ', '_edges')
|
||||
|
||||
def __init__(self, node, neighbor, succ=True):
|
||||
self._node = node
|
||||
@@ -118,14 +122,18 @@ class MemgraphEdgeKeyDict(collections.abc.Mapping):
|
||||
def _get_edges(self):
|
||||
if not self._edges:
|
||||
if self._succ:
|
||||
self._edges = list(filter(lambda e: e.to_vertex == self._neighbor, self._node.out_edges))
|
||||
self._edges = list(filter(
|
||||
lambda e: e.to_vertex == self._neighbor,
|
||||
self._node.out_edges))
|
||||
else:
|
||||
self._edges = list(filter(lambda e: e.from_vertex == self._neighbor, self._node.in_edges))
|
||||
self._edges = list(filter(
|
||||
lambda e: e.from_vertex == self._neighbor,
|
||||
self._node.in_edges))
|
||||
return self._edges
|
||||
|
||||
|
||||
class UnhashableProperties(collections.abc.Mapping):
|
||||
__slots__ = "_properties"
|
||||
__slots__ = ('_properties')
|
||||
|
||||
def __init__(self, properties):
|
||||
self._properties = properties
|
||||
@@ -147,7 +155,7 @@ class UnhashableProperties(collections.abc.Mapping):
|
||||
|
||||
|
||||
class MemgraphNodeDict(collections.abc.Mapping):
|
||||
__slots__ = ("_ctx",)
|
||||
__slots__ = ('_ctx',)
|
||||
|
||||
def __init__(self, ctx):
|
||||
self._ctx = ctx
|
||||
@@ -179,7 +187,8 @@ class MemgraphNodeDict(collections.abc.Mapping):
|
||||
|
||||
|
||||
class MemgraphDiGraphBase:
|
||||
def __init__(self, incoming_graph_data=None, ctx=None, multi=True, **kwargs):
|
||||
def __init__(self, incoming_graph_data=None, ctx=None, multi=True,
|
||||
**kwargs):
|
||||
# NOTE: We assume that our graph will never be given any initial data
|
||||
# because we already pull our data from the Memgraph database. This
|
||||
# assert is triggered by certain NetworkX procedures because they
|
||||
@@ -192,30 +201,23 @@ class MemgraphDiGraphBase:
|
||||
# modify the graph's internal attributes and don't try to populate it
|
||||
# with initial data or modify it.
|
||||
|
||||
self.node_dict_factory = lambda: MemgraphNodeDict(ctx) if ctx else self._error
|
||||
self.node_dict_factory = lambda: MemgraphNodeDict(ctx) \
|
||||
if ctx else self._error
|
||||
self.node_attr_dict_factory = self._error
|
||||
|
||||
self.adjlist_outer_dict_factory = lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) if ctx else self._error
|
||||
self.adjlist_outer_dict_factory = \
|
||||
lambda: MemgraphAdjlistOuterDict(ctx, multi=multi) \
|
||||
if ctx else self._error
|
||||
self.adjlist_inner_dict_factory = self._error
|
||||
self.edge_key_dict_factory = self._error
|
||||
self.edge_attr_dict_factory = self._error
|
||||
|
||||
# NOTE: We forbid any mutating operations because our graph is
|
||||
# immutable and pulls its data from the Memgraph database.
|
||||
for f in [
|
||||
"add_node",
|
||||
"add_nodes_from",
|
||||
"remove_node",
|
||||
"remove_nodes_from",
|
||||
"add_edge",
|
||||
"add_edges_from",
|
||||
"add_weighted_edges_from",
|
||||
"new_edge_key",
|
||||
"remove_edge",
|
||||
"remove_edges_from",
|
||||
"update",
|
||||
"clear",
|
||||
]:
|
||||
for f in ['add_node', 'add_nodes_from', 'remove_node',
|
||||
'remove_nodes_from', 'add_edge', 'add_edges_from',
|
||||
'add_weighted_edges_from', 'new_edge_key', 'remove_edge',
|
||||
'remove_edges_from', 'update', 'clear']:
|
||||
setattr(self, f, lambda *args, **kwargs: self._error())
|
||||
|
||||
super().__init__(None, **kwargs)
|
||||
@@ -229,29 +231,33 @@ class MemgraphDiGraphBase:
|
||||
self._pred = MemgraphAdjlistOuterDict(ctx, succ=False, multi=multi)
|
||||
|
||||
def _error(self):
|
||||
raise RuntimeError("Modification operations are not supported")
|
||||
raise RuntimeError('Modification operations are not supported')
|
||||
|
||||
|
||||
class MemgraphMultiDiGraph(MemgraphDiGraphBase, nx.MultiDiGraph):
|
||||
def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
|
||||
super().__init__(incoming_graph_data=incoming_graph_data, ctx=ctx, multi=True, **kwargs)
|
||||
super().__init__(incoming_graph_data=incoming_graph_data,
|
||||
ctx=ctx, multi=True, **kwargs)
|
||||
|
||||
|
||||
def MemgraphMultiGraph(incoming_graph_data=None, ctx=None, **kwargs):
|
||||
return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data, ctx=ctx, **kwargs).to_undirected(as_view=True)
|
||||
return MemgraphMultiDiGraph(incoming_graph_data=incoming_graph_data,
|
||||
ctx=ctx, **kwargs).to_undirected(as_view=True)
|
||||
|
||||
|
||||
class MemgraphDiGraph(MemgraphDiGraphBase, nx.DiGraph):
|
||||
def __init__(self, incoming_graph_data=None, ctx=None, **kwargs):
|
||||
super().__init__(incoming_graph_data=incoming_graph_data, ctx=ctx, multi=False, **kwargs)
|
||||
super().__init__(incoming_graph_data=incoming_graph_data,
|
||||
ctx=ctx, multi=False, **kwargs)
|
||||
|
||||
|
||||
def MemgraphGraph(incoming_graph_data=None, ctx=None, **kwargs):
|
||||
return MemgraphDiGraph(incoming_graph_data=incoming_graph_data, ctx=ctx, **kwargs).to_undirected(as_view=True)
|
||||
return MemgraphDiGraph(incoming_graph_data=incoming_graph_data,
|
||||
ctx=ctx, **kwargs).to_undirected(as_view=True)
|
||||
|
||||
|
||||
class PropertiesDictionary(collections.abc.Mapping):
|
||||
__slots__ = ("_ctx", "_prop", "_len")
|
||||
__slots__ = ('_ctx', '_prop', '_len')
|
||||
|
||||
def __init__(self, ctx, prop):
|
||||
self._ctx = ctx
|
||||
@@ -264,7 +270,8 @@ class PropertiesDictionary(collections.abc.Mapping):
|
||||
try:
|
||||
return vertex.properties[self._prop]
|
||||
except KeyError:
|
||||
raise KeyError(("{} doesn\t have the required " + "property '{}'").format(vertex, self._prop))
|
||||
raise KeyError(("{} doesn\t have the required " +
|
||||
"property '{}'").format(vertex, self._prop))
|
||||
|
||||
def __iter__(self):
|
||||
for v in self._ctx.graph.vertices:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,20 +1,23 @@
|
||||
import sys
|
||||
import mgp
|
||||
|
||||
try:
|
||||
import networkx as nx
|
||||
except ImportError as import_error:
|
||||
sys.stderr.write(
|
||||
"\n" "NOTE: Please install networkx to be able to use wcc module.\n" "Using Python:\n" + sys.version + "\n"
|
||||
)
|
||||
'\n'
|
||||
'NOTE: Please install networkx to be able to use wcc module.\n'
|
||||
'Using Python:\n'
|
||||
+ sys.version +
|
||||
'\n')
|
||||
raise import_error
|
||||
|
||||
|
||||
@mgp.read_proc
|
||||
def get_components(
|
||||
vertices: mgp.List[mgp.Vertex], edges: mgp.List[mgp.Edge]
|
||||
) -> mgp.Record(n_components=int, components=mgp.List[mgp.List[mgp.Vertex]]):
|
||||
"""
|
||||
def get_components(vertices: mgp.List[mgp.Vertex],
|
||||
edges: mgp.List[mgp.Edge]
|
||||
) -> mgp.Record(n_components=int,
|
||||
components=mgp.List[mgp.List[mgp.Vertex]]):
|
||||
'''
|
||||
This procedure finds weakly connected components of a given subgraph of a
|
||||
directed graph.
|
||||
|
||||
@@ -38,7 +41,7 @@ def get_components(
|
||||
WITH collect(n) AS nodes, collect(e) AS edges
|
||||
CALL wcc.get_components(nodes, edges) YIELD *
|
||||
RETURN n_components, components;
|
||||
"""
|
||||
'''
|
||||
g = nx.DiGraph()
|
||||
g.add_nodes_from(vertices)
|
||||
g.add_edges_from([(edge.from_vertex, edge.to_vertex) for edge in edges])
|
||||
|
||||
@@ -10,14 +10,6 @@ set(CPACK_PACKAGE_VENDOR "Memgraph Ltd.")
|
||||
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY
|
||||
"High performance, in-memory, transactional graph database")
|
||||
|
||||
# Setting arhitecture extension for deb packages
|
||||
set(MG_ARCH_EXTENSION_DEB "all")
|
||||
if (${MG_ARCH} STREQUAL "x86_64")
|
||||
set(MG_ARCH_EXTENSION_DEB "amd64")
|
||||
elseif (${MG_ARCH} STREQUAL "ARM64")
|
||||
set(MG_ARCH_EXTENSION_DEB "arm64")
|
||||
endif()
|
||||
|
||||
# DEB specific
|
||||
# Instead of using "name <email>" format, we use "email (name)" to prevent
|
||||
# errors due to full stop, '.' at the end of "Ltd". (See: RFC 822)
|
||||
@@ -25,7 +17,7 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "tech@memgraph.com (Memgraph Ltd.)")
|
||||
set(CPACK_DEBIAN_PACKAGE_SECTION non-free/database)
|
||||
set(CPACK_DEBIAN_PACKAGE_HOMEPAGE https://memgraph.com)
|
||||
set(CPACK_DEBIAN_PACKAGE_VERSION "${MEMGRAPH_VERSION_DEB}")
|
||||
set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_${MG_ARCH_EXTENSION_DEB}.deb")
|
||||
set(CPACK_DEBIAN_FILE_NAME "memgraph_${MEMGRAPH_VERSION_DEB}_amd64.deb")
|
||||
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/debian/conffiles;"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/debian/copyright;"
|
||||
@@ -41,20 +33,21 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
|
||||
applications driver by real-time connected data.")
|
||||
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
|
||||
# We also depend on `python3` because we embed it in Memgraph.
|
||||
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6")
|
||||
|
||||
# Setting arhitecture extension for rpm packages
|
||||
set(MG_ARCH_EXTENSION_RPM "noarch")
|
||||
if (${MG_ARCH} STREQUAL "x86_64")
|
||||
set(MG_ARCH_EXTENSION_RPM "x86_64")
|
||||
elseif (${MG_ARCH} STREQUAL "ARM64")
|
||||
set(MG_ARCH_EXTENSION_RPM "aarch64")
|
||||
endif()
|
||||
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
|
||||
|
||||
# RPM specific
|
||||
|
||||
set(MG_ARCH_EXTENSION "noarch")
|
||||
|
||||
if (${MG_ARCH} STREQUAL "x86_64")
|
||||
set(MG_ARCH_EXTENSION "x86_64")
|
||||
elseif (${MG_ARCH} STREQUAL "ARM64")
|
||||
set(MG_ARCH_EXTENSION "aarch64")
|
||||
endif()
|
||||
|
||||
set(CPACK_RPM_PACKAGE_URL https://memgraph.com)
|
||||
set(CPACK_RPM_PACKAGE_VERSION "${MEMGRAPH_VERSION_RPM}")
|
||||
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION_RPM}.rpm")
|
||||
set(CPACK_RPM_FILE_NAME "memgraph-${MEMGRAPH_VERSION_RPM}-1.${MG_ARCH_EXTENSION}.rpm")
|
||||
set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
|
||||
/var /var/lib /var/log /etc/logrotate.d
|
||||
/lib /lib/systemd /lib/systemd/system /lib/systemd/system/memgraph.service)
|
||||
@@ -67,7 +60,7 @@ It aims to deliver developers the speed, simplicity and scale required to build
|
||||
the next generation of applications driver by real-time connected data.")
|
||||
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
|
||||
# We also depend on `python3` because we embed it in Memgraph.
|
||||
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc >= 6, logrotate")
|
||||
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0")
|
||||
|
||||
# All variables must be set before including.
|
||||
include(CPack)
|
||||
|
||||
@@ -1,21 +1,19 @@
|
||||
FROM debian:bullseye
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG BINARY_NAME
|
||||
ARG EXTENSION
|
||||
ARG TARGETARCH
|
||||
ARG release
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
|
||||
--no-install-recommends \
|
||||
openssl libcurl4 libssl1.1 libseccomp2 python3 libpython3.9 python3-pip \
|
||||
--no-install-recommends \
|
||||
&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
|
||||
|
||||
COPY "${BINARY_NAME}${TARGETARCH}.${EXTENSION}" /
|
||||
COPY ${release} /
|
||||
|
||||
# Install memgraph package
|
||||
RUN dpkg -i "${BINARY_NAME}${TARGETARCH}.deb"
|
||||
RUN dpkg -i ${release}
|
||||
|
||||
# Memgraph listens for Bolt Protocol on this port by default.
|
||||
EXPOSE 7687
|
||||
|
||||
31
release/docker/memgraph_rpm.dockerfile
Normal file
31
release/docker/memgraph_rpm.dockerfile
Normal file
@@ -0,0 +1,31 @@
|
||||
FROM dokken/centos-stream-9
|
||||
# NOTE: If you change the base distro update release/package as well.
|
||||
|
||||
ARG release
|
||||
|
||||
RUN yum update && yum install -y \
|
||||
openssl libcurl libseccomp python3 python3-pip \
|
||||
--nobest --allowerasing \
|
||||
&& rm -rf /tmp/* \
|
||||
&& yum clean all
|
||||
|
||||
RUN pip3 install networkx==2.4 numpy==1.21.4 scipy==1.7.3
|
||||
|
||||
COPY ${release} /
|
||||
|
||||
# Install memgraph package
|
||||
RUN rpm -i ${release}
|
||||
|
||||
# Memgraph listens for Bolt Protocol on this port by default.
|
||||
EXPOSE 7687
|
||||
# Snapshots and logging volumes
|
||||
VOLUME /var/log/memgraph
|
||||
VOLUME /var/lib/memgraph
|
||||
# Configuration volume
|
||||
VOLUME /etc/memgraph
|
||||
|
||||
USER memgraph
|
||||
WORKDIR /usr/lib/memgraph
|
||||
|
||||
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
|
||||
CMD [""]
|
||||
@@ -55,10 +55,7 @@ image_name="memgraph:${version}"
|
||||
image_package_name="memgraph-${version}-docker.tar.gz"
|
||||
|
||||
# Build docker image.
|
||||
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
|
||||
--build-arg BINARY_NAME=${package_name} \
|
||||
--build-arg EXTENSION=${extension} \
|
||||
--build-arg TARGETARCH="" .
|
||||
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} --build-arg release=${package_name}.${extension} .
|
||||
docker save ${image_name} ${latest_image} | gzip > ${image_package_name}
|
||||
rm "${package_name}.${extension}"
|
||||
echo "Built Docker image at '${working_dir}/${image_package_name}'"
|
||||
|
||||
@@ -104,9 +104,7 @@ def retry(retry_limit, timeout=100):
|
||||
except Exception:
|
||||
time.sleep(timeout)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return inner_func
|
||||
|
||||
|
||||
@@ -165,15 +163,8 @@ def format_version(variant, version, offering, distance=None, shorthash=None, su
|
||||
|
||||
# Parse arguments.
|
||||
parser = argparse.ArgumentParser(description="Get the current version of Memgraph.")
|
||||
parser.add_argument(
|
||||
"--open-source",
|
||||
action="store_true",
|
||||
help="set the current offering to 'open-source'",
|
||||
)
|
||||
parser.add_argument(
|
||||
"version",
|
||||
help="manual version override, if supplied the version isn't " "determined using git",
|
||||
)
|
||||
parser.add_argument("--open-source", action="store_true", help="set the current offering to 'open-source'")
|
||||
parser.add_argument("version", help="manual version override, if supplied the version isn't " "determined using git")
|
||||
parser.add_argument("suffix", help="custom suffix for the current version being built")
|
||||
parser.add_argument(
|
||||
"--variant",
|
||||
@@ -182,9 +173,7 @@ parser.add_argument(
|
||||
help="which variant of the version string should be generated",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--memgraph-root-dir",
|
||||
help="The root directory of the checked out " "Memgraph repository.",
|
||||
default=".",
|
||||
"--memgraph-root-dir", help="The root directory of the checked out " "Memgraph repository.", default="."
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -267,27 +256,14 @@ for version in versions:
|
||||
if current_version is None:
|
||||
raise Exception("You are attempting to determine the version for a very " "old version of Memgraph!")
|
||||
version, branch, master_branch_merge = current_version
|
||||
distance = int(
|
||||
get_output(
|
||||
"git",
|
||||
"rev-list",
|
||||
"--count",
|
||||
"--first-parent",
|
||||
master_branch_merge + ".." + current_hash,
|
||||
)
|
||||
)
|
||||
distance = int(get_output("git", "rev-list", "--count", "--first-parent", master_branch_merge + ".." + current_hash))
|
||||
version_str = ".".join(map(str, version)) + ".0"
|
||||
if distance == 0:
|
||||
print(format_version(args.variant, version_str, offering, suffix=args.suffix), end="")
|
||||
else:
|
||||
print(
|
||||
format_version(
|
||||
args.variant,
|
||||
version_str,
|
||||
offering,
|
||||
distance=distance,
|
||||
shorthash=current_hash_short,
|
||||
suffix=args.suffix,
|
||||
args.variant, version_str, offering, distance=distance, shorthash=current_hash_short, suffix=args.suffix
|
||||
),
|
||||
end="",
|
||||
)
|
||||
|
||||
@@ -7,8 +7,8 @@ RUN yum -y update \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7-x86_64.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-7.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
14
release/package/centos-8/Dockerfile
Normal file
14
release/package/centos-8/Dockerfile
Normal file
@@ -0,0 +1,14 @@
|
||||
FROM centos:8
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
RUN dnf -y update \
|
||||
&& dnf install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-8.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -1,14 +0,0 @@
|
||||
FROM quay.io/centos/centos:stream9
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
RUN yum -y update \
|
||||
&& yum install -y wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-centos-9-x86_64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10-amd64.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-10.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
FROM debian:11
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-arm64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11-amd64.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-debian-11.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -4,10 +4,10 @@ services:
|
||||
build:
|
||||
context: centos-7
|
||||
container_name: "mgbuild_centos-7"
|
||||
mgbuild_centos-9:
|
||||
mgbuild_centos-8:
|
||||
build:
|
||||
context: centos-9
|
||||
container_name: "mgbuild_centos-9"
|
||||
context: centos-8
|
||||
container_name: "mgbuild_centos-8"
|
||||
mgbuild_debian-10:
|
||||
build:
|
||||
context: debian-10
|
||||
@@ -24,7 +24,3 @@ services:
|
||||
build:
|
||||
context: ubuntu-20.04
|
||||
container_name: "mgbuild_ubuntu-20.04"
|
||||
mgbuild_ubuntu-22.04:
|
||||
build:
|
||||
context: ubuntu-22.04
|
||||
container_name: "mgbuild_ubuntu-22.04"
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
SUPPORTED_OS=(centos-7 centos-9 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04 ubuntu-22.04 debian-11-arm)
|
||||
SUPPORTED_OS=(centos-7 centos-8 debian-10 debian-11 ubuntu-18.04 ubuntu-20.04)
|
||||
PROJECT_ROOT="$SCRIPT_DIR/../.."
|
||||
TOOLCHAIN_VERSION="toolchain-v4"
|
||||
ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
|
||||
@@ -67,18 +67,14 @@ make_package () {
|
||||
# environment/os/{os}.sh does not come within the toolchain package. When
|
||||
# migrating to the next version of toolchain do that, and remove the
|
||||
# TOOLCHAIN_RUN_DEPS installation from here.
|
||||
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
|
||||
echo "Installing dependencies..."
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
|
||||
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
|
||||
|
||||
echo "Building targeted package..."
|
||||
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
|
||||
if [[ "$os" == "debian-11-arm" ]]; then
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
|
||||
else
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
|
||||
fi
|
||||
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release $telemetry_id_override_flag .."
|
||||
# ' is used instead of " because we need to run make within the allowed
|
||||
# container resources.
|
||||
# shellcheck disable=SC2016
|
||||
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04-amd64.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-18.04.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -10,8 +10,8 @@ RUN apt update && apt install -y \
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04-amd64.tar.gz -C /opt
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-20.04.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
FROM ubuntu:22.04
|
||||
|
||||
ARG TOOLCHAIN_VERSION
|
||||
|
||||
# Stops tzdata interactive configuration.
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt update && apt install -y \
|
||||
ca-certificates wget git
|
||||
# Do NOT be smart here and clean the cache because the container is used in the
|
||||
# stateful context.
|
||||
|
||||
RUN wget -q https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/${TOOLCHAIN_VERSION}/${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
-O ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz \
|
||||
&& tar xzvf ${TOOLCHAIN_VERSION}-binaries-ubuntu-22.04-amd64.tar.gz -C /opt
|
||||
|
||||
ENTRYPOINT ["sleep", "infinity"]
|
||||
@@ -22,7 +22,7 @@
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(auth_password_permit_null, true, "Set to false to disable null passwords.");
|
||||
|
||||
inline constexpr std::string_view default_password_regex = ".+";
|
||||
constexpr std::string_view default_password_regex = ".+";
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_string(auth_password_strength_regex, default_password_regex.data(),
|
||||
"The regular expression that should be used to match the entire "
|
||||
@@ -84,8 +84,6 @@ std::string PermissionToString(Permission permission) {
|
||||
return "MODULE_WRITE";
|
||||
case Permission::WEBSOCKET:
|
||||
return "WEBSOCKET";
|
||||
case Permission::LABELS:
|
||||
return "LABELS";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,107 +183,19 @@ bool operator==(const Permissions &first, const Permissions &second) {
|
||||
|
||||
bool operator!=(const Permissions &first, const Permissions &second) { return !(first == second); }
|
||||
|
||||
LabelPermissions::LabelPermissions(const std::unordered_set<std::string> &grants,
|
||||
const std::unordered_set<std::string> &denies)
|
||||
: grants_(grants), denies_(denies) {}
|
||||
|
||||
PermissionLevel LabelPermissions::Has(const std::string &permission) const {
|
||||
if (denies_.find(permission) != denies_.end()) {
|
||||
return PermissionLevel::DENY;
|
||||
}
|
||||
|
||||
if (grants_.find(permission) != denies_.end()) {
|
||||
return PermissionLevel::GRANT;
|
||||
}
|
||||
|
||||
return PermissionLevel::NEUTRAL;
|
||||
}
|
||||
|
||||
void LabelPermissions::Grant(const std::string &permission) {
|
||||
auto deniedPermissionIter = denies_.find(permission);
|
||||
|
||||
if (deniedPermissionIter != denies_.end()) {
|
||||
denies_.erase(deniedPermissionIter);
|
||||
}
|
||||
|
||||
if (grants_.find(permission) == grants_.end()) {
|
||||
grants_.insert(permission);
|
||||
}
|
||||
}
|
||||
|
||||
void LabelPermissions::Revoke(const std::string &permission) {
|
||||
auto deniedPermissionIter = denies_.find(permission);
|
||||
auto grantedPermissionIter = grants_.find(permission);
|
||||
|
||||
if (deniedPermissionIter != denies_.end()) {
|
||||
denies_.erase(deniedPermissionIter);
|
||||
}
|
||||
|
||||
if (grantedPermissionIter != grants_.end()) {
|
||||
grants_.erase(grantedPermissionIter);
|
||||
}
|
||||
}
|
||||
|
||||
void LabelPermissions::Deny(const std::string &permission) {
|
||||
auto grantedPermissionIter = grants_.find(permission);
|
||||
|
||||
if (grantedPermissionIter != grants_.end()) {
|
||||
grants_.erase(grantedPermissionIter);
|
||||
}
|
||||
|
||||
if (denies_.find(permission) == denies_.end()) {
|
||||
denies_.insert(permission);
|
||||
}
|
||||
}
|
||||
|
||||
std::unordered_set<std::string> LabelPermissions::GetGrants() const { return grants_; }
|
||||
|
||||
std::unordered_set<std::string> LabelPermissions::GetDenies() const { return denies_; }
|
||||
|
||||
nlohmann::json LabelPermissions::Serialize() const {
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["grants"] = grants_;
|
||||
data["denies"] = denies_;
|
||||
return data;
|
||||
}
|
||||
|
||||
LabelPermissions LabelPermissions::Deserialize(const nlohmann::json &data) {
|
||||
if (!data.is_object()) {
|
||||
throw AuthException("Couldn't load permissions data!");
|
||||
}
|
||||
|
||||
return {LabelPermissions(data["grants"], data["denies"])};
|
||||
}
|
||||
|
||||
std::unordered_set<std::string> LabelPermissions::grants() const { return grants_; }
|
||||
std::unordered_set<std::string> LabelPermissions::denies() const { return denies_; }
|
||||
|
||||
bool operator==(const LabelPermissions &first, const LabelPermissions &second) {
|
||||
return first.grants() == second.grants() && first.denies() == second.denies();
|
||||
}
|
||||
|
||||
bool operator!=(const LabelPermissions &first, const LabelPermissions &second) { return !(first == second); }
|
||||
|
||||
Role::Role(const std::string &rolename) : rolename_(utils::ToLowerCase(rolename)) {}
|
||||
|
||||
Role::Role(const std::string &rolename, const Permissions &permissions)
|
||||
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
|
||||
|
||||
Role::Role(const std::string &rolename, const Permissions &permissions, const LabelPermissions &labelPermissions)
|
||||
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions), labelPermissions_(labelPermissions) {}
|
||||
|
||||
const std::string &Role::rolename() const { return rolename_; }
|
||||
const Permissions &Role::permissions() const { return permissions_; }
|
||||
Permissions &Role::permissions() { return permissions_; }
|
||||
|
||||
LabelPermissions &Role::labelPermissions() { return labelPermissions_; }
|
||||
|
||||
nlohmann::json Role::Serialize() const {
|
||||
nlohmann::json data = nlohmann::json::object();
|
||||
data["rolename"] = rolename_;
|
||||
data["permissions"] = permissions_.Serialize();
|
||||
data["labelPermissions"] = labelPermissions_.Serialize();
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -297,9 +207,7 @@ Role Role::Deserialize(const nlohmann::json &data) {
|
||||
throw AuthException("Couldn't load role data!");
|
||||
}
|
||||
auto permissions = Permissions::Deserialize(data["permissions"]);
|
||||
auto labelPermissions = LabelPermissions::Deserialize(data["labelPermissions"]);
|
||||
|
||||
return {data["rolename"], permissions, labelPermissions};
|
||||
return {data["rolename"], permissions};
|
||||
}
|
||||
|
||||
bool operator==(const Role &first, const Role &second) {
|
||||
@@ -311,13 +219,6 @@ User::User(const std::string &username) : username_(utils::ToLowerCase(username)
|
||||
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
|
||||
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
|
||||
|
||||
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
|
||||
const LabelPermissions &labelPermissions)
|
||||
: username_(utils::ToLowerCase(username)),
|
||||
password_hash_(password_hash),
|
||||
permissions_(permissions),
|
||||
labelPermissions_(labelPermissions) {}
|
||||
|
||||
bool User::CheckPassword(const std::string &password) {
|
||||
if (password_hash_.empty()) return true;
|
||||
return VerifyPassword(password, password_hash_);
|
||||
@@ -370,8 +271,6 @@ const std::string &User::username() const { return username_; }
|
||||
const Permissions &User::permissions() const { return permissions_; }
|
||||
Permissions &User::permissions() { return permissions_; }
|
||||
|
||||
LabelPermissions &User::labelPermissions() { return labelPermissions_; }
|
||||
|
||||
const Role *User::role() const {
|
||||
if (role_.has_value()) {
|
||||
return &role_.value();
|
||||
@@ -384,7 +283,6 @@ nlohmann::json User::Serialize() const {
|
||||
data["username"] = username_;
|
||||
data["password_hash"] = password_hash_;
|
||||
data["permissions"] = permissions_.Serialize();
|
||||
data["labelPermissions"] = labelPermissions_.Serialize();
|
||||
// The role shouldn't be serialized here, it is stored as a foreign key.
|
||||
return data;
|
||||
}
|
||||
@@ -397,14 +295,11 @@ User User::Deserialize(const nlohmann::json &data) {
|
||||
throw AuthException("Couldn't load user data!");
|
||||
}
|
||||
auto permissions = Permissions::Deserialize(data["permissions"]);
|
||||
auto labelPermissions = LabelPermissions::Deserialize(data["labelPermissions"]);
|
||||
|
||||
return {data["username"], data["password_hash"], permissions, labelPermissions};
|
||||
return {data["username"], data["password_hash"], permissions};
|
||||
}
|
||||
|
||||
bool operator==(const User &first, const User &second) {
|
||||
return first.username_ == second.username_ && first.password_hash_ == second.password_hash_ &&
|
||||
first.permissions_ == second.permissions_ && first.role_ == second.role_;
|
||||
}
|
||||
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
#include <string>
|
||||
|
||||
#include <json/json.hpp>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace memgraph::auth {
|
||||
// These permissions must have values that are applicable for usage in a
|
||||
@@ -39,8 +38,7 @@ enum class Permission : uint64_t {
|
||||
STREAM = 1U << 17U,
|
||||
MODULE_READ = 1U << 18U,
|
||||
MODULE_WRITE = 1U << 19U,
|
||||
WEBSOCKET = 1U << 20U,
|
||||
LABELS = 1U << 21U
|
||||
WEBSOCKET = 1U << 20U
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
@@ -90,52 +88,16 @@ bool operator==(const Permissions &first, const Permissions &second);
|
||||
|
||||
bool operator!=(const Permissions &first, const Permissions &second);
|
||||
|
||||
class LabelPermissions final {
|
||||
public:
|
||||
LabelPermissions(const std::unordered_set<std::string> &grants = {},
|
||||
const std::unordered_set<std::string> &denies = {});
|
||||
|
||||
PermissionLevel Has(const std::string &permission) const;
|
||||
|
||||
void Grant(const std::string &permission);
|
||||
|
||||
void Revoke(const std::string &permission);
|
||||
|
||||
void Deny(const std::string &permission);
|
||||
|
||||
std::unordered_set<std::string> GetGrants() const;
|
||||
std::unordered_set<std::string> GetDenies() const;
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
|
||||
/// @throw AuthException if unable to deserialize.
|
||||
static LabelPermissions Deserialize(const nlohmann::json &data);
|
||||
|
||||
std::unordered_set<std::string> grants() const;
|
||||
std::unordered_set<std::string> denies() const;
|
||||
|
||||
private:
|
||||
std::unordered_set<std::string> grants_{};
|
||||
std::unordered_set<std::string> denies_{};
|
||||
};
|
||||
|
||||
bool operator==(const LabelPermissions &first, const LabelPermissions &second);
|
||||
|
||||
bool operator!=(const LabelPermissions &first, const LabelPermissions &second);
|
||||
class Role final {
|
||||
public:
|
||||
Role(const std::string &rolename);
|
||||
|
||||
Role(const std::string &rolename, const Permissions &permissions);
|
||||
|
||||
Role(const std::string &rolename, const Permissions &permissions, const LabelPermissions &labelPermissions);
|
||||
|
||||
const std::string &rolename() const;
|
||||
const Permissions &permissions() const;
|
||||
Permissions &permissions();
|
||||
|
||||
LabelPermissions &labelPermissions();
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
|
||||
/// @throw AuthException if unable to deserialize.
|
||||
@@ -146,7 +108,6 @@ class Role final {
|
||||
private:
|
||||
std::string rolename_;
|
||||
Permissions permissions_;
|
||||
LabelPermissions labelPermissions_;
|
||||
};
|
||||
|
||||
bool operator==(const Role &first, const Role &second);
|
||||
@@ -158,9 +119,6 @@ class User final {
|
||||
|
||||
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
|
||||
|
||||
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
|
||||
const LabelPermissions &labelPermissions);
|
||||
|
||||
/// @throw AuthException if unable to verify the password.
|
||||
bool CheckPassword(const std::string &password);
|
||||
|
||||
@@ -180,8 +138,6 @@ class User final {
|
||||
|
||||
const Role *role() const;
|
||||
|
||||
LabelPermissions &labelPermissions();
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
|
||||
/// @throw AuthException if unable to deserialize.
|
||||
@@ -194,9 +150,7 @@ class User final {
|
||||
std::string password_hash_;
|
||||
Permissions permissions_;
|
||||
std::optional<Role> role_;
|
||||
LabelPermissions labelPermissions_;
|
||||
};
|
||||
|
||||
bool operator==(const User &first, const User &second);
|
||||
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -18,24 +18,19 @@ roles_config = config["roles"]
|
||||
# Initialize LDAP server.
|
||||
tls = None
|
||||
if server_config["encryption"] != "disabled":
|
||||
cert_file = server_config["cert_file"] if server_config["cert_file"] else None
|
||||
cert_file = server_config["cert_file"] if server_config["cert_file"] \
|
||||
else None
|
||||
key_file = server_config["key_file"] if server_config["key_file"] else None
|
||||
ca_file = server_config["ca_file"] if server_config["ca_file"] else None
|
||||
validate = ssl.CERT_REQUIRED if server_config["validate_cert"] else ssl.CERT_NONE
|
||||
tls = ldap3.Tls(
|
||||
local_private_key_file=key_file,
|
||||
local_certificate_file=cert_file,
|
||||
ca_certs_file=ca_file,
|
||||
validate=validate,
|
||||
)
|
||||
validate = ssl.CERT_REQUIRED if server_config["validate_cert"] \
|
||||
else ssl.CERT_NONE
|
||||
tls = ldap3.Tls(local_private_key_file=key_file,
|
||||
local_certificate_file=cert_file,
|
||||
ca_certs_file=ca_file,
|
||||
validate=validate)
|
||||
use_ssl = server_config["encryption"] == "ssl"
|
||||
server = ldap3.Server(
|
||||
server_config["host"],
|
||||
port=server_config["port"],
|
||||
tls=tls,
|
||||
use_ssl=use_ssl,
|
||||
get_info=ldap3.ALL,
|
||||
)
|
||||
server = ldap3.Server(server_config["host"], port=server_config["port"],
|
||||
tls=tls, use_ssl=use_ssl, get_info=ldap3.ALL)
|
||||
|
||||
|
||||
# Main authentication/authorization function.
|
||||
@@ -45,12 +40,14 @@ def authenticate(username, password):
|
||||
return {"authenticated": False, "role": ""}
|
||||
|
||||
# Create the DN of the user
|
||||
dn = users_config["prefix"] + ldap3.utils.dn.escape_rdn(username) + users_config["suffix"]
|
||||
dn = users_config["prefix"] + ldap3.utils.dn.escape_rdn(username) + \
|
||||
users_config["suffix"]
|
||||
|
||||
# Bind to the server
|
||||
conn = ldap3.Connection(server, dn, password)
|
||||
if server_config["encryption"] == "starttls" and not conn.start_tls():
|
||||
print("ERROR: Couldn't issue STARTTLS to the LDAP server!", file=sys.stderr)
|
||||
print("ERROR: Couldn't issue STARTTLS to the LDAP server!",
|
||||
file=sys.stderr)
|
||||
return {"authenticated": False, "role": ""}
|
||||
if not conn.bind():
|
||||
return {"authenticated": False, "role": ""}
|
||||
@@ -59,32 +56,25 @@ def authenticate(username, password):
|
||||
if roles_config["root_dn"] != "":
|
||||
# search for role
|
||||
search_filter = "(&(objectclass={objclass})({attr}={value}))".format(
|
||||
objclass=roles_config["root_objectclass"],
|
||||
attr=roles_config["user_attribute"],
|
||||
value=ldap3.utils.conv.escape_filter_chars(dn),
|
||||
)
|
||||
succ = conn.search(
|
||||
roles_config["root_dn"],
|
||||
search_filter,
|
||||
search_scope=ldap3.LEVEL,
|
||||
attributes=[roles_config["role_attribute"]],
|
||||
)
|
||||
objclass=roles_config["root_objectclass"],
|
||||
attr=roles_config["user_attribute"],
|
||||
value=ldap3.utils.conv.escape_filter_chars(dn))
|
||||
succ = conn.search(roles_config["root_dn"], search_filter,
|
||||
search_scope=ldap3.LEVEL,
|
||||
attributes=[roles_config["role_attribute"]])
|
||||
if not succ or len(conn.entries) == 0:
|
||||
return {"authenticated": True, "role": ""}
|
||||
if len(conn.entries) > 1:
|
||||
roles = list(map(lambda x: x[roles_config["role_attribute"]].value, conn.entries))
|
||||
roles = list(map(lambda x: x[roles_config["role_attribute"]].value,
|
||||
conn.entries))
|
||||
# Because we don't know exactly which role the user should have
|
||||
# we authorize the user with an empty role.
|
||||
print(
|
||||
"WARNING: Found more than one role for " "user '" + username + "':",
|
||||
", ".join(roles) + "!",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print("WARNING: Found more than one role for "
|
||||
"user '" + username + "':", ", ".join(roles) + "!",
|
||||
file=sys.stderr)
|
||||
return {"authenticated": True, "role": ""}
|
||||
return {
|
||||
"authenticated": True,
|
||||
"role": conn.entries[0][roles_config["role_attribute"]].value,
|
||||
}
|
||||
return {"authenticated": True,
|
||||
"role": conn.entries[0][roles_config["role_attribute"]].value}
|
||||
else:
|
||||
return {"authenticated": True, "role": ""}
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
|
||||
namespace memgraph::communication::bolt {
|
||||
|
||||
inline constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
|
||||
inline constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
|
||||
static constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
|
||||
static constexpr uint8_t kProtocol[4] = {0x00, 0x00, 0x00, 0x01};
|
||||
|
||||
enum class Signature : uint8_t {
|
||||
Noop = 0x00,
|
||||
@@ -95,9 +95,9 @@ enum class Marker : uint8_t {
|
||||
Struct16 = 0xDD,
|
||||
};
|
||||
|
||||
inline constexpr uint8_t MarkerString = 0, MarkerList = 1, MarkerMap = 2;
|
||||
inline constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList, Marker::TinyMap};
|
||||
inline constexpr Marker Marker8[3] = {Marker::String8, Marker::List8, Marker::Map8};
|
||||
inline constexpr Marker Marker16[3] = {Marker::String16, Marker::List16, Marker::Map16};
|
||||
inline constexpr Marker Marker32[3] = {Marker::String32, Marker::List32, Marker::Map32};
|
||||
static constexpr uint8_t MarkerString = 0, MarkerList = 1, MarkerMap = 2;
|
||||
static constexpr Marker MarkerTiny[3] = {Marker::TinyString, Marker::TinyList, Marker::TinyMap};
|
||||
static constexpr Marker Marker8[3] = {Marker::String8, Marker::List8, Marker::Map8};
|
||||
static constexpr Marker Marker16[3] = {Marker::String16, Marker::List16, Marker::Map16};
|
||||
static constexpr Marker Marker32[3] = {Marker::String32, Marker::List32, Marker::Map32};
|
||||
} // namespace memgraph::communication::bolt
|
||||
|
||||
@@ -19,17 +19,17 @@ namespace memgraph::communication::bolt {
|
||||
/**
|
||||
* Sizes related to the chunk defined in Bolt protocol.
|
||||
*/
|
||||
inline constexpr size_t kChunkHeaderSize = 2;
|
||||
inline constexpr size_t kChunkMaxDataSize = 65535;
|
||||
inline constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
|
||||
static constexpr size_t kChunkHeaderSize = 2;
|
||||
static constexpr size_t kChunkMaxDataSize = 65535;
|
||||
static constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
|
||||
|
||||
/**
|
||||
* Handshake size defined in the Bolt protocol.
|
||||
*/
|
||||
inline constexpr size_t kHandshakeSize = 20;
|
||||
static constexpr size_t kHandshakeSize = 20;
|
||||
|
||||
inline constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
|
||||
static constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
|
||||
|
||||
inline constexpr int kPullAll = -1;
|
||||
inline constexpr int kPullLast = -1;
|
||||
static constexpr int kPullAll = -1;
|
||||
static constexpr int kPullLast = -1;
|
||||
} // namespace memgraph::communication::bolt
|
||||
|
||||
@@ -78,18 +78,14 @@ bool ClientContext::use_ssl() { return use_ssl_; }
|
||||
|
||||
ServerContext::ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file,
|
||||
bool verify_peer) {
|
||||
namespace ssl = boost::asio::ssl;
|
||||
ctx_.emplace(ssl::context::tls_server);
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
ctx_->set_options(ssl::context::default_workarounds | ssl::context::no_sslv2 | ssl::context::no_sslv3 |
|
||||
ssl::context::single_dh_use);
|
||||
ctx_.emplace(boost::asio::ssl::context::tls_server);
|
||||
ctx_->set_default_verify_paths();
|
||||
// TODO: add support for encrypted private keys
|
||||
// TODO: add certificate revocation list (CRL)
|
||||
boost::system::error_code ec;
|
||||
ctx_->use_certificate_chain_file(cert_file, ec);
|
||||
MG_ASSERT(!ec, "Couldn't load server certificate from file: {}", cert_file);
|
||||
ctx_->use_private_key_file(key_file, ssl::context::pem, ec);
|
||||
ctx_->use_private_key_file(key_file, boost::asio::ssl::context::pem, ec);
|
||||
MG_ASSERT(!ec, "Couldn't load server private key from file: {}", key_file);
|
||||
|
||||
ctx_->set_options(SSL_OP_NO_SSLv3, ec);
|
||||
@@ -104,7 +100,7 @@ ServerContext::ServerContext(const std::string &key_file, const std::string &cer
|
||||
if (verify_peer) {
|
||||
// Enable verification of the client certificate.
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
ctx_->set_verify_mode(ssl::verify_peer | ssl::verify_fail_if_no_peer_cert, ec);
|
||||
ctx_->set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert, ec);
|
||||
MG_ASSERT(!ec, "Setting SSL verification mode failed!");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string_view>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/strand.hpp>
|
||||
#include <boost/beast/core.hpp>
|
||||
#include <boost/system/detail/error_code.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/v2/pool.hpp"
|
||||
#include "communication/v2/session.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
template <class TSession, class TSessionData>
|
||||
class Listener final : public std::enable_shared_from_this<Listener<TSession, TSessionData>> {
|
||||
using tcp = boost::asio::ip::tcp;
|
||||
using SessionHandler = Session<TSession, TSessionData>;
|
||||
using std::enable_shared_from_this<Listener<TSession, TSessionData>>::shared_from_this;
|
||||
|
||||
public:
|
||||
Listener(const Listener &) = delete;
|
||||
Listener(Listener &&) = delete;
|
||||
Listener &operator=(const Listener &) = delete;
|
||||
Listener &operator=(Listener &&) = delete;
|
||||
~Listener() {}
|
||||
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Listener> Create(Args &&...args) {
|
||||
return std::shared_ptr<Listener>{new Listener(std::forward<Args>(args)...)};
|
||||
}
|
||||
|
||||
void Start() { DoAccept(); }
|
||||
|
||||
bool IsRunning() const noexcept { return alive_.load(std::memory_order_relaxed); }
|
||||
|
||||
private:
|
||||
Listener(boost::asio::io_context &io_context, TSessionData *data, ServerContext *server_context,
|
||||
tcp::endpoint &endpoint, const std::string_view service_name, const uint64_t inactivity_timeout_sec)
|
||||
: io_context_(io_context),
|
||||
data_(data),
|
||||
server_context_(server_context),
|
||||
acceptor_(io_context_),
|
||||
endpoint_{endpoint},
|
||||
service_name_{service_name},
|
||||
inactivity_timeout_{inactivity_timeout_sec} {
|
||||
boost::system::error_code ec;
|
||||
// Open the acceptor
|
||||
acceptor_.open(endpoint.protocol(), ec);
|
||||
if (ec) {
|
||||
OnError(ec, "open");
|
||||
return;
|
||||
}
|
||||
|
||||
// Allow address reuse
|
||||
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
|
||||
if (ec) {
|
||||
OnError(ec, "set_option");
|
||||
return;
|
||||
}
|
||||
|
||||
// Bind to the server address
|
||||
acceptor_.bind(endpoint, ec);
|
||||
if (ec) {
|
||||
spdlog::error(
|
||||
utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint, "https://memgr.ph/socket"));
|
||||
OnError(ec, "bind");
|
||||
return;
|
||||
}
|
||||
|
||||
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
|
||||
if (ec) {
|
||||
OnError(ec, "listen");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void DoAccept() {
|
||||
acceptor_.async_accept(io_context_,
|
||||
[shared_this = shared_from_this()](auto ec, boost::asio::ip::tcp::socket &&socket) {
|
||||
shared_this->OnAccept(ec, std::move(socket));
|
||||
});
|
||||
}
|
||||
|
||||
void OnAccept(boost::system::error_code ec, tcp::socket socket) {
|
||||
if (ec) {
|
||||
return OnError(ec, "accept");
|
||||
}
|
||||
|
||||
auto session = SessionHandler::Create(std::move(socket), data_, *server_context_, endpoint_, inactivity_timeout_,
|
||||
service_name_);
|
||||
session->Start();
|
||||
DoAccept();
|
||||
}
|
||||
|
||||
void OnError(const boost::system::error_code &ec, const std::string_view what) {
|
||||
spdlog::error("Listener failed on {}: {}", what, ec.message());
|
||||
alive_.store(false, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
boost::asio::io_context &io_context_;
|
||||
TSessionData *data_;
|
||||
ServerContext *server_context_;
|
||||
tcp::acceptor acceptor_;
|
||||
|
||||
tcp::endpoint endpoint_;
|
||||
std::string_view service_name_;
|
||||
std::chrono::seconds inactivity_timeout_;
|
||||
|
||||
std::atomic<bool> alive_;
|
||||
};
|
||||
} // namespace memgraph::communication::v2
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/asio/executor_work_guard.hpp>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
class IOContextThreadPool final {
|
||||
private:
|
||||
using IOContext = boost::asio::io_context;
|
||||
using IOContextGuard = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
|
||||
|
||||
public:
|
||||
explicit IOContextThreadPool(size_t pool_size) : guard_{io_context_.get_executor()}, pool_size_{pool_size} {
|
||||
MG_ASSERT(pool_size != 0, "Pool size must be greater then 0!");
|
||||
}
|
||||
|
||||
IOContextThreadPool(const IOContextThreadPool &) = delete;
|
||||
IOContextThreadPool &operator=(const IOContextThreadPool &) = delete;
|
||||
IOContextThreadPool(IOContextThreadPool &&) = delete;
|
||||
IOContextThreadPool &operator=(IOContextThreadPool &&) = delete;
|
||||
~IOContextThreadPool() = default;
|
||||
|
||||
void Run() {
|
||||
background_threads_.reserve(pool_size_);
|
||||
for (size_t i = 0; i < pool_size_; ++i) {
|
||||
background_threads_.emplace_back([this]() { io_context_.run(); });
|
||||
}
|
||||
running_ = true;
|
||||
}
|
||||
|
||||
void Shutdown() {
|
||||
io_context_.stop();
|
||||
running_ = false;
|
||||
}
|
||||
|
||||
void AwaitShutdown() { background_threads_.clear(); }
|
||||
|
||||
bool IsRunning() const noexcept { return running_; }
|
||||
|
||||
IOContext &GetIOContext() noexcept { return io_context_; }
|
||||
|
||||
private:
|
||||
/// The pool of io_context.
|
||||
IOContext io_context_;
|
||||
IOContextGuard guard_;
|
||||
size_t pool_size_;
|
||||
std::vector<std::jthread> background_threads_;
|
||||
bool running_{false};
|
||||
};
|
||||
} // namespace memgraph::communication::v2
|
||||
@@ -1,128 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/ip/address.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/init.hpp"
|
||||
#include "communication/v2/listener.hpp"
|
||||
#include "communication/v2/pool.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/thread.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
using Socket = boost::asio::ip::tcp::socket;
|
||||
using ServerEndpoint = boost::asio::ip::tcp::endpoint;
|
||||
/**
|
||||
* Communication server.
|
||||
*
|
||||
* Listens for incoming connections on the server port and assigns them to the
|
||||
* connection listener. The listener and session are implemented using asio
|
||||
* async model. Currently the implemented model is thread per core model
|
||||
* opposed to io_context per core. The reasoning for opting for the former model
|
||||
* is the robustness to the multiple resource demanding queries that can be split
|
||||
* across multiple threads, and then a single thread would not block io_context,
|
||||
* unlike in the latter model where it is possible that thread that accepts
|
||||
* request is being blocked by demanding query.
|
||||
* All logic is contained within handlers that are being dispatched
|
||||
* on a single strand per session. The only exception is write which is
|
||||
* synchronous since the nature of the clients conenction is synchronous as
|
||||
* well.
|
||||
*
|
||||
* Current Server architecture:
|
||||
* incoming connection -> server -> listener -> session
|
||||
|
||||
*
|
||||
* @tparam TSession the server can handle different Sessions, each session
|
||||
* represents a different protocol so the same network infrastructure
|
||||
* can be used for handling different protocols
|
||||
* @tparam TSessionData the class with objects that will be forwarded to the
|
||||
* session
|
||||
*/
|
||||
template <typename TSession, typename TSessionData>
|
||||
class Server final {
|
||||
using ServerHandler = Server<TSession, TSessionData>;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Constructs and binds server to endpoint, operates on session data and
|
||||
* invokes workers_count workers
|
||||
*/
|
||||
Server(ServerEndpoint &endpoint, TSessionData *session_data, ServerContext *server_context,
|
||||
const int inactivity_timeout_sec, const std::string_view service_name,
|
||||
size_t workers_count = std::thread::hardware_concurrency())
|
||||
: endpoint_{endpoint},
|
||||
service_name_{service_name},
|
||||
context_thread_pool_{workers_count},
|
||||
listener_{Listener<TSession, TSessionData>::Create(context_thread_pool_.GetIOContext(), session_data,
|
||||
server_context, endpoint_, service_name_,
|
||||
inactivity_timeout_sec)} {}
|
||||
|
||||
~Server() { MG_ASSERT(!IsRunning(), "Server wasn't shutdown properly"); }
|
||||
|
||||
Server(const Server &) = delete;
|
||||
Server(Server &&) = delete;
|
||||
Server &operator=(const Server &) = delete;
|
||||
Server &operator=(Server &&) = delete;
|
||||
|
||||
const auto &Endpoint() const {
|
||||
MG_ASSERT(IsRunning(), "You can't get the server endpoint when it's not running!");
|
||||
return endpoint_;
|
||||
}
|
||||
|
||||
bool Start() {
|
||||
if (IsRunning()) {
|
||||
spdlog::error("The server is already running");
|
||||
return false;
|
||||
}
|
||||
listener_->Start();
|
||||
|
||||
spdlog::info("{} server is fully armed and operational", service_name_);
|
||||
spdlog::info("{} listening on {}", service_name_, endpoint_.address());
|
||||
context_thread_pool_.Run();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Shutdown() {
|
||||
context_thread_pool_.Shutdown();
|
||||
spdlog::info("{} shutting down...", service_name_);
|
||||
}
|
||||
|
||||
void AwaitShutdown() { context_thread_pool_.AwaitShutdown(); }
|
||||
|
||||
bool IsRunning() const noexcept { return context_thread_pool_.IsRunning() && listener_->IsRunning(); }
|
||||
|
||||
private:
|
||||
ServerEndpoint endpoint_;
|
||||
std::string service_name_;
|
||||
|
||||
IOContextThreadPool context_thread_pool_;
|
||||
std::shared_ptr<Listener<TSession, TSessionData>> listener_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::communication::v2
|
||||
@@ -1,516 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <boost/asio/bind_executor.hpp>
|
||||
#include <boost/asio/buffer.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/read.hpp>
|
||||
#include <boost/asio/socket_base.hpp>
|
||||
#include <boost/asio/ssl/stream.hpp>
|
||||
#include <boost/asio/ssl/stream_base.hpp>
|
||||
#include <boost/asio/steady_timer.hpp>
|
||||
#include <boost/asio/strand.hpp>
|
||||
#include <boost/asio/system_context.hpp>
|
||||
#include <boost/asio/write.hpp>
|
||||
#include <boost/beast/core/tcp_stream.hpp>
|
||||
#include <boost/beast/http.hpp>
|
||||
#include <boost/beast/websocket.hpp>
|
||||
#include <boost/beast/websocket/rfc6455.hpp>
|
||||
#include <boost/system/detail/error_code.hpp>
|
||||
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
namespace memgraph::communication::v2 {
|
||||
|
||||
/**
|
||||
* This is used to provide input to user Sessions. All Sessions used with the
|
||||
* network stack should use this class as their input stream.
|
||||
*/
|
||||
using InputStream = communication::Buffer::ReadEnd;
|
||||
using tcp = boost::asio::ip::tcp;
|
||||
|
||||
/**
|
||||
* This is used to provide output from user Sessions. All Sessions used with the
|
||||
* network stack should use this class for their output stream.
|
||||
*/
|
||||
class OutputStream final {
|
||||
public:
|
||||
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
|
||||
: write_function_(write_function) {}
|
||||
|
||||
OutputStream(const OutputStream &) = delete;
|
||||
OutputStream(OutputStream &&) = delete;
|
||||
OutputStream &operator=(const OutputStream &) = delete;
|
||||
OutputStream &operator=(OutputStream &&) = delete;
|
||||
~OutputStream() = default;
|
||||
|
||||
bool Write(const uint8_t *data, size_t len, bool have_more = false) { return write_function_(data, len, have_more); }
|
||||
|
||||
bool Write(const std::string &str, bool have_more = false) {
|
||||
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(), have_more);
|
||||
}
|
||||
|
||||
private:
|
||||
std::function<bool(const uint8_t *, size_t, bool)> write_function_;
|
||||
};
|
||||
|
||||
/**
|
||||
* This class is used internally in the communication stack to handle all user
|
||||
* Websocket Sessions. It handles socket ownership, inactivity timeout and protocol
|
||||
* wrapping.
|
||||
*/
|
||||
template <typename TSession, typename TSessionData>
|
||||
class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TSession, TSessionData>> {
|
||||
using WebSocket = boost::beast::websocket::stream<boost::beast::tcp_stream>;
|
||||
using std::enable_shared_from_this<WebsocketSession<TSession, TSessionData>>::shared_from_this;
|
||||
|
||||
public:
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<WebsocketSession> Create(Args &&...args) {
|
||||
return std::shared_ptr<WebsocketSession>(new WebsocketSession(std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
// Start the asynchronous accept operation
|
||||
template <class Body, class Allocator>
|
||||
void DoAccept(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> req) {
|
||||
execution_active_ = true;
|
||||
// Set suggested timeout settings for the websocket
|
||||
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
|
||||
boost::asio::socket_base::keep_alive option(true);
|
||||
|
||||
// Set a decorator to change the Server of the handshake
|
||||
ws_.set_option(boost::beast::websocket::stream_base::decorator([&req](boost::beast::websocket::response_type &res) {
|
||||
res.set(boost::beast::http::field::server, std::string("Memgraph Bolt WS"));
|
||||
|
||||
// We need to do this to support WASM clients, which explicitly send this flag
|
||||
// in their upgrade request
|
||||
// Neo4j client breaks when this flag is sent
|
||||
if (const auto secondary_protocol = req.base().find(boost::beast::http::field::sec_websocket_protocol);
|
||||
secondary_protocol != res.base().end() && secondary_protocol->value() == "binary") {
|
||||
res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
|
||||
}
|
||||
}));
|
||||
ws_.binary(true);
|
||||
|
||||
// Accept the websocket handshake
|
||||
ws_.async_accept(
|
||||
req, boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnAccept, shared_from_this())));
|
||||
}
|
||||
|
||||
bool Write(const uint8_t *data, size_t len) {
|
||||
if (!IsConnected()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
boost::system::error_code ec;
|
||||
ws_.write(boost::asio::buffer(data, len), ec);
|
||||
if (ec) {
|
||||
OnError(ec, "write");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
// Take ownership of the socket
|
||||
explicit WebsocketSession(tcp::socket &&socket, TSessionData *data, tcp::endpoint endpoint,
|
||||
std::string_view service_name)
|
||||
: ws_(std::move(socket)),
|
||||
strand_{boost::asio::make_strand(ws_.get_executor())},
|
||||
output_stream_([this](const uint8_t *data, size_t len, bool /*have_more*/) { return Write(data, len); }),
|
||||
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
|
||||
endpoint_{endpoint},
|
||||
remote_endpoint_{ws_.next_layer().socket().remote_endpoint()},
|
||||
service_name_{service_name} {}
|
||||
|
||||
void OnAccept(boost::beast::error_code ec) {
|
||||
if (ec) {
|
||||
return OnError(ec, "accept");
|
||||
}
|
||||
|
||||
// Read a message
|
||||
DoRead();
|
||||
}
|
||||
|
||||
void DoRead() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
// Read a message into our buffer
|
||||
auto buffer = input_buffer_.write_end()->Allocate();
|
||||
ws_.async_read_some(
|
||||
boost::asio::buffer(buffer.data, buffer.len),
|
||||
boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnRead, shared_from_this())));
|
||||
}
|
||||
|
||||
void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
|
||||
// This indicates that the WebsocketSession was closed
|
||||
if (ec == boost::beast::websocket::error::closed) {
|
||||
return;
|
||||
}
|
||||
if (ec) {
|
||||
OnError(ec, "read");
|
||||
}
|
||||
input_buffer_.write_end()->Written(bytes_transferred);
|
||||
|
||||
try {
|
||||
session_.Execute();
|
||||
DoRead();
|
||||
} catch (const SessionClosedException &e) {
|
||||
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
DoClose();
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::error(
|
||||
"Exception was thrown while processing event in {} session "
|
||||
"associated with {}:{}",
|
||||
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
|
||||
spdlog::debug("Exception message: {}", e.what());
|
||||
DoClose();
|
||||
}
|
||||
}
|
||||
|
||||
void OnError(const boost::system::error_code &ec, const std::string_view action) {
|
||||
spdlog::error("Websocket Bolt session error: {} on {}", ec.message(), action);
|
||||
|
||||
DoClose();
|
||||
}
|
||||
|
||||
void DoClose() {
|
||||
ws_.async_close(
|
||||
boost::beast::websocket::close_code::normal,
|
||||
boost::asio::bind_executor(
|
||||
strand_, [shared_this = shared_from_this()](boost::beast::error_code ec) { shared_this->OnClose(ec); }));
|
||||
}
|
||||
|
||||
void OnClose(const boost::system::error_code &ec) {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
if (ec) {
|
||||
return OnError(ec, "close");
|
||||
}
|
||||
}
|
||||
|
||||
bool IsConnected() const { return ws_.is_open() && execution_active_; }
|
||||
|
||||
WebSocket ws_;
|
||||
boost::asio::strand<WebSocket::executor_type> strand_;
|
||||
|
||||
communication::Buffer input_buffer_;
|
||||
OutputStream output_stream_;
|
||||
TSession session_;
|
||||
tcp::endpoint endpoint_;
|
||||
tcp::endpoint remote_endpoint_;
|
||||
std::string_view service_name_;
|
||||
bool execution_active_{false};
|
||||
};
|
||||
|
||||
/**
|
||||
* This class is used internally in the communication stack to handle all user
|
||||
* Sessions. It handles socket ownership, inactivity timeout and protocol
|
||||
* wrapping.
|
||||
*/
|
||||
template <typename TSession, typename TSessionData>
|
||||
class Session final : public std::enable_shared_from_this<Session<TSession, TSessionData>> {
|
||||
using TCPSocket = tcp::socket;
|
||||
using SSLSocket = boost::asio::ssl::stream<TCPSocket>;
|
||||
using std::enable_shared_from_this<Session<TSession, TSessionData>>::shared_from_this;
|
||||
|
||||
public:
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Session> Create(Args &&...args) {
|
||||
return std::shared_ptr<Session>(new Session(std::forward<Args>(args)...));
|
||||
}
|
||||
|
||||
Session(const Session &) = delete;
|
||||
Session(Session &&) = delete;
|
||||
Session &operator=(const Session &) = delete;
|
||||
Session &operator=(Session &&) = delete;
|
||||
~Session() = default;
|
||||
|
||||
bool Start() {
|
||||
if (execution_active_) {
|
||||
return false;
|
||||
}
|
||||
execution_active_ = true;
|
||||
timeout_timer_.async_wait(boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
|
||||
|
||||
if (std::holds_alternative<SSLSocket>(socket_)) {
|
||||
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoHandshake(); });
|
||||
} else {
|
||||
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Write(const uint8_t *data, size_t len, bool have_more = false) {
|
||||
if (!IsConnected()) {
|
||||
return false;
|
||||
}
|
||||
return std::visit(
|
||||
utils::Overloaded{[shared_this = shared_from_this(), data, len, have_more](TCPSocket &socket) mutable {
|
||||
boost::system::error_code ec;
|
||||
while (len > 0) {
|
||||
const auto sent = socket.send(boost::asio::buffer(data, len),
|
||||
MSG_NOSIGNAL | (have_more ? MSG_MORE : 0), ec);
|
||||
if (ec) {
|
||||
shared_this->OnError(ec);
|
||||
return false;
|
||||
}
|
||||
data += sent;
|
||||
len -= sent;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
[shared_this = shared_from_this(), data, len](SSLSocket &socket) mutable {
|
||||
boost::system::error_code ec;
|
||||
while (len > 0) {
|
||||
const auto sent = socket.write_some(boost::asio::buffer(data, len), ec);
|
||||
if (ec) {
|
||||
shared_this->OnError(ec);
|
||||
return false;
|
||||
}
|
||||
data += sent;
|
||||
len -= sent;
|
||||
}
|
||||
return true;
|
||||
}},
|
||||
socket_);
|
||||
}
|
||||
|
||||
bool IsConnected() const {
|
||||
return std::visit([this](const auto &socket) { return execution_active_ && socket.lowest_layer().is_open(); },
|
||||
socket_);
|
||||
}
|
||||
|
||||
private:
|
||||
explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &server_context, tcp::endpoint endpoint,
|
||||
const std::chrono::seconds inactivity_timeout_sec, std::string_view service_name)
|
||||
: socket_(CreateSocket(std::move(socket), server_context)),
|
||||
strand_{boost::asio::make_strand(GetExecutor())},
|
||||
output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
|
||||
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
|
||||
data_{data},
|
||||
endpoint_{endpoint},
|
||||
remote_endpoint_{GetRemoteEndpoint()},
|
||||
service_name_{service_name},
|
||||
timeout_seconds_(inactivity_timeout_sec),
|
||||
timeout_timer_(GetExecutor()) {
|
||||
ExecuteForSocket([](auto &&socket) {
|
||||
socket.lowest_layer().set_option(tcp::no_delay(true)); // enable PSH
|
||||
socket.lowest_layer().set_option(boost::asio::socket_base::keep_alive(true)); // enable SO_KEEPALIVE
|
||||
socket.lowest_layer().non_blocking(false);
|
||||
});
|
||||
timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
|
||||
spdlog::info("Accepted a connection from {}: {}:{}", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
}
|
||||
|
||||
void DoRead() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
timeout_timer_.expires_after(timeout_seconds_);
|
||||
ExecuteForSocket([this](auto &&socket) {
|
||||
auto buffer = input_buffer_.write_end()->Allocate();
|
||||
socket.async_read_some(
|
||||
boost::asio::buffer(buffer.data, buffer.len),
|
||||
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnRead, shared_from_this())));
|
||||
});
|
||||
}
|
||||
|
||||
bool IsWebsocketUpgrade(boost::beast::http::request_parser<boost::beast::http::string_body> &parser) {
|
||||
boost::system::error_code error_code_parsing;
|
||||
parser.put(boost::asio::buffer(input_buffer_.read_end()->data(), input_buffer_.read_end()->size()),
|
||||
error_code_parsing);
|
||||
if (error_code_parsing) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return boost::beast::websocket::is_upgrade(parser.get());
|
||||
}
|
||||
|
||||
void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
|
||||
if (ec) {
|
||||
return OnError(ec);
|
||||
}
|
||||
input_buffer_.write_end()->Written(bytes_transferred);
|
||||
|
||||
// Can be a websocket connection only on the first read, since it is not
|
||||
// expected from clients to upgrade from tcp to websocket
|
||||
if (!has_received_msg_) {
|
||||
has_received_msg_ = true;
|
||||
boost::beast::http::request_parser<boost::beast::http::string_body> parser;
|
||||
|
||||
if (IsWebsocketUpgrade(parser)) {
|
||||
spdlog::info("Switching {} to websocket connection", remote_endpoint_);
|
||||
if (std::holds_alternative<TCPSocket>(socket_)) {
|
||||
auto sock = std::get<TCPSocket>(std::move(socket_));
|
||||
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), data_, endpoint_, service_name_)
|
||||
->DoAccept(parser.release());
|
||||
execution_active_ = false;
|
||||
return;
|
||||
}
|
||||
spdlog::error("Error while upgrading connection to websocket");
|
||||
DoShutdown();
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
session_.Execute();
|
||||
DoRead();
|
||||
} catch (const SessionClosedException &e) {
|
||||
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
|
||||
remote_endpoint_.port());
|
||||
DoShutdown();
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::error(
|
||||
"Exception was thrown while processing event in {} session "
|
||||
"associated with {}:{}",
|
||||
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
|
||||
spdlog::debug("Exception message: {}", e.what());
|
||||
DoShutdown();
|
||||
}
|
||||
}
|
||||
|
||||
void OnError(const boost::system::error_code &ec) {
|
||||
if (ec == boost::asio::error::operation_aborted) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (ec == boost::asio::error::eof) {
|
||||
spdlog::info("Session closed by peer");
|
||||
} else {
|
||||
spdlog::error("Session error: {}", ec.message());
|
||||
}
|
||||
|
||||
DoShutdown();
|
||||
}
|
||||
|
||||
void DoShutdown() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
execution_active_ = false;
|
||||
timeout_timer_.cancel();
|
||||
ExecuteForSocket([](auto &socket) {
|
||||
boost::system::error_code ec;
|
||||
auto &lowest_layer = socket.lowest_layer();
|
||||
lowest_layer.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
|
||||
if (ec) {
|
||||
spdlog::error("Session shutdown failed: {}", ec.what());
|
||||
}
|
||||
lowest_layer.close();
|
||||
});
|
||||
}
|
||||
|
||||
void DoHandshake() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
if (auto *socket = std::get_if<SSLSocket>(&socket_); socket) {
|
||||
socket->async_handshake(
|
||||
boost::asio::ssl::stream_base::server,
|
||||
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnHandshake, shared_from_this())));
|
||||
}
|
||||
}
|
||||
|
||||
void OnHandshake(const boost::system::error_code &ec) {
|
||||
if (ec) {
|
||||
return OnError(ec);
|
||||
}
|
||||
DoRead();
|
||||
}
|
||||
|
||||
void OnClose(const boost::system::error_code &ec) {
|
||||
if (ec) {
|
||||
return OnError(ec);
|
||||
}
|
||||
}
|
||||
|
||||
void OnTimeout() {
|
||||
if (!IsConnected()) {
|
||||
return;
|
||||
}
|
||||
// Check whether the deadline has passed. We compare the deadline against
|
||||
// the current time since a new asynchronous operation may have moved the
|
||||
// deadline before this actor had a chance to run.
|
||||
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
|
||||
// The deadline has passed. Stop the session. The other actors will
|
||||
// terminate as soon as possible.
|
||||
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
|
||||
DoShutdown();
|
||||
} else {
|
||||
// Put the actor back to sleep.
|
||||
timeout_timer_.async_wait(
|
||||
boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
|
||||
}
|
||||
}
|
||||
|
||||
std::variant<TCPSocket, SSLSocket> CreateSocket(tcp::socket &&socket, ServerContext &context) {
|
||||
if (context.use_ssl()) {
|
||||
ssl_context_.emplace(context.context_clone());
|
||||
return SSLSocket{std::move(socket), *ssl_context_};
|
||||
}
|
||||
|
||||
return TCPSocket{std::move(socket)};
|
||||
}
|
||||
|
||||
auto GetExecutor() {
|
||||
return std::visit(utils::Overloaded{[](auto &&socket) { return socket.get_executor(); }}, socket_);
|
||||
}
|
||||
|
||||
auto GetRemoteEndpoint() const {
|
||||
return std::visit(utils::Overloaded{[](const auto &socket) { return socket.lowest_layer().remote_endpoint(); }},
|
||||
socket_);
|
||||
}
|
||||
|
||||
template <typename F>
|
||||
decltype(auto) ExecuteForSocket(F &&fun) {
|
||||
return std::visit(utils::Overloaded{std::forward<F>(fun)}, socket_);
|
||||
}
|
||||
|
||||
std::variant<TCPSocket, SSLSocket> socket_;
|
||||
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ssl_context_;
|
||||
boost::asio::strand<tcp::socket::executor_type> strand_;
|
||||
|
||||
communication::Buffer input_buffer_;
|
||||
OutputStream output_stream_;
|
||||
TSession session_;
|
||||
TSessionData *data_;
|
||||
tcp::endpoint endpoint_;
|
||||
tcp::endpoint remote_endpoint_;
|
||||
std::string_view service_name_;
|
||||
std::chrono::seconds timeout_seconds_;
|
||||
boost::asio::steady_timer timeout_timer_;
|
||||
bool execution_active_{false};
|
||||
bool has_received_msg_{false};
|
||||
};
|
||||
} // namespace memgraph::communication::v2
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
|
||||
|
||||
#include <list>
|
||||
#include <memory>
|
||||
|
||||
|
||||
@@ -44,10 +44,10 @@ class QuoteEscapeFormatter : public spdlog::custom_flag_formatter {
|
||||
void format(const spdlog::details::log_msg &msg, const std::tm & /*time*/, spdlog::memory_buf_t &dest) override {
|
||||
for (const auto c : msg.payload) {
|
||||
if (c == '"') {
|
||||
static constexpr std::string_view escaped_quote = "\\\"";
|
||||
constexpr std::string_view escaped_quote = "\\\"";
|
||||
dest.append(escaped_quote.data(), escaped_quote.data() + escaped_quote.size());
|
||||
} else if (c == '\n') {
|
||||
static constexpr std::string_view escaped_newline = "\\n";
|
||||
constexpr std::string_view escaped_newline = "\\n";
|
||||
dest.append(escaped_newline.data(), escaped_newline.data() + escaped_newline.size());
|
||||
} else {
|
||||
dest.push_back(c);
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include <spdlog/sinks/base_sink.h>
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
|
||||
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
@@ -57,8 +57,6 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
|
||||
return auth::Permission::MODULE_WRITE;
|
||||
case query::AuthQuery::Privilege::WEBSOCKET:
|
||||
return auth::Permission::WEBSOCKET;
|
||||
case query::AuthQuery::Privilege::LABELS:
|
||||
return auth::Permission::LABELS;
|
||||
}
|
||||
}
|
||||
} // namespace memgraph::glue
|
||||
|
||||
@@ -16,11 +16,10 @@
|
||||
|
||||
namespace memgraph::integrations {
|
||||
|
||||
inline constexpr int64_t kDefaultCheckBatchLimit{1};
|
||||
inline constexpr int64_t kMinimumStartBatchLimit{1};
|
||||
inline constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
|
||||
inline constexpr std::chrono::milliseconds kMinimumInterval{1};
|
||||
inline constexpr int64_t kMinimumSize{1};
|
||||
constexpr int64_t kDefaultCheckBatchLimit{1};
|
||||
constexpr std::chrono::milliseconds kDefaultCheckTimeout{30000};
|
||||
constexpr std::chrono::milliseconds kMinimumInterval{1};
|
||||
constexpr int64_t kMinimumSize{1};
|
||||
const std::string kReducted{"<REDUCTED>"};
|
||||
|
||||
} // namespace memgraph::integrations
|
||||
|
||||
@@ -74,36 +74,6 @@ utils::BasicResult<std::string, std::vector<Message>> GetBatch(RdKafka::KafkaCon
|
||||
|
||||
return std::move(batch);
|
||||
}
|
||||
|
||||
void CheckAndDestroyLastAssignmentIfNeeded(RdKafka::KafkaConsumer &consumer, const ConsumerInfo &info,
|
||||
std::vector<RdKafka::TopicPartition *> &last_assignment) {
|
||||
if (!last_assignment.empty()) {
|
||||
if (const auto err = consumer.assign(last_assignment); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerStartFailedException(info.consumer_name,
|
||||
fmt::format("Couldn't restore commited offsets: '{}'", RdKafka::err2str(err)));
|
||||
}
|
||||
RdKafka::TopicPartition::destroy(last_assignment);
|
||||
}
|
||||
}
|
||||
|
||||
void TryToConsumeBatch(RdKafka::KafkaConsumer &consumer, const ConsumerInfo &info,
|
||||
const ConsumerFunction &consumer_function, const std::vector<Message> &batch) {
|
||||
consumer_function(batch);
|
||||
std::vector<RdKafka::TopicPartition *> partitions;
|
||||
utils::OnScopeExit clear_partitions([&]() { RdKafka::TopicPartition::destroy(partitions); });
|
||||
|
||||
if (const auto err = consumer.assignment(partitions); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerCommitFailedException(
|
||||
info.consumer_name, fmt::format("Couldn't get assignment to commit offsets: {}", RdKafka::err2str(err)));
|
||||
}
|
||||
if (const auto err = consumer.position(partitions); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerCommitFailedException(info.consumer_name,
|
||||
fmt::format("Couldn't get offsets from librdkafka {}", RdKafka::err2str(err)));
|
||||
}
|
||||
if (const auto err = consumer.commitSync(partitions); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerCommitFailedException(info.consumer_name, RdKafka::err2str(err));
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Message::Message(std::unique_ptr<RdKafka::Message> &&message) : message_{std::move(message)} {
|
||||
@@ -215,10 +185,8 @@ Consumer::Consumer(ConsumerInfo info, ConsumerFunction consumer_function)
|
||||
std::inserter(topic_names_from_metadata, topic_names_from_metadata.begin()),
|
||||
[](const auto topic_metadata) { return topic_metadata->topic(); });
|
||||
|
||||
static constexpr size_t max_topic_name_length = 249;
|
||||
static constexpr auto is_valid_topic_name = [](const auto c) {
|
||||
return std::isalnum(c) || c == '.' || c == '_' || c == '-';
|
||||
};
|
||||
constexpr size_t max_topic_name_length = 249;
|
||||
constexpr auto is_valid_topic_name = [](const auto c) { return std::isalnum(c) || c == '.' || c == '_' || c == '-'; };
|
||||
|
||||
for (const auto &topic_name : info_.topics) {
|
||||
if (topic_name.size() > max_topic_name_length ||
|
||||
@@ -251,21 +219,10 @@ void Consumer::Start() {
|
||||
StartConsuming();
|
||||
}
|
||||
|
||||
void Consumer::StartWithLimit(const uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
|
||||
if (is_running_) {
|
||||
throw ConsumerRunningException(info_.consumer_name);
|
||||
void Consumer::StartIfStopped() {
|
||||
if (!is_running_) {
|
||||
StartConsuming();
|
||||
}
|
||||
if (limit_batches < kMinimumStartBatchLimit) {
|
||||
throw ConsumerStartFailedException(
|
||||
info_.consumer_name, fmt::format("Batch limit has to be greater than or equal to {}", kMinimumStartBatchLimit));
|
||||
}
|
||||
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
|
||||
throw ConsumerStartFailedException(
|
||||
info_.consumer_name,
|
||||
fmt::format("Timeout has to be greater than or equal to {} milliseconds", kMinimumInterval.count()));
|
||||
}
|
||||
|
||||
StartConsumingWithLimit(limit_batches, timeout);
|
||||
}
|
||||
|
||||
void Consumer::Stop() {
|
||||
@@ -285,7 +242,7 @@ void Consumer::StopIfRunning() {
|
||||
}
|
||||
}
|
||||
|
||||
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
|
||||
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
|
||||
const ConsumerFunction &check_consumer_function) const {
|
||||
// NOLINTNEXTLINE (modernize-use-nullptr)
|
||||
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
|
||||
@@ -385,10 +342,16 @@ void Consumer::StartConsuming() {
|
||||
|
||||
is_running_.store(true);
|
||||
|
||||
CheckAndDestroyLastAssignmentIfNeeded(*consumer_, info_, last_assignment_);
|
||||
if (!last_assignment_.empty()) {
|
||||
if (const auto err = consumer_->assign(last_assignment_); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerStartFailedException(info_.consumer_name,
|
||||
fmt::format("Couldn't restore commited offsets: '{}'", RdKafka::err2str(err)));
|
||||
}
|
||||
RdKafka::TopicPartition::destroy(last_assignment_);
|
||||
}
|
||||
|
||||
thread_ = std::thread([this] {
|
||||
static constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
|
||||
constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
|
||||
const auto full_thread_name = "Cons#" + info_.consumer_name;
|
||||
|
||||
utils::ThreadSetName(full_thread_name.substr(0, kMaxThreadNameSize));
|
||||
@@ -396,18 +359,33 @@ void Consumer::StartConsuming() {
|
||||
while (is_running_) {
|
||||
auto maybe_batch = GetBatch(*consumer_, info_, is_running_);
|
||||
if (maybe_batch.HasError()) {
|
||||
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
|
||||
spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
|
||||
maybe_batch.GetError());
|
||||
break;
|
||||
}
|
||||
const auto &batch = maybe_batch.GetValue();
|
||||
|
||||
if (batch.empty()) {
|
||||
continue;
|
||||
}
|
||||
if (batch.empty()) continue;
|
||||
|
||||
spdlog::info("Kafka consumer {} is processing a batch", info_.consumer_name);
|
||||
|
||||
try {
|
||||
TryToConsumeBatch(*consumer_, info_, consumer_function_, batch);
|
||||
consumer_function_(batch);
|
||||
std::vector<RdKafka::TopicPartition *> partitions;
|
||||
utils::OnScopeExit clear_partitions([&]() { RdKafka::TopicPartition::destroy(partitions); });
|
||||
|
||||
if (const auto err = consumer_->assignment(partitions); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerCheckFailedException(
|
||||
info_.consumer_name, fmt::format("Couldn't get assignment to commit offsets: {}", RdKafka::err2str(err)));
|
||||
}
|
||||
if (const auto err = consumer_->position(partitions); err != RdKafka::ERR_NO_ERROR) {
|
||||
throw ConsumerCheckFailedException(
|
||||
info_.consumer_name, fmt::format("Couldn't get offsets from librdkafka {}", RdKafka::err2str(err)));
|
||||
}
|
||||
if (const auto err = consumer_->commitSync(partitions); err != RdKafka::ERR_NO_ERROR) {
|
||||
spdlog::warn("Committing offset of consumer {} failed: {}", info_.consumer_name, RdKafka::err2str(err));
|
||||
break;
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
|
||||
break;
|
||||
@@ -418,44 +396,6 @@ void Consumer::StartConsuming() {
|
||||
});
|
||||
}
|
||||
|
||||
void Consumer::StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
|
||||
MG_ASSERT(!is_running_, "Cannot start already running consumer!");
|
||||
|
||||
if (is_running_.exchange(true)) {
|
||||
throw ConsumerRunningException(info_.consumer_name);
|
||||
}
|
||||
utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
|
||||
|
||||
CheckAndDestroyLastAssignmentIfNeeded(*consumer_, info_, last_assignment_);
|
||||
|
||||
const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
|
||||
for (uint64_t batch_count = 0; batch_count < limit_batches;) {
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
if (now - start >= timeout_to_use) {
|
||||
throw ConsumerStartFailedException(info_.consumer_name, "Timeout reached");
|
||||
}
|
||||
|
||||
const auto maybe_batch = GetBatch(*consumer_, info_, is_running_);
|
||||
if (maybe_batch.HasError()) {
|
||||
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
|
||||
}
|
||||
const auto &batch = maybe_batch.GetValue();
|
||||
|
||||
if (batch.empty()) {
|
||||
continue;
|
||||
}
|
||||
++batch_count;
|
||||
|
||||
spdlog::info("Kafka consumer {} is processing a batch", info_.consumer_name);
|
||||
|
||||
TryToConsumeBatch(*consumer_, info_, consumer_function_, batch);
|
||||
|
||||
spdlog::info("Kafka consumer {} finished processing", info_.consumer_name);
|
||||
}
|
||||
}
|
||||
|
||||
void Consumer::StopConsuming() {
|
||||
is_running_.store(false);
|
||||
if (thread_.joinable()) thread_.join();
|
||||
|
||||
@@ -113,19 +113,11 @@ class Consumer final : public RdKafka::EventCb {
|
||||
/// This method will start a new thread which will poll all the topics for messages.
|
||||
///
|
||||
/// @throws ConsumerRunningException if the consumer is already running
|
||||
/// @throws ConsumerStartFailedException if the commited offsets cannot be restored
|
||||
void Start();
|
||||
|
||||
/// Starts consuming messages.
|
||||
/// Starts consuming messages if it is not started already.
|
||||
///
|
||||
/// This method will start a new thread which will poll all the topics for messages.
|
||||
///
|
||||
/// @param limit_batches the consumer will only consume the given number of batches.
|
||||
/// @param timeout the maximum duration during which the command should run.
|
||||
///
|
||||
/// @throws ConsumerRunningException if the consumer is already running
|
||||
/// @throws ConsumerStartFailedException if the commited offsets cannot be restored
|
||||
void StartWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
|
||||
void StartIfStopped();
|
||||
|
||||
/// Stops consuming messages.
|
||||
///
|
||||
@@ -144,9 +136,9 @@ class Consumer final : public RdKafka::EventCb {
|
||||
/// used.
|
||||
/// @param check_consumer_function a function to feed the received messages in, only used during this dry-run.
|
||||
///
|
||||
/// @throws ConsumerRunningException if the consumer is already running.
|
||||
/// @throws ConsumerRunningException if the consumer is alredy running.
|
||||
/// @throws ConsumerCheckFailedException if check isn't successful.
|
||||
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
|
||||
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
|
||||
const ConsumerFunction &check_consumer_function) const;
|
||||
|
||||
/// Returns true if the consumer is actively consuming messages.
|
||||
@@ -165,7 +157,6 @@ class Consumer final : public RdKafka::EventCb {
|
||||
void event_cb(RdKafka::Event &event) override;
|
||||
|
||||
void StartConsuming();
|
||||
void StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
|
||||
|
||||
void StopConsuming();
|
||||
|
||||
@@ -187,6 +178,7 @@ class Consumer final : public RdKafka::EventCb {
|
||||
ConsumerFunction consumer_function_;
|
||||
mutable std::atomic<bool> is_running_{false};
|
||||
mutable std::vector<RdKafka::TopicPartition *> last_assignment_; // Protected by is_running_
|
||||
std::optional<int64_t> limit_batches_{std::nullopt};
|
||||
std::unique_ptr<RdKafka::KafkaConsumer, std::function<void(RdKafka::KafkaConsumer *)>> consumer_;
|
||||
std::thread thread_;
|
||||
ConsumerRebalanceCb cb_;
|
||||
|
||||
@@ -64,16 +64,4 @@ class TopicNotFoundException : public KafkaStreamException {
|
||||
TopicNotFoundException(const std::string_view consumer_name, const std::string_view topic_name)
|
||||
: KafkaStreamException("Kafka consumer {} cannot find topic {}", consumer_name, topic_name) {}
|
||||
};
|
||||
|
||||
class ConsumerCommitFailedException : public KafkaStreamException {
|
||||
public:
|
||||
ConsumerCommitFailedException(const std::string_view consumer_name, const std::string_view error)
|
||||
: KafkaStreamException("Committing offset of consumer {} failed: {}", consumer_name, error) {}
|
||||
};
|
||||
|
||||
class ConsumerReadMessagesFailedException : public KafkaStreamException {
|
||||
public:
|
||||
ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
|
||||
: KafkaStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
|
||||
};
|
||||
} // namespace memgraph::integrations::kafka
|
||||
|
||||
@@ -11,14 +11,13 @@
|
||||
|
||||
#include "integrations/pulsar/consumer.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <pulsar/Client.h>
|
||||
#include <pulsar/InitialPosition.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
|
||||
#include "integrations/constants.hpp"
|
||||
#include "integrations/pulsar/exceptions.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
@@ -34,10 +33,6 @@ namespace {
|
||||
template <typename T>
|
||||
concept PulsarConsumer = utils::SameAsAnyOf<T, pulsar_client::Consumer, pulsar_client::Reader>;
|
||||
|
||||
template <typename TFunc>
|
||||
concept PulsarMessageGetter =
|
||||
std::same_as<const pulsar_client::Message &, std::invoke_result_t<TFunc, const Message &>>;
|
||||
|
||||
pulsar_client::Result ConsumeMessage(pulsar_client::Consumer &consumer, pulsar_client::Message &message,
|
||||
int remaining_timeout_in_ms) {
|
||||
return consumer.receive(message, remaining_timeout_in_ms);
|
||||
@@ -102,26 +97,6 @@ pulsar_client::Client CreateClient(const std::string &service_url) {
|
||||
conf.setLogger(new SpdlogLoggerFactory);
|
||||
return {service_url, conf};
|
||||
}
|
||||
|
||||
template <PulsarConsumer TConsumer, PulsarMessageGetter TPulsarMessageGetter>
|
||||
void TryToConsumeBatch(TConsumer &consumer, const ConsumerInfo &info, const ConsumerFunction &consumer_function,
|
||||
pulsar_client::MessageId &last_message_id, const std::vector<Message> &batch,
|
||||
const TPulsarMessageGetter &message_getter) {
|
||||
consumer_function(batch);
|
||||
|
||||
auto has_message_failed = [&consumer, &info, &last_message_id, &message_getter](const auto &message) {
|
||||
if (const auto result = consumer.acknowledge(message_getter(message)); result != pulsar_client::ResultOk) {
|
||||
spdlog::warn("Acknowledging a message of consumer {} failed: {}", info.consumer_name, result);
|
||||
return true;
|
||||
}
|
||||
last_message_id = message_getter(message).getMessageId();
|
||||
return false;
|
||||
};
|
||||
|
||||
if (std::ranges::any_of(batch, has_message_failed)) {
|
||||
throw ConsumerAcknowledgeMessagesFailedException(info.consumer_name);
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Message::Message(pulsar_client::Message &&message) : message_{std::move(message)} {}
|
||||
@@ -162,24 +137,6 @@ void Consumer::Start() {
|
||||
StartConsuming();
|
||||
}
|
||||
|
||||
void Consumer::StartWithLimit(const uint64_t limit_batches,
|
||||
const std::optional<std::chrono::milliseconds> timeout) const {
|
||||
if (is_running_) {
|
||||
throw ConsumerRunningException(info_.consumer_name);
|
||||
}
|
||||
if (limit_batches < kMinimumStartBatchLimit) {
|
||||
throw ConsumerStartFailedException(
|
||||
info_.consumer_name, fmt::format("Batch limit has to be greater than or equal to {}", kMinimumStartBatchLimit));
|
||||
}
|
||||
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
|
||||
throw ConsumerStartFailedException(
|
||||
info_.consumer_name,
|
||||
fmt::format("Timeout has to be greater than or equal to {} milliseconds", kMinimumInterval.count()));
|
||||
}
|
||||
|
||||
StartConsumingWithLimit(limit_batches, timeout);
|
||||
}
|
||||
|
||||
void Consumer::Stop() {
|
||||
if (!is_running_) {
|
||||
throw ConsumerStoppedException(info_.consumer_name);
|
||||
@@ -197,7 +154,7 @@ void Consumer::StopIfRunning() {
|
||||
}
|
||||
}
|
||||
|
||||
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
|
||||
void Consumer::Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
|
||||
const ConsumerFunction &check_consumer_function) const {
|
||||
// NOLINTNEXTLINE (modernize-use-nullptr)
|
||||
if (timeout.value_or(kMinimumInterval) < kMinimumInterval) {
|
||||
@@ -274,7 +231,7 @@ void Consumer::StartConsuming() {
|
||||
is_running_.store(true);
|
||||
|
||||
thread_ = std::thread([this] {
|
||||
static constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
|
||||
constexpr auto kMaxThreadNameSize = utils::GetMaxThreadNameSize();
|
||||
const auto full_thread_name = "Cons#" + info_.consumer_name;
|
||||
|
||||
utils::ThreadSetName(full_thread_name.substr(0, kMaxThreadNameSize));
|
||||
@@ -283,7 +240,9 @@ void Consumer::StartConsuming() {
|
||||
auto maybe_batch = GetBatch(consumer_, info_, is_running_, last_message_id_);
|
||||
|
||||
if (maybe_batch.HasError()) {
|
||||
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
|
||||
spdlog::warn("Error happened in consumer {} while fetching messages: {}!", info_.consumer_name,
|
||||
maybe_batch.GetError());
|
||||
break;
|
||||
}
|
||||
|
||||
const auto &batch = maybe_batch.GetValue();
|
||||
@@ -295,8 +254,18 @@ void Consumer::StartConsuming() {
|
||||
spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
|
||||
|
||||
try {
|
||||
TryToConsumeBatch(consumer_, info_, consumer_function_, last_message_id_, batch,
|
||||
[&](const Message &message) -> const pulsar_client::Message & { return message.message_; });
|
||||
consumer_function_(batch);
|
||||
|
||||
if (std::any_of(batch.begin(), batch.end(), [&](const auto &message) {
|
||||
if (const auto result = consumer_.acknowledge(message.message_); result != pulsar_client::ResultOk) {
|
||||
spdlog::warn("Acknowledging a message of consumer {} failed: {}", info_.consumer_name, result);
|
||||
return true;
|
||||
}
|
||||
last_message_id_ = message.message_.getMessageId();
|
||||
return false;
|
||||
})) {
|
||||
break;
|
||||
}
|
||||
} catch (const std::exception &e) {
|
||||
spdlog::warn("Error happened in consumer {} while processing a batch: {}!", info_.consumer_name, e.what());
|
||||
break;
|
||||
@@ -308,43 +277,6 @@ void Consumer::StartConsuming() {
|
||||
});
|
||||
}
|
||||
|
||||
void Consumer::StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const {
|
||||
if (is_running_.exchange(true)) {
|
||||
throw ConsumerRunningException(info_.consumer_name);
|
||||
}
|
||||
utils::OnScopeExit restore_is_running([this] { is_running_.store(false); });
|
||||
|
||||
const auto timeout_to_use = timeout.value_or(kDefaultCheckTimeout);
|
||||
const auto start = std::chrono::steady_clock::now();
|
||||
|
||||
for (uint64_t batch_count = 0; batch_count < limit_batches;) {
|
||||
const auto now = std::chrono::steady_clock::now();
|
||||
if (now - start >= timeout_to_use) {
|
||||
throw ConsumerCheckFailedException(info_.consumer_name, "Timeout reached");
|
||||
}
|
||||
|
||||
const auto maybe_batch = GetBatch(consumer_, info_, is_running_, last_message_id_);
|
||||
|
||||
if (maybe_batch.HasError()) {
|
||||
throw ConsumerReadMessagesFailedException(info_.consumer_name, maybe_batch.GetError());
|
||||
}
|
||||
|
||||
const auto &batch = maybe_batch.GetValue();
|
||||
|
||||
if (batch.empty()) {
|
||||
continue;
|
||||
}
|
||||
++batch_count;
|
||||
|
||||
spdlog::info("Pulsar consumer {} is processing a batch", info_.consumer_name);
|
||||
|
||||
TryToConsumeBatch(consumer_, info_, consumer_function_, last_message_id_, batch,
|
||||
[](const Message &message) -> const pulsar_client::Message & { return message.message_; });
|
||||
|
||||
spdlog::info("Pulsar consumer {} finished processing", info_.consumer_name);
|
||||
}
|
||||
}
|
||||
|
||||
void Consumer::StopConsuming() {
|
||||
is_running_.store(false);
|
||||
if (thread_.joinable()) {
|
||||
|
||||
@@ -58,27 +58,25 @@ class Consumer final {
|
||||
|
||||
bool IsRunning() const;
|
||||
void Start();
|
||||
void StartWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
|
||||
void Stop();
|
||||
void StopIfRunning();
|
||||
|
||||
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<uint64_t> limit_batches,
|
||||
void Check(std::optional<std::chrono::milliseconds> timeout, std::optional<int64_t> limit_batches,
|
||||
const ConsumerFunction &check_consumer_function) const;
|
||||
|
||||
const ConsumerInfo &Info() const;
|
||||
|
||||
private:
|
||||
void StartConsuming();
|
||||
void StartConsumingWithLimit(uint64_t limit_batches, std::optional<std::chrono::milliseconds> timeout) const;
|
||||
void StopConsuming();
|
||||
|
||||
ConsumerInfo info_;
|
||||
mutable pulsar_client::Client client_;
|
||||
mutable pulsar_client::Consumer consumer_;
|
||||
pulsar_client::Consumer consumer_;
|
||||
ConsumerFunction consumer_function_;
|
||||
|
||||
mutable std::atomic<bool> is_running_{false};
|
||||
mutable pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()}; // Protected by is_running_
|
||||
pulsar_client::MessageId last_message_id_{pulsar_client::MessageId::earliest()};
|
||||
std::thread thread_;
|
||||
};
|
||||
} // namespace memgraph::integrations::pulsar
|
||||
|
||||
@@ -55,16 +55,4 @@ class TopicNotFoundException : public PulsarStreamException {
|
||||
TopicNotFoundException(const std::string &consumer_name, const std::string &topic_name)
|
||||
: PulsarStreamException("Pulsar consumer {} cannot find topic {}", consumer_name, topic_name) {}
|
||||
};
|
||||
|
||||
class ConsumerReadMessagesFailedException : public PulsarStreamException {
|
||||
public:
|
||||
ConsumerReadMessagesFailedException(const std::string_view consumer_name, const std::string_view error)
|
||||
: PulsarStreamException("Error happened in consumer {} while fetching messages: {}", consumer_name, error) {}
|
||||
};
|
||||
|
||||
class ConsumerAcknowledgeMessagesFailedException : public PulsarStreamException {
|
||||
public:
|
||||
explicit ConsumerAcknowledgeMessagesFailedException(const std::string_view consumer_name)
|
||||
: PulsarStreamException("Acknowledging a message of consumer {} has failed!", consumer_name) {}
|
||||
};
|
||||
} // namespace memgraph::integrations::pulsar
|
||||
|
||||
@@ -9,52 +9,34 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "io/network/addrinfo.hpp"
|
||||
#include <netdb.h>
|
||||
#include <cstring>
|
||||
|
||||
#include <concepts>
|
||||
#include <iterator>
|
||||
#include "io/network/addrinfo.hpp"
|
||||
|
||||
#include "io/network/network_error.hpp"
|
||||
|
||||
namespace memgraph::io::network {
|
||||
|
||||
static_assert(std::forward_iterator<AddrInfo::Iterator> && std::equality_comparable<AddrInfo::Iterator>);
|
||||
AddrInfo::AddrInfo(struct addrinfo *info) : info(info) {}
|
||||
|
||||
AddrInfo::AddrInfo(const Endpoint &endpoint) : AddrInfo(endpoint.address, endpoint.port) {}
|
||||
AddrInfo::~AddrInfo() { freeaddrinfo(info); }
|
||||
|
||||
AddrInfo AddrInfo::Get(const char *addr, const char *port) {
|
||||
struct addrinfo hints;
|
||||
memset(&hints, 0, sizeof(struct addrinfo));
|
||||
|
||||
hints.ai_family = AF_UNSPEC; // IPv4 and IPv6
|
||||
hints.ai_socktype = SOCK_STREAM; // TCP socket
|
||||
hints.ai_flags = AI_PASSIVE;
|
||||
|
||||
struct addrinfo *result;
|
||||
auto status = getaddrinfo(addr, port, &hints, &result);
|
||||
|
||||
AddrInfo::AddrInfo(const std::string &addr, uint16_t port) : info_{nullptr, nullptr} {
|
||||
addrinfo hints{
|
||||
.ai_flags = AI_PASSIVE,
|
||||
.ai_family = AF_UNSPEC, // IPv4 and IPv6
|
||||
.ai_socktype = SOCK_STREAM // TCP socket
|
||||
};
|
||||
addrinfo *info = nullptr;
|
||||
auto status = getaddrinfo(addr.c_str(), std::to_string(port).c_str(), &hints, &info);
|
||||
if (status != 0) throw NetworkError(gai_strerror(status));
|
||||
info_ = std::unique_ptr<addrinfo, decltype(&freeaddrinfo)>(info, &freeaddrinfo);
|
||||
|
||||
return AddrInfo(result);
|
||||
}
|
||||
|
||||
AddrInfo::Iterator::Iterator(addrinfo *p) noexcept : ptr_(p) {}
|
||||
|
||||
AddrInfo::Iterator::reference AddrInfo::Iterator::operator*() const noexcept { return *ptr_; }
|
||||
|
||||
AddrInfo::Iterator::pointer AddrInfo::Iterator::operator->() const noexcept { return ptr_; }
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
AddrInfo::Iterator AddrInfo::Iterator::operator++(int) noexcept {
|
||||
auto it = *this;
|
||||
++(*this);
|
||||
return it;
|
||||
}
|
||||
AddrInfo::Iterator &AddrInfo::Iterator::operator++() noexcept {
|
||||
ptr_ = ptr_->ai_next;
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool operator==(const AddrInfo::Iterator &lhs, const AddrInfo::Iterator &rhs) noexcept { return lhs.ptr_ == rhs.ptr_; };
|
||||
|
||||
bool operator!=(const AddrInfo::Iterator &lhs, const AddrInfo::Iterator &rhs) noexcept { return !(lhs == rhs); };
|
||||
|
||||
void swap(AddrInfo::Iterator &lhs, AddrInfo::Iterator &rhs) noexcept { std::swap(lhs.ptr_, rhs.ptr_); };
|
||||
|
||||
AddrInfo::operator struct addrinfo *() { return info; }
|
||||
} // namespace memgraph::io::network
|
||||
|
||||
@@ -11,14 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <netdb.h>
|
||||
|
||||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
|
||||
namespace memgraph::io::network {
|
||||
|
||||
/**
|
||||
@@ -26,38 +18,16 @@ namespace memgraph::io::network {
|
||||
* see: man 3 getaddrinfo
|
||||
*/
|
||||
class AddrInfo {
|
||||
explicit AddrInfo(struct addrinfo *info);
|
||||
|
||||
public:
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using value_type = addrinfo;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using pointer = addrinfo *;
|
||||
using reference = addrinfo &;
|
||||
~AddrInfo();
|
||||
|
||||
Iterator() = default;
|
||||
Iterator(const Iterator &) = default;
|
||||
explicit Iterator(addrinfo *p) noexcept;
|
||||
Iterator &operator=(const Iterator &) = default;
|
||||
reference operator*() const noexcept;
|
||||
pointer operator->() const noexcept;
|
||||
Iterator operator++(int) noexcept;
|
||||
Iterator &operator++() noexcept;
|
||||
static AddrInfo Get(const char *addr, const char *port);
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) noexcept;
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) noexcept;
|
||||
friend void swap(Iterator &lhs, Iterator &rhs) noexcept;
|
||||
|
||||
private:
|
||||
addrinfo *ptr_{nullptr};
|
||||
};
|
||||
|
||||
AddrInfo(const std::string &addr, uint16_t port);
|
||||
explicit AddrInfo(const Endpoint &endpoint);
|
||||
|
||||
auto begin() const noexcept { return Iterator(info_.get()); }
|
||||
auto end() const noexcept { return Iterator{nullptr}; }
|
||||
operator struct addrinfo *();
|
||||
|
||||
private:
|
||||
std::unique_ptr<addrinfo, void (*)(addrinfo *)> info_;
|
||||
struct addrinfo *info;
|
||||
};
|
||||
} // namespace memgraph::io::network
|
||||
|
||||
@@ -9,25 +9,39 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "io/network/socket.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include <arpa/inet.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <netdb.h>
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <poll.h>
|
||||
#include <sys/epoll.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "io/network/addrinfo.hpp"
|
||||
#include "io/network/socket.hpp"
|
||||
#include "utils/likely.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::io::network {
|
||||
|
||||
Socket::Socket(Socket &&other) noexcept : socket_(other.socket_), endpoint_(std::move(other.endpoint_)) {
|
||||
Socket::Socket(Socket &&other) {
|
||||
socket_ = other.socket_;
|
||||
endpoint_ = std::move(other.endpoint_);
|
||||
other.socket_ = -1;
|
||||
}
|
||||
|
||||
Socket &Socket::operator=(Socket &&other) noexcept {
|
||||
Socket &Socket::operator=(Socket &&other) {
|
||||
if (this != &other) {
|
||||
if (socket_ != -1) close(socket_);
|
||||
socket_ = other.socket_;
|
||||
endpoint_ = std::move(other.endpoint_);
|
||||
other.socket_ = -1;
|
||||
@@ -35,8 +49,9 @@ Socket &Socket::operator=(Socket &&other) noexcept {
|
||||
return *this;
|
||||
}
|
||||
|
||||
Socket::~Socket() noexcept {
|
||||
if (socket_ != -1) close(socket_);
|
||||
Socket::~Socket() {
|
||||
if (socket_ == -1) return;
|
||||
close(socket_);
|
||||
}
|
||||
|
||||
void Socket::Close() {
|
||||
@@ -55,27 +70,33 @@ bool Socket::IsOpen() const { return socket_ != -1; }
|
||||
bool Socket::Connect(const Endpoint &endpoint) {
|
||||
if (socket_ != -1) return false;
|
||||
|
||||
for (const auto &it : AddrInfo{endpoint}) {
|
||||
int sfd = socket(it.ai_family, it.ai_socktype, it.ai_protocol);
|
||||
auto info = AddrInfo::Get(endpoint.address.c_str(), std::to_string(endpoint.port).c_str());
|
||||
|
||||
for (struct addrinfo *it = info; it != nullptr; it = it->ai_next) {
|
||||
int sfd = socket(it->ai_family, it->ai_socktype, it->ai_protocol);
|
||||
if (sfd == -1) continue;
|
||||
if (connect(sfd, it.ai_addr, it.ai_addrlen) == 0) {
|
||||
if (connect(sfd, it->ai_addr, it->ai_addrlen) == 0) {
|
||||
socket_ = sfd;
|
||||
endpoint_ = endpoint;
|
||||
break;
|
||||
} else {
|
||||
// If the connect failed close the file descriptor to prevent file
|
||||
// descriptors being leaked
|
||||
close(sfd);
|
||||
}
|
||||
// If the connect failed close the file descriptor to prevent file
|
||||
// descriptors being leaked
|
||||
close(sfd);
|
||||
}
|
||||
|
||||
return !(socket_ == -1);
|
||||
if (socket_ == -1) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Socket::Bind(const Endpoint &endpoint) {
|
||||
if (socket_ != -1) return false;
|
||||
|
||||
for (const auto &it : AddrInfo{endpoint}) {
|
||||
int sfd = socket(it.ai_family, it.ai_socktype, it.ai_protocol);
|
||||
auto info = AddrInfo::Get(endpoint.address.c_str(), std::to_string(endpoint.port).c_str());
|
||||
|
||||
for (struct addrinfo *it = info; it != nullptr; it = it->ai_next) {
|
||||
int sfd = socket(it->ai_family, it->ai_socktype, it->ai_protocol);
|
||||
if (sfd == -1) continue;
|
||||
|
||||
int on = 1;
|
||||
@@ -86,13 +107,14 @@ bool Socket::Bind(const Endpoint &endpoint) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (bind(sfd, it.ai_addr, it.ai_addrlen) == 0) {
|
||||
if (bind(sfd, it->ai_addr, it->ai_addrlen) == 0) {
|
||||
socket_ = sfd;
|
||||
break;
|
||||
} else {
|
||||
// If the bind failed close the file descriptor to prevent file
|
||||
// descriptors being leaked
|
||||
close(sfd);
|
||||
}
|
||||
// If the bind failed close the file descriptor to prevent file
|
||||
// descriptors being leaked
|
||||
close(sfd);
|
||||
}
|
||||
|
||||
if (socket_ == -1) return false;
|
||||
@@ -100,7 +122,7 @@ bool Socket::Bind(const Endpoint &endpoint) {
|
||||
// detect bound port, used when the server binds to a random port
|
||||
struct sockaddr_in6 portdata;
|
||||
socklen_t portdatalen = sizeof(portdata);
|
||||
if (getsockname(socket_, reinterpret_cast<sockaddr *>(&portdata), &portdatalen) < 0) {
|
||||
if (getsockname(socket_, (struct sockaddr *)&portdata, &portdatalen) < 0) {
|
||||
// If the getsockname failed close the file descriptor to prevent file
|
||||
// descriptors being leaked
|
||||
close(socket_);
|
||||
@@ -114,35 +136,36 @@ bool Socket::Bind(const Endpoint &endpoint) {
|
||||
}
|
||||
|
||||
void Socket::SetNonBlocking() {
|
||||
const unsigned flags = fcntl(socket_, F_GETFL);
|
||||
constexpr unsigned o_nonblock = O_NONBLOCK;
|
||||
int flags = fcntl(socket_, F_GETFL, 0);
|
||||
MG_ASSERT(flags != -1, "Can't get socket mode");
|
||||
MG_ASSERT(fcntl(socket_, F_SETFL, flags | o_nonblock) != -1, "Can't set socket nonblocking");
|
||||
flags |= O_NONBLOCK;
|
||||
MG_ASSERT(fcntl(socket_, F_SETFL, flags) != -1, "Can't set socket nonblocking");
|
||||
}
|
||||
|
||||
void Socket::SetKeepAlive() {
|
||||
int optval = 1;
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_KEEPALIVE, &optval, sizeof(optval)), "Can't set socket keep alive");
|
||||
socklen_t optlen = sizeof(optval);
|
||||
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_SOCKET, SO_KEEPALIVE, &optval, optlen), "Can't set socket keep alive");
|
||||
|
||||
optval = 20; // wait 20s before sending keep-alive packets
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPIDLE, (void *)&optval, sizeof(optval)),
|
||||
"Can't set socket keep alive");
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPIDLE, (void *)&optval, optlen), "Can't set socket keep alive");
|
||||
|
||||
optval = 4; // 4 keep-alive packets must fail to close
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPCNT, (void *)&optval, sizeof(optval)), "Can't set socket keep alive");
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPCNT, (void *)&optval, optlen), "Can't set socket keep alive");
|
||||
|
||||
optval = 15; // send keep-alive packets every 15s
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPINTVL, (void *)&optval, sizeof(optval)),
|
||||
"Can't set socket keep alive");
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_KEEPINTVL, (void *)&optval, optlen), "Can't set socket keep alive");
|
||||
}
|
||||
|
||||
void Socket::SetNoDelay() {
|
||||
int optval = 1;
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_NODELAY, (void *)&optval, sizeof(optval)), "Can't set socket no delay");
|
||||
socklen_t optlen = sizeof(optval);
|
||||
|
||||
MG_ASSERT(!setsockopt(socket_, SOL_TCP, TCP_NODELAY, (void *)&optval, optlen), "Can't set socket no delay");
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-make-member-function-const)
|
||||
void Socket::SetTimeout(int64_t sec, int64_t usec) {
|
||||
void Socket::SetTimeout(long sec, long usec) {
|
||||
struct timeval tv;
|
||||
tv.tv_sec = sec;
|
||||
tv.tv_usec = usec;
|
||||
@@ -153,7 +176,7 @@ void Socket::SetTimeout(int64_t sec, int64_t usec) {
|
||||
}
|
||||
|
||||
int Socket::ErrorStatus() const {
|
||||
int optval = 0;
|
||||
int optval;
|
||||
socklen_t optlen = sizeof(optval);
|
||||
auto status = getsockopt(socket_, SOL_SOCKET, SO_ERROR, &optval, &optlen);
|
||||
MG_ASSERT(!status, "getsockopt failed");
|
||||
@@ -166,22 +189,21 @@ std::optional<Socket> Socket::Accept() {
|
||||
sockaddr_storage addr;
|
||||
socklen_t addr_size = sizeof addr;
|
||||
char addr_decoded[INET6_ADDRSTRLEN];
|
||||
void *addr_src;
|
||||
unsigned short port;
|
||||
|
||||
int sfd = accept(socket_, reinterpret_cast<sockaddr *>(&addr), &addr_size);
|
||||
int sfd = accept(socket_, (struct sockaddr *)&addr, &addr_size);
|
||||
if (UNLIKELY(sfd == -1)) return std::nullopt;
|
||||
|
||||
void *addr_src = nullptr;
|
||||
uint16_t port = 0;
|
||||
|
||||
if (addr.ss_family == AF_INET) {
|
||||
addr_src = &reinterpret_cast<sockaddr_in &>(addr).sin_addr;
|
||||
port = ntohs(reinterpret_cast<sockaddr_in &>(addr).sin_port);
|
||||
addr_src = (void *)&(((sockaddr_in *)&addr)->sin_addr);
|
||||
port = ntohs(((sockaddr_in *)&addr)->sin_port);
|
||||
} else {
|
||||
addr_src = &reinterpret_cast<sockaddr_in6 &>(addr).sin6_addr;
|
||||
port = ntohs(reinterpret_cast<sockaddr_in6 &>(addr).sin6_port);
|
||||
addr_src = (void *)&(((sockaddr_in6 *)&addr)->sin6_addr);
|
||||
port = ntohs(((sockaddr_in6 *)&addr)->sin6_port);
|
||||
}
|
||||
|
||||
inet_ntop(addr.ss_family, addr_src, addr_decoded, sizeof(addr_decoded));
|
||||
inet_ntop(addr.ss_family, addr_src, addr_decoded, INET6_ADDRSTRLEN);
|
||||
|
||||
Endpoint endpoint(addr_decoded, port);
|
||||
|
||||
@@ -191,11 +213,9 @@ std::optional<Socket> Socket::Accept() {
|
||||
bool Socket::Write(const uint8_t *data, size_t len, bool have_more) {
|
||||
// MSG_NOSIGNAL is here to disable raising a SIGPIPE signal when a
|
||||
// connection dies mid-write, the socket will only return an EPIPE error.
|
||||
constexpr unsigned msg_nosignal = MSG_NOSIGNAL;
|
||||
constexpr unsigned msg_more = MSG_MORE;
|
||||
const unsigned flags = msg_nosignal | (have_more ? msg_more : 0);
|
||||
int flags = MSG_NOSIGNAL | (have_more ? MSG_MORE : 0);
|
||||
while (len > 0) {
|
||||
auto written = send(socket_, data, len, static_cast<int>(flags));
|
||||
auto written = send(socket_, data, len, flags);
|
||||
if (written == -1) {
|
||||
if (errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) {
|
||||
// Terminal error, return failure.
|
||||
@@ -233,8 +253,7 @@ bool Socket::WaitForReadyRead() {
|
||||
// event occurs.
|
||||
int ret = poll(&p, 1, -1);
|
||||
if (ret < 1) return false;
|
||||
constexpr unsigned pollin = POLLIN;
|
||||
return static_cast<unsigned>(p.revents) & pollin;
|
||||
return p.revents & POLLIN;
|
||||
}
|
||||
|
||||
bool Socket::WaitForReadyWrite() {
|
||||
@@ -246,8 +265,7 @@ bool Socket::WaitForReadyWrite() {
|
||||
// event occurs.
|
||||
int ret = poll(&p, 1, -1);
|
||||
if (ret < 1) return false;
|
||||
constexpr unsigned pollout = POLLOUT;
|
||||
return static_cast<unsigned>(p.revents) & pollout;
|
||||
return p.revents & POLLOUT;
|
||||
}
|
||||
|
||||
} // namespace memgraph::io::network
|
||||
|
||||
@@ -27,12 +27,12 @@ namespace memgraph::io::network {
|
||||
*/
|
||||
class Socket {
|
||||
public:
|
||||
Socket() noexcept = default;
|
||||
Socket() = default;
|
||||
Socket(const Socket &) = delete;
|
||||
Socket &operator=(const Socket &) = delete;
|
||||
Socket(Socket &&) noexcept;
|
||||
Socket &operator=(Socket &&) noexcept;
|
||||
~Socket() noexcept;
|
||||
Socket(Socket &&);
|
||||
Socket &operator=(Socket &&);
|
||||
~Socket();
|
||||
|
||||
/**
|
||||
* Closes the socket if it is open.
|
||||
@@ -118,7 +118,7 @@ class Socket {
|
||||
* @param sec timeout seconds value
|
||||
* @param usec timeout microseconds value
|
||||
*/
|
||||
void SetTimeout(int64_t sec, int64_t usec);
|
||||
void SetTimeout(long sec, long usec);
|
||||
|
||||
/**
|
||||
* Checks if there are any errors on a socket. Returns 0 if there are none.
|
||||
|
||||
@@ -81,8 +81,8 @@
|
||||
#include "communication/bolt/v1/exceptions.hpp"
|
||||
#include "communication/bolt/v1/session.hpp"
|
||||
#include "communication/init.hpp"
|
||||
#include "communication/v2/server.hpp"
|
||||
#include "communication/v2/session.hpp"
|
||||
#include "communication/server.hpp"
|
||||
#include "communication/session.hpp"
|
||||
#include "glue/communication.hpp"
|
||||
|
||||
#include "auth/auth.hpp"
|
||||
@@ -252,11 +252,6 @@ DEFINE_double(query_execution_timeout_sec, 600,
|
||||
"Maximum allowed query execution time. Queries exceeding this "
|
||||
"limit will be aborted. Value of 0 means no limit.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint64(replication_replica_check_frequency_sec, 1,
|
||||
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
|
||||
"The MAIN instance allocates a new thread for each REPLICA.");
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint64(
|
||||
memory_limit, 0,
|
||||
@@ -265,7 +260,7 @@ DEFINE_uint64(
|
||||
|
||||
namespace {
|
||||
using namespace std::literals;
|
||||
inline constexpr std::array isolation_level_mappings{
|
||||
constexpr std::array isolation_level_mappings{
|
||||
std::pair{"SNAPSHOT_ISOLATION"sv, memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION},
|
||||
std::pair{"READ_COMMITTED"sv, memgraph::storage::IsolationLevel::READ_COMMITTED},
|
||||
std::pair{"READ_UNCOMMITTED"sv, memgraph::storage::IsolationLevel::READ_UNCOMMITTED}};
|
||||
@@ -353,7 +348,7 @@ DEFINE_bool(also_log_to_stderr, false, "Log messages go to stderr in addition to
|
||||
DEFINE_string(log_file, "", "Path to where the log should be stored.");
|
||||
|
||||
namespace {
|
||||
inline constexpr std::array log_level_mappings{
|
||||
constexpr std::array log_level_mappings{
|
||||
std::pair{"TRACE"sv, spdlog::level::trace}, std::pair{"DEBUG"sv, spdlog::level::debug},
|
||||
std::pair{"INFO"sv, spdlog::level::info}, std::pair{"WARNING"sv, spdlog::level::warn},
|
||||
std::pair{"ERROR"sv, spdlog::level::err}, std::pair{"CRITICAL"sv, spdlog::level::critical}};
|
||||
@@ -390,7 +385,7 @@ spdlog::level::level_enum ParseLogLevel() {
|
||||
}
|
||||
|
||||
// 5 weeks * 7 days
|
||||
inline constexpr auto log_retention_count = 35;
|
||||
constexpr auto log_retention_count = 35;
|
||||
void CreateLoggerFromSink(const auto &sinks, const auto log_level) {
|
||||
auto logger = std::make_shared<spdlog::logger>("memgraph_log", sinks.begin(), sinks.end());
|
||||
logger->set_level(log_level);
|
||||
@@ -461,7 +456,7 @@ struct SessionData {
|
||||
#endif
|
||||
};
|
||||
|
||||
inline constexpr std::string_view default_user_role_regex = "[a-zA-Z0-9_.+-@]+";
|
||||
constexpr std::string_view default_user_role_regex = "[a-zA-Z0-9_.+-@]+";
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_string(auth_user_or_role_name_regex, default_user_role_regex.data(),
|
||||
"Set to the regular expression that each user or role name must fulfill.");
|
||||
@@ -501,7 +496,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
|
||||
|
||||
if (first_user) {
|
||||
spdlog::info("{} is first created user. Granting all privileges.", username);
|
||||
GrantPrivilege(username, memgraph::query::kPrivilegesAll, {"*"});
|
||||
GrantPrivilege(username, memgraph::query::kPrivilegesAll);
|
||||
}
|
||||
|
||||
return user_added;
|
||||
@@ -747,9 +742,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
|
||||
}
|
||||
|
||||
void GrantPrivilege(const std::string &user_or_role,
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels) override {
|
||||
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
|
||||
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
|
||||
// TODO (mferencevic): should we first check that the
|
||||
// privilege is granted/denied/revoked before
|
||||
// unconditionally granting/denying/revoking it?
|
||||
@@ -758,9 +752,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
|
||||
}
|
||||
|
||||
void DenyPrivilege(const std::string &user_or_role,
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels) override {
|
||||
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
|
||||
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
|
||||
// TODO (mferencevic): should we first check that the
|
||||
// privilege is granted/denied/revoked before
|
||||
// unconditionally granting/denying/revoking it?
|
||||
@@ -769,9 +762,8 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
|
||||
}
|
||||
|
||||
void RevokePrivilege(const std::string &user_or_role,
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels) override {
|
||||
EditPermissions(user_or_role, privileges, labels, [](auto *permissions, const auto &permission) {
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges) override {
|
||||
EditPermissions(user_or_role, privileges, [](auto *permissions, const auto &permission) {
|
||||
// TODO (mferencevic): should we first check that the
|
||||
// privilege is granted/denied/revoked before
|
||||
// unconditionally granting/denying/revoking it?
|
||||
@@ -782,8 +774,7 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
|
||||
private:
|
||||
template <class TEditFun>
|
||||
void EditPermissions(const std::string &user_or_role,
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels, const TEditFun &edit_fun) {
|
||||
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges, const TEditFun &edit_fun) {
|
||||
if (!std::regex_match(user_or_role, name_regex_)) {
|
||||
throw memgraph::query::QueryRuntimeException("Invalid user or role name.");
|
||||
}
|
||||
@@ -803,17 +794,11 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
|
||||
for (const auto &permission : permissions) {
|
||||
edit_fun(&user->permissions(), permission);
|
||||
}
|
||||
for (const auto &label : labels) {
|
||||
edit_fun(&user->labelPermissions(), label);
|
||||
}
|
||||
locked_auth->SaveUser(*user);
|
||||
} else {
|
||||
for (const auto &permission : permissions) {
|
||||
edit_fun(&role->permissions(), permission);
|
||||
}
|
||||
for (const auto &label : labels) {
|
||||
edit_fun(&role->labelPermissions(), label);
|
||||
}
|
||||
locked_auth->SaveRole(*role);
|
||||
}
|
||||
} catch (const memgraph::auth::AuthException &e) {
|
||||
@@ -857,14 +842,13 @@ class AuthChecker final : public memgraph::query::AuthChecker {
|
||||
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
|
||||
};
|
||||
|
||||
class BoltSession final : public memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
|
||||
memgraph::communication::v2::OutputStream> {
|
||||
class BoltSession final : public memgraph::communication::bolt::Session<memgraph::communication::InputStream,
|
||||
memgraph::communication::OutputStream> {
|
||||
public:
|
||||
BoltSession(SessionData *data, const memgraph::communication::v2::ServerEndpoint &endpoint,
|
||||
memgraph::communication::v2::InputStream *input_stream,
|
||||
memgraph::communication::v2::OutputStream *output_stream)
|
||||
: memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
|
||||
memgraph::communication::v2::OutputStream>(input_stream, output_stream),
|
||||
BoltSession(SessionData *data, const memgraph::io::network::Endpoint &endpoint,
|
||||
memgraph::communication::InputStream *input_stream, memgraph::communication::OutputStream *output_stream)
|
||||
: memgraph::communication::bolt::Session<memgraph::communication::InputStream,
|
||||
memgraph::communication::OutputStream>(input_stream, output_stream),
|
||||
db_(data->db),
|
||||
interpreter_(data->interpreter_context),
|
||||
auth_(data->auth),
|
||||
@@ -874,8 +858,8 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
|
||||
endpoint_(endpoint) {
|
||||
}
|
||||
|
||||
using memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
|
||||
memgraph::communication::v2::OutputStream>::TEncoder;
|
||||
using memgraph::communication::bolt::Session<memgraph::communication::InputStream,
|
||||
memgraph::communication::OutputStream>::TEncoder;
|
||||
|
||||
void BeginTransaction() override { interpreter_.BeginTransaction(); }
|
||||
|
||||
@@ -893,8 +877,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
|
||||
}
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (memgraph::utils::license::global_license_checker.IsValidLicenseFast()) {
|
||||
audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query,
|
||||
memgraph::storage::PropertyValue(params_pv));
|
||||
audit_log_->Record(endpoint_.address, user_ ? *username : "", query, memgraph::storage::PropertyValue(params_pv));
|
||||
}
|
||||
#endif
|
||||
try {
|
||||
@@ -1013,10 +996,10 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
|
||||
#ifdef MG_ENTERPRISE
|
||||
memgraph::audit::Log *audit_log_;
|
||||
#endif
|
||||
memgraph::communication::v2::ServerEndpoint endpoint_;
|
||||
memgraph::io::network::Endpoint endpoint_;
|
||||
};
|
||||
|
||||
using ServerT = memgraph::communication::v2::Server<BoltSession, SessionData>;
|
||||
using ServerT = memgraph::communication::Server<BoltSession, SessionData>;
|
||||
using memgraph::communication::ServerContext;
|
||||
|
||||
// Needed to correctly handle memgraph destruction from a signal handler.
|
||||
@@ -1085,22 +1068,6 @@ int main(int argc, char **argv) {
|
||||
if (maybe_exc) {
|
||||
spdlog::error(memgraph::utils::MessageWithLink("Unable to load support for embedded Python: {}.", *maybe_exc,
|
||||
"https://memgr.ph/python"));
|
||||
} else {
|
||||
// Change how we load dynamic libraries on Python by using RTLD_NOW and
|
||||
// RTLD_DEEPBIND flags. This solves an issue with using the wrong version of
|
||||
// libstd.
|
||||
auto gil = memgraph::py::EnsureGIL();
|
||||
// NOLINTNEXTLINE(hicpp-signed-bitwise)
|
||||
auto *flag = PyLong_FromLong(RTLD_NOW | RTLD_DEEPBIND);
|
||||
auto *setdl = PySys_GetObject("setdlopenflags");
|
||||
MG_ASSERT(setdl);
|
||||
auto *arg = PyTuple_New(1);
|
||||
MG_ASSERT(arg);
|
||||
MG_ASSERT(PyTuple_SetItem(arg, 0, flag) == 0);
|
||||
PyObject_CallObject(setdl, arg);
|
||||
Py_DECREF(flag);
|
||||
Py_DECREF(setdl);
|
||||
Py_DECREF(arg);
|
||||
}
|
||||
} else {
|
||||
spdlog::error(
|
||||
@@ -1231,7 +1198,6 @@ int main(int argc, char **argv) {
|
||||
&db,
|
||||
{.query = {.allow_load_csv = FLAGS_allow_load_csv},
|
||||
.execution_timeout_sec = FLAGS_query_execution_timeout_sec,
|
||||
.replication_replica_check_frequency = std::chrono::seconds(FLAGS_replication_replica_check_frequency_sec),
|
||||
.default_kafka_bootstrap_servers = FLAGS_kafka_bootstrap_servers,
|
||||
.default_pulsar_service_url = FLAGS_pulsar_service_url,
|
||||
.stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries,
|
||||
@@ -1275,10 +1241,8 @@ int main(int argc, char **argv) {
|
||||
memgraph::utils::MessageWithLink("Using non-secure Bolt connection (without SSL).", "https://memgr.ph/ssl"));
|
||||
}
|
||||
|
||||
auto server_endpoint = memgraph::communication::v2::ServerEndpoint{
|
||||
boost::asio::ip::address::from_string(FLAGS_bolt_address), static_cast<uint16_t>(FLAGS_bolt_port)};
|
||||
ServerT server(server_endpoint, &session_data, &context, FLAGS_bolt_session_inactivity_timeout, service_name,
|
||||
FLAGS_bolt_num_workers);
|
||||
ServerT server({FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)}, &session_data, &context,
|
||||
FLAGS_bolt_session_inactivity_timeout, service_name, FLAGS_bolt_num_workers);
|
||||
|
||||
// Setup telemetry
|
||||
std::optional<memgraph::telemetry::Telemetry> telemetry;
|
||||
|
||||
@@ -9,5 +9,4 @@ target_link_libraries(mg-memory mg-utils fmt)
|
||||
|
||||
if (ENABLE_JEMALLOC)
|
||||
target_link_libraries(mg-memory Jemalloc::Jemalloc)
|
||||
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1)
|
||||
endif()
|
||||
|
||||
@@ -170,7 +170,7 @@ enum class CsvParserState {
|
||||
EXPECT_DELIMITER,
|
||||
};
|
||||
|
||||
bool SubstringStartsWith(const std::string_view str, size_t pos, const std::string_view what) {
|
||||
bool SubstringStartsWith(const std::string_view &str, size_t pos, const std::string_view &what) {
|
||||
return memgraph::utils::StartsWith(memgraph::utils::Substr(str, pos), what);
|
||||
}
|
||||
|
||||
|
||||
@@ -21,8 +21,6 @@ struct InterpreterConfig {
|
||||
|
||||
// The default execution timeout is 10 minutes.
|
||||
double execution_timeout_sec{600.0};
|
||||
// The same as \ref memgraph::storage::replication::ReplicationClientConfig
|
||||
std::chrono::seconds replication_replica_check_frequency{1};
|
||||
|
||||
std::string default_kafka_bootstrap_servers;
|
||||
std::string default_pulsar_service_url;
|
||||
|
||||
@@ -14,6 +14,6 @@
|
||||
#include <string>
|
||||
|
||||
namespace memgraph::query {
|
||||
inline constexpr uint16_t kDefaultReplicationPort = 10000;
|
||||
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
|
||||
constexpr uint16_t kDefaultReplicationPort = 10000;
|
||||
constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -310,11 +310,11 @@ class DbAccessor final {
|
||||
return std::make_optional<VertexAccessor>(*value);
|
||||
}
|
||||
|
||||
storage::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
|
||||
storage::PropertyId NameToProperty(const std::string_view &name) { return accessor_->NameToProperty(name); }
|
||||
|
||||
storage::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
|
||||
storage::LabelId NameToLabel(const std::string_view &name) { return accessor_->NameToLabel(name); }
|
||||
|
||||
storage::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
|
||||
storage::EdgeTypeId NameToEdgeType(const std::string_view &name) { return accessor_->NameToEdgeType(name); }
|
||||
|
||||
const std::string &PropertyToName(storage::PropertyId prop) const { return accessor_->PropertyToName(prop); }
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ const char *kInternalPropertyId = "__mg_id__";
|
||||
const char *kInternalVertexLabel = "__mg_vertex__";
|
||||
|
||||
/// A helper function that escapes label, edge type and property names.
|
||||
std::string EscapeName(const std::string_view value) {
|
||||
std::string EscapeName(const std::string_view &value) {
|
||||
std::string out;
|
||||
out.reserve(value.size() + 2);
|
||||
out.append(1, '`');
|
||||
|
||||
@@ -852,9 +852,7 @@ cpp<#
|
||||
: arguments_(arguments),
|
||||
function_name_(function_name),
|
||||
function_(NameToFunction(function_name_)) {
|
||||
if (!function_) {
|
||||
throw SemanticException("Function '{}' doesn't exist.", function_name);
|
||||
}
|
||||
DMG_ASSERT(function_, "Unexpected missing function: {}", function_name_);
|
||||
}
|
||||
cpp<#)
|
||||
(:private
|
||||
@@ -2239,11 +2237,9 @@ cpp<#
|
||||
(user "std::string" :scope :public)
|
||||
(role "std::string" :scope :public)
|
||||
(user-or-role "std::string" :scope :public)
|
||||
|
||||
(password "Expression *" :initval "nullptr" :scope :public
|
||||
:slk-save #'slk-save-ast-pointer
|
||||
:slk-load (slk-load-ast-pointer "Expression"))
|
||||
(labels "std::vector<std::string>" :scope :public)
|
||||
(privileges "std::vector<Privilege>" :scope :public))
|
||||
(:public
|
||||
(lcp:define-enum action
|
||||
@@ -2255,7 +2251,7 @@ cpp<#
|
||||
(lcp:define-enum privilege
|
||||
(create delete match merge set remove index stats auth constraint
|
||||
dump replication durability read_file free_memory trigger config stream module_read module_write
|
||||
websocket labels)
|
||||
websocket)
|
||||
(:serialize))
|
||||
#>cpp
|
||||
AuthQuery() = default;
|
||||
@@ -2266,14 +2262,13 @@ cpp<#
|
||||
#>cpp
|
||||
AuthQuery(Action action, std::string user, std::string role,
|
||||
std::string user_or_role, Expression *password,
|
||||
std::vector<std::string> labels ,std::vector<Privilege> privileges)
|
||||
std::vector<Privilege> privileges)
|
||||
: action_(action),
|
||||
user_(user),
|
||||
role_(role),
|
||||
user_or_role_(user_or_role),
|
||||
password_(password),
|
||||
labels_(labels),
|
||||
privileges_(privileges){}
|
||||
privileges_(privileges) {}
|
||||
cpp<#)
|
||||
(:private
|
||||
#>cpp
|
||||
@@ -2298,8 +2293,7 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
|
||||
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
|
||||
AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM,
|
||||
AuthQuery::Privilege::MODULE_READ, AuthQuery::Privilege::MODULE_WRITE,
|
||||
AuthQuery::Privilege::WEBSOCKET,
|
||||
AuthQuery::Privilege::LABELS};
|
||||
AuthQuery::Privilege::WEBSOCKET};
|
||||
cpp<#
|
||||
|
||||
(lcp:define-class info-query (query)
|
||||
@@ -2395,9 +2389,6 @@ cpp<#
|
||||
(lcp:define-enum sync-mode
|
||||
(sync async)
|
||||
(:serialize))
|
||||
(lcp:define-enum replica-state
|
||||
(ready replicating recovery invalid)
|
||||
(:serialize))
|
||||
#>cpp
|
||||
ReplicationQuery() = default;
|
||||
|
||||
@@ -2638,39 +2629,5 @@ cpp<#
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
(lcp:define-class foreach (clause)
|
||||
((named_expression "NamedExpression *" :initval "nullptr" :scope :public
|
||||
:slk-save #'slk-save-ast-pointer
|
||||
:slk-load (slk-load-ast-pointer "Expression"))
|
||||
(clauses "std::vector<Clause *>"
|
||||
:scope :public
|
||||
:slk-save #'slk-save-ast-vector
|
||||
:slk-load (slk-load-ast-vector "Clause")))
|
||||
(:public
|
||||
#>cpp
|
||||
Foreach() = default;
|
||||
|
||||
bool Accept(HierarchicalTreeVisitor &visitor) override {
|
||||
if (visitor.PreVisit(*this)) {
|
||||
named_expression_->Accept(visitor);
|
||||
for (auto &clause : clauses_) {
|
||||
clause->Accept(visitor);
|
||||
}
|
||||
}
|
||||
return visitor.PostVisit(*this);
|
||||
}
|
||||
cpp<#)
|
||||
(:protected
|
||||
#>cpp
|
||||
Foreach(NamedExpression *expression, std::vector<Clause *> clauses)
|
||||
: named_expression_(expression), clauses_(clauses) {}
|
||||
cpp<#)
|
||||
(:private
|
||||
#>cpp
|
||||
friend class AstStorage;
|
||||
cpp<#)
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
(lcp:pop-namespace) ;; namespace query
|
||||
(lcp:pop-namespace) ;; namespace memgraph
|
||||
|
||||
@@ -93,7 +93,6 @@ class CreateSnapshotQuery;
|
||||
class StreamQuery;
|
||||
class SettingQuery;
|
||||
class VersionQuery;
|
||||
class Foreach;
|
||||
|
||||
using TreeCompositeVisitor = utils::CompositeVisitor<
|
||||
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
|
||||
@@ -102,7 +101,7 @@ using TreeCompositeVisitor = utils::CompositeVisitor<
|
||||
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
|
||||
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
|
||||
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
|
||||
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach>;
|
||||
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv>;
|
||||
|
||||
using TreeLeafVisitor = utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
|
||||
namespace memgraph::query::frontend {
|
||||
|
||||
@@ -646,28 +645,28 @@ antlrcpp::Any CypherMainVisitor::visitKafkaCreateStreamConfig(MemgraphCypher::Ka
|
||||
|
||||
if (ctx->TOPICS()) {
|
||||
ThrowIfExists(memory_, KafkaConfigKey::TOPICS);
|
||||
static constexpr auto topics_key = static_cast<uint8_t>(KafkaConfigKey::TOPICS);
|
||||
constexpr auto topics_key = static_cast<uint8_t>(KafkaConfigKey::TOPICS);
|
||||
GetTopicNames(memory_[topics_key], ctx->topicNames(), *this);
|
||||
return {};
|
||||
}
|
||||
|
||||
if (ctx->CONSUMER_GROUP()) {
|
||||
ThrowIfExists(memory_, KafkaConfigKey::CONSUMER_GROUP);
|
||||
static constexpr auto consumer_group_key = static_cast<uint8_t>(KafkaConfigKey::CONSUMER_GROUP);
|
||||
constexpr auto consumer_group_key = static_cast<uint8_t>(KafkaConfigKey::CONSUMER_GROUP);
|
||||
memory_[consumer_group_key] = JoinSymbolicNamesWithDotsAndMinus(*this, *ctx->consumerGroup);
|
||||
return {};
|
||||
}
|
||||
|
||||
if (ctx->CONFIGS()) {
|
||||
ThrowIfExists(memory_, KafkaConfigKey::CONFIGS);
|
||||
static constexpr auto configs_key = static_cast<uint8_t>(KafkaConfigKey::CONFIGS);
|
||||
constexpr auto configs_key = static_cast<uint8_t>(KafkaConfigKey::CONFIGS);
|
||||
memory_.emplace(configs_key, ctx->configsMap->accept(this).as<std::unordered_map<Expression *, Expression *>>());
|
||||
return {};
|
||||
}
|
||||
|
||||
if (ctx->CREDENTIALS()) {
|
||||
ThrowIfExists(memory_, KafkaConfigKey::CREDENTIALS);
|
||||
static constexpr auto credentials_key = static_cast<uint8_t>(KafkaConfigKey::CREDENTIALS);
|
||||
constexpr auto credentials_key = static_cast<uint8_t>(KafkaConfigKey::CREDENTIALS);
|
||||
memory_.emplace(credentials_key,
|
||||
ctx->credentialsMap->accept(this).as<std::unordered_map<Expression *, Expression *>>());
|
||||
return {};
|
||||
@@ -775,23 +774,6 @@ antlrcpp::Any CypherMainVisitor::visitDropStream(MemgraphCypher::DropStreamConte
|
||||
antlrcpp::Any CypherMainVisitor::visitStartStream(MemgraphCypher::StartStreamContext *ctx) {
|
||||
auto *stream_query = storage_->Create<StreamQuery>();
|
||||
stream_query->action_ = StreamQuery::Action::START_STREAM;
|
||||
|
||||
if (ctx->BATCH_LIMIT()) {
|
||||
if (!ctx->batchLimit->numberLiteral() || !ctx->batchLimit->numberLiteral()->integerLiteral()) {
|
||||
throw SemanticException("Batch limit should be an integer literal!");
|
||||
}
|
||||
stream_query->batch_limit_ = ctx->batchLimit->accept(this);
|
||||
}
|
||||
if (ctx->TIMEOUT()) {
|
||||
if (!ctx->timeout->numberLiteral() || !ctx->timeout->numberLiteral()->integerLiteral()) {
|
||||
throw SemanticException("Timeout should be an integer literal!");
|
||||
}
|
||||
if (!ctx->BATCH_LIMIT()) {
|
||||
throw SemanticException("Parameter TIMEOUT can only be defined if BATCH_LIMIT is defined");
|
||||
}
|
||||
stream_query->timeout_ = ctx->timeout->accept(this);
|
||||
}
|
||||
|
||||
stream_query->stream_name_ = ctx->streamName()->symbolicName()->accept(this).as<std::string>();
|
||||
return stream_query;
|
||||
}
|
||||
@@ -974,8 +956,7 @@ antlrcpp::Any CypherMainVisitor::visitSingleQuery(MemgraphCypher::SingleQueryCon
|
||||
utils::IsSubtype(clause_type, SetProperty::kType) ||
|
||||
utils::IsSubtype(clause_type, SetProperties::kType) || utils::IsSubtype(clause_type, SetLabels::kType) ||
|
||||
utils::IsSubtype(clause_type, RemoveProperty::kType) ||
|
||||
utils::IsSubtype(clause_type, RemoveLabels::kType) || utils::IsSubtype(clause_type, Merge::kType) ||
|
||||
utils::IsSubtype(clause_type, Foreach::kType)) {
|
||||
utils::IsSubtype(clause_type, RemoveLabels::kType) || utils::IsSubtype(clause_type, Merge::kType)) {
|
||||
if (has_return) {
|
||||
throw SemanticException("Update clause can't be used after RETURN.");
|
||||
}
|
||||
@@ -1055,9 +1036,6 @@ antlrcpp::Any CypherMainVisitor::visitClause(MemgraphCypher::ClauseContext *ctx)
|
||||
if (ctx->loadCsv()) {
|
||||
return static_cast<Clause *>(ctx->loadCsv()->accept(this).as<LoadCsv *>());
|
||||
}
|
||||
if (ctx->foreach ()) {
|
||||
return static_cast<Clause *>(ctx->foreach ()->accept(this).as<Foreach *>());
|
||||
}
|
||||
// TODO: implement other clauses.
|
||||
throw utils::NotYetImplemented("clause '{}'", ctx->getText());
|
||||
return 0;
|
||||
@@ -1285,11 +1263,7 @@ antlrcpp::Any CypherMainVisitor::visitGrantPrivilege(MemgraphCypher::GrantPrivil
|
||||
auth->user_or_role_ = ctx->userOrRole->accept(this).as<std::string>();
|
||||
if (ctx->privilegeList()) {
|
||||
for (auto *privilege : ctx->privilegeList()->privilege()) {
|
||||
if (privilege->LABELS()) {
|
||||
auth->labels_ = privilege->labelList()->accept(this).as<std::vector<std::string>>();
|
||||
} else {
|
||||
auth->privileges_.push_back(privilege->accept(this));
|
||||
}
|
||||
auth->privileges_.push_back(privilege->accept(this));
|
||||
}
|
||||
} else {
|
||||
/* grant all privileges */
|
||||
@@ -1307,11 +1281,7 @@ antlrcpp::Any CypherMainVisitor::visitDenyPrivilege(MemgraphCypher::DenyPrivileg
|
||||
auth->user_or_role_ = ctx->userOrRole->accept(this).as<std::string>();
|
||||
if (ctx->privilegeList()) {
|
||||
for (auto *privilege : ctx->privilegeList()->privilege()) {
|
||||
if (privilege->LABELS()) {
|
||||
auth->labels_ = privilege->labelList()->accept(this).as<std::vector<std::string>>();
|
||||
} else {
|
||||
auth->privileges_.push_back(privilege->accept(this));
|
||||
}
|
||||
auth->privileges_.push_back(privilege->accept(this));
|
||||
}
|
||||
} else {
|
||||
/* deny all privileges */
|
||||
@@ -1329,11 +1299,7 @@ antlrcpp::Any CypherMainVisitor::visitRevokePrivilege(MemgraphCypher::RevokePriv
|
||||
auth->user_or_role_ = ctx->userOrRole->accept(this).as<std::string>();
|
||||
if (ctx->privilegeList()) {
|
||||
for (auto *privilege : ctx->privilegeList()->privilege()) {
|
||||
if (privilege->LABELS()) {
|
||||
auth->labels_ = privilege->labelList()->accept(this).as<std::vector<std::string>>();
|
||||
} else {
|
||||
auth->privileges_.push_back(privilege->accept(this));
|
||||
}
|
||||
auth->privileges_.push_back(privilege->accept(this));
|
||||
}
|
||||
} else {
|
||||
/* revoke all privileges */
|
||||
@@ -1342,22 +1308,6 @@ antlrcpp::Any CypherMainVisitor::visitRevokePrivilege(MemgraphCypher::RevokePriv
|
||||
return auth;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return AuthQuery*
|
||||
*/
|
||||
antlrcpp::Any CypherMainVisitor::visitLabelList(MemgraphCypher::LabelListContext *ctx) {
|
||||
std::vector<std::string> labels;
|
||||
for (auto *label : ctx->label()) {
|
||||
if (label->ASTERISK()) {
|
||||
labels.push_back("*");
|
||||
} else {
|
||||
labels.push_back(label->symbolicName()->accept(this).as<std::string>());
|
||||
}
|
||||
}
|
||||
|
||||
return labels;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return AuthQuery::Privilege
|
||||
*/
|
||||
@@ -1383,10 +1333,6 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
|
||||
if (ctx->MODULE_READ()) return AuthQuery::Privilege::MODULE_READ;
|
||||
if (ctx->MODULE_WRITE()) return AuthQuery::Privilege::MODULE_WRITE;
|
||||
if (ctx->WEBSOCKET()) return AuthQuery::Privilege::WEBSOCKET;
|
||||
if (ctx->LABELS()) {
|
||||
// fill labels in authquery
|
||||
return AuthQuery::Privilege::LABELS;
|
||||
}
|
||||
LOG_FATAL("Should not get here - unknown privilege!");
|
||||
}
|
||||
|
||||
@@ -2158,30 +2104,13 @@ antlrcpp::Any CypherMainVisitor::visitFunctionInvocation(MemgraphCypher::Functio
|
||||
storage_->Create<Aggregation>(expressions[1], expressions[0], Aggregation::Op::COLLECT_MAP));
|
||||
}
|
||||
|
||||
auto is_user_defined_function = [](const std::string &function_name) {
|
||||
// Dots are present only in user-defined functions, since modules are case-sensitive, so must be user-defined
|
||||
// functions. Builtin functions should be case insensitive.
|
||||
return function_name.find('.') != std::string::npos;
|
||||
};
|
||||
|
||||
// Don't cache queries which call user-defined functions. User-defined function's return
|
||||
// types can vary depending on whether the module is reloaded, therefore the cache would
|
||||
// be invalid.
|
||||
if (is_user_defined_function(function_name)) {
|
||||
query_info_.is_cacheable = false;
|
||||
}
|
||||
|
||||
auto function = NameToFunction(function_name);
|
||||
if (!function) throw SemanticException("Function '{}' doesn't exist.", function_name);
|
||||
return static_cast<Expression *>(storage_->Create<Function>(function_name, expressions));
|
||||
}
|
||||
|
||||
antlrcpp::Any CypherMainVisitor::visitFunctionName(MemgraphCypher::FunctionNameContext *ctx) {
|
||||
auto function_name = ctx->getText();
|
||||
// Dots are present only in user-defined functions, since modules are case-sensitive, so must be user-defined
|
||||
// functions. Builtin functions should be case insensitive.
|
||||
if (function_name.find('.') != std::string::npos) {
|
||||
return function_name;
|
||||
}
|
||||
return utils::ToUpperCase(function_name);
|
||||
return utils::ToUpperCase(ctx->getText());
|
||||
}
|
||||
|
||||
antlrcpp::Any CypherMainVisitor::visitDoubleLiteral(MemgraphCypher::DoubleLiteralContext *ctx) {
|
||||
@@ -2354,37 +2283,6 @@ antlrcpp::Any CypherMainVisitor::visitFilterExpression(MemgraphCypher::FilterExp
|
||||
return 0;
|
||||
}
|
||||
|
||||
antlrcpp::Any CypherMainVisitor::visitForeach(MemgraphCypher::ForeachContext *ctx) {
|
||||
auto *for_each = storage_->Create<Foreach>();
|
||||
|
||||
auto *named_expr = storage_->Create<NamedExpression>();
|
||||
named_expr->expression_ = ctx->expression()->accept(this);
|
||||
named_expr->name_ = std::string(ctx->variable()->accept(this).as<std::string>());
|
||||
for_each->named_expression_ = named_expr;
|
||||
|
||||
for (auto *update_clause_ctx : ctx->updateClause()) {
|
||||
if (auto *set = update_clause_ctx->set(); set) {
|
||||
auto set_items = visitSet(set).as<std::vector<Clause *>>();
|
||||
std::copy(set_items.begin(), set_items.end(), std::back_inserter(for_each->clauses_));
|
||||
} else if (auto *remove = update_clause_ctx->remove(); remove) {
|
||||
auto remove_items = visitRemove(remove).as<std::vector<Clause *>>();
|
||||
std::copy(remove_items.begin(), remove_items.end(), std::back_inserter(for_each->clauses_));
|
||||
} else if (auto *merge = update_clause_ctx->merge(); merge) {
|
||||
for_each->clauses_.push_back(visitMerge(merge).as<Merge *>());
|
||||
} else if (auto *create = update_clause_ctx->create(); create) {
|
||||
for_each->clauses_.push_back(visitCreate(create).as<Create *>());
|
||||
} else if (auto *cypher_delete = update_clause_ctx->cypherDelete(); cypher_delete) {
|
||||
for_each->clauses_.push_back(visitCypherDelete(cypher_delete).as<Delete *>());
|
||||
} else {
|
||||
auto *nested_for_each = update_clause_ctx->foreach ();
|
||||
MG_ASSERT(nested_for_each != nullptr, "Unexpected clause in FOREACH");
|
||||
for_each->clauses_.push_back(visitForeach(nested_for_each).as<Foreach *>());
|
||||
}
|
||||
}
|
||||
|
||||
return for_each;
|
||||
}
|
||||
|
||||
LabelIx CypherMainVisitor::AddLabel(const std::string &name) { return storage_->GetLabelIx(name); }
|
||||
|
||||
PropertyIx CypherMainVisitor::AddProperty(const std::string &name) { return storage_->GetPropertyIx(name); }
|
||||
|
||||
@@ -473,11 +473,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
|
||||
*/
|
||||
antlrcpp::Any visitPrivilege(MemgraphCypher::PrivilegeContext *ctx) override;
|
||||
|
||||
/**
|
||||
* @return AuthQuery::LabelList
|
||||
*/
|
||||
antlrcpp::Any visitLabelList(MemgraphCypher::LabelListContext *ctx) override;
|
||||
|
||||
/**
|
||||
* @return AuthQuery*
|
||||
*/
|
||||
@@ -849,11 +844,6 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
|
||||
*/
|
||||
antlrcpp::Any visitFilterExpression(MemgraphCypher::FilterExpressionContext *) override;
|
||||
|
||||
/**
|
||||
* @return Foreach*
|
||||
*/
|
||||
antlrcpp::Any visitForeach(MemgraphCypher::ForeachContext *ctx) override;
|
||||
|
||||
public:
|
||||
Query *query() { return query_; }
|
||||
const static std::string kAnonPrefix;
|
||||
|
||||
@@ -279,7 +279,7 @@ idInColl : variable IN expression ;
|
||||
|
||||
functionInvocation : functionName '(' ( DISTINCT )? ( expression ( ',' expression )* )? ')' ;
|
||||
|
||||
functionName : symbolicName ( '.' symbolicName )* ;
|
||||
functionName : symbolicName ;
|
||||
|
||||
listComprehension : '[' filterExpression ( '|' expression )? ']' ;
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ memgraphCypherKeyword : cypherKeyword
|
||||
| DUMP
|
||||
| EXECUTE
|
||||
| FOR
|
||||
| FOREACH
|
||||
| FREE
|
||||
| FROM
|
||||
| GLOBAL
|
||||
@@ -56,7 +55,6 @@ memgraphCypherKeyword : cypherKeyword
|
||||
| IDENTIFIED
|
||||
| ISOLATION
|
||||
| KAFKA
|
||||
| LABELS
|
||||
| LEVEL
|
||||
| LOAD
|
||||
| LOCK
|
||||
@@ -165,19 +163,8 @@ clause : cypherMatch
|
||||
| cypherReturn
|
||||
| callProcedure
|
||||
| loadCsv
|
||||
| foreach
|
||||
;
|
||||
|
||||
updateClause : set
|
||||
| remove
|
||||
| create
|
||||
| merge
|
||||
| cypherDelete
|
||||
| foreach
|
||||
;
|
||||
|
||||
foreach : FOREACH '(' variable IN expression '|' updateClause+ ')' ;
|
||||
|
||||
streamQuery : checkStream
|
||||
| createStream
|
||||
| dropStream
|
||||
@@ -255,15 +242,10 @@ privilege : CREATE
|
||||
| MODULE_READ
|
||||
| MODULE_WRITE
|
||||
| WEBSOCKET
|
||||
| LABELS labels=labelList
|
||||
;
|
||||
|
||||
privilegeList : privilege ( ',' privilege )* ;
|
||||
|
||||
labelList : COLON label ( ',' COLON label )* ;
|
||||
|
||||
label : ( '*' | symbolicName ) ;
|
||||
|
||||
showPrivileges : SHOW PRIVILEGES FOR userOrRole=userOrRoleName ;
|
||||
|
||||
showRoleForUser : SHOW ROLE FOR user=userOrRoleName ;
|
||||
@@ -357,7 +339,7 @@ pulsarCreateStream : CREATE PULSAR STREAM streamName ( pulsarCreateStreamConfig
|
||||
|
||||
dropStream : DROP STREAM streamName ;
|
||||
|
||||
startStream : START STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ;
|
||||
startStream : START STREAM streamName ;
|
||||
|
||||
startAllStreams : START ALL STREAMS ;
|
||||
|
||||
|
||||
@@ -54,7 +54,6 @@ DUMP : D U M P ;
|
||||
DURABILITY : D U R A B I L I T Y ;
|
||||
EXECUTE : E X E C U T E ;
|
||||
FOR : F O R ;
|
||||
FOREACH : F O R E A C H;
|
||||
FREE : F R E E ;
|
||||
FREE_MEMORY : F R E E UNDERSCORE M E M O R Y ;
|
||||
FROM : F R O M ;
|
||||
@@ -66,7 +65,6 @@ IDENTIFIED : I D E N T I F I E D ;
|
||||
IGNORE : I G N O R E ;
|
||||
ISOLATION : I S O L A T I O N ;
|
||||
KAFKA : K A F K A ;
|
||||
LABELS : L A B E L S ;
|
||||
LEVEL : L E V E L ;
|
||||
LOAD : L O A D ;
|
||||
LOCK : L O C K ;
|
||||
|
||||
@@ -15,9 +15,7 @@
|
||||
|
||||
#include "query/frontend/semantic/symbol_generator.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
#include <ranges>
|
||||
#include <unordered_set>
|
||||
#include <variant>
|
||||
|
||||
@@ -41,56 +39,35 @@ std::unordered_map<std::string, Identifier *> GeneratePredefinedIdentifierMap(
|
||||
} // namespace
|
||||
|
||||
SymbolGenerator::SymbolGenerator(SymbolTable *symbol_table, const std::vector<Identifier *> &predefined_identifiers)
|
||||
: symbol_table_(symbol_table),
|
||||
predefined_identifiers_{GeneratePredefinedIdentifierMap(predefined_identifiers)},
|
||||
scopes_(1, Scope()) {}
|
||||
: symbol_table_(symbol_table), predefined_identifiers_{GeneratePredefinedIdentifierMap(predefined_identifiers)} {}
|
||||
|
||||
std::optional<Symbol> SymbolGenerator::FindSymbolInScope(const std::string &name, const Scope &scope,
|
||||
Symbol::Type type) {
|
||||
if (auto it = scope.symbols.find(name); it != scope.symbols.end()) {
|
||||
const auto &symbol = it->second;
|
||||
auto SymbolGenerator::CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type, int token_position) {
|
||||
auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position);
|
||||
scope_.symbols[name] = symbol;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
auto SymbolGenerator::GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type) {
|
||||
auto search = scope_.symbols.find(name);
|
||||
if (search != scope_.symbols.end()) {
|
||||
auto symbol = search->second;
|
||||
// Unless we have `ANY` type, check that types match.
|
||||
if (type != Symbol::Type::ANY && symbol.type() != Symbol::Type::ANY && type != symbol.type()) {
|
||||
throw TypeMismatchError(name, Symbol::TypeToString(symbol.type()), Symbol::TypeToString(type));
|
||||
}
|
||||
return symbol;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
auto SymbolGenerator::CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type, int token_position) {
|
||||
auto symbol = symbol_table_->CreateSymbol(name, user_declared, type, token_position);
|
||||
scopes_.back().symbols[name] = symbol;
|
||||
return symbol;
|
||||
}
|
||||
|
||||
auto SymbolGenerator::GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared, Symbol::Type type) {
|
||||
auto &scope = scopes_.back();
|
||||
if (auto maybe_symbol = FindSymbolInScope(name, scope, type); maybe_symbol) {
|
||||
return *maybe_symbol;
|
||||
}
|
||||
return CreateSymbol(name, user_declared, type);
|
||||
}
|
||||
|
||||
auto SymbolGenerator::GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type) {
|
||||
// NOLINTNEXTLINE
|
||||
for (auto scope = scopes_.rbegin(); scope != scopes_.rend(); ++scope) {
|
||||
if (auto maybe_symbol = FindSymbolInScope(name, *scope, type); maybe_symbol) {
|
||||
return *maybe_symbol;
|
||||
}
|
||||
return search->second;
|
||||
}
|
||||
return CreateSymbol(name, user_declared, type);
|
||||
}
|
||||
|
||||
void SymbolGenerator::VisitReturnBody(ReturnBody &body, Where *where) {
|
||||
auto &scope = scopes_.back();
|
||||
for (auto &expr : body.named_expressions) {
|
||||
expr->Accept(*this);
|
||||
}
|
||||
std::vector<Symbol> user_symbols;
|
||||
if (body.all_identifiers) {
|
||||
// Carry over user symbols because '*' appeared.
|
||||
for (const auto &sym_pair : scope.symbols) {
|
||||
for (auto sym_pair : scope_.symbols) {
|
||||
if (!sym_pair.second.user_declared()) {
|
||||
continue;
|
||||
}
|
||||
@@ -104,18 +81,18 @@ void SymbolGenerator::VisitReturnBody(ReturnBody &body, Where *where) {
|
||||
// declares only those established through named expressions. New declarations
|
||||
// must not be visible inside named expressions themselves.
|
||||
bool removed_old_names = false;
|
||||
if ((!where && body.order_by.empty()) || scope.has_aggregation) {
|
||||
if ((!where && body.order_by.empty()) || scope_.has_aggregation) {
|
||||
// WHERE and ORDER BY need to see both the old and new symbols, unless we
|
||||
// have an aggregation. Therefore, we can clear the symbols immediately if
|
||||
// there is neither ORDER BY nor WHERE, or we have an aggregation.
|
||||
scope.symbols.clear();
|
||||
scope_.symbols.clear();
|
||||
removed_old_names = true;
|
||||
}
|
||||
// Create symbols for named expressions.
|
||||
std::unordered_set<std::string> new_names;
|
||||
for (const auto &user_sym : user_symbols) {
|
||||
new_names.insert(user_sym.name());
|
||||
scope.symbols[user_sym.name()] = user_sym;
|
||||
scope_.symbols[user_sym.name()] = user_sym;
|
||||
}
|
||||
for (auto &named_expr : body.named_expressions) {
|
||||
const auto &name = named_expr->name_;
|
||||
@@ -126,35 +103,35 @@ void SymbolGenerator::VisitReturnBody(ReturnBody &body, Where *where) {
|
||||
// new symbol would have a more specific type.
|
||||
named_expr->MapTo(CreateSymbol(name, true, Symbol::Type::ANY, named_expr->token_position_));
|
||||
}
|
||||
scope.in_order_by = true;
|
||||
scope_.in_order_by = true;
|
||||
for (const auto &order_pair : body.order_by) {
|
||||
order_pair.expression->Accept(*this);
|
||||
}
|
||||
scope.in_order_by = false;
|
||||
scope_.in_order_by = false;
|
||||
if (body.skip) {
|
||||
scope.in_skip = true;
|
||||
scope_.in_skip = true;
|
||||
body.skip->Accept(*this);
|
||||
scope.in_skip = false;
|
||||
scope_.in_skip = false;
|
||||
}
|
||||
if (body.limit) {
|
||||
scope.in_limit = true;
|
||||
scope_.in_limit = true;
|
||||
body.limit->Accept(*this);
|
||||
scope.in_limit = false;
|
||||
scope_.in_limit = false;
|
||||
}
|
||||
if (where) where->Accept(*this);
|
||||
if (!removed_old_names) {
|
||||
// We have an ORDER BY or WHERE, but no aggregation, which means we didn't
|
||||
// clear the old symbols, so do it now. We cannot just call clear, because
|
||||
// we've added new symbols.
|
||||
for (auto sym_it = scope.symbols.begin(); sym_it != scope.symbols.end();) {
|
||||
for (auto sym_it = scope_.symbols.begin(); sym_it != scope_.symbols.end();) {
|
||||
if (new_names.find(sym_it->first) == new_names.end()) {
|
||||
sym_it = scope.symbols.erase(sym_it);
|
||||
sym_it = scope_.symbols.erase(sym_it);
|
||||
} else {
|
||||
sym_it++;
|
||||
}
|
||||
}
|
||||
}
|
||||
scopes_.back().has_aggregation = false;
|
||||
scope_.has_aggregation = false;
|
||||
}
|
||||
|
||||
// Query
|
||||
@@ -168,7 +145,7 @@ bool SymbolGenerator::PreVisit(SingleQuery &) {
|
||||
// Union
|
||||
|
||||
bool SymbolGenerator::PreVisit(CypherUnion &) {
|
||||
scopes_.back() = Scope();
|
||||
scope_ = Scope();
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -189,11 +166,11 @@ bool SymbolGenerator::PostVisit(CypherUnion &cypher_union) {
|
||||
// Clauses
|
||||
|
||||
bool SymbolGenerator::PreVisit(Create &) {
|
||||
scopes_.back().in_create = true;
|
||||
scope_.in_create = true;
|
||||
return true;
|
||||
}
|
||||
bool SymbolGenerator::PostVisit(Create &) {
|
||||
scopes_.back().in_create = false;
|
||||
scope_.in_create = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -206,7 +183,7 @@ bool SymbolGenerator::PreVisit(CallProcedure &call_proc) {
|
||||
|
||||
bool SymbolGenerator::PostVisit(CallProcedure &call_proc) {
|
||||
for (auto *ident : call_proc.result_identifiers_) {
|
||||
if (HasSymbolLocalScope(ident->name_)) {
|
||||
if (HasSymbol(ident->name_)) {
|
||||
throw RedeclareVariableError(ident->name_);
|
||||
}
|
||||
ident->MapTo(CreateSymbol(ident->name_, true));
|
||||
@@ -217,7 +194,7 @@ bool SymbolGenerator::PostVisit(CallProcedure &call_proc) {
|
||||
bool SymbolGenerator::PreVisit(LoadCsv &load_csv) { return false; }
|
||||
|
||||
bool SymbolGenerator::PostVisit(LoadCsv &load_csv) {
|
||||
if (HasSymbolLocalScope(load_csv.row_var_->name_)) {
|
||||
if (HasSymbol(load_csv.row_var_->name_)) {
|
||||
throw RedeclareVariableError(load_csv.row_var_->name_);
|
||||
}
|
||||
load_csv.row_var_->MapTo(CreateSymbol(load_csv.row_var_->name_, true));
|
||||
@@ -225,47 +202,45 @@ bool SymbolGenerator::PostVisit(LoadCsv &load_csv) {
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Return &ret) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_return = true;
|
||||
scope_.in_return = true;
|
||||
VisitReturnBody(ret.body_);
|
||||
scope.in_return = false;
|
||||
scope_.in_return = false;
|
||||
return false; // We handled the traversal ourselves.
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(Return &) {
|
||||
for (const auto &name_symbol : scopes_.back().symbols) curr_return_names_.insert(name_symbol.first);
|
||||
for (const auto &name_symbol : scope_.symbols) curr_return_names_.insert(name_symbol.first);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(With &with) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_with = true;
|
||||
scope_.in_with = true;
|
||||
VisitReturnBody(with.body_, with.where_);
|
||||
scope.in_with = false;
|
||||
scope_.in_with = false;
|
||||
return false; // We handled the traversal ourselves.
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Where &) {
|
||||
scopes_.back().in_where = true;
|
||||
scope_.in_where = true;
|
||||
return true;
|
||||
}
|
||||
bool SymbolGenerator::PostVisit(Where &) {
|
||||
scopes_.back().in_where = false;
|
||||
scope_.in_where = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Merge &) {
|
||||
scopes_.back().in_merge = true;
|
||||
scope_.in_merge = true;
|
||||
return true;
|
||||
}
|
||||
bool SymbolGenerator::PostVisit(Merge &) {
|
||||
scopes_.back().in_merge = false;
|
||||
scope_.in_merge = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(Unwind &unwind) {
|
||||
const auto &name = unwind.named_expression_->name_;
|
||||
if (HasSymbolLocalScope(name)) {
|
||||
if (HasSymbol(name)) {
|
||||
throw RedeclareVariableError(name);
|
||||
}
|
||||
unwind.named_expression_->MapTo(CreateSymbol(name, true));
|
||||
@@ -273,70 +248,55 @@ bool SymbolGenerator::PostVisit(Unwind &unwind) {
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Match &) {
|
||||
scopes_.back().in_match = true;
|
||||
scope_.in_match = true;
|
||||
return true;
|
||||
}
|
||||
bool SymbolGenerator::PostVisit(Match &) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_match = false;
|
||||
scope_.in_match = false;
|
||||
// Check variables in property maps after visiting Match, so that they can
|
||||
// reference symbols out of bind order.
|
||||
for (auto &ident : scope.identifiers_in_match) {
|
||||
if (!HasSymbolLocalScope(ident->name_) && !ConsumePredefinedIdentifier(ident->name_))
|
||||
for (auto &ident : scope_.identifiers_in_match) {
|
||||
if (!HasSymbol(ident->name_) && !ConsumePredefinedIdentifier(ident->name_))
|
||||
throw UnboundVariableError(ident->name_);
|
||||
ident->MapTo(scope.symbols[ident->name_]);
|
||||
ident->MapTo(scope_.symbols[ident->name_]);
|
||||
}
|
||||
scope.identifiers_in_match.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Foreach &for_each) {
|
||||
const auto &name = for_each.named_expression_->name_;
|
||||
scopes_.emplace_back(Scope());
|
||||
scopes_.back().in_foreach = true;
|
||||
for_each.named_expression_->MapTo(
|
||||
CreateSymbol(name, true, Symbol::Type::ANY, for_each.named_expression_->token_position_));
|
||||
return true;
|
||||
}
|
||||
bool SymbolGenerator::PostVisit([[maybe_unused]] Foreach &for_each) {
|
||||
scopes_.pop_back();
|
||||
scope_.identifiers_in_match.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Expressions
|
||||
|
||||
SymbolGenerator::ReturnType SymbolGenerator::Visit(Identifier &ident) {
|
||||
auto &scope = scopes_.back();
|
||||
if (scope.in_skip || scope.in_limit) {
|
||||
throw SemanticException("Variables are not allowed in {}.", scope.in_skip ? "SKIP" : "LIMIT");
|
||||
if (scope_.in_skip || scope_.in_limit) {
|
||||
throw SemanticException("Variables are not allowed in {}.", scope_.in_skip ? "SKIP" : "LIMIT");
|
||||
}
|
||||
Symbol symbol;
|
||||
if (scope.in_pattern && !(scope.in_node_atom || scope.visiting_edge)) {
|
||||
if (scope_.in_pattern && !(scope_.in_node_atom || scope_.visiting_edge)) {
|
||||
// If we are in the pattern, and outside of a node or an edge, the
|
||||
// identifier is the pattern name.
|
||||
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, Symbol::Type::PATH);
|
||||
} else if (scope.in_pattern && scope.in_pattern_atom_identifier) {
|
||||
symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, Symbol::Type::PATH);
|
||||
} else if (scope_.in_pattern && scope_.in_pattern_atom_identifier) {
|
||||
// Patterns used to create nodes and edges cannot redeclare already
|
||||
// established bindings. Declaration only happens in single node
|
||||
// patterns and in edge patterns. OpenCypher example,
|
||||
// `MATCH (n) CREATE (n)` should throw an error that `n` is already
|
||||
// declared. While `MATCH (n) CREATE (n) -[:R]-> (n)` is allowed,
|
||||
// since `n` now references the bound node instead of declaring it.
|
||||
if ((scope.in_create_node || scope.in_create_edge) && HasSymbolLocalScope(ident.name_)) {
|
||||
if ((scope_.in_create_node || scope_.in_create_edge) && HasSymbol(ident.name_)) {
|
||||
throw RedeclareVariableError(ident.name_);
|
||||
}
|
||||
auto type = Symbol::Type::VERTEX;
|
||||
if (scope.visiting_edge) {
|
||||
if (scope_.visiting_edge) {
|
||||
// Edge referencing is not allowed (like in Neo4j):
|
||||
// `MATCH (n) - [r] -> (n) - [r] -> (n) RETURN r` is not allowed.
|
||||
if (HasSymbolLocalScope(ident.name_)) {
|
||||
if (HasSymbol(ident.name_)) {
|
||||
throw RedeclareVariableError(ident.name_);
|
||||
}
|
||||
type = scope.visiting_edge->IsVariable() ? Symbol::Type::EDGE_LIST : Symbol::Type::EDGE;
|
||||
type = scope_.visiting_edge->IsVariable() ? Symbol::Type::EDGE_LIST : Symbol::Type::EDGE;
|
||||
}
|
||||
symbol = GetOrCreateSymbolLocalScope(ident.name_, ident.user_declared_, type);
|
||||
} else if (scope.in_pattern && !scope.in_pattern_atom_identifier && scope.in_match) {
|
||||
if (scope.in_edge_range && scope.visiting_edge->identifier_->name_ == ident.name_) {
|
||||
symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, type);
|
||||
} else if (scope_.in_pattern && !scope_.in_pattern_atom_identifier && scope_.in_match) {
|
||||
if (scope_.in_edge_range && scope_.visiting_edge->identifier_->name_ == ident.name_) {
|
||||
// Prevent variable path bounds to reference the identifier which is bound
|
||||
// by the variable path itself.
|
||||
throw UnboundVariableError(ident.name_);
|
||||
@@ -344,30 +304,30 @@ SymbolGenerator::ReturnType SymbolGenerator::Visit(Identifier &ident) {
|
||||
// Variables in property maps or bounds of variable length path during MATCH
|
||||
// can reference symbols bound later in the same MATCH. We collect them
|
||||
// here, so that they can be checked after visiting Match.
|
||||
scope.identifiers_in_match.emplace_back(&ident);
|
||||
scope_.identifiers_in_match.emplace_back(&ident);
|
||||
} else {
|
||||
// Everything else references a bound symbol.
|
||||
if (!HasSymbol(ident.name_) && !ConsumePredefinedIdentifier(ident.name_)) throw UnboundVariableError(ident.name_);
|
||||
symbol = GetOrCreateSymbol(ident.name_, ident.user_declared_, Symbol::Type::ANY);
|
||||
symbol = scope_.symbols[ident.name_];
|
||||
}
|
||||
ident.MapTo(symbol);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Aggregation &aggr) {
|
||||
auto &scope = scopes_.back();
|
||||
// Check if the aggregation can be used in this context. This check should
|
||||
// probably move to a separate phase, which checks if the query is well
|
||||
// formed.
|
||||
if ((!scope.in_return && !scope.in_with) || scope.in_order_by || scope.in_skip || scope.in_limit || scope.in_where) {
|
||||
if ((!scope_.in_return && !scope_.in_with) || scope_.in_order_by || scope_.in_skip || scope_.in_limit ||
|
||||
scope_.in_where) {
|
||||
throw SemanticException("Aggregation functions are only allowed in WITH and RETURN.");
|
||||
}
|
||||
if (scope.in_aggregation) {
|
||||
if (scope_.in_aggregation) {
|
||||
throw SemanticException(
|
||||
"Using aggregation functions inside aggregation functions is not "
|
||||
"allowed.");
|
||||
}
|
||||
if (scope.num_if_operators) {
|
||||
if (scope_.num_if_operators) {
|
||||
// Neo allows aggregations here and produces very interesting behaviors.
|
||||
// To simplify implementation at this moment we decided to completely
|
||||
// disallow aggregations inside of the CASE.
|
||||
@@ -381,23 +341,23 @@ bool SymbolGenerator::PreVisit(Aggregation &aggr) {
|
||||
// Currently, we only have aggregation operators which return numbers.
|
||||
auto aggr_name = Aggregation::OpToString(aggr.op_) + std::to_string(aggr.symbol_pos_);
|
||||
aggr.MapTo(CreateSymbol(aggr_name, false, Symbol::Type::NUMBER));
|
||||
scope.in_aggregation = true;
|
||||
scope.has_aggregation = true;
|
||||
scope_.in_aggregation = true;
|
||||
scope_.has_aggregation = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(Aggregation &) {
|
||||
scopes_.back().in_aggregation = false;
|
||||
scope_.in_aggregation = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(IfOperator &) {
|
||||
++scopes_.back().num_if_operators;
|
||||
++scope_.num_if_operators;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(IfOperator &) {
|
||||
--scopes_.back().num_if_operators;
|
||||
--scope_.num_if_operators;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -441,36 +401,33 @@ bool SymbolGenerator::PreVisit(Extract &extract) {
|
||||
// Pattern and its subparts.
|
||||
|
||||
bool SymbolGenerator::PreVisit(Pattern &pattern) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_pattern = true;
|
||||
if ((scope.in_create || scope.in_merge) && pattern.atoms_.size() == 1U) {
|
||||
scope_.in_pattern = true;
|
||||
if ((scope_.in_create || scope_.in_merge) && pattern.atoms_.size() == 1U) {
|
||||
MG_ASSERT(utils::IsSubtype(*pattern.atoms_[0], NodeAtom::kType), "Expected a single NodeAtom in Pattern");
|
||||
scope.in_create_node = true;
|
||||
scope_.in_create_node = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(Pattern &) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_pattern = false;
|
||||
scope.in_create_node = false;
|
||||
scope_.in_pattern = false;
|
||||
scope_.in_create_node = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(NodeAtom &node_atom) {
|
||||
auto &scope = scopes_.back();
|
||||
auto check_node_semantic = [&node_atom, &scope, this](const bool props_or_labels) {
|
||||
auto check_node_semantic = [&node_atom, this](const bool props_or_labels) {
|
||||
const auto &node_name = node_atom.identifier_->name_;
|
||||
if ((scope.in_create || scope.in_merge) && props_or_labels && HasSymbolLocalScope(node_name)) {
|
||||
if ((scope_.in_create || scope_.in_merge) && props_or_labels && HasSymbol(node_name)) {
|
||||
throw SemanticException("Cannot create node '" + node_name +
|
||||
"' with labels or properties, because it is already declared.");
|
||||
}
|
||||
scope.in_pattern_atom_identifier = true;
|
||||
scope_.in_pattern_atom_identifier = true;
|
||||
node_atom.identifier_->Accept(*this);
|
||||
scope.in_pattern_atom_identifier = false;
|
||||
scope_.in_pattern_atom_identifier = false;
|
||||
};
|
||||
|
||||
scope.in_node_atom = true;
|
||||
scope_.in_node_atom = true;
|
||||
if (auto *properties = std::get_if<std::unordered_map<PropertyIx, Expression *>>(&node_atom.properties_)) {
|
||||
bool props_or_labels = !properties->empty() || !node_atom.labels_.empty();
|
||||
|
||||
@@ -490,21 +447,20 @@ bool SymbolGenerator::PreVisit(NodeAtom &node_atom) {
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(NodeAtom &) {
|
||||
scopes_.back().in_node_atom = false;
|
||||
scope_.in_node_atom = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.visiting_edge = &edge_atom;
|
||||
if (scope.in_create || scope.in_merge) {
|
||||
scope.in_create_edge = true;
|
||||
scope_.visiting_edge = &edge_atom;
|
||||
if (scope_.in_create || scope_.in_merge) {
|
||||
scope_.in_create_edge = true;
|
||||
if (edge_atom.edge_types_.size() != 1U) {
|
||||
throw SemanticException(
|
||||
"A single relationship type must be specified "
|
||||
"when creating an edge.");
|
||||
}
|
||||
if (scope.in_create && // Merge allows bidirectionality
|
||||
if (scope_.in_create && // Merge allows bidirectionality
|
||||
edge_atom.direction_ == EdgeAtom::Direction::BOTH) {
|
||||
throw SemanticException(
|
||||
"Bidirectional relationship are not supported "
|
||||
@@ -524,15 +480,15 @@ bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) {
|
||||
std::get<ParameterLookup *>(edge_atom.properties_)->Accept(*this);
|
||||
}
|
||||
if (edge_atom.IsVariable()) {
|
||||
scope.in_edge_range = true;
|
||||
scope_.in_edge_range = true;
|
||||
if (edge_atom.lower_bound_) {
|
||||
edge_atom.lower_bound_->Accept(*this);
|
||||
}
|
||||
if (edge_atom.upper_bound_) {
|
||||
edge_atom.upper_bound_->Accept(*this);
|
||||
}
|
||||
scope.in_edge_range = false;
|
||||
scope.in_pattern = false;
|
||||
scope_.in_edge_range = false;
|
||||
scope_.in_pattern = false;
|
||||
if (edge_atom.filter_lambda_.expression) {
|
||||
VisitWithIdentifiers(edge_atom.filter_lambda_.expression,
|
||||
{edge_atom.filter_lambda_.inner_edge, edge_atom.filter_lambda_.inner_node});
|
||||
@@ -549,36 +505,34 @@ bool SymbolGenerator::PreVisit(EdgeAtom &edge_atom) {
|
||||
VisitWithIdentifiers(edge_atom.weight_lambda_.expression,
|
||||
{edge_atom.weight_lambda_.inner_edge, edge_atom.weight_lambda_.inner_node});
|
||||
}
|
||||
scope.in_pattern = true;
|
||||
scope_.in_pattern = true;
|
||||
}
|
||||
scope.in_pattern_atom_identifier = true;
|
||||
scope_.in_pattern_atom_identifier = true;
|
||||
edge_atom.identifier_->Accept(*this);
|
||||
scope.in_pattern_atom_identifier = false;
|
||||
scope_.in_pattern_atom_identifier = false;
|
||||
if (edge_atom.total_weight_) {
|
||||
if (HasSymbolLocalScope(edge_atom.total_weight_->name_)) {
|
||||
if (HasSymbol(edge_atom.total_weight_->name_)) {
|
||||
throw RedeclareVariableError(edge_atom.total_weight_->name_);
|
||||
}
|
||||
edge_atom.total_weight_->MapTo(GetOrCreateSymbolLocalScope(
|
||||
edge_atom.total_weight_->name_, edge_atom.total_weight_->user_declared_, Symbol::Type::NUMBER));
|
||||
edge_atom.total_weight_->MapTo(GetOrCreateSymbol(edge_atom.total_weight_->name_,
|
||||
edge_atom.total_weight_->user_declared_, Symbol::Type::NUMBER));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(EdgeAtom &) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.visiting_edge = nullptr;
|
||||
scope.in_create_edge = false;
|
||||
scope_.visiting_edge = nullptr;
|
||||
scope_.in_create_edge = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
void SymbolGenerator::VisitWithIdentifiers(Expression *expr, const std::vector<Identifier *> &identifiers) {
|
||||
auto &scope = scopes_.back();
|
||||
std::vector<std::pair<std::optional<Symbol>, Identifier *>> prev_symbols;
|
||||
// Collect previous symbols if they exist.
|
||||
for (const auto &identifier : identifiers) {
|
||||
std::optional<Symbol> prev_symbol;
|
||||
auto prev_symbol_it = scope.symbols.find(identifier->name_);
|
||||
if (prev_symbol_it != scope.symbols.end()) {
|
||||
auto prev_symbol_it = scope_.symbols.find(identifier->name_);
|
||||
if (prev_symbol_it != scope_.symbols.end()) {
|
||||
prev_symbol = prev_symbol_it->second;
|
||||
}
|
||||
identifier->MapTo(CreateSymbol(identifier->name_, identifier->user_declared_));
|
||||
@@ -591,20 +545,14 @@ void SymbolGenerator::VisitWithIdentifiers(Expression *expr, const std::vector<I
|
||||
const auto &prev_symbol = prev.first;
|
||||
const auto &identifier = prev.second;
|
||||
if (prev_symbol) {
|
||||
scope.symbols[identifier->name_] = *prev_symbol;
|
||||
scope_.symbols[identifier->name_] = *prev_symbol;
|
||||
} else {
|
||||
scope.symbols.erase(identifier->name_);
|
||||
scope_.symbols.erase(identifier->name_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool SymbolGenerator::HasSymbol(const std::string &name) const {
|
||||
return std::ranges::any_of(scopes_, [&name](const auto &scope) { return scope.symbols.contains(name); });
|
||||
}
|
||||
|
||||
bool SymbolGenerator::HasSymbolLocalScope(const std::string &name) const {
|
||||
return scopes_.back().symbols.contains(name);
|
||||
}
|
||||
bool SymbolGenerator::HasSymbol(const std::string &name) { return scope_.symbols.find(name) != scope_.symbols.end(); }
|
||||
|
||||
bool SymbolGenerator::ConsumePredefinedIdentifier(const std::string &name) {
|
||||
auto it = predefined_identifiers_.find(name);
|
||||
|
||||
@@ -15,9 +15,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
#include "query/exceptions.hpp"
|
||||
#include "query/frontend/ast/ast.hpp"
|
||||
#include "query/frontend/semantic/symbol_table.hpp"
|
||||
@@ -62,8 +59,6 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
bool PostVisit(Unwind &) override;
|
||||
bool PreVisit(Match &) override;
|
||||
bool PostVisit(Match &) override;
|
||||
bool PreVisit(Foreach &) override;
|
||||
bool PostVisit(Foreach &) override;
|
||||
|
||||
// Expressions
|
||||
ReturnType Visit(Identifier &) override;
|
||||
@@ -112,7 +107,6 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
bool in_order_by{false};
|
||||
bool in_where{false};
|
||||
bool in_match{false};
|
||||
bool in_foreach{false};
|
||||
// True when visiting a pattern atom (node or edge) identifier, which can be
|
||||
// reused or created in the pattern itself.
|
||||
bool in_pattern_atom_identifier{false};
|
||||
@@ -131,10 +125,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
int num_if_operators{0};
|
||||
};
|
||||
|
||||
static std::optional<Symbol> FindSymbolInScope(const std::string &name, const Scope &scope, Symbol::Type type);
|
||||
|
||||
bool HasSymbol(const std::string &name) const;
|
||||
bool HasSymbolLocalScope(const std::string &name) const;
|
||||
bool HasSymbol(const std::string &name);
|
||||
|
||||
// @return true if it added a predefined identifier with that name
|
||||
bool ConsumePredefinedIdentifier(const std::string &name);
|
||||
@@ -144,10 +135,9 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
auto CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
|
||||
int token_position = -1);
|
||||
|
||||
auto GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY);
|
||||
// Returns the symbol by name. If the mapping already exists, checks if the
|
||||
// types match. Otherwise, returns a new symbol.
|
||||
auto GetOrCreateSymbolLocalScope(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY);
|
||||
auto GetOrCreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY);
|
||||
|
||||
void VisitReturnBody(ReturnBody &body, Where *where = nullptr);
|
||||
|
||||
@@ -158,7 +148,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
// Identifiers which are injected from outside the query. Each identifier
|
||||
// is mapped by its name.
|
||||
std::unordered_map<std::string, Identifier *> predefined_identifiers_;
|
||||
std::vector<Scope> scopes_;
|
||||
Scope scope_;
|
||||
std::unordered_set<std::string> prev_return_names_;
|
||||
std::unordered_set<std::string> curr_return_names_;
|
||||
};
|
||||
|
||||
@@ -204,9 +204,7 @@ const trie::Trie kKeywords = {"union",
|
||||
"pulsar",
|
||||
"service_url",
|
||||
"version",
|
||||
"websocket",
|
||||
"foreach",
|
||||
"labels"};
|
||||
"websocket"};
|
||||
|
||||
// Unicode codepoints that are allowed at the start of the unescaped name.
|
||||
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(
|
||||
|
||||
@@ -22,9 +22,6 @@
|
||||
|
||||
#include "query/db_accessor.hpp"
|
||||
#include "query/exceptions.hpp"
|
||||
#include "query/procedure/cypher_types.hpp"
|
||||
#include "query/procedure/mg_procedure_impl.hpp"
|
||||
#include "query/procedure/module.hpp"
|
||||
#include "query/typed_value.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/temporal.hpp"
|
||||
@@ -310,9 +307,9 @@ void FType(const char *name, const TypedValue *args, int64_t nargs, int64_t pos
|
||||
}
|
||||
return;
|
||||
}
|
||||
static constexpr int64_t required_args = FTypeRequiredArgs<ArgType, ArgTypes...>();
|
||||
static constexpr int64_t optional_args = FTypeOptionalArgs<ArgType, ArgTypes...>();
|
||||
static constexpr int64_t total_args = required_args + optional_args;
|
||||
constexpr int64_t required_args = FTypeRequiredArgs<ArgType, ArgTypes...>();
|
||||
constexpr int64_t optional_args = FTypeOptionalArgs<ArgType, ArgTypes...>();
|
||||
constexpr int64_t total_args = required_args + optional_args;
|
||||
if constexpr (optional_args > 0) {
|
||||
if (nargs < required_args || nargs > total_args) {
|
||||
throw QueryRuntimeException("'{}' requires between {} and {} arguments.", name, required_args, total_args);
|
||||
@@ -813,7 +810,7 @@ TypedValue StringMatchOperator(const TypedValue *args, int64_t nargs, const Func
|
||||
|
||||
// Check if s1 starts with s2.
|
||||
struct StartsWithPredicate {
|
||||
static constexpr const char *name = "startsWith";
|
||||
constexpr static const char *name = "startsWith";
|
||||
bool operator()(const TypedValue::TString &s1, const TypedValue::TString &s2) const {
|
||||
if (s1.size() < s2.size()) return false;
|
||||
return std::equal(s2.begin(), s2.end(), s1.begin());
|
||||
@@ -823,7 +820,7 @@ auto StartsWith = StringMatchOperator<StartsWithPredicate>;
|
||||
|
||||
// Check if s1 ends with s2.
|
||||
struct EndsWithPredicate {
|
||||
static constexpr const char *name = "endsWith";
|
||||
constexpr static const char *name = "endsWith";
|
||||
bool operator()(const TypedValue::TString &s1, const TypedValue::TString &s2) const {
|
||||
if (s1.size() < s2.size()) return false;
|
||||
return std::equal(s2.rbegin(), s2.rend(), s1.rbegin());
|
||||
@@ -833,7 +830,7 @@ auto EndsWith = StringMatchOperator<EndsWithPredicate>;
|
||||
|
||||
// Check if s1 contains s2.
|
||||
struct ContainsPredicate {
|
||||
static constexpr const char *name = "contains";
|
||||
constexpr static const char *name = "contains";
|
||||
bool operator()(const TypedValue::TString &s1, const TypedValue::TString &s2) const {
|
||||
if (s1.size() < s2.size()) return false;
|
||||
return s1.find(s2) != std::string::npos;
|
||||
@@ -1177,53 +1174,6 @@ TypedValue Duration(const TypedValue *args, int64_t nargs, const FunctionContext
|
||||
MapNumericParameters<Number>(parameter_mappings, args[0].ValueMap());
|
||||
return TypedValue(utils::Duration(duration_parameters), ctx.memory);
|
||||
}
|
||||
|
||||
std::function<TypedValue(const TypedValue *, const int64_t, const FunctionContext &)> UserFunction(
|
||||
const mgp_func &func, const std::string &fully_qualified_name) {
|
||||
return [func, fully_qualified_name](const TypedValue *args, int64_t nargs, const FunctionContext &ctx) -> TypedValue {
|
||||
/// Find function is called to aquire the lock on Module pointer while user-defined function is executed
|
||||
const auto &maybe_found =
|
||||
procedure::FindFunction(procedure::gModuleRegistry, fully_qualified_name, utils::NewDeleteResource());
|
||||
if (!maybe_found) {
|
||||
throw QueryRuntimeException(
|
||||
"Function '{}' has been unloaded. Please check query modules to confirm that function is loaded in Memgraph.",
|
||||
fully_qualified_name);
|
||||
}
|
||||
/// Explicit extraction of module pointer, to clearly state that the lock is aquired.
|
||||
// NOLINTNEXTLINE(clang-diagnostic-unused-variable)
|
||||
const auto &module_ptr = (*maybe_found).first;
|
||||
|
||||
const auto &func_cb = func.cb;
|
||||
mgp_memory memory{ctx.memory};
|
||||
mgp_func_context functx{ctx.db_accessor, ctx.view};
|
||||
auto graph = mgp_graph::NonWritableGraph(*ctx.db_accessor, ctx.view);
|
||||
|
||||
std::vector<TypedValue> args_list;
|
||||
args_list.reserve(nargs);
|
||||
for (std::size_t i = 0; i < nargs; ++i) {
|
||||
args_list.emplace_back(args[i]);
|
||||
}
|
||||
|
||||
auto function_argument_list = mgp_list(ctx.memory);
|
||||
procedure::ConstructArguments(args_list, func, fully_qualified_name, function_argument_list, graph);
|
||||
|
||||
mgp_func_result maybe_res;
|
||||
func_cb(&function_argument_list, &functx, &maybe_res, &memory);
|
||||
if (maybe_res.error_msg) {
|
||||
throw QueryRuntimeException(*maybe_res.error_msg);
|
||||
}
|
||||
|
||||
if (!maybe_res.value) {
|
||||
throw QueryRuntimeException(
|
||||
"Function '{}' didn't set the result nor the error message. Please either set the result by using "
|
||||
"mgp_func_result_set_value or the error by using mgp_func_result_set_error_msg.",
|
||||
fully_qualified_name);
|
||||
}
|
||||
|
||||
return {*(maybe_res.value), ctx.memory};
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::function<TypedValue(const TypedValue *, int64_t, const FunctionContext &ctx)> NameToFunction(
|
||||
@@ -1309,14 +1259,6 @@ std::function<TypedValue(const TypedValue *, int64_t, const FunctionContext &ctx
|
||||
if (function_name == "LOCALDATETIME") return LocalDateTime;
|
||||
if (function_name == "DURATION") return Duration;
|
||||
|
||||
const auto &maybe_found =
|
||||
procedure::FindFunction(procedure::gModuleRegistry, function_name, utils::NewDeleteResource());
|
||||
|
||||
if (maybe_found) {
|
||||
const auto *func = (*maybe_found).second;
|
||||
return UserFunction(*func, function_name);
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
|
||||
@@ -716,7 +716,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
|
||||
template <class TRecordAccessor>
|
||||
storage::PropertyValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
|
||||
storage::PropertyValue GetProperty(const TRecordAccessor &record_accessor, const std::string_view &name) {
|
||||
auto maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
|
||||
if (maybe_prop.HasError() && maybe_prop.GetError() == storage::Error::NONEXISTENT_OBJECT) {
|
||||
// This is a very nasty and temporary hack in order to make MERGE work.
|
||||
|
||||
@@ -160,8 +160,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
void RegisterReplica(const std::string &name, const std::string &socket_address,
|
||||
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout,
|
||||
const std::chrono::seconds replica_check_frequency) override {
|
||||
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout) override {
|
||||
if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
|
||||
// replica can't register another replica
|
||||
throw QueryRuntimeException("Replica can't register another replica!");
|
||||
@@ -183,9 +182,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::kDefaultReplicationPort);
|
||||
if (maybe_ip_and_port) {
|
||||
auto [ip, port] = *maybe_ip_and_port;
|
||||
auto ret = db_->RegisterReplica(
|
||||
name, {std::move(ip), port}, repl_mode,
|
||||
{.timeout = timeout, .replica_check_frequency = replica_check_frequency, .ssl = std::nullopt});
|
||||
auto ret =
|
||||
db_->RegisterReplica(name, {std::move(ip), port}, repl_mode, {.timeout = timeout, .ssl = std::nullopt});
|
||||
if (ret.HasError()) {
|
||||
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
|
||||
}
|
||||
@@ -232,25 +230,6 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
replica.timeout = *repl_info.timeout;
|
||||
}
|
||||
|
||||
replica.current_timestamp_of_replica = repl_info.timestamp_info.current_timestamp_of_replica;
|
||||
replica.current_number_of_timestamp_behind_master =
|
||||
repl_info.timestamp_info.current_number_of_timestamp_behind_master;
|
||||
|
||||
switch (repl_info.state) {
|
||||
case storage::replication::ReplicaState::READY:
|
||||
replica.state = ReplicationQuery::ReplicaState::READY;
|
||||
break;
|
||||
case storage::replication::ReplicaState::REPLICATING:
|
||||
replica.state = ReplicationQuery::ReplicaState::REPLICATING;
|
||||
break;
|
||||
case storage::replication::ReplicaState::RECOVERY:
|
||||
replica.state = ReplicationQuery::ReplicaState::RECOVERY;
|
||||
break;
|
||||
case storage::replication::ReplicaState::INVALID:
|
||||
replica.state = ReplicationQuery::ReplicaState::INVALID;
|
||||
break;
|
||||
}
|
||||
|
||||
return replica;
|
||||
};
|
||||
|
||||
@@ -282,8 +261,6 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
|
||||
std::string rolename = auth_query->role_;
|
||||
std::string user_or_role = auth_query->user_or_role_;
|
||||
std::vector<AuthQuery::Privilege> privileges = auth_query->privileges_;
|
||||
std::vector<std::string> labels = auth_query->labels_;
|
||||
// std::vector<storage::LabelId> labels = NamesToLabels(labels, db_accessor);
|
||||
auto password = EvaluateOptionalExpression(auth_query->password_, &evaluator);
|
||||
|
||||
Callback callback;
|
||||
@@ -298,8 +275,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
|
||||
|
||||
if (license_check_result.HasError() && enterprise_only_methods.contains(auth_query->action_)) {
|
||||
throw utils::BasicException(
|
||||
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "advanced authentication
|
||||
features"));
|
||||
utils::license::LicenseCheckErrorToString(license_check_result.GetError(), "advanced authentication features"));
|
||||
}
|
||||
|
||||
switch (auth_query->action_) {
|
||||
@@ -314,7 +290,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
|
||||
// If the license is not valid we create users with admin access
|
||||
if (!valid_enterprise_license) {
|
||||
spdlog::warn("Granting all the privileges to {}.", username);
|
||||
auth->GrantPrivilege(username, kPrivilegesAll, {});
|
||||
auth->GrantPrivilege(username, kPrivilegesAll);
|
||||
}
|
||||
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
@@ -389,20 +365,20 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
|
||||
};
|
||||
return callback;
|
||||
case AuthQuery::Action::GRANT_PRIVILEGE:
|
||||
callback.fn = [auth, user_or_role, privileges, labels] {
|
||||
auth->GrantPrivilege(user_or_role, privileges, labels);
|
||||
callback.fn = [auth, user_or_role, privileges] {
|
||||
auth->GrantPrivilege(user_or_role, privileges);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
};
|
||||
return callback;
|
||||
case AuthQuery::Action::DENY_PRIVILEGE:
|
||||
callback.fn = [auth, user_or_role, privileges, labels] {
|
||||
auth->DenyPrivilege(user_or_role, privileges, labels);
|
||||
callback.fn = [auth, user_or_role, privileges] {
|
||||
auth->DenyPrivilege(user_or_role, privileges);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
};
|
||||
return callback;
|
||||
case AuthQuery::Action::REVOKE_PRIVILEGE: {
|
||||
callback.fn = [auth, user_or_role, privileges, labels] {
|
||||
auth->RevokePrivilege(user_or_role, privileges, labels);
|
||||
callback.fn = [auth, user_or_role, privileges] {
|
||||
auth->RevokePrivilege(user_or_role, privileges);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
};
|
||||
return callback;
|
||||
@@ -472,7 +448,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
return callback;
|
||||
}
|
||||
case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: {
|
||||
callback.header = {"replication role"};
|
||||
callback.header = {"replication mode"};
|
||||
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}] {
|
||||
auto mode = handler.ShowReplicationRole();
|
||||
switch (mode) {
|
||||
@@ -491,28 +467,21 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
const auto &sync_mode = repl_query->sync_mode_;
|
||||
auto socket_address = repl_query->socket_address_->Accept(evaluator);
|
||||
auto timeout = EvaluateOptionalExpression(repl_query->timeout_, &evaluator);
|
||||
const auto replica_check_frequency = interpreter_context->config.replication_replica_check_frequency;
|
||||
std::optional<double> maybe_timeout;
|
||||
if (timeout.IsDouble()) {
|
||||
maybe_timeout = timeout.ValueDouble();
|
||||
} else if (timeout.IsInt()) {
|
||||
maybe_timeout = static_cast<double>(timeout.ValueInt());
|
||||
}
|
||||
if (maybe_timeout && *maybe_timeout <= 0.0) {
|
||||
throw utils::BasicException("Parameter TIMEOUT must be strictly greater than 0.");
|
||||
}
|
||||
|
||||
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name, socket_address, sync_mode,
|
||||
maybe_timeout, replica_check_frequency]() mutable {
|
||||
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, maybe_timeout,
|
||||
replica_check_frequency);
|
||||
maybe_timeout]() mutable {
|
||||
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, maybe_timeout);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
};
|
||||
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::REGISTER_REPLICA,
|
||||
fmt::format("Replica {} is registered.", repl_query->replica_name_));
|
||||
return callback;
|
||||
}
|
||||
|
||||
case ReplicationQuery::Action::DROP_REPLICA: {
|
||||
const auto &name = repl_query->replica_name_;
|
||||
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, name]() mutable {
|
||||
@@ -523,15 +492,8 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
fmt::format("Replica {} is dropped.", repl_query->replica_name_));
|
||||
return callback;
|
||||
}
|
||||
|
||||
case ReplicationQuery::Action::SHOW_REPLICAS: {
|
||||
callback.header = {"name",
|
||||
"socket_address",
|
||||
"sync_mode",
|
||||
"timeout",
|
||||
"current_timestamp_of_replica",
|
||||
"number_of_timestamp_behind_master",
|
||||
"state"};
|
||||
callback.header = {"name", "socket_address", "sync_mode", "timeout"};
|
||||
callback.fn = [handler = ReplQueryHandler{interpreter_context->db}, replica_nfields = callback.header.size()] {
|
||||
const auto &replicas = handler.ShowReplicas();
|
||||
auto typed_replicas = std::vector<std::vector<TypedValue>>{};
|
||||
@@ -542,7 +504,6 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
|
||||
typed_replica.emplace_back(TypedValue(replica.name));
|
||||
typed_replica.emplace_back(TypedValue(replica.socket_address));
|
||||
|
||||
switch (replica.sync_mode) {
|
||||
case ReplicationQuery::SyncMode::SYNC:
|
||||
typed_replica.emplace_back(TypedValue("sync"));
|
||||
@@ -551,32 +512,13 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
typed_replica.emplace_back(TypedValue("async"));
|
||||
break;
|
||||
}
|
||||
|
||||
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.sync_mode)));
|
||||
if (replica.timeout) {
|
||||
typed_replica.emplace_back(TypedValue(*replica.timeout));
|
||||
} else {
|
||||
typed_replica.emplace_back(TypedValue());
|
||||
}
|
||||
|
||||
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.current_timestamp_of_replica)));
|
||||
typed_replica.emplace_back(
|
||||
TypedValue(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master)));
|
||||
|
||||
switch (replica.state) {
|
||||
case ReplicationQuery::ReplicaState::READY:
|
||||
typed_replica.emplace_back(TypedValue("ready"));
|
||||
break;
|
||||
case ReplicationQuery::ReplicaState::REPLICATING:
|
||||
typed_replica.emplace_back(TypedValue("replicating"));
|
||||
break;
|
||||
case ReplicationQuery::ReplicaState::RECOVERY:
|
||||
typed_replica.emplace_back(TypedValue("recovery"));
|
||||
break;
|
||||
case ReplicationQuery::ReplicaState::INVALID:
|
||||
typed_replica.emplace_back(TypedValue("invalid"));
|
||||
break;
|
||||
}
|
||||
|
||||
typed_replicas.emplace_back(std::move(typed_replica));
|
||||
}
|
||||
return typed_replicas;
|
||||
@@ -612,7 +554,7 @@ std::vector<std::string> EvaluateTopicNames(ExpressionEvaluator &evaluator,
|
||||
Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, ExpressionEvaluator &evaluator,
|
||||
InterpreterContext *interpreter_context,
|
||||
const std::string *username) {
|
||||
static constexpr std::string_view kDefaultConsumerGroup = "mg_consumer";
|
||||
constexpr std::string_view kDefaultConsumerGroup = "mg_consumer";
|
||||
std::string consumer_group{stream_query->consumer_group_.empty() ? kDefaultConsumerGroup
|
||||
: stream_query->consumer_group_};
|
||||
|
||||
@@ -710,26 +652,12 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters ¶mete
|
||||
return callback;
|
||||
}
|
||||
case StreamQuery::Action::START_STREAM: {
|
||||
const auto batch_limit = GetOptionalValue<int64_t>(stream_query->batch_limit_, evaluator);
|
||||
const auto timeout = GetOptionalValue<std::chrono::milliseconds>(stream_query->timeout_, evaluator);
|
||||
|
||||
if (batch_limit.has_value()) {
|
||||
if (batch_limit.value() < 0) {
|
||||
throw utils::BasicException("Parameter BATCH_LIMIT cannot hold negative value");
|
||||
}
|
||||
|
||||
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_, batch_limit, timeout]() {
|
||||
interpreter_context->streams.StartWithLimit(stream_name, static_cast<uint64_t>(batch_limit.value()), timeout);
|
||||
return std::vector<std::vector<TypedValue>>{};
|
||||
};
|
||||
} else {
|
||||
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() {
|
||||
interpreter_context->streams.Start(stream_name);
|
||||
return std::vector<std::vector<TypedValue>>{};
|
||||
};
|
||||
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::START_STREAM,
|
||||
fmt::format("Started stream {}.", stream_query->stream_name_));
|
||||
}
|
||||
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_]() {
|
||||
interpreter_context->streams.Start(stream_name);
|
||||
return std::vector<std::vector<TypedValue>>{};
|
||||
};
|
||||
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::START_STREAM,
|
||||
fmt::format("Started stream {}.", stream_query->stream_name_));
|
||||
return callback;
|
||||
}
|
||||
case StreamQuery::Action::START_ALL_STREAMS: {
|
||||
@@ -798,16 +726,10 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters ¶mete
|
||||
return callback;
|
||||
}
|
||||
case StreamQuery::Action::CHECK_STREAM: {
|
||||
callback.header = {"queries", "raw messages"};
|
||||
|
||||
const auto batch_limit = GetOptionalValue<int64_t>(stream_query->batch_limit_, evaluator);
|
||||
if (batch_limit.has_value() && batch_limit.value() < 0) {
|
||||
throw utils::BasicException("Parameter BATCH_LIMIT cannot hold negative value");
|
||||
}
|
||||
|
||||
callback.header = {"query", "parameters"};
|
||||
callback.fn = [interpreter_context, stream_name = stream_query->stream_name_,
|
||||
timeout = GetOptionalValue<std::chrono::milliseconds>(stream_query->timeout_, evaluator),
|
||||
batch_limit]() mutable {
|
||||
batch_limit = GetOptionalValue<int64_t>(stream_query->batch_limit_, evaluator)]() mutable {
|
||||
return interpreter_context->streams.Check(stream_name, timeout, batch_limit);
|
||||
};
|
||||
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::CHECK_STREAM,
|
||||
@@ -977,7 +899,7 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
|
||||
// Set up temporary memory for a single Pull. Initial memory comes from the
|
||||
// stack. 256 KiB should fit on the stack and should be more than enough for a
|
||||
// single `Pull`.
|
||||
static constexpr size_t stack_size = 256UL * 1024UL;
|
||||
constexpr size_t stack_size = 256 * 1024;
|
||||
char stack_data[stack_size];
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
|
||||
@@ -47,7 +47,7 @@ extern const Event FailedQuery;
|
||||
|
||||
namespace memgraph::query {
|
||||
|
||||
inline constexpr size_t kExecutionMemoryBlockSize = 1UL * 1024UL * 1024UL;
|
||||
static constexpr size_t kExecutionMemoryBlockSize = 1U * 1024U * 1024U;
|
||||
|
||||
class AuthQueryHandler {
|
||||
public:
|
||||
@@ -99,16 +99,14 @@ class AuthQueryHandler {
|
||||
virtual std::vector<std::vector<TypedValue>> GetPrivileges(const std::string &user_or_role) = 0;
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
virtual void GrantPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels) = 0;
|
||||
virtual void GrantPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges) = 0;
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
virtual void DenyPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels) = 0;
|
||||
virtual void DenyPrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges) = 0;
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
virtual void RevokePrivilege(const std::string &user_or_role, const std::vector<AuthQuery::Privilege> &privileges,
|
||||
const std::vector<std::string> &labels) = 0;
|
||||
virtual void RevokePrivilege(const std::string &user_or_role,
|
||||
const std::vector<AuthQuery::Privilege> &privileges) = 0;
|
||||
};
|
||||
|
||||
enum class QueryHandlerResult { COMMIT, ABORT, NOTHING };
|
||||
@@ -129,9 +127,6 @@ class ReplicationQueryHandler {
|
||||
std::string socket_address;
|
||||
ReplicationQuery::SyncMode sync_mode;
|
||||
std::optional<double> timeout;
|
||||
uint64_t current_timestamp_of_replica;
|
||||
uint64_t current_number_of_timestamp_behind_master;
|
||||
ReplicationQuery::ReplicaState state;
|
||||
};
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
@@ -142,8 +137,7 @@ class ReplicationQueryHandler {
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
virtual void RegisterReplica(const std::string &name, const std::string &socket_address,
|
||||
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout,
|
||||
const std::chrono::seconds replica_check_frequency) = 0;
|
||||
const ReplicationQuery::SyncMode sync_mode, const std::optional<double> timeout) = 0;
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
virtual void DropReplica(const std::string &replica_name) = 0;
|
||||
|
||||
@@ -61,7 +61,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
|
||||
static constexpr double kFilter{1.5};
|
||||
static constexpr double kEdgeUniquenessFilter{1.5};
|
||||
static constexpr double kUnwind{1.3};
|
||||
static constexpr double kForeach{1.0};
|
||||
};
|
||||
|
||||
struct CardParam {
|
||||
@@ -73,7 +72,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
|
||||
|
||||
struct MiscParam {
|
||||
static constexpr double kUnwindNoLiteral{10.0};
|
||||
static constexpr double kForeachNoLiteral{10.0};
|
||||
};
|
||||
|
||||
using HierarchicalLogicalOperatorVisitor::PostVisit;
|
||||
@@ -195,23 +193,6 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(Foreach &foreach) override {
|
||||
// Foreach cost depends both on the number elements in the list that get unwound
|
||||
// as well as the total clauses that get called for each unwounded element.
|
||||
// First estimate cardinality and then increment the cost.
|
||||
|
||||
double foreach_elements{0};
|
||||
if (auto *literal = utils::Downcast<query::ListLiteral>(foreach.expression_)) {
|
||||
foreach_elements = literal->elements_.size();
|
||||
} else {
|
||||
foreach_elements = MiscParam::kForeachNoLiteral;
|
||||
}
|
||||
|
||||
cardinality_ *= foreach_elements;
|
||||
IncrementCost(CostParam::kForeach);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Visit(Once &) override { return true; }
|
||||
|
||||
auto cost() const { return cost_; }
|
||||
|
||||
@@ -45,7 +45,6 @@
|
||||
#include "utils/fnv.hpp"
|
||||
#include "utils/likely.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/pmr/unordered_map.hpp"
|
||||
#include "utils/pmr/unordered_set.hpp"
|
||||
#include "utils/pmr/vector.hpp"
|
||||
@@ -106,7 +105,6 @@ extern const Event DistinctOperator;
|
||||
extern const Event UnionOperator;
|
||||
extern const Event CartesianOperator;
|
||||
extern const Event CallProcedureOperator;
|
||||
extern const Event ForeachOperator;
|
||||
} // namespace EventCounter
|
||||
|
||||
namespace memgraph::query::plan {
|
||||
@@ -3697,7 +3695,7 @@ std::unordered_map<std::string, int64_t> CallProcedure::GetAndResetCounters() {
|
||||
|
||||
namespace {
|
||||
|
||||
void CallCustomProcedure(const std::string_view fully_qualified_procedure_name, const mgp_proc &proc,
|
||||
void CallCustomProcedure(const std::string_view &fully_qualified_procedure_name, const mgp_proc &proc,
|
||||
const std::vector<Expression *> &args, mgp_graph &graph, ExpressionEvaluator *evaluator,
|
||||
utils::MemoryResource *memory, std::optional<size_t> memory_limit, mgp_result *result) {
|
||||
static_assert(std::uses_allocator_v<mgp_value, utils::Allocator<mgp_value>>,
|
||||
@@ -3705,12 +3703,46 @@ void CallCustomProcedure(const std::string_view fully_qualified_procedure_name,
|
||||
"containers aware of that");
|
||||
// Build and type check procedure arguments.
|
||||
mgp_list proc_args(memory);
|
||||
std::vector<TypedValue> args_list;
|
||||
args_list.reserve(args.size());
|
||||
for (auto *expression : args) {
|
||||
args_list.emplace_back(expression->Accept(*evaluator));
|
||||
proc_args.elems.reserve(args.size());
|
||||
if (args.size() < proc.args.size() ||
|
||||
// Rely on `||` short circuit so we can avoid potential overflow of
|
||||
// proc.args.size() + proc.opt_args.size() by subtracting.
|
||||
(args.size() - proc.args.size() > proc.opt_args.size())) {
|
||||
if (proc.args.empty() && proc.opt_args.empty()) {
|
||||
throw QueryRuntimeException("'{}' requires no arguments.", fully_qualified_procedure_name);
|
||||
} else if (proc.opt_args.empty()) {
|
||||
throw QueryRuntimeException("'{}' requires exactly {} {}.", fully_qualified_procedure_name, proc.args.size(),
|
||||
proc.args.size() == 1U ? "argument" : "arguments");
|
||||
} else {
|
||||
throw QueryRuntimeException("'{}' requires between {} and {} arguments.", fully_qualified_procedure_name,
|
||||
proc.args.size(), proc.args.size() + proc.opt_args.size());
|
||||
}
|
||||
}
|
||||
for (size_t i = 0; i < args.size(); ++i) {
|
||||
auto arg = args[i]->Accept(*evaluator);
|
||||
std::string_view name;
|
||||
const query::procedure::CypherType *type{nullptr};
|
||||
if (proc.args.size() > i) {
|
||||
name = proc.args[i].first;
|
||||
type = proc.args[i].second;
|
||||
} else {
|
||||
MG_ASSERT(proc.opt_args.size() > i - proc.args.size());
|
||||
name = std::get<0>(proc.opt_args[i - proc.args.size()]);
|
||||
type = std::get<1>(proc.opt_args[i - proc.args.size()]);
|
||||
}
|
||||
if (!type->SatisfiesType(arg)) {
|
||||
throw QueryRuntimeException("'{}' argument named '{}' at position {} must be of type {}.",
|
||||
fully_qualified_procedure_name, name, i, type->GetPresentableName());
|
||||
}
|
||||
proc_args.elems.emplace_back(std::move(arg), &graph);
|
||||
}
|
||||
// Fill missing optional arguments with their default values.
|
||||
MG_ASSERT(args.size() >= proc.args.size());
|
||||
size_t passed_in_opt_args = args.size() - proc.args.size();
|
||||
MG_ASSERT(passed_in_opt_args <= proc.opt_args.size());
|
||||
for (size_t i = passed_in_opt_args; i < proc.opt_args.size(); ++i) {
|
||||
proc_args.elems.emplace_back(std::get<2>(proc.opt_args[i]), &graph);
|
||||
}
|
||||
procedure::ConstructArguments(args_list, proc, fully_qualified_procedure_name, proc_args, graph);
|
||||
if (memory_limit) {
|
||||
SPDLOG_INFO("Running '{}' with memory limit of {}", fully_qualified_procedure_name,
|
||||
utils::GetReadableSize(*memory_limit));
|
||||
@@ -3798,7 +3830,7 @@ class CallProcedureCursor : public Cursor {
|
||||
// generator like procedures which yield a new result on each invocation.
|
||||
auto *memory = context.evaluation_context.memory;
|
||||
auto memory_limit = EvaluateMemoryLimit(&evaluator, self_->memory_limit_, self_->memory_scale_);
|
||||
auto graph = mgp_graph::WritableGraph(*context.db_accessor, graph_view, context);
|
||||
mgp_graph graph{context.db_accessor, graph_view, &context};
|
||||
CallCustomProcedure(self_->procedure_name_, *proc, self_->arguments_, graph, &evaluator, memory, memory_limit,
|
||||
&result_);
|
||||
|
||||
@@ -3992,85 +4024,4 @@ UniqueCursorPtr LoadCsv::MakeCursor(utils::MemoryResource *mem) const {
|
||||
return MakeUniqueCursorPtr<LoadCsvCursor>(mem, this, mem);
|
||||
};
|
||||
|
||||
class ForeachCursor : public Cursor {
|
||||
public:
|
||||
explicit ForeachCursor(const Foreach &foreach, utils::MemoryResource *mem)
|
||||
: loop_variable_symbol_(foreach.loop_variable_symbol_),
|
||||
input_(foreach.input_->MakeCursor(mem)),
|
||||
updates_(foreach.update_clauses_->MakeCursor(mem)),
|
||||
expression(foreach.expression_) {}
|
||||
|
||||
bool Pull(Frame &frame, ExecutionContext &context) override {
|
||||
SCOPED_PROFILE_OP(op_name_);
|
||||
|
||||
if (!input_->Pull(frame, context)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor,
|
||||
storage::View::NEW);
|
||||
TypedValue expr_result = expression->Accept(evaluator);
|
||||
|
||||
if (expr_result.IsNull()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!expr_result.IsList()) {
|
||||
throw QueryRuntimeException("FOREACH expression must resolve to a list, but got '{}'.", expr_result.type());
|
||||
}
|
||||
|
||||
const auto &cache_ = expr_result.ValueList();
|
||||
for (const auto &index : cache_) {
|
||||
frame[loop_variable_symbol_] = index;
|
||||
while (updates_->Pull(frame, context)) {
|
||||
}
|
||||
ResetUpdates();
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void Shutdown() override { input_->Shutdown(); }
|
||||
|
||||
void ResetUpdates() { updates_->Reset(); }
|
||||
|
||||
void Reset() override {
|
||||
input_->Reset();
|
||||
ResetUpdates();
|
||||
}
|
||||
|
||||
private:
|
||||
const Symbol loop_variable_symbol_;
|
||||
const UniqueCursorPtr input_;
|
||||
const UniqueCursorPtr updates_;
|
||||
Expression *expression;
|
||||
const char *op_name_{"Foreach"};
|
||||
};
|
||||
|
||||
Foreach::Foreach(std::shared_ptr<LogicalOperator> input, std::shared_ptr<LogicalOperator> updates, Expression *expr,
|
||||
Symbol loop_variable_symbol)
|
||||
: input_(input ? std::move(input) : std::make_shared<Once>()),
|
||||
update_clauses_(std::move(updates)),
|
||||
expression_(expr),
|
||||
loop_variable_symbol_(loop_variable_symbol) {}
|
||||
|
||||
UniqueCursorPtr Foreach::MakeCursor(utils::MemoryResource *mem) const {
|
||||
EventCounter::IncrementCounter(EventCounter::ForeachOperator);
|
||||
return MakeUniqueCursorPtr<ForeachCursor>(mem, *this, mem);
|
||||
}
|
||||
|
||||
std::vector<Symbol> Foreach::ModifiedSymbols(const SymbolTable &table) const {
|
||||
auto symbols = input_->ModifiedSymbols(table);
|
||||
symbols.emplace_back(loop_variable_symbol_);
|
||||
return symbols;
|
||||
}
|
||||
|
||||
bool Foreach::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
|
||||
if (visitor.PreVisit(*this)) {
|
||||
input_->Accept(visitor);
|
||||
update_clauses_->Accept(visitor);
|
||||
}
|
||||
return visitor.PostVisit(*this);
|
||||
}
|
||||
|
||||
} // namespace memgraph::query::plan
|
||||
|
||||
@@ -131,7 +131,6 @@ class Union;
|
||||
class Cartesian;
|
||||
class CallProcedure;
|
||||
class LoadCsv;
|
||||
class Foreach;
|
||||
|
||||
using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
|
||||
Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
|
||||
@@ -140,7 +139,7 @@ using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
|
||||
Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
|
||||
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
|
||||
EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
|
||||
Optional, Unwind, Distinct, Union, Cartesian, CallProcedure, LoadCsv, Foreach>;
|
||||
Optional, Unwind, Distinct, Union, Cartesian, CallProcedure, LoadCsv>;
|
||||
|
||||
using LogicalOperatorLeafVisitor = utils::LeafVisitor<Once>;
|
||||
|
||||
@@ -2262,42 +2261,6 @@ at once. Instead, each call of the callback should return a single row of the ta
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
(lcp:define-class foreach (logical-operator)
|
||||
((input "std::shared_ptr<LogicalOperator>" :scope :public
|
||||
:slk-save #'slk-save-operator-pointer
|
||||
:slk-load #'slk-load-operator-pointer)
|
||||
(update-clauses "std::shared_ptr<LogicalOperator>" :scope :public
|
||||
:slk-save #'slk-save-operator-pointer
|
||||
:slk-load #'slk-load-operator-pointer)
|
||||
(expression "Expression *" :scope :public
|
||||
:slk-save #'slk-save-ast-pointer
|
||||
:slk-load (slk-load-ast-pointer "Expression"))
|
||||
(loop-variable-symbol "Symbol" :scope :public))
|
||||
|
||||
(:documentation
|
||||
"Iterates over a collection of elements and applies one or more update
|
||||
clauses.
|
||||
")
|
||||
(:public
|
||||
#>cpp
|
||||
Foreach() = default;
|
||||
Foreach(std::shared_ptr<LogicalOperator> input,
|
||||
std::shared_ptr<LogicalOperator> updates,
|
||||
Expression *named_expr,
|
||||
Symbol loop_variable_symbol);
|
||||
|
||||
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
|
||||
UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override;
|
||||
std::vector<Symbol> ModifiedSymbols(const SymbolTable &) const override;
|
||||
bool HasSingleInput() const override { return true; }
|
||||
std::shared_ptr<LogicalOperator> input() const override { return input_; }
|
||||
void set_input(std::shared_ptr<LogicalOperator> input) override {
|
||||
input_ = std::move(input);
|
||||
}
|
||||
cpp<#)
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
(lcp:pop-namespace) ;; plan
|
||||
(lcp:pop-namespace) ;; query
|
||||
(lcp:pop-namespace) ;; memgraph
|
||||
|
||||
@@ -17,10 +17,8 @@
|
||||
#include <variant>
|
||||
|
||||
#include "query/exceptions.hpp"
|
||||
#include "query/frontend/ast/ast.hpp"
|
||||
#include "query/frontend/ast/ast_visitor.hpp"
|
||||
#include "query/plan/preprocess.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
|
||||
namespace memgraph::query::plan {
|
||||
|
||||
@@ -528,18 +526,6 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
|
||||
// as `expr1 < n.prop AND n.prop < expr2`.
|
||||
}
|
||||
|
||||
static void ParseForeach(query::Foreach &foreach, SingleQueryPart &query_part, AstStorage &storage,
|
||||
SymbolTable &symbol_table) {
|
||||
for (auto *clause : foreach.clauses_) {
|
||||
if (auto *merge = utils::Downcast<query::Merge>(clause)) {
|
||||
query_part.merge_matching.emplace_back(Matching{});
|
||||
AddMatching({merge->pattern_}, nullptr, symbol_table, storage, query_part.merge_matching.back());
|
||||
} else if (auto *nested = utils::Downcast<query::Foreach>(clause)) {
|
||||
ParseForeach(*nested, query_part, storage, symbol_table);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Converts a Query to multiple QueryParts. In the process new Ast nodes may be
|
||||
// created, e.g. filter expressions.
|
||||
std::vector<SingleQueryPart> CollectSingleQueryParts(SymbolTable &symbol_table, AstStorage &storage,
|
||||
@@ -560,8 +546,6 @@ std::vector<SingleQueryPart> CollectSingleQueryParts(SymbolTable &symbol_table,
|
||||
if (auto *merge = utils::Downcast<query::Merge>(clause)) {
|
||||
query_part->merge_matching.emplace_back(Matching{});
|
||||
AddMatching({merge->pattern_}, nullptr, symbol_table, storage, query_part->merge_matching.back());
|
||||
} else if (auto *foreach = utils::Downcast<query::Foreach>(clause)) {
|
||||
ParseForeach(*foreach, *query_part, storage, symbol_table);
|
||||
} else if (utils::IsSubtype(*clause, With::kType) || utils::IsSubtype(*clause, query::Unwind::kType) ||
|
||||
utils::IsSubtype(*clause, query::CallProcedure::kType) ||
|
||||
utils::IsSubtype(*clause, query::LoadCsv::kType)) {
|
||||
|
||||
@@ -306,10 +306,6 @@ struct Matching {
|
||||
/// will produce the second `merge_matching` element. This way, if someone
|
||||
/// traverses `remaining_clauses`, the order of appearance of `Merge` clauses is
|
||||
/// in the same order as their respective `merge_matching` elements.
|
||||
/// An exception to the above rule is Foreach. Its update clauses will not be contained in
|
||||
/// the `remaining_clauses`, but rather inside the foreach itself. The order guarantee is not
|
||||
/// violated because the update clauses of the foreach are immediately processed in
|
||||
/// the `RuleBasedPlanner` as if as they were pushed into the `remaining_clauses`.
|
||||
struct SingleQueryPart {
|
||||
/// @brief All `MATCH` clauses merged into one @c Matching.
|
||||
Matching matching;
|
||||
@@ -324,10 +320,6 @@ struct SingleQueryPart {
|
||||
///
|
||||
/// Since @c Merge is contained in `remaining_clauses`, this vector contains
|
||||
/// matching in the same order as @c Merge appears.
|
||||
//
|
||||
/// Foreach @c does not violate this gurantee. However, update clauses are not stored
|
||||
/// in the `remaining_clauses` but rather in the `Foreach` itself and are guranteed
|
||||
/// to be processed in the same order by the semantics of the `RuleBasedPlanner`.
|
||||
std::vector<Matching> merge_matching{};
|
||||
/// @brief All the remaining clauses (without @c Match).
|
||||
std::vector<Clause *> remaining_clauses{};
|
||||
|
||||
@@ -241,12 +241,6 @@ bool PlanPrinter::PreVisit(query::plan::Cartesian &op) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PlanPrinter::PreVisit(query::plan::Foreach &op) {
|
||||
WithPrintLn([](auto &out) { out << "* Foreach"; });
|
||||
Branch(*op.update_clauses_);
|
||||
op.input_->Accept(*this);
|
||||
return false;
|
||||
}
|
||||
#undef PRE_VISIT
|
||||
|
||||
bool PlanPrinter::DefaultPreVisit() {
|
||||
@@ -889,21 +883,6 @@ bool PlanToJsonVisitor::PreVisit(Cartesian &op) {
|
||||
output_ = std::move(self);
|
||||
return false;
|
||||
}
|
||||
bool PlanToJsonVisitor::PreVisit(Foreach &op) {
|
||||
json self;
|
||||
self["name"] = "Foreach";
|
||||
self["loop_variable_symbol"] = ToJson(op.loop_variable_symbol_);
|
||||
self["expression"] = ToJson(op.expression_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
|
||||
op.update_clauses_->Accept(*this);
|
||||
self["update_clauses"] = PopOutput();
|
||||
|
||||
output_ = std::move(self);
|
||||
return false;
|
||||
}
|
||||
|
||||
} // namespace impl
|
||||
|
||||
|
||||
@@ -92,7 +92,6 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(Unwind &) override;
|
||||
bool PreVisit(CallProcedure &) override;
|
||||
bool PreVisit(LoadCsv &) override;
|
||||
bool PreVisit(Foreach &) override;
|
||||
|
||||
bool Visit(Once &) override;
|
||||
|
||||
@@ -205,7 +204,6 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(Union &) override;
|
||||
|
||||
bool PreVisit(Unwind &) override;
|
||||
bool PreVisit(Foreach &) override;
|
||||
bool PreVisit(CallProcedure &) override;
|
||||
bool PreVisit(LoadCsv &) override;
|
||||
|
||||
|
||||
@@ -79,11 +79,6 @@ bool ReadWriteTypeChecker::PreVisit(CallProcedure &op) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool ReadWriteTypeChecker::PreVisit([[maybe_unused]] Foreach &op) {
|
||||
UpdateType(RWType::RW);
|
||||
return false;
|
||||
}
|
||||
|
||||
#undef PRE_VISIT
|
||||
|
||||
bool ReadWriteTypeChecker::Visit(Once &op) { return false; }
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user