Compare commits

..

3 Commits

Author SHA1 Message Date
Marko Barišić
307c2e0881 Turn e2e tests back on for release build workflows (#1725) 2024-02-15 16:22:36 +01:00
Deda
b9b221cfe2 Update BSL license date 2024-02-15 15:50:41 +01:00
Marko Barišić
6698d48a82 Add rules for rc workflows (#1722) 2024-02-15 15:48:50 +01:00
232 changed files with 2133 additions and 5612 deletions

View File

@@ -3,6 +3,7 @@ name: Bug report
about: Create a report to help us improve
title: ""
labels: bug
assignees: gitbuda
---
**Memgraph version**

View File

@@ -336,6 +336,53 @@ jobs:
# multiple paths could be defined
build/logs
experimental_build_ha:
name: "High availability build"
runs-on: [self-hosted, Linux, X64, Diff]
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v4
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
source /opt/toolchain-v4/activate
./init
cd build
cmake -DCMAKE_BUILD_TYPE=Release -DMG_EXPERIMENTAL_HIGH_AVAILABILITY=ON ..
make -j$THREADS
- name: Run unit tests
run: |
source /opt/toolchain-v4/activate
cd build
ctest -R memgraph__unit --output-on-failure -j$THREADS
- name: Run e2e tests
if: false
run: |
cd tests
./setup.sh /opt/toolchain-v4/activate
source ve3/bin/activate_e2e
cd e2e
./run.sh "Coordinator"
./run.sh "Client initiated failover"
./run.sh "Uninitialized cluster"
- name: Save test data
uses: actions/upload-artifact@v4
if: always()
with:
name: "Test data(High availability build)"
path: |
# multiple paths could be defined
build/logs
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]

View File

@@ -1,162 +0,0 @@
name: Release build test
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: true
on:
workflow_dispatch:
inputs:
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
default: 'Release'
options:
- Release
- RelWithDebInfo
push:
branches:
- "release/**"
tags:
- "v*.*.*-rc*"
- "v*.*-rc*"
schedule:
# UTC
- cron: "0 22 * * *"
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
BUILD_TYPE: ${{ github.event.inputs.build_type || 'Release' }}
jobs:
Debian10:
uses: ./.github/workflows/release_debian10.yaml
with:
build_type: ${{ github.event.inputs.build_type || 'Release' }}
secrets: inherit
Ubuntu20_04:
uses: ./.github/workflows/release_ubuntu2004.yaml
with:
build_type: ${{ github.event.inputs.build_type || 'Release' }}
secrets: inherit
PackageDebian10:
if: github.ref_type == 'tag'
needs: [Debian10]
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-10 $BUILD_TYPE
- name: "Upload package"
uses: actions/upload-artifact@v4
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
PackageUbuntu20_04:
if: github.ref_type == 'tag'
needs: [Ubuntu20_04]
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04 $BUILD_TYPE
- name: "Upload package"
uses: actions/upload-artifact@v4
with:
name: ubuntu-22.04
path: build/output/ubuntu-22.04/memgraph*.deb
PackageUbuntu20_04_ARM:
if: github.ref_type == 'tag'
needs: [Ubuntu20_04]
runs-on: [self-hosted, DockerMgBuild, ARM64]
# M1 Mac mini is sometimes slower
timeout-minutes: 90
steps:
- name: "Set up repository"
uses: actions/checkout@v4
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04-arm $BUILD_TYPE
- name: "Upload package"
uses: actions/upload-artifact@v4
with:
name: ubuntu-22.04-aarch64
path: build/output/ubuntu-22.04-arm/memgraph*.deb
PackageDebian11:
if: github.ref_type == 'tag'
needs: [Debian10, Ubuntu20_04]
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v4
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 $BUILD_TYPE
- name: "Upload package"
uses: actions/upload-artifact@v4
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
PackageDebian11_ARM:
if: github.ref_type == 'tag'
needs: [Debian10, Ubuntu20_04]
runs-on: [self-hosted, DockerMgBuild, ARM64]
# M1 Mac mini is sometimes slower
timeout-minutes: 90
steps:
- name: "Set up repository"
uses: actions/checkout@v4
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm $BUILD_TYPE
- name: "Upload package"
uses: actions/upload-artifact@v4
with:
name: debian-11-aarch64
path: build/output/debian-11-arm/memgraph*.deb
PushToS3:
if: github.ref_type == 'tag'
needs: [PackageDebian10, PackageDebian11, PackageDebian11_ARM, PackageUbuntu20_04, PackageUbuntu20_04_ARM]
runs-on: ubuntu-latest
steps:
- name: Download artifacts
uses: actions/download-artifact@v4
with:
# name: # if name input parameter is not provided, all artifacts are downloaded
# and put in directories named after each one.
path: build/output/release
- name: Upload to S3
uses: jakejarvis/s3-sync-action@v0.5.1
env:
AWS_S3_BUCKET: "deps.memgraph.io"
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
AWS_REGION: "eu-west-1"
SOURCE_DIR: "build/output/release"
DEST_DIR: "memgraph-unofficial/${{ github.ref_name }}/"

View File

@@ -1,12 +1,9 @@
name: Release Debian 10
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: true
on:
workflow_call:
inputs:
build_type:
type: string
description: "Memgraph Build type. Default value is Release."
default: 'Release'
workflow_dispatch:
inputs:
build_type:
@@ -16,9 +13,16 @@ on:
options:
- Release
- RelWithDebInfo
push:
branches:
- "release/**"
tags:
- "v*.*.*-rc*"
- "v*.*-rc*"
schedule:
- cron: "0 22 * * *"
env:
OS: "Debian10"
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
@@ -115,7 +119,7 @@ jobs:
- name: Save code coverage
uses: actions/upload-artifact@v4
with:
name: "Code coverage(Coverage build)-${{ env.OS }}"
name: "Code coverage(Coverage build)"
path: tools/github/generated/code_coverage.tar.gz
debug_build:
@@ -169,7 +173,7 @@ jobs:
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v4
with:
name: "Code coverage(Debug build)-${{ env.OS }}"
name: "Code coverage(Debug build)"
path: tools/github/cppcheck_and_clang_format.txt
debug_integration_test:
@@ -246,7 +250,7 @@ jobs:
- name: Save enterprise DEB package
uses: actions/upload-artifact@v4
with:
name: "Enterprise DEB package-${{ env.OS}}"
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Run GQL Behave tests
@@ -259,7 +263,7 @@ jobs:
- name: Save quality assurance status
uses: actions/upload-artifact@v4
with:
name: "GQL Behave Status-${{ env.OS }}"
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
@@ -459,5 +463,5 @@ jobs:
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: "Jepsen Report-${{ env.OS }}"
name: "Jepsen Report"
path: tests/jepsen/Jepsen.tar.gz

View File

@@ -1,12 +1,9 @@
name: Release Ubuntu 20.04
concurrency:
group: ${{ github.workflow }}-${{ github.ref_name }}
cancel-in-progress: true
on:
workflow_call:
inputs:
build_type:
type: string
description: "Memgraph Build type. Default value is Release."
default: 'Release'
workflow_dispatch:
inputs:
build_type:
@@ -16,9 +13,16 @@ on:
options:
- Release
- RelWithDebInfo
push:
branches:
- "release/**"
tags:
- "v*.*.*-rc*"
- "v*.*-rc*"
schedule:
- cron: "0 22 * * *"
env:
OS: "Ubuntu 20.04"
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
@@ -111,7 +115,7 @@ jobs:
- name: Save code coverage
uses: actions/upload-artifact@v4
with:
name: "Code coverage(Coverage build)-${{ env.OS }}"
name: "Code coverage(Coverage build)"
path: tools/github/generated/code_coverage.tar.gz
debug_build:
@@ -165,7 +169,7 @@ jobs:
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v4
with:
name: "Code coverage(Debug build)-${{ env.OS }}"
name: "Code coverage(Debug build)"
path: tools/github/cppcheck_and_clang_format.txt
debug_integration_test:
@@ -242,7 +246,7 @@ jobs:
- name: Save enterprise DEB package
uses: actions/upload-artifact@v4
with:
name: "Enterprise DEB package-${{ env.OS }}"
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Run GQL Behave tests
@@ -255,7 +259,7 @@ jobs:
- name: Save quality assurance status
uses: actions/upload-artifact@v4
with:
name: "GQL Behave Status-${{ env.OS }}"
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html

View File

@@ -211,13 +211,8 @@ set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
# ** Static linking is allowed only for executables! **
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
# Use lld linker to speedup build and use less memory.
add_link_options(-fuse-ld=lld)
# NOTE: Moving to latest Clang (probably starting from 15), lld stopped to work
# without explicit link_directories call.
string(REPLACE ":" " " LD_LIBS $ENV{LD_LIBRARY_PATH})
separate_arguments(LD_LIBS)
link_directories(${LD_LIBS})
# Use lld linker to speedup build
add_link_options(-fuse-ld=lld) # TODO: use mold linker
# release flags
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
@@ -276,6 +271,18 @@ endif()
set(libs_dir ${CMAKE_SOURCE_DIR}/libs)
add_subdirectory(libs EXCLUDE_FROM_ALL)
option(MG_EXPERIMENTAL_HIGH_AVAILABILITY "Feature flag for experimental high availability" OFF)
if (NOT MG_ENTERPRISE AND MG_EXPERIMENTAL_HIGH_AVAILABILITY)
set(MG_EXPERIMENTAL_HIGH_AVAILABILITY OFF)
message(FATAL_ERROR "MG_EXPERIMENTAL_HIGH_AVAILABILITY can only be used with enterpise version of the code.")
endif ()
if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
add_compile_definitions(MG_EXPERIMENTAL_HIGH_AVAILABILITY)
endif ()
# Optional subproject configuration -------------------------------------------
option(TEST_COVERAGE "Generate coverage reports from running memgraph" OFF)
option(TOOLS "Build tools binaries" ON)
option(QUERY_MODULES "Build query modules containing custom procedures" ON)

View File

@@ -5,4 +5,11 @@
* GCC 11.2 (toolchain-v4) doesn't compile on Fedora 38, multiple definitions of enum issue
* spdlog 1.10/11 doesn't work with fmt 10.0.0
## [os](os/README.md)
## os
Under the `os` directory, you can find scripts to install all required system
dependencies on operating systems where Memgraph natively builds. The testing
script helps to see how to install all packages (in the case of a new package),
or make any adjustments in the overall system setup. Also, the testing script
helps check if Memgraph runs on a freshly installed operating system (with no
packages installed).

View File

@@ -1,8 +0,0 @@
# Memgraph Environment OSes
Under the this (`os`) directory, you can find scripts to install all required system
dependencies on operating systems where Memgraph natively builds. The testing
script helps to see how to install all packages (in the case of a new package),
or make any adjustments in the overall system setup. Also, the testing script
helps check if Memgraph runs on a freshly installed operating system (with no
packages installed).

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -7,7 +9,7 @@ check_operating_system "amzn-2"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
git gcc gcc-c++ make # generic build tools
gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
@@ -61,8 +63,6 @@ MEMGRAPH_BUILD_DEPS=(
cyrus-sasl-devel
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -61,8 +63,6 @@ MEMGRAPH_BUILD_DEPS=(
cyrus-sasl-devel
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -7,10 +9,8 @@ check_operating_system "centos-9"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
wget # used for archive download
coreutils-common gcc gcc-c++ make # generic build tools
# NOTE: Pure libcurl conflicts with libcurl-minimal
libcurl-devel # cmake build requires it
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
@@ -64,8 +64,6 @@ MEMGRAPH_BUILD_DEPS=(
cyrus-sasl-devel
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
@@ -125,9 +123,7 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
# --nobest is used because of libipt because we install custom versions
# because libipt-devel is not available on CentOS 9 Stream
yum update -y --nobest
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do

View File

@@ -1,10 +1,10 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
# IMPORTANT: Deprecated since memgraph v2.12.0.
check_operating_system "debian-10"
check_architecture "x86_64"

View File

@@ -1,10 +1,10 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
# IMPORTANT: Deprecated since memgraph v2.12.0.
check_operating_system "debian-11"
check_architecture "arm64" "aarch64"

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -59,8 +61,6 @@ MEMGRAPH_BUILD_DEPS=(
libsasl2-dev
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)

View File

@@ -1,134 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-12"
check_architecture "arm64" "aarch64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
curl # snappy
file # for libunwind
libssl-dev # for libevent
libgmp-dev
gperf # for proxygen
git # for fbthrift
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 liblzma5 python3 # for gdb
libcurl4 # for cmake
file # for CPack
libreadline8 # for cmake and llvm
libffi8 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
netcat # tests are using nc to wait for memgraph
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-7.0 golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-7.0 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb
apt-get update
apt-get install -y apt-transport-https dotnet-sdk-7.0
fi
continue
fi
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1,136 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "debian-12"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils gcc g++ build-essential make # generic build tools
wget # used for archive download
gnupg # used for archive signature verification
tar gzip bzip2 xz-utils unzip # used for archive unpacking
zlib1g-dev # zlib library used for all builds
libexpat1-dev libipt-dev libbabeltrace-dev liblzma-dev python3-dev texinfo # for gdb
libcurl4-openssl-dev # for cmake
libreadline-dev # for cmake and llvm
libffi-dev libxml2-dev # for llvm
libedit-dev libpcre2-dev libpcre3-dev automake bison # for swig
curl # snappy
file # for libunwind
libssl-dev # for libevent
libgmp-dev
gperf # for proxygen
git # for fbthrift
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz-utils # used for archive unpacking
zlib1g # zlib library used for all builds
libexpat1 libipt2 libbabeltrace1 liblzma5 python3 # for gdb
libcurl4 # for cmake
file # for CPack
libreadline8 # for cmake and llvm
libffi8 libxml2 # for llvm
libssl-dev # for libevent
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
libpython3-dev python3-dev # for query modules
libssl-dev
libseccomp-dev
netcat-traditional # tests are using nc to wait for memgraph
python3 virtualenv python3-virtualenv python3-pip # for qa, macro_benchmark and stress tests
python3-yaml # for the configuration generator
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless custom-maven3.9.3 # for driver tests
dotnet-sdk-7.0 golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
apt update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-7.0 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/12/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
dpkg -i packages-microsoft-prod.deb
apt-get update
apt-get install -y apt-transport-https dotnet-sdk-7.0
fi
continue
fi
apt install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1,10 +1,10 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
# IMPORTANT: Deprecated since memgraph v2.12.0.
check_operating_system "fedora-36"
check_architecture "x86_64"
@@ -27,7 +27,6 @@ TOOLCHAIN_BUILD_DEPS=(
libipt libipt-devel # intel
patch
perl # for openssl
git
)
TOOLCHAIN_RUN_DEPS=(

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -25,7 +27,6 @@ TOOLCHAIN_BUILD_DEPS=(
libipt libipt-devel # intel
patch
perl # for openssl
git
)
TOOLCHAIN_RUN_DEPS=(
@@ -57,16 +58,6 @@ MEMGRAPH_BUILD_DEPS=(
libtool # for protobuf code generation
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}

View File

@@ -1,117 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "fedora-39"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
coreutils-common gcc gcc-c++ make # generic build tools
wget # used for archive download
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel texinfo libbabeltrace-devel # for gdb
curl libcurl-devel # for cmake
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
perl # for openssl
git
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv python3-virtualenvwrapper python3-pyyaml nmap-ncat # for tests
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
if [ -v LD_LIBRARY_PATH ]; then
# On Fedora 38 yum/dnf and python11 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
fi
local missing=""
for pkg in $1; do
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
if [ -v OLD_LD_LIBRARY_PATH ]; then
echo "Restoring LD_LIBRARY_PATH..."
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests don't work without the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
dnf install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1,188 +0,0 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
# TODO(gitbuda): Rocky gets automatically updates -> figure out how to handle it.
check_operating_system "rocky-9.3"
check_architecture "x86_64"
TOOLCHAIN_BUILD_DEPS=(
wget # used for archive download
coreutils-common gcc gcc-c++ make # generic build tools
# NOTE: Pure libcurl conflicts with libcurl-minimal
libcurl-devel # cmake build requires it
gnupg2 # used for archive signature verification
tar gzip bzip2 xz unzip # used for archive unpacking
zlib-devel # zlib library used for all builds
expat-devel xz-devel python3-devel perl-Unicode-EastAsianWidth texinfo libbabeltrace-devel # for gdb
readline-devel # for cmake and llvm
libffi-devel libxml2-devel # for llvm
libedit-devel pcre-devel pcre2-devel automake bison # for swig
file
openssl-devel
gmp-devel
gperf
diffutils
libipt libipt-devel # intel
patch
)
TOOLCHAIN_RUN_DEPS=(
make # generic build tools
tar gzip bzip2 xz # used for archive unpacking
zlib # zlib library used for all builds
expat xz-libs python3 # for gdb
readline # for cmake and llvm
libffi libxml2 # for llvm
openssl-devel
perl # for openssl
)
MEMGRAPH_BUILD_DEPS=(
git # source code control
make cmake pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
python3-devel # for query modules
openssl-devel
libseccomp-devel
python3 python3-pip python3-virtualenv nmap-ncat # for qa, macro_benchmark and stress tests
#
# IMPORTANT: python3-yaml does NOT exist on CentOS
# Install it manually using `pip3 install PyYAML`
#
PyYAML # Package name here does not correspond to the yum package!
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang custom-golang1.18.9 # for driver tests
zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.3 # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
echo "$1"
}
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
if [ ! -f "/opt/apache-maven-3.9.3/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "python3-virtualenv" ]; then
continue
fi
if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root."
exit 1
fi
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.3 ]; then
install_custom_maven "3.9.3"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == perl-Unicode-EastAsianWidth ]; then
if ! dnf list installed perl-Unicode-EastAsianWidth >/dev/null 2>/dev/null; then
dnf install -y https://dl.rockylinux.org/pub/rocky/9/CRB/x86_64/os/Packages/p/perl-Unicode-EastAsianWidth-12.0-7.el9.noarch.rpm
fi
continue
fi
if [ "$pkg" == texinfo ]; then
if ! dnf list installed texinfo >/dev/null 2>/dev/null; then
dnf install -y https://dl.rockylinux.org/pub/rocky/9/CRB/x86_64/os/Packages/t/texinfo-6.7-15.el9.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libbabeltrace-devel ]; then
if ! dnf list installed libbabeltrace-devel >/dev/null 2>/dev/null; then
dnf install -y https://dl.rockylinux.org/pub/rocky/9/devel/x86_64/os/Packages/l/libbabeltrace-devel-1.5.8-10.el9.x86_64.rpm
fi
continue
fi
if [ "$pkg" == libipt-devel ]; then
if ! dnf list installed libipt-devel >/dev/null 2>/dev/null; then
dnf install -y https://dl.rockylinux.org/pub/rocky/9/devel/x86_64/os/Packages/l/libipt-devel-2.0.4-5.el9.x86_64.rpm
fi
continue
fi
if [ "$pkg" == PyYAML ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install --user PyYAML
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
fi
continue
fi
if [ "$pkg" == python3-virtualenv ]; then
if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
pip3 install virtualenv
pip3 install virtualenvwrapper
else # Running using sudo.
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenv"
sudo -H -u "$SUDO_USER" bash -c "pip3 install virtualenvwrapper"
fi
continue
fi
yum install -y "$pkg"
done
}
deps=$2"[*]"
"$1" "${!deps}"

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -18,10 +20,6 @@ MEMGRAPH_BUILD_DEPS=(
pkg
)
MEMGRAPH_TEST_DEPS=(
pkg
)
MEMGRAPH_RUN_DEPS=(
pkg
)

View File

@@ -1,10 +1,10 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
# IMPORTANT: Deprecated since memgraph v2.12.0.
check_operating_system "ubuntu-18.04"
check_architecture "x86_64"

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -58,8 +60,6 @@ MEMGRAPH_BUILD_DEPS=(
libsasl2-dev
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -58,8 +60,6 @@ MEMGRAPH_BUILD_DEPS=(
libsasl2-dev
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)

View File

@@ -1,5 +1,7 @@
#!/bin/bash
set -Eeuo pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
@@ -58,8 +60,6 @@ MEMGRAPH_BUILD_DEPS=(
libsasl2-dev
)
MEMGRAPH_TEST_DEPS="${MEMGRAPH_BUILD_DEPS[*]}"
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)

View File

@@ -2,4 +2,3 @@ archives
build
output
*.tar.gz
tmp_build.sh

View File

@@ -1,48 +0,0 @@
#!/bin/bash -e
# NOTE: Copy this under memgraph/environment/toolchain/vN/tmp_build.sh, edit and test.
pushd () { command pushd "$@" > /dev/null; }
popd () { command popd "$@" > /dev/null; }
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
CPUS=$( grep -c processor < /proc/cpuinfo )
cd "$DIR"
source "$DIR/../../util.sh"
DISTRO="$(operating_system)"
TOOLCHAIN_VERSION=5
NAME=toolchain-v$TOOLCHAIN_VERSION
PREFIX=/opt/$NAME
function log_tool_name () {
echo ""
echo ""
echo "#### $1 ####"
echo ""
echo ""
}
# HERE: Remove/clear dependencies from a given toolchain.
mkdir -p archives && pushd archives
# HERE: Download dependencies here.
popd
mkdir -p build
pushd build
source $PREFIX/activate
export CC=$PREFIX/bin/clang
export CXX=$PREFIX/bin/clang++
export CFLAGS="$CFLAGS -fPIC"
export PATH=$PREFIX/bin:$PATH
export LD_LIBRARY_PATH=$PREFIX/lib64
COMMON_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$PREFIX
-DCMAKE_PREFIX_PATH=$PREFIX
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_C_COMPILER=$CC
-DCMAKE_CXX_COMPILER=$CXX
-DBUILD_SHARED_LIBS=OFF
-DCMAKE_CXX_STANDARD=20
-DBUILD_TESTING=OFF
-DCMAKE_REQUIRED_INCLUDES=$PREFIX/include
-DCMAKE_POSITION_INDEPENDENT_CODE=ON"
# HERE: Add dependencies to test below.

View File

@@ -307,7 +307,7 @@ if [ ! -f $PREFIX/bin/ld.gold ]; then
fi
log_tool_name "GDB $GDB_VERSION"
if [[ ! -f "$PREFIX/bin/gdb" && "$DISTRO" -ne "amzn-2" ]]; then
if [ ! -f $PREFIX/bin/gdb ]; then
if [ -d gdb-$GDB_VERSION ]; then
rm -rf gdb-$GDB_VERSION
fi
@@ -671,6 +671,7 @@ PROXYGEN_SHA256=5360a8ccdfb2f5a6c7b3eed331ec7ab0e2c792d579c6fff499c85c516c11fe14
WANGLE_SHA256=1002e9c32b6f4837f6a760016e3b3e22f3509880ef3eaad191c80dc92655f23f
# WANGLE_SHA256=0e493c03572bb27fe9ca03a9da5023e52fde99c95abdcaa919bb6190e7e69532
FLEX_VERSION=2.6.4
FMT_SHA256=78b8c0a72b1c35e4443a7e308df52498252d1cefc2b08c9a97bc9ee6cfe61f8b
FMT_VERSION=10.1.1
# NOTE: spdlog depends on exact fmt versions -> UPGRADE fmt and spdlog TOGETHER.
@@ -689,8 +690,8 @@ LZ4_VERSION=1.9.4
SNAPPY_SHA256=75c1fbb3d618dd3a0483bff0e26d0a92b495bbe5059c8b4f1c962b478b6e06e7
SNAPPY_VERSION=1.1.9
XZ_VERSION=5.2.5 # for LZMA
ZLIB_VERSION=1.3.1
ZSTD_VERSION=1.5.5
ZLIB_VERSION=1.3
ZSTD_VERSION=1.5.0
pushd archives
if [ ! -f boost_$BOOST_VERSION_UNDERSCORES.tar.gz ]; then
@@ -699,7 +700,7 @@ if [ ! -f boost_$BOOST_VERSION_UNDERSCORES.tar.gz ]; then
wget https://boostorg.jfrog.io/artifactory/main/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORES.tar.gz -O boost_$BOOST_VERSION_UNDERSCORES.tar.gz
fi
if [ ! -f bzip2-$BZIP2_VERSION.tar.gz ]; then
wget https://sourceware.org/pub/bzip2/bzip2-$BZIP2_VERSION.tar.gz -O bzip2-$BZIP2_VERSION.tar.gz
wget https://sourceforge.net/projects/bzip2/files/bzip2-$BZIP2_VERSION.tar.gz -O bzip2-$BZIP2_VERSION.tar.gz
fi
if [ ! -f double-conversion-$DOUBLE_CONVERSION_VERSION.tar.gz ]; then
wget https://github.com/google/double-conversion/archive/refs/tags/v$DOUBLE_CONVERSION_VERSION.tar.gz -O double-conversion-$DOUBLE_CONVERSION_VERSION.tar.gz
@@ -707,7 +708,9 @@ fi
if [ ! -f fizz-$FBLIBS_VERSION.tar.gz ]; then
wget https://github.com/facebookincubator/fizz/releases/download/v$FBLIBS_VERSION/fizz-v$FBLIBS_VERSION.tar.gz -O fizz-$FBLIBS_VERSION.tar.gz
fi
if [ ! -f flex-$FLEX_VERSION.tar.gz ]; then
wget https://github.com/westes/flex/releases/download/v$FLEX_VERSION/flex-$FLEX_VERSION.tar.gz -O flex-$FLEX_VERSION.tar.gz
fi
if [ ! -f fmt-$FMT_VERSION.tar.gz ]; then
wget https://github.com/fmtlib/fmt/archive/refs/tags/$FMT_VERSION.tar.gz -O fmt-$FMT_VERSION.tar.gz
fi
@@ -762,6 +765,14 @@ echo "$BZIP2_SHA256 bzip2-$BZIP2_VERSION.tar.gz" | sha256sum -c
echo "$DOUBLE_CONVERSION_SHA256 double-conversion-$DOUBLE_CONVERSION_VERSION.tar.gz" | sha256sum -c
# verify fizz
echo "$FIZZ_SHA256 fizz-$FBLIBS_VERSION.tar.gz" | sha256sum -c
# verify flex
if [ ! -f flex-$FLEX_VERSION.tar.gz.sig ]; then
wget https://github.com/westes/flex/releases/download/v$FLEX_VERSION/flex-$FLEX_VERSION.tar.gz.sig
fi
if false; then
$GPG --keyserver $KEYSERVER --recv-keys 0xE4B29C8D64885307
$GPG --verify flex-$FLEX_VERSION.tar.gz.sig flex-$FLEX_VERSION.tar.gz
fi
# verify fmt
echo "$FMT_SHA256 fmt-$FMT_VERSION.tar.gz" | sha256sum -c
# verify spdlog
@@ -1014,6 +1025,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
if [ -d gflags ]; then
rm -rf gflags
fi
git clone https://github.com/memgraph/gflags.git gflags
pushd gflags
git checkout $GFLAGS_COMMIT_HASH
@@ -1022,7 +1034,7 @@ if [ ! -d $PREFIX/include/gflags ]; then
cmake .. $COMMON_CMAKE_FLAGS \
-DREGISTER_INSTALL_PREFIX=OFF \
-DBUILD_gflags_nothreads_LIB=OFF \
-DGFLAGS_NO_FILENAMES=1
-DGFLAGS_NO_FILENAMES=0
make -j$CPUS install
popd && popd
fi
@@ -1220,6 +1232,18 @@ if false; then
fi
fi
log_tool_name "flex $FLEX_VERSION"
if [ ! -f $PREFIX/include/FlexLexer.h ]; then
if [ -d flex-$FLEX_VERSION ]; then
rm -rf flex-$FLEX_VERSION
fi
tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
pushd flex-$FLEX_VERSION
./configure $COMMON_CONFIGURE_FLAGS
make -j$CPUS install
popd
fi
popd
# NOTE: It's important/clean (e.g., easier upload to S3) to have a separated
# folder to the output archive.

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -283,7 +283,7 @@ inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memor
}
// mgp_graph
inline bool graph_is_transactional(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_transactional, graph); }
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }

1
libs/.gitignore vendored
View File

@@ -7,4 +7,3 @@
!pulsar.patch
!antlr4.10.1.patch
!rocksdb8.1.1.patch
!nuraft2.1.0.patch

View File

@@ -16,7 +16,7 @@ set(GFLAGS_NOTHREADS OFF)
# NOTE: config/generate.py depends on the gflags help XML format.
find_package(gflags REQUIRED)
find_package(fmt 8.0.1 REQUIRED)
find_package(fmt 8.0.1)
find_package(ZLIB 1.2.11 REQUIRED)
set(LIB_DIR ${CMAKE_CURRENT_SOURCE_DIR})

View File

@@ -5,7 +5,7 @@ index ee9b58c..31359a9 100644
@@ -48,7 +48,7 @@ option(LIBRDTSC_USE_PMU "Enables PMU usage on ARM platforms" OFF)
# | Library Build and Install Properties |
# +--------------------------------------------------------+
-add_library(rdtsc SHARED
+add_library(rdtsc
src/cycles.c
@@ -14,7 +14,7 @@ index ee9b58c..31359a9 100644
@@ -72,15 +72,6 @@ target_include_directories(rdtsc
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include
)
-# Install directory changes depending on build mode
-if (CMAKE_BUILD_TYPE MATCHES "^[Dd]ebug")
- # During debug, the library will be installed into a local directory
@@ -27,15 +27,3 @@ index ee9b58c..31359a9 100644
# Specifying what to export when installing (GNUInstallDirs required)
install(TARGETS rdtsc
EXPORT librstsc-config
diff --git a/include/librdtsc/common_timer.h b/include/librdtsc/common_timer.h
index a6922d8..080dc77 100644
--- a/include/librdtsc/common_timer.h
+++ b/include/librdtsc/common_timer.h
@@ -2,6 +2,7 @@
#define LIBRDTSC_COMMON_TIMER_H
#include <librdtsc/common.h>
+#include <librdtsc/cycles.h>
extern uint64_t rdtsc_get_tsc_freq_arch();
extern uint64_t rdtsc_get_tsc_freq();

View File

@@ -1,24 +0,0 @@
diff --git a/include/libnuraft/asio_service_options.hxx b/include/libnuraft/asio_service_options.hxx
index 8fe1ec9..9497355 100644
--- a/include/libnuraft/asio_service_options.hxx
+++ b/include/libnuraft/asio_service_options.hxx
@@ -17,6 +17,7 @@ limitations under the License.
#pragma once
+#include <cstdint>
#include <functional>
#include <string>
#include <system_error>
diff --git a/include/libnuraft/callback.hxx b/include/libnuraft/callback.hxx
index 7b71624..d48c1e2 100644
--- a/include/libnuraft/callback.hxx
+++ b/include/libnuraft/callback.hxx
@@ -18,6 +18,7 @@ limitations under the License.
#ifndef _CALLBACK_H_
#define _CALLBACK_H_
+#include <cstdint>
#include <functional>
#include <string>

21
libs/rocksdb.patch Normal file
View File

@@ -0,0 +1,21 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6761929..6a369af 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -220,6 +220,7 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
endif()
endif()
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated-copy -Wno-unused-but-set-variable")
endif()
include(CheckCCompilerFlag)
@@ -997,7 +998,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(ROCKSDB_BUILD_SHARED)
install(
- TARGETS ${ROCKSDB_SHARED_LIB}
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
EXPORT RocksDBTargets
COMPONENT runtime
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"

View File

@@ -168,11 +168,12 @@ pushd antlr4
git apply ../antlr4.10.1.patch
popd
cppitertools_ref="v2.1" # 2021-01-15
# cppitertools v2.0 2019-12-23
cppitertools_ref="cb3635456bdb531121b82b4d2e3afc7ae1f56d47"
repo_clone_try_double "${primary_urls[cppitertools]}" "${secondary_urls[cppitertools]}" "cppitertools" "$cppitertools_ref"
# rapidcheck
rapidcheck_tag="1c91f40e64d87869250cfb610376c629307bf77d" # (2023-08-15)
rapidcheck_tag="7bc7d302191a4f3d0bf005692677126136e02f60" # (2020-05-04)
repo_clone_try_double "${primary_urls[rapidcheck]}" "${secondary_urls[rapidcheck]}" "rapidcheck" "$rapidcheck_tag"
# google benchmark
@@ -220,7 +221,7 @@ repo_clone_try_double "${primary_urls[pymgclient]}" "${secondary_urls[pymgclient
mgconsole_tag="v1.4.0" # (2023-05-21)
repo_clone_try_double "${primary_urls[mgconsole]}" "${secondary_urls[mgconsole]}" "mgconsole" "$mgconsole_tag" true
spdlog_tag="v1.12.0" # (2022-11-02)
spdlog_tag="v1.9.2" # (2021-08-12)
repo_clone_try_double "${primary_urls[spdlog]}" "${secondary_urls[spdlog]}" "spdlog" "$spdlog_tag" true
# librdkafka
@@ -285,6 +286,5 @@ repo_clone_try_double "${primary_urls[range-v3]}" "${secondary_urls[range-v3]}"
nuraft_tag="v2.1.0"
repo_clone_try_double "${primary_urls[nuraft]}" "${secondary_urls[nuraft]}" "nuraft" "$nuraft_tag" true
pushd nuraft
git apply ../nuraft2.1.0.patch
./prepare.sh
popd

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2028-21-01
CHANGE DATE: 2028-28-02
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -1,11 +0,0 @@
# Memgraph Release Packaging
## Design Decisions
* Being able to build and test memgraph locally under Docker because multi-OS support
* Being able to push Docker images to the centralized repo (DockerHub)
* Being able to build and test memgraph under CI under Docker
* Being able to iterate on the code changes when adding support for new toolchains and OSe
* Being able to pick and choose toolchains, OSes, build type, hardware architecture
* Take care of enterprise keys, telemetry
* Create generator for Dockerfiles

View File

@@ -35,42 +35,16 @@ DEFINE_VALIDATED_string(auth_module_executable, "", "Absolute path to the auth m
}
return true;
});
DEFINE_bool(auth_module_create_missing_user, true, "Set to false to disable creation of missing users.");
DEFINE_bool(auth_module_create_missing_role, true, "Set to false to disable creation of missing roles.");
DEFINE_bool(auth_module_manage_roles, true, "Set to false to disable management of roles through the auth module.");
DEFINE_VALIDATED_int32(auth_module_timeout_ms, 10000,
"Timeout (in milliseconds) used when waiting for a "
"response from the auth module.",
FLAG_IN_RANGE(100, 1800000));
// DEPRECATED FLAGS
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
DEFINE_VALIDATED_HIDDEN_bool(auth_module_create_missing_user, true,
"Set to false to disable creation of missing users.", {
spdlog::warn(
"auth_module_create_missing_user flag is deprecated. It not possible to create "
"users through the module anymore.");
return true;
});
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
DEFINE_VALIDATED_HIDDEN_bool(auth_module_create_missing_role, true,
"Set to false to disable creation of missing roles.", {
spdlog::warn(
"auth_module_create_missing_role flag is deprecated. It not possible to create "
"roles through the module anymore.");
return true;
});
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables, misc-unused-parameters)
DEFINE_VALIDATED_HIDDEN_bool(
auth_module_manage_roles, true, "Set to false to disable management of roles through the auth module.", {
spdlog::warn(
"auth_module_manage_roles flag is deprecated. It not possible to create roles through the module anymore.");
return true;
});
namespace memgraph::auth {
const Auth::Epoch Auth::kStartEpoch = 1;
namespace {
#ifdef MG_ENTERPRISE
/**
@@ -218,17 +192,6 @@ void MigrateVersions(kvstore::KVStore &store) {
version_str = kVersionV1;
}
}
auto ParseJson(std::string_view str) {
nlohmann::json data;
try {
data = nlohmann::json::parse(str);
} catch (const nlohmann::json::parse_error &e) {
throw AuthException("Couldn't load auth data!");
}
return data;
}
}; // namespace
Auth::Auth(std::string storage_directory, Config config)
@@ -236,11 +199,8 @@ Auth::Auth(std::string storage_directory, Config config)
MigrateVersions(storage_);
}
std::optional<UserOrRole> Auth::Authenticate(const std::string &username, const std::string &password) {
std::optional<User> Auth::Authenticate(const std::string &username, const std::string &password) {
if (module_.IsUsed()) {
/*
* MODULE AUTH STORAGE
*/
const auto license_check_result = license::global_license_checker.IsEnterpriseValid(utils::global_settings);
if (license_check_result.HasError()) {
spdlog::warn(license::LicenseCheckErrorToString(license_check_result.GetError(), "authentication modules"));
@@ -265,64 +225,108 @@ std::optional<UserOrRole> Auth::Authenticate(const std::string &username, const
auto is_authenticated = ret_authenticated.get<bool>();
const auto &rolename = ret_role.get<std::string>();
// Check if role is present
auto role = GetRole(rolename);
if (!role) {
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the role '{}' doesn't exist.",
username, rolename, "https://memgr.ph/auth"));
return std::nullopt;
}
// Authenticate the user.
if (!is_authenticated) return std::nullopt;
return RoleWUsername{username, std::move(*role)};
}
/**
* TODO
* The auth module should not update auth data.
* There is now way to replicate it and we should not be storing sensitive data if we don't have to.
*/
/*
* LOCAL AUTH STORAGE
*/
auto user = GetUser(username);
if (!user) {
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the user doesn't exist.", username,
"https://memgr.ph/auth"));
return std::nullopt;
}
if (!user->CheckPassword(password)) {
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the password is not correct.",
username, "https://memgr.ph/auth"));
return std::nullopt;
}
if (user->UpgradeHash(password)) {
// Find or create the user and return it.
auto user = GetUser(username);
if (!user) {
if (FLAGS_auth_module_create_missing_user) {
user = AddUser(username, password);
if (!user) {
spdlog::warn(utils::MessageWithLink(
"Couldn't create the missing user '{}' using the auth module because the user already exists as a role.",
username, "https://memgr.ph/auth"));
return std::nullopt;
}
} else {
spdlog::warn(utils::MessageWithLink(
"Couldn't authenticate user '{}' using the auth module because the user doesn't exist.", username,
"https://memgr.ph/auth"));
return std::nullopt;
}
} else {
UpdatePassword(*user, password);
}
if (FLAGS_auth_module_manage_roles) {
if (!rolename.empty()) {
auto role = GetRole(rolename);
if (!role) {
if (FLAGS_auth_module_create_missing_role) {
role = AddRole(rolename);
if (!role) {
spdlog::warn(
utils::MessageWithLink("Couldn't authenticate user '{}' using the auth module because the user's "
"role '{}' already exists as a user.",
username, rolename, "https://memgr.ph/auth"));
return std::nullopt;
}
SaveRole(*role);
} else {
spdlog::warn(utils::MessageWithLink(
"Couldn't authenticate user '{}' using the auth module because the user's role '{}' doesn't exist.",
username, rolename, "https://memgr.ph/auth"));
return std::nullopt;
}
}
user->SetRole(*role);
} else {
user->ClearRole();
}
}
SaveUser(*user);
}
return user;
} else {
auto user = GetUser(username);
if (!user) {
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the user doesn't exist.", username,
"https://memgr.ph/auth"));
return std::nullopt;
}
if (!user->CheckPassword(password)) {
spdlog::warn(utils::MessageWithLink("Couldn't authenticate user '{}' because the password is not correct.",
username, "https://memgr.ph/auth"));
return std::nullopt;
}
if (user->UpgradeHash(password)) {
SaveUser(*user);
}
return user;
return user;
}
}
void Auth::LinkUser(User &user) const {
auto link = storage_.Get(kLinkPrefix + user.username());
std::optional<User> Auth::GetUser(const std::string &username_orig) const {
auto username = utils::ToLowerCase(username_orig);
auto existing_user = storage_.Get(kUserPrefix + username);
if (!existing_user) return std::nullopt;
nlohmann::json data;
try {
data = nlohmann::json::parse(*existing_user);
} catch (const nlohmann::json::parse_error &e) {
throw AuthException("Couldn't load user data!");
}
auto user = User::Deserialize(data);
auto link = storage_.Get(kLinkPrefix + username);
if (link) {
auto role = GetRole(*link);
if (role) {
user.SetRole(*role);
}
}
}
std::optional<User> Auth::GetUser(const std::string &username_orig) const {
if (module_.IsUsed()) return std::nullopt; // User's are not supported when using module
auto username = utils::ToLowerCase(username_orig);
auto existing_user = storage_.Get(kUserPrefix + username);
if (!existing_user) return std::nullopt;
auto user = User::Deserialize(ParseJson(*existing_user));
LinkUser(user);
return user;
}
void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
DisableIfModuleUsed();
bool success = false;
if (const auto *role = user.role(); role != nullptr) {
success = storage_.PutMultiple(
@@ -334,10 +338,6 @@ void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
if (!success) {
throw AuthException("Couldn't save user '{}'!", user.username());
}
// Durability updated -> new epoch
UpdateEpoch();
// All changes to the user end up calling this function, so no need to add a delta anywhere else
if (system_tx) {
#ifdef MG_ENTERPRISE
@@ -347,7 +347,6 @@ void Auth::SaveUser(const User &user, system::Transaction *system_tx) {
}
void Auth::UpdatePassword(auth::User &user, const std::optional<std::string> &password) {
DisableIfModuleUsed();
// Check if null
if (!password) {
if (!config_.password_permit_null) {
@@ -379,7 +378,6 @@ void Auth::UpdatePassword(auth::User &user, const std::optional<std::string> &pa
std::optional<User> Auth::AddUser(const std::string &username, const std::optional<std::string> &password,
system::Transaction *system_tx) {
DisableIfModuleUsed();
if (!NameRegexMatch(username)) {
throw AuthException("Invalid user name.");
}
@@ -394,17 +392,12 @@ std::optional<User> Auth::AddUser(const std::string &username, const std::option
}
bool Auth::RemoveUser(const std::string &username_orig, system::Transaction *system_tx) {
DisableIfModuleUsed();
auto username = utils::ToLowerCase(username_orig);
if (!storage_.Get(kUserPrefix + username)) return false;
std::vector<std::string> keys({kLinkPrefix + username, kUserPrefix + username});
if (!storage_.DeleteMultiple(keys)) {
throw AuthException("Couldn't remove user '{}'!", username);
}
// Durability updated -> new epoch
UpdateEpoch();
// Handling drop user delta
if (system_tx) {
#ifdef MG_ENTERPRISE
@@ -419,12 +412,9 @@ std::vector<auth::User> Auth::AllUsers() const {
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
auto username = it->first.substr(kUserPrefix.size());
if (username != utils::ToLowerCase(username)) continue;
try {
User user = auth::User::Deserialize(ParseJson(it->second)); // Will throw on failure
LinkUser(user);
ret.emplace_back(std::move(user));
} catch (AuthException &) {
continue;
auto user = GetUser(username);
if (user) {
ret.push_back(std::move(*user));
}
}
return ret;
@@ -435,12 +425,9 @@ std::vector<std::string> Auth::AllUsernames() const {
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
auto username = it->first.substr(kUserPrefix.size());
if (username != utils::ToLowerCase(username)) continue;
try {
// Check if serialized correctly
memgraph::auth::User::Deserialize(ParseJson(it->second)); // Will throw on failure
ret.emplace_back(std::move(username));
} catch (AuthException &) {
continue;
auto user = GetUser(username);
if (user) {
ret.push_back(username);
}
}
return ret;
@@ -448,24 +435,25 @@ std::vector<std::string> Auth::AllUsernames() const {
bool Auth::HasUsers() const { return storage_.begin(kUserPrefix) != storage_.end(kUserPrefix); }
bool Auth::AccessControlled() const { return HasUsers() || module_.IsUsed(); }
std::optional<Role> Auth::GetRole(const std::string &rolename_orig) const {
auto rolename = utils::ToLowerCase(rolename_orig);
auto existing_role = storage_.Get(kRolePrefix + rolename);
if (!existing_role) return std::nullopt;
return Role::Deserialize(ParseJson(*existing_role));
nlohmann::json data;
try {
data = nlohmann::json::parse(*existing_role);
} catch (const nlohmann::json::parse_error &e) {
throw AuthException("Couldn't load role data!");
}
return Role::Deserialize(data);
}
void Auth::SaveRole(const Role &role, system::Transaction *system_tx) {
if (!storage_.Put(kRolePrefix + role.rolename(), role.Serialize().dump())) {
throw AuthException("Couldn't save role '{}'!", role.rolename());
}
// Durability updated -> new epoch
UpdateEpoch();
// All changes to the role end up calling this function, so no need to add a delta anywhere else
if (system_tx) {
#ifdef MG_ENTERPRISE
@@ -498,10 +486,6 @@ bool Auth::RemoveRole(const std::string &rolename_orig, system::Transaction *sys
if (!storage_.DeleteMultiple(keys)) {
throw AuthException("Couldn't remove role '{}'!", rolename);
}
// Durability updated -> new epoch
UpdateEpoch();
// Handling drop role delta
if (system_tx) {
#ifdef MG_ENTERPRISE
@@ -516,8 +500,11 @@ std::vector<auth::Role> Auth::AllRoles() const {
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
auto rolename = it->first.substr(kRolePrefix.size());
if (rolename != utils::ToLowerCase(rolename)) continue;
Role role = memgraph::auth::Role::Deserialize(ParseJson(it->second)); // Will throw on failure
ret.emplace_back(std::move(role));
if (auto role = GetRole(rolename)) {
ret.push_back(*role);
} else {
throw AuthException("Couldn't load role '{}'!", rolename);
}
}
return ret;
}
@@ -527,19 +514,14 @@ std::vector<std::string> Auth::AllRolenames() const {
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
auto rolename = it->first.substr(kRolePrefix.size());
if (rolename != utils::ToLowerCase(rolename)) continue;
try {
// Check that the data is serialized correctly
memgraph::auth::Role::Deserialize(ParseJson(it->second));
ret.emplace_back(std::move(rolename));
} catch (AuthException &) {
continue;
if (auto role = GetRole(rolename)) {
ret.push_back(rolename);
}
}
return ret;
}
std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig) const {
DisableIfModuleUsed();
const auto rolename = utils::ToLowerCase(rolename_orig);
std::vector<auth::User> ret;
for (auto it = storage_.begin(kLinkPrefix); it != storage_.end(kLinkPrefix); ++it) {
@@ -558,176 +540,51 @@ std::vector<auth::User> Auth::AllUsersForRole(const std::string &rolename_orig)
}
#ifdef MG_ENTERPRISE
Auth::Result Auth::GrantDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx) {
using enum Auth::Result;
if (module_.IsUsed()) {
if (auto role = GetRole(name)) {
GrantDatabase(db, *role, system_tx);
return SUCCESS;
}
return NO_ROLE;
}
bool Auth::GrantDatabaseToUser(const std::string &db, const std::string &name, system::Transaction *system_tx) {
if (auto user = GetUser(name)) {
GrantDatabase(db, *user, system_tx);
return SUCCESS;
}
if (auto role = GetRole(name)) {
GrantDatabase(db, *role, system_tx);
return SUCCESS;
}
return NO_USER_ROLE;
}
void Auth::GrantDatabase(const std::string &db, User &user, system::Transaction *system_tx) {
if (db == kAllDatabases) {
user.db_access().GrantAll();
} else {
user.db_access().Grant(db);
}
SaveUser(user, system_tx);
}
void Auth::GrantDatabase(const std::string &db, Role &role, system::Transaction *system_tx) {
if (db == kAllDatabases) {
role.db_access().GrantAll();
} else {
role.db_access().Grant(db);
}
SaveRole(role, system_tx);
}
Auth::Result Auth::DenyDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx) {
using enum Auth::Result;
if (module_.IsUsed()) {
if (auto role = GetRole(name)) {
DenyDatabase(db, *role, system_tx);
return SUCCESS;
if (db == kAllDatabases) {
user->db_access().GrantAll();
} else {
user->db_access().Add(db);
}
return NO_ROLE;
SaveUser(*user, system_tx);
return true;
}
return false;
}
bool Auth::RevokeDatabaseFromUser(const std::string &db, const std::string &name, system::Transaction *system_tx) {
if (auto user = GetUser(name)) {
DenyDatabase(db, *user, system_tx);
return SUCCESS;
}
if (auto role = GetRole(name)) {
DenyDatabase(db, *role, system_tx);
return SUCCESS;
}
return NO_USER_ROLE;
}
void Auth::DenyDatabase(const std::string &db, User &user, system::Transaction *system_tx) {
if (db == kAllDatabases) {
user.db_access().DenyAll();
} else {
user.db_access().Deny(db);
}
SaveUser(user, system_tx);
}
void Auth::DenyDatabase(const std::string &db, Role &role, system::Transaction *system_tx) {
if (db == kAllDatabases) {
role.db_access().DenyAll();
} else {
role.db_access().Deny(db);
}
SaveRole(role, system_tx);
}
Auth::Result Auth::RevokeDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx) {
using enum Auth::Result;
if (module_.IsUsed()) {
if (auto role = GetRole(name)) {
RevokeDatabase(db, *role, system_tx);
return SUCCESS;
if (db == kAllDatabases) {
user->db_access().DenyAll();
} else {
user->db_access().Remove(db);
}
return NO_ROLE;
SaveUser(*user, system_tx);
return true;
}
if (auto user = GetUser(name)) {
RevokeDatabase(db, *user, system_tx);
return SUCCESS;
}
if (auto role = GetRole(name)) {
RevokeDatabase(db, *role, system_tx);
return SUCCESS;
}
return NO_USER_ROLE;
}
void Auth::RevokeDatabase(const std::string &db, User &user, system::Transaction *system_tx) {
if (db == kAllDatabases) {
user.db_access().RevokeAll();
} else {
user.db_access().Revoke(db);
}
SaveUser(user, system_tx);
}
void Auth::RevokeDatabase(const std::string &db, Role &role, system::Transaction *system_tx) {
if (db == kAllDatabases) {
role.db_access().RevokeAll();
} else {
role.db_access().Revoke(db);
}
SaveRole(role, system_tx);
return false;
}
void Auth::DeleteDatabase(const std::string &db, system::Transaction *system_tx) {
for (auto it = storage_.begin(kUserPrefix); it != storage_.end(kUserPrefix); ++it) {
auto username = it->first.substr(kUserPrefix.size());
try {
User user = auth::User::Deserialize(ParseJson(it->second));
LinkUser(user);
user.db_access().Revoke(db);
SaveUser(user, system_tx);
} catch (AuthException &) {
continue;
}
}
for (auto it = storage_.begin(kRolePrefix); it != storage_.end(kRolePrefix); ++it) {
auto rolename = it->first.substr(kRolePrefix.size());
try {
auto role = memgraph::auth::Role::Deserialize(ParseJson(it->second));
role.db_access().Revoke(db);
SaveRole(role, system_tx);
} catch (AuthException &) {
continue;
if (auto user = GetUser(username)) {
user->db_access().Delete(db);
SaveUser(*user, system_tx);
}
}
}
Auth::Result Auth::SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx) {
using enum Auth::Result;
if (module_.IsUsed()) {
if (auto role = GetRole(name)) {
SetMainDatabase(db, *role, system_tx);
return SUCCESS;
}
return NO_ROLE;
}
bool Auth::SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx) {
if (auto user = GetUser(name)) {
SetMainDatabase(db, *user, system_tx);
return SUCCESS;
if (!user->db_access().SetDefault(db)) {
throw AuthException("Couldn't set default database '{}' for user '{}'!", db, name);
}
SaveUser(*user, system_tx);
return true;
}
if (auto role = GetRole(name)) {
SetMainDatabase(db, *role, system_tx);
return SUCCESS;
}
return NO_USER_ROLE;
}
void Auth::SetMainDatabase(std::string_view db, User &user, system::Transaction *system_tx) {
if (!user.db_access().SetMain(db)) {
throw AuthException("Couldn't set default database '{}' for '{}'!", db, user.username());
}
SaveUser(user, system_tx);
}
void Auth::SetMainDatabase(std::string_view db, Role &role, system::Transaction *system_tx) {
if (!role.db_access().SetMain(db)) {
throw AuthException("Couldn't set default database '{}' for '{}'!", db, role.rolename());
}
SaveRole(role, system_tx);
return false;
}
#endif

View File

@@ -29,18 +29,6 @@ using SynchedAuth = memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph
static const constexpr char *const kAllDatabases = "*";
struct RoleWUsername : Role {
template <typename... Args>
RoleWUsername(std::string_view username, Args &&...args) : Role{std::forward<Args>(args)...}, username_{username} {}
std::string username() { return username_; }
const std::string &username() const { return username_; }
private:
std::string username_;
};
using UserOrRole = std::variant<User, RoleWUsername>;
/**
* This class serves as the main Authentication/Authorization storage.
* It provides functions for managing Users, Roles, Permissions and FineGrainedAccessPermissions.
@@ -73,25 +61,6 @@ class Auth final {
std::regex password_regex{password_regex_str};
};
struct Epoch {
Epoch() : epoch_{0} {}
Epoch(unsigned e) : epoch_{e} {}
Epoch operator++() { return ++epoch_; }
bool operator==(const Epoch &rhs) const = default;
private:
unsigned epoch_;
};
static const Epoch kStartEpoch;
enum class Result {
SUCCESS,
NO_USER_ROLE,
NO_ROLE,
};
explicit Auth(std::string storage_directory, Config config);
/**
@@ -120,7 +89,7 @@ class Auth final {
* @return a user when the username and password match, nullopt otherwise
* @throw AuthException if unable to authenticate for whatever reason.
*/
std::optional<UserOrRole> Authenticate(const std::string &username, const std::string &password);
std::optional<User> Authenticate(const std::string &username, const std::string &password);
/**
* Gets a user from the storage.
@@ -132,8 +101,6 @@ class Auth final {
*/
std::optional<User> GetUser(const std::string &username) const;
void LinkUser(User &user) const;
/**
* Saves a user object to the storage.
*
@@ -196,13 +163,6 @@ class Auth final {
*/
bool HasUsers() const;
/**
* Returns whether the access is controlled by authentication/authorization.
*
* @return `true` if auth needs to run
*/
bool AccessControlled() const;
/**
* Gets a role from the storage.
*
@@ -213,37 +173,6 @@ class Auth final {
*/
std::optional<Role> GetRole(const std::string &rolename) const;
std::optional<UserOrRole> GetUserOrRole(const std::optional<std::string> &username,
const std::optional<std::string> &rolename) const {
auto expect = [](bool condition, std::string &&msg) {
if (!condition) throw AuthException(std::move(msg));
};
// Special case if we are using a module; we must find the specified role
if (module_.IsUsed()) {
expect(username && rolename, "When using a module, a role needs to be connected to a username.");
const auto role = GetRole(*rolename);
expect(role != std::nullopt, "No role named " + *rolename);
return UserOrRole(auth::RoleWUsername{*username, *role});
}
// First check if we need to find a role
if (username && rolename) {
const auto role = GetRole(*rolename);
expect(role != std::nullopt, "No role named " + *rolename);
return UserOrRole(auth::RoleWUsername{*username, *role});
}
// We are only looking for a user
if (username) {
const auto user = GetUser(*username);
expect(user != std::nullopt, "No user named " + *username);
return *user;
}
// No user or role
return std::nullopt;
}
/**
* Saves a role object to the storage.
*
@@ -300,6 +229,16 @@ class Auth final {
std::vector<User> AllUsersForRole(const std::string &rolename) const;
#ifdef MG_ENTERPRISE
/**
* @brief Revoke access to individual database for a user.
*
* @param db name of the database to revoke
* @param name user's username
* @return true on success
* @throw AuthException if unable to find or update the user
*/
bool RevokeDatabaseFromUser(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
/**
* @brief Grant access to individual database for a user.
*
@@ -308,33 +247,7 @@ class Auth final {
* @return true on success
* @throw AuthException if unable to find or update the user
*/
Result GrantDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
void GrantDatabase(const std::string &db, User &user, system::Transaction *system_tx = nullptr);
void GrantDatabase(const std::string &db, Role &role, system::Transaction *system_tx = nullptr);
/**
* @brief Revoke access to individual database for a user.
*
* @param db name of the database to revoke
* @param name user's username
* @return true on success
* @throw AuthException if unable to find or update the user
*/
Result DenyDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
void DenyDatabase(const std::string &db, User &user, system::Transaction *system_tx = nullptr);
void DenyDatabase(const std::string &db, Role &role, system::Transaction *system_tx = nullptr);
/**
* @brief Revoke access to individual database for a user.
*
* @param db name of the database to revoke
* @param name user's username
* @return true on success
* @throw AuthException if unable to find or update the user
*/
Result RevokeDatabase(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
void RevokeDatabase(const std::string &db, User &user, system::Transaction *system_tx = nullptr);
void RevokeDatabase(const std::string &db, Role &role, system::Transaction *system_tx = nullptr);
bool GrantDatabaseToUser(const std::string &db, const std::string &name, system::Transaction *system_tx = nullptr);
/**
* @brief Delete a database from all users.
@@ -352,17 +265,9 @@ class Auth final {
* @return true on success
* @throw AuthException if unable to find or update the user
*/
Result SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx = nullptr);
void SetMainDatabase(std::string_view db, User &user, system::Transaction *system_tx = nullptr);
void SetMainDatabase(std::string_view db, Role &role, system::Transaction *system_tx = nullptr);
bool SetMainDatabase(std::string_view db, const std::string &name, system::Transaction *system_tx = nullptr);
#endif
bool UpToDate(Epoch &e) const {
bool res = e == epoch_;
e = epoch_;
return res;
}
private:
/**
* @brief
@@ -373,18 +278,11 @@ class Auth final {
*/
bool NameRegexMatch(const std::string &user_or_role) const;
void UpdateEpoch() { ++epoch_; }
void DisableIfModuleUsed() const {
if (module_.IsUsed()) throw AuthException("Operation not permited when using an authentication module.");
}
// Even though the `kvstore::KVStore` class is guaranteed to be thread-safe,
// Auth is not thread-safe because modifying users and roles might require
// more than one operation on the storage.
kvstore::KVStore storage_;
auth::Module module_;
Config config_;
Epoch epoch_{kStartEpoch};
};
} // namespace memgraph::auth

View File

@@ -8,12 +8,10 @@
#pragma once
#include <cstdint>
#include <json/json.hpp>
#include <optional>
#include <string>
#include <json/json.hpp>
namespace memgraph::auth {
/// Need to be stable, auth durability depends on this
enum class PasswordHashAlgorithm : uint8_t { BCRYPT = 0, SHA256 = 1, SHA256_MULTIPLE = 2 };

View File

@@ -425,11 +425,10 @@ Role::Role(const std::string &rolename, const Permissions &permissions)
: rolename_(utils::ToLowerCase(rolename)), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
Role::Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
FineGrainedAccessHandler fine_grained_access_handler)
: rolename_(utils::ToLowerCase(rolename)),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)),
db_access_(std::move(db_access)) {}
fine_grained_access_handler_(std::move(fine_grained_access_handler)) {}
#endif
const std::string &Role::rolename() const { return rolename_; }
@@ -455,10 +454,8 @@ nlohmann::json Role::Serialize() const {
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
data[kFineGrainedAccessHandler] = fine_grained_access_handler_.Serialize();
data[kDatabases] = db_access_.Serialize();
} else {
data[kFineGrainedAccessHandler] = {};
data[kDatabases] = {};
}
#endif
return data;
@@ -474,21 +471,12 @@ Role Role::Deserialize(const nlohmann::json &data) {
auto permissions = Permissions::Deserialize(data[kPermissions]);
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
Databases db_access;
if (data[kDatabases].is_structured()) {
db_access = Databases::Deserialize(data[kDatabases]);
} else {
// Back-compatibility
spdlog::warn("Role without specified database access. Given access to the default database.");
db_access.Grant(dbms::kDefaultDB);
db_access.SetMain(dbms::kDefaultDB);
}
FineGrainedAccessHandler fine_grained_access_handler;
// We can have an empty fine_grained if the user was created without a valid license
if (data[kFineGrainedAccessHandler].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
}
return {data[kRoleName], permissions, std::move(fine_grained_access_handler), std::move(db_access)};
return {data[kRoleName], permissions, std::move(fine_grained_access_handler)};
}
#endif
return {data[kRoleName], permissions};
@@ -505,7 +493,7 @@ bool operator==(const Role &first, const Role &second) {
}
#ifdef MG_ENTERPRISE
void Databases::Grant(std::string_view db) {
void Databases::Add(std::string_view db) {
if (allow_all_) {
grants_dbs_.clear();
allow_all_ = false;
@@ -514,19 +502,19 @@ void Databases::Grant(std::string_view db) {
denies_dbs_.erase(std::string{db}); // TODO: C++23 use transparent key compare
}
void Databases::Deny(const std::string &db) {
void Databases::Remove(const std::string &db) {
denies_dbs_.emplace(db);
grants_dbs_.erase(db);
}
void Databases::Revoke(const std::string &db) {
void Databases::Delete(const std::string &db) {
denies_dbs_.erase(db);
if (!allow_all_) {
grants_dbs_.erase(db);
}
// Reset if default deleted
if (main_db_ == db) {
main_db_ = "";
if (default_db_ == db) {
default_db_ = "";
}
}
@@ -542,16 +530,9 @@ void Databases::DenyAll() {
denies_dbs_.clear();
}
void Databases::RevokeAll() {
allow_all_ = false;
grants_dbs_.clear();
denies_dbs_.clear();
main_db_ = "";
}
bool Databases::SetMain(std::string_view db) {
bool Databases::SetDefault(std::string_view db) {
if (!Contains(db)) return false;
main_db_ = db;
default_db_ = db;
return true;
}
@@ -559,11 +540,11 @@ bool Databases::SetMain(std::string_view db) {
return !denies_dbs_.contains(db) && (allow_all_ || grants_dbs_.contains(db));
}
const std::string &Databases::GetMain() const {
if (!Contains(main_db_)) {
throw AuthException("No access to the set default database \"{}\".", main_db_);
const std::string &Databases::GetDefault() const {
if (!Contains(default_db_)) {
throw AuthException("No access to the set default database \"{}\".", default_db_);
}
return main_db_;
return default_db_;
}
nlohmann::json Databases::Serialize() const {
@@ -571,7 +552,7 @@ nlohmann::json Databases::Serialize() const {
data[kGrants] = grants_dbs_;
data[kDenies] = denies_dbs_;
data[kAllowAll] = allow_all_;
data[kDefault] = main_db_;
data[kDefault] = default_db_;
return data;
}
@@ -738,16 +719,15 @@ User User::Deserialize(const nlohmann::json &data) {
} else {
// Back-compatibility
spdlog::warn("User without specified database access. Given access to the default database.");
db_access.Grant(dbms::kDefaultDB);
db_access.SetMain(dbms::kDefaultDB);
db_access.Add(dbms::kDefaultDB);
db_access.SetDefault(dbms::kDefaultDB);
}
FineGrainedAccessHandler fine_grained_access_handler;
// We can have an empty fine_grained if the user was created without a valid license
if (data[kFineGrainedAccessHandler].is_object()) {
fine_grained_access_handler = FineGrainedAccessHandler::Deserialize(data[kFineGrainedAccessHandler]);
}
return {data[kUsername], std::move(password_hash), permissions, std::move(fine_grained_access_handler),
std::move(db_access)};
return {data[kUsername], std::move(password_hash), permissions, std::move(fine_grained_access_handler), db_access};
}
#endif
return {data[kUsername], std::move(password_hash), permissions};

View File

@@ -205,96 +205,7 @@ class FineGrainedAccessHandler final {
bool operator==(const FineGrainedAccessHandler &first, const FineGrainedAccessHandler &second);
#endif
#ifdef MG_ENTERPRISE
class Databases final {
public:
Databases() : grants_dbs_{std::string{dbms::kDefaultDB}}, allow_all_(false), main_db_(dbms::kDefaultDB) {}
Databases(const Databases &) = default;
Databases &operator=(const Databases &) = default;
Databases(Databases &&) noexcept = default;
Databases &operator=(Databases &&) noexcept = default;
~Databases() = default;
/**
* @brief Add database to the list of granted access. @note allow_all_ will be false after execution
*
* @param db name of the database to grant access to
*/
void Grant(std::string_view db);
/**
* @brief Remove database to the list of granted access.
* @note if allow_all_ is set, the flag will remain set and the
* database will be added to the set of denied databases.
*
* @param db name of the database to grant access to
*/
void Deny(const std::string &db);
/**
* @brief Called when database is dropped. Removes it from granted (if allow_all is false) and denied set.
* @note allow_all_ is not changed
*
* @param db name of the database to grant access to
*/
void Revoke(const std::string &db);
/**
* @brief Set allow_all_ to true and clears grants and denied sets.
*/
void GrantAll();
/**
* @brief Set allow_all_ to false and clears grants and denied sets.
*/
void DenyAll();
/**
* @brief Set allow_all_ to false and clears grants and denied sets.
*/
void RevokeAll();
/**
* @brief Set the default database.
*/
bool SetMain(std::string_view db);
/**
* @brief Checks if access is grated to the database.
*
* @param db name of the database
* @return true if allow_all and not denied or granted
*/
bool Contains(std::string_view db) const;
bool Denies(std::string_view db_name) const { return denies_dbs_.contains(db_name); }
bool Grants(std::string_view db_name) const { return allow_all_ || grants_dbs_.contains(db_name); }
bool GetAllowAll() const { return allow_all_; }
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
const std::string &GetMain() const;
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static Databases Deserialize(const nlohmann::json &data);
private:
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
std::string default_db = std::string{dbms::kDefaultDB})
: grants_dbs_(std::move(grant)),
denies_dbs_(std::move(deny)),
allow_all_(allow_all),
main_db_(std::move(default_db)) {}
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
std::string main_db_; //!< user's default database
};
#endif
class Role {
class Role final {
public:
Role() = default;
@@ -302,7 +213,7 @@ class Role {
Role(const std::string &rolename, const Permissions &permissions);
#ifdef MG_ENTERPRISE
Role(const std::string &rolename, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
FineGrainedAccessHandler fine_grained_access_handler);
#endif
Role(const Role &) = default;
Role &operator=(const Role &) = default;
@@ -313,23 +224,12 @@ class Role {
const std::string &rolename() const;
const Permissions &permissions() const;
Permissions &permissions();
Permissions GetPermissions() const { return permissions_; }
#ifdef MG_ENTERPRISE
const FineGrainedAccessHandler &fine_grained_access_handler() const;
FineGrainedAccessHandler &fine_grained_access_handler();
const FineGrainedAccessPermissions &GetFineGrainedAccessLabelPermissions() const;
const FineGrainedAccessPermissions &GetFineGrainedAccessEdgeTypePermissions() const;
#endif
#ifdef MG_ENTERPRISE
Databases &db_access() { return db_access_; }
const Databases &db_access() const { return db_access_; }
bool DeniesDB(std::string_view db_name) const { return db_access_.Denies(db_name); }
bool GrantsDB(std::string_view db_name) const { return db_access_.Grants(db_name); }
bool HasAccess(std::string_view db_name) const { return !DeniesDB(db_name) && GrantsDB(db_name); }
#endif
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
@@ -342,12 +242,93 @@ class Role {
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
Databases db_access_;
#endif
};
bool operator==(const Role &first, const Role &second);
#ifdef MG_ENTERPRISE
class Databases final {
public:
Databases() : grants_dbs_{std::string{dbms::kDefaultDB}}, allow_all_(false), default_db_(dbms::kDefaultDB) {}
Databases(const Databases &) = default;
Databases &operator=(const Databases &) = default;
Databases(Databases &&) noexcept = default;
Databases &operator=(Databases &&) noexcept = default;
~Databases() = default;
/**
* @brief Add database to the list of granted access. @note allow_all_ will be false after execution
*
* @param db name of the database to grant access to
*/
void Add(std::string_view db);
/**
* @brief Remove database to the list of granted access.
* @note if allow_all_ is set, the flag will remain set and the
* database will be added to the set of denied databases.
*
* @param db name of the database to grant access to
*/
void Remove(const std::string &db);
/**
* @brief Called when database is dropped. Removes it from granted (if allow_all is false) and denied set.
* @note allow_all_ is not changed
*
* @param db name of the database to grant access to
*/
void Delete(const std::string &db);
/**
* @brief Set allow_all_ to true and clears grants and denied sets.
*/
void GrantAll();
/**
* @brief Set allow_all_ to false and clears grants and denied sets.
*/
void DenyAll();
/**
* @brief Set the default database.
*/
bool SetDefault(std::string_view db);
/**
* @brief Checks if access is grated to the database.
*
* @param db name of the database
* @return true if allow_all and not denied or granted
*/
bool Contains(std::string_view db) const;
bool GetAllowAll() const { return allow_all_; }
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
const std::string &GetDefault() const;
nlohmann::json Serialize() const;
/// @throw AuthException if unable to deserialize.
static Databases Deserialize(const nlohmann::json &data);
private:
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
std::string default_db = std::string{dbms::kDefaultDB})
: grants_dbs_(std::move(grant)),
denies_dbs_(std::move(deny)),
allow_all_(allow_all),
default_db_(std::move(default_db)) {}
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
std::string default_db_; //!< user's default database
};
#endif
// TODO (mferencevic): Implement password expiry.
class User final {
public:
@@ -407,18 +388,6 @@ class User final {
#ifdef MG_ENTERPRISE
Databases &db_access() { return database_access_; }
const Databases &db_access() const { return database_access_; }
bool DeniesDB(std::string_view db_name) const {
bool denies = database_access_.Denies(db_name);
if (role_) denies |= role_->DeniesDB(db_name);
return denies;
}
bool GrantsDB(std::string_view db_name) const {
bool grants = database_access_.Grants(db_name);
if (role_) grants |= role_->GrantsDB(db_name);
return grants;
}
bool HasAccess(std::string_view db_name) const { return !DeniesDB(db_name) && GrantsDB(db_name); }
#endif
nlohmann::json Serialize() const;
@@ -434,7 +403,7 @@ class User final {
Permissions permissions_;
#ifdef MG_ENTERPRISE
FineGrainedAccessHandler fine_grained_access_handler_;
Databases database_access_{};
Databases database_access_;
#endif
std::optional<Role> role_;
};

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -403,7 +403,7 @@ nlohmann::json Module::Call(const nlohmann::json &params, int timeout_millisec)
return ret;
}
bool Module::IsUsed() const { return !module_executable_path_.empty(); }
bool Module::IsUsed() { return !module_executable_path_.empty(); }
void Module::Shutdown() {
if (pid_ == -1) return;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
@@ -49,7 +49,7 @@ class Module final {
/// specified executable path and can thus be used.
///
/// @return boolean indicating whether the module can be used
bool IsUsed() const;
bool IsUsed();
~Module();

View File

@@ -18,9 +18,11 @@
#include "utils/enum.hpp"
namespace memgraph::slk {
// Serialize code for auth::Role
void Save(const auth::Role &self, Builder *builder) { memgraph::slk::Save(self.Serialize().dump(), builder); }
// Serialize code for auth::Role
void Save(const auth::Role &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.Serialize().dump(), builder);
}
namespace {
auth::Role LoadAuthRole(memgraph::slk::Reader *reader) {
std::string tmp;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,9 +15,6 @@
#include "communication/bolt/v1/value.hpp"
#include "utils/logging.hpp"
#include "communication/bolt/v1/fmt.hpp"
#include "io/network/fmt.hpp"
namespace {
constexpr uint8_t kBoltV43Version[4] = {0x00, 0x00, 0x03, 0x04};
constexpr uint8_t kEmptyBoltVersion[4] = {0x00, 0x00, 0x00, 0x00};

View File

@@ -1,27 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#if FMT_VERSION > 90000
#include <fmt/ostream.h>
#include "communication/bolt/v1/value.hpp"
template <>
class fmt::formatter<memgraph::communication::bolt::Value> : public fmt::ostream_formatter {};
template <>
class fmt::formatter<std::vector<memgraph::communication::bolt::Value>> : public fmt::ostream_formatter {};
template <>
class fmt::formatter<std::map<std::string, memgraph::communication::bolt::Value>> : public fmt::ostream_formatter {};
#endif

View File

@@ -1,20 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#if FMT_VERSION > 90000
#include <fmt/ostream.h>
#include <boost/asio/ip/tcp.hpp>
template <>
class fmt::formatter<boost::asio::ip::tcp::endpoint> : public fmt::ostream_formatter {};
#endif

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -21,7 +21,6 @@
#include <boost/beast/core.hpp>
#include "communication/context.hpp"
#include "communication/fmt.hpp"
#include "communication/http/session.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
@@ -83,7 +82,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TRequestHand
return;
}
spdlog::info("HTTP server is listening on {}", endpoint);
spdlog::info("HTTP server is listening on {}:{}", endpoint.address(), endpoint.port());
}
void DoAccept() {

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -23,7 +23,6 @@
#include "communication/session.hpp"
#include "io/network/epoll.hpp"
#include "io/network/fmt.hpp"
#include "io/network/socket.hpp"
#include "utils/logging.hpp"
#include "utils/signals.hpp"

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,7 +22,6 @@
#include "communication/init.hpp"
#include "communication/listener.hpp"
#include "io/network/fmt.hpp"
#include "io/network/socket.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -26,7 +26,6 @@
#include <boost/asio/ip/tcp.hpp>
#include "communication/context.hpp"
#include "communication/fmt.hpp"
#include "communication/init.hpp"
#include "communication/v2/listener.hpp"
#include "communication/v2/pool.hpp"
@@ -130,7 +129,7 @@ bool Server<TSession, TSessionContext>::Start() {
listener_->Start();
spdlog::info("{} server is fully armed and operational", service_name_);
spdlog::info("{} listening on {}", service_name_, endpoint_);
spdlog::info("{} listening on {}", service_name_, endpoint_.address());
context_thread_pool_.Run();
return true;

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -47,7 +47,6 @@
#include "communication/buffer.hpp"
#include "communication/context.hpp"
#include "communication/exceptions.hpp"
#include "communication/fmt.hpp"
#include "dbms/global.hpp"
#include "utils/event_counter.hpp"
#include "utils/logging.hpp"
@@ -213,11 +212,14 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
session_.Execute();
DoRead();
} catch (const SessionClosedException &e) {
spdlog::info("{} client {} closed the connection.", service_name_, remote_endpoint_);
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
remote_endpoint_.port());
DoClose();
} catch (const std::exception &e) {
spdlog::error("Exception was thrown while processing event in {} session associated with {}", service_name_,
remote_endpoint_);
spdlog::error(
"Exception was thrown while processing event in {} session "
"associated with {}:{}",
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
spdlog::debug("Exception message: {}", e.what());
DoClose();
}
@@ -374,7 +376,8 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
socket.lowest_layer().non_blocking(false);
});
timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
spdlog::info("Accepted a connection from {}: {}", service_name_, remote_endpoint_);
spdlog::info("Accepted a connection from {}: {}:{}", service_name_, remote_endpoint_.address(),
remote_endpoint_.port());
}
void DoRead() {
@@ -434,11 +437,14 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
session_.Execute();
DoRead();
} catch (const SessionClosedException &e) {
spdlog::info("{} client {} closed the connection.", service_name_, remote_endpoint_);
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
remote_endpoint_.port());
DoShutdown();
} catch (const std::exception &e) {
spdlog::error("Exception was thrown while processing event in {} session associated with {}", service_name_,
remote_endpoint_);
spdlog::error(
"Exception was thrown while processing event in {} session "
"associated with {}:{}",
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
spdlog::debug("Exception message: {}", e.what());
DoShutdown();
}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,44 +12,19 @@
#include "communication/websocket/auth.hpp"
#include <string>
#include "utils/variant_helpers.hpp"
namespace memgraph::communication::websocket {
bool SafeAuth::Authenticate(const std::string &username, const std::string &password) const {
user_or_role_ = auth_->Lock()->Authenticate(username, password);
return user_or_role_.has_value();
return auth_->Lock()->Authenticate(username, password).has_value();
}
bool SafeAuth::HasPermission(const auth::Permission permission) const {
auto locked_auth = auth_->ReadLock();
// Update if cache invalidated
if (!locked_auth->UpToDate(auth_epoch_) && user_or_role_) {
bool success = true;
std::visit(utils::Overloaded{[&](auth::User &user) {
auto tmp = locked_auth->GetUser(user.username());
if (!tmp) success = false;
user = std::move(*tmp);
},
[&](auth::Role &role) {
auto tmp = locked_auth->GetRole(role.rolename());
if (!tmp) success = false;
role = std::move(*tmp);
}},
*user_or_role_);
// Missing user/role; delete from cache
if (!success) user_or_role_.reset();
bool SafeAuth::HasUserPermission(const std::string &username, const auth::Permission permission) const {
if (const auto user = auth_->ReadLock()->GetUser(username); user) {
return user->GetPermissions().Has(permission) == auth::PermissionLevel::GRANT;
}
// Check permissions
if (user_or_role_) {
return std::visit(utils::Overloaded{[&](auto &user_or_role) {
return user_or_role.GetPermissions().Has(permission) == auth::PermissionLevel::GRANT;
}},
*user_or_role_);
}
// NOTE: websocket authenticates only if there is a user, so no need to check if access controlled
return false;
}
bool SafeAuth::AccessControlled() const { return auth_->ReadLock()->AccessControlled(); }
bool SafeAuth::HasAnyUsers() const { return auth_->ReadLock()->HasUsers(); }
} // namespace memgraph::communication::websocket

View File

@@ -21,9 +21,9 @@ class AuthenticationInterface {
public:
virtual bool Authenticate(const std::string &username, const std::string &password) const = 0;
virtual bool HasPermission(auth::Permission permission) const = 0;
virtual bool HasUserPermission(const std::string &username, auth::Permission permission) const = 0;
virtual bool AccessControlled() const = 0;
virtual bool HasAnyUsers() const = 0;
};
class SafeAuth : public AuthenticationInterface {
@@ -32,13 +32,11 @@ class SafeAuth : public AuthenticationInterface {
bool Authenticate(const std::string &username, const std::string &password) const override;
bool HasPermission(auth::Permission permission) const override;
bool HasUserPermission(const std::string &username, auth::Permission permission) const override;
bool AccessControlled() const override;
bool HasAnyUsers() const override;
private:
auth::SynchedAuth *auth_;
mutable std::optional<auth::UserOrRole> user_or_role_;
mutable auth::Auth::Epoch auth_epoch_{};
};
} // namespace memgraph::communication::websocket

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,7 +10,6 @@
// licenses/APL.txt.
#include "communication/websocket/listener.hpp"
#include "communication/fmt.hpp"
namespace memgraph::communication::websocket {
namespace {
@@ -62,7 +61,7 @@ Listener::Listener(boost::asio::io_context &ioc, ServerContext *context, tcp::en
return;
}
spdlog::info("WebSocket server is listening on {}", endpoint);
spdlog::info("WebSocket server is listening on {}:{}", endpoint.address(), endpoint.port());
}
void Listener::DoAccept() {

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -80,7 +80,7 @@ bool Session::Run() {
return false;
}
authenticated_ = !auth_.AccessControlled();
authenticated_ = !auth_.HasAnyUsers();
connected_.store(true, std::memory_order_relaxed);
// run on the strand
@@ -162,7 +162,7 @@ utils::BasicResult<std::string> Session::Authorize(const nlohmann::json &creds)
return {"Authentication failed!"};
}
#ifdef MG_ENTERPRISE
if (!auth_.HasPermission(auth::Permission::WEBSOCKET)) {
if (!auth_.HasUserPermission(creds.at("username").get<std::string>(), auth::Permission::WEBSOCKET)) {
return {"Authorization failed!"};
}
#endif

View File

@@ -11,10 +11,10 @@ target_sources(mg-coordination
include/coordination/coordinator_slk.hpp
include/coordination/coordinator_instance.hpp
include/coordination/coordinator_handlers.hpp
include/coordination/constants.hpp
include/coordination/instance_status.hpp
include/coordination/replication_instance.hpp
include/coordination/raft_state.hpp
include/coordination/rpc_errors.hpp
include/nuraft/coordinator_log_store.hpp
include/nuraft/coordinator_state_machine.hpp

View File

@@ -17,7 +17,6 @@
#include "coordination/coordinator_config.hpp"
#include "coordination/coordinator_rpc.hpp"
#include "replication_coordination_glue/messages.hpp"
#include "utils/result.hpp"
namespace memgraph::coordination {
@@ -46,10 +45,6 @@ auto CoordinatorClient::InstanceDownTimeoutSec() const -> std::chrono::seconds {
return config_.instance_down_timeout_sec;
}
auto CoordinatorClient::InstanceGetUUIDFrequencySec() const -> std::chrono::seconds {
return config_.instance_get_uuid_frequency_sec;
}
void CoordinatorClient::StartFrequentCheck() {
if (instance_checker_.IsRunning()) {
return;
@@ -145,31 +140,5 @@ auto CoordinatorClient::SendUnregisterReplicaRpc(std::string const &instance_nam
return false;
}
auto CoordinatorClient::SendGetInstanceUUIDRpc() const
-> utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>> {
try {
auto stream{rpc_client_.Stream<GetInstanceUUIDRpc>()};
auto res = stream.AwaitResponse();
return res.uuid;
} catch (const rpc::RpcFailedException &) {
spdlog::error("RPC error occured while sending GetInstance UUID RPC");
return GetInstanceUUIDError::RPC_EXCEPTION;
}
}
auto CoordinatorClient::SendEnableWritingOnMainRpc() const -> bool {
try {
auto stream{rpc_client_.Stream<EnableWritingOnMainRpc>()};
if (!stream.AwaitResponse().success) {
spdlog::error("Failed to receive successful RPC response for enabling writing on main!");
return false;
}
return true;
} catch (rpc::RpcFailedException const &) {
spdlog::error("Failed to enable writing on main!");
}
return false;
}
} // namespace memgraph::coordination
#endif

View File

@@ -45,18 +45,6 @@ void CoordinatorHandlers::Register(memgraph::coordination::CoordinatorServer &se
spdlog::info("Received UnregisterReplicaRpc on coordinator server");
CoordinatorHandlers::UnregisterReplicaHandler(replication_handler, req_reader, res_builder);
});
server.Register<coordination::EnableWritingOnMainRpc>(
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
spdlog::info("Received EnableWritingOnMainRpc on coordinator server");
CoordinatorHandlers::EnableWritingOnMainHandler(replication_handler, req_reader, res_builder);
});
server.Register<coordination::GetInstanceUUIDRpc>(
[&replication_handler](slk::Reader *req_reader, slk::Builder *res_builder) -> void {
spdlog::info("Received GetInstanceUUIDRpc on coordinator server");
CoordinatorHandlers::GetInstanceUUIDHandler(replication_handler, req_reader, res_builder);
});
}
void CoordinatorHandlers::SwapMainUUIDHandler(replication::ReplicationHandler &replication_handler,
@@ -80,6 +68,12 @@ void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHan
slk::Reader *req_reader, slk::Builder *res_builder) {
spdlog::info("Executing DemoteMainToReplicaHandler");
if (!replication_handler.IsMain()) {
spdlog::error("Setting to replica must be performed on main.");
slk::Save(coordination::DemoteMainToReplicaRes{false}, res_builder);
return;
}
coordination::DemoteMainToReplicaReq req;
slk::Load(&req, req_reader);
@@ -89,18 +83,11 @@ void CoordinatorHandlers::DemoteMainToReplicaHandler(replication::ReplicationHan
if (!replication_handler.SetReplicationRoleReplica(clients_config, std::nullopt)) {
spdlog::error("Demoting main to replica failed!");
slk::Save(coordination::DemoteMainToReplicaRes{false}, res_builder);
slk::Save(coordination::PromoteReplicaToMainRes{false}, res_builder);
return;
}
slk::Save(coordination::DemoteMainToReplicaRes{true}, res_builder);
}
void CoordinatorHandlers::GetInstanceUUIDHandler(replication::ReplicationHandler &replication_handler,
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
spdlog::info("Executing GetInstanceUUIDHandler");
slk::Save(coordination::GetInstanceUUIDRes{replication_handler.GetReplicaUUID()}, res_builder);
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
}
void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHandler &replication_handler,
@@ -132,7 +119,7 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
// registering replicas
for (auto const &config : req.replication_clients_info | ranges::views::transform(converter)) {
auto instance_client = replication_handler.RegisterReplica(config);
auto instance_client = replication_handler.RegisterReplica(config, false);
if (instance_client.HasError()) {
using enum memgraph::replication::RegisterReplicaError;
switch (instance_client.GetError()) {
@@ -161,7 +148,7 @@ void CoordinatorHandlers::PromoteReplicaToMainHandler(replication::ReplicationHa
}
}
}
spdlog::info("Promote replica to main was success {}", std::string(req.main_uuid_));
spdlog::error(fmt::format("FICO : Promote replica to main was success {}", std::string(req.main_uuid_)));
slk::Save(coordination::PromoteReplicaToMainRes{true}, res_builder);
}
@@ -197,22 +184,5 @@ void CoordinatorHandlers::UnregisterReplicaHandler(replication::ReplicationHandl
}
}
void CoordinatorHandlers::EnableWritingOnMainHandler(replication::ReplicationHandler &replication_handler,
slk::Reader * /*req_reader*/, slk::Builder *res_builder) {
if (!replication_handler.IsMain()) {
spdlog::error("Enable writing on main must be performed on main!");
slk::Save(coordination::EnableWritingOnMainRes{false}, res_builder);
return;
}
if (!replication_handler.GetReplState().EnableWritingOnMain()) {
spdlog::error("Enabling writing on main failed!");
slk::Save(coordination::EnableWritingOnMainRes{false}, res_builder);
return;
}
slk::Save(coordination::EnableWritingOnMainRes{true}, res_builder);
}
} // namespace memgraph::dbms
#endif

View File

@@ -14,11 +14,9 @@
#include "coordination/coordinator_instance.hpp"
#include "coordination/coordinator_exceptions.hpp"
#include "coordination/fmt.hpp"
#include "nuraft/coordinator_state_machine.hpp"
#include "nuraft/coordinator_state_manager.hpp"
#include "utils/counter.hpp"
#include "utils/functional.hpp"
#include <range/v3/view.hpp>
#include <shared_mutex>
@@ -49,12 +47,9 @@ CoordinatorInstance::CoordinatorInstance()
spdlog::trace("Instance {} performing replica successful callback", repl_instance_name);
auto &repl_instance = find_repl_instance(self, repl_instance_name);
// We need to get replicas UUID from time to time to ensure replica is listening to correct main
// and that it didn't go down for less time than we could notice
// We need to get id of main replica is listening to
// and swap if necessary
if (!repl_instance.EnsureReplicaHasCorrectMainUUID(self->GetMainUUID())) {
spdlog::error("Failed to swap uuid for replica instance {} which is alive", repl_instance.InstanceName());
spdlog::error(
fmt::format("Failed to swap uuid for replica instance {} which is alive", repl_instance.InstanceName()));
return;
}
@@ -66,6 +61,14 @@ CoordinatorInstance::CoordinatorInstance()
spdlog::trace("Instance {} performing replica failure callback", repl_instance_name);
auto &repl_instance = find_repl_instance(self, repl_instance_name);
repl_instance.OnFailPing();
// We need to restart main uuid from instance since it was "down" at least a second
// There is slight delay, if we choose to use isAlive, instance can be down and back up in less than
// our isAlive time difference, which would lead to instance setting UUID to nullopt and stopping accepting any
// incoming RPCs from valid main
// TODO(antoniofilipovic) this needs here more complex logic
// We need to get id of main replica is listening to on successful ping
// and swap it to correct uuid if it failed
repl_instance.ResetMainUUID();
};
main_succ_cb_ = [find_repl_instance](CoordinatorInstance *self, std::string_view repl_instance_name) -> void {
@@ -84,11 +87,6 @@ CoordinatorInstance::CoordinatorInstance()
auto const curr_main_uuid = self->GetMainUUID();
if (curr_main_uuid == repl_instance_uuid.value()) {
if (!repl_instance.EnableWritingOnMain()) {
spdlog::error("Failed to enable writing on main instance {}", repl_instance_name);
return;
}
repl_instance.OnSuccessPing();
return;
}
@@ -127,6 +125,9 @@ CoordinatorInstance::CoordinatorInstance()
auto CoordinatorInstance::ShowInstances() const -> std::vector<InstanceStatus> {
auto const coord_instances = raft_state_.GetAllCoordinators();
std::vector<InstanceStatus> instances_status;
instances_status.reserve(repl_instances_.size() + coord_instances.size());
auto const stringify_repl_role = [](ReplicationInstance const &instance) -> std::string {
if (!instance.IsAlive()) return "unknown";
if (instance.IsMain()) return "main";
@@ -148,7 +149,8 @@ auto CoordinatorInstance::ShowInstances() const -> std::vector<InstanceStatus> {
// CoordinatorState to every instance, we can be smarter about this using our RPC.
};
auto instances_status = utils::fmap(coord_instance_to_status, coord_instances);
std::ranges::transform(coord_instances, std::back_inserter(instances_status), coord_instance_to_status);
{
auto lock = std::shared_lock{coord_instance_lock_};
std::ranges::transform(repl_instances_, std::back_inserter(instances_status), repl_instance_to_status);
@@ -187,9 +189,10 @@ auto CoordinatorInstance::TryFailover() -> void {
}
}
auto repl_clients_info = repl_instances_ | ranges::views::filter(is_not_new_main) |
ranges::views::transform(&ReplicationInstance::ReplicationClientInfo) |
ranges::to<ReplicationClientsInfo>();
ReplicationClientsInfo repl_clients_info;
repl_clients_info.reserve(repl_instances_.size() - 1);
std::ranges::transform(repl_instances_ | ranges::views::filter(is_not_new_main),
std::back_inserter(repl_clients_info), &ReplicationInstance::ReplicationClientInfo);
if (!new_main->PromoteToMain(new_main_uuid, std::move(repl_clients_info), main_succ_cb_, main_fail_cb_)) {
spdlog::warn("Failover failed since promoting replica to main failed!");
@@ -205,10 +208,6 @@ auto CoordinatorInstance::SetReplicationInstanceToMain(std::string instance_name
-> SetInstanceToMainCoordinatorStatus {
auto lock = std::lock_guard{coord_instance_lock_};
if (std::ranges::any_of(repl_instances_, &ReplicationInstance::IsMain)) {
return SetInstanceToMainCoordinatorStatus::MAIN_ALREADY_EXISTS;
}
auto const is_new_main = [&instance_name](ReplicationInstance const &instance) {
return instance.InstanceName() == instance_name;
};

View File

@@ -68,35 +68,6 @@ void UnregisterReplicaRes::Load(UnregisterReplicaRes *self, memgraph::slk::Reade
memgraph::slk::Load(self, reader);
}
void EnableWritingOnMainRes::Save(EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void EnableWritingOnMainRes::Load(EnableWritingOnMainRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
void EnableWritingOnMainReq::Save(EnableWritingOnMainReq const &self, memgraph::slk::Builder *builder) {}
void EnableWritingOnMainReq::Load(EnableWritingOnMainReq *self, memgraph::slk::Reader *reader) {}
// GetInstanceUUID
void GetInstanceUUIDReq::Save(const GetInstanceUUIDReq &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void GetInstanceUUIDReq::Load(GetInstanceUUIDReq *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
void GetInstanceUUIDRes::Save(const GetInstanceUUIDRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self, builder);
}
void GetInstanceUUIDRes::Load(GetInstanceUUIDRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(self, reader);
}
} // namespace coordination
constexpr utils::TypeInfo coordination::PromoteReplicaToMainReq::kType{utils::TypeId::COORD_FAILOVER_REQ,
@@ -118,22 +89,8 @@ constexpr utils::TypeInfo coordination::UnregisterReplicaReq::kType{utils::TypeI
constexpr utils::TypeInfo coordination::UnregisterReplicaRes::kType{utils::TypeId::COORD_UNREGISTER_REPLICA_RES,
"UnregisterReplicaRes", nullptr};
constexpr utils::TypeInfo coordination::EnableWritingOnMainReq::kType{utils::TypeId::COORD_ENABLE_WRITING_ON_MAIN_REQ,
"CoordEnableWritingOnMainReq", nullptr};
constexpr utils::TypeInfo coordination::EnableWritingOnMainRes::kType{utils::TypeId::COORD_ENABLE_WRITING_ON_MAIN_RES,
"CoordEnableWritingOnMainRes", nullptr};
constexpr utils::TypeInfo coordination::GetInstanceUUIDReq::kType{utils::TypeId::COORD_GET_UUID_REQ, "CoordGetUUIDReq",
nullptr};
constexpr utils::TypeInfo coordination::GetInstanceUUIDRes::kType{utils::TypeId::COORD_GET_UUID_RES, "CoordGetUUIDRes",
nullptr};
namespace slk {
// PromoteReplicaToMainRpc
void Save(const memgraph::coordination::PromoteReplicaToMainRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.success, builder);
}
@@ -152,7 +109,6 @@ void Load(memgraph::coordination::PromoteReplicaToMainReq *self, memgraph::slk::
memgraph::slk::Load(&self->replication_clients_info, reader);
}
// DemoteMainToReplicaRpc
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.replication_client_info, builder);
}
@@ -169,8 +125,6 @@ void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::R
memgraph::slk::Load(&self->success, reader);
}
// UnregisterReplicaRpc
void Save(memgraph::coordination::UnregisterReplicaReq const &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.instance_name, builder);
}
@@ -187,32 +141,6 @@ void Load(memgraph::coordination::UnregisterReplicaRes *self, memgraph::slk::Rea
memgraph::slk::Load(&self->success, reader);
}
void Save(memgraph::coordination::EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.success, builder);
}
void Load(memgraph::coordination::EnableWritingOnMainRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(&self->success, reader);
}
// GetInstanceUUIDRpc
void Save(const memgraph::coordination::GetInstanceUUIDReq & /*self*/, memgraph::slk::Builder * /*builder*/) {
/* nothing to serialize*/
}
void Load(memgraph::coordination::GetInstanceUUIDReq * /*self*/, memgraph::slk::Reader * /*reader*/) {
/* nothing to serialize*/
}
void Save(const memgraph::coordination::GetInstanceUUIDRes &self, memgraph::slk::Builder *builder) {
memgraph::slk::Save(self.uuid, builder);
}
void Load(memgraph::coordination::GetInstanceUUIDRes *self, memgraph::slk::Reader *reader) {
memgraph::slk::Load(&self->uuid, reader);
}
} // namespace slk
} // namespace memgraph

View File

@@ -1,60 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#if FMT_VERSION > 90000
#include <fmt/ostream.h>
#include <string>
#include <libnuraft/nuraft.hxx>
#include "utils/logging.hpp"
inline std::string ToString(const nuraft::cmd_result_code &code) {
switch (code) {
case nuraft::cmd_result_code::OK:
return "OK";
case nuraft::cmd_result_code::FAILED:
return "FAILED";
case nuraft::cmd_result_code::RESULT_NOT_EXIST_YET:
return "RESULT_NOT_EXIST_YET";
case nuraft::cmd_result_code::TERM_MISMATCH:
return "TERM_MISMATCH";
case nuraft::cmd_result_code::SERVER_IS_LEAVING:
return "SERVER_IS_LEAVING";
case nuraft::cmd_result_code::CANNOT_REMOVE_LEADER:
return "CANNOT_REMOVE_LEADER";
case nuraft::cmd_result_code::SERVER_NOT_FOUND:
return "SERVER_NOT_FOUND";
case nuraft::cmd_result_code::SERVER_IS_JOINING:
return "SERVER_IS_JOINING";
case nuraft::cmd_result_code::CONFIG_CHANGING:
return "CONFIG_CHANGING";
case nuraft::cmd_result_code::SERVER_ALREADY_EXISTS:
return "SERVER_ALREADY_EXISTS";
case nuraft::cmd_result_code::BAD_REQUEST:
return "BAD_REQUEST";
case nuraft::cmd_result_code::NOT_LEADER:
return "NOT_LEADER";
case nuraft::cmd_result_code::TIMEOUT:
return "TIMEOUT";
case nuraft::cmd_result_code::CANCELLED:
return "CANCELLED";
}
LOG_FATAL("ToString of a nuraft::cmd_result_code -> check missing switch case");
}
inline std::ostream &operator<<(std::ostream &os, const nuraft::cmd_result_code &code) {
os << ToString(code);
return os;
}
template <>
class fmt::formatter<nuraft::cmd_result_code> : public fmt::ostream_formatter {};
#endif

View File

@@ -9,6 +9,14 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::coordination {
enum class GetInstanceUUIDError { NO_RESPONSE, RPC_EXCEPTION };
#ifdef MG_EXPERIMENTAL_HIGH_AVAILABILITY
constexpr bool allow_ha = true;
#else
constexpr bool allow_ha = false;
#endif
} // namespace memgraph::coordination

View File

@@ -11,14 +11,12 @@
#pragma once
#include "utils/uuid.hpp"
#ifdef MG_ENTERPRISE
#include "coordination/coordinator_config.hpp"
#include "rpc/client.hpp"
#include "rpc_errors.hpp"
#include "utils/result.hpp"
#include "utils/scheduler.hpp"
#include "utils/uuid.hpp"
namespace memgraph::coordination {
@@ -48,7 +46,7 @@ class CoordinatorClient {
auto SocketAddress() const -> std::string;
[[nodiscard]] auto DemoteToReplica() const -> bool;
// TODO: (andi) Consistent naming
auto SendPromoteReplicaToMainRpc(const utils::UUID &uuid, ReplicationClientsInfo replication_clients_info) const
-> bool;
@@ -56,10 +54,6 @@ class CoordinatorClient {
auto SendUnregisterReplicaRpc(std::string const &instance_name) const -> bool;
auto SendEnableWritingOnMainRpc() const -> bool;
auto SendGetInstanceUUIDRpc() const -> memgraph::utils::BasicResult<GetInstanceUUIDError, std::optional<utils::UUID>>;
auto ReplicationClientInfo() const -> ReplClientInfo;
auto SetCallbacks(HealthCheckCallback succ_cb, HealthCheckCallback fail_cb) -> void;
@@ -68,8 +62,6 @@ class CoordinatorClient {
auto InstanceDownTimeoutSec() const -> std::chrono::seconds;
auto InstanceGetUUIDFrequencySec() const -> std::chrono::seconds;
friend bool operator==(CoordinatorClient const &first, CoordinatorClient const &second) {
return first.config_ == second.config_;
}
@@ -77,6 +69,7 @@ class CoordinatorClient {
private:
utils::Scheduler instance_checker_;
// TODO: (andi) Pimpl?
communication::ClientContext rpc_context_;
mutable rpc::Client rpc_client_;

View File

@@ -30,7 +30,6 @@ struct CoordinatorClientConfig {
uint16_t port{};
std::chrono::seconds instance_health_check_frequency_sec{1};
std::chrono::seconds instance_down_timeout_sec{5};
std::chrono::seconds instance_get_uuid_frequency_sec{10};
auto SocketAddress() const -> std::string { return ip_address + ":" + std::to_string(port); }

View File

@@ -36,11 +36,6 @@ class CoordinatorHandlers {
static void UnregisterReplicaHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
slk::Builder *res_builder);
static void EnableWritingOnMainHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
slk::Builder *res_builder);
static void GetInstanceUUIDHandler(replication::ReplicationHandler &replication_handler, slk::Reader *req_reader,
slk::Builder *res_builder);
};
} // namespace memgraph::dbms

View File

@@ -111,56 +111,6 @@ struct UnregisterReplicaRes {
using UnregisterReplicaRpc = rpc::RequestResponse<UnregisterReplicaReq, UnregisterReplicaRes>;
struct EnableWritingOnMainReq {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(EnableWritingOnMainReq *self, memgraph::slk::Reader *reader);
static void Save(EnableWritingOnMainReq const &self, memgraph::slk::Builder *builder);
EnableWritingOnMainReq() = default;
};
struct EnableWritingOnMainRes {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(EnableWritingOnMainRes *self, memgraph::slk::Reader *reader);
static void Save(EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder);
explicit EnableWritingOnMainRes(bool success) : success(success) {}
EnableWritingOnMainRes() = default;
bool success;
};
using EnableWritingOnMainRpc = rpc::RequestResponse<EnableWritingOnMainReq, EnableWritingOnMainRes>;
struct GetInstanceUUIDReq {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(GetInstanceUUIDReq *self, memgraph::slk::Reader *reader);
static void Save(const GetInstanceUUIDReq &self, memgraph::slk::Builder *builder);
GetInstanceUUIDReq() = default;
};
struct GetInstanceUUIDRes {
static const utils::TypeInfo kType;
static const utils::TypeInfo &GetTypeInfo() { return kType; }
static void Load(GetInstanceUUIDRes *self, memgraph::slk::Reader *reader);
static void Save(const GetInstanceUUIDRes &self, memgraph::slk::Builder *builder);
explicit GetInstanceUUIDRes(std::optional<utils::UUID> uuid) : uuid(uuid) {}
GetInstanceUUIDRes() = default;
std::optional<utils::UUID> uuid;
};
using GetInstanceUUIDRpc = rpc::RequestResponse<GetInstanceUUIDReq, GetInstanceUUIDRes>;
} // namespace memgraph::coordination
// SLK serialization declarations
@@ -178,20 +128,12 @@ void Load(memgraph::coordination::DemoteMainToReplicaRes *self, memgraph::slk::R
void Save(const memgraph::coordination::DemoteMainToReplicaReq &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::DemoteMainToReplicaReq *self, memgraph::slk::Reader *reader);
// GetInstanceUUIDRpc
void Save(const memgraph::coordination::GetInstanceUUIDReq &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::GetInstanceUUIDReq *self, memgraph::slk::Reader *reader);
void Save(const memgraph::coordination::GetInstanceUUIDRes &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::GetInstanceUUIDRes *self, memgraph::slk::Reader *reader);
// UnregisterReplicaRpc
void Save(memgraph::coordination::UnregisterReplicaRes const &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::UnregisterReplicaRes *self, memgraph::slk::Reader *reader);
void Save(memgraph::coordination::UnregisterReplicaReq const &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::UnregisterReplicaReq *self, memgraph::slk::Reader *reader);
void Save(memgraph::coordination::EnableWritingOnMainRes const &self, memgraph::slk::Builder *builder);
void Load(memgraph::coordination::EnableWritingOnMainRes *self, memgraph::slk::Reader *reader);
} // namespace memgraph::slk
#endif

View File

@@ -15,6 +15,8 @@
#include <flags/replication.hpp>
#include <optional>
#include <libnuraft/nuraft.hxx>
namespace memgraph::coordination {

View File

@@ -39,7 +39,6 @@ enum class UnregisterInstanceCoordinatorStatus : uint8_t {
enum class SetInstanceToMainCoordinatorStatus : uint8_t {
NO_INSTANCE_WITH_NAME,
MAIN_ALREADY_EXISTS,
NOT_COORDINATOR,
SUCCESS,
COULD_NOT_PROMOTE_TO_MAIN,

View File

@@ -18,7 +18,6 @@
#include "replication_coordination_glue/role.hpp"
#include <libnuraft/nuraft.hxx>
#include "utils/result.hpp"
#include "utils/uuid.hpp"
namespace memgraph::coordination {
@@ -38,9 +37,6 @@ class ReplicationInstance {
auto OnSuccessPing() -> void;
auto OnFailPing() -> bool;
auto IsReadyForUUIDPing() -> bool;
void UpdateReplicaLastResponseUUID();
auto IsAlive() const -> bool;
@@ -66,12 +62,9 @@ class ReplicationInstance {
auto SendSwapAndUpdateUUID(const utils::UUID &new_main_uuid) -> bool;
auto SendUnregisterReplicaRpc(std::string const &instance_name) -> bool;
auto SendGetInstanceUUID() -> utils::BasicResult<coordination::GetInstanceUUIDError, std::optional<utils::UUID>>;
// TODO: (andi) Inconsistent API
auto GetClient() -> CoordinatorClient &;
auto EnableWritingOnMain() -> bool;
auto SetNewMainUUID(utils::UUID const &main_uuid) -> void;
auto ResetMainUUID() -> void;
auto GetMainUUID() const -> const std::optional<utils::UUID> &;
@@ -81,7 +74,6 @@ class ReplicationInstance {
replication_coordination_glue::ReplicationRole replication_role_;
std::chrono::system_clock::time_point last_response_time_{};
bool is_alive_{false};
std::chrono::system_clock::time_point last_check_of_uuid_{};
// for replica this is main uuid of current main
// for "main" main this same as in CoordinatorData

View File

@@ -14,7 +14,6 @@
#include "coordination/replication_instance.hpp"
#include "replication_coordination_glue/handler.hpp"
#include "utils/result.hpp"
namespace memgraph::coordination {
@@ -40,11 +39,6 @@ auto ReplicationInstance::OnFailPing() -> bool {
return is_alive_;
}
auto ReplicationInstance::IsReadyForUUIDPing() -> bool {
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - last_check_of_uuid_) >
client_.InstanceGetUUIDFrequencySec();
}
auto ReplicationInstance::InstanceName() const -> std::string { return client_.InstanceName(); }
auto ReplicationInstance::SocketAddress() const -> std::string { return client_.SocketAddress(); }
auto ReplicationInstance::IsAlive() const -> bool { return is_alive_; }
@@ -97,20 +91,10 @@ auto ReplicationInstance::ResetMainUUID() -> void { main_uuid_ = std::nullopt; }
auto ReplicationInstance::GetMainUUID() const -> std::optional<utils::UUID> const & { return main_uuid_; }
auto ReplicationInstance::EnsureReplicaHasCorrectMainUUID(utils::UUID const &curr_main_uuid) -> bool {
if (!IsReadyForUUIDPing()) {
return true;
if (!main_uuid_ || *main_uuid_ != curr_main_uuid) {
return SendSwapAndUpdateUUID(curr_main_uuid);
}
auto res = SendGetInstanceUUID();
if (res.HasError()) {
return false;
}
UpdateReplicaLastResponseUUID();
if (res.GetValue().has_value() && res.GetValue().value() == curr_main_uuid) {
return true;
}
return SendSwapAndUpdateUUID(curr_main_uuid);
return true;
}
auto ReplicationInstance::SendSwapAndUpdateUUID(const utils::UUID &new_main_uuid) -> bool {
@@ -125,14 +109,5 @@ auto ReplicationInstance::SendUnregisterReplicaRpc(std::string const &instance_n
return client_.SendUnregisterReplicaRpc(instance_name);
}
auto ReplicationInstance::EnableWritingOnMain() -> bool { return client_.SendEnableWritingOnMainRpc(); }
auto ReplicationInstance::SendGetInstanceUUID()
-> utils::BasicResult<coordination::GetInstanceUUIDError, std::optional<utils::UUID>> {
return client_.SendGetInstanceUUIDRpc();
}
void ReplicationInstance::UpdateReplicaLastResponseUUID() { last_check_of_uuid_ = std::chrono::system_clock::now(); }
} // namespace memgraph::coordination
#endif

View File

@@ -19,7 +19,6 @@
#include "storage/v2/durability/durability.hpp"
#include "storage/v2/durability/snapshot.hpp"
#include "storage/v2/durability/version.hpp"
#include "storage/v2/fmt.hpp"
#include "storage/v2/indices/label_index_stats.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/inmemory/unique_constraints.hpp"

0
src/dbms/utils.hpp Normal file
View File

View File

@@ -10,7 +10,6 @@
// licenses/APL.txt.
#pragma once
#include <algorithm>
#include <atomic>
#include <compare>
#include <cstdint>

View File

@@ -19,14 +19,13 @@
// Bolt server flags.
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(experimental_enabled, "",
"Experimental features to be used, comma seperated. Options [system-replication, high-availability]");
"Experimental features to be used, comma seperated. Options [system-replication]");
using namespace std::string_view_literals;
namespace memgraph::flags {
auto const mapping = std::map{std::pair{"system-replication"sv, Experiments::SYSTEM_REPLICATION},
std::pair{"high-availability"sv, Experiments::HIGH_AVAILABILITY}};
auto const mapping = std::map{std::pair{"system-replication"sv, Experiments::SYSTEM_REPLICATION}};
auto ExperimentsInstance() -> Experiments & {
static auto instance = Experiments{};

View File

@@ -23,7 +23,6 @@ namespace memgraph::flags {
// old experiments can be reused once code cleanup has happened
enum class Experiments : uint8_t {
SYSTEM_REPLICATION = 1 << 0,
HIGH_AVAILABILITY = 1 << 1,
};
bool AreExperimentsEnabled(Experiments experiments);

View File

@@ -22,8 +22,6 @@ DEFINE_uint32(raft_server_id, 0, "Unique ID of the raft server.");
DEFINE_uint32(instance_down_timeout_sec, 5, "Time duration after which an instance is considered down.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(instance_health_check_frequency_sec, 1, "The time duration between two health checks/pings.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(instance_get_uuid_frequency_sec, 10, "The time duration between two instance uuid checks.");
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)

View File

@@ -24,8 +24,6 @@ DECLARE_uint32(raft_server_id);
DECLARE_uint32(instance_down_timeout_sec);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint32(instance_health_check_frequency_sec);
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint32(instance_get_uuid_frequency_sec);
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)

View File

@@ -6,6 +6,5 @@ target_sources(mg-glue PRIVATE auth.cpp
SessionHL.cpp
ServerT.cpp
MonitoringServerT.cpp
run_id.cpp
query_user.cpp)
run_id.cpp)
target_link_libraries(mg-glue mg-query mg-auth mg-audit mg-flags)

View File

@@ -11,7 +11,6 @@
#include <optional>
#include <utility>
#include "auth/auth.hpp"
#include "gflags/gflags.h"
#include "audit/log.hpp"
@@ -20,22 +19,17 @@
#include "glue/SessionHL.hpp"
#include "glue/auth_checker.hpp"
#include "glue/communication.hpp"
#include "glue/query_user.hpp"
#include "glue/run_id.hpp"
#include "license/license.hpp"
#include "query/auth_checker.hpp"
#include "query/discard_value_stream.hpp"
#include "query/interpreter_context.hpp"
#include "query/query_user.hpp"
#include "utils/event_map.hpp"
#include "utils/spin_lock.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::metrics {
extern const Event ActiveBoltSessions;
} // namespace memgraph::metrics
namespace {
auto ToQueryExtras(const memgraph::communication::bolt::Value &extra) -> memgraph::query::QueryExtras {
auto const &as_map = extra.ValueMap();
@@ -103,24 +97,20 @@ std::vector<memgraph::communication::bolt::Value> TypedValueResultStreamBase::De
}
return decoded_values;
}
TypedValueResultStreamBase::TypedValueResultStreamBase(memgraph::storage::Storage *storage) : storage_(storage) {}
namespace memgraph::glue {
#ifdef MG_ENTERPRISE
void MultiDatabaseAuth(memgraph::query::QueryUserOrRole *user, std::string_view db) {
if (user && !user->IsAuthorized({}, std::string(db), &memgraph::query::session_long_policy)) {
inline static void MultiDatabaseAuth(const std::optional<auth::User> &user, std::string_view db) {
if (user && !AuthChecker::IsUserAuthorized(*user, {}, std::string(db))) {
throw memgraph::communication::bolt::ClientError(
"You are not authorized on the database \"{}\"! Please contact your database administrator.", db);
}
}
#endif
} // namespace
namespace memgraph::glue {
#ifdef MG_ENTERPRISE
std::string SessionHL::GetDefaultDB() {
if (user_or_role_) {
return user_or_role_->GetDefaultDB();
if (user_.has_value()) {
return user_->db_access().GetDefault();
}
return std::string{memgraph::dbms::kDefaultDB};
}
@@ -142,18 +132,13 @@ bool SessionHL::Authenticate(const std::string &username, const std::string &pas
interpreter_.ResetUser();
{
auto locked_auth = auth_->Lock();
if (locked_auth->AccessControlled()) {
const auto user_or_role = locked_auth->Authenticate(username, password);
if (user_or_role.has_value()) {
user_or_role_ = AuthChecker::GenQueryUser(auth_, *user_or_role);
interpreter_.SetUser(AuthChecker::GenQueryUser(auth_, *user_or_role));
if (locked_auth->HasUsers()) {
user_ = locked_auth->Authenticate(username, password);
if (user_.has_value()) {
interpreter_.SetUser(user_->username());
} else {
res = false;
}
} else {
// No access control -> give empty user
user_or_role_ = AuthChecker::GenQueryUser(auth_, std::nullopt);
interpreter_.SetUser(AuthChecker::GenQueryUser(auth_, std::nullopt));
}
}
#ifdef MG_ENTERPRISE
@@ -210,17 +195,21 @@ std::pair<std::vector<std::string>, std::optional<int>> SessionHL::Interpret(
}
#ifdef MG_ENTERPRISE
const std::string *username{nullptr};
if (user_) {
username = &user_->username();
}
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
auto &db = interpreter_.current_db_.db_acc_;
const auto username = user_or_role_ ? (user_or_role_->username() ? *user_or_role_->username() : "") : "";
audit_log_->Record(endpoint_.address().to_string(), username, query, memgraph::storage::PropertyValue(params_pv),
db ? db->get()->name() : "no known database");
audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query,
memgraph::storage::PropertyValue(params_pv), db ? db->get()->name() : "no known database");
}
#endif
try {
auto result = interpreter_.Prepare(query, params_pv, ToQueryExtras(extra));
const std::string db_name = result.db ? *result.db : "";
if (user_or_role_ && !user_or_role_->IsAuthorized(result.privileges, db_name, &query::session_long_policy)) {
if (user_ && !AuthChecker::IsUserAuthorized(*user_, result.privileges, db_name)) {
interpreter_.Abort();
if (db_name.empty()) {
throw memgraph::communication::bolt::ClientError(
@@ -322,7 +311,7 @@ void SessionHL::Configure(const std::map<std::string, memgraph::communication::b
// Check if the underlying database needs to be updated
if (update) {
MultiDatabaseAuth(user_or_role_.get(), db);
MultiDatabaseAuth(user_, db);
interpreter_.SetCurrentDB(db, in_explicit_db_);
}
#endif
@@ -349,7 +338,7 @@ SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context,
// Metrics update
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions);
#ifdef MG_ENTERPRISE
interpreter_.OnChangeCB([&](std::string_view db_name) { MultiDatabaseAuth(user_or_role_.get(), db_name); });
interpreter_.OnChangeCB([&](std::string_view db_name) { MultiDatabaseAuth(user_, db_name); });
#endif
interpreter_context_->interpreters.WithLock([this](auto &interpreters) { interpreters.insert(&interpreter_); });
}

View File

@@ -15,7 +15,6 @@
#include "communication/v2/server.hpp"
#include "communication/v2/session.hpp"
#include "dbms/database.hpp"
#include "glue/query_user.hpp"
#include "query/interpreter.hpp"
namespace memgraph::glue {
@@ -83,7 +82,7 @@ class SessionHL final : public memgraph::communication::bolt::Session<memgraph::
memgraph::query::InterpreterContext *interpreter_context_;
memgraph::query::Interpreter interpreter_;
std::unique_ptr<query::QueryUserOrRole> user_or_role_;
std::optional<memgraph::auth::User> user_;
#ifdef MG_ENTERPRISE
memgraph::audit::Log *audit_log_;
bool in_explicit_db_{false}; //!< If true, the user has defined the database to use via metadata

View File

@@ -14,74 +14,53 @@
#include "auth/auth.hpp"
#include "auth/models.hpp"
#include "glue/auth.hpp"
#include "glue/query_user.hpp"
#include "license/license.hpp"
#include "query/auth_checker.hpp"
#include "query/constants.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/query_user.hpp"
#include "utils/logging.hpp"
#include "utils/synchronized.hpp"
#include "utils/variant_helpers.hpp"
#ifdef MG_ENTERPRISE
namespace {
bool IsAuthorizedLabels(const memgraph::auth::UserOrRole &user_or_role, const memgraph::query::DbAccessor *dba,
const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
bool IsUserAuthorizedLabels(const memgraph::auth::User &user, const memgraph::query::DbAccessor *dba,
const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return std::all_of(labels.begin(), labels.end(), [dba, &user_or_role, fine_grained_privilege](const auto &label) {
return std::visit(memgraph::utils::Overloaded{[&](auto &user_or_role) {
return user_or_role.GetFineGrainedAccessLabelPermissions().Has(
dba->LabelToName(label), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) ==
memgraph::auth::PermissionLevel::GRANT;
}},
user_or_role);
return std::all_of(labels.begin(), labels.end(), [dba, &user, fine_grained_privilege](const auto &label) {
return user.GetFineGrainedAccessLabelPermissions().Has(
dba->LabelToName(label), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) == memgraph::auth::PermissionLevel::GRANT;
});
}
bool IsAuthorizedGloballyLabels(const memgraph::auth::UserOrRole &user_or_role,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
bool IsUserAuthorizedGloballyLabels(const memgraph::auth::User &user,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return std::visit(memgraph::utils::Overloaded{[&](auto &user_or_role) {
return user_or_role.GetFineGrainedAccessLabelPermissions().Has(memgraph::query::kAsterisk,
fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}},
user_or_role);
return user.GetFineGrainedAccessLabelPermissions().Has(memgraph::query::kAsterisk, fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}
bool IsAuthorizedGloballyEdges(const memgraph::auth::UserOrRole &user_or_role,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
bool IsUserAuthorizedGloballyEdges(const memgraph::auth::User &user,
const memgraph::auth::FineGrainedPermission fine_grained_permission) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return std::visit(memgraph::utils::Overloaded{[&](auto &user_or_role) {
return user_or_role.GetFineGrainedAccessEdgeTypePermissions().Has(memgraph::query::kAsterisk,
fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}},
user_or_role);
return user.GetFineGrainedAccessEdgeTypePermissions().Has(memgraph::query::kAsterisk, fine_grained_permission) ==
memgraph::auth::PermissionLevel::GRANT;
}
bool IsAuthorizedEdgeType(const memgraph::auth::UserOrRole &user_or_role, const memgraph::query::DbAccessor *dba,
const memgraph::storage::EdgeTypeId &edgeType,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
bool IsUserAuthorizedEdgeType(const memgraph::auth::User &user, const memgraph::query::DbAccessor *dba,
const memgraph::storage::EdgeTypeId &edgeType,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return std::visit(memgraph::utils::Overloaded{[&](auto &user_or_role) {
return user_or_role.GetFineGrainedAccessEdgeTypePermissions().Has(
dba->EdgeTypeToName(edgeType),
memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege)) ==
memgraph::auth::PermissionLevel::GRANT;
}},
user_or_role);
return user.GetFineGrainedAccessEdgeTypePermissions().Has(
dba->EdgeTypeToName(edgeType), memgraph::glue::FineGrainedPrivilegeToFineGrainedPermission(
fine_grained_privilege)) == memgraph::auth::PermissionLevel::GRANT;
}
} // namespace
#endif
@@ -89,54 +68,47 @@ namespace memgraph::glue {
AuthChecker::AuthChecker(memgraph::auth::SynchedAuth *auth) : auth_(auth) {}
std::shared_ptr<query::QueryUserOrRole> AuthChecker::GenQueryUser(const std::optional<std::string> &username,
const std::optional<std::string> &rolename) const {
const auto user_or_role = auth_->ReadLock()->GetUserOrRole(username, rolename);
if (user_or_role) {
return std::make_shared<QueryUserOrRole>(auth_, *user_or_role);
bool AuthChecker::IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::string &db_name) const {
std::optional<memgraph::auth::User> maybe_user;
{
auto locked_auth = auth_->ReadLock();
if (!locked_auth->HasUsers()) {
return true;
}
if (username.has_value()) {
maybe_user = locked_auth->GetUser(*username);
}
}
// No user or role
return std::make_shared<QueryUserOrRole>(auth_);
}
std::unique_ptr<query::QueryUserOrRole> AuthChecker::GenQueryUser(auth::SynchedAuth *auth,
const std::optional<auth::UserOrRole> &user_or_role) {
if (user_or_role) {
return std::visit(
utils::Overloaded{[&](auto &user_or_role) { return std::make_unique<QueryUserOrRole>(auth, user_or_role); }},
*user_or_role);
}
// No user or role
return std::make_unique<QueryUserOrRole>(auth);
return maybe_user.has_value() && IsUserAuthorized(*maybe_user, privileges, db_name);
}
#ifdef MG_ENTERPRISE
std::unique_ptr<memgraph::query::FineGrainedAuthChecker> AuthChecker::GetFineGrainedAuthChecker(
std::shared_ptr<query::QueryUserOrRole> user_or_role, const memgraph::query::DbAccessor *dba) const {
const std::string &username, const memgraph::query::DbAccessor *dba) const {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return {};
}
if (!user_or_role || !*user_or_role) {
throw query::QueryRuntimeException("No user specified for fine grained authorization!");
}
// Convert from query user to auth user or role
try {
auto glue_user = dynamic_cast<glue::QueryUserOrRole &>(*user_or_role);
if (glue_user.user_) {
return std::make_unique<glue::FineGrainedAuthChecker>(std::move(*glue_user.user_), dba);
auto user = user_.Lock();
if (username != user->username()) {
auto maybe_user = auth_->ReadLock()->GetUser(username);
if (!maybe_user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist .", username);
}
*user = std::move(*maybe_user);
}
if (glue_user.role_) {
return std::make_unique<glue::FineGrainedAuthChecker>(
auth::RoleWUsername{*glue_user.username(), std::move(*glue_user.role_)}, dba);
}
DMG_ASSERT(false, "Glue user has neither user not role");
} catch (std::bad_cast &e) {
DMG_ASSERT(false, "Using a non-glue user in glue...");
}
return std::make_unique<memgraph::glue::FineGrainedAuthChecker>(*user, dba);
// Should never get here
return {};
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthChecker::ClearCache() const {
user_.WithLock([](auto &user) mutable { user = {}; });
}
#endif
@@ -144,7 +116,7 @@ bool AuthChecker::IsUserAuthorized(const memgraph::auth::User &user,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::string &db_name) { // NOLINT
#ifdef MG_ENTERPRISE
if (!db_name.empty() && !user.HasAccess(db_name)) {
if (!db_name.empty() && !user.db_access().Contains(db_name)) {
return false;
}
#endif
@@ -155,34 +127,9 @@ bool AuthChecker::IsUserAuthorized(const memgraph::auth::User &user,
});
}
bool AuthChecker::IsRoleAuthorized(const memgraph::auth::Role &role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::string &db_name) { // NOLINT
#ifdef MG_ENTERPRISE
if (!db_name.empty() && !role.HasAccess(db_name)) {
return false;
}
#endif
const auto role_permissions = role.permissions();
return std::all_of(privileges.begin(), privileges.end(), [&role_permissions](const auto privilege) {
return role_permissions.Has(memgraph::glue::PrivilegeToPermission(privilege)) ==
memgraph::auth::PermissionLevel::GRANT;
});
}
bool AuthChecker::IsUserOrRoleAuthorized(const memgraph::auth::UserOrRole &user_or_role,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::string &db_name) {
return std::visit(
utils::Overloaded{
[&](const auth::User &user) -> bool { return AuthChecker::IsUserAuthorized(user, privileges, db_name); },
[&](const auth::Role &role) -> bool { return AuthChecker::IsRoleAuthorized(role, privileges, db_name); }},
user_or_role);
}
#ifdef MG_ENTERPRISE
FineGrainedAuthChecker::FineGrainedAuthChecker(auth::UserOrRole user_or_role, const memgraph::query::DbAccessor *dba)
: user_or_role_{std::move(user_or_role)}, dba_(dba){};
FineGrainedAuthChecker::FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba)
: user_{std::move(user)}, dba_(dba){};
bool FineGrainedAuthChecker::Has(const memgraph::query::VertexAccessor &vertex, const memgraph::storage::View view,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
@@ -200,22 +147,22 @@ bool FineGrainedAuthChecker::Has(const memgraph::query::VertexAccessor &vertex,
}
}
return IsAuthorizedLabels(user_or_role_, dba_, *maybe_labels, fine_grained_privilege);
return IsUserAuthorizedLabels(user_, dba_, *maybe_labels, fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const memgraph::query::EdgeAccessor &edge,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsAuthorizedEdgeType(user_or_role_, dba_, edge.EdgeType(), fine_grained_privilege);
return IsUserAuthorizedEdgeType(user_, dba_, edge.EdgeType(), fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const std::vector<memgraph::storage::LabelId> &labels,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsAuthorizedLabels(user_or_role_, dba_, labels, fine_grained_privilege);
return IsUserAuthorizedLabels(user_, dba_, labels, fine_grained_privilege);
}
bool FineGrainedAuthChecker::Has(const memgraph::storage::EdgeTypeId &edge_type,
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
return IsAuthorizedEdgeType(user_or_role_, dba_, edge_type, fine_grained_privilege);
return IsUserAuthorizedEdgeType(user_, dba_, edge_type, fine_grained_privilege);
}
bool FineGrainedAuthChecker::HasGlobalPrivilegeOnVertices(
@@ -223,7 +170,7 @@ bool FineGrainedAuthChecker::HasGlobalPrivilegeOnVertices(
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return IsAuthorizedGloballyLabels(user_or_role_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
return IsUserAuthorizedGloballyLabels(user_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
}
bool FineGrainedAuthChecker::HasGlobalPrivilegeOnEdges(
@@ -231,7 +178,7 @@ bool FineGrainedAuthChecker::HasGlobalPrivilegeOnEdges(
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
return true;
}
return IsAuthorizedGloballyEdges(user_or_role_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
return IsUserAuthorizedGloballyEdges(user_, FineGrainedPrivilegeToFineGrainedPermission(fine_grained_privilege));
};
#endif
} // namespace memgraph::glue

View File

@@ -22,59 +22,53 @@ namespace memgraph::glue {
class AuthChecker : public query::AuthChecker {
public:
explicit AuthChecker(auth::SynchedAuth *auth);
explicit AuthChecker(memgraph::auth::SynchedAuth *auth);
std::shared_ptr<query::QueryUserOrRole> GenQueryUser(const std::optional<std::string> &username,
const std::optional<std::string> &rolename) const override;
static std::unique_ptr<query::QueryUserOrRole> GenQueryUser(auth::SynchedAuth *auth,
const std::optional<auth::UserOrRole> &user_or_role);
bool IsUserAuthorized(const std::optional<std::string> &username,
const std::vector<query::AuthQuery::Privilege> &privileges,
const std::string &db_name) const override;
#ifdef MG_ENTERPRISE
std::unique_ptr<query::FineGrainedAuthChecker> GetFineGrainedAuthChecker(std::shared_ptr<query::QueryUserOrRole> user,
const query::DbAccessor *dba) const override;
std::unique_ptr<memgraph::query::FineGrainedAuthChecker> GetFineGrainedAuthChecker(
const std::string &username, const memgraph::query::DbAccessor *dba) const override;
void ClearCache() const override;
#endif
[[nodiscard]] static bool IsUserAuthorized(const auth::User &user,
const std::vector<query::AuthQuery::Privilege> &privileges,
[[nodiscard]] static bool IsUserAuthorized(const memgraph::auth::User &user,
const std::vector<memgraph::query::AuthQuery::Privilege> &privileges,
const std::string &db_name = "");
[[nodiscard]] static bool IsRoleAuthorized(const auth::Role &role,
const std::vector<query::AuthQuery::Privilege> &privileges,
const std::string &db_name = "");
[[nodiscard]] static bool IsUserOrRoleAuthorized(const auth::UserOrRole &user_or_role,
const std::vector<query::AuthQuery::Privilege> &privileges,
const std::string &db_name = "");
private:
auth::SynchedAuth *auth_;
mutable utils::Synchronized<auth::UserOrRole, utils::SpinLock> user_or_role_; // cached user
memgraph::auth::SynchedAuth *auth_;
mutable memgraph::utils::Synchronized<auth::User, memgraph::utils::SpinLock> user_; // cached user
};
#ifdef MG_ENTERPRISE
class FineGrainedAuthChecker : public query::FineGrainedAuthChecker {
public:
explicit FineGrainedAuthChecker(auth::UserOrRole user, const query::DbAccessor *dba);
explicit FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba);
bool Has(const query::VertexAccessor &vertex, storage::View view,
bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const query::EdgeAccessor &edge,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const std::vector<storage::LabelId> &labels,
bool Has(const std::vector<memgraph::storage::LabelId> &labels,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool Has(const storage::EdgeTypeId &edge_type,
bool Has(const memgraph::storage::EdgeTypeId &edge_type,
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnVertices(query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnVertices(
memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnEdges(query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
bool HasGlobalPrivilegeOnEdges(
memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
private:
auth::UserOrRole user_or_role_;
const query::DbAccessor *dba_;
auth::User user_;
const memgraph::query::DbAccessor *dba_;
};
#endif
} // namespace memgraph::glue

View File

@@ -15,7 +15,6 @@
#include <fmt/format.h>
#include "auth/auth.hpp"
#include "auth/models.hpp"
#include "dbms/constants.hpp"
#include "glue/auth.hpp"
@@ -124,29 +123,6 @@ std::vector<std::vector<memgraph::query::TypedValue>> ShowRolePrivileges(
}
#ifdef MG_ENTERPRISE
std::vector<std::vector<memgraph::query::TypedValue>> ShowDatabasePrivileges(
const std::optional<memgraph::auth::Role> &role) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast() || !role) {
return {};
}
const auto &db = role->db_access();
const auto &allows = db.GetAllowAll();
const auto &grants = db.GetGrants();
const auto &denies = db.GetDenies();
std::vector<memgraph::query::TypedValue> res; // First element is a list of granted databases, second of revoked ones
if (allows) {
res.emplace_back("*");
} else {
std::vector<memgraph::query::TypedValue> grants_vec(grants.cbegin(), grants.cend());
res.emplace_back(std::move(grants_vec));
}
std::vector<memgraph::query::TypedValue> denies_vec(denies.cbegin(), denies.cend());
res.emplace_back(std::move(denies_vec));
return {res};
}
std::vector<std::vector<memgraph::query::TypedValue>> ShowDatabasePrivileges(
const std::optional<memgraph::auth::User> &user) {
if (!memgraph::license::global_license_checker.IsEnterpriseValidFast() || !user) {
@@ -154,15 +130,9 @@ std::vector<std::vector<memgraph::query::TypedValue>> ShowDatabasePrivileges(
}
const auto &db = user->db_access();
auto allows = db.GetAllowAll();
auto grants = db.GetGrants();
auto denies = db.GetDenies();
if (const auto *role = user->role()) {
const auto &role_db = role->db_access();
allows |= role_db.GetAllowAll();
grants.insert(role_db.GetGrants().begin(), role_db.GetGrants().end());
denies.insert(role_db.GetDenies().begin(), role_db.GetDenies().end());
}
const auto &allows = db.GetAllowAll();
const auto &grants = db.GetGrants();
const auto &denies = db.GetDenies();
std::vector<memgraph::query::TypedValue> res; // First element is a list of granted databases, second of revoked ones
if (allows) {
@@ -317,7 +287,7 @@ bool AuthQueryHandler::CreateUser(const std::string &username, const std::option
,
system_tx);
#ifdef MG_ENTERPRISE
GrantDatabase(auth::kAllDatabases, username, system_tx);
GrantDatabaseToUser(auth::kAllDatabases, username, system_tx);
SetMainDatabase(dbms::kDefaultDB, username, system_tx);
#endif
}
@@ -364,97 +334,51 @@ bool AuthQueryHandler::CreateRole(const std::string &rolename, system::Transacti
}
#ifdef MG_ENTERPRISE
void AuthQueryHandler::GrantDatabase(const std::string &db_name, const std::string &user_or_role,
system::Transaction *system_tx) {
bool AuthQueryHandler::RevokeDatabaseFromUser(const std::string &db_name, const std::string &username,
system::Transaction *system_tx) {
try {
auto locked_auth = auth_->Lock();
const auto res = locked_auth->GrantDatabase(db_name, user_or_role, system_tx);
switch (res) {
using enum auth::Auth::Result;
case SUCCESS:
return;
case NO_USER_ROLE:
throw query::QueryRuntimeException("No user nor role '{}' found.", user_or_role);
case NO_ROLE:
throw query::QueryRuntimeException("Using auth module, no role '{}' found.", user_or_role);
break;
}
auto user = locked_auth->GetUser(username);
if (!user) return false;
return locked_auth->RevokeDatabaseFromUser(db_name, username, system_tx);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::DenyDatabase(const std::string &db_name, const std::string &user_or_role,
system::Transaction *system_tx) {
bool AuthQueryHandler::GrantDatabaseToUser(const std::string &db_name, const std::string &username,
system::Transaction *system_tx) {
try {
auto locked_auth = auth_->Lock();
const auto res = locked_auth->DenyDatabase(db_name, user_or_role, system_tx);
switch (res) {
using enum auth::Auth::Result;
case SUCCESS:
return;
case NO_USER_ROLE:
throw query::QueryRuntimeException("No user nor role '{}' found.", user_or_role);
case NO_ROLE:
throw query::QueryRuntimeException("Using auth module, no role '{}' found.", user_or_role);
break;
}
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::RevokeDatabase(const std::string &db_name, const std::string &user_or_role,
system::Transaction *system_tx) {
try {
auto locked_auth = auth_->Lock();
const auto res = locked_auth->RevokeDatabase(db_name, user_or_role, system_tx);
switch (res) {
using enum auth::Auth::Result;
case SUCCESS:
return;
case NO_USER_ROLE:
throw query::QueryRuntimeException("No user nor role '{}' found.", user_or_role);
case NO_ROLE:
throw query::QueryRuntimeException("Using auth module, no role '{}' found.", user_or_role);
break;
}
auto user = locked_auth->GetUser(username);
if (!user) return false;
return locked_auth->GrantDatabaseToUser(db_name, username, system_tx);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
std::vector<std::vector<memgraph::query::TypedValue>> AuthQueryHandler::GetDatabasePrivileges(
const std::string &user_or_role) {
const std::string &username) {
try {
auto locked_auth = auth_->ReadLock();
if (auto user = locked_auth->GetUser(user_or_role)) {
return ShowDatabasePrivileges(user);
auto user = locked_auth->GetUser(username);
if (!user) {
throw memgraph::query::QueryRuntimeException("User '{}' doesn't exist.", username);
}
if (auto role = locked_auth->GetRole(user_or_role)) {
return ShowDatabasePrivileges(role);
}
throw memgraph::query::QueryRuntimeException("Neither user nor role '{}' exist.", user_or_role);
return ShowDatabasePrivileges(user);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}
}
void AuthQueryHandler::SetMainDatabase(std::string_view db_name, const std::string &user_or_role,
bool AuthQueryHandler::SetMainDatabase(std::string_view db_name, const std::string &username,
system::Transaction *system_tx) {
try {
auto locked_auth = auth_->Lock();
const auto res = locked_auth->SetMainDatabase(db_name, user_or_role, system_tx);
switch (res) {
using enum auth::Auth::Result;
case SUCCESS:
return;
case NO_USER_ROLE:
throw query::QueryRuntimeException("No user nor role '{}' found.", user_or_role);
case NO_ROLE:
throw query::QueryRuntimeException("Using auth module, no role '{}' found.", user_or_role);
break;
}
auto user = locked_auth->GetUser(username);
if (!user) return false;
return locked_auth->SetMainDatabase(db_name, username, system_tx);
} catch (const memgraph::auth::AuthException &e) {
throw memgraph::query::QueryRuntimeException(e.what());
}

View File

@@ -37,19 +37,15 @@ class AuthQueryHandler final : public memgraph::query::AuthQueryHandler {
system::Transaction *system_tx) override;
#ifdef MG_ENTERPRISE
void GrantDatabase(const std::string &db_name, const std::string &user_or_role,
system::Transaction *system_tx) override;
bool RevokeDatabaseFromUser(const std::string &db_name, const std::string &username,
system::Transaction *system_tx) override;
void DenyDatabase(const std::string &db_name, const std::string &user_or_role,
system::Transaction *system_tx) override;
bool GrantDatabaseToUser(const std::string &db_name, const std::string &username,
system::Transaction *system_tx) override;
void RevokeDatabase(const std::string &db_name, const std::string &user_or_role,
system::Transaction *system_tx) override;
std::vector<std::vector<memgraph::query::TypedValue>> GetDatabasePrivileges(const std::string &username) override;
std::vector<std::vector<memgraph::query::TypedValue>> GetDatabasePrivileges(const std::string &user_or_role) override;
void SetMainDatabase(std::string_view db_name, const std::string &user_or_role,
system::Transaction *system_tx) override;
bool SetMainDatabase(std::string_view db_name, const std::string &username, system::Transaction *system_tx) override;
void DeleteDatabase(std::string_view db_name, system::Transaction *system_tx) override;
#endif

View File

@@ -1,41 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "glue/query_user.hpp"
#include "glue/auth_checker.hpp"
namespace memgraph::glue {
bool QueryUserOrRole::IsAuthorized(const std::vector<query::AuthQuery::Privilege> &privileges,
const std::string &db_name, query::UserPolicy *policy) const {
auto locked_auth = auth_->Lock();
// Check policy and update if behind (and policy permits it)
if (policy->DoUpdate() && !locked_auth->UpToDate(auth_epoch_)) {
if (user_) user_ = locked_auth->GetUser(user_->username());
if (role_) role_ = locked_auth->GetRole(role_->rolename());
}
if (user_) return AuthChecker::IsUserAuthorized(*user_, privileges, db_name);
if (role_) return AuthChecker::IsRoleAuthorized(*role_, privileges, db_name);
return !policy->DoUpdate() || !locked_auth->AccessControlled();
}
#ifdef MG_ENTERPRISE
std::string QueryUserOrRole::GetDefaultDB() const {
if (user_) return user_->db_access().GetMain();
if (role_) return role_->db_access().GetMain();
return std::string{dbms::kDefaultDB};
}
#endif
} // namespace memgraph::glue

View File

@@ -1,57 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <optional>
#include "auth/auth.hpp"
#include "query/query_user.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::glue {
struct QueryUserOrRole : public query::QueryUserOrRole {
bool IsAuthorized(const std::vector<query::AuthQuery::Privilege> &privileges, const std::string &db_name,
query::UserPolicy *policy) const override;
#ifdef MG_ENTERPRISE
std::string GetDefaultDB() const override;
#endif
explicit QueryUserOrRole(auth::SynchedAuth *auth) : query::QueryUserOrRole{std::nullopt, std::nullopt}, auth_{auth} {}
QueryUserOrRole(auth::SynchedAuth *auth, auth::UserOrRole user_or_role)
: query::QueryUserOrRole{std::visit(
utils::Overloaded{[](const auto &user_or_role) { return user_or_role.username(); }},
user_or_role),
std::visit(utils::Overloaded{[&](const auth::User &) -> std::optional<std::string> {
return std::nullopt;
},
[&](const auth::Role &role) -> std::optional<std::string> {
return role.rolename();
}},
user_or_role)},
auth_{auth} {
std::visit(utils::Overloaded{[&](auth::User &&user) { user_.emplace(std::move(user)); },
[&](auth::Role &&role) { role_.emplace(std::move(role)); }},
std::move(user_or_role));
}
private:
friend class AuthChecker;
auth::SynchedAuth *auth_;
mutable std::optional<auth::User> user_{};
mutable std::optional<auth::Role> role_{};
mutable auth::Auth::Epoch auth_epoch_{auth::Auth::kStartEpoch};
};
} // namespace memgraph::glue

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,7 +22,6 @@
#include "integrations/constants.hpp"
#include "integrations/kafka/exceptions.hpp"
#include "integrations/kafka/fmt.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/on_scope_exit.hpp"

View File

@@ -1,25 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#if FMT_VERSION > 90000
#include <fmt/ostream.h>
#include <librdkafka/rdkafkacpp.h>
inline std::ostream &operator<<(std::ostream &os, const RdKafka::ErrorCode &code) {
os << RdKafka::err2str(code);
return os;
}
template <>
class fmt::formatter<RdKafka::ErrorCode> : public fmt::ostream_formatter {};
#endif

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,12 +15,12 @@
#include <chrono>
#include <thread>
#include <fmt/format.h>
#include <pulsar/Client.h>
#include <pulsar/InitialPosition.h>
#include "integrations/constants.hpp"
#include "integrations/pulsar/exceptions.hpp"
#include "integrations/pulsar/fmt.hpp"
#include "utils/concepts.hpp"
#include "utils/logging.hpp"
#include "utils/on_scope_exit.hpp"

View File

@@ -1,21 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#if FMT_VERSION > 90000
#include <fmt/ostream.h>
#include "integrations/pulsar/consumer.hpp"
template <>
class fmt::formatter<memgraph::integrations::pulsar::pulsar_client::Result> : public fmt::ostream_formatter {};
#endif

View File

@@ -1,21 +0,0 @@
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#if FMT_VERSION > 90000
#include <fmt/ostream.h>
#include "io/network/endpoint.hpp"
template <>
class fmt::formatter<memgraph::io::network::Endpoint> : public fmt::ostream_formatter {};
#endif

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,7 +11,6 @@
#pragma once
#include <cstddef>
#include <cstdint>
namespace memgraph::io::network {

View File

@@ -160,14 +160,13 @@ class KVStore final {
* and behaves as if all of those pairs are stored in a single iterable
* collection of std::pair<std::string, std::string>.
*/
class iterator final {
class iterator final : public std::iterator<std::input_iterator_tag, // iterator_category
std::pair<std::string, std::string>, // value_type
long, // difference_type
const std::pair<std::string, std::string> *, // pointer
const std::pair<std::string, std::string> & // reference
> {
public:
using iterator_concept [[maybe_unused]] = std::input_iterator_tag;
using value_type = std::pair<std::string, std::string>;
using difference_type = long;
using pointer = const std::pair<std::string, std::string> *;
using reference = const std::pair<std::string, std::string> &;
explicit iterator(const KVStore *kvstore, const std::string &prefix = "", bool at_end = false);
iterator(const iterator &other) = delete;

View File

@@ -27,7 +27,6 @@
#include "helpers.hpp"
#include "license/license_sender.hpp"
#include "memory/global_memory_control.hpp"
#include "query/auth_checker.hpp"
#include "query/auth_query_handler.hpp"
#include "query/config.hpp"
#include "query/discard_value_stream.hpp"
@@ -58,13 +57,8 @@ constexpr uint64_t kMgVmMaxMapCount = 262144;
void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbms::DatabaseAccess &db_acc,
std::string cypherl_file_path, memgraph::audit::Log *audit_log = nullptr) {
memgraph::query::Interpreter interpreter(&ctx, db_acc);
// Temporary empty user
// TODO: Double check with buda
memgraph::query::AllowEverythingAuthChecker tmp_auth_checker;
auto tmp_user = tmp_auth_checker.GenQueryUser(std::nullopt, std::nullopt);
interpreter.SetUser(tmp_user);
std::ifstream file(cypherl_file_path);
if (!file.is_open()) {
spdlog::trace("Could not find init file {}", cypherl_file_path);
return;
@@ -365,7 +359,6 @@ int main(int argc, char **argv) {
#ifdef MG_ENTERPRISE
.instance_down_timeout_sec = std::chrono::seconds(FLAGS_instance_down_timeout_sec),
.instance_health_check_frequency_sec = std::chrono::seconds(FLAGS_instance_health_check_frequency_sec),
.instance_get_uuid_frequency_sec = std::chrono::seconds(FLAGS_instance_get_uuid_frequency_sec),
#endif
.default_kafka_bootstrap_servers = FLAGS_kafka_bootstrap_servers,
.default_pulsar_service_url = FLAGS_pulsar_service_url,

View File

@@ -61,12 +61,10 @@ void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t
// This needs to be before, to throw exception in case of too big alloc
if (*commit) [[likely]] {
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
bool ok = GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
if (!ok) return nullptr;
GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
}
// This needs to be here so it doesn't get incremented in case the first TrackAlloc throws an exception
bool ok = memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
if (!ok) return nullptr;
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
}
auto *ptr = old_hooks->alloc(extent_hooks, new_addr, size, alignment, zero, commit, arena_ind);
@@ -120,14 +118,10 @@ static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, siz
return err;
}
[[maybe_unused]] auto blocker = memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker{};
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
bool ok = GetQueriesMemoryControl().TrackAllocOnCurrentThread(length);
DMG_ASSERT(ok);
GetQueriesMemoryControl().TrackAllocOnCurrentThread(length);
}
auto ok = memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
DMG_ASSERT(ok);
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
return false;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2024 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -28,12 +28,6 @@ void *newImpl(const std::size_t size) {
return ptr;
}
[[maybe_unused]] auto blocker = memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker{};
auto maybe_msg = memgraph::utils::MemoryErrorStatus().msg();
if (maybe_msg) {
throw memgraph::utils::OutOfMemoryException{std::move(*maybe_msg)};
}
throw std::bad_alloc{};
}
@@ -43,21 +37,11 @@ void *newImpl(const std::size_t size, const std::align_val_t align) {
return ptr;
}
[[maybe_unused]] auto blocker = memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker{};
auto maybe_msg = memgraph::utils::MemoryErrorStatus().msg();
if (maybe_msg) {
throw memgraph::utils::OutOfMemoryException{std::move(*maybe_msg)};
}
throw std::bad_alloc{};
}
void *newNoExcept(const std::size_t size) noexcept {
[[maybe_unused]] auto blocker = memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker{};
return malloc(size);
}
void *newNoExcept(const std::size_t size) noexcept { return malloc(size); }
void *newNoExcept(const std::size_t size, const std::align_val_t align) noexcept {
[[maybe_unused]] auto blocker = memgraph::utils::MemoryTracker::OutOfMemoryExceptionBlocker{};
return aligned_alloc(size, static_cast<std::size_t>(align));
}

View File

@@ -54,14 +54,14 @@ void QueriesMemoryControl::EraseThreadToTransactionId(const std::thread::id &thr
}
}
bool QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {
void QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {
auto thread_id_to_transaction_id_accessor = thread_id_to_transaction_id.access();
// we might be just constructing mapping between thread id and transaction id
// so we miss this allocation
auto thread_id_to_transaction_id_elem = thread_id_to_transaction_id_accessor.find(std::this_thread::get_id());
if (thread_id_to_transaction_id_elem == thread_id_to_transaction_id_accessor.end()) {
return true;
return;
}
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
@@ -71,10 +71,10 @@ bool QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {
// It can happen that some allocation happens between mapping thread to
// transaction id, so we miss this allocation
if (transaction_id_to_tracker == transaction_id_to_tracker_accessor.end()) [[unlikely]] {
return true;
return;
}
auto &query_tracker = transaction_id_to_tracker->tracker;
return query_tracker.TrackAlloc(size);
query_tracker.TrackAlloc(size);
}
void QueriesMemoryControl::TrackFreeOnCurrentThread(size_t size) {

View File

@@ -62,7 +62,7 @@ class QueriesMemoryControl {
// Find tracker for current thread if exists, track
// query allocation and procedure allocation if
// necessary
bool TrackAllocOnCurrentThread(size_t size);
void TrackAllocOnCurrentThread(size_t size);
// Find tracker for current thread if exists, track
// query allocation and procedure allocation if

Some files were not shown because too many files have changed in this diff Show More