tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into T635-add-vertex-degree-to-query-planner

Browse Source
This commit is contained in:
Josip Mrden
2023-06-23 12:25:25 +02:00
parent 79d91b68e2 b25e9968ee
commit 6574c220b7
54 changed files with 1332 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
config/generate.py
View File

@@ -23,6 +23,14 @@ def wrap_text(s, initial_indent="# "):
def extract_flags(binary_path):
ret = {}
data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8")
# If something is printed out before the help output, it will break the the
# XML parsing -> filter out if something is not XML line because something
# can be logged before gflags output (e.g. during the global objects init).
# This gets called during memgraph build phase to generate default config
# file later installed under /etc/memgraph/memgraph.conf
# NOTE: Don't use \n in the gflags description strings.
# NOTE: Check here if gflags version changes because of the XML format.
data = "\n".join([line for line in data.split("\n") if line.startswith("<")])
root = ET.fromstring(data)
for child in root:
if child.tag == "usage" and child.text.lower().count("warning"):

26
docs/csv-import-tool/README.md
View File

@@ -202,3 +202,29 @@ for row in csv.reader(stream, delimiter=',', doublequote=True,
For more information about the meaning of the above values, see:
https://docs.python.org/3/library/csv.html#csv.Dialect
## Errors
1. [Skipping duplicate node with ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-1)
2. [Skipping bad relationship with START_ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-2)
3. [Skipping bad relationship with END_ID '{}'. For more details, visit:
memgr.ph/csv-import-tool.](#error-3)
## Skipping duplicate node with ID {} {#error-1}
Duplicate nodes are nodes that have an ID that is the same as another node that
was already imported. You can instruct the importer to ignore all duplicate
nodes (instead of raising an error) by using the `--skip-duplicate-nodes` flag.
## Skipping bad relationship with START_ID {} {#error-2}
A node with the id `START_ID` doesn't exist. You can instruct the importer to
ignore all bad relationships (instead of raising an error) that refer to nodes
that don't exist in the node files by using the `--skip-bad-relationships` flag.
## Skipping bad relationship with END_ID {} {#error-3}
A node with the id `END_ID` doesn't exist. You can instruct the importer to
ignore all bad relationships (instead of raising an error) that refer to nodes
that don't exist in the node files by using the `--skip-bad-relationships` flag.

10
environment/README.md Normal file
View File

@@ -0,0 +1,10 @@
# Memgraph Operating Environments
## os
Under the `os` directory, you can find scripts to install all required system
dependencies on operating systems where Memgraph natively builds. The testing
script helps to see how to install all packages (in the case of a new package),
or make any adjustments in the overall system setup. Also, the testing script
helps check if Memgraph runs on a freshly installed operating system (with no
packages installed).

3
environment/os/.gitignore vendored
View File

@@ -1,3 +1,6 @@
*.deb
*.deb.*
*.rpm
*.rpm.*
*.tar.gz
*.tar.gz.*

44
environment/os/amzn-2.sh
View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make # build system
make cmake # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -57,9 +57,18 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
which nodejs golang custom-golang1.18.9 zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.2 # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -73,6 +82,18 @@ check() {
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""}
LD_LIBRARY_PATH=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
@@ -103,8 +124,27 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
yum update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == jdk-17 ]; then
if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then
wget --no-check-certificate -c --header "Cookie: oraclelicense=accept-securebackup-cookie" https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
rpm -Uvh jdk-17_linux-x64_bin.rpm
# NOTE: Set Java 11 as default.
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm

44
environment/os/centos-7.sh
View File

@@ -39,7 +39,7 @@ TOOLCHAIN_RUN_DEPS=(
)
MEMGRAPH_BUILD_DEPS=(
make pkgconfig # build system
make cmake pkgconfig # build system
curl wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -56,9 +56,19 @@ MEMGRAPH_BUILD_DEPS=(
sbcl # for custom Lisp C++ preprocessing
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests
which mono-complete dotnet-sdk-3.1 golang custom-golang1.18.9 # for driver tests
nodejs zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.2 # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -68,6 +78,18 @@ list() {
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == git ]; then
if ! which "git" >/dev/null; then
missing="git $missing"
@@ -110,7 +132,25 @@ install() {
yum update -y
yum install -y wget python3 python3-pip
yum install -y git
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == jdk-17 ]; then
if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then
wget https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm
rpm -ivh jdk-17_linux-x64_bin.rpm
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
if [ "$pkg" == libipt ]; then
if ! yum list installed libipt >/dev/null 2>/dev/null; then
yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm

35
environment/os/centos-9.sh
View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
make cmake pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -56,10 +56,20 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
which nodejs golang custom-golang1.18.9 # for driver tests
zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.2 # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -69,6 +79,18 @@ list() {
check() {
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == "PyYAML" ]; then
if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
@@ -103,7 +125,16 @@ install() {
fi
yum update -y
yum install -y wget git python3 python3-pip
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
# Since there is no support for libipt-devel for CentOS 9 we install
# Fedoras version of same libs, they are the same version but released
# for different OS

65
environment/os/debian-10.sh
View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless oracle-java17-installer custom-maven3.9.2 # for driver tests
dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -64,7 +73,28 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
@@ -75,8 +105,15 @@ deb http://deb.debian.org/debian/ buster-updates main contrib non-free
deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free
deb http://security.debian.org/debian-security buster/updates main contrib non-free
deb-src http://security.debian.org/debian-security buster/updates main contrib non-free
EOF
apt --allow-releaseinfo-change update
cat >/etc/apt/sources.list.d/java.list << EOF
deb http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main
deb-src http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main
EOF
cd "$DIR"
apt install -y gnupg
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EA8CACC073C3DB2A
apt --allow-releaseinfo-change update
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
@@ -85,8 +122,26 @@ EOF
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == oracle-java17-installer ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
echo oracle-java17-installer shared/accepted-oracle-license-v1-3 select true | /usr/bin/debconf-set-selections
echo oracle-java17-installer shared/accepted-oracle-license-v1-3 seen true | /usr/bin/debconf-set-selections
apt install -y "$pkg"
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

54
environment/os/debian-11-arm.sh
View File

@@ -54,10 +54,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
golang nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.2 # for driver tests
golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -65,7 +74,28 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
@@ -89,7 +119,25 @@ EOF
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

56
environment/os/debian-11.sh
View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -54,10 +54,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.2 # for driver tests
dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -65,7 +74,28 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
@@ -89,7 +119,25 @@ EOF
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb

60
environment/os/fedora-36.sh
View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkgconf-pkg-config # build system
make cmake pkgconf-pkg-config # build system
wget # for downloading libs
libuuid-devel java-11-openjdk # required by antlr
readline-devel # for memgraph console
@@ -52,10 +52,21 @@ MEMGRAPH_BUILD_DEPS=(
libcurl-devel # mg-requests
rpm-build rpmlint # for RPM package building
doxygen graphviz # source documentation generators
which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
java-11-openjdk-devel java-17-openjdk-devel custom-maven3.9.2 # for driver tests
which zip unzip
nodejs golang custom-golang1.18.9 # for driver tests
sbcl # for custom Lisp C++ preprocessing
autoconf # for jemalloc code generation
libtool # for protobuf code generation
cyrus-sasl-devel
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -64,11 +75,25 @@ list() {
check() {
local missing=""
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
if [ -v LD_LIBRARY_PATH ]; then
# On Fedora yum/dnf and python10 use newer glibc which is not compatible
# with ours, so we need to momentarely disable env
local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
LD_LIBRARY_PATH=""
fi
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
@@ -77,7 +102,10 @@ check() {
echo "MISSING PACKAGES: $missing"
exit 1
fi
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
if [ -v OLD_LD_LIBRARY_PATH ]; then
echo "Restoring LD_LIBRARY_PATH..."
LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
fi
}
install() {
@@ -94,7 +122,25 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
dnf update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == java-17-openjdk-devel ]; then
if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then
dnf install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java java-11-openjdk.x86_64
update-alternatives --set javac java-11-openjdk.x86_64
fi
continue
fi
dnf install -y "$pkg"
done
}

145
environment/os/run.sh Executable file
View File

@@ -0,0 +1,145 @@
#!/bin/bash
set -Eeuo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
IFS=' '
# NOTE: docker_image_name could be local image build based on release/package images.
# NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space.
# "docker_container_type script_name docker_image_name"
OPERATING_SYSTEMS=(
"mgrun amzn-2 amazonlinux:2"
"mgrun centos-7 centos:7"
"mgrun centos-9 dokken/centos-stream-9"
"mgrun debian-10 debian:10"
"mgrun debian-11 debian:11"
"mgrun fedora-36 fedora:36"
"mgrun ubuntu-18.04 ubuntu:18.04"
"mgrun ubuntu-20.04 ubuntu:20.04"
"mgrun ubuntu-22.04 ubuntu:22.04"
# "mgbuild centos-7 package-mgbuild_centos-7"
)
if [ ! "$(docker info)" ]; then
echo "ERROR: Docker is required"
exit 1
fi
print_help () {
echo -e "$0 all\t\t\t\t => start + init all containers in the background"
echo -e "$0 check\t\t\t\t => check all containers"
echo -e "$0 delete\t\t\t\t => stop + remove all containers"
echo -e "$0 copy src_container dst_container => copy build package from src to dst container"
exit 1
}
# NOTE: This is an idempotent operation!
# TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function.
docker_run () {
cnt_name="$1"
cnt_image="$2"
if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
echo "Cleanup of the old exited container..."
docker rm $cnt_name
fi
docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity
fi
echo "The $cnt_image container is active under $cnt_name name!"
}
docker_exec () {
cnt_name="$1"
cnt_cmd="$2"
docker exec -it "$cnt_name" bash -c "$cnt_cmd"
}
docker_stop_and_rm () {
cnt_name="$1"
if [ "$(docker ps -q -f name=$cnt_name)" ]; then
docker stop "$1"
if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then
docker rm "$1"
fi
fi
}
# TODO(gitbuda): Make the call to `install NEW_DEPS` configurable, the question what else is useful?
start_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_run "$docker_name" "$docker_image"
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""
done
}
check_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
echo ""
echo "~~~~ OPERATING ON $docker_image as $docker_name..."
docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh check NEW_DEPS"
echo "---- DONE EVERYHING FOR $docker_image as $docker_name..."
echo ""
done
}
delete_all () {
for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do
read -a script_docker <<< "$script_docker_pair"
docker_container_type="${script_docker[0]}"
script_name="${script_docker[1]}"
docker_image="${script_docker[2]}"
docker_name="${docker_container_type}_$script_name"
docker_stop_and_rm "$docker_name"
echo "~~~~ $docker_image as $docker_name DELETED"
done
}
# TODO(gitbuda): Copy file between containers is a useful util, also delete, + consider copying of a whole folder.
# TODO(gitbuda): Add args: src_cnt dst_cnt abs_path; both file and recursive folder, always delete + copy.
copy_build_package () {
src_container="$1"
dst_container="$2"
src="$src_container:/memgraph/build/output"
tmp_dst="$SCRIPT_DIR/../../build"
mkdir -p "$tmp_dst"
rm -rf "$tmp_dst/output"
dst="$dst_container:/"
docker cp "$src" "$tmp_dst"
docker cp "$tmp_dst/output" "$dst"
}
if [ "$#" -eq 0 ]; then
print_help
else
case $1 in
all)
start_all
;;
check)
check_all
;;
delete)
delete_all
;;
copy) # src_container dst_container
if [ "$#" -ne 3 ]; then
print_help
fi
copy_build_package "$2" "$3"
;;
*)
print_help
;;
esac
fi

11
environment/os/template.sh
View File

@@ -6,6 +6,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "$DIR/../util.sh"
check_operating_system "todo-os-name"
check_architecture "todo-arch-name"
TOOLCHAIN_BUILD_DEPS=(
pkg
@@ -19,6 +20,16 @@ MEMGRAPH_BUILD_DEPS=(
pkg
)
MEMGRAPH_RUN_DEPS=(
pkg
)
# NEW_DEPS is useful when you won't to test the installation of a new package.
# During the test you can put here packages like wget curl tar gzip
NEW_DEPS=(
pkg
)
list() {
echo "$1"
}

60
environment/os/ubuntu-18.04.sh
View File

@@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
make cmake pkg-config # build system
curl wget # downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # memgraph console
@@ -53,9 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests
mono-runtime mono-mcs nodejs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # driver tests
custom-golang1.18.9 # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -63,11 +73,53 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
apt install -y $1
apt update -y
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}
deps=$2"[*]"

61
environment/os/ubuntu-20.04.sh
View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-3.1 golang nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # for driver tests
dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -64,12 +73,35 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
apt update
export DEBIAN_FRONTEND=noninteractive
apt update -y
apt install -y wget
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -77,8 +109,16 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-3.1 ]; then
if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -88,6 +128,15 @@ install() {
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

56
environment/os/ubuntu-22.04-arm.sh
View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-6.0 golang nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # for driver tests
dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -64,7 +73,28 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
@@ -78,7 +108,16 @@ install() {
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -88,6 +127,15 @@ install() {
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

60
environment/os/ubuntu-22.04.sh
View File

@@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=(
MEMGRAPH_BUILD_DEPS=(
git # source code control
make pkg-config # build system
make cmake pkg-config # build system
curl wget # for downloading libs
uuid-dev default-jre-headless # required by antlr
libreadline-dev # for memgraph console
@@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=(
libcurl4-openssl-dev # mg-requests
sbcl # for custom Lisp C++ preprocessing
doxygen graphviz # source documentation generators
mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests
dotnet-sdk-6.0 golang nodejs npm
mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # for driver tests
dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm # for driver tests
autoconf # for jemalloc code generation
libtool # for protobuf code generation
libsasl2-dev
)
MEMGRAPH_RUN_DEPS=(
logrotate openssl python3 libseccomp2
)
NEW_DEPS=(
wget curl tar gzip
)
list() {
@@ -64,12 +73,34 @@ list() {
}
check() {
check_all_dpkg "$1"
local missing=""
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then
missing="$pkg $missing"
fi
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then
missing="$pkg $missing"
fi
continue
fi
if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then
missing="$pkg $missing"
fi
done
if [ "$missing" != "" ]; then
echo "MISSING PACKAGES: $missing"
exit 1
fi
}
install() {
cd "$DIR"
apt update
apt update -y
apt install -y wget
# If GitHub Actions runner is installed, append LANG to the environment.
# Python related tests doesn't work the LANG export.
if [ -d "/home/gh/actions-runner" ]; then
@@ -77,8 +108,16 @@ install() {
else
echo "NOTE: export LANG=en_US.utf8"
fi
apt install -y wget
for pkg in $1; do
if [ "$pkg" == custom-maven3.9.2 ]; then
install_custom_maven "3.9.2"
continue
fi
if [ "$pkg" == custom-golang1.18.9 ]; then
install_custom_golang "1.18.9"
continue
fi
if [ "$pkg" == dotnet-sdk-6.0 ]; then
if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then
wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
@@ -88,6 +127,15 @@ install() {
fi
continue
fi
if [ "$pkg" == openjdk-17-jdk-headless ]; then
if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then
apt install -y "$pkg"
# The default Java version should be Java 11
update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java
update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac
fi
continue
fi
apt install -y "$pkg"
done
}

23
environment/util.sh
View File

@@ -76,3 +76,26 @@ function install_all_apt() {
apt install -y "$pkg"
done
}
function install_custom_golang() {
# NOTE: The official https://go.dev/doc/manage-install doesn't seem to be working.
GOVERSION="$1"
GOINSTALLDIR="/opt/go$GOVERSION"
GOROOT="$GOINSTALLDIR/go" # GOPATH=$HOME/go
if [ ! -f "$GOROOT/bin/go" ]; then
curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz
mkdir -p "$GOINSTALLDIR"
tar -C "$GOINSTALLDIR" -xzf go$GOVERSION.linux-amd64.tar.gz
fi
echo "go $GOVERSION installed under $GOROOT"
}
function install_custom_maven() {
MVNVERSION="$1"
MVNINSTALLDIR="/opt/apache-maven-$MVNVERSION"
if [ ! -f "$MVNINSTALLDIR/bin/mvn" ]; then
curl -LO "https://dlcdn.apache.org/maven/maven-3/$MVNVERSION/binaries/apache-maven-$MVNVERSION-bin.tar.gz"
tar -C "/opt" -xzf "apache-maven-$MVNVERSION-bin.tar.gz"
fi
echo "maven $MVNVERSION installed under $MVNINSTALLDIR"
}

4
init
View File

@@ -77,7 +77,9 @@ fi
# Fix for centos 7 during release
if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then
python3 -m pip uninstall -y virtualenv
if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then
python3 -m pip uninstall -y virtualenv
fi
python3 -m pip install virtualenv
fi

1
libs/CMakeLists.txt
View File

@@ -12,6 +12,7 @@ find_package(Boost 1.78 REQUIRED)
find_package(BZip2 1.0.6 REQUIRED)
find_package(Threads REQUIRED)
set(GFLAGS_NOTHREADS OFF)
# NOTE: config/generate.py depends on the gflags help XML format.
find_package(gflags REQUIRED)
find_package(fmt 8.0.1)
find_package(Jemalloc REQUIRED)

16
release/CMakeLists.txt
View File

@@ -48,7 +48,7 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY}
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6")
set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)")
# Setting arhitecture extension for rpm packages
set(MG_ARCH_EXTENSION_RPM "noarch")
@@ -69,6 +69,12 @@ set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION
set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils")
set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/rpm/memgraph.spec.in")
set(CPACK_RPM_PACKAGE_LICENSE "Memgraph License")
# CPACK deduces dependency to libstdc++ which:
# * can't be easily installed on Centos 7 (the one from the toolchain,
# only required to avoid printing issue within query modules)
# * it causes issues with glibcxx 2.4
# `if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")`
set(CPACK_RPM_PACKAGE_AUTOREQ " no")
# Description formatting is important, no line must be greater than 80 characters.
set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database.
@@ -77,13 +83,7 @@ the next generation of applications driver by real-time connected data.")
# Add `openssl` package to dependencies list. Used to generate SSL certificates.
# We also depend on `python3` because we embed it in Memgraph.
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc++ >= 3.4.29, logrotate")
# If amzn-2
if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")
# It causes issues with glibcxx 2.4
set(CPACK_RPM_PACKAGE_AUTOREQ " no")
endif()
set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, logrotate")
# All variables must be set before including.
include(CPack)

3
release/docker/package_docker
View File

@@ -55,7 +55,8 @@ image_name="memgraph:${version}"
image_package_name="memgraph-${version}-docker.tar.gz"
# Build docker image.
docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
# NOTE: --pull is here to always pull that latest base image because of security patches.
docker build --pull -t ${image_name} ${tag_latest} -f ${dockerfile_path} \
--build-arg BINARY_NAME=${package_name} \
--build-arg EXTENSION=${extension} \
--build-arg TARGETARCH="" .

18
release/package/run.sh
View File

@@ -17,7 +17,8 @@ ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate"
HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output"
print_help () {
echo "$0 init|package {os} [--for-docker|--for-platform]|docker|test"
# TODO(gitbuda): Update the release/package/run.sh help
echo "$0 init|package|docker|test {os} [--for-docker|--for-platform]"
echo ""
echo " OSs: ${SUPPORTED_OS[*]}"
exit 1
@@ -35,7 +36,7 @@ make_package () {
package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm "
fi
if [[ "$os" =~ ^"debian".* ]]; then
docker exec "$build_container" bash -c "apt update"
docker exec "$build_container" bash -c "apt --allow-releaseinfo-change -y update"
package_command=" cpack -G DEB --config ../CPackConfig.cmake "
fi
if [[ "$os" =~ ^"ubuntu".* ]]; then
@@ -64,6 +65,7 @@ make_package () {
git fetch origin master:master
fi
docker exec "$build_container" mkdir -p /memgraph
# TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI.
docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/"
container_build_dir="/memgraph/build"
@@ -74,6 +76,8 @@ make_package () {
# environment/os/{os}.sh does not come within the toolchain package. When
# migrating to the next version of toolchain do that, and remove the
# TOOLCHAIN_RUN_DEPS installation from here.
# TODO(gitbuda): On the other side, having this here allows updating deps
# wihout reruning the build containers.
echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..."
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS"
docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS"
@@ -83,6 +87,7 @@ make_package () {
docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'"
docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init"
docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*"
# TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX
if [[ "$os" =~ "-arm" ]]; then
docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .."
else
@@ -108,8 +113,13 @@ make_package () {
case "$1" in
init)
cd "$SCRIPT_DIR"
docker-compose build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
docker-compose up -d
if ! which "docker-compose" >/dev/null; then
docker_compose_cmd="docker compose"
else
docker_compose_cmd="docker-compose"
fi
$docker_compose_cmd build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}"
$docker_compose_cmd up -d
;;
docker)

2
src/memgraph.cpp
View File

@@ -891,7 +891,7 @@ int main(int argc, char **argv) {
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit,
.restore_replicas_on_startup = true,
.restore_replication_state_on_startup = true,
.items_per_batch = FLAGS_storage_items_per_batch,
.recovery_thread_count = FLAGS_storage_recovery_thread_count,
.allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery},

6
src/mg_import_csv.cpp
View File

@@ -436,7 +436,7 @@ void ProcessNodeRow(memgraph::storage::Storage *store, const std::vector<std::st
if (it != node_id_map->end()) {
if (FLAGS_skip_duplicate_nodes) {
spdlog::warn(memgraph::utils::MessageWithLink("Skipping duplicate node with ID '{}'.", node_id,
"https://memgr.ph/csv"));
"https://memgr.ph/csv-import-tool"));
return;
} else {
throw LoadException("Node with ID '{}' already exists", node_id);
@@ -529,7 +529,7 @@ void ProcessRelationshipsRow(memgraph::storage::Storage *store, const std::vecto
if (it == node_id_map.end()) {
if (FLAGS_skip_bad_relationships) {
spdlog::warn(memgraph::utils::MessageWithLink("Skipping bad relationship with START_ID '{}'.", node_id,
"https://memgr.ph/csv"));
"https://memgr.ph/csv-import-tool"));
return;
} else {
throw LoadException("Node with ID '{}' does not exist", node_id);
@@ -547,7 +547,7 @@ void ProcessRelationshipsRow(memgraph::storage::Storage *store, const std::vecto
if (it == node_id_map.end()) {
if (FLAGS_skip_bad_relationships) {
spdlog::warn(memgraph::utils::MessageWithLink("Skipping bad relationship with END_ID '{}'.", node_id,
"https://memgr.ph/csv"));
"https://memgr.ph/csv-import-tool"));
return;
} else {
throw LoadException("Node with ID '{}' does not exist", node_id);

2
src/query/constants.hpp
View File

@@ -14,8 +14,6 @@
#include <string>
namespace memgraph::query {
inline constexpr uint16_t kDefaultReplicationPort = 10000;
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
inline const std::string kAsterisk = "*";
inline constexpr uint16_t kComputeStatisticsNumResults = 7;
} // namespace memgraph::query

2
src/query/exceptions.hpp
View File

@@ -254,7 +254,7 @@ class ReplicationException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
explicit ReplicationException(const std::string &message)
: utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICA' query.",
: utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICAS' query.",
message) {}
};

5
src/query/frontend/ast/ast.hpp
View File

@@ -3022,6 +3022,7 @@ class LoadCsv : public memgraph::query::Clause {
bool ignore_bad_;
memgraph::query::Expression *delimiter_{nullptr};
memgraph::query::Expression *quote_{nullptr};
memgraph::query::Expression *nullif_{nullptr};
memgraph::query::Identifier *row_var_{nullptr};
LoadCsv *Clone(AstStorage *storage) const override {
@@ -3031,18 +3032,20 @@ class LoadCsv : public memgraph::query::Clause {
object->ignore_bad_ = ignore_bad_;
object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr;
object->quote_ = quote_ ? quote_->Clone(storage) : nullptr;
object->nullif_ = nullif_;
object->row_var_ = row_var_ ? row_var_->Clone(storage) : nullptr;
return object;
}
protected:
explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, Expression *delimiter, Expression *quote,
Identifier *row_var)
Expression *nullif, Identifier *row_var)
: file_(file),
with_header_(with_header),
ignore_bad_(ignore_bad),
delimiter_(delimiter),
quote_(quote),
nullif_(nullif),
row_var_(row_var) {
DMG_ASSERT(row_var, "LoadCsv cannot take nullptr for identifier");
}

5
src/query/frontend/ast/cypher_main_visitor.cpp
View File

@@ -362,6 +362,11 @@ antlrcpp::Any CypherMainVisitor::visitLoadCsv(MemgraphCypher::LoadCsvContext *ct
// handle skip bad row option
load_csv->ignore_bad_ = ctx->IGNORE() && ctx->BAD();
// handle character sequence which will correspond to nulls
if (ctx->NULLIF()) {
load_csv->nullif_ = std::any_cast<Expression *>(ctx->nullif()->accept(this));
}
// handle delimiter
if (ctx->DELIMITER()) {
if (ctx->delimiter()->literal()->StringLiteral()) {

4
src/query/frontend/opencypher/grammar/MemgraphCypher.g4
View File

@@ -59,6 +59,7 @@ memgraphCypherKeyword : cypherKeyword
| GRANT
| HEADER
| IDENTIFIED
| NULLIF
| ISOLATION
| IN_MEMORY_ANALYTICAL
| IN_MEMORY_TRANSACTIONAL
@@ -224,6 +225,7 @@ loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
( IGNORE BAD ) ?
( DELIMITER delimiter ) ?
( QUOTE quote ) ?
( NULLIF nullif ) ?
AS rowVar ;
csvFile : literal ;
@@ -232,6 +234,8 @@ delimiter : literal ;
quote : literal ;
nullif : literal ;
rowVar : variable ;
userOrRoleName : symbolicName ;

1
src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4
View File

@@ -85,6 +85,7 @@ MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ;
NEXT : N E X T ;
NO : N O ;
NOTHING : N O T H I N G ;
NULLIF : N U L L I F ;
PASSWORD : P A S S W O R D ;
PORT : P O R T ;
PRIVILEGES : P R I V I L E G E S ;

20
src/query/interpreter.cpp
View File

@@ -177,7 +177,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
throw QueryRuntimeException("Port number invalid!");
}
if (!db_->SetReplicaRole(
io::network::Endpoint(query::kDefaultReplicationServerIp, static_cast<uint16_t>(*port)))) {
io::network::Endpoint(storage::replication::kDefaultReplicationServerIp, static_cast<uint16_t>(*port)))) {
throw QueryRuntimeException("Couldn't set role to replica!");
}
}
@@ -186,9 +186,9 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
ReplicationQuery::ReplicationRole ShowReplicationRole() const override {
switch (db_->GetReplicationRole()) {
case storage::ReplicationRole::MAIN:
case storage::replication::ReplicationRole::MAIN:
return ReplicationQuery::ReplicationRole::MAIN;
case storage::ReplicationRole::REPLICA:
case storage::replication::ReplicationRole::REPLICA:
return ReplicationQuery::ReplicationRole::REPLICA;
}
throw QueryRuntimeException("Couldn't show replication role - invalid role set!");
@@ -198,11 +198,15 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
void RegisterReplica(const std::string &name, const std::string &socket_address,
const ReplicationQuery::SyncMode sync_mode,
const std::chrono::seconds replica_check_frequency) override {
if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) {
// replica can't register another replica
throw QueryRuntimeException("Replica can't register another replica!");
}
if (name == storage::replication::kReservedReplicationRoleName) {
throw QueryRuntimeException("This replica name is reserved and can not be used as replica name!");
}
storage::replication::ReplicationMode repl_mode;
switch (sync_mode) {
case ReplicationQuery::SyncMode::ASYNC: {
@@ -216,7 +220,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
}
auto maybe_ip_and_port =
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::kDefaultReplicationPort);
io::network::Endpoint::ParseSocketOrIpAddress(socket_address, storage::replication::kDefaultReplicationPort);
if (maybe_ip_and_port) {
auto [ip, port] = *maybe_ip_and_port;
auto ret = db_->RegisterReplica(name, {std::move(ip), port}, repl_mode,
@@ -232,7 +236,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
void DropReplica(const std::string &replica_name) override {
if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) {
// replica can't unregister a replica
throw QueryRuntimeException("Replica can't unregister a replica!");
}
@@ -243,7 +247,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
using Replica = ReplicationQueryHandler::Replica;
std::vector<Replica> ShowReplicas() const override {
if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) {
if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) {
// replica can't show registered replicas (it shouldn't have any)
throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!");
}
@@ -3063,7 +3067,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
UpdateTypeCount(rw_type);
if (const auto query_type = query_execution->prepared_query->rw_type;
interpreter_context_->db->GetReplicationRole() == storage::ReplicationRole::REPLICA &&
interpreter_context_->db->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA &&
(query_type == RWType::W || query_type == RWType::RW)) {
query_execution = nullptr;
throw QueryException("Write query forbidden on the replica!");

35
src/query/plan/operator.cpp
View File

@@ -4637,13 +4637,14 @@ UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const {
}
LoadCsv::LoadCsv(std::shared_ptr<LogicalOperator> input, Expression *file, bool with_header, bool ignore_bad,
Expression *delimiter, Expression *quote, Symbol row_var)
Expression *delimiter, Expression *quote, Expression *nullif, Symbol row_var)
: input_(input ? input : (std::make_shared<Once>())),
file_(file),
with_header_(with_header),
ignore_bad_(ignore_bad),
delimiter_(delimiter),
quote_(quote),
nullif_(nullif),
row_var_(row_var) {
MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__);
}
@@ -4674,22 +4675,31 @@ auto ToOptionalString(ExpressionEvaluator *evaluator, Expression *expression) ->
return std::nullopt;
};
TypedValue CsvRowToTypedList(csv::Reader::Row &row) {
TypedValue CsvRowToTypedList(csv::Reader::Row &row, std::optional<utils::pmr::string> &nullif) {
auto *mem = row.get_allocator().GetMemoryResource();
auto typed_columns = utils::pmr::vector<TypedValue>(mem);
typed_columns.reserve(row.size());
for (auto &column : row) {
typed_columns.emplace_back(std::move(column));
if (!nullif.has_value() || column != nullif.value()) {
typed_columns.emplace_back(std::move(column));
} else {
typed_columns.emplace_back();
}
}
return {std::move(typed_columns), mem};
}
TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header) {
TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header,
std::optional<utils::pmr::string> &nullif) {
// a valid row has the same number of elements as the header
auto *mem = row.get_allocator().GetMemoryResource();
utils::pmr::map<utils::pmr::string, TypedValue> m(mem);
for (auto i = 0; i < row.size(); ++i) {
m.emplace(std::move(header[i]), std::move(row[i]));
if (!nullif.has_value() || row[i] != nullif.value()) {
m.emplace(std::move(header[i]), std::move(row[i]));
} else {
m.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(header[i])), std::forward_as_tuple());
}
}
return {std::move(m), mem};
}
@@ -4701,6 +4711,7 @@ class LoadCsvCursor : public Cursor {
const UniqueCursorPtr input_cursor_;
bool did_pull_;
std::optional<csv::Reader> reader_{};
std::optional<utils::pmr::string> nullif_;
public:
LoadCsvCursor(const LoadCsv *self, utils::MemoryResource *mem)
@@ -4718,6 +4729,7 @@ class LoadCsvCursor : public Cursor {
// without massacring the code even worse than I did here
if (UNLIKELY(!reader_)) {
reader_ = MakeReader(&context.evaluation_context);
nullif_ = ParseNullif(&context.evaluation_context);
}
if (input_cursor_->Pull(frame, context)) {
@@ -4733,10 +4745,10 @@ class LoadCsvCursor : public Cursor {
return false;
}
if (!reader_->HasHeader()) {
frame[self_->row_var_] = CsvRowToTypedList(*row);
frame[self_->row_var_] = CsvRowToTypedList(*row, nullif_);
} else {
frame[self_->row_var_] =
CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory));
CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory), nullif_);
}
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(self_->row_var_.name())) {
context.frame_change_collector->ResetTrackingValue(self_->row_var_.name());
@@ -4768,6 +4780,15 @@ class LoadCsvCursor : public Cursor {
csv::Reader::Config(self_->with_header_, self_->ignore_bad_, std::move(maybe_delim), std::move(maybe_quote)),
utils::NewDeleteResource());
}
std::optional<utils::pmr::string> ParseNullif(EvaluationContext *eval_context) {
Frame frame(0);
SymbolTable symbol_table;
DbAccessor *dba = nullptr;
auto evaluator = ExpressionEvaluator(&frame, symbol_table, *eval_context, dba, storage::View::OLD);
return ToOptionalString(&evaluator, self_->nullif_);
}
};
UniqueCursorPtr LoadCsv::MakeCursor(utils::MemoryResource *mem) const {

4
src/query/plan/operator.hpp
View File

@@ -2227,7 +2227,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator {
LoadCsv() = default;
LoadCsv(std::shared_ptr<LogicalOperator> input, Expression *file, bool with_header, bool ignore_bad,
Expression *delimiter, Expression *quote, Symbol row_var);
Expression *delimiter, Expression *quote, Expression *nullif, Symbol row_var);
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override;
std::vector<Symbol> OutputSymbols(const SymbolTable &) const override;
@@ -2243,6 +2243,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator {
bool ignore_bad_;
Expression *delimiter_{nullptr};
Expression *quote_{nullptr};
Expression *nullif_{nullptr};
Symbol row_var_;
std::unique_ptr<LogicalOperator> Clone(AstStorage *storage) const override {
@@ -2253,6 +2254,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator {
object->ignore_bad_ = ignore_bad_;
object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr;
object->quote_ = quote_ ? quote_->Clone(storage) : nullptr;
object->nullif_ = nullif_;
object->row_var_ = row_var_;
return object;
}

4
src/query/plan/pretty_print.cpp
View File

@@ -895,6 +895,10 @@ bool PlanToJsonVisitor::PreVisit(query::plan::LoadCsv &op) {
self["quote"] = ToJson(op.quote_);
}
if (op.nullif_) {
self["nullif"] = ToJson(op.nullif_);
}
self["row_variable"] = ToJson(op.row_var_);
op.input_->Accept(*this);

7
src/query/plan/rule_based_planner.hpp
View File

@@ -226,10 +226,9 @@ class RuleBasedPlanner {
const auto &row_sym = context.symbol_table->at(*load_csv->row_var_);
context.bound_symbols.insert(row_sym);
input_op =
std::make_unique<plan::LoadCsv>(std::move(input_op), load_csv->file_, load_csv->with_header_,
load_csv->ignore_bad_, load_csv->delimiter_, load_csv->quote_, row_sym);
input_op = std::make_unique<plan::LoadCsv>(std::move(input_op), load_csv->file_, load_csv->with_header_,
load_csv->ignore_bad_, load_csv->delimiter_, load_csv->quote_,
load_csv->nullif_, row_sym);
} else if (auto *foreach = utils::Downcast<query::Foreach>(clause)) {
context.is_write_query = true;
input_op = HandleForeachClause(foreach, std::move(input_op), *context.symbol_table, context.bound_symbols,

16
src/query/procedure/module.hpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -131,7 +131,12 @@ class ModuleRegistry final {
private:
class SharedLibraryHandle {
public:
SharedLibraryHandle(const std::string &shared_library, int mode) : handle_{dlopen(shared_library.c_str(), mode)} {}
SharedLibraryHandle(const std::string &shared_library, int mode, const std::string &hint_message = "")
: handle_{dlopen(shared_library.c_str(), mode)} {
if (!handle_) {
spdlog::warn("Unable to load {}. {}", shared_library, hint_message);
}
}
SharedLibraryHandle(const SharedLibraryHandle &) = delete;
SharedLibraryHandle(SharedLibraryHandle &&) = delete;
SharedLibraryHandle operator=(const SharedLibraryHandle &) = delete;
@@ -147,10 +152,13 @@ class ModuleRegistry final {
void *handle_;
};
inline static const std::string kLibstdcppWarning =
"Query modules might not work as expected. Printing non-string values from query modules might not work. Please "
"install libstdc++ or compile from source with the recent toolchain (all included).";
#if __has_feature(address_sanitizer)
// This is why we need RTLD_NODELETE and we must not use RTLD_DEEPBIND with
// ASAN: https://github.com/google/sanitizers/issues/89
SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE};
SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE, kLibstdcppWarning};
#else
// The reason behind opening share library during runtime is to avoid issues
// with loading symbols from stdlib. We have encounter issues with locale
@@ -161,7 +169,7 @@ class ModuleRegistry final {
// mentioned library will be first performed in the already existing binded
// libraries and then the global namespace.
// RTLD_DEEPBIND => https://linux.die.net/man/3/dlopen
SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND};
SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND, kLibstdcppWarning};
#endif
std::vector<std::filesystem::path> modules_dirs_;
std::filesystem::path internal_module_dir_;

2
src/storage/v2/config.hpp
View File

@@ -49,7 +49,7 @@ struct Config {
uint64_t wal_file_flush_every_n_tx{100000};
bool snapshot_on_exit{false};
bool restore_replicas_on_startup{false};
bool restore_replication_state_on_startup{false};
uint64_t items_per_batch{1'000'000};
uint64_t recovery_thread_count{8};

6
src/storage/v2/durability/durability.cpp
View File

@@ -74,6 +74,12 @@ std::vector<SnapshotDurabilityInfo> GetSnapshotFiles(const std::filesystem::path
if (utils::DirExists(snapshot_directory)) {
for (const auto &item : std::filesystem::directory_iterator(snapshot_directory, error_code)) {
if (!item.is_regular_file()) continue;
if (!utils::HasReadAccess(item.path())) {
spdlog::warn(
"Skipping snapshot file '{}' because it is not readable, check file ownership and read permissions!",
item.path());
continue;
}
try {
auto info = ReadSnapshotInfo(item.path());
if (uuid.empty() || info.uuid == uuid) {

1
src/storage/v2/durability/snapshot.cpp
View File

@@ -26,6 +26,7 @@
#include "storage/v2/vertex.hpp"
#include "storage/v2/vertex_accessor.hpp"
#include "utils/concepts.hpp"
#include "utils/file.hpp"
#include "utils/file_locker.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"

4
src/storage/v2/replication/enums.hpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,6 +13,8 @@
#include <cstdint>
namespace memgraph::storage::replication {
enum class ReplicationRole : uint8_t { MAIN, REPLICA };
enum class ReplicationMode : std::uint8_t { SYNC, ASYNC };
enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, INVALID };

34
src/storage/v2/replication/replication_persistence_helper.cpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,21 +10,24 @@
// licenses/APL.txt.
#include "storage/v2/replication/replication_persistence_helper.hpp"
#include "storage/v2/replication/enums.hpp"
#include "utils/logging.hpp"
namespace {
const std::string kReplicaName = "replica_name";
const std::string kIpAddress = "replica_ip_address";
const std::string kPort = "replica_port";
const std::string kSyncMode = "replica_sync_mode";
const std::string kCheckFrequency = "replica_check_frequency";
const std::string kSSLKeyFile = "replica_ssl_key_file";
const std::string kSSLCertFile = "replica_ssl_cert_file";
inline constexpr auto *kReplicaName = "replica_name";
inline constexpr auto *kIpAddress = "replica_ip_address";
inline constexpr auto *kPort = "replica_port";
inline constexpr auto *kSyncMode = "replica_sync_mode";
inline constexpr auto *kCheckFrequency = "replica_check_frequency";
inline constexpr auto *kSSLKeyFile = "replica_ssl_key_file";
inline constexpr auto *kSSLCertFile = "replica_ssl_cert_file";
inline constexpr auto *kReplicationRole = "replication_role";
} // namespace
namespace memgraph::storage::replication {
nlohmann::json ReplicaStatusToJSON(ReplicaStatus &&status) {
nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status) {
auto data = nlohmann::json::object();
data[kReplicaName] = std::move(status.name);
@@ -42,11 +45,15 @@ nlohmann::json ReplicaStatusToJSON(ReplicaStatus &&status) {
data[kSSLCertFile] = nullptr;
}
if (status.role.has_value()) {
data[kReplicationRole] = *status.role;
}
return data;
}
std::optional<ReplicaStatus> JSONToReplicaStatus(nlohmann::json &&data) {
ReplicaStatus replica_status;
std::optional<ReplicationStatus> JSONToReplicationStatus(nlohmann::json &&data) {
ReplicationStatus replica_status;
const auto get_failed_message = [](const std::string_view message, const std::string_view nested_message) {
return fmt::format("Failed to deserialize replica's configuration: {} : {}", message, nested_message);
@@ -70,6 +77,11 @@ std::optional<ReplicaStatus> JSONToReplicaStatus(nlohmann::json &&data) {
data.at(kSSLKeyFile).get_to(replica_status.ssl->key_file);
data.at(kSSLCertFile).get_to(replica_status.ssl->cert_file);
}
if (data.find(kReplicationRole) != data.end()) {
replica_status.role = replication::ReplicationRole::MAIN;
data.at(kReplicationRole).get_to(replica_status.role.value());
}
} catch (const nlohmann::json::type_error &exception) {
spdlog::error(get_failed_message("Invalid type conversion", exception.what()));
return std::nullopt;

15
src/storage/v2/replication/replication_persistence_helper.hpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -23,18 +23,23 @@
namespace memgraph::storage::replication {
struct ReplicaStatus {
inline constexpr auto *kReservedReplicationRoleName{"__replication_role"};
inline constexpr uint16_t kDefaultReplicationPort = 10000;
inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0";
struct ReplicationStatus {
std::string name;
std::string ip_address;
uint16_t port;
ReplicationMode sync_mode;
std::chrono::seconds replica_check_frequency;
std::optional<ReplicationClientConfig::SSL> ssl;
std::optional<ReplicationRole> role;
friend bool operator==(const ReplicaStatus &, const ReplicaStatus &) = default;
friend bool operator==(const ReplicationStatus &, const ReplicationStatus &) = default;
};
nlohmann::json ReplicaStatusToJSON(ReplicaStatus &&status);
nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status);
std::optional<ReplicaStatus> JSONToReplicaStatus(nlohmann::json &&data);
std::optional<ReplicationStatus> JSONToReplicationStatus(nlohmann::json &&data);
} // namespace memgraph::storage::replication

165
src/storage/v2/storage.cpp
View File

@@ -335,13 +335,6 @@ Storage::Storage(Config config)
uuid_(utils::GenerateUUID()),
epoch_id_(utils::GenerateUUID()),
global_locker_(file_retainer_.AddLocker()) {
if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED &&
replication_role_ == ReplicationRole::MAIN) {
spdlog::warn(
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider "
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
"without write-ahead logs this instance is not replicating any data.");
}
if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED ||
config_.durability.snapshot_on_exit || config_.durability.recover_on_startup) {
// Create the directory initially to crash the database in case of
@@ -437,14 +430,29 @@ Storage::Storage(Config config)
commit_log_.emplace(timestamp_);
}
if (config_.durability.restore_replicas_on_startup) {
spdlog::info("Replica's configuration will be stored and will be automatically restored in case of a crash.");
if (config_.durability.restore_replication_state_on_startup) {
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
utils::EnsureDirOrDie(config_.durability.storage_directory / durability::kReplicationDirectory);
storage_ =
std::make_unique<kvstore::KVStore>(config_.durability.storage_directory / durability::kReplicationDirectory);
RestoreReplicas();
RestoreReplicationRole();
if (replication_role_ == replication::ReplicationRole::MAIN) {
RestoreReplicas();
}
} else {
spdlog::warn("Replicas' configuration will NOT be stored. When the server restarts, replicas will be forgotten.");
spdlog::warn(
"Replicastion configuration will NOT be stored. When the server restarts, replication state will be "
"forgotten.");
}
if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED &&
replication_role_ == replication::ReplicationRole::MAIN) {
spdlog::warn(
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider "
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
"without write-ahead logs this instance is not replicating any data.");
}
}
@@ -968,7 +976,7 @@ utils::BasicResult<StorageDataManipulationError, void> Storage::Accessor::Commit
// modifications before they are written to disk.
// Replica can log only the write transaction received from Main
// so the Wal files are consistent
if (storage_->replication_role_ == ReplicationRole::MAIN || desired_commit_timestamp.has_value()) {
if (storage_->replication_role_ == replication::ReplicationRole::MAIN || desired_commit_timestamp.has_value()) {
could_replicate_all_sync_replicas = storage_->AppendToWalDataManipulation(transaction_, *commit_timestamp_);
}
@@ -982,7 +990,8 @@ utils::BasicResult<StorageDataManipulationError, void> Storage::Accessor::Commit
transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release);
// Replica can only update the last commit timestamp with
// the commits received from main.
if (storage_->replication_role_ == ReplicationRole::MAIN || desired_commit_timestamp.has_value()) {
if (storage_->replication_role_ == replication::ReplicationRole::MAIN ||
desired_commit_timestamp.has_value()) {
// Update the last commit timestamp
storage_->last_commit_timestamp_.store(*commit_timestamp_);
}
@@ -1447,7 +1456,7 @@ Transaction Storage::CreateTransaction(IsolationLevel isolation_level, StorageMo
// of any query on replica to the last commited transaction
// which is timestamp_ as only commit of transaction with writes
// can change the value of it.
if (replication_role_ == ReplicationRole::REPLICA) {
if (replication_role_ == replication::ReplicationRole::REPLICA) {
start_timestamp = timestamp_;
} else {
start_timestamp = timestamp_++;
@@ -1752,7 +1761,7 @@ bool Storage::AppendToWalDataManipulation(const Transaction &transaction, uint64
// A single transaction will always be contained in a single WAL file.
auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire);
if (replication_role_.load() == ReplicationRole::MAIN) {
if (replication_role_.load() == replication::ReplicationRole::MAIN) {
replication_clients_.WithLock([&](auto &clients) {
for (auto &client : clients) {
client->StartTransactionReplication(wal_file_->SequenceNumber());
@@ -1940,7 +1949,7 @@ bool Storage::AppendToWalDataDefinition(durability::StorageGlobalOperation opera
auto finalized_on_all_replicas = true;
wal_file_->AppendOperation(operation, label, properties, final_commit_timestamp);
{
if (replication_role_.load() == ReplicationRole::MAIN) {
if (replication_role_.load() == replication::ReplicationRole::MAIN) {
replication_clients_.WithLock([&](auto &clients) {
for (auto &client : clients) {
client->StartTransactionReplication(wal_file_->SequenceNumber());
@@ -1960,7 +1969,7 @@ bool Storage::AppendToWalDataDefinition(durability::StorageGlobalOperation opera
}
utils::BasicResult<Storage::CreateSnapshotError> Storage::CreateSnapshot(std::optional<bool> is_periodic) {
if (replication_role_.load() != ReplicationRole::MAIN) {
if (replication_role_.load() != replication::ReplicationRole::MAIN) {
return CreateSnapshotError::DisabledForReplica;
}
@@ -2054,20 +2063,38 @@ uint64_t Storage::CommitTimestamp(const std::optional<uint64_t> desired_commit_t
bool Storage::SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config) {
// We don't want to restart the server if we're already a REPLICA
if (replication_role_ == ReplicationRole::REPLICA) {
if (replication_role_ == replication::ReplicationRole::REPLICA) {
return false;
}
auto port = endpoint.port; // assigning because we will move the endpoint
replication_server_ = std::make_unique<ReplicationServer>(this, std::move(endpoint), config);
replication_role_.store(ReplicationRole::REPLICA);
if (ShouldStoreAndRestoreReplicationState()) {
// Only thing that matters here is the role saved as REPLICA and the listening port
auto data = replication::ReplicationStatusToJSON(
replication::ReplicationStatus{.name = replication::kReservedReplicationRoleName,
.ip_address = "",
.port = port,
.sync_mode = replication::ReplicationMode::SYNC,
.replica_check_frequency = std::chrono::seconds(0),
.ssl = std::nullopt,
.role = replication::ReplicationRole::REPLICA});
if (!storage_->Put(replication::kReservedReplicationRoleName, data.dump())) {
spdlog::error("Error when saving REPLICA replication role in settings.");
return false;
}
}
replication_role_.store(replication::ReplicationRole::REPLICA);
return true;
}
bool Storage::SetMainReplicationRole() {
// We don't want to generate new epoch_id and do the
// cleanup if we're already a MAIN
if (replication_role_ == ReplicationRole::MAIN) {
if (replication_role_ == replication::ReplicationRole::MAIN) {
return false;
}
@@ -2090,14 +2117,33 @@ bool Storage::SetMainReplicationRole() {
epoch_id_ = utils::GenerateUUID();
}
replication_role_.store(ReplicationRole::MAIN);
if (ShouldStoreAndRestoreReplicationState()) {
// Only thing that matters here is the role saved as MAIN
auto data = replication::ReplicationStatusToJSON(
replication::ReplicationStatus{.name = replication::kReservedReplicationRoleName,
.ip_address = "",
.port = 0,
.sync_mode = replication::ReplicationMode::SYNC,
.replica_check_frequency = std::chrono::seconds(0),
.ssl = std::nullopt,
.role = replication::ReplicationRole::MAIN});
if (!storage_->Put(replication::kReservedReplicationRoleName, data.dump())) {
spdlog::error("Error when saving MAIN replication role in settings.");
return false;
}
}
replication_role_.store(replication::ReplicationRole::MAIN);
return true;
}
utils::BasicResult<Storage::RegisterReplicaError> Storage::RegisterReplica(
std::string name, io::network::Endpoint endpoint, const replication::ReplicationMode replication_mode,
const replication::RegistrationMode registration_mode, const replication::ReplicationClientConfig &config) {
MG_ASSERT(replication_role_.load() == ReplicationRole::MAIN, "Only main instance can register a replica!");
MG_ASSERT(replication_role_.load() == replication::ReplicationRole::MAIN,
"Only main instance can register a replica!");
const bool name_exists = replication_clients_.WithLock([&](auto &clients) {
return std::any_of(clients.begin(), clients.end(), [&name](const auto &client) { return client->Name() == name; });
@@ -2116,14 +2162,15 @@ utils::BasicResult<Storage::RegisterReplicaError> Storage::RegisterReplica(
return RegisterReplicaError::END_POINT_EXISTS;
}
if (ShouldStoreAndRestoreReplicas()) {
auto data = replication::ReplicaStatusToJSON(
replication::ReplicaStatus{.name = name,
.ip_address = endpoint.address,
.port = endpoint.port,
.sync_mode = replication_mode,
.replica_check_frequency = config.replica_check_frequency,
.ssl = config.ssl});
if (ShouldStoreAndRestoreReplicationState()) {
auto data = replication::ReplicationStatusToJSON(
replication::ReplicationStatus{.name = name,
.ip_address = endpoint.address,
.port = endpoint.port,
.sync_mode = replication_mode,
.replica_check_frequency = config.replica_check_frequency,
.ssl = config.ssl,
.role = replication::ReplicationRole::REPLICA});
if (!storage_->Put(name, data.dump())) {
spdlog::error("Error when saving replica {} in settings.", name);
return RegisterReplicaError::COULD_NOT_BE_PERSISTED;
@@ -2159,8 +2206,9 @@ utils::BasicResult<Storage::RegisterReplicaError> Storage::RegisterReplica(
}
bool Storage::UnregisterReplica(const std::string &name) {
MG_ASSERT(replication_role_.load() == ReplicationRole::MAIN, "Only main instance can unregister a replica!");
if (ShouldStoreAndRestoreReplicas()) {
MG_ASSERT(replication_role_.load() == replication::ReplicationRole::MAIN,
"Only main instance can unregister a replica!");
if (ShouldStoreAndRestoreReplicationState()) {
if (!storage_->Delete(name)) {
spdlog::error("Error when removing replica {} from settings.", name);
return false;
@@ -2183,7 +2231,7 @@ std::optional<replication::ReplicaState> Storage::GetReplicaState(const std::str
});
}
ReplicationRole Storage::GetReplicationRole() const { return replication_role_; }
replication::ReplicationRole Storage::GetReplicationRole() const { return replication_role_; }
std::vector<Storage::ReplicaInfo> Storage::ReplicasInfo() {
return replication_clients_.WithLock([](auto &clients) {
@@ -2207,6 +2255,46 @@ utils::BasicResult<Storage::SetIsolationLevelError> Storage::SetIsolationLevel(I
return {};
}
void Storage::RestoreReplicationRole() {
if (!ShouldStoreAndRestoreReplicationState()) {
return;
}
spdlog::info("Restoring replication role.");
uint16_t port = replication::kDefaultReplicationPort;
for (const auto &[replica_name, replica_data] : *storage_) {
const auto maybe_replica_status = replication::JSONToReplicationStatus(nlohmann::json::parse(replica_data));
if (!maybe_replica_status.has_value()) {
LOG_FATAL("Cannot parse previously saved configuration of replica {}.", replica_name);
}
if (replica_name != replication::kReservedReplicationRoleName) {
continue;
}
auto replica_status = *maybe_replica_status;
if (!replica_status.role.has_value()) {
replication_role_.store(replication::ReplicationRole::MAIN);
} else {
replication_role_.store(*replica_status.role);
port = replica_status.port;
}
break;
}
if (replication_role_ == replication::ReplicationRole::REPLICA) {
io::network::Endpoint endpoint(replication::kDefaultReplicationServerIp, port);
replication_server_ =
std::make_unique<ReplicationServer>(this, std::move(endpoint), replication::ReplicationServerConfig{});
}
spdlog::info("Replication role restored to {}.",
replication_role_ == replication::ReplicationRole::MAIN ? "MAIN" : "REPLICA");
}
IsolationLevel Storage::GetIsolationLevel() const noexcept { return isolation_level_; }
void Storage::SetStorageMode(StorageMode storage_mode) {
@@ -2217,8 +2305,7 @@ void Storage::SetStorageMode(StorageMode storage_mode) {
StorageMode Storage::GetStorageMode() { return storage_mode_; }
void Storage::RestoreReplicas() {
MG_ASSERT(memgraph::storage::ReplicationRole::MAIN == GetReplicationRole());
if (!ShouldStoreAndRestoreReplicas()) {
if (!ShouldStoreAndRestoreReplicationState()) {
return;
}
spdlog::info("Restoring replicas.");
@@ -2226,7 +2313,7 @@ void Storage::RestoreReplicas() {
for (const auto &[replica_name, replica_data] : *storage_) {
spdlog::info("Restoring replica {}.", replica_name);
const auto maybe_replica_status = replication::JSONToReplicaStatus(nlohmann::json::parse(replica_data));
const auto maybe_replica_status = replication::JSONToReplicationStatus(nlohmann::json::parse(replica_data));
if (!maybe_replica_status.has_value()) {
LOG_FATAL("Cannot parse previously saved configuration of replica {}.", replica_name);
}
@@ -2235,6 +2322,10 @@ void Storage::RestoreReplicas() {
MG_ASSERT(replica_status.name == replica_name, "Expected replica name is '{}', but got '{}'", replica_status.name,
replica_name);
if (replica_name == replication::kReservedReplicationRoleName) {
continue;
}
auto ret =
RegisterReplica(std::move(replica_status.name), {std::move(replica_status.ip_address), replica_status.port},
replica_status.sync_mode, replication::RegistrationMode::CAN_BE_INVALID,
@@ -2251,6 +2342,6 @@ void Storage::RestoreReplicas() {
}
}
bool Storage::ShouldStoreAndRestoreReplicas() const { return nullptr != storage_; }
bool Storage::ShouldStoreAndRestoreReplicationState() const { return nullptr != storage_; }
} // namespace memgraph::storage

11
src/storage/v2/storage.hpp
View File

@@ -50,6 +50,7 @@
#include "rpc/server.hpp"
#include "storage/v2/replication/config.hpp"
#include "storage/v2/replication/enums.hpp"
#include "storage/v2/replication/replication_persistence_helper.hpp"
#include "storage/v2/replication/rpc.hpp"
#include "storage/v2/replication/serialization.hpp"
#include "storage/v2/storage_error.hpp"
@@ -188,8 +189,6 @@ struct StorageInfo {
uint64_t disk_usage;
};
enum class ReplicationRole : uint8_t { MAIN, REPLICA };
class Storage final {
public:
/// @throw std::system_error
@@ -516,7 +515,7 @@ class Storage final {
std::optional<replication::ReplicaState> GetReplicaState(std::string_view name);
ReplicationRole GetReplicationRole() const;
replication::ReplicationRole GetReplicationRole() const;
struct TimestampInfo {
uint64_t current_timestamp_of_replica;
@@ -580,9 +579,11 @@ class Storage final {
uint64_t CommitTimestamp(std::optional<uint64_t> desired_commit_timestamp = {});
void RestoreReplicationRole();
void RestoreReplicas();
bool ShouldStoreAndRestoreReplicas() const;
bool ShouldStoreAndRestoreReplicationState() const;
// Main storage lock.
//
@@ -703,7 +704,7 @@ class Storage final {
using ReplicationClientList = utils::Synchronized<std::vector<std::unique_ptr<ReplicationClient>>, utils::SpinLock>;
ReplicationClientList replication_clients_;
std::atomic<ReplicationRole> replication_role_{ReplicationRole::MAIN};
std::atomic<replication::ReplicationRole> replication_role_{replication::ReplicationRole::MAIN};
};
} // namespace memgraph::storage

4
src/utils/file.cpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -82,6 +82,8 @@ bool RenamePath(const std::filesystem::path &src, const std::filesystem::path &d
return !error_code;
}
bool HasReadAccess(const std::filesystem::path &path) { return access(path.c_str(), R_OK) == 0; }
static_assert(std::is_same_v<off_t, ssize_t>, "off_t must fit into ssize_t!");
InputFile::~InputFile() { Close(); }

5
src/utils/file.hpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -63,6 +63,9 @@ bool CopyFile(const std::filesystem::path &src, const std::filesystem::path &dst
/// don't exist, the renaming fails. Symlinks are not followed.
bool RenamePath(const std::filesystem::path &src, const std::filesystem::path &dst);
/// Checks if process has read access to the file.
bool HasReadAccess(const std::filesystem::path &path);
/// Buffer size used for `InputFile` and `OutputFile` implementations. Using
/// system calls is very expensive and we can't afford to call either `read` or
/// `write` for each of our (very small) logical reads/writes. Because of that,

3
tests/e2e/load_csv/CMakeLists.txt
View File

@@ -8,3 +8,6 @@ endfunction()
copy_load_csv_e2e_python_files(load_csv.py)
copy_load_csv_e2e_files(simple.csv)
copy_load_csv_e2e_python_files(load_csv_nullif.py)
copy_load_csv_e2e_files(nullif.csv)

53
tests/e2e/load_csv/load_csv_nullif.py Normal file
View File

@@ -0,0 +1,53 @@
# Copyright 2022 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import os
import sys
from pathlib import Path
import pytest
from gqlalchemy import Memgraph
NULLIF_CSV_FILE = "nullif.csv"
def get_file_path(file: str) -> str:
parent_path = Path(__file__).parent.absolute()
return os.path.join(parent_path, file)
def test_given_csv_when_nullif_then_all_identical_rows_are_null():
memgraph = Memgraph("localhost", 7687)
results = list(
memgraph.execute_and_fetch(
f"""LOAD CSV FROM '{get_file_path(NULLIF_CSV_FILE)}'
WITH HEADER NULLIF 'N/A' AS row
CREATE (n:Person {{name: row.name, age: row.age,
percentage: row.percentage, works_in_IT: row.works_in_IT}})
RETURN n
"""
)
)
expected_properties = [
{"age": "10", "percentage": "15.0", "works_in_IT": "false"},
{"name": "John", "percentage": "35.4", "works_in_IT": "false"},
{"name": "Milewa", "age": "34", "works_in_IT": "false"},
{"name": "Lucas", "age": "50", "percentage": "12.5"},
]
properties = [result["n"]._properties for result in results]
assert expected_properties == properties
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-rA"]))

5
tests/e2e/load_csv/nullif.csv Normal file
View File

@@ -0,0 +1,5 @@
name,age,percentage,works_in_IT
N/A,10,15.0,false
John,N/A,35.4,false
Milewa,34,N/A,false
Lucas,50,12.5,N/A
1 name age percentage works_in_IT
2 N/A 10 15.0 false
3 John N/A 35.4 false
4 Milewa 34 N/A false
5 Lucas 50 12.5 N/A

11
tests/e2e/load_csv/workloads.yaml
View File

@@ -1,3 +1,10 @@
nullif_cluster: &nullif_cluster
cluster:
main:
args: ["--bolt-port", "7687", "--log-level=TRACE"]
log_file: "load_csv_log_file.txt"
validation_queries: []
load_csv_cluster: &load_csv_cluster
cluster:
main:
@@ -9,6 +16,10 @@ load_csv_cluster: &load_csv_cluster
validation_queries: []
workloads:
- name: "LOAD CSV nullif"
binary: "tests/e2e/pytest_runner.sh"
args: ["load_csv/load_csv_nullif.py"]
<<: *nullif_cluster
- name: "MATCH + LOAD CSV"
binary: "tests/e2e/pytest_runner.sh"
args: ["load_csv/load_csv.py"]

143
tests/e2e/replication/show_while_creating_invalid_state.py
View File

@@ -9,17 +9,16 @@
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import sys
import os
import pytest
import random
import sys
import tempfile
from common import execute_and_fetch_all
from mg_utils import mg_sleep_and_assert
import interactive_mg_runner
import mgclient
import tempfile
import pytest
from common import execute_and_fetch_all
from mg_utils import mg_sleep_and_assert
interactive_mg_runner.SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
interactive_mg_runner.PROJECT_DIR = os.path.normpath(
@@ -340,6 +339,138 @@ def test_basic_recovery(connection):
assert interactive_mg_runner.MEMGRAPH_INSTANCES[f"replica_{index}"].query(QUERY_TO_CHECK) == res_from_main
def test_replication_role_recovery(connection):
# Goal of this test is to check the recovery of main and replica role.
# 0/ We start all replicas manually: we want to be able to kill them ourselves without relying on external tooling to kill processes.
# 1/ We try to add a replica with reserved name which results in an exception
# 2/ We check that all replicas have the correct state: they should all be ready.
# 3/ We kill main.
# 4/ We re-start main. We check that main indeed has the role main and replicas still have the correct state.
# 5/ We kill the replica.
# 6/ We observed that the replica result is in invalid state.
# 7/ We start the replica again. We observe that indeed the replica has the replica state.
# 8/ We observe that main has the replica ready.
# 9/ We kill the replica again.
# 10/ We add data to main.
# 11/ We start the replica again. We observe that the replica has the same
# data as main because it synced and added lost data.
# 0/
data_directory = tempfile.TemporaryDirectory()
CONFIGURATION = {
"replica": {
"args": ["--bolt-port", "7688", "--log-level=TRACE"],
"log_file": "replica.log",
"setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"],
"data_directory": f"{data_directory.name}/replica",
},
"main": {
"args": ["--bolt-port", "7687", "--log-level=TRACE", "--storage-recover-on-startup=true"],
"log_file": "main.log",
"setup_queries": [],
"data_directory": f"{data_directory.name}/main",
},
}
interactive_mg_runner.start_all(CONFIGURATION)
cursor = connection(7687, "main").cursor()
# We want to execute manually and not via the configuration, otherwise re-starting main would also execute these registration.
execute_and_fetch_all(cursor, "REGISTER REPLICA replica SYNC TO '127.0.0.1:10001';")
# When we restart the replica, it does not need this query anymore since it needs to remember state
CONFIGURATION = {
"replica": {
"args": ["--bolt-port", "7688", "--log-level=TRACE"],
"log_file": "replica.log",
"setup_queries": [],
"data_directory": f"{data_directory.name}/replica",
},
"main": {
"args": ["--bolt-port", "7687", "--log-level=TRACE", "--storage-recover-on-startup=true"],
"log_file": "main.log",
"setup_queries": [],
"data_directory": f"{data_directory.name}/main",
},
}
# 1/
with pytest.raises(mgclient.DatabaseError):
execute_and_fetch_all(cursor, "REGISTER REPLICA __replication_role SYNC TO '127.0.0.1:10002';")
# 2/
expected_data = {
("replica", "127.0.0.1:10001", "sync", 0, 0, "ready"),
}
actual_data = set(execute_and_fetch_all(cursor, "SHOW REPLICAS;"))
assert actual_data == expected_data
def check_roles():
assert "main" == interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query("SHOW REPLICATION ROLE;")[0][0]
assert "replica" == interactive_mg_runner.MEMGRAPH_INSTANCES["replica"].query("SHOW REPLICATION ROLE;")[0][0]
check_roles()
# 3/
interactive_mg_runner.kill(CONFIGURATION, "main")
# 4/
interactive_mg_runner.start(CONFIGURATION, "main")
cursor = connection(7687, "main").cursor()
check_roles()
def retrieve_data():
return set(execute_and_fetch_all(cursor, "SHOW REPLICAS;"))
actual_data = mg_sleep_and_assert(expected_data, retrieve_data)
assert actual_data == expected_data
# 5/
interactive_mg_runner.kill(CONFIGURATION, "replica")
# 6/
expected_data = {
("replica", "127.0.0.1:10001", "sync", 0, 0, "invalid"),
}
actual_data = mg_sleep_and_assert(expected_data, retrieve_data)
assert actual_data == expected_data
# 7/
interactive_mg_runner.start(CONFIGURATION, "replica")
check_roles()
# 8/
expected_data = {
("replica", "127.0.0.1:10001", "sync", 0, 0, "ready"),
}
actual_data = mg_sleep_and_assert(expected_data, retrieve_data)
assert actual_data == expected_data
# 9/
interactive_mg_runner.kill(CONFIGURATION, "replica")
# 10/
with pytest.raises(mgclient.DatabaseError):
execute_and_fetch_all(cursor, "CREATE (n:First)")
# 11/
interactive_mg_runner.start(CONFIGURATION, "replica")
check_roles()
expected_data = {
("replica", "127.0.0.1:10001", "sync", 2, 0, "ready"),
}
actual_data = mg_sleep_and_assert(expected_data, retrieve_data)
assert actual_data == expected_data
QUERY_TO_CHECK = "MATCH (node) return node;"
res_from_main = execute_and_fetch_all(cursor, QUERY_TO_CHECK)
assert len(res_from_main) == 1
assert res_from_main == interactive_mg_runner.MEMGRAPH_INSTANCES["replica"].query(QUERY_TO_CHECK)
def test_conflict_at_startup(connection):
# Goal of this test is to check starting up several instance with different replicas' configuration directory works as expected.
# main_1 and main_2 have different directory.

85
tests/unit/replication_persistence_helper.cpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,73 +19,82 @@
#include <optional>
#include <string>
class ReplicationPersistanceHelperTest : public ::testing::Test {
using namespace memgraph::storage::replication;
class ReplicationPersistanceHelperTest : public testing::Test {
protected:
void SetUp() override {}
void TearDown() override {}
memgraph::storage::replication::ReplicaStatus CreateReplicaStatus(
std::string name, std::string ip_address, uint16_t port,
memgraph::storage::replication::ReplicationMode sync_mode, std::chrono::seconds replica_check_frequency,
std::optional<memgraph::storage::replication::ReplicationClientConfig::SSL> ssl) const {
return memgraph::storage::replication::ReplicaStatus{.name = name,
.ip_address = ip_address,
.port = port,
.sync_mode = sync_mode,
.replica_check_frequency = replica_check_frequency,
.ssl = ssl};
ReplicationStatus CreateReplicationStatus(std::string name, std::string ip_address, uint16_t port,
ReplicationMode sync_mode, std::chrono::seconds replica_check_frequency,
std::optional<ReplicationClientConfig::SSL> ssl,
std::optional<ReplicationRole> role) const {
return ReplicationStatus{.name = name,
.ip_address = ip_address,
.port = port,
.sync_mode = sync_mode,
.replica_check_frequency = replica_check_frequency,
.ssl = ssl,
.role = role};
}
static_assert(
sizeof(memgraph::storage::replication::ReplicaStatus) == 152,
"Most likely you modified ReplicaStatus without updating the tests. Please modify CreateReplicaStatus. ");
sizeof(ReplicationStatus) == 160,
"Most likely you modified ReplicationStatus without updating the tests. Please modify CreateReplicationStatus. ");
};
TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesInitialized) {
auto replicas_status = CreateReplicaStatus(
"name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC, std::chrono::seconds(1),
memgraph::storage::replication::ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"});
auto replicas_status = CreateReplicationStatus(
"name", "ip_address", 0, ReplicationMode::SYNC, std::chrono::seconds(1),
ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}, ReplicationRole::REPLICA);
auto json_status = memgraph::storage::replication::ReplicaStatusToJSON(
memgraph::storage::replication::ReplicaStatus(replicas_status));
auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status));
auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status));
auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status));
ASSERT_EQ(replicas_status, *replicas_status_converted);
}
TEST_F(ReplicationPersistanceHelperTest, BasicTestOnlyMandatoryAttributesInitialized) {
auto replicas_status =
CreateReplicaStatus("name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC,
std::chrono::seconds(1), std::nullopt);
auto replicas_status = CreateReplicationStatus("name", "ip_address", 0, ReplicationMode::SYNC,
std::chrono::seconds(1), std::nullopt, std::nullopt);
auto json_status = memgraph::storage::replication::ReplicaStatusToJSON(
memgraph::storage::replication::ReplicaStatus(replicas_status));
auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status));
auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status));
auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status));
ASSERT_EQ(replicas_status, *replicas_status_converted);
}
TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesButSSLInitialized) {
auto replicas_status =
CreateReplicaStatus("name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC,
std::chrono::seconds(1), std::nullopt);
auto replicas_status = CreateReplicationStatus("name", "ip_address", 0, ReplicationMode::SYNC,
std::chrono::seconds(1), std::nullopt, ReplicationRole::MAIN);
auto json_status = memgraph::storage::replication::ReplicaStatusToJSON(
memgraph::storage::replication::ReplicaStatus(replicas_status));
auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status));
auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status));
auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status));
ASSERT_EQ(replicas_status, *replicas_status_converted);
}
TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesButTimeoutInitialized) {
auto replicas_status = CreateReplicaStatus(
"name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC, std::chrono::seconds(1),
memgraph::storage::replication::ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"});
auto replicas_status = CreateReplicationStatus(
"name", "ip_address", 0, ReplicationMode::SYNC, std::chrono::seconds(1),
ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}, ReplicationRole::REPLICA);
auto json_status = memgraph::storage::replication::ReplicaStatusToJSON(
memgraph::storage::replication::ReplicaStatus(replicas_status));
auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status));
auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status));
auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status));
ASSERT_EQ(replicas_status, *replicas_status_converted);
}
TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesButReplicationRoleInitialized) {
// this one is importand for backwards compatibility
auto replicas_status = CreateReplicationStatus(
"name", "ip_address", 0, ReplicationMode::SYNC, std::chrono::seconds(1),
ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}, std::nullopt);
auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status));
auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status));
ASSERT_EQ(replicas_status, *replicas_status_converted);
}

12
tests/unit/storage_v2_replication.cpp
View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -652,9 +652,9 @@ TEST_F(ReplicationTest, ReplicationInformation) {
memgraph::storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID)
.HasError());
ASSERT_EQ(main_store.GetReplicationRole(), memgraph::storage::ReplicationRole::MAIN);
ASSERT_EQ(replica_store1.GetReplicationRole(), memgraph::storage::ReplicationRole::REPLICA);
ASSERT_EQ(replica_store2.GetReplicationRole(), memgraph::storage::ReplicationRole::REPLICA);
ASSERT_EQ(main_store.GetReplicationRole(), memgraph::storage::replication::ReplicationRole::MAIN);
ASSERT_EQ(replica_store1.GetReplicationRole(), memgraph::storage::replication::ReplicationRole::REPLICA);
ASSERT_EQ(replica_store2.GetReplicationRole(), memgraph::storage::replication::ReplicationRole::REPLICA);
const auto replicas_info = main_store.ReplicasInfo();
ASSERT_EQ(replicas_info.size(), 2);
@@ -730,7 +730,7 @@ TEST_F(ReplicationTest, ReplicationReplicaWithExistingEndPoint) {
TEST_F(ReplicationTest, RestoringReplicationAtStartupAftgerDroppingReplica) {
auto main_config = configuration;
main_config.durability.restore_replicas_on_startup = true;
main_config.durability.restore_replication_state_on_startup = true;
auto main_store = std::make_unique<memgraph::storage::Storage>(main_config);
memgraph::storage::Storage replica_store1(configuration);
@@ -773,7 +773,7 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartupAftgerDroppingReplica) {
TEST_F(ReplicationTest, RestoringReplicationAtStartup) {
auto main_config = configuration;
main_config.durability.restore_replicas_on_startup = true;
main_config.durability.restore_replication_state_on_startup = true;
auto main_store = std::make_unique<memgraph::storage::Storage>(main_config);
memgraph::storage::Storage replica_store1(configuration);
replica_store1.SetReplicaRole(memgraph::io::network::Endpoint{local_host, ports[0]});