Compare commits

...

31 Commits

Author SHA1 Message Date
jbajic
061d0f8aa6 Refactor update vertices 2022-05-16 10:32:41 +02:00
jbajic
9c53a9943b Add update vertices 2022-05-13 14:12:55 +02:00
jbajic
9685e1d3c4 Add edge updating property 2022-05-13 13:30:38 +02:00
jbajic
bdf794e229 Create edge 2022-05-12 16:22:35 +02:00
János Benjamin Antal
3dd90e2acb Update the storage interface 2022-05-02 09:09:31 +02:00
János Benjamin Antal
62945c3b3a Merge remote-tracking branch 'origin/master' into T0804-MG-base-thrift-project 2022-04-29 15:48:00 +02:00
János Benjamin Antal
e1704ff2d0 Revert "Use attribute instead of UNLIKELY"
This reverts commit 5780ee6c5b.
2022-04-29 15:46:49 +02:00
János Benjamin Antal
7f0c53196b Revert unvanted change 2022-04-29 15:19:23 +02:00
Jeremy B
b782271be8 Fix shared module loading
* Moving function add_query_module from CMakeLists from tests/e2e/magic_functions to tests/e2e

* Adding failing test copying behavior when loading c module (.so) into memgraph.

* Fixing issue where NO_ERROR status returned  MgpTransAddFixedResult was converted to false

* Removing unnecessary transformation

* removing incorrect parameterization of test

* re-adding parametrized transformation
2022-04-28 20:28:44 +02:00
Jure Bajic
a8ffcfa046 Update license year
* Update license year

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
2022-04-27 13:31:37 +02:00
Jure Bajic
7b78665cd8 Implement Bolt over WebSocket with asio
* Replace server implementation with asio

* Add support for bolt over WebSocket
2022-04-27 10:13:16 +02:00
János Benjamin Antal
2da28c2c87 Add meta service interface 2022-04-22 17:06:48 +02:00
János Benjamin Antal
b36cce2428 Add thrift demo 2022-03-31 09:05:52 +02:00
János Benjamin Antal
6afcfcbb89 Merge commit 'bf01c58ed9950a1b2d05a884aad9544e8193b7f3' into T0804-MG-base-thrift-project 2022-03-24 15:25:39 +01:00
János Benjamin Antal
d21798c350 Implement scanVertices 2022-03-24 08:51:37 +01:00
János Benjamin Antal
6fe2293f4a Merge remote-tracking branch 'origin/master' into T0804-MG-base-thrift-project 2022-02-28 15:34:53 +01:00
János Benjamin Antal
2f1acff7d7 Replace outdated wrapper header file 2022-02-28 13:27:44 +01:00
János Benjamin Antal
e298f11968 Add storage client demo 2022-02-10 09:07:33 +01:00
János Benjamin Antal
5a4c0f7a1b Resolve conflict of ANTLR generated function and macro from glog 2022-02-08 12:33:39 +01:00
János Benjamin Antal
52960a8877 Add base of server and client 2022-02-07 15:14:29 +01:00
János Benjamin Antal
4811024e75 Add createVertices to Thrift 2022-02-07 15:13:19 +01:00
János Benjamin Antal
9193fcade3 CMake cleanup 2022-02-07 15:12:23 +01:00
János Benjamin Antal
066cfa32e3 Fix typo 2022-02-07 15:12:13 +01:00
János Benjamin Antal
77c27482f7 Link generated thrift interface to memgraph 2022-01-28 15:17:45 +01:00
János Benjamin Antal
8f42632fd2 CMake cleanup 2022-01-28 15:14:58 +01:00
János Benjamin Antal
5780ee6c5b Use attribute instead of UNLIKELY 2022-01-28 13:42:21 +01:00
János Benjamin Antal
542928b690 Fix namespace 2022-01-28 10:17:33 +01:00
János Benjamin Antal
67d39597d7 Remove coroutines from folly 2022-01-28 10:16:11 +01:00
János Benjamin Antal
bd0efa1159 Use patch files 2022-01-28 10:15:48 +01:00
János Benjamin Antal
e60dd252d0 Use std::unordered_map 2022-01-27 08:21:03 +01:00
János Benjamin Antal
84b78ded07 Add base of thrift interface 2022-01-27 07:52:39 +01:00
50 changed files with 2732 additions and 79 deletions

View File

@@ -184,7 +184,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-Werror=switch -Werror=switch-bool -Werror=return-type \
-Werror=return-stack-address \
-Wno-c99-designator")
-Wno-c99-designator \
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO

297
cmake/FindSodium.cmake Normal file
View File

@@ -0,0 +1,297 @@
# Written in 2016 by Henrik Steffen Gaßmann <henrik@gassmann.onl>
#
# To the extent possible under law, the author(s) have dedicated all
# copyright and related and neighboring rights to this software to the
# public domain worldwide. This software is distributed without any warranty.
#
# You should have received a copy of the CC0 Public Domain Dedication
# along with this software. If not, see
#
# http://creativecommons.org/publicdomain/zero/1.0/
#
########################################################################
# Tries to find the local libsodium installation.
#
# On Windows the sodium_DIR environment variable is used as a default
# hint which can be overridden by setting the corresponding cmake variable.
#
# Once done the following variables will be defined:
#
# sodium_FOUND
# sodium_INCLUDE_DIR
# sodium_LIBRARY_DEBUG
# sodium_LIBRARY_RELEASE
#
#
# Furthermore an imported "sodium" target is created.
#
if (CMAKE_C_COMPILER_ID STREQUAL "GNU"
OR CMAKE_C_COMPILER_ID STREQUAL "Clang")
set(_GCC_COMPATIBLE 1)
endif()
# static library option
if (NOT DEFINED sodium_USE_STATIC_LIBS)
option(sodium_USE_STATIC_LIBS "enable to statically link against sodium" OFF)
endif()
if(NOT (sodium_USE_STATIC_LIBS EQUAL sodium_USE_STATIC_LIBS_LAST))
unset(sodium_LIBRARY CACHE)
unset(sodium_LIBRARY_DEBUG CACHE)
unset(sodium_LIBRARY_RELEASE CACHE)
unset(sodium_DLL_DEBUG CACHE)
unset(sodium_DLL_RELEASE CACHE)
set(sodium_USE_STATIC_LIBS_LAST ${sodium_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
endif()
########################################################################
# UNIX
if (UNIX)
# import pkg-config
find_package(PkgConfig QUIET)
if (PKG_CONFIG_FOUND)
pkg_check_modules(sodium_PKG QUIET libsodium)
endif()
if(sodium_USE_STATIC_LIBS)
foreach(_libname ${sodium_PKG_STATIC_LIBRARIES})
if (NOT _libname MATCHES "^lib.*\\.a$") # ignore strings already ending with .a
list(INSERT sodium_PKG_STATIC_LIBRARIES 0 "lib${_libname}.a")
endif()
endforeach()
list(REMOVE_DUPLICATES sodium_PKG_STATIC_LIBRARIES)
# if pkgconfig for libsodium doesn't provide
# static lib info, then override PKG_STATIC here..
if (NOT sodium_PKG_STATIC_FOUND)
set(sodium_PKG_STATIC_LIBRARIES libsodium.a)
endif()
set(XPREFIX sodium_PKG_STATIC)
else()
if (NOT sodium_PKG_FOUND)
set(sodium_PKG_LIBRARIES sodium)
endif()
set(XPREFIX sodium_PKG)
endif()
find_path(sodium_INCLUDE_DIR sodium.h
HINTS ${${XPREFIX}_INCLUDE_DIRS}
)
find_library(sodium_LIBRARY_DEBUG NAMES ${${XPREFIX}_LIBRARIES}
HINTS ${${XPREFIX}_LIBRARY_DIRS}
)
find_library(sodium_LIBRARY_RELEASE NAMES ${${XPREFIX}_LIBRARIES}
HINTS ${${XPREFIX}_LIBRARY_DIRS}
)
########################################################################
# Windows
elseif (WIN32)
set(sodium_DIR "$ENV{sodium_DIR}" CACHE FILEPATH "sodium install directory")
mark_as_advanced(sodium_DIR)
find_path(sodium_INCLUDE_DIR sodium.h
HINTS ${sodium_DIR}
PATH_SUFFIXES include
)
if (MSVC)
# detect target architecture
file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp" [=[
#if defined _M_IX86
#error ARCH_VALUE x86_32
#elif defined _M_X64
#error ARCH_VALUE x86_64
#endif
#error ARCH_VALUE unknown
]=])
try_compile(_UNUSED_VAR "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/arch.cpp"
OUTPUT_VARIABLE _COMPILATION_LOG
)
string(REGEX REPLACE ".*ARCH_VALUE ([a-zA-Z0-9_]+).*" "\\1" _TARGET_ARCH "${_COMPILATION_LOG}")
# construct library path
if (_TARGET_ARCH STREQUAL "x86_32")
string(APPEND _PLATFORM_PATH "Win32")
elseif(_TARGET_ARCH STREQUAL "x86_64")
string(APPEND _PLATFORM_PATH "x64")
else()
message(FATAL_ERROR "the ${_TARGET_ARCH} architecture is not supported by Findsodium.cmake.")
endif()
string(APPEND _PLATFORM_PATH "/$$CONFIG$$")
if (MSVC_VERSION LESS 1900)
math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 60")
else()
math(EXPR _VS_VERSION "${MSVC_VERSION} / 10 - 50")
endif()
string(APPEND _PLATFORM_PATH "/v${_VS_VERSION}")
if (sodium_USE_STATIC_LIBS)
string(APPEND _PLATFORM_PATH "/static")
else()
string(APPEND _PLATFORM_PATH "/dynamic")
endif()
string(REPLACE "$$CONFIG$$" "Debug" _DEBUG_PATH_SUFFIX "${_PLATFORM_PATH}")
string(REPLACE "$$CONFIG$$" "Release" _RELEASE_PATH_SUFFIX "${_PLATFORM_PATH}")
find_library(sodium_LIBRARY_DEBUG libsodium.lib
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX}
)
find_library(sodium_LIBRARY_RELEASE libsodium.lib
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX}
)
if (NOT sodium_USE_STATIC_LIBS)
set(CMAKE_FIND_LIBRARY_SUFFIXES_BCK ${CMAKE_FIND_LIBRARY_SUFFIXES})
set(CMAKE_FIND_LIBRARY_SUFFIXES ".dll")
find_library(sodium_DLL_DEBUG libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_DEBUG_PATH_SUFFIX}
)
find_library(sodium_DLL_RELEASE libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES ${_RELEASE_PATH_SUFFIX}
)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK})
endif()
elseif(_GCC_COMPATIBLE)
if (sodium_USE_STATIC_LIBS)
find_library(sodium_LIBRARY_DEBUG libsodium.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
find_library(sodium_LIBRARY_RELEASE libsodium.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
else()
find_library(sodium_LIBRARY_DEBUG libsodium.dll.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
find_library(sodium_LIBRARY_RELEASE libsodium.dll.a
HINTS ${sodium_DIR}
PATH_SUFFIXES lib
)
file(GLOB _DLL
LIST_DIRECTORIES false
RELATIVE "${sodium_DIR}/bin"
"${sodium_DIR}/bin/libsodium*.dll"
)
find_library(sodium_DLL_DEBUG ${_DLL} libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES bin
)
find_library(sodium_DLL_RELEASE ${_DLL} libsodium
HINTS ${sodium_DIR}
PATH_SUFFIXES bin
)
endif()
else()
message(FATAL_ERROR "this platform is not supported by FindSodium.cmake")
endif()
########################################################################
# unsupported
else()
message(FATAL_ERROR "this platform is not supported by FindSodium.cmake")
endif()
########################################################################
# common stuff
# extract sodium version
if (sodium_INCLUDE_DIR)
set(_VERSION_HEADER "${_INCLUDE_DIR}/sodium/version.h")
if (EXISTS _VERSION_HEADER)
file(READ "${_VERSION_HEADER}" _VERSION_HEADER_CONTENT)
string(REGEX REPLACE ".*#[ \t]*define[ \t]*SODIUM_VERSION_STRING[ \t]*\"([^\n]*)\".*" "\\1"
sodium_VERSION "${_VERSION_HEADER_CONTENT}")
set(sodium_VERSION "${sodium_VERSION}" PARENT_SCOPE)
endif()
endif()
# communicate results
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(
Sodium # The name must be either uppercase or match the filename case.
REQUIRED_VARS
sodium_LIBRARY_RELEASE
sodium_LIBRARY_DEBUG
sodium_INCLUDE_DIR
VERSION_VAR
sodium_VERSION
)
if(Sodium_FOUND)
set(sodium_LIBRARIES
optimized ${sodium_LIBRARY_RELEASE} debug ${sodium_LIBRARY_DEBUG})
endif()
# mark file paths as advanced
mark_as_advanced(sodium_INCLUDE_DIR)
mark_as_advanced(sodium_LIBRARY_DEBUG)
mark_as_advanced(sodium_LIBRARY_RELEASE)
if (WIN32)
mark_as_advanced(sodium_DLL_DEBUG)
mark_as_advanced(sodium_DLL_RELEASE)
endif()
# create imported target
if(sodium_USE_STATIC_LIBS)
set(_LIB_TYPE STATIC)
else()
set(_LIB_TYPE SHARED)
endif()
if(NOT TARGET sodium)
add_library(sodium ${_LIB_TYPE} IMPORTED)
endif()
set_target_properties(sodium PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${sodium_INCLUDE_DIR}"
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
)
if (sodium_USE_STATIC_LIBS)
set_target_properties(sodium PROPERTIES
INTERFACE_COMPILE_DEFINITIONS "SODIUM_STATIC"
IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}"
IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}"
)
else()
if (UNIX)
set_target_properties(sodium PROPERTIES
IMPORTED_LOCATION "${sodium_LIBRARY_RELEASE}"
IMPORTED_LOCATION_DEBUG "${sodium_LIBRARY_DEBUG}"
)
elseif (WIN32)
set_target_properties(sodium PROPERTIES
IMPORTED_IMPLIB "${sodium_LIBRARY_RELEASE}"
IMPORTED_IMPLIB_DEBUG "${sodium_LIBRARY_DEBUG}"
)
if (NOT (sodium_DLL_DEBUG MATCHES ".*-NOTFOUND"))
set_target_properties(sodium PROPERTIES
IMPORTED_LOCATION_DEBUG "${sodium_DLL_DEBUG}"
)
endif()
if (NOT (sodium_DLL_RELEASE MATCHES ".*-NOTFOUND"))
set_target_properties(sodium PROPERTIES
IMPORTED_LOCATION_RELWITHDEBINFO "${sodium_DLL_RELEASE}"
IMPORTED_LOCATION_MINSIZEREL "${sodium_DLL_RELEASE}"
IMPORTED_LOCATION_RELEASE "${sodium_DLL_RELEASE}"
)
endif()
endif()
endif()

169
cmake/MgThrift.cmake Normal file
View File

@@ -0,0 +1,169 @@
find_package(Gflags)
find_package(Folly)
find_package(wangle)
find_package(fizz)
find_package(proxygen)
find_package(FBThrift)
set(THRIFTCPP2 "FBThrift::thriftcpp2")
set(THRIFT1 ${FBTHRIFT_COMPILER})
include(${FBTHRIFT_INCLUDE_DIR}/thrift/ThriftLibrary.cmake)
set(MG_INTERFACE_TARGET_NAME_PREFIX "mg-interface")
function(_mg_thrift_generate
file_name
services
language
options
file_path
output_path
include_prefix
)
cmake_parse_arguments(THRIFT_GENERATE # Prefix
"" # Options
"" # One Value args
"THRIFT_INCLUDE_DIRECTORIES" # Multi-value args
"${ARGN}")
set(thrift_include_directories)
foreach(dir ${THRIFT_GENERATE_THRIFT_INCLUDE_DIRECTORIES})
list(APPEND thrift_include_directories "-I" "${dir}")
endforeach()
set("${file_name}-${language}-HEADERS"
${output_path}/gen-${language}/${file_name}_constants.h
${output_path}/gen-${language}/${file_name}_data.h
${output_path}/gen-${language}/${file_name}_metadata.h
${output_path}/gen-${language}/${file_name}_types.h
${output_path}/gen-${language}/${file_name}_types.tcc
)
set("${file_name}-${language}-SOURCES"
${output_path}/gen-${language}/${file_name}_constants.cpp
${output_path}/gen-${language}/${file_name}_data.cpp
${output_path}/gen-${language}/${file_name}_types.cpp
)
if(NOT "${options}" MATCHES "no_metadata")
set("${file_name}-${language}-SOURCES"
${${file_name}-${language}-SOURCES}
${output_path}/gen-${language}/${file_name}_metadata.cpp
)
endif()
foreach(service ${services})
set("${file_name}-${language}-HEADERS"
${${file_name}-${language}-HEADERS}
${output_path}/gen-${language}/${service}.h
${output_path}/gen-${language}/${service}.tcc
${output_path}/gen-${language}/${service}AsyncClient.h
${output_path}/gen-${language}/${service}_custom_protocol.h
)
set("${file_name}-${language}-SOURCES"
${${file_name}-${language}-SOURCES}
${output_path}/gen-${language}/${service}.cpp
${output_path}/gen-${language}/${service}AsyncClient.cpp
)
endforeach()
if("${include_prefix}" STREQUAL "")
set(include_prefix_text "")
else()
set(include_prefix_text "include_prefix=${include_prefix}")
if(NOT "${options}" STREQUAL "")
set(include_prefix_text ",${include_prefix_text}")
endif()
endif()
set(gen_language ${language})
if("${language}" STREQUAL "cpp2")
set(gen_language "mstch_cpp2")
elseif("${language}" STREQUAL "py3")
set(gen_language "mstch_py3")
file(WRITE "${output_path}/gen-${language}/${file_name}/__init__.py")
endif()
add_custom_command(
OUTPUT ${${file_name}-${language}-HEADERS}
${${file_name}-${language}-SOURCES}
COMMAND ${THRIFT1}
--gen "${gen_language}:${options}${include_prefix_text}"
-o ${output_path}
${thrift_include_directories}
"${file_path}/${file_name}.thrift"
DEPENDS
${THRIFT1}
"${file_path}/${file_name}.thrift"
COMMENT "Generating ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language} files. Output: ${output_path}"
)
add_custom_target(
${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-target ALL
DEPENDS ${${language}-${language}-HEADERS}
${${file_name}-${language}-SOURCES}
)
set("${file_name}-${language}-SOURCES" ${${file_name}-${language}-SOURCES} PARENT_SCOPE)
install(
DIRECTORY gen-${language}
DESTINATION include/${include_prefix}
FILES_MATCHING PATTERN "*.h")
install(
DIRECTORY gen-${language}
DESTINATION include/${include_prefix}
FILES_MATCHING PATTERN "*.tcc")
endfunction()
function(_mg_thrift_object
file_name
services
language
options
file_path
output_path
include_prefix
)
_mg_thrift_generate(
"${file_name}"
"${services}"
"${language}"
"${options}"
"${file_path}"
"${output_path}"
"${include_prefix}"
"${ARGN}"
)
bypass_source_check(${${file_name}-${language}-SOURCES})
add_library(
"${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj"
OBJECT
${${file_name}-${language}-SOURCES}
)
add_dependencies(
"${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj"
"${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-target"
)
target_include_directories(${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj PUBLIC ${output_path})
message(STATUS "MgThrift will create the Object file : ${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-${language}-obj")
endfunction()
function(mg_thrift_library
file_name
services
file_path
output_path
include_prefix
)
_mg_thrift_object(
"${file_name}"
"${services}"
"cpp2"
"stack_arguments" # options
"${file_path}"
"${output_path}"
"${include_prefix}"
THRIFT_INCLUDE_DIRECTORIES "${FBTHRIFT_INCLUDE_DIR}"
)
set(LIBRARY_NAME "${MG_INTERFACE_TARGET_NAME_PREFIX}-${file_name}-cpp2")
add_library(
"${LIBRARY_NAME}"
$<TARGET_OBJECTS:${LIBRARY_NAME}-obj>
)
target_link_libraries("${LIBRARY_NAME}" ${THRIFTCPP2})
message("MgThrift will create the library file : ${LIBRARY_NAME}")
endfunction()

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2026-18-02
CHANGE DATE: 2026-27-04
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -1,6 +1,7 @@
# CMake configuration for the main memgraph library and executable
# add memgraph sub libraries, ordered by dependency
add_subdirectory(interface)
add_subdirectory(lisp)
add_subdirectory(utils)
add_subdirectory(requests)
@@ -37,7 +38,7 @@ set(mg_single_node_v2_sources
)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-interface-storage-cpp2)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -78,14 +78,18 @@ bool ClientContext::use_ssl() { return use_ssl_; }
ServerContext::ServerContext(const std::string &key_file, const std::string &cert_file, const std::string &ca_file,
bool verify_peer) {
ctx_.emplace(boost::asio::ssl::context::tls_server);
namespace ssl = boost::asio::ssl;
ctx_.emplace(ssl::context::tls_server);
// NOLINTNEXTLINE(hicpp-signed-bitwise)
ctx_->set_options(ssl::context::default_workarounds | ssl::context::no_sslv2 | ssl::context::no_sslv3 |
ssl::context::single_dh_use);
ctx_->set_default_verify_paths();
// TODO: add support for encrypted private keys
// TODO: add certificate revocation list (CRL)
boost::system::error_code ec;
ctx_->use_certificate_chain_file(cert_file, ec);
MG_ASSERT(!ec, "Couldn't load server certificate from file: {}", cert_file);
ctx_->use_private_key_file(key_file, boost::asio::ssl::context::pem, ec);
ctx_->use_private_key_file(key_file, ssl::context::pem, ec);
MG_ASSERT(!ec, "Couldn't load server private key from file: {}", key_file);
ctx_->set_options(SSL_OP_NO_SSLv3, ec);
@@ -100,7 +104,7 @@ ServerContext::ServerContext(const std::string &key_file, const std::string &cer
if (verify_peer) {
// Enable verification of the client certificate.
// NOLINTNEXTLINE(hicpp-signed-bitwise)
ctx_->set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert, ec);
ctx_->set_verify_mode(ssl::verify_peer | ssl::verify_fail_if_no_peer_cert, ec);
MG_ASSERT(!ec, "Setting SSL verification mode failed!");
}
}

View File

@@ -0,0 +1,135 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <string_view>
#include <thread>
#include <vector>
#include <spdlog/spdlog.h>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core.hpp>
#include <boost/system/detail/error_code.hpp>
#include "communication/context.hpp"
#include "communication/v2/pool.hpp"
#include "communication/v2/session.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
namespace memgraph::communication::v2 {
template <class TSession, class TSessionData>
class Listener final : public std::enable_shared_from_this<Listener<TSession, TSessionData>> {
using tcp = boost::asio::ip::tcp;
using SessionHandler = Session<TSession, TSessionData>;
using std::enable_shared_from_this<Listener<TSession, TSessionData>>::shared_from_this;
public:
Listener(const Listener &) = delete;
Listener(Listener &&) = delete;
Listener &operator=(const Listener &) = delete;
Listener &operator=(Listener &&) = delete;
~Listener() {}
template <typename... Args>
static std::shared_ptr<Listener> Create(Args &&...args) {
return std::shared_ptr<Listener>{new Listener(std::forward<Args>(args)...)};
}
void Start() { DoAccept(); }
bool IsRunning() const noexcept { return alive_.load(std::memory_order_relaxed); }
private:
Listener(boost::asio::io_context &io_context, TSessionData *data, ServerContext *server_context,
tcp::endpoint &endpoint, const std::string_view service_name, const uint64_t inactivity_timeout_sec)
: io_context_(io_context),
data_(data),
server_context_(server_context),
acceptor_(io_context_),
endpoint_{endpoint},
service_name_{service_name},
inactivity_timeout_{inactivity_timeout_sec} {
boost::system::error_code ec;
// Open the acceptor
acceptor_.open(endpoint.protocol(), ec);
if (ec) {
OnError(ec, "open");
return;
}
// Allow address reuse
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
if (ec) {
OnError(ec, "set_option");
return;
}
// Bind to the server address
acceptor_.bind(endpoint, ec);
if (ec) {
spdlog::error(
utils::MessageWithLink("Cannot bind to socket on endpoint {}.", endpoint, "https://memgr.ph/socket"));
OnError(ec, "bind");
return;
}
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
if (ec) {
OnError(ec, "listen");
return;
}
}
void DoAccept() {
acceptor_.async_accept(io_context_,
[shared_this = shared_from_this()](auto ec, boost::asio::ip::tcp::socket &&socket) {
shared_this->OnAccept(ec, std::move(socket));
});
}
void OnAccept(boost::system::error_code ec, tcp::socket socket) {
if (ec) {
return OnError(ec, "accept");
}
auto session = SessionHandler::Create(std::move(socket), data_, *server_context_, endpoint_, inactivity_timeout_,
service_name_);
session->Start();
DoAccept();
}
void OnError(const boost::system::error_code &ec, const std::string_view what) {
spdlog::error("Listener failed on {}: {}", what, ec.message());
alive_.store(false, std::memory_order_relaxed);
}
boost::asio::io_context &io_context_;
TSessionData *data_;
ServerContext *server_context_;
tcp::acceptor acceptor_;
tcp::endpoint endpoint_;
std::string_view service_name_;
std::chrono::seconds inactivity_timeout_;
std::atomic<bool> alive_;
};
} // namespace memgraph::communication::v2

View File

@@ -0,0 +1,68 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstddef>
#include <thread>
#include <vector>
#include <boost/asio/executor_work_guard.hpp>
#include <boost/asio/io_context.hpp>
#include "utils/logging.hpp"
namespace memgraph::communication::v2 {
class IOContextThreadPool final {
private:
using IOContext = boost::asio::io_context;
using IOContextGuard = boost::asio::executor_work_guard<boost::asio::io_context::executor_type>;
public:
explicit IOContextThreadPool(size_t pool_size) : guard_{io_context_.get_executor()}, pool_size_{pool_size} {
MG_ASSERT(pool_size != 0, "Pool size must be greater then 0!");
}
IOContextThreadPool(const IOContextThreadPool &) = delete;
IOContextThreadPool &operator=(const IOContextThreadPool &) = delete;
IOContextThreadPool(IOContextThreadPool &&) = delete;
IOContextThreadPool &operator=(IOContextThreadPool &&) = delete;
~IOContextThreadPool() = default;
void Run() {
background_threads_.reserve(pool_size_);
for (size_t i = 0; i < pool_size_; ++i) {
background_threads_.emplace_back([this]() { io_context_.run(); });
}
running_ = true;
}
void Shutdown() {
io_context_.stop();
running_ = false;
}
void AwaitShutdown() { background_threads_.clear(); }
bool IsRunning() const noexcept { return running_; }
IOContext &GetIOContext() noexcept { return io_context_; }
private:
/// The pool of io_context.
IOContext io_context_;
IOContextGuard guard_;
size_t pool_size_;
std::vector<std::jthread> background_threads_;
bool running_{false};
};
} // namespace memgraph::communication::v2

View File

@@ -0,0 +1,128 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <algorithm>
#include <cstdint>
#include <iostream>
#include <memory>
#include <optional>
#include <string>
#include <thread>
#include <vector>
#include <fmt/format.h>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/address.hpp>
#include <boost/asio/ip/tcp.hpp>
#include "communication/context.hpp"
#include "communication/init.hpp"
#include "communication/v2/listener.hpp"
#include "communication/v2/pool.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
#include "utils/thread.hpp"
namespace memgraph::communication::v2 {
using Socket = boost::asio::ip::tcp::socket;
using ServerEndpoint = boost::asio::ip::tcp::endpoint;
/**
* Communication server.
*
* Listens for incoming connections on the server port and assigns them to the
* connection listener. The listener and session are implemented using asio
* async model. Currently the implemented model is thread per core model
* opposed to io_context per core. The reasoning for opting for the former model
* is the robustness to the multiple resource demanding queries that can be split
* across multiple threads, and then a single thread would not block io_context,
* unlike in the latter model where it is possible that thread that accepts
* request is being blocked by demanding query.
* All logic is contained within handlers that are being dispatched
* on a single strand per session. The only exception is write which is
* synchronous since the nature of the clients conenction is synchronous as
* well.
*
* Current Server architecture:
* incoming connection -> server -> listener -> session
*
* @tparam TSession the server can handle different Sessions, each session
* represents a different protocol so the same network infrastructure
* can be used for handling different protocols
* @tparam TSessionData the class with objects that will be forwarded to the
* session
*/
template <typename TSession, typename TSessionData>
class Server final {
using ServerHandler = Server<TSession, TSessionData>;
public:
/**
* Constructs and binds server to endpoint, operates on session data and
* invokes workers_count workers
*/
Server(ServerEndpoint &endpoint, TSessionData *session_data, ServerContext *server_context,
const int inactivity_timeout_sec, const std::string_view service_name,
size_t workers_count = std::thread::hardware_concurrency())
: endpoint_{endpoint},
service_name_{service_name},
context_thread_pool_{workers_count},
listener_{Listener<TSession, TSessionData>::Create(context_thread_pool_.GetIOContext(), session_data,
server_context, endpoint_, service_name_,
inactivity_timeout_sec)} {}
~Server() { MG_ASSERT(!IsRunning(), "Server wasn't shutdown properly"); }
Server(const Server &) = delete;
Server(Server &&) = delete;
Server &operator=(const Server &) = delete;
Server &operator=(Server &&) = delete;
const auto &Endpoint() const {
MG_ASSERT(IsRunning(), "You can't get the server endpoint when it's not running!");
return endpoint_;
}
bool Start() {
if (IsRunning()) {
spdlog::error("The server is already running");
return false;
}
listener_->Start();
spdlog::info("{} server is fully armed and operational", service_name_);
spdlog::info("{} listening on {}", service_name_, endpoint_.address());
context_thread_pool_.Run();
return true;
}
void Shutdown() {
context_thread_pool_.Shutdown();
spdlog::info("{} shutting down...", service_name_);
}
void AwaitShutdown() { context_thread_pool_.AwaitShutdown(); }
bool IsRunning() const noexcept { return context_thread_pool_.IsRunning() && listener_->IsRunning(); }
private:
ServerEndpoint endpoint_;
std::string service_name_;
IOContextThreadPool context_thread_pool_;
std::shared_ptr<Listener<TSession, TSessionData>> listener_;
};
} // namespace memgraph::communication::v2

View File

@@ -0,0 +1,513 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <deque>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <spdlog/spdlog.h>
#include <boost/asio/bind_executor.hpp>
#include <boost/asio/buffer.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/read.hpp>
#include <boost/asio/socket_base.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/asio/ssl/stream_base.hpp>
#include <boost/asio/steady_timer.hpp>
#include <boost/asio/strand.hpp>
#include <boost/asio/system_context.hpp>
#include <boost/asio/write.hpp>
#include <boost/beast/core/tcp_stream.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/websocket.hpp>
#include <boost/beast/websocket/rfc6455.hpp>
#include <boost/system/detail/error_code.hpp>
#include "communication/context.hpp"
#include "communication/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::communication::v2 {
/**
* This is used to provide input to user Sessions. All Sessions used with the
* network stack should use this class as their input stream.
*/
using InputStream = communication::Buffer::ReadEnd;
using tcp = boost::asio::ip::tcp;
/**
* This is used to provide output from user Sessions. All Sessions used with the
* network stack should use this class for their output stream.
*/
class OutputStream final {
public:
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
: write_function_(write_function) {}
OutputStream(const OutputStream &) = delete;
OutputStream(OutputStream &&) = delete;
OutputStream &operator=(const OutputStream &) = delete;
OutputStream &operator=(OutputStream &&) = delete;
~OutputStream() = default;
bool Write(const uint8_t *data, size_t len, bool have_more = false) { return write_function_(data, len, have_more); }
bool Write(const std::string &str, bool have_more = false) {
return Write(reinterpret_cast<const uint8_t *>(str.data()), str.size(), have_more);
}
private:
std::function<bool(const uint8_t *, size_t, bool)> write_function_;
};
/**
* This class is used internally in the communication stack to handle all user
* Websocket Sessions. It handles socket ownership, inactivity timeout and protocol
* wrapping.
*/
template <typename TSession, typename TSessionData>
class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TSession, TSessionData>> {
using WebSocket = boost::beast::websocket::stream<boost::beast::tcp_stream>;
using std::enable_shared_from_this<WebsocketSession<TSession, TSessionData>>::shared_from_this;
public:
template <typename... Args>
static std::shared_ptr<WebsocketSession> Create(Args &&...args) {
return std::shared_ptr<WebsocketSession>(new WebsocketSession(std::forward<Args>(args)...));
}
// Start the asynchronous accept operation
template <class Body, class Allocator>
void DoAccept(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> req) {
execution_active_ = true;
// Set suggested timeout settings for the websocket
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
boost::asio::socket_base::keep_alive option(true);
// Set a decorator to change the Server of the handshake
ws_.set_option(boost::beast::websocket::stream_base::decorator([](boost::beast::websocket::response_type &res) {
res.set(boost::beast::http::field::server, std::string("Memgraph Bolt WS"));
res.set(boost::beast::http::field::sec_websocket_protocol, "binary");
}));
ws_.binary(true);
// Accept the websocket handshake
ws_.async_accept(
req, boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnAccept, shared_from_this())));
}
bool Write(const uint8_t *data, size_t len) {
if (!IsConnected()) {
return false;
}
boost::system::error_code ec;
ws_.write(boost::asio::buffer(data, len), ec);
if (ec) {
OnError(ec, "write");
return false;
}
return true;
}
private:
// Take ownership of the socket
explicit WebsocketSession(tcp::socket &&socket, TSessionData *data, tcp::endpoint endpoint,
std::string_view service_name)
: ws_(std::move(socket)),
strand_{boost::asio::make_strand(ws_.get_executor())},
output_stream_([this](const uint8_t *data, size_t len, bool /*have_more*/) { return Write(data, len); }),
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
endpoint_{endpoint},
remote_endpoint_{ws_.next_layer().socket().remote_endpoint()},
service_name_{service_name} {}
void OnAccept(boost::beast::error_code ec) {
if (ec) {
return OnError(ec, "accept");
}
// Read a message
DoRead();
}
void DoRead() {
if (!IsConnected()) {
return;
}
// Read a message into our buffer
auto buffer = input_buffer_.write_end()->Allocate();
ws_.async_read_some(
boost::asio::buffer(buffer.data, buffer.len),
boost::asio::bind_executor(strand_, std::bind_front(&WebsocketSession::OnRead, shared_from_this())));
}
void OnRead(const boost::system::error_code &ec, [[maybe_unused]] const size_t bytes_transferred) {
// This indicates that the WebsocketSession was closed
if (ec == boost::beast::websocket::error::closed) {
return;
}
if (ec) {
OnError(ec, "read");
}
input_buffer_.write_end()->Written(bytes_transferred);
try {
session_.Execute();
DoRead();
} catch (const SessionClosedException &e) {
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
remote_endpoint_.port());
DoClose();
} catch (const std::exception &e) {
spdlog::error(
"Exception was thrown while processing event in {} session "
"associated with {}:{}",
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
spdlog::debug("Exception message: {}", e.what());
DoClose();
}
}
void OnError(const boost::system::error_code &ec, const std::string_view action) {
spdlog::error("Websocket Bolt session error: {} on {}", ec.message(), action);
DoClose();
}
void DoClose() {
ws_.async_close(
boost::beast::websocket::close_code::normal,
boost::asio::bind_executor(
strand_, [shared_this = shared_from_this()](boost::beast::error_code ec) { shared_this->OnClose(ec); }));
}
void OnClose(const boost::system::error_code &ec) {
if (!IsConnected()) {
return;
}
if (ec) {
return OnError(ec, "close");
}
}
bool IsConnected() const { return ws_.is_open() && execution_active_; }
WebSocket ws_;
boost::asio::strand<WebSocket::executor_type> strand_;
communication::Buffer input_buffer_;
OutputStream output_stream_;
TSession session_;
tcp::endpoint endpoint_;
tcp::endpoint remote_endpoint_;
std::string_view service_name_;
bool execution_active_{false};
};
/**
* This class is used internally in the communication stack to handle all user
* Sessions. It handles socket ownership, inactivity timeout and protocol
* wrapping.
*/
template <typename TSession, typename TSessionData>
class Session final : public std::enable_shared_from_this<Session<TSession, TSessionData>> {
using TCPSocket = tcp::socket;
using SSLSocket = boost::asio::ssl::stream<TCPSocket>;
using std::enable_shared_from_this<Session<TSession, TSessionData>>::shared_from_this;
public:
template <typename... Args>
static std::shared_ptr<Session> Create(Args &&...args) {
return std::shared_ptr<Session>(new Session(std::forward<Args>(args)...));
}
Session(const Session &) = delete;
Session(Session &&) = delete;
Session &operator=(const Session &) = delete;
Session &operator=(Session &&) = delete;
~Session() {
if (IsConnected()) {
spdlog::error("Session: Destructor called while execution is active");
}
}
bool Start() {
if (execution_active_) {
return false;
}
execution_active_ = true;
timeout_timer_.async_wait(boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
if (std::holds_alternative<SSLSocket>(socket_)) {
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoHandshake(); });
} else {
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
}
return true;
}
bool Write(const uint8_t *data, size_t len, bool have_more = false) {
if (!IsConnected()) {
return false;
}
return std::visit(
utils::Overloaded{[shared_this = shared_from_this(), data, len, have_more](TCPSocket &socket) mutable {
boost::system::error_code ec;
while (len > 0) {
const auto sent = socket.send(boost::asio::buffer(data, len),
MSG_NOSIGNAL | (have_more ? MSG_MORE : 0), ec);
if (ec) {
shared_this->OnError(ec);
return false;
}
data += sent;
len -= sent;
}
return true;
},
[shared_this = shared_from_this(), data, len](SSLSocket &socket) mutable {
boost::system::error_code ec;
while (len > 0) {
const auto sent = socket.write_some(boost::asio::buffer(data, len), ec);
if (ec) {
shared_this->OnError(ec);
return false;
}
data += sent;
len -= sent;
}
return true;
}},
socket_);
}
bool IsConnected() const {
return std::visit([this](const auto &socket) { return execution_active_ && socket.lowest_layer().is_open(); },
socket_);
}
private:
explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &server_context, tcp::endpoint endpoint,
const std::chrono::seconds inactivity_timeout_sec, std::string_view service_name)
: socket_(CreateSocket(std::move(socket), server_context)),
strand_{boost::asio::make_strand(GetExecutor())},
output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
data_{data},
endpoint_{endpoint},
remote_endpoint_{GetRemoteEndpoint()},
service_name_{service_name},
timeout_seconds_(inactivity_timeout_sec),
timeout_timer_(GetExecutor()) {
ExecuteForSocket([](auto &&socket) {
socket.lowest_layer().set_option(tcp::no_delay(true)); // enable PSH
socket.lowest_layer().set_option(boost::asio::socket_base::keep_alive(true)); // enable SO_KEEPALIVE
socket.lowest_layer().non_blocking(false);
});
timeout_timer_.expires_at(boost::asio::steady_timer::time_point::max());
spdlog::info("Accepted a connection from {}:", service_name_, remote_endpoint_.address(), remote_endpoint_.port());
}
void DoRead() {
if (!IsConnected()) {
return;
}
timeout_timer_.expires_after(timeout_seconds_);
ExecuteForSocket([this](auto &&socket) {
auto buffer = input_buffer_.write_end()->Allocate();
socket.async_read_some(
boost::asio::buffer(buffer.data, buffer.len),
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnRead, shared_from_this())));
});
}
bool IsWebsocketUpgrade(boost::beast::http::request_parser<boost::beast::http::string_body> &parser) {
boost::system::error_code error_code_parsing;
parser.put(boost::asio::buffer(input_buffer_.read_end()->data(), input_buffer_.read_end()->size()),
error_code_parsing);
if (error_code_parsing) {
return false;
}
return boost::beast::websocket::is_upgrade(parser.get());
}
void OnRead(const boost::system::error_code &ec, const size_t bytes_transferred) {
if (ec) {
return OnError(ec);
}
input_buffer_.write_end()->Written(bytes_transferred);
// Can be a websocket connection only on the first read, since it is not
// expected from clients to upgrade from tcp to websocket
if (!has_received_msg_) {
has_received_msg_ = true;
boost::beast::http::request_parser<boost::beast::http::string_body> parser;
if (IsWebsocketUpgrade(parser)) {
spdlog::info("Switching {} to websocket connection", remote_endpoint_);
if (std::holds_alternative<TCPSocket>(socket_)) {
auto sock = std::get<TCPSocket>(std::move(socket_));
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), data_, endpoint_, service_name_)
->DoAccept(parser.release());
execution_active_ = false;
return;
}
spdlog::error("Error while upgrading connection to websocket");
DoShutdown();
}
}
try {
session_.Execute();
DoRead();
} catch (const SessionClosedException &e) {
spdlog::info("{} client {}:{} closed the connection.", service_name_, remote_endpoint_.address(),
remote_endpoint_.port());
DoShutdown();
} catch (const std::exception &e) {
spdlog::error(
"Exception was thrown while processing event in {} session "
"associated with {}:{}",
service_name_, remote_endpoint_.address(), remote_endpoint_.port());
spdlog::debug("Exception message: {}", e.what());
DoShutdown();
}
}
void OnError(const boost::system::error_code &ec) {
if (ec == boost::asio::error::operation_aborted) {
return;
}
execution_active_ = false;
if (ec == boost::asio::error::eof) {
spdlog::info("Session closed by peer");
} else {
spdlog::error("Session error: {}", ec.message());
}
DoShutdown();
}
void DoShutdown() {
if (!IsConnected()) {
return;
}
execution_active_ = false;
timeout_timer_.cancel();
ExecuteForSocket([](auto &socket) {
boost::system::error_code ec;
auto &lowest_layer = socket.lowest_layer();
lowest_layer.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec);
if (ec) {
spdlog::error("Session shutdown failed: {}", ec.what());
}
lowest_layer.close();
});
}
void DoHandshake() {
if (!IsConnected()) {
return;
}
if (auto *socket = std::get_if<SSLSocket>(&socket_); socket) {
socket->async_handshake(
boost::asio::ssl::stream_base::server,
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnHandshake, shared_from_this())));
}
}
void OnHandshake(const boost::system::error_code &ec) {
if (ec) {
return OnError(ec);
}
DoRead();
}
void OnClose(const boost::system::error_code &ec) {
if (ec) {
return OnError(ec);
}
}
void OnTimeout() {
if (!IsConnected()) {
return;
}
// Check whether the deadline has passed. We compare the deadline against
// the current time since a new asynchronous operation may have moved the
// deadline before this actor had a chance to run.
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
// The deadline has passed. Stop the session. The other actors will
// terminate as soon as possible.
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
DoShutdown();
} else {
// Put the actor back to sleep.
timeout_timer_.async_wait(
boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
}
}
std::variant<TCPSocket, SSLSocket> CreateSocket(tcp::socket &&socket, ServerContext &context) {
if (context.use_ssl()) {
ssl_context_.emplace(context.context_clone());
return SSLSocket{std::move(socket), *ssl_context_};
}
return TCPSocket{std::move(socket)};
}
auto GetExecutor() {
return std::visit(utils::Overloaded{[](auto &&socket) { return socket.get_executor(); }}, socket_);
}
auto GetRemoteEndpoint() const {
return std::visit(utils::Overloaded{[](const auto &socket) { return socket.lowest_layer().remote_endpoint(); }},
socket_);
}
template <typename F>
decltype(auto) ExecuteForSocket(F &&fun) {
return std::visit(utils::Overloaded{std::forward<F>(fun)}, socket_);
}
std::variant<TCPSocket, SSLSocket> socket_;
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ssl_context_;
boost::asio::strand<tcp::socket::executor_type> strand_;
communication::Buffer input_buffer_;
OutputStream output_stream_;
TSession session_;
TSessionData *data_;
tcp::endpoint endpoint_;
tcp::endpoint remote_endpoint_;
std::string_view service_name_;
std::chrono::seconds timeout_seconds_;
boost::asio::steady_timer timeout_timer_;
bool execution_active_{false};
bool has_received_msg_{false};
};
} // namespace memgraph::communication::v2

View File

@@ -11,8 +11,6 @@
#pragma once
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
#include <list>
#include <memory>

View File

@@ -11,8 +11,6 @@
#pragma once
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
#include <thread>
#include <spdlog/sinks/base_sink.h>

View File

@@ -11,8 +11,6 @@
#pragma once
#define BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
#include <deque>
#include <memory>
#include <optional>

1
src/interface/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
gen-cpp2

View File

@@ -0,0 +1,19 @@
include(MgThrift)
set(MG_INTERFACE_PATH ${CMAKE_CURRENT_SOURCE_DIR})
mg_thrift_library(
"storage" #file_name
"Storage" #services
"${MG_INTERFACE_PATH}" #file_path
"${MG_INTERFACE_PATH}" #output_path
"interface" #include_prefix
)
mg_thrift_library(
"meta" #file_name
"Meta" #services
"${MG_INTERFACE_PATH}" #file_path
"${MG_INTERFACE_PATH}" #output_path
"interface" #include_prefix
)

74
src/interface/meta.thrift Normal file
View File

@@ -0,0 +1,74 @@
namespace cpp2 interface.meta
typedef i64 LabelId
typedef i64 IndexId
struct Result {
1: bool success;
}
struct CreatePrimaryLabelRequest {
1: binary name;
2: list<binary> primary_keys;
}
struct CreateLabelResponse {
1: Result result;
2: LabelId label_id;
}
struct GetLabelInfosRequest {
// The empty list means return all of the label infos
1: list<binary> label_names;
}
struct LabelInfo {
1: binary name;
2: LabelId label_id;
3: list<binary> primary_keys;
}
struct GetLabelInfoResponse {
1: Result result;
2: LabelInfo label_info;
}
struct GetLabelInfosResponse {
1: Result result;
2: list<LabelInfo> label_infos;
}
struct CreateIndexRequest {
1: LabelId label_id;
2: list<binary> property_names;
}
struct CreateIndexResponse {
1: Result result;
2: IndexId index_id;
}
struct IndexInfo {
1: LabelId label_id;
2: list<binary> property_names;
}
struct GetIndexInfosResponse {
1: Result result;
2: list<IndexInfo> index_infos;
}
service Meta {
CreateLabelResponse createPrimaryLabel(1: CreatePrimaryLabelRequest req);
CreateLabelResponse createSecondaryLabel(1: binary label_name);
Result dropLabel(1: binary label_name);
GetLabelInfoResponse getLabelInfo(1:binary label_name);
GetLabelInfosResponse getLabelInfos(1: list<binary> label_names);
CreateIndexResponse createIndex(1: CreateIndexRequest req);
// Don't have index names, and the label doesn't identify the index uniquely, therefore
Result dropIndex(1: IndexId index_id);
GetIndexInfosResponse getIndexInfos();
GetIndexInfosResponse getIndexInfosForLabel(1: LabelId label_id);
}

View File

@@ -0,0 +1,302 @@
namespace cpp2 interface.storage
// https://stackoverflow.com/a/34234874/6639989
cpp_include "storage/v2/view.hpp"
typedef i64 VertexId
typedef i64 Gid
// TODO(antaljanosbenjamin): Use this after introducing 128 bit vertex ids
// struct VertexId {
// 1: i64 upper_half;
// 2: i64 lower_half;
// }
struct Label {
1: i64 id;
}
struct EdgeType {
1: binary name;
}
struct EdgeId {
1: VertexId src;
// QUESTION(antaljanosbenjamin): is it okay to have vertex based (edge id = vertex id + edge id inside vertex)?
2: Gid gid;
}
struct Date {
1: i16 year;
2: byte month;
3: byte day;
}
struct LocalTime {
1: byte hour;
2: byte minute;
3: byte second;
4: i16 millisecond;
5: i16 microsecond;
}
struct LocalDateTime {
1: Date date;
2: LocalTime local_time;
}
struct Duration {
1: i64 milliseconds;
}
union Value {
1: Null null_v;
2: bool bool_v;
3: i64 int_v;
4: double double_v;
5: binary string_v;
6: list<Value> list_v;
7: map<binary, Value> (cpp.template = "std::unordered_map") map_v (cpp2.ref_type = "unique");
8: Vertex vertex_v (cpp2.ref_type = "unique");
9: Edge edge_v (cpp2.ref_type = "unique");
10: Path path_v (cpp2.ref_type = "unique");
11: Date date_v;
12: LocalTime local_time_v;
13: LocalDateTime local_date_time_v;
14: Duration duration_v;
}
struct Null {
}
struct Vertex {
1: VertexId id;
// TODO(antaljanosbenjamin): Change to sperate primary and secondary labels when schema is implemented
2: list<Label> labels;
}
struct Edge {
1: VertexId src;
2: VertexId dst;
3: EdgeType type;
}
struct PathPart {
1: Vertex dst;
2: Edge edge;
}
struct Path {
1: Vertex src;
2: list<PathPart> parts;
}
struct ValuesMap {
1: map<i64, Value> (cpp.template = "std::unordered_map") values_map;
}
struct MappedValues {
1: list<ValuesMap> properties;
}
struct ListedValues {
1: list<list<Value>> properties;
}
union Values {
// This struct is necessary because depending on the request the response
// has two different formats:
// 1. When the request specifies the returned properties, then they are
// returned in that order, therefore no extra mapping is necessary.
// 2. When the request doesn't specify the returned properties, then all
// of the properties are returned. In this case the `mapped` field is
// used. To extract the <key,value> pairs from this struct the
// mapping of i64 -> property name has to be used.
1: ListedValues listed;
2: MappedValues mapped;
}
struct Expression {
1: binary alias;
2: binary expression;
}
struct Filter {
1: binary filter_expression;
}
enum OrderingDirection {
ASCENDING = 1;
DESCENDING = 2;
}
struct OrderBy {
1: Expression expression;
2: OrderingDirection direction;
}
struct Result {
// Just placeholder data for now
1: bool success;
}
enum View {
OLD = 0,
NEW = 1
} (cpp.enum_strict, cpp.type = "memgraph::storage::View")
struct ScanVerticesRequest {
1: i64 transaction_id;
2: optional i64 start_id;
// Special values are accepted:
// * __mg__id (Vertex, but without labels)
// * __mg__labels (Vertex, but without the id)
// If both of them is specified, then it will result in a single, fully populated vertex
// QUESTION(antaljanosbenjamin): Does the `__mg__labels` is necessary? What about passing the `labels` function
// as an expression? Maybe it is an optimization. For communicating the vertex id
// the Vertex struct is really handy.
3: optional list<binary> props_to_return;
4: list<Expression> expressions;
5: optional i64 limit;
6: View view;
7: optional Filter filter;
}
struct ScanVerticesResponse {
1: Result result;
2: Values values;
3: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
// contains the next start_id if there is any
4: optional VertexId next_start_id;
}
union VertexOrEdgeIds {
1: list<VertexId> vertex_ids;
2: list<EdgeId> edge_ids;
}
struct GetPropertiesRequest {
1: i64 transaction_id;
2: VertexOrEdgeIds vertex_or_edge_ids;
3: list<binary> property_names;
4: list<Expression> expressions;
5: bool only_unique = false;
6: optional list<OrderBy> order_by;
7: optional i64 limit;
8: optional Filter filter;
}
struct GetPropertiesResponse {
1: Values values;
2: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
}
enum EdgeDirection {
OUT = 1;
IN = 2;
BOTH = 3;
}
struct ExpandOneRequest {
1: i64 transaction_id;
2: list<VertexId> src_vertices;
3: list<EdgeType> edge_types;
4: EdgeDirection direction;
5: bool only_unique_neighbor_rows = false;
// The empty optional means return all of the properties, while an empty
// list means do not return any properties
// TODO(antaljanosbenjamin): All of the special values should be communicated through a single vertex object
// after schema is implemented
// Special values are accepted:
// * __mg__labels
6: optional list<binary> src_vertex_properties;
// TODO(antaljanosbenjamin): All of the special values should be communicated through a single vertex object
// after schema is implemented
// Special values are accepted:
// * __mg__dst_id (Vertex, but without labels)
// * __mg__type (binary)
7: optional list<binary> edge_properties;
// QUESTION(antaljanosbenjamin): Maybe also add possibility to expressions evaluated on the source vertex?
// List of expressions evaluated on edges
8: list<Expression> expressions;
9: optional list<OrderBy> order_by;
10: optional i64 limit;
11: optional Filter filter;
}
struct ExpandOneResultRow {
// NOTE: This struct could be a single Values with columns something like this:
// src_vertex(Vertex), vertex_prop1(Value), vertex_prop2(Value), edges(list<Value>)
// where edges might be a list of:
// 1. list<Value> if only a defined list of edge properties are returned
// 2. map<binary, Value> if all of the edge properties are returned
// The drawback of this is currently the key of the map is always interpreted as a string in Value, not as an
// integer, which should be in case of mapped properties.
1: Vertex src_vertex;
2: optional Values src_vertex_properties;
3: Values edges;
}
struct ExpandOneResponse {
// This approach might not suit the expand with per shard parrallelization,
// because the property_name_map has to be accessed from multiple threads
// in order to avoid duplicated keys (two threads might map the same
// property with different numbers) and multiple passes (to unify the
// mapping amond result returned from different shards).
1: list<ExpandOneResultRow> result;
2: optional map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
}
struct NewVertex {
1: list<i64> label_ids;
2: map<i64, Value> properties;
}
struct CreateVerticesRequest {
1: required i64 transaction_id;
2: map<i64, binary> (cpp.template = "std::unordered_map") labels_name_map;
3: map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
4: list<NewVertex> new_vertices;
}
struct NewEdge {
1: required VertexId src;
2: required VertexId dest;
3: required EdgeType type;
4: map<i64, Value> properties;
}
struct CreateEdgesRequest {
1: required i64 transaction_id;
3: map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
4: list<NewEdge> new_edges;
}
struct UpdateVertex {
1: required VertexId vertex_id;
2: list<i64> create_label_ids;
3: list<i64> delete_label_ids;
4: map<i64, Value> properties;
}
// Null signifies removel of label or property
struct UpdateVerticesRequest {
1: required i64 transaction_id;
2: map<i64, binary> (cpp.template = "std::unordered_map") labels_name_map;
3: map<i64, binary> (cpp.template = "std::unordered_map") property_name_map;
4: list<UpdateVertex> update_vertices;
}
service Storage {
i64 startTransaction()
Result commitTransaction(1: i64 transaction_id)
void abortTransaction(1: i64 transaction_id)
Result createVertices(1: CreateVerticesRequest req)
Result updateVertices(1: UpdateVerticesRequest req)
Result createEdges(1: CreateEdgesRequest req)
ScanVerticesResponse scanVertices(1: ScanVerticesRequest req)
GetPropertiesResponse getProperties(1: GetPropertiesRequest req)
ExpandOneResponse expandOne(1: ExpandOneRequest req)
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -26,16 +26,22 @@
#include <thread>
#include <fmt/format.h>
#include <folly/init/Init.h>
#include <folly/io/async/AsyncSocket.h>
#include <folly/io/async/AsyncTransport.h>
#include <folly/io/async/EventBase.h>
#include <gflags/gflags.h>
#include <spdlog/common.h>
#include <spdlog/sinks/daily_file_sink.h>
#include <spdlog/sinks/dist_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include <thrift/lib/cpp2/async/HeaderClientChannel.h>
#include "communication/bolt/v1/constants.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "helpers.hpp"
#include "interface/gen-cpp2/Storage.h"
#include "py/py.hpp"
#include "query/auth_checker.hpp"
#include "query/discard_value_stream.hpp"
@@ -81,8 +87,8 @@
#include "communication/bolt/v1/exceptions.hpp"
#include "communication/bolt/v1/session.hpp"
#include "communication/init.hpp"
#include "communication/server.hpp"
#include "communication/session.hpp"
#include "communication/v2/server.hpp"
#include "communication/v2/session.hpp"
#include "glue/communication.hpp"
#include "auth/auth.hpp"
@@ -842,13 +848,14 @@ class AuthChecker final : public memgraph::query::AuthChecker {
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
};
class BoltSession final : public memgraph::communication::bolt::Session<memgraph::communication::InputStream,
memgraph::communication::OutputStream> {
class BoltSession final : public memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream> {
public:
BoltSession(SessionData *data, const memgraph::io::network::Endpoint &endpoint,
memgraph::communication::InputStream *input_stream, memgraph::communication::OutputStream *output_stream)
: memgraph::communication::bolt::Session<memgraph::communication::InputStream,
memgraph::communication::OutputStream>(input_stream, output_stream),
BoltSession(SessionData *data, const memgraph::communication::v2::ServerEndpoint &endpoint,
memgraph::communication::v2::InputStream *input_stream,
memgraph::communication::v2::OutputStream *output_stream)
: memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream>(input_stream, output_stream),
db_(data->db),
interpreter_(data->interpreter_context),
auth_(data->auth),
@@ -858,8 +865,8 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
endpoint_(endpoint) {
}
using memgraph::communication::bolt::Session<memgraph::communication::InputStream,
memgraph::communication::OutputStream>::TEncoder;
using memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream>::TEncoder;
void BeginTransaction() override { interpreter_.BeginTransaction(); }
@@ -877,7 +884,8 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
}
#ifdef MG_ENTERPRISE
if (memgraph::utils::license::global_license_checker.IsValidLicenseFast()) {
audit_log_->Record(endpoint_.address, user_ ? *username : "", query, memgraph::storage::PropertyValue(params_pv));
audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query,
memgraph::storage::PropertyValue(params_pv));
}
#endif
try {
@@ -996,10 +1004,10 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
#ifdef MG_ENTERPRISE
memgraph::audit::Log *audit_log_;
#endif
memgraph::io::network::Endpoint endpoint_;
memgraph::communication::v2::ServerEndpoint endpoint_;
};
using ServerT = memgraph::communication::Server<BoltSession, SessionData>;
using ServerT = memgraph::communication::v2::Server<BoltSession, SessionData>;
using memgraph::communication::ServerContext;
// Needed to correctly handle memgraph destruction from a signal handler.
@@ -1241,8 +1249,10 @@ int main(int argc, char **argv) {
memgraph::utils::MessageWithLink("Using non-secure Bolt connection (without SSL).", "https://memgr.ph/ssl"));
}
ServerT server({FLAGS_bolt_address, static_cast<uint16_t>(FLAGS_bolt_port)}, &session_data, &context,
FLAGS_bolt_session_inactivity_timeout, service_name, FLAGS_bolt_num_workers);
auto server_endpoint = memgraph::communication::v2::ServerEndpoint{
boost::asio::ip::address::from_string(FLAGS_bolt_address), static_cast<uint16_t>(FLAGS_bolt_port)};
ServerT server(server_endpoint, &session_data, &context, FLAGS_bolt_session_inactivity_timeout, service_name,
FLAGS_bolt_num_workers);
// Setup telemetry
std::optional<memgraph::telemetry::Telemetry> telemetry;
@@ -1287,8 +1297,19 @@ int main(int argc, char **argv) {
MG_ASSERT(server.Start(), "Couldn't start the Bolt server!");
websocket_server.Start();
server.AwaitShutdown();
websocket_server.AwaitShutdown();
folly::init(&argc, &argv);
folly::EventBase base;
int port = 7779; // The port on which server is listening
// Create a async client socket and connect it. Change
// the ip address to where the server is listening
auto socket(folly::AsyncSocket::newSocket(&base, "127.0.0.1", port));
// Create a HeaderClientChannel object which is used in creating
// client object
auto client_channel = apache::thrift::HeaderClientChannel::newChannel(std::move(socket));
// Create a client object
interface::storage::StorageAsyncClient client(std::move(client_channel));
memgraph::query::procedure::gModuleRegistry.UnloadAllModules();

View File

@@ -17,8 +17,6 @@
parser grammar Cypher;
options { tokenVocab=CypherLexer; }
cypher : statement ';'? EOF ;
statement : query ;

View File

@@ -30,7 +30,7 @@ memgraphCypherKeyword : cypherKeyword
| BATCH_SIZE
| BEFORE
| BOOTSTRAP_SERVERS
| CHECK
| CHECK_
| CLEAR
| COMMIT
| COMMITTED
@@ -361,7 +361,7 @@ stopAllStreams : STOP ALL STREAMS ;
showStreams : SHOW STREAMS ;
checkStream : CHECK STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ;
checkStream : CHECK_ STREAM streamName ( BATCH_LIMIT batchLimit=literal ) ? ( TIMEOUT timeout=literal ) ? ;
settingName : literal ;

View File

@@ -35,7 +35,8 @@ BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ;
BATCH_SIZE : B A T C H UNDERSCORE S I Z E ;
BEFORE : B E F O R E ;
BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ;
CHECK : C H E C K ;
/* This is a workaround to make sure the generated C++ function doesn't conlift with the CHECK macro from glog */
CHECK_ : C H E C K ;
CLEAR : C L E A R ;
COMMIT : C O M M I T ;
COMMITTED : C O M M I T T E D ;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -2109,7 +2109,7 @@ template <typename T>
concept AccessorWithProperties = requires(T value, storage::PropertyId property_id,
storage::PropertyValue property_value) {
{ value.ClearProperties() } -> std::same_as<storage::Result<std::map<storage::PropertyId, storage::PropertyValue>>>;
{value.SetProperty(property_id, property_value)};
{ value.SetProperty(property_id, property_value) };
};
/// Helper function that sets the given values on either a Vertex or an Edge.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -43,6 +43,17 @@
// NOLINTNEXTLINE(google-build-using-namespace)
using namespace memgraph::query::procedure;
#define MG_STATIC_ASSERT_NOEXCEPT(x) \
do { \
_Pragma("clang diagnostic push"); \
_Pragma("clang diagnostic ignored \"-Wunevaluated-expression\""); \
static_assert(noexcept(x)); \
_Pragma("clang diagnostic pop"); \
} while (false)
#define MG_EXECUTE_NOEXCEPT(x) \
MG_STATIC_ASSERT_NOEXCEPT(x); \
x
namespace {
void *MgpAlignedAllocImpl(memgraph::utils::MemoryResource &memory, const size_t size_in_bytes, const size_t alignment) {
@@ -823,7 +834,6 @@ DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(int, int64_t);
DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(double, double);
DEFINE_MGP_VALUE_MAKE_WITH_MEMORY(string, const char *);
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define DEFINE_MGP_VALUE_MAKE(type) \
mgp_error mgp_value_make_##type(mgp_##type *val, mgp_value **result) { \
return WrapExceptions([val] { return NewRawMgpObject<mgp_value>(val->GetMemoryResource(), val); }, result); \
@@ -1687,9 +1697,7 @@ mgp_error mgp_vertex_copy(mgp_vertex *v, mgp_memory *memory, mgp_vertex **result
void mgp_vertex_destroy(mgp_vertex *v) { DeleteRawMgpObject(v); }
mgp_error mgp_vertex_equal(mgp_vertex *v1, mgp_vertex *v2, int *result) {
// NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression)
static_assert(noexcept(*result = *v1 == *v2 ? 1 : 0));
*result = *v1 == *v2 ? 1 : 0;
MG_EXECUTE_NOEXCEPT(*result = *v1 == *v2 ? 1 : 0);
return MGP_ERROR_NO_ERROR;
}
@@ -1947,9 +1955,7 @@ mgp_error mgp_edge_copy(mgp_edge *e, mgp_memory *memory, mgp_edge **result) {
void mgp_edge_destroy(mgp_edge *e) { DeleteRawMgpObject(e); }
mgp_error mgp_edge_equal(mgp_edge *e1, mgp_edge *e2, int *result) {
// NOLINTNEXTLINE(clang-diagnostic-unevaluated-expression)
static_assert(noexcept(*result = *e1 == *e2 ? 1 : 0));
*result = *e1 == *e2 ? 1 : 0;
MG_EXECUTE_NOEXCEPT(*result = *e1 == *e2 ? 1 : 0);
return MGP_ERROR_NO_ERROR;
}

View File

@@ -832,8 +832,8 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) {
return with_error(error);
}
for (auto &trans : module_def->transformations) {
const bool was_result_added = MgpTransAddFixedResult(&trans.second);
if (!was_result_added) {
const bool success = MGP_ERROR_NO_ERROR == MgpTransAddFixedResult(&trans.second);
if (!success) {
const auto error =
fmt::format("Unable to add result to transformation in module {}; add result failed", file_path);
return with_error(error);

View File

@@ -97,6 +97,10 @@ class LabelIndex {
Iterator begin() { return Iterator(this, index_accessor_.begin()); }
Iterator end() { return Iterator(this, index_accessor_.end()); }
Iterator IterateFrom(Vertex *vertex) {
return Iterator(this, index_accessor_.find_equal_or_greater(Entry{vertex, 0}));
}
private:
utils::SkipList<Entry>::Accessor index_accessor_;
LabelId label_;

View File

@@ -85,6 +85,7 @@ VerticesIterable::VerticesIterable(AllVerticesIterable vertices) : type_(Type::A
}
VerticesIterable::VerticesIterable(LabelIndex::Iterable vertices) : type_(Type::BY_LABEL) {
// TODO(antaljanosbenjamin): vertices_accessor_ should be initialized here
new (&vertices_by_label_) LabelIndex::Iterable(std::move(vertices));
}
@@ -169,6 +170,22 @@ VerticesIterable::Iterator VerticesIterable::end() {
}
}
VerticesIterable::Iterator VerticesIterable::IterateFrom(const Gid vertex_id) {
switch (type_) {
case Type::ALL:
return Iterator(all_vertices_.IterateFrom(vertex_id));
case Type::BY_LABEL: {
auto vertex_it = vertices_accessor_.find_equal_or_greater(vertex_id);
if (vertex_it == vertices_accessor_.end()) {
return end();
}
return Iterator(vertices_by_label_.IterateFrom(&(*vertex_it)));
}
case Type::BY_LABEL_PROPERTY:
LOG_FATAL("IterateFrom is not supported for label property indices!");
}
}
VerticesIterable::Iterator::Iterator(AllVerticesIterable::Iterator it) : type_(Type::ALL) {
new (&all_it_) AllVerticesIterable::Iterator(std::move(it));
}

View File

@@ -94,8 +94,10 @@ class AllVerticesIterable final {
constraints_(constraints),
config_(config) {}
Iterator begin() { return Iterator(this, vertices_accessor_.begin()); }
Iterator end() { return Iterator(this, vertices_accessor_.end()); }
Iterator begin() { return Iterator{this, vertices_accessor_.begin()}; }
Iterator end() { return Iterator{this, vertices_accessor_.end()}; }
Iterator IterateFrom(Gid vertex_id) { return Iterator{this, vertices_accessor_.find_equal_or_greater(vertex_id)}; }
};
/// Generic access to different kinds of vertex iterations.
@@ -108,8 +110,13 @@ class VerticesIterable final {
Type type_;
union {
AllVerticesIterable all_vertices_;
LabelIndex::Iterable vertices_by_label_;
LabelPropertyIndex::Iterable vertices_by_label_property_;
struct {
utils::SkipList<Vertex>::Accessor vertices_accessor_;
union {
LabelIndex::Iterable vertices_by_label_;
LabelPropertyIndex::Iterable vertices_by_label_property_;
};
};
};
public:
@@ -158,6 +165,9 @@ class VerticesIterable final {
Iterator begin();
Iterator end();
// TODO(antaljanosbenjamin): write tests for this
Iterator IterateFrom(Gid vertex_id);
};
/// Structure used to return information about existing indices in the storage.

View File

@@ -15,8 +15,8 @@ namespace memgraph::storage {
/// Indicator for obtaining the state before or after a transaction & command.
enum class View {
OLD,
NEW,
OLD = 0,
NEW = 1,
};
} // namespace memgraph::storage

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2021 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,3 +1,11 @@
# Set up C++ functions for e2e tests
function(add_query_module target_name src)
add_library(${target_name} SHARED ${src})
SET_TARGET_PROPERTIES(${target_name} PROPERTIES PREFIX "")
target_include_directories(${target_name} PRIVATE ${CMAKE_SOURCE_DIR}/include)
endfunction()
function(copy_e2e_python_files TARGET_PREFIX FILE_NAME)
add_custom_target(memgraph__e2e__${TARGET_PREFIX}__${FILE_NAME} ALL
COMMAND ${CMAKE_COMMAND} -E copy

View File

@@ -1,10 +1,3 @@
# Set up C++ functions for e2e tests
function(add_query_module target_name src)
add_library(${target_name} SHARED ${src})
SET_TARGET_PROPERTIES(${target_name} PROPERTIES PREFIX "")
target_include_directories(${target_name} PRIVATE ${CMAKE_SOURCE_DIR}/include)
endfunction()
# Set up Python functions for e2e tests
function(copy_magic_functions_e2e_python_files FILE_NAME)
copy_e2e_python_files(functions ${FILE_NAME})

View File

@@ -18,12 +18,14 @@ import time
from multiprocessing import Process, Value
import common
TRANSFORMATIONS_TO_CHECK = [
TRANSFORMATIONS_TO_CHECK_C = [
"empty_transformation"]
TRANSFORMATIONS_TO_CHECK_PY = [
"kafka_transform.simple",
"kafka_transform.with_parameters"]
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
def test_simple(kafka_producer, kafka_topics, connection, transformation):
assert len(kafka_topics) > 0
cursor = connection.cursor()
@@ -44,7 +46,7 @@ def test_simple(kafka_producer, kafka_topics, connection, transformation):
cursor, topic, common.SIMPLE_MSG)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
def test_separate_consumers(
kafka_producer,
kafka_topics,
@@ -125,7 +127,7 @@ def test_start_from_last_committed_offset(
cursor, kafka_topics[0], message)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
def test_check_stream(
kafka_producer,
kafka_topics,
@@ -307,7 +309,7 @@ def test_restart_after_error(kafka_producer, kafka_topics, connection):
cursor, "MATCH (n:VERTEX { id : 42 }) RETURN n")
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
def test_bootstrap_server(
kafka_producer,
kafka_topics,
@@ -334,7 +336,7 @@ def test_bootstrap_server(
cursor, topic, common.SIMPLE_MSG)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
def test_bootstrap_server_empty(
kafka_producer,
kafka_topics,
@@ -352,7 +354,7 @@ def test_bootstrap_server_empty(
)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK)
@pytest.mark.parametrize("transformation", TRANSFORMATIONS_TO_CHECK_PY)
def test_set_offset(kafka_producer, kafka_topics, connection, transformation):
assert len(kafka_topics) > 0
cursor = connection.cursor()
@@ -450,6 +452,14 @@ def test_info_procedure(kafka_topics, connection):
(local, configs, consumer_group, reducted_credentials, kafka_topics)]
common.validate_info(stream_info, expected_stream_info)
@pytest.mark.parametrize("transformation",TRANSFORMATIONS_TO_CHECK_C)
def test_load_c_transformations(connection, transformation):
cursor = connection.cursor()
query = "CALL mg.transformations() YIELD * WITH name WHERE name STARTS WITH 'c_transformations." + transformation + "' RETURN name"
result = common.execute_and_fetch_all(
cursor, query)
assert len(result) == 1
assert result[0][0] == "c_transformations." + transformation
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-rA"]))

View File

@@ -1,2 +1,3 @@
copy_streams_e2e_python_files(kafka_transform.py)
copy_streams_e2e_python_files(pulsar_transform.py)
add_query_module(c_transformations c_transformations.cpp)

View File

@@ -0,0 +1,22 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "mg_procedure.h"
extern "C" int mgp_init_module(mgp_module *module, mgp_memory *memory) {
static const auto no_op_cb = [](mgp_messages *msg, mgp_graph *graph, mgp_result *result, mgp_memory *memory) {};
if (MGP_ERROR_NO_ERROR != mgp_module_add_transformation(module, "empty_transformation", no_op_cb)) {
return 1;
}
return 0;
}

View File

@@ -55,3 +55,9 @@ target_link_libraries(${test_prefix}ssl_client mg-communication)
add_manual_test(ssl_server.cpp)
target_link_libraries(${test_prefix}ssl_server mg-communication)
add_manual_test(storage_server.cpp storage_service.cpp storage_service.hpp)
target_link_libraries(${test_prefix}storage_server mg-query mg-interface-storage-cpp2)
add_manual_test(storage_client_demo.cpp)
target_link_libraries(${test_prefix}storage_client_demo mg-interface-storage-cpp2 mg-utils)

View File

@@ -0,0 +1,336 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <chrono>
#include <cstdint>
#include <iostream>
#include <optional>
#include <ratio>
#include <thread>
#include <bits/ranges_algo.h>
#include <folly/Executor.h>
#include <folly/Unit.h>
#include <folly/executors/IOThreadPoolExecutor.h>
#include <folly/executors/ThreadedExecutor.h>
#include <folly/executors/thread_factory/NamedThreadFactory.h>
#include <folly/init/Init.h>
#include <folly/io/async/AsyncSocket.h>
#include <folly/io/async/EventBase.h>
#include <folly/io/async/ScopedEventBaseThread.h>
#include <thrift/lib/cpp2/async/ClientStreamBridge.h>
#include <thrift/lib/cpp2/async/HeaderClientChannel.h>
#include <thrift/lib/cpp2/async/RocketClientChannel.h>
#include "interface/gen-cpp2/StorageAsyncClient.h"
#include "interface/gen-cpp2/storage_types.h"
#include "storage/v2/view.hpp"
#include "utils/logging.hpp"
DEFINE_string(host, "127.0.0.1", "Storage Server host");
DEFINE_int32(port, 7779, "Storage Server port");
using interface::storage::CreateVerticesRequest;
using interface::storage::StorageAsyncClient;
using interface::storage::Value;
using interface::storage::Vertex;
void Print(std::ostream &os, const Value &value) {
switch (value.getType()) {
case Value::Type::null_v:
os << "<null>";
break;
case Value::Type::bool_v:
os << value.get_bool_v();
break;
case Value::Type::int_v:
os << value.get_int_v();
break;
case Value::Type::string_v:
os << value.get_string_v();
break;
default:
os << "UNKNOWN TYPE";
break;
}
}
std::optional<int64_t> ScanVertices(const std::shared_ptr<StorageAsyncClient> &client, int64_t transaction_id,
const std::optional<int64_t> &start_id,
const std::optional<std::vector<std::string>> &props_to_return,
const std::optional<int64_t> &limit,
const memgraph::storage::View view = memgraph::storage::View::NEW) {
interface::storage::ScanVerticesRequest req;
req.transaction_id_ref() = transaction_id;
req.view_ref() = view;
if (limit.has_value()) {
req.limit_ref() = limit.value();
}
if (start_id.has_value()) {
req.start_id_ref() = start_id.value();
}
if (props_to_return.has_value()) {
req.props_to_return_ref() = props_to_return.value();
}
return client->future_scanVertices(req)
.then(
[&props_to_return](folly::Try<interface::storage::ScanVerticesResponse> &&result) -> std::optional<int64_t> {
if (result.hasException()) {
LOG(INFO) << "FAILED: " << result.exception().get_exception()->what() << std::endl;
return std::nullopt;
}
if (!*result->result_ref()->success()) {
throw std::runtime_error("scan vertices failed");
}
const auto &response = result.value();
MG_ASSERT(props_to_return.has_value() != response.property_name_map_ref().has_value());
if (props_to_return.has_value()) {
const auto values_list_ref = response.values_ref()->listed_ref()->properties_ref();
const auto &prop_names = props_to_return.value();
if (values_list_ref->empty()) {
return std::nullopt;
}
MG_ASSERT(values_list_ref->at(0).size() == prop_names.size() + 1UL);
for (const auto &values : *values_list_ref) {
auto value_it = values.begin();
std::cout << "Vertex(";
Print(std::cout, *value_it++);
std::cout << ')';
auto name_it = prop_names.begin();
while (value_it != values.end()) {
std::cout << "\n\t" << *name_it << ": ";
Print(std::cout, *value_it);
++value_it;
++name_it;
}
std::cout << std::endl;
}
} else {
const auto values_list_ref = response.values_ref()->mapped_ref()->properties();
const auto mapped_prop_names_ref = response.property_name_map_ref();
for (const auto &values_map : *values_list_ref) {
const auto inner_values_map_ref = values_map.values_map_ref();
std::cout << "Vertex(";
Print(std::cout, inner_values_map_ref->at(1));
std ::cout << ')';
for (const auto &[prop_id, value] : *inner_values_map_ref) {
if (prop_id != 1) {
std::cout << "\n\t" << mapped_prop_names_ref->at(prop_id) << ": ";
Print(std::cout, value);
}
}
std::cout << std::endl;
}
}
if (response.next_start_id_ref().has_value()) {
return *response.get_next_start_id();
}
return std::nullopt;
})
.get();
}
void CreateVertex(const std::shared_ptr<StorageAsyncClient> &client, std::vector<std::string_view> labels,
std::vector<std::string_view> property_names) {
interface::storage::CreateVerticesRequest request {};
auto &new_vertex = request.new_vertices_ref()->emplace_back();
int prop_count = 1;
for (const auto prop : property_names) {
request.property_name_map()->emplace(prop_count, prop);
interface::storage::Value prop_value {};
prop_value.set_int_v(prop_count);
new_vertex.properties_ref()->emplace(prop_count, std::move(prop_value));
prop_count++;
}
int label_count = 1;
for (const auto label : labels) {
request.labels_name_map_ref()->emplace(label_count, label);
new_vertex.label_ids_ref()->push_back(label_count);
label_count++;
}
client->future_startTransaction()
.then([client, request = std::move(request)](folly::Try<int64_t> &&result) mutable {
if (result.hasException()) {
LOG(INFO) << "FAILED1: " << result.exception().get_exception()->what() << std::endl;
return folly::makeFuture<interface::storage::Result>(std::runtime_error("failed to start transaction"));
}
const auto transaction_id = result.value();
request.transaction_id_ref() = transaction_id;
LOG(INFO) << "Sending message...";
return client->future_createVertices(request) /*.via(evb)*/.then(
[transaction_id, client](folly::Try<interface::storage::Result> &&reply) {
LOG(INFO) << "Closing transaction";
if (reply.hasException()) {
LOG(INFO) << "FAILED2: " << reply.exception().get_exception()->what() << std::endl;
return client->future_abortTransaction(transaction_id).then([](folly::Try<void> &&reply) {
return folly::makeFuture<interface::storage::Result>(std::runtime_error("vertex creation failed"));
});
} else {
LOG(INFO) << "SUCCESS\n";
return client->future_commitTransaction(transaction_id);
}
});
})
.get();
}
void CreateEdge(const std::shared_ptr<StorageAsyncClient> &client, uint64_t src, uint64_t dest, std::string type,
std::vector<std::string_view> property_names) {
interface::storage::CreateEdgesRequest request {};
auto &new_edge = request.new_edges_ref()->emplace_back();
new_edge.src_ref().emplace(src);
new_edge.dest_ref().emplace(dest);
new_edge.type_ref()->name_ref().emplace(type);
int prop_count = 1;
for (const auto prop : property_names) {
request.property_name_map()->emplace(prop_count, prop);
interface::storage::Value prop_value {};
prop_value.set_int_v(prop_count);
new_edge.properties_ref()->emplace(prop_count, std::move(prop_value));
prop_count++;
}
client->future_startTransaction()
.then([client, request = std::move(request)](folly::Try<int64_t> &&result) mutable {
if (result.hasException()) {
LOG(INFO) << "FAILED1: " << result.exception().get_exception()->what() << std::endl;
return folly::makeFuture<interface::storage::Result>(std::runtime_error("failed to start transaction"));
}
const auto transaction_id = result.value();
request.transaction_id_ref() = transaction_id;
LOG(INFO) << "Sending message...";
return client->future_createEdges(request).then(
[transaction_id, client](folly::Try<interface::storage::Result> &&reply) {
LOG(INFO) << "Closing transaction";
if (reply.hasException()) {
LOG(INFO) << "FAILED2: " << reply.exception().get_exception()->what() << std::endl;
return client->future_abortTransaction(transaction_id).then([](folly::Try<void> &&reply) {
return folly::makeFuture<interface::storage::Result>(std::runtime_error("edge creation failed"));
});
}
LOG(INFO) << "SUCCESS\n";
return client->future_commitTransaction(transaction_id);
});
})
.get();
}
void UpdateVertexProperties(const std::shared_ptr<StorageAsyncClient> &client, uint64_t vertex_id,
std::vector<std::string_view> add_labels, std::vector<std::string_view> remove_labels,
std::vector<std::string_view> property_names) {
interface::storage::UpdateVerticesRequest request {};
auto &update_vertex = request.update_vertices_ref()->emplace_back();
int prop_count = 10;
for (const auto prop : property_names) {
request.property_name_map()->emplace(prop_count, prop);
interface::storage::Value prop_value {};
prop_value.set_int_v(prop_count);
update_vertex.properties_ref()->emplace(prop_count, std::move(prop_value));
prop_count++;
}
int label_count = 1;
for (const auto add_label : add_labels) {
request.labels_name_map_ref()->emplace(label_count, add_label);
update_vertex.create_label_ids_ref()->push_back(label_count);
label_count++;
}
for (const auto remove_label : remove_labels) {
request.labels_name_map_ref()->emplace(label_count, remove_label);
update_vertex.delete_label_ids_ref()->push_back(label_count);
label_count++;
}
client->future_startTransaction()
.then([client, request = std::move(request)](folly::Try<int64_t> &&result) mutable {
if (result.hasException()) {
LOG(INFO) << "FAILED1: " << result.exception().get_exception()->what() << std::endl;
return folly::makeFuture<interface::storage::Result>(std::runtime_error("failed to start transaction"));
}
const auto transaction_id = result.value();
request.transaction_id_ref() = transaction_id;
LOG(INFO) << "Sending message...";
return client->future_updateVertices(request).then(
[transaction_id, client](folly::Try<interface::storage::Result> &&reply) {
LOG(INFO) << "Closing transaction";
if (reply.hasException()) {
LOG(INFO) << "FAILED2: " << reply.exception().get_exception()->what() << std::endl;
return client->future_abortTransaction(transaction_id).then([](folly::Try<void> &&reply) {
return folly::makeFuture<interface::storage::Result>(std::runtime_error("updating vertices failed"));
});
}
LOG(INFO) << "SUCCESS\n";
return client->future_commitTransaction(transaction_id);
});
})
.get();
}
int main(int argc, char *argv[]) {
FLAGS_logtostderr = true;
folly::init(&argc, &argv);
auto threadFactory = std::make_shared<folly::NamedThreadFactory>("io-thread");
auto ioThreadPool = std::make_shared<folly::IOThreadPoolExecutor>(8, std::move(threadFactory));
auto *evb = ioThreadPool->getEventBase();
folly::AsyncSocket::UniquePtr socket_ptr;
evb->runImmediatelyOrRunInEventBaseThreadAndWait(
[&socket_ptr, evb]() { socket_ptr = folly::AsyncSocket::newSocket(evb, FLAGS_host, FLAGS_port); });
auto headerClientChannel = apache::thrift::HeaderClientChannel::newChannel(std::move(socket_ptr));
std::shared_ptr<StorageAsyncClient> client(
new StorageAsyncClient(std::move(headerClientChannel)),
[evb](StorageAsyncClient *p) { evb->runImmediatelyOrRunInEventBaseThreadAndWait([p] { delete p; }); });
LOG(INFO) << "Start... ";
CreateVertex(client, {"label1", "label2"}, {"proop", "prooop2"});
CreateVertex(client, {"label1", "label2"}, {"proop", "prooop3"});
CreateVertex(client, {"label1", "label2"}, {"proop", "prooop4"});
CreateVertex(client, {"label1", "label3"}, {"proop", "prooop2"});
CreateVertex(client, {"label1", "label4"}, {"proop", "prooop3"});
CreateVertex(client, {"label1", "label5"}, {"proop", "prooop4"});
// TODO: Until we have hash-based vertex id implemented this is temporary
CreateEdge(client, 0, 1, "type1", {"proop", "prooop2"});
CreateEdge(client, 1, 2, "type1", {"proop", "prooop2"});
CreateEdge(client, 2, 3, "type5", {});
// This should fail
// CreateEdge(client, 12121212, 2121212121, "type5", {});
UpdateVertexProperties(client, 0, {"label3"}, {"label1", "label2"}, {"proop", "new_prop"});
UpdateVertexProperties(client, 1, {"el_label", "el_labelo"}, {}, {"proop", "new_new_prop"});
UpdateVertexProperties(client, 1, {"el_label"}, {"el_labelo"}, {"proop", "new_news_new_prop"});
const auto transaction_id = client->future_startTransaction().get();
std::vector<std::string> props{"proop", "prooop2"};
auto res = ScanVertices(client, transaction_id, std::nullopt, props, 3);
std::cout << "RES:" << res.value() << std::endl;
res = ScanVertices(client, transaction_id, *res, props, 3);
std::cout << "RES:" << res.value_or(-1) << std::endl;
res = ScanVertices(client, transaction_id, std::nullopt, std::nullopt, 3);
std::cout << "RES:" << res.value() << std::endl;
res = ScanVertices(client, transaction_id, *res, std::nullopt, 3);
std::cout << "RES:" << res.value_or(-1) << std::endl;
return 1;
}

View File

@@ -0,0 +1,80 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <string>
#include <folly/init/Init.h>
#include <folly/portability/GFlags.h>
#include <proxygen/httpserver/HTTPServerOptions.h>
#include <spdlog/common.h>
#include <thrift/lib/cpp/concurrency/ThreadManager.h>
#include <thrift/lib/cpp/thrift_config.h>
#include <thrift/lib/cpp2/server/ThriftProcessor.h>
#include <thrift/lib/cpp2/server/ThriftServer.h>
#include <thrift/lib/cpp2/transport/rocket/server/RocketRoutingHandler.h>
#include "query/exceptions.hpp"
#include "query/frontend/opencypher/generated/MemgraphCypher.h"
#include "query/frontend/opencypher/generated/MemgraphCypherLexer.h"
#include "query/interpret/eval.hpp"
#include "storage_service.hpp"
using apache::thrift::RocketRoutingHandler;
using apache::thrift::ThriftServer;
using apache::thrift::ThriftServerAsyncProcessorFactory;
using manual::storage::StorageServiceHandler;
std::unique_ptr<RocketRoutingHandler> createRoutingHandler(std::shared_ptr<ThriftServer> server) {
return std::make_unique<RocketRoutingHandler>(*server);
}
template <typename ServiceHandler>
std::shared_ptr<ThriftServer> createServer(memgraph::storage::Storage &db, int32_t port) {
auto handler = std::make_shared<ServiceHandler>(db);
auto proc_factory = std::make_shared<ThriftServerAsyncProcessorFactory<ServiceHandler>>(handler);
auto server = std::make_shared<ThriftServer>();
server->setPort(port);
server->setInterface(handler);
return server;
}
int main(int argc, char **argv) {
spdlog::set_level(spdlog::level::trace);
// Main storage and execution engines initialization
memgraph::storage::Config db_config{
.gc = {.type = memgraph::storage::Config::Gc::Type::PERIODIC, .interval = std::chrono::seconds(1000)},
.items = {.properties_on_edges = true},
.durability = {.storage_directory = "data",
.recover_on_startup = false,
.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT,
.snapshot_interval = std::chrono::seconds(15),
.snapshot_retention_count = 2,
.wal_file_size_kibibytes = 20480,
.wal_file_flush_every_n_tx = 1,
.snapshot_on_exit = true},
.transaction = {.isolation_level = memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION}};
memgraph::storage::Storage db(db_config);
folly::init(&argc, &argv);
auto storage_server = createServer<StorageServiceHandler>(db, 7779);
std::shared_ptr<apache::thrift::concurrency::ThreadManager> threadManager(
apache::thrift::concurrency::PriorityThreadManager::newPriorityThreadManager(8));
threadManager->setNamePrefix("executor");
threadManager->start();
storage_server->setThreadManager(threadManager);
std::cout << "alma\n";
LOG(ERROR) << "Storage Server running on port: " << 7779;
storage_server->serve();
return 0;
}

View File

@@ -0,0 +1,386 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage_service.hpp"
#include <algorithm>
#include <functional>
#include <iterator>
#include <stdexcept>
#include "interface/gen-cpp2/storage_types.h"
#include "spdlog/spdlog.h"
#include "storage/v2/id_types.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/result.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/vertex_accessor.hpp"
#include "storage/v2/view.hpp"
#include "utils/result.hpp"
namespace {
// The response in the handler functions are const, so we cannot move out from them
memgraph::storage::PropertyValue ThriftValueToPropertyValue(const interface::storage::Value &value) {
using Type = interface::storage::Value::Type;
using PropertyValue = memgraph::storage::PropertyValue;
using ThriftValue = interface::storage::Value;
switch (value.getType()) {
case Type::null_v:
return PropertyValue{};
case Type::bool_v:
return PropertyValue{value.get_bool_v()};
case Type::int_v:
return PropertyValue{value.get_int_v()};
case Type::double_v:
return PropertyValue{value.get_double_v()};
case Type::string_v:
return PropertyValue{value.get_string_v()};
case Type::list_v: {
std::vector<PropertyValue> result;
const auto &list = value.get_list_v();
result.resize(list.size());
std::transform(list.begin(), list.end(), std::back_insert_iterator(result), ThriftValueToPropertyValue);
return PropertyValue{std::move(result)};
}
case Type::map_v: {
std::map<std::string, PropertyValue> result;
const auto &map = value.get_map_v();
std::transform(map->begin(), map->end(), std::inserter(result, result.end()),
[](const std::pair<std::string, ThriftValue> &p) -> decltype(result)::value_type {
return {std::move(p.first), ThriftValueToPropertyValue(p.second)};
});
return PropertyValue{std::move(result)};
}
case Type::date_v:
case Type::local_time_v:
case Type::local_date_time_v:
case Type::duration_v:
case Type::__EMPTY__:
case Type::vertex_v:
case Type::edge_v:
case Type::path_v:
// TODO(antaljanosbenjamin): Handle temporal types, assert on entities
return PropertyValue{};
}
}
interface::storage::Value PropertyValueToThriftValue(memgraph::storage::PropertyValue &&value) {
using Type = memgraph::storage::PropertyValue::Type;
using PropertyValue = memgraph::storage::PropertyValue;
using ThriftValue = interface::storage::Value;
interface::storage::Value thrift_value {};
switch (value.type()) {
case Type::Null:
thrift_value.set_null_v();
break;
case Type::Bool:
thrift_value.set_bool_v(value.ValueBool());
break;
case Type::Int:
thrift_value.set_int_v(value.ValueInt());
break;
case Type::Double:
thrift_value.set_double_v(value.ValueDouble());
break;
case Type::String:
thrift_value.set_string_v(std::move(value.ValueString()));
break;
case Type::List: {
std::vector<ThriftValue> result;
auto list = std::move(value.ValueList());
result.resize(list.size());
std::transform(std::make_move_iterator(list.begin()), std::make_move_iterator(list.end()),
std::back_insert_iterator(result), PropertyValueToThriftValue);
thrift_value.set_list_v(std::move(result));
break;
}
case Type::Map: {
std::unordered_map<std::string, ThriftValue> result{};
auto map = std::move(value.ValueMap());
std::transform(std::make_move_iterator(map.begin()), std::make_move_iterator(map.end()),
std::inserter(result, result.end()),
[](std::pair<std::string, PropertyValue> &&p) -> decltype(result)::value_type {
return {std::move(p.first), PropertyValueToThriftValue(std::move(p.second))};
});
thrift_value.set_map_v(std::move(result));
}
case Type::TemporalData:
// TODO(antaljanosbenjamin): Handle temporal types, assert on entities
break;
}
return thrift_value;
}
static const interface::storage::Value kNullValue = std::invoke([] {
interface::storage::Value value;
value.set_null_v();
return value;
});
} // namespace
namespace manual::storage {
int64_t StorageServiceHandler::startTransaction() {
spdlog::info("Starting transaction");
static std::atomic<int64_t> counter{0};
const auto transaction_id = ++counter;
active_transactions_.insert(transaction_id, std::make_shared<memgraph::storage::Storage::Accessor>(db_.Access()));
return transaction_id;
};
void StorageServiceHandler::commitTransaction(::interface::storage::Result &result, int64_t transaction_id) {
spdlog::info("Commiting transaction");
if (auto accessor_it = active_transactions_.find(transaction_id); accessor_it != active_transactions_.end()) {
result.success_ref() = accessor_it->second->Commit().HasError();
} else {
result.success_ref() = false;
}
};
void StorageServiceHandler::abortTransaction(int64_t transaction_id) {
spdlog::info("Aborting transaction");
if (auto accessor_it = active_transactions_.find(transaction_id); accessor_it != active_transactions_.end()) {
accessor_it->second->Abort();
}
};
void StorageServiceHandler::createVertices(::interface::storage::Result &result,
const ::interface::storage::CreateVerticesRequest &req) {
spdlog::info("Creating vertex...");
result.success_ref() = false;
auto accessor = active_transactions_.at(req.get_transaction_id());
const auto &labels_map = req.get_labels_name_map();
const auto &property_names_map = req.get_property_name_map();
// auto accessor = db_.Access();
for (auto &new_vertex : *req.new_vertices_ref()) {
auto vertex = accessor->CreateVertex();
for (const auto label_id : new_vertex.get_label_ids()) {
if (const auto result = vertex.AddLabel(accessor->NameToLabel(labels_map.at(label_id))); result.HasError()) {
return;
}
}
for (auto &[prop_id, prop] : *new_vertex.properties_ref()) {
if (const auto result = vertex.SetProperty(accessor->NameToProperty(property_names_map.at(prop_id)),
ThriftValueToPropertyValue(prop));
result.HasError()) {
return;
}
}
}
spdlog::info("Vertex creation done!");
}
void StorageServiceHandler::createEdges(::interface::storage::Result &result,
const ::interface::storage::CreateEdgesRequest &req) {
spdlog::info("Creating edges...");
result.success_ref() = false;
auto accessor = active_transactions_.at(req.get_transaction_id());
const auto &property_names_map = req.get_property_name_map();
for (auto &new_edge : *req.new_edges_ref()) {
const auto src = new_edge.src;
const auto dest = new_edge.dest;
const auto type = new_edge.type.name().value();
auto from_node = accessor->FindVertex(memgraph::storage::Gid::FromInt(src), memgraph::storage::View::NEW);
if (!from_node) {
throw std::runtime_error("Source node must be in the storage");
}
auto to_node = accessor->FindVertex(memgraph::storage::Gid::FromInt(dest), memgraph::storage::View::NEW);
if (!to_node) {
throw std::runtime_error("Destination node must be in the storage");
}
auto edge = accessor->CreateEdge(&*from_node, &*to_node, accessor->NameToEdgeType(type));
for (const auto &[prop_id, prop] : new_edge.get_properties()) {
if (const auto result = edge->SetProperty(accessor->NameToProperty(property_names_map.at(prop_id)),
ThriftValueToPropertyValue(prop));
result.HasError()) {
return;
}
}
}
spdlog::info("Edges creation done!");
}
void StorageServiceHandler::updateVertices(::interface::storage::Result &result,
const ::interface::storage::UpdateVerticesRequest &req) {
spdlog::info("Updating vertices...");
result.success_ref() = false;
auto accessor = active_transactions_.at(req.get_transaction_id());
const auto &labels_map = req.get_labels_name_map();
const auto &property_names_map = req.get_property_name_map();
for (const auto &update_vertex : req.get_update_vertices()) {
auto vertex =
accessor->FindVertex(memgraph::storage::Gid::FromInt(update_vertex.vertex_id), memgraph::storage::View::NEW);
if (!vertex) {
throw std::runtime_error("Vertex not found in storage");
}
for (const auto label_id : update_vertex.get_create_label_ids()) {
if (const auto result = vertex->AddLabel(accessor->NameToLabel(labels_map.at(label_id))); result.HasError()) {
return;
}
}
for (const auto label_id : update_vertex.get_delete_label_ids()) {
if (const auto result = vertex->RemoveLabel(accessor->NameToLabel(labels_map.at(label_id))); result.HasError()) {
return;
}
}
for (auto &[prop_id, prop] : *update_vertex.properties_ref()) {
if (const auto result = vertex->SetProperty(accessor->NameToProperty(property_names_map.at(prop_id)),
ThriftValueToPropertyValue(prop));
result.HasError()) {
return;
}
}
}
spdlog::info("Updating vertices done!");
}
static_assert(sizeof(apache::thrift::optional_field_ref<interface::storage::Values &>) > 16);
std::function<memgraph::utils::BasicResult<std::string>(memgraph::storage::VertexAccessor)> CreateVertexProcessor(
memgraph::storage::Storage::Accessor &db_accessor, interface::storage::Values &values,
apache::thrift::optional_field_ref<std::unordered_map<int64_t, std::string> &> property_name_map_ref,
const apache::thrift::optional_field_ref<const std::vector<::std::string> &> &props_to_return_ref,
const memgraph::storage::View view) {
if (!props_to_return_ref.has_value()) {
values.set_mapped();
auto &mapped_values = values.mutable_mapped();
auto result_props_ref = mapped_values.properties_ref();
property_name_map_ref.ensure();
auto &property_name_map = *property_name_map_ref;
static constexpr auto vertex_id_id = 1L;
auto internal_id_to_thrift_id = [&db_accessor, &property_name_map,
cache = std::unordered_map<memgraph::storage::PropertyId, int64_t>{},
next_id = vertex_id_id + 1l](memgraph::storage::PropertyId prop_id) mutable {
if (const auto it = cache.find(prop_id); it != cache.end()) {
return it->second;
}
const auto &name = db_accessor.PropertyToName(prop_id);
property_name_map.emplace(next_id, name);
cache.emplace(prop_id, next_id);
return next_id++;
};
return [result_props_ref = std::move(result_props_ref), view,
internal_id_to_thrift_id = std::move(internal_id_to_thrift_id)](
memgraph::storage::VertexAccessor vertex_acc) mutable -> memgraph::utils::BasicResult<std::string> {
auto props_res = vertex_acc.Properties(view);
if (props_res.HasError()) {
// TODO(antaljanosbenjamin): More fine grained error handling
return std::string{"Vertex deleted"};
}
interface::storage::ValuesMap values_map;
auto row_ref = values_map.values_map_ref();
interface::storage::Value vertex_id;
vertex_id.set_int_v(vertex_acc.Gid().AsInt());
row_ref->reserve(props_res->size() + 1);
row_ref->emplace(vertex_id_id, std::move(vertex_id));
for (auto &[prop_id, prop] : *props_res) {
const auto thrift_id = internal_id_to_thrift_id(prop_id);
row_ref->emplace(thrift_id, PropertyValueToThriftValue(std::move(prop)));
}
result_props_ref->push_back(std::move(values_map));
return {};
};
}
values.set_listed({});
auto &listed_values = values.mutable_listed();
auto result_props_ref = listed_values.properties_ref();
if (props_to_return_ref->empty()) {
// Return only the vertex ids
return [result_props_ref = std::move(result_props_ref),
view](memgraph::storage::VertexAccessor vertex_acc) mutable -> memgraph::utils::BasicResult<std::string> {
if (!vertex_acc.IsVisible(view)) {
return {};
}
auto &values_list = result_props_ref->emplace_back();
auto &value = values_list.emplace_back();
value.set_int_v(vertex_acc.Gid().AsInt());
return {};
};
}
// Return properties in order
std::vector<memgraph::storage::PropertyId> prop_ids_to_return{};
prop_ids_to_return.reserve(props_to_return_ref->size());
std::transform(props_to_return_ref->begin(), props_to_return_ref->end(), std::back_inserter(prop_ids_to_return),
std::bind_front(&memgraph::storage::Storage::Accessor::NameToProperty, &db_accessor));
return [result_props_ref = std::move(result_props_ref), prop_ids_to_return = std::move(prop_ids_to_return),
view](memgraph::storage::VertexAccessor vertex_acc) mutable -> memgraph::utils::BasicResult<std::string> {
if (!vertex_acc.IsVisible(view)) {
return {};
}
std::vector<interface::storage::Value> row{};
row.reserve(prop_ids_to_return.size() + 1);
{
interface::storage::Value vertex_id;
vertex_id.set_int_v(vertex_acc.Gid().AsInt());
row.push_back(std::move(vertex_id));
}
for (const auto &prop_id : prop_ids_to_return) {
auto property_result = vertex_acc.GetProperty(prop_id, view);
if (property_result.HasError()) {
// TODO(antaljanosbenjamin): More fine grained error handling
return std::string{"Vertex deleted"};
}
row.push_back(PropertyValueToThriftValue(std::move(property_result.GetValue())));
}
result_props_ref->push_back(std::move(row));
return {};
};
}
void StorageServiceHandler::scanVertices(::interface::storage::ScanVerticesResponse &resp,
const ::interface::storage::ScanVerticesRequest &req) {
resp.result_ref()->success_ref() = false;
auto accessor = active_transactions_.at(req.get_transaction_id());
// TODO(antaljanosbenjamin): handle filter
const auto view = req.get_view();
auto vertices = accessor->Vertices(view);
auto it = vertices.begin();
if (const auto *start_id = req.get_start_id(); start_id != nullptr) {
it = vertices.IterateFrom(memgraph::storage::Gid::FromInt(*start_id));
}
auto count = 0;
auto vertex_processor = CreateVertexProcessor(*accessor, *resp.values(), resp.property_name_map_ref(),
req.props_to_return_ref(), req.get_view());
const auto limit = std::invoke([&]() -> int64_t {
if (req.limit_ref().has_value()) {
return *req.limit_ref();
}
return 50;
});
while (count < limit && it != vertices.end()) {
if (const auto res = vertex_processor(*it); res.HasError()) {
throw std::runtime_error{res.GetError()};
};
++it;
++count;
}
if (it != vertices.end()) {
resp.next_start_id_ref() = (*it).Gid().AsInt();
}
resp.result_ref()->success_ref() = true;
}
} // namespace manual::storage

View File

@@ -0,0 +1,48 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include <folly/concurrency/ConcurrentHashMap.h>
#include "interface/gen-cpp2/Storage.h"
#include "interface/gen-cpp2/storage_types.h"
#include "storage/v2/storage.hpp"
namespace manual::storage {
// TODO(antaljanosbenjamin):
// - Check out different approaches of how Thrift message members can be read/write
class StorageServiceHandler final : public interface::storage::StorageSvIf {
public:
explicit StorageServiceHandler(memgraph::storage::Storage &db) : db_{db} {}
int64_t startTransaction() override;
void commitTransaction(::interface::storage::Result &result, int64_t transaction_id) override;
void abortTransaction(int64_t transaction_id) override;
void createVertices(::interface::storage::Result &result,
const ::interface::storage::CreateVerticesRequest &req) override;
void updateVertices(::interface::storage::Result &result,
const ::interface::storage::UpdateVerticesRequest &req) override;
void createEdges(::interface::storage::Result &result, const ::interface::storage::CreateEdgesRequest &req) override;
void scanVertices(::interface::storage::ScanVerticesResponse &resp,
const ::interface::storage::ScanVerticesRequest &req) override;
private:
memgraph::storage::Storage &db_;
folly::ConcurrentHashMap<int64_t, std::shared_ptr<memgraph::storage::Storage::Accessor>> active_transactions_;
};
} // namespace manual::storage