Compare commits
2 Commits
add-gnuplo
...
MG-determi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18edf17496 | ||
|
|
eeb7ab6272 |
2
.github/workflows/diff.yaml
vendored
2
.github/workflows/diff.yaml
vendored
@@ -99,7 +99,7 @@ jobs:
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
python3 -m black --check --diff ${file}
|
||||
python3 -m isort --check-only --profile "black" --diff ${file}
|
||||
python3 -m isort --check-only --diff ${file}
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ repos:
|
||||
hooks:
|
||||
- id: isort
|
||||
name: isort (python)
|
||||
args: ["--profile", "black"]
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v13.0.0
|
||||
hooks:
|
||||
|
||||
181
CMakeLists.txt
181
CMakeLists.txt
@@ -4,7 +4,6 @@ cmake_minimum_required(VERSION 3.8)
|
||||
|
||||
# !! IMPORTANT !! run ./project_root/init.sh before cmake command
|
||||
# to download dependencies
|
||||
|
||||
if(NOT UNIX)
|
||||
message(FATAL_ERROR "Unsupported operating system.")
|
||||
endif()
|
||||
@@ -19,6 +18,7 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM TRUE)
|
||||
find_program(CCACHE_FOUND ccache)
|
||||
option(USE_CCACHE "ccache:" ON)
|
||||
message(STATUS "CCache: ${USE_CCACHE}")
|
||||
|
||||
if(CCACHE_FOUND AND USE_CCACHE)
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
|
||||
@@ -28,7 +28,8 @@ endif(CCACHE_FOUND AND USE_CCACHE)
|
||||
# NOTE: must be choosen before use of project() or enable_language()
|
||||
find_program(CLANG_FOUND clang)
|
||||
find_program(CLANGXX_FOUND clang++)
|
||||
if (CLANG_FOUND AND CLANGXX_FOUND)
|
||||
|
||||
if(CLANG_FOUND AND CLANGXX_FOUND)
|
||||
set(CMAKE_C_COMPILER ${CLANG_FOUND})
|
||||
set(CMAKE_CXX_COMPILER ${CLANGXX_FOUND})
|
||||
else()
|
||||
@@ -36,12 +37,11 @@ else()
|
||||
endif()
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
project(memgraph)
|
||||
|
||||
# Install licenses.
|
||||
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
|
||||
DESTINATION share/doc/memgraph)
|
||||
DESTINATION share/doc/memgraph)
|
||||
|
||||
# For more information about how to release a new version of Memgraph, see
|
||||
# `release/README.md`.
|
||||
@@ -61,61 +61,65 @@ set(MEMGRAPH_OVERRIDE_VERSION "")
|
||||
set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "")
|
||||
|
||||
# Variables used to generate the versions.
|
||||
if (MG_ENTERPRISE)
|
||||
if(MG_ENTERPRISE)
|
||||
set(get_version_offering "")
|
||||
else()
|
||||
set(get_version_offering "--open-source")
|
||||
endif()
|
||||
|
||||
set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
|
||||
|
||||
# Get version that should be used in the binary.
|
||||
execute_process(
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
|
||||
message(FATAL_ERROR "Unable to get Memgraph version.")
|
||||
message(FATAL_ERROR "Unable to get Memgraph version.")
|
||||
else()
|
||||
MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
|
||||
MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
|
||||
endif()
|
||||
|
||||
# Get version that should be used in the DEB package.
|
||||
execute_process(
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant deb
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant deb
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
|
||||
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
|
||||
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
|
||||
else()
|
||||
MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
|
||||
MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
|
||||
endif()
|
||||
|
||||
# Get version that should be used in the RPM package.
|
||||
execute_process(
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant rpm
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
|
||||
RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
|
||||
COMMAND "${get_version_script}" ${get_version_offering}
|
||||
--variant rpm
|
||||
"${MEMGRAPH_OVERRIDE_VERSION}"
|
||||
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
|
||||
"--memgraph-root-dir"
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}"
|
||||
)
|
||||
|
||||
if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
|
||||
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
|
||||
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
|
||||
else()
|
||||
MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
|
||||
MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
|
||||
endif()
|
||||
|
||||
# We want the above variables to be updated each time something is committed to
|
||||
@@ -135,22 +139,24 @@ endif()
|
||||
# unnecessary recalculations of the release version. The release version only
|
||||
# changes on every `git commit` or `git checkout`. That is why we watch the
|
||||
# following files for changes:
|
||||
# - `.git/HEAD` -> changes each time a `git checkout` is issued
|
||||
# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
|
||||
# you are on a branch) and you have to monitor the file of the specific
|
||||
# branch to detect when a `git commit` was issued
|
||||
# - `.git/HEAD` -> changes each time a `git checkout` is issued
|
||||
# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
|
||||
# you are on a branch) and you have to monitor the file of the specific
|
||||
# branch to detect when a `git commit` was issued
|
||||
# More details about the contents of the `.git` directory and the specific
|
||||
# files used can be seen here:
|
||||
# https://git-scm.com/book/en/v2/Git-Internals-Git-References
|
||||
set(git_directory "${CMAKE_SOURCE_DIR}/.git")
|
||||
if (EXISTS "${git_directory}")
|
||||
|
||||
if(EXISTS "${git_directory}")
|
||||
set_property(DIRECTORY APPEND PROPERTY
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
|
||||
file(STRINGS "${git_directory}/HEAD" git_head_data)
|
||||
if (git_head_data MATCHES "^ref: ")
|
||||
|
||||
if(git_head_data MATCHES "^ref: ")
|
||||
string(SUBSTRING "${git_head_data}" 5 -1 git_head_ref)
|
||||
set_property(DIRECTORY APPEND PROPERTY
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
|
||||
CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -159,16 +165,19 @@ endif()
|
||||
# setup CMake module path, defines path for include() and find_package()
|
||||
# https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake)
|
||||
|
||||
# custom function definitions
|
||||
include(functions)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# We want out of source builds, so that cmake generated files don't get mixed
|
||||
# with source files. This allows for easier clean up.
|
||||
disallow_in_source_build()
|
||||
add_custom_target(clean_all
|
||||
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
|
||||
COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
|
||||
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
|
||||
COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# build flags -----------------------------------------------------------------
|
||||
@@ -179,18 +188,19 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 20)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
# c99-designator is disabled because of required mixture of designated and
|
||||
# non-designated initializers in Python Query Module code (`py_module.cpp`).
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
|
||||
-Werror=switch -Werror=switch-bool -Werror=implicit-fallthrough \
|
||||
-Werror=return-type \
|
||||
-Werror=switch -Werror=switch-bool -Werror=return-type \
|
||||
-Werror=return-stack-address \
|
||||
-Wno-c99-designator \
|
||||
-DBOOST_STACKTRACE_USE_ADDR2LINE \
|
||||
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
|
||||
|
||||
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
|
||||
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
|
||||
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
|
||||
|
||||
# Statically link libgcc and libstdc++, the GCC allows this according to:
|
||||
# https://gcc.gnu.org/onlinedocs/gcc-10.2.0/libstdc++/manual/manual/license.html
|
||||
@@ -208,42 +218,47 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
|
||||
|
||||
SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
|
||||
|
||||
#debug flags
|
||||
# debug flags
|
||||
set(PREFERRED_DEBUGGER "gdb" CACHE STRING
|
||||
"Tunes the debug output for your preferred debugger (gdb or lldb).")
|
||||
if ("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
|
||||
elseif ("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
|
||||
"Tunes the debug output for your preferred debugger (gdb or lldb).")
|
||||
|
||||
if("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
|
||||
elseif("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
|
||||
"${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
|
||||
else()
|
||||
message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
|
||||
"'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g")
|
||||
message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
|
||||
"'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "-g")
|
||||
endif()
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# default build type is debug
|
||||
if (NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE "Debug")
|
||||
if(NOT CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE "Debug")
|
||||
endif()
|
||||
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
|
||||
|
||||
# setup external dependencies -------------------------------------------------
|
||||
|
||||
# threading
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# optional readline
|
||||
option(USE_READLINE "Use GNU Readline library if available (default ON). \
|
||||
Set this to OFF to prevent linking with Readline even if it is available." ON)
|
||||
if (USE_READLINE)
|
||||
|
||||
if(USE_READLINE)
|
||||
find_package(Readline)
|
||||
if (READLINE_FOUND)
|
||||
|
||||
if(READLINE_FOUND)
|
||||
add_definitions(-DHAS_READLINE)
|
||||
endif()
|
||||
endif()
|
||||
@@ -259,24 +274,27 @@ option(ASAN "Build with Address Sanitizer. To get a reasonable performance optio
|
||||
option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
|
||||
option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
|
||||
|
||||
if (TEST_COVERAGE)
|
||||
if(TEST_COVERAGE)
|
||||
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
if (NOT lower_build_type STREQUAL "debug")
|
||||
|
||||
if(NOT lower_build_type STREQUAL "debug")
|
||||
message(FATAL_ERROR "Generating test coverage unsupported in non Debug builds. Current build type is '${CMAKE_BUILD_TYPE}'")
|
||||
endif()
|
||||
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
|
||||
endif()
|
||||
|
||||
if (MG_ENTERPRISE)
|
||||
if(MG_ENTERPRISE)
|
||||
add_definitions(-DMG_ENTERPRISE)
|
||||
endif()
|
||||
|
||||
set(ENABLE_JEMALLOC ON)
|
||||
|
||||
if (ASAN)
|
||||
if(ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
set(ENABLE_JEMALLOC OFF)
|
||||
|
||||
# Enable Addres sanitizer and get nicer stack traces in error messages.
|
||||
# NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
|
||||
# distribution to symbolize the stack traces (note that ideally the
|
||||
@@ -285,19 +303,20 @@ if (ASAN)
|
||||
# provide it in separate ASAN_SYMBOLIZER_PATH environment variable.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
|
||||
|
||||
# To detect Stack-use-after-return bugs set run-time flag:
|
||||
# ASAN_OPTIONS=detect_stack_use_after_return=1
|
||||
# ASAN_OPTIONS=detect_stack_use_after_return=1
|
||||
# To check initialization order bugs set run-time flag:
|
||||
# ASAN_OPTIONS=check_initialization_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# dynamically initialized global from another translation unit, which is
|
||||
# not yet initialized
|
||||
# ASAN_OPTIONS=strict_init_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# any dynamically initialized global from another translation unit.
|
||||
# ASAN_OPTIONS=check_initialization_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# dynamically initialized global from another translation unit, which is
|
||||
# not yet initialized
|
||||
# ASAN_OPTIONS=strict_init_order=true
|
||||
# This mode reports an error if initializer for a global variable accesses
|
||||
# any dynamically initialized global from another translation unit.
|
||||
endif()
|
||||
|
||||
if (TSAN)
|
||||
if(TSAN)
|
||||
# ThreadSanitizer generally requires all code to be compiled with -fsanitize=thread.
|
||||
# If some code (e.g. dynamic libraries) is not compiled with the flag, it can
|
||||
# lead to false positive race reports, false negative race reports and/or
|
||||
@@ -310,19 +329,21 @@ if (TSAN)
|
||||
# (unwinding stack on each memory access is too expensive).
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
|
||||
|
||||
# By default ThreadSanitizer uses addr2line utility to symbolize reports.
|
||||
# llvm-symbolizer is faster, consumes less memory and produces much better
|
||||
# reports. To use it set runtime flag:
|
||||
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
|
||||
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
|
||||
# For more runtime flags see: https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
|
||||
endif()
|
||||
|
||||
if (UBSAN)
|
||||
if(UBSAN)
|
||||
# Compile with UBSAN but disable vptr check. This is disabled because it
|
||||
# requires linking with clang++ to make sure C++ specific parts of the
|
||||
# runtime library and c++ standard libraries are present.
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
|
||||
|
||||
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
|
||||
# Make sure llvm-symbolizer binary is in path.
|
||||
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
|
||||
@@ -341,7 +362,7 @@ add_subdirectory(release)
|
||||
option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
|
||||
message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
|
||||
|
||||
if (MG_ENABLE_TESTING)
|
||||
if(MG_ENABLE_TESTING)
|
||||
enable_testing()
|
||||
add_subdirectory(tests)
|
||||
endif()
|
||||
|
||||
@@ -171,7 +171,7 @@ benchmark_tag="v1.6.0"
|
||||
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
|
||||
|
||||
# google test
|
||||
googletest_tag="release-1.12.1"
|
||||
googletest_tag="release-1.8.0"
|
||||
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
|
||||
|
||||
# libbcrypt
|
||||
|
||||
@@ -21,7 +21,6 @@ add_subdirectory(auth)
|
||||
add_subdirectory(parser)
|
||||
add_subdirectory(expr)
|
||||
add_subdirectory(coordinator)
|
||||
add_subdirectory(functions)
|
||||
|
||||
if (MG_ENTERPRISE)
|
||||
add_subdirectory(audit)
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string_view>
|
||||
|
||||
namespace memgraph::common {
|
||||
|
||||
enum class ErrorCode : uint8_t {
|
||||
SERIALIZATION_ERROR,
|
||||
NONEXISTENT_OBJECT,
|
||||
DELETED_OBJECT,
|
||||
VERTEX_HAS_EDGES,
|
||||
PROPERTIES_DISABLED,
|
||||
VERTEX_ALREADY_INSERTED,
|
||||
// Schema Violations
|
||||
SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL,
|
||||
SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
|
||||
SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
|
||||
SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
|
||||
SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
|
||||
SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
|
||||
|
||||
OBJECT_NOT_FOUND,
|
||||
};
|
||||
|
||||
constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
|
||||
switch (code) {
|
||||
case ErrorCode::SERIALIZATION_ERROR:
|
||||
return "SERIALIZATION_ERROR";
|
||||
case ErrorCode::NONEXISTENT_OBJECT:
|
||||
return "NONEXISTENT_OBJECT";
|
||||
case ErrorCode::DELETED_OBJECT:
|
||||
return "DELETED_OBJECT";
|
||||
case ErrorCode::VERTEX_HAS_EDGES:
|
||||
return "VERTEX_HAS_EDGES";
|
||||
case ErrorCode::PROPERTIES_DISABLED:
|
||||
return "PROPERTIES_DISABLED";
|
||||
case ErrorCode::VERTEX_ALREADY_INSERTED:
|
||||
return "VERTEX_ALREADY_INSERTED";
|
||||
case ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
|
||||
return "SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL";
|
||||
case ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
|
||||
return "SCHEMA_VERTEX_PROPERTY_WRONG_TYPE";
|
||||
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
|
||||
return "SCHEMA_VERTEX_UPDATE_PRIMARY_KEY";
|
||||
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
|
||||
return "SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL";
|
||||
case ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
return "SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY";
|
||||
case ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
|
||||
case ErrorCode::OBJECT_NOT_FOUND:
|
||||
return "OBJECT_NOT_FOUND";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace memgraph::common
|
||||
@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
|
||||
}
|
||||
case Signature::Route: {
|
||||
if constexpr (bolt_minor >= 3) {
|
||||
return HandleRoute<TSession>(session);
|
||||
if (signature == Signature::Route) return HandleRoute<TSession>(session);
|
||||
} else {
|
||||
spdlog::trace("Supported only in bolt v4.3");
|
||||
return State::Close;
|
||||
|
||||
@@ -71,9 +71,6 @@ struct QueueInner {
|
||||
// starvation by sometimes randomizing priorities, rather than following a strict
|
||||
// prioritization.
|
||||
std::deque<Message> queue;
|
||||
|
||||
uint64_t submitted = 0;
|
||||
uint64_t calls_to_pop = 0;
|
||||
};
|
||||
|
||||
/// There are two reasons to implement our own Queue instead of using
|
||||
@@ -89,8 +86,6 @@ class Queue {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->submitted++;
|
||||
|
||||
inner_->queue.emplace_back(std::move(message));
|
||||
} // lock dropped before notifying condition variable
|
||||
|
||||
@@ -101,9 +96,6 @@ class Queue {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->calls_to_pop++;
|
||||
inner_->cv.notify_all();
|
||||
|
||||
while (inner_->queue.empty()) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
@@ -113,15 +105,6 @@ class Queue {
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
void BlockOnQuiescence() const {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
while (inner_->calls_to_pop <= inner_->submitted) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
|
||||
@@ -146,7 +129,9 @@ class CoordinatorWorker {
|
||||
|
||||
public:
|
||||
CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
|
||||
: io_(std::move(io)), queue_(std::move(queue)), coordinator_{std::move(io_), {}, std::move(coordinator)} {}
|
||||
: io_(std::move(io)),
|
||||
queue_(std::move(queue)),
|
||||
coordinator_{std::move(io_.ForkLocal()), {}, std::move(coordinator)} {}
|
||||
|
||||
CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
|
||||
CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
|
||||
@@ -155,12 +140,15 @@ class CoordinatorWorker {
|
||||
~CoordinatorWorker() = default;
|
||||
|
||||
void Run() {
|
||||
bool should_continue = true;
|
||||
while (should_continue) {
|
||||
while (true) {
|
||||
Message message = queue_.Pop();
|
||||
|
||||
should_continue = std::visit([this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); },
|
||||
std::move(message));
|
||||
const bool should_continue = std::visit(
|
||||
[this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); }, std::move(message));
|
||||
|
||||
if (!should_continue) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -228,7 +228,7 @@ Hlc ShardMap::IncrementShardMapVersion() noexcept {
|
||||
return shard_map_version;
|
||||
}
|
||||
|
||||
// TODO(antaljanosbenjamin) use a single map for all name id
|
||||
// TODO(antaljanosbenjamin) use a single map for all name id
|
||||
// mapping and a single counter to maintain the next id
|
||||
std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
@@ -248,25 +248,6 @@ std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
|
||||
|
||||
Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
|
||||
|
||||
boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
|
||||
return boost::uuids::uuid{0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
static_cast<unsigned char>(shard_id >> 56U),
|
||||
static_cast<unsigned char>(shard_id >> 48U),
|
||||
static_cast<unsigned char>(shard_id >> 40U),
|
||||
static_cast<unsigned char>(shard_id >> 32U),
|
||||
static_cast<unsigned char>(shard_id >> 24U),
|
||||
static_cast<unsigned char>(shard_id >> 16U),
|
||||
static_cast<unsigned char>(shard_id >> 8U),
|
||||
static_cast<unsigned char>(shard_id)};
|
||||
}
|
||||
|
||||
std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
|
||||
std::set<boost::uuids::uuid> initialized) {
|
||||
std::vector<ShardToInitialize> ret{};
|
||||
@@ -283,11 +264,10 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
|
||||
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
|
||||
bool machine_contains_shard = false;
|
||||
|
||||
for (auto &aas : shard.peers) {
|
||||
for (auto &aas : shard) {
|
||||
if (initialized.contains(aas.address.unique_id)) {
|
||||
machine_contains_shard = true;
|
||||
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
mutated = true;
|
||||
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
|
||||
aas.status = Status::CONSENSUS_PARTICIPANT;
|
||||
}
|
||||
@@ -311,14 +291,11 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
|
||||
}
|
||||
}
|
||||
|
||||
if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
|
||||
// increment version for each new uuid for deterministic creation
|
||||
IncrementShardMapVersion();
|
||||
|
||||
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
|
||||
Address address = storage_manager;
|
||||
|
||||
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
|
||||
address.unique_id = NewShardUuid(shard_map_version.logical_id);
|
||||
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
|
||||
spdlog::info("assigning shard manager to shard");
|
||||
|
||||
@@ -337,7 +314,7 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
|
||||
.status = Status::INITIALIZING,
|
||||
};
|
||||
|
||||
shard.peers.emplace_back(aas);
|
||||
shard.emplace_back(aas);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -360,9 +337,9 @@ bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, cons
|
||||
MG_ASSERT(!shards_in_map.contains(key));
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
// Finding the ShardMetadata that the new PrimaryKey should map to.
|
||||
// Finding the Shard that the new PrimaryKey should map to.
|
||||
auto prev = std::prev(shards_in_map.upper_bound(key));
|
||||
ShardMetadata duplicated_shard = prev->second;
|
||||
Shard duplicated_shard = prev->second;
|
||||
|
||||
// Apply the split
|
||||
shards_in_map[key] = duplicated_shard;
|
||||
@@ -383,7 +360,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
|
||||
labels.emplace(std::move(label_name), label_id);
|
||||
|
||||
PrimaryKey initial_key = SchemaToMinKey(schema);
|
||||
ShardMetadata empty_shard = {};
|
||||
Shard empty_shard = {};
|
||||
|
||||
Shards shards = {
|
||||
{initial_key, empty_shard},
|
||||
@@ -406,7 +383,6 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
|
||||
void ShardMap::AddServer(Address server_address) {
|
||||
// Find a random place for the server to plug in
|
||||
}
|
||||
|
||||
std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
|
||||
if (const auto it = labels.find(label); it != labels.end()) {
|
||||
return it->second;
|
||||
@@ -479,7 +455,7 @@ Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey
|
||||
return shards;
|
||||
}
|
||||
|
||||
ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
|
||||
Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
LabelId label_id = labels.at(label_name);
|
||||
@@ -492,7 +468,7 @@ ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const Primar
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
|
||||
Shard ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
@@ -556,12 +532,12 @@ EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new
|
||||
bool ShardMap::ClusterInitialized() const {
|
||||
for (const auto &[label_id, label_space] : label_spaces) {
|
||||
for (const auto &[low_key, shard] : label_space.shards) {
|
||||
if (shard.peers.size() < label_space.replication_factor) {
|
||||
if (shard.size() < label_space.replication_factor) {
|
||||
spdlog::info("label_space below desired replication factor");
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto &aas : shard.peers) {
|
||||
for (const auto &aas : shard) {
|
||||
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
|
||||
return false;
|
||||
|
||||
@@ -76,35 +76,8 @@ struct AddressAndStatus {
|
||||
};
|
||||
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
|
||||
struct ShardMetadata {
|
||||
std::vector<AddressAndStatus> peers;
|
||||
uint64_t version;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
|
||||
using utils::print_helpers::operator<<;
|
||||
|
||||
in << "ShardMetadata { peers: ";
|
||||
in << shard.peers;
|
||||
in << " version: ";
|
||||
in << shard.version;
|
||||
in << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
|
||||
friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
|
||||
|
||||
friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
|
||||
if (lhs.peers != rhs.peers) {
|
||||
return lhs.peers < rhs.peers;
|
||||
}
|
||||
|
||||
return lhs.version < rhs.version;
|
||||
}
|
||||
};
|
||||
|
||||
using Shards = std::map<PrimaryKey, ShardMetadata>;
|
||||
using Shard = std::vector<AddressAndStatus>;
|
||||
using Shards = std::map<PrimaryKey, Shard>;
|
||||
using LabelName = std::string;
|
||||
using PropertyName = std::string;
|
||||
using EdgeTypeName = std::string;
|
||||
@@ -126,7 +99,7 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
|
||||
struct LabelSpace {
|
||||
std::vector<SchemaProperty> schema;
|
||||
// Maps between the smallest primary key stored in the shard and the shard
|
||||
std::map<PrimaryKey, ShardMetadata> shards;
|
||||
std::map<PrimaryKey, Shard> shards;
|
||||
size_t replication_factor;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
|
||||
@@ -187,9 +160,9 @@ struct ShardMap {
|
||||
|
||||
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
|
||||
|
||||
ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
|
||||
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
|
||||
|
||||
ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
|
||||
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
|
||||
|
||||
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
|
||||
|
||||
|
||||
@@ -17,4 +17,4 @@ target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
|
||||
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser mg-functions)
|
||||
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
#include "expr/exceptions.hpp"
|
||||
#include "expr/interpret/frame.hpp"
|
||||
#include "expr/semantic/symbol_table.hpp"
|
||||
#include "functions/awesome_memgraph_functions.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
@@ -36,8 +35,8 @@ template <typename TypedValue, typename EvaluationContext, typename DbAccessor,
|
||||
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
|
||||
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
public:
|
||||
ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
|
||||
StorageView view)
|
||||
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
|
||||
DbAccessor *dba, StorageView view)
|
||||
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
|
||||
|
||||
using ExpressionVisitor<TypedValue>::Visit;
|
||||
@@ -101,28 +100,6 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
#undef BINARY_OPERATOR_VISITOR
|
||||
#undef UNARY_OPERATOR_VISITOR
|
||||
|
||||
void HandleObjectAccessError(Error &shard_error, const std::string_view accessed_object) {
|
||||
switch (shard_error) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access {} on a deleted object.", accessed_object);
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access {} from a node object doesn't exist.", accessed_object);
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
case Error::OBJECT_NOT_FOUND:
|
||||
throw ExpressionRuntimeException("Unexpected error when accessing {}.", accessed_object);
|
||||
case Error::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
|
||||
case Error::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
|
||||
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
|
||||
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
|
||||
case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(AndOperator &op) override {
|
||||
auto value1 = op.expression1_->Accept(*this);
|
||||
if (value1.IsBool() && !value1.ValueBool()) {
|
||||
@@ -419,7 +396,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
|
||||
}
|
||||
if (has_label.HasError()) {
|
||||
HandleObjectAccessError(has_label.GetError().code, "labels");
|
||||
switch (has_label.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
|
||||
}
|
||||
}
|
||||
return *has_label;
|
||||
}
|
||||
@@ -428,7 +415,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
|
||||
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
|
||||
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
|
||||
return vertex.HasLabel(label);
|
||||
auto has_label = vertex.HasLabel(label);
|
||||
return !has_label;
|
||||
}
|
||||
|
||||
TypedValue Visit(LabelsTest &labels_test) override {
|
||||
@@ -491,7 +479,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
|
||||
TypedValue Visit(Function &function) override {
|
||||
functions::FunctionContext<DbAccessor> function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
|
||||
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
|
||||
// Stack allocate evaluated arguments when there's a small number of them.
|
||||
if (function.arguments_.size() <= 8) {
|
||||
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
|
||||
@@ -756,7 +744,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
|
||||
}
|
||||
if (maybe_prop.HasError()) {
|
||||
HandleObjectAccessError(maybe_prop.GetError().code, "property");
|
||||
switch (maybe_prop.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
throw ExpressionRuntimeException("Unexpected error when getting a property.");
|
||||
}
|
||||
}
|
||||
return conv_(*maybe_prop, ctx_->memory);
|
||||
}
|
||||
@@ -775,14 +773,24 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
|
||||
}
|
||||
if (maybe_prop.HasError()) {
|
||||
HandleObjectAccessError(maybe_prop.GetError().code, "property");
|
||||
switch (maybe_prop.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
throw ExpressionRuntimeException("Unexpected error when getting a property.");
|
||||
}
|
||||
}
|
||||
return conv_(*maybe_prop, ctx_->memory);
|
||||
}
|
||||
|
||||
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
|
||||
|
||||
Frame *frame_;
|
||||
Frame<TypedValue> *frame_;
|
||||
const SymbolTable *symbol_table_;
|
||||
const EvaluationContext *ctx_;
|
||||
DbAccessor *dba_;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -20,12 +20,13 @@
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
template <typename TypedValue>
|
||||
class Frame {
|
||||
public:
|
||||
/// Create a Frame of given size backed by a utils::NewDeleteResource()
|
||||
explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
|
||||
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
|
||||
|
||||
Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
|
||||
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
|
||||
|
||||
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
|
||||
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
|
||||
@@ -34,7 +35,6 @@ class Frame {
|
||||
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
|
||||
|
||||
auto &elems() { return elems_; }
|
||||
const auto &elems() const { return elems_; }
|
||||
|
||||
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
|
||||
|
||||
@@ -42,18 +42,4 @@ class Frame {
|
||||
utils::pmr::vector<TypedValue> elems_;
|
||||
};
|
||||
|
||||
class FrameWithValidity final : public Frame {
|
||||
public:
|
||||
explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {}
|
||||
|
||||
FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
|
||||
|
||||
bool IsValid() const noexcept { return is_valid_; }
|
||||
void MakeValid() noexcept { is_valid_ = true; }
|
||||
void MakeInvalid() noexcept { is_valid_ = false; }
|
||||
|
||||
private:
|
||||
bool is_valid_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::expr
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
add_library(mg-functions INTERFACE)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,13 +15,13 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "common/errors.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "storage/v3/edge_accessor.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
@@ -71,101 +71,106 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
|
||||
}
|
||||
}
|
||||
|
||||
communication::bolt::Vertex ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex,
|
||||
const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View /*view*/) {
|
||||
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(
|
||||
const query::v2::accessors::VertexAccessor &vertex, const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View /*view*/) {
|
||||
auto id = communication::bolt::Id::FromUint(0);
|
||||
|
||||
auto labels = vertex.Labels();
|
||||
std::vector<std::string> new_labels;
|
||||
new_labels.reserve(labels.size());
|
||||
for (const auto &label : labels) {
|
||||
new_labels.push_back(request_router->LabelToName(label.id));
|
||||
new_labels.push_back(shard_request_manager->LabelToName(label.id));
|
||||
}
|
||||
|
||||
auto properties = vertex.Properties();
|
||||
std::map<std::string, Value> new_properties;
|
||||
for (const auto &[prop, property_value] : properties) {
|
||||
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
|
||||
new_properties[shard_request_manager->PropertyToName(prop)] = ToBoltValue(property_value);
|
||||
}
|
||||
return communication::bolt::Vertex{id, new_labels, new_properties};
|
||||
}
|
||||
|
||||
communication::bolt::Edge ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge,
|
||||
const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View /*view*/) {
|
||||
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(
|
||||
const query::v2::accessors::EdgeAccessor &edge, const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View /*view*/) {
|
||||
// TODO(jbajic) Fix bolt communication
|
||||
auto id = communication::bolt::Id::FromUint(0);
|
||||
auto from = communication::bolt::Id::FromUint(0);
|
||||
auto to = communication::bolt::Id::FromUint(0);
|
||||
const auto &type = request_router->EdgeTypeToName(edge.EdgeType());
|
||||
const auto &type = shard_request_manager->EdgeTypeToName(edge.EdgeType());
|
||||
|
||||
auto properties = edge.Properties();
|
||||
std::map<std::string, Value> new_properties;
|
||||
for (const auto &[prop, property_value] : properties) {
|
||||
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
|
||||
new_properties[shard_request_manager->PropertyToName(prop)] = ToBoltValue(property_value);
|
||||
}
|
||||
return communication::bolt::Edge{id, from, to, type, new_properties};
|
||||
}
|
||||
|
||||
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path & /*edge*/,
|
||||
const query::v2::RequestRouterInterface * /*request_router*/,
|
||||
storage::v3::View /*view*/) {
|
||||
storage::v3::Result<communication::bolt::Path> ToBoltPath(
|
||||
const query::v2::accessors::Path & /*edge*/, const msgs::ShardRequestManagerInterface * /*shard_request_manager*/,
|
||||
storage::v3::View /*view*/) {
|
||||
// TODO(jbajic) Fix bolt communication
|
||||
MG_ASSERT(false, "Path is unimplemented!");
|
||||
return {};
|
||||
return {storage::v3::Error::DELETED_OBJECT};
|
||||
}
|
||||
|
||||
Value ToBoltValue(const query::v2::TypedValue &value, const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View view) {
|
||||
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value,
|
||||
const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View view) {
|
||||
switch (value.type()) {
|
||||
case query::v2::TypedValue::Type::Null:
|
||||
return {};
|
||||
return Value();
|
||||
case query::v2::TypedValue::Type::Bool:
|
||||
return {value.ValueBool()};
|
||||
return Value(value.ValueBool());
|
||||
case query::v2::TypedValue::Type::Int:
|
||||
return {value.ValueInt()};
|
||||
return Value(value.ValueInt());
|
||||
case query::v2::TypedValue::Type::Double:
|
||||
return {value.ValueDouble()};
|
||||
return Value(value.ValueDouble());
|
||||
case query::v2::TypedValue::Type::String:
|
||||
return {std::string(value.ValueString())};
|
||||
return Value(std::string(value.ValueString()));
|
||||
case query::v2::TypedValue::Type::List: {
|
||||
std::vector<Value> values;
|
||||
values.reserve(value.ValueList().size());
|
||||
for (const auto &v : value.ValueList()) {
|
||||
auto value = ToBoltValue(v, request_router, view);
|
||||
values.emplace_back(std::move(value));
|
||||
auto maybe_value = ToBoltValue(v, shard_request_manager, view);
|
||||
if (maybe_value.HasError()) return maybe_value.GetError();
|
||||
values.emplace_back(std::move(*maybe_value));
|
||||
}
|
||||
return {std::move(values)};
|
||||
return Value(std::move(values));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Map: {
|
||||
std::map<std::string, Value> map;
|
||||
for (const auto &kv : value.ValueMap()) {
|
||||
auto value = ToBoltValue(kv.second, request_router, view);
|
||||
map.emplace(kv.first, std::move(value));
|
||||
auto maybe_value = ToBoltValue(kv.second, shard_request_manager, view);
|
||||
if (maybe_value.HasError()) return maybe_value.GetError();
|
||||
map.emplace(kv.first, std::move(*maybe_value));
|
||||
}
|
||||
return {std::move(map)};
|
||||
return Value(std::move(map));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Vertex: {
|
||||
auto vertex = ToBoltVertex(value.ValueVertex(), request_router, view);
|
||||
return {std::move(vertex)};
|
||||
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), shard_request_manager, view);
|
||||
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
|
||||
return Value(std::move(*maybe_vertex));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Edge: {
|
||||
auto edge = ToBoltEdge(value.ValueEdge(), request_router, view);
|
||||
return {std::move(edge)};
|
||||
auto maybe_edge = ToBoltEdge(value.ValueEdge(), shard_request_manager, view);
|
||||
if (maybe_edge.HasError()) return maybe_edge.GetError();
|
||||
return Value(std::move(*maybe_edge));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Path: {
|
||||
auto path = ToBoltPath(value.ValuePath(), request_router, view);
|
||||
return {std::move(path)};
|
||||
auto maybe_path = ToBoltPath(value.ValuePath(), shard_request_manager, view);
|
||||
if (maybe_path.HasError()) return maybe_path.GetError();
|
||||
return Value(std::move(*maybe_path));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Date:
|
||||
return {value.ValueDate()};
|
||||
return Value(value.ValueDate());
|
||||
case query::v2::TypedValue::Type::LocalTime:
|
||||
return {value.ValueLocalTime()};
|
||||
return Value(value.ValueLocalTime());
|
||||
case query::v2::TypedValue::Type::LocalDateTime:
|
||||
return {value.ValueLocalDateTime()};
|
||||
return Value(value.ValueLocalDateTime());
|
||||
case query::v2::TypedValue::Type::Duration:
|
||||
return {value.ValueDuration()};
|
||||
return Value(value.ValueDuration());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -15,12 +15,11 @@
|
||||
#include "communication/bolt/v1/value.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
class EdgeAccessor;
|
||||
@@ -32,37 +31,40 @@ namespace memgraph::glue::v2 {
|
||||
|
||||
/// @param storage::v3::VertexAccessor for converting to
|
||||
/// communication::bolt::Vertex.
|
||||
/// @param query::v2::RequestRouterInterface *request_router getting label and property names.
|
||||
/// @param msgs::ShardRequestManagerInterface *shard_request_manager getting label and property names.
|
||||
/// @param storage::v3::View for deciding which vertex attributes are visible.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Vertex ToBoltVertex(const storage::v3::VertexAccessor &vertex,
|
||||
const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(
|
||||
const storage::v3::VertexAccessor &vertex, const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View view);
|
||||
|
||||
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
|
||||
/// @param query::v2::RequestRouterInterface *request_router getting edge type and property names.
|
||||
/// @param msgs::ShardRequestManagerInterface *shard_request_manager getting edge type and property names.
|
||||
/// @param storage::v3::View for deciding which edge attributes are visible.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Edge ToBoltEdge(const storage::v3::EdgeAccessor &edge,
|
||||
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(
|
||||
const storage::v3::EdgeAccessor &edge, const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View view);
|
||||
|
||||
/// @param query::v2::Path for converting to communication::bolt::Path.
|
||||
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
|
||||
/// @param msgs::ShardRequestManagerInterface *shard_request_manager ToBoltVertex and ToBoltEdge.
|
||||
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path &path,
|
||||
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Path> ToBoltPath(
|
||||
const query::v2::accessors::Path &path, const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View view);
|
||||
|
||||
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
|
||||
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
|
||||
/// @param msgs::ShardRequestManagerInterface *shard_request_manager ToBoltVertex and ToBoltEdge.
|
||||
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Value ToBoltValue(const query::v2::TypedValue &value,
|
||||
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Value> ToBoltValue(
|
||||
const query::v2::TypedValue &value, const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View view);
|
||||
|
||||
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
|
||||
|
||||
@@ -72,7 +74,8 @@ storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &val
|
||||
|
||||
communication::bolt::Value ToBoltValue(msgs::Value value);
|
||||
|
||||
communication::bolt::Value ToBoltValue(msgs::Value value, const query::v2::RequestRouterInterface *request_router,
|
||||
communication::bolt::Value ToBoltValue(msgs::Value value,
|
||||
const msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
storage::v3::View view);
|
||||
|
||||
} // namespace memgraph::glue::v2
|
||||
|
||||
@@ -20,8 +20,6 @@
|
||||
#include <boost/uuid/uuid_generators.hpp>
|
||||
#include <boost/uuid/uuid_io.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
struct PartialAddress {
|
||||
@@ -60,39 +58,18 @@ struct Address {
|
||||
uint16_t last_known_port;
|
||||
|
||||
static Address TestAddress(uint16_t port) {
|
||||
MG_ASSERT(port <= 255);
|
||||
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static_cast<unsigned char>(port)},
|
||||
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
.last_known_port = port,
|
||||
};
|
||||
}
|
||||
|
||||
// NB: don't use this in test code because it is non-deterministic
|
||||
static Address UniqueLocalAddress() {
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
};
|
||||
}
|
||||
|
||||
/// `Coordinator`s have constant UUIDs because there is at most one per ip/port pair.
|
||||
Address ForkLocalCoordinator() {
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
.last_known_ip = last_known_ip,
|
||||
.last_known_port = last_known_port,
|
||||
};
|
||||
}
|
||||
|
||||
/// `ShardManager`s have constant UUIDs because there is at most one per ip/port pair.
|
||||
Address ForkLocalShardManager() {
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
.last_known_ip = last_known_ip,
|
||||
.last_known_port = last_known_port,
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns a new ID with the same IP and port but a unique UUID.
|
||||
Address ForkUniqueAddress() {
|
||||
return Address{
|
||||
|
||||
@@ -35,13 +35,10 @@ class Shared {
|
||||
std::optional<T> item_;
|
||||
bool consumed_ = false;
|
||||
bool waiting_ = false;
|
||||
bool filled_ = false;
|
||||
std::function<bool()> wait_notifier_ = nullptr;
|
||||
std::function<void()> fill_notifier_ = nullptr;
|
||||
std::function<bool()> simulator_notifier_ = nullptr;
|
||||
|
||||
public:
|
||||
explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
|
||||
: wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
|
||||
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
|
||||
Shared() = default;
|
||||
Shared(Shared &&) = delete;
|
||||
Shared &operator=(Shared &&) = delete;
|
||||
@@ -67,7 +64,8 @@ class Shared {
|
||||
waiting_ = true;
|
||||
|
||||
while (!item_) {
|
||||
if (wait_notifier_) [[unlikely]] {
|
||||
bool simulator_progressed = false;
|
||||
if (simulator_notifier_) [[unlikely]] {
|
||||
// We can't hold our own lock while notifying
|
||||
// the simulator because notifying the simulator
|
||||
// involves acquiring the simulator's mutex
|
||||
@@ -79,7 +77,7 @@ class Shared {
|
||||
// so we have to get out of its way to avoid
|
||||
// a cyclical deadlock.
|
||||
lock.unlock();
|
||||
std::invoke(wait_notifier_);
|
||||
simulator_progressed = std::invoke(simulator_notifier_);
|
||||
lock.lock();
|
||||
if (item_) {
|
||||
// item may have been filled while we
|
||||
@@ -87,7 +85,8 @@ class Shared {
|
||||
// the simulator of our waiting_ status.
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
}
|
||||
if (!simulator_progressed) [[likely]] {
|
||||
cv_.wait(lock);
|
||||
}
|
||||
MG_ASSERT(!consumed_, "Future consumed twice!");
|
||||
@@ -118,19 +117,11 @@ class Shared {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
|
||||
MG_ASSERT(!filled_, "Promise filled twice!");
|
||||
MG_ASSERT(!item_, "Promise filled twice!");
|
||||
|
||||
item_ = item;
|
||||
filled_ = true;
|
||||
} // lock released before condition variable notification
|
||||
|
||||
if (fill_notifier_) {
|
||||
spdlog::trace("calling fill notifier");
|
||||
std::invoke(fill_notifier_);
|
||||
} else {
|
||||
spdlog::trace("not calling fill notifier");
|
||||
}
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
@@ -262,9 +253,8 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
|
||||
std::function<void()> fill_notifier) {
|
||||
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
|
||||
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
|
||||
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
|
||||
|
||||
Future<T> future = Future<T>(shared);
|
||||
Promise<T> promise = Promise<T>(shared);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -31,10 +31,9 @@ class LocalTransport {
|
||||
: local_transport_handle_(std::move(local_transport_handle)) {}
|
||||
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
|
||||
std::function<void()> fill_notifier, Duration timeout) {
|
||||
return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(
|
||||
to_address, from_address, std::move(request), timeout, fill_notifier);
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
|
||||
return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address,
|
||||
std::move(request), timeout);
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
|
||||
@@ -140,12 +140,8 @@ class LocalTransportHandle {
|
||||
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
|
||||
Duration timeout, std::function<void()> fill_notifier) {
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
|
||||
// set null notifier for when the Future::Wait is called
|
||||
nullptr,
|
||||
// set notifier for when Promise::Fill is called
|
||||
std::forward<std::function<void()>>(fill_notifier));
|
||||
Duration timeout) {
|
||||
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
|
||||
|
||||
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
|
||||
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <compare>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
@@ -40,8 +39,6 @@ struct LatencyHistogramSummary {
|
||||
Duration p100;
|
||||
Duration sum;
|
||||
|
||||
friend bool operator==(const LatencyHistogramSummary &lhs, const LatencyHistogramSummary &rhs) = default;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummary &histo) {
|
||||
in << "{ \"count\": " << histo.count;
|
||||
in << ", \"p0\": " << histo.p0.count();
|
||||
@@ -83,8 +80,6 @@ struct LatencyHistogramSummaries {
|
||||
return output;
|
||||
}
|
||||
|
||||
friend bool operator==(const LatencyHistogramSummaries &lhs, const LatencyHistogramSummaries &rhs) = default;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummaries &histo) {
|
||||
using memgraph::utils::print_helpers::operator<<;
|
||||
in << histo.latencies;
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
class ReadinessToken {
|
||||
size_t id_;
|
||||
|
||||
public:
|
||||
explicit ReadinessToken(size_t id) : id_(id) {}
|
||||
size_t GetId() const { return id_; }
|
||||
};
|
||||
|
||||
class Inner {
|
||||
std::condition_variable cv_;
|
||||
std::mutex mu_;
|
||||
std::vector<ReadinessToken> ready_;
|
||||
std::optional<std::function<bool()>> tick_simulator_;
|
||||
|
||||
public:
|
||||
void Notify(ReadinessToken readiness_token) {
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
ready_.emplace_back(readiness_token);
|
||||
} // mutex dropped
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
ReadinessToken Await() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
while (ready_.empty()) {
|
||||
if (tick_simulator_) [[unlikely]] {
|
||||
// This avoids a deadlock in a similar way that
|
||||
// Future::Wait will release its mutex while
|
||||
// interacting with the simulator, due to
|
||||
// the fact that the simulator may cause
|
||||
// notifications that we are interested in.
|
||||
lock.unlock();
|
||||
std::invoke(tick_simulator_.value());
|
||||
lock.lock();
|
||||
} else {
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
|
||||
ReadinessToken ret = ready_.back();
|
||||
ready_.pop_back();
|
||||
return ret;
|
||||
}
|
||||
|
||||
void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
tick_simulator_ = tick_simulator;
|
||||
}
|
||||
};
|
||||
|
||||
class Notifier {
|
||||
std::shared_ptr<Inner> inner_;
|
||||
|
||||
public:
|
||||
Notifier() : inner_(std::make_shared<Inner>()) {}
|
||||
Notifier(const Notifier &) = default;
|
||||
Notifier &operator=(const Notifier &) = default;
|
||||
Notifier(Notifier &&old) = default;
|
||||
Notifier &operator=(Notifier &&old) = default;
|
||||
~Notifier() = default;
|
||||
|
||||
void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
|
||||
|
||||
ReadinessToken Await() const { return inner_->Await(); }
|
||||
|
||||
void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
|
||||
};
|
||||
|
||||
} // namespace memgraph::io
|
||||
@@ -19,6 +19,7 @@
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
@@ -91,43 +92,33 @@ struct ReadResponse {
|
||||
};
|
||||
|
||||
template <class... ReadReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
|
||||
return TypeInfoForVariant(response.read_return);
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &read_response) {
|
||||
return TypeInfoForVariant(read_response.read_return);
|
||||
}
|
||||
|
||||
template <class ReadReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* read_response */) {
|
||||
return typeid(ReadReturn);
|
||||
}
|
||||
|
||||
template <class ReadOperation>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
|
||||
return typeid(ReadOperation);
|
||||
}
|
||||
|
||||
template <class... ReadOperations>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
|
||||
return TypeInfoForVariant(request.operation);
|
||||
}
|
||||
|
||||
template <class... WriteReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
|
||||
return TypeInfoForVariant(response.write_return);
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &write_response) {
|
||||
return TypeInfoForVariant(write_response.write_return);
|
||||
}
|
||||
|
||||
template <class WriteReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* write_response */) {
|
||||
return typeid(WriteReturn);
|
||||
}
|
||||
|
||||
template <class WriteOperation>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* write_request */) {
|
||||
return typeid(WriteOperation);
|
||||
}
|
||||
|
||||
template <class... WriteOperations>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
|
||||
return TypeInfoForVariant(request.operation);
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &write_request) {
|
||||
return TypeInfoForVariant(write_request.operation);
|
||||
}
|
||||
|
||||
/// AppendRequest is a raft-level message that the Leader
|
||||
@@ -191,7 +182,7 @@ struct PendingClientRequest {
|
||||
|
||||
struct Leader {
|
||||
std::map<Address, FollowerTracker> followers;
|
||||
std::map<LogIndex, PendingClientRequest> pending_client_requests;
|
||||
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
|
||||
Time last_broadcast = Time::min();
|
||||
|
||||
std::string static ToString() { return "\tLeader \t"; }
|
||||
@@ -321,12 +312,19 @@ class Raft {
|
||||
}
|
||||
|
||||
void Run() {
|
||||
// debug - gvolfing
|
||||
uint64_t tick_count = 0;
|
||||
|
||||
while (!io_.ShouldShutDown()) {
|
||||
const auto now = io_.Now();
|
||||
if (now >= next_cron_) {
|
||||
next_cron_ = Cron();
|
||||
}
|
||||
|
||||
// debug - gvolfing
|
||||
spdlog::info("Raft::Run() awakened on thread {}, Cron tick: {}", std::this_thread::get_id(), tick_count);
|
||||
tick_count++;
|
||||
|
||||
const Duration receive_timeout = RandomTimeout(kMinimumReceiveTimeout, kMaximumReceiveTimeout);
|
||||
|
||||
auto request_result =
|
||||
@@ -692,7 +690,7 @@ class Raft {
|
||||
|
||||
return Leader{
|
||||
.followers = std::move(followers),
|
||||
.pending_client_requests = std::map<LogIndex, PendingClientRequest>(),
|
||||
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -856,9 +854,7 @@ class Raft {
|
||||
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
|
||||
std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
|
||||
Address from_address) {
|
||||
auto type_info = TypeInfoFor(req);
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
Log("handling ReadOperation<" + demangled_name + ">");
|
||||
Log("handling ReadOperation");
|
||||
ReadOperation read_operation = req.operation;
|
||||
|
||||
ReadResponseValue read_return = replicated_state_.Read(read_operation);
|
||||
|
||||
@@ -14,12 +14,10 @@
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
@@ -38,14 +36,6 @@ using memgraph::io::rsm::WriteRequest;
|
||||
using memgraph::io::rsm::WriteResponse;
|
||||
using memgraph::utils::BasicResult;
|
||||
|
||||
template <typename RequestT, typename ResponseT>
|
||||
struct AsyncRequest {
|
||||
Time start_time;
|
||||
RequestT request;
|
||||
Notifier notifier;
|
||||
ResponseFuture<ResponseT> future;
|
||||
};
|
||||
|
||||
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
|
||||
typename ReadResponseT>
|
||||
class RsmClient {
|
||||
@@ -57,15 +47,23 @@ class RsmClient {
|
||||
|
||||
/// State for single async read/write operations. In the future this could become a map
|
||||
/// of async operations that can be accessed via an ID etc...
|
||||
std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
|
||||
std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
|
||||
std::optional<Time> async_read_before_;
|
||||
std::optional<ResponseFuture<ReadResponse<ReadResponseT>>> async_read_;
|
||||
ReadRequestT current_read_request_;
|
||||
|
||||
std::optional<Time> async_write_before_;
|
||||
std::optional<ResponseFuture<WriteResponse<WriteResponseT>>> async_write_;
|
||||
WriteRequestT current_write_request_;
|
||||
|
||||
void SelectRandomLeader() {
|
||||
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
|
||||
size_t addr_index = io_.Rand(addr_distrib);
|
||||
leader_ = server_addrs_[addr_index];
|
||||
|
||||
spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
|
||||
spdlog::debug(
|
||||
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
|
||||
"a RSM Candidate) trying a random one at index {} with address {}",
|
||||
addr_index, leader_.ToString());
|
||||
}
|
||||
|
||||
template <typename ResponseT>
|
||||
@@ -93,76 +91,107 @@ class RsmClient {
|
||||
~RsmClient() = default;
|
||||
|
||||
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
|
||||
Notifier notifier;
|
||||
const ReadinessToken readiness_token{0};
|
||||
SendAsyncWriteRequest(req, notifier, readiness_token);
|
||||
auto poll_result = AwaitAsyncWriteRequest(readiness_token);
|
||||
while (!poll_result) {
|
||||
poll_result = AwaitAsyncWriteRequest(readiness_token);
|
||||
}
|
||||
return poll_result.value();
|
||||
WriteRequest<WriteRequestT> client_req;
|
||||
client_req.operation = req;
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const Time before = io_.Now();
|
||||
|
||||
do {
|
||||
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
|
||||
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
|
||||
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
|
||||
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
|
||||
|
||||
if (response_result.HasError()) {
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
return response_result.GetError();
|
||||
}
|
||||
|
||||
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
|
||||
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
|
||||
|
||||
if (write_response.success) {
|
||||
return std::move(write_response.write_return);
|
||||
}
|
||||
|
||||
PossiblyRedirectLeader(write_response);
|
||||
} while (io_.Now() < before + overall_timeout);
|
||||
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
|
||||
Notifier notifier;
|
||||
const ReadinessToken readiness_token{0};
|
||||
SendAsyncReadRequest(req, notifier, readiness_token);
|
||||
auto poll_result = AwaitAsyncReadRequest(readiness_token);
|
||||
while (!poll_result) {
|
||||
poll_result = AwaitAsyncReadRequest(readiness_token);
|
||||
}
|
||||
return poll_result.value();
|
||||
ReadRequest<ReadRequestT> read_req;
|
||||
read_req.operation = req;
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const Time before = io_.Now();
|
||||
|
||||
do {
|
||||
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
|
||||
|
||||
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
|
||||
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
|
||||
|
||||
// receive response
|
||||
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
|
||||
|
||||
if (get_response_result.HasError()) {
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
return get_response_result.GetError();
|
||||
}
|
||||
|
||||
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
|
||||
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
|
||||
|
||||
if (read_get_response.success) {
|
||||
return std::move(read_get_response.read_return);
|
||||
}
|
||||
|
||||
PossiblyRedirectLeader(read_get_response);
|
||||
} while (io_.Now() < before + overall_timeout);
|
||||
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
/// AsyncRead methods
|
||||
void SendAsyncReadRequest(const ReadRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
|
||||
void SendAsyncReadRequest(const ReadRequestT &req) {
|
||||
MG_ASSERT(!async_read_);
|
||||
|
||||
ReadRequest<ReadRequestT> read_req = {.operation = req};
|
||||
|
||||
AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
|
||||
.start_time = io_.Now(),
|
||||
.request = std::move(req),
|
||||
.notifier = notifier,
|
||||
.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
|
||||
leader_, read_req, notifier, readiness_token),
|
||||
};
|
||||
|
||||
async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
|
||||
if (!async_read_before_) {
|
||||
async_read_before_ = io_.Now();
|
||||
}
|
||||
current_read_request_ = std::move(req);
|
||||
async_read_ = io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
|
||||
}
|
||||
|
||||
void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest() {
|
||||
MG_ASSERT(async_read_);
|
||||
|
||||
ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
|
||||
|
||||
async_request.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
|
||||
leader_, read_req, async_request.notifier, readiness_token);
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
|
||||
if (!async_request.future.IsReady()) {
|
||||
if (!async_read_->IsReady()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return AwaitAsyncReadRequest(readiness_token);
|
||||
return AwaitAsyncReadRequest();
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest() {
|
||||
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(*async_read_).Wait();
|
||||
async_read_.reset();
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
|
||||
const bool past_time_out = io_.Now() < *async_read_before_ + overall_timeout;
|
||||
const bool result_has_error = get_response_result.HasError();
|
||||
|
||||
if (result_has_error && past_time_out) {
|
||||
// TODO static assert the exact type of error.
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
async_reads_.erase(readiness_token.GetId());
|
||||
async_read_before_ = std::nullopt;
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
if (!result_has_error) {
|
||||
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
|
||||
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
|
||||
@@ -170,69 +199,54 @@ class RsmClient {
|
||||
PossiblyRedirectLeader(read_get_response);
|
||||
|
||||
if (read_get_response.success) {
|
||||
async_reads_.erase(readiness_token.GetId());
|
||||
spdlog::debug("returning read_return for RSM request");
|
||||
async_read_before_ = std::nullopt;
|
||||
return std::move(read_get_response.read_return);
|
||||
}
|
||||
} else {
|
||||
SendAsyncReadRequest(current_read_request_);
|
||||
} else if (result_has_error) {
|
||||
SelectRandomLeader();
|
||||
SendAsyncReadRequest(current_read_request_);
|
||||
}
|
||||
|
||||
ResendAsyncReadRequest(readiness_token);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
/// AsyncWrite methods
|
||||
void SendAsyncWriteRequest(const WriteRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
|
||||
void SendAsyncWriteRequest(const WriteRequestT &req) {
|
||||
MG_ASSERT(!async_write_);
|
||||
|
||||
WriteRequest<WriteRequestT> write_req = {.operation = req};
|
||||
|
||||
AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
|
||||
.start_time = io_.Now(),
|
||||
.request = std::move(req),
|
||||
.notifier = notifier,
|
||||
.future = io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
|
||||
leader_, write_req, notifier, readiness_token),
|
||||
};
|
||||
|
||||
async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
|
||||
if (!async_write_before_) {
|
||||
async_write_before_ = io_.Now();
|
||||
}
|
||||
current_write_request_ = std::move(req);
|
||||
async_write_ = io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, write_req);
|
||||
}
|
||||
|
||||
void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest() {
|
||||
MG_ASSERT(async_write_);
|
||||
|
||||
WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
|
||||
|
||||
async_request.future =
|
||||
io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
|
||||
leader_, write_req, async_request.notifier, readiness_token);
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
|
||||
if (!async_request.future.IsReady()) {
|
||||
if (!async_write_->IsReady()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return AwaitAsyncWriteRequest(readiness_token);
|
||||
return AwaitAsyncWriteRequest();
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest() {
|
||||
ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(*async_write_).Wait();
|
||||
async_write_.reset();
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
|
||||
const bool past_time_out = io_.Now() < *async_write_before_ + overall_timeout;
|
||||
const bool result_has_error = get_response_result.HasError();
|
||||
|
||||
if (result_has_error && past_time_out) {
|
||||
// TODO static assert the exact type of error.
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
async_writes_.erase(readiness_token.GetId());
|
||||
async_write_before_ = std::nullopt;
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
if (!result_has_error) {
|
||||
ResponseEnvelope<WriteResponse<WriteResponseT>> &&get_response_envelope =
|
||||
std::move(get_response_result.GetValue());
|
||||
@@ -241,15 +255,14 @@ class RsmClient {
|
||||
PossiblyRedirectLeader(write_get_response);
|
||||
|
||||
if (write_get_response.success) {
|
||||
async_writes_.erase(readiness_token.GetId());
|
||||
async_write_before_ = std::nullopt;
|
||||
return std::move(write_get_response.write_return);
|
||||
}
|
||||
} else {
|
||||
SendAsyncWriteRequest(current_write_request_);
|
||||
} else if (result_has_error) {
|
||||
SelectRandomLeader();
|
||||
SendAsyncWriteRequest(current_write_request_);
|
||||
}
|
||||
|
||||
ResendAsyncWriteRequest(readiness_token);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -41,7 +41,7 @@ class Simulator {
|
||||
Io<SimulatorTransport> Register(Address address) {
|
||||
std::uniform_int_distribution<uint64_t> seed_distrib;
|
||||
uint64_t seed = seed_distrib(rng_);
|
||||
return Io{SimulatorTransport(simulator_handle_, address, seed), address};
|
||||
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
|
||||
}
|
||||
|
||||
void IncrementServerCountAndWaitForQuiescentState(Address address) {
|
||||
@@ -49,14 +49,5 @@ class Simulator {
|
||||
}
|
||||
|
||||
SimulatorStats Stats() { return simulator_handle_->Stats(); }
|
||||
|
||||
std::shared_ptr<SimulatorHandle> GetSimulatorHandle() const { return simulator_handle_; }
|
||||
|
||||
std::function<bool()> GetSimulatorTickClosure() {
|
||||
std::function<bool()> tick_closure = [handle_copy = simulator_handle_] {
|
||||
return handle_copy->MaybeTickSimulator();
|
||||
};
|
||||
return tick_closure;
|
||||
}
|
||||
};
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -23,12 +23,6 @@ namespace memgraph::io::simulator {
|
||||
void SimulatorHandle::ShutDown() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
should_shut_down_ = true;
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
std::move(dop).promise.TimeOut();
|
||||
it = promises_.erase(it);
|
||||
}
|
||||
can_receive_.clear();
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
@@ -52,94 +46,92 @@ void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address addre
|
||||
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
|
||||
|
||||
if (all_servers_blocked) {
|
||||
spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
|
||||
server_addresses_.size());
|
||||
return;
|
||||
}
|
||||
|
||||
spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
|
||||
server_addresses_.size());
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
|
||||
bool SortInFlight(const std::pair<Address, OpaqueMessage> &lhs, const std::pair<Address, OpaqueMessage> &rhs) {
|
||||
// NB: never sort based on the request ID etc...
|
||||
// This should only be used from std::stable_sort
|
||||
// because by comparing on the from_address alone,
|
||||
// we expect the sender ordering to remain
|
||||
// deterministic.
|
||||
const auto &[addr_1, om_1] = lhs;
|
||||
const auto &[addr_2, om_2] = rhs;
|
||||
return om_1.from_address < om_2.from_address;
|
||||
}
|
||||
|
||||
bool SimulatorHandle::MaybeTickSimulator() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
const size_t blocked_servers = blocked_on_receive_.size();
|
||||
|
||||
if (should_shut_down_ || blocked_servers < server_addresses_.size()) {
|
||||
// Don't judge, this is a workaround for poc. gvolfing
|
||||
server_count_++;
|
||||
if (server_count_ == 3) {
|
||||
is_quiescent_state_achieved_ = true;
|
||||
}
|
||||
|
||||
// if (!is_quiescent_state_achieved_) {
|
||||
if (!is_quiescent_state_achieved_) {
|
||||
// return SimulatorProgression::DidSetUpOneServer;
|
||||
cv_.notify_all(); // -> deadlock
|
||||
// return false;// -> deadlock
|
||||
return true; // -> deadlock
|
||||
}
|
||||
// if (blocked_servers < server_addresses_.size() && is_quiescent_state_achieved_) {
|
||||
if (blocked_servers < server_addresses_.size()) {
|
||||
// we only need to advance the simulator when all
|
||||
// servers have reached a quiescent state, blocked
|
||||
// on their own futures or receive methods.
|
||||
return false;
|
||||
}
|
||||
|
||||
// We allow the simulator to progress the state of the system only
|
||||
// after all servers are blocked on receive.
|
||||
spdlog::trace("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ simulator tick ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
|
||||
stats_.simulator_ticks++;
|
||||
blocked_on_receive_.clear();
|
||||
|
||||
cv_.notify_all();
|
||||
|
||||
bool timed_anything_out = TimeoutPromisesPastDeadline();
|
||||
TimeoutPromisesPastDeadline();
|
||||
|
||||
if (timed_anything_out) {
|
||||
spdlog::trace("simulator progressing: timed out a request");
|
||||
}
|
||||
auto sort_based_on_message_id = [](const std::pair<Address, OpaqueMessage> &lhs,
|
||||
const std::pair<Address, OpaqueMessage> &rhs) {
|
||||
// return a.second.request_id < b.second.request_id;
|
||||
if (lhs.second.from_address != rhs.second.from_address) {
|
||||
return lhs.second.from_address < rhs.second.from_address;
|
||||
}
|
||||
|
||||
const Duration clock_advance = std::chrono::microseconds{time_distrib_(rng_)};
|
||||
|
||||
// We don't always want to advance the clock with every message that we deliver because
|
||||
// when we advance it for every message, it causes timeouts to occur for any "request path"
|
||||
// over a certain length. Alternatively, we don't want to simply deliver multiple messages
|
||||
// in a single simulator tick because that would reduce the amount of concurrent message
|
||||
// mixing that may naturally occur in production. This approach is to mod the random clock
|
||||
// advance by a prime number (hopefully avoiding most harmonic effects that would be introduced
|
||||
// by only advancing the clock by an even amount etc...) and only advancing the clock close to
|
||||
// half of the time.
|
||||
if (clock_advance.count() % 97 > 49) {
|
||||
spdlog::trace("simulator progressing: clock advanced by {}", clock_advance.count());
|
||||
cluster_wide_time_microseconds_ += clock_advance;
|
||||
stats_.elapsed_time = cluster_wide_time_microseconds_ - config_.start_time;
|
||||
}
|
||||
|
||||
if (cluster_wide_time_microseconds_ >= config_.abort_time) {
|
||||
spdlog::error(
|
||||
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
|
||||
"in an expected amount of time. The SimulatorConfig.rng_seed for this run is {}",
|
||||
config_.rng_seed);
|
||||
throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
|
||||
}
|
||||
return lhs.second.request_id < rhs.second.request_id;
|
||||
};
|
||||
std::sort(in_flight_.begin(), in_flight_.end(), sort_based_on_message_id);
|
||||
|
||||
if (in_flight_.empty()) {
|
||||
// return early here because there are no messages to schedule
|
||||
|
||||
// We tick the clock forward when all servers are blocked but
|
||||
// there are no in-flight messages to schedule delivery of.
|
||||
// const Duration clock_advance = std::chrono::microseconds{Rand(time_distrib_)};
|
||||
// cluster_wide_time_microseconds_ += clock_advance;
|
||||
|
||||
if (cluster_wide_time_microseconds_ >= config_.abort_time) {
|
||||
if (should_shut_down_) {
|
||||
return false;
|
||||
}
|
||||
spdlog::error(
|
||||
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
|
||||
"in an expected amount of time.");
|
||||
throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
|
||||
}
|
||||
// spdlog::info(
|
||||
// "Time increased with {} to {}", clock_advance.count(),
|
||||
// std::chrono::duration_cast<std::chrono::milliseconds>(cluster_wide_time_microseconds_.time_since_epoch())
|
||||
// .count());
|
||||
return true;
|
||||
}
|
||||
|
||||
std::stable_sort(in_flight_.begin(), in_flight_.end(), SortInFlight);
|
||||
|
||||
if (config_.scramble_messages && in_flight_.size() > 1) {
|
||||
if (config_.scramble_messages) {
|
||||
// scramble messages
|
||||
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
|
||||
const size_t swap_index = swap_distrib(rng_);
|
||||
const size_t swap_index = RandLocked(swap_distrib);
|
||||
std::swap(in_flight_[swap_index], in_flight_.back());
|
||||
}
|
||||
|
||||
auto [to_address, opaque_message] = std::move(in_flight_.back());
|
||||
in_flight_.pop_back();
|
||||
event_log_.AddAndLog(opaque_message, "Handle ");
|
||||
|
||||
const int drop_threshold = drop_distrib_(rng_);
|
||||
const int drop_threshold = RandLocked(drop_distrib_);
|
||||
const bool should_drop = drop_threshold < config_.drop_percent;
|
||||
|
||||
if (should_drop) {
|
||||
@@ -149,6 +141,7 @@ bool SimulatorHandle::MaybeTickSimulator() {
|
||||
PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
|
||||
|
||||
if (promises_.contains(promise_key)) {
|
||||
event_log_.AddAndLog(opaque_message, "Promise contains");
|
||||
// complete waiting promise if it's there
|
||||
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
|
||||
promises_.erase(promise_key);
|
||||
@@ -158,22 +151,24 @@ bool SimulatorHandle::MaybeTickSimulator() {
|
||||
if (should_drop || normal_timeout) {
|
||||
stats_.timed_out_requests++;
|
||||
dop.promise.TimeOut();
|
||||
spdlog::trace("simulator timing out request ");
|
||||
} else {
|
||||
stats_.total_responses++;
|
||||
Duration response_latency = cluster_wide_time_microseconds_ - dop.requested_at;
|
||||
auto type_info = opaque_message.type_info;
|
||||
dop.promise.Fill(std::move(opaque_message), response_latency);
|
||||
histograms_.Measure(type_info, response_latency);
|
||||
spdlog::trace("simulator replying to request");
|
||||
}
|
||||
} else if (should_drop) {
|
||||
event_log_.AddAndLog(opaque_message, "Dropped");
|
||||
// don't add it anywhere, let it drop
|
||||
spdlog::trace("simulator silently dropping request");
|
||||
} else {
|
||||
event_log_.AddAndLog(opaque_message, "To can_receive");
|
||||
// add to can_receive_ if not
|
||||
spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
|
||||
opaque_message.to_address.last_known_port);
|
||||
// This might be needed I am not sure, it can cause deadlock if you uncomment. Didn't find the issue yet.
|
||||
// MG_ASSERT(server_addresses_.contains(to_address));
|
||||
// MG_ASSERT(blocked_on_receive_.contains(to_address));
|
||||
// blocked_on_receive_.erase(to_address);
|
||||
|
||||
const auto &[om_vec, inserted] =
|
||||
can_receive_.try_emplace(to_address.ToPartialAddress(), std::vector<OpaqueMessage>());
|
||||
om_vec->second.emplace_back(std::move(opaque_message));
|
||||
@@ -187,6 +182,11 @@ Time SimulatorHandle::Now() const {
|
||||
return cluster_wide_time_microseconds_;
|
||||
}
|
||||
|
||||
Time SimulatorHandle::NowLocked() const {
|
||||
// std::unique_lock<std::mutex> lock(mu_);
|
||||
return cluster_wide_time_microseconds_;
|
||||
}
|
||||
|
||||
SimulatorStats SimulatorHandle::Stats() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
return stats_;
|
||||
|
||||
@@ -17,13 +17,20 @@
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <queue>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
#include <sstream>
|
||||
|
||||
#include <boost/lexical_cast.hpp>
|
||||
#include <boost/stacktrace.hpp>
|
||||
#include <boost/uuid/uuid_io.hpp>
|
||||
|
||||
#include "fmt/format.h"
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/message_conversion.hpp"
|
||||
@@ -32,9 +39,50 @@
|
||||
#include "io/simulator/simulator_stats.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
|
||||
struct EventDescriptor {
|
||||
boost::asio::ip::address from_address_ip;
|
||||
uint16_t from_address_port;
|
||||
boost::asio::ip::address to_address_ip;
|
||||
uint16_t to_address_port;
|
||||
uint64_t request_id;
|
||||
std::string_view caller;
|
||||
utils::TypeInfoRef type_info;
|
||||
uint64_t event_id;
|
||||
Time time;
|
||||
};
|
||||
|
||||
class SimulatorHandle;
|
||||
class EventLog {
|
||||
std::vector<EventDescriptor> event_log_;
|
||||
static constexpr const char *eventlog_signal_string_ = "EVENTLOG_UPDATE";
|
||||
|
||||
void LogEvent(const EventDescriptor &event) const {
|
||||
spdlog::info(
|
||||
fmt::format("{} ({})-> caller: {}, from_address: {}:{}, to_address: {}:{}, request_id: {}, type: {}, event_id: "
|
||||
"{}, thread_id: {}",
|
||||
eventlog_signal_string_,
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(event.time.time_since_epoch()).count(),
|
||||
event.caller, event.from_address_ip.to_string(), event.from_address_port,
|
||||
event.to_address_ip.to_string(), event.to_address_port, event.request_id,
|
||||
event.type_info.get().name(), event.event_id, std::this_thread::get_id()));
|
||||
}
|
||||
|
||||
public:
|
||||
SimulatorHandle *handle{nullptr};
|
||||
|
||||
void AddAndLog(const OpaqueMessage &message, std::string_view caller);
|
||||
|
||||
void LogAllEvents() const {
|
||||
for (const auto &event : event_log_) {
|
||||
LogEvent(event);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class SimulatorHandle {
|
||||
mutable std::mutex mu_{};
|
||||
mutable std::condition_variable cv_;
|
||||
@@ -48,40 +96,48 @@ class SimulatorHandle {
|
||||
// messages that are sent to servers that may later receive them
|
||||
std::map<PartialAddress, std::vector<OpaqueMessage>> can_receive_;
|
||||
|
||||
// maybe a boolean here
|
||||
bool is_quiescent_state_achieved_ = false;
|
||||
std::atomic<int> server_count_;
|
||||
|
||||
Time cluster_wide_time_microseconds_;
|
||||
bool should_shut_down_ = false;
|
||||
SimulatorStats stats_;
|
||||
std::set<Address> blocked_on_receive_;
|
||||
std::set<Address> server_addresses_;
|
||||
std::mt19937 rng_;
|
||||
std::uniform_int_distribution<int> time_distrib_{0, 30000};
|
||||
std::uniform_int_distribution<int> time_distrib_{5, 50};
|
||||
std::uniform_int_distribution<int> drop_distrib_{0, 99};
|
||||
SimulatorConfig config_;
|
||||
MessageHistogramCollector histograms_;
|
||||
RequestId request_id_counter_{0};
|
||||
EventLog event_log_;
|
||||
|
||||
bool TimeoutPromisesPastDeadline() {
|
||||
bool timed_anything_out = false;
|
||||
void TimeoutPromisesPastDeadline() {
|
||||
const Time now = cluster_wide_time_microseconds_;
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
if (dop.deadline < now && config_.perform_timeouts) {
|
||||
spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
|
||||
const bool timed_out = dop.deadline < now;
|
||||
if (timed_out && config_.perform_timeouts) {
|
||||
// spdlog::info("timing out request from requester {}.", promise_key.requester_address.ToString());
|
||||
spdlog::info("timing out request from requester {}. bool timed_out: {}",
|
||||
promise_key.requester_address.ToString(), timed_out);
|
||||
std::move(dop).promise.TimeOut();
|
||||
it = promises_.erase(it);
|
||||
|
||||
stats_.timed_out_requests++;
|
||||
timed_anything_out = true;
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
return timed_anything_out;
|
||||
}
|
||||
|
||||
public:
|
||||
explicit SimulatorHandle(SimulatorConfig config)
|
||||
: cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
|
||||
: server_count_(0), cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {
|
||||
spdlog::info("SimulatorHandle constructed.");
|
||||
event_log_.handle = this;
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies();
|
||||
|
||||
@@ -107,48 +163,41 @@ class SimulatorHandle {
|
||||
|
||||
template <Message Request, Message Response>
|
||||
ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
|
||||
std::function<bool()> &&maybe_tick_simulator,
|
||||
std::function<void()> &&fill_notifier) {
|
||||
std::function<bool()> &&maybe_tick_simulator) {
|
||||
auto type_info = TypeInfoFor(request);
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
|
||||
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<Response>>(
|
||||
// set notifier for when the Future::Wait is called
|
||||
std::forward<std::function<bool()>>(maybe_tick_simulator),
|
||||
// set notifier for when Promise::Fill is called
|
||||
std::forward<std::function<void()>>(fill_notifier));
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifier<ResponseResult<Response>>(
|
||||
std::forward<std::function<bool()>>(maybe_tick_simulator));
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
RequestId request_id = ++request_id_counter_;
|
||||
RequestId request_id = ++request_id_counter_;
|
||||
|
||||
const Time deadline = cluster_wide_time_microseconds_ + timeout;
|
||||
const Time deadline = cluster_wide_time_microseconds_ + timeout;
|
||||
|
||||
std::any message(request);
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message),
|
||||
.type_info = type_info};
|
||||
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
|
||||
std::any message(request);
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message),
|
||||
.type_info = type_info};
|
||||
event_log_.AddAndLog(om, "SubmitRequest");
|
||||
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
|
||||
|
||||
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
|
||||
OpaquePromise opaque_promise(std::move(promise).ToUnique());
|
||||
DeadlineAndOpaquePromise dop{
|
||||
.requested_at = cluster_wide_time_microseconds_,
|
||||
.deadline = deadline,
|
||||
.promise = std::move(opaque_promise),
|
||||
};
|
||||
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
|
||||
OpaquePromise opaque_promise(std::move(promise).ToUnique());
|
||||
DeadlineAndOpaquePromise dop{
|
||||
.requested_at = cluster_wide_time_microseconds_,
|
||||
.deadline = deadline,
|
||||
.promise = std::move(opaque_promise),
|
||||
};
|
||||
|
||||
MG_ASSERT(!promises_.contains(promise_key));
|
||||
MG_ASSERT(!promises_.contains(promise_key));
|
||||
|
||||
promises_.emplace(std::move(promise_key), std::move(dop));
|
||||
promises_.emplace(std::move(promise_key), std::move(dop));
|
||||
|
||||
stats_.total_messages++;
|
||||
stats_.total_requests++;
|
||||
} // lock dropped here
|
||||
stats_.total_messages++;
|
||||
stats_.total_requests++;
|
||||
|
||||
cv_.notify_all();
|
||||
|
||||
@@ -159,6 +208,8 @@ class SimulatorHandle {
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
blocked_on_receive_.emplace(receiver);
|
||||
|
||||
const Time deadline = cluster_wide_time_microseconds_ + timeout;
|
||||
|
||||
auto partial_address = receiver.ToPartialAddress();
|
||||
@@ -170,57 +221,83 @@ class SimulatorHandle {
|
||||
OpaqueMessage message = std::move(can_rx.back());
|
||||
can_rx.pop_back();
|
||||
|
||||
event_log_.AddAndLog(message, "Receivexxxxxx");
|
||||
|
||||
// TODO(tyler) search for item in can_receive_ that matches the desired types, rather
|
||||
// than asserting that the last item in can_rx matches.
|
||||
auto m_opt = std::move(message).Take<Ms...>();
|
||||
MG_ASSERT(m_opt.has_value(), "Wrong message type received compared to the expected type");
|
||||
|
||||
blocked_on_receive_.erase(receiver);
|
||||
|
||||
return std::move(m_opt).value();
|
||||
}
|
||||
}
|
||||
|
||||
if (!should_shut_down_) {
|
||||
if (!blocked_on_receive_.contains(receiver)) {
|
||||
blocked_on_receive_.emplace(receiver);
|
||||
spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
|
||||
cv_.notify_all();
|
||||
}
|
||||
lock.unlock();
|
||||
auto simulator_progress = MaybeTickSimulator();
|
||||
lock.lock();
|
||||
if (!should_shut_down_ && !simulator_progress) {
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
|
||||
|
||||
blocked_on_receive_.erase(receiver);
|
||||
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
|
||||
spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
|
||||
auto type_info = TypeInfoFor(message);
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
std::any message_any(std::move(message));
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message_any),
|
||||
.type_info = type_info};
|
||||
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
std::any message_any(std::move(message));
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message_any),
|
||||
.type_info = type_info};
|
||||
event_log_.AddAndLog(om, "Sendxxxxxxxxx");
|
||||
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
|
||||
|
||||
stats_.total_messages++;
|
||||
} // lock dropped before cv notification
|
||||
stats_.total_messages++;
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
Time Now() const;
|
||||
|
||||
Time NowLocked() const;
|
||||
|
||||
template <class D = std::poisson_distribution<>, class Return = uint64_t>
|
||||
Return Rand(D distrib) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
return distrib(rng_);
|
||||
return RandLocked<D, Return>(std::forward<D>(distrib));
|
||||
}
|
||||
|
||||
template <class D = std::poisson_distribution<>, class Return = uint64_t>
|
||||
Return RandLocked(D distrib) {
|
||||
Return res = distrib(rng_);
|
||||
spdlog::info("Getting random value from thread {}: {}. Stacktrace:\n{}", std::this_thread::get_id(), res,
|
||||
boost::stacktrace::stacktrace());
|
||||
return res;
|
||||
}
|
||||
|
||||
SimulatorStats Stats();
|
||||
};
|
||||
|
||||
inline void EventLog::AddAndLog(const OpaqueMessage &message, std::string_view caller) {
|
||||
EventDescriptor event{.from_address_ip = message.from_address.last_known_ip,
|
||||
.from_address_port = message.from_address.last_known_port,
|
||||
.to_address_ip = message.to_address.last_known_ip,
|
||||
.to_address_port = message.to_address.last_known_port,
|
||||
.request_id = message.request_id,
|
||||
.caller = caller,
|
||||
.type_info = message.type_info,
|
||||
.event_id = event_log_.size(),
|
||||
.time = handle->NowLocked()};
|
||||
|
||||
event_log_.push_back(event);
|
||||
LogEvent(event);
|
||||
}
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -13,10 +13,6 @@
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "io/time.hpp"
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
struct SimulatorStats {
|
||||
uint64_t total_messages = 0;
|
||||
@@ -25,22 +21,5 @@ struct SimulatorStats {
|
||||
uint64_t total_requests = 0;
|
||||
uint64_t total_responses = 0;
|
||||
uint64_t simulator_ticks = 0;
|
||||
Duration elapsed_time;
|
||||
|
||||
friend bool operator==(const SimulatorStats & /* lhs */, const SimulatorStats & /* rhs */) = default;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const SimulatorStats &stats) {
|
||||
auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(stats.elapsed_time).count();
|
||||
|
||||
std::string formated = fmt::format(
|
||||
"SimulatorStats {{ total_messages: {}, dropped_messages: {}, timed_out_requests: {}, total_requests: {}, "
|
||||
"total_responses: {}, simulator_ticks: {}, elapsed_time: {}ms }}",
|
||||
stats.total_messages, stats.dropped_messages, stats.timed_out_requests, stats.total_requests,
|
||||
stats.total_responses, stats.simulator_ticks, elapsed_ms);
|
||||
|
||||
in << formated;
|
||||
|
||||
return in;
|
||||
}
|
||||
};
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
#include <utility>
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/simulator/simulator_handle.hpp"
|
||||
#include "io/time.hpp"
|
||||
|
||||
@@ -26,7 +25,7 @@ using memgraph::io::Time;
|
||||
|
||||
class SimulatorTransport {
|
||||
std::shared_ptr<SimulatorHandle> simulator_handle_;
|
||||
Address address_;
|
||||
const Address address_;
|
||||
std::mt19937 rng_;
|
||||
|
||||
public:
|
||||
@@ -34,14 +33,11 @@ class SimulatorTransport {
|
||||
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
|
||||
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
|
||||
std::function<void()> notification, Duration timeout) {
|
||||
std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
|
||||
return handle_copy->MaybeTickSimulator();
|
||||
};
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
|
||||
std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
|
||||
|
||||
return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(
|
||||
to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
|
||||
return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address, std::move(request),
|
||||
timeout, std::move(maybe_tick_simulator));
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
@@ -58,9 +54,11 @@ class SimulatorTransport {
|
||||
|
||||
bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); }
|
||||
|
||||
template <class D = std::poisson_distribution<>, class Return = uint64_t>
|
||||
template <class D = std::poisson_distribution<uint64_t>, class Return = uint64_t>
|
||||
Return Rand(D distrib) {
|
||||
return distrib(rng_);
|
||||
// debug - gvolfing
|
||||
auto ret = distrib(rng_);
|
||||
return ret;
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() { return simulator_handle_->ResponseLatencies(); }
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#include "io/errors.hpp"
|
||||
#include "io/future.hpp"
|
||||
#include "io/message_histogram_collector.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
@@ -85,9 +84,7 @@ class Io {
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = nullptr;
|
||||
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, fill_notifier,
|
||||
timeout);
|
||||
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, timeout);
|
||||
}
|
||||
|
||||
/// Issue a request that times out after the default timeout. This tends
|
||||
@@ -96,30 +93,7 @@ class Io {
|
||||
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
|
||||
const Duration timeout = default_timeout_;
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = nullptr;
|
||||
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
|
||||
fill_notifier, timeout);
|
||||
}
|
||||
|
||||
/// Issue a request that will notify a Notifier when it is filled or times out.
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RequestT request, Notifier notifier,
|
||||
ReadinessToken readiness_token) {
|
||||
const Duration timeout = default_timeout_;
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
|
||||
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
|
||||
fill_notifier, timeout);
|
||||
}
|
||||
|
||||
/// Issue a request that will notify a Notifier when it is filled or times out.
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT request, Notifier notifier,
|
||||
ReadinessToken readiness_token, Duration timeout) {
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
|
||||
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
|
||||
fill_notifier, timeout);
|
||||
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request), timeout);
|
||||
}
|
||||
|
||||
/// Wait for an explicit number of microseconds for a request of one of the
|
||||
@@ -163,14 +137,7 @@ class Io {
|
||||
Address GetAddress() { return address_; }
|
||||
void SetAddress(Address address) { address_ = address; }
|
||||
|
||||
Io<I> ForkLocal(boost::uuids::uuid uuid) {
|
||||
Address new_address{
|
||||
.unique_id = uuid,
|
||||
.last_known_ip = address_.last_known_ip,
|
||||
.last_known_port = address_.last_known_port,
|
||||
};
|
||||
return Io(implementation_, new_address);
|
||||
}
|
||||
Io<I> ForkLocal() { return Io(implementation_, address_.ForkUniqueAddress()); }
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
|
||||
};
|
||||
|
||||
@@ -42,7 +42,6 @@ struct MachineConfig {
|
||||
boost::asio::ip::address listen_ip;
|
||||
uint16_t listen_port;
|
||||
size_t shard_worker_threads = std::max(static_cast<unsigned int>(1), std::thread::hardware_concurrency());
|
||||
bool sync_message_handling = false;
|
||||
};
|
||||
|
||||
} // namespace memgraph::machine_manager
|
||||
|
||||
@@ -78,10 +78,10 @@ class MachineManager {
|
||||
MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
|
||||
: io_(io),
|
||||
config_(config),
|
||||
coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
|
||||
shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
|
||||
coordinator_address_} {
|
||||
auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
|
||||
coordinator_address_(io.GetAddress().ForkUniqueAddress()),
|
||||
shard_manager_{io.ForkLocal(), config.shard_worker_threads, coordinator_address_} {
|
||||
auto coordinator_io = io.ForkLocal();
|
||||
coordinator_io.SetAddress(coordinator_address_);
|
||||
CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
|
||||
coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
|
||||
}
|
||||
@@ -101,23 +101,11 @@ class MachineManager {
|
||||
Address CoordinatorAddress() { return coordinator_address_; }
|
||||
|
||||
void Run() {
|
||||
while (true) {
|
||||
MaybeBlockOnSyncHandling();
|
||||
|
||||
if (io_.ShouldShutDown()) {
|
||||
break;
|
||||
}
|
||||
|
||||
while (!io_.ShouldShutDown()) {
|
||||
const auto now = io_.Now();
|
||||
|
||||
uint64_t now_us = now.time_since_epoch().count();
|
||||
uint64_t next_us = next_cron_.time_since_epoch().count();
|
||||
|
||||
if (now >= next_cron_) {
|
||||
spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
|
||||
next_cron_ = Cron();
|
||||
} else {
|
||||
spdlog::info("now {} < next_cron_ {}", now_us, next_us);
|
||||
}
|
||||
|
||||
Duration receive_timeout = std::max(next_cron_, now) - now;
|
||||
@@ -206,27 +194,10 @@ class MachineManager {
|
||||
}
|
||||
|
||||
private:
|
||||
// This method exists for controlling concurrency
|
||||
// during deterministic simulation testing.
|
||||
void MaybeBlockOnSyncHandling() {
|
||||
if (!config_.sync_message_handling) {
|
||||
return;
|
||||
}
|
||||
|
||||
// block on coordinator
|
||||
coordinator_queue_.BlockOnQuiescence();
|
||||
|
||||
// block on shards
|
||||
shard_manager_.BlockOnQuiescence();
|
||||
}
|
||||
|
||||
Time Cron() {
|
||||
spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
|
||||
coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
|
||||
MaybeBlockOnSyncHandling();
|
||||
Time ret = shard_manager_.Cron();
|
||||
MaybeBlockOnSyncHandling();
|
||||
return ret;
|
||||
return shard_manager_.Cron();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -33,7 +33,6 @@
|
||||
#include <spdlog/sinks/dist_sink.h>
|
||||
#include <spdlog/sinks/stdout_color_sinks.h>
|
||||
|
||||
#include "common/errors.hpp"
|
||||
#include "communication/bolt/v1/constants.hpp"
|
||||
#include "communication/websocket/auth.hpp"
|
||||
#include "communication/websocket/server.hpp"
|
||||
@@ -454,7 +453,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
|
||||
|
||||
std::map<std::string, memgraph::communication::bolt::Value> Pull(TEncoder *encoder, std::optional<int> n,
|
||||
std::optional<int> qid) override {
|
||||
TypedValueResultStream stream(encoder, interpreter_.GetRequestRouter());
|
||||
TypedValueResultStream stream(encoder, interpreter_.GetShardRequestManager());
|
||||
return PullResults(stream, n, qid);
|
||||
}
|
||||
|
||||
@@ -481,9 +480,20 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
|
||||
const auto &summary = interpreter_.Pull(&stream, n, qid);
|
||||
std::map<std::string, memgraph::communication::bolt::Value> decoded_summary;
|
||||
for (const auto &kv : summary) {
|
||||
auto bolt_value = memgraph::glue::v2::ToBoltValue(kv.second, interpreter_.GetRequestRouter(),
|
||||
memgraph::storage::v3::View::NEW);
|
||||
decoded_summary.emplace(kv.first, std::move(bolt_value));
|
||||
auto maybe_value = memgraph::glue::v2::ToBoltValue(kv.second, interpreter_.GetShardRequestManager(),
|
||||
memgraph::storage::v3::View::NEW);
|
||||
if (maybe_value.HasError()) {
|
||||
switch (maybe_value.GetError()) {
|
||||
case memgraph::storage::v3::Error::DELETED_OBJECT:
|
||||
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
|
||||
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
|
||||
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
case memgraph::storage::v3::Error::VERTEX_ALREADY_INSERTED:
|
||||
throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming summary.");
|
||||
}
|
||||
}
|
||||
decoded_summary.emplace(kv.first, std::move(*maybe_value));
|
||||
}
|
||||
return decoded_summary;
|
||||
} catch (const memgraph::query::v2::QueryException &e) {
|
||||
@@ -497,22 +507,35 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
|
||||
/// before forwarding the calls to original TEncoder.
|
||||
class TypedValueResultStream {
|
||||
public:
|
||||
TypedValueResultStream(TEncoder *encoder, const memgraph::query::v2::RequestRouterInterface *request_router)
|
||||
: encoder_(encoder), request_router_(request_router) {}
|
||||
TypedValueResultStream(TEncoder *encoder, const memgraph::msgs::ShardRequestManagerInterface *shard_request_manager)
|
||||
: encoder_(encoder), shard_request_manager_(shard_request_manager) {}
|
||||
|
||||
void Result(const std::vector<memgraph::query::v2::TypedValue> &values) {
|
||||
std::vector<memgraph::communication::bolt::Value> decoded_values;
|
||||
decoded_values.reserve(values.size());
|
||||
for (const auto &v : values) {
|
||||
auto bolt_value = memgraph::glue::v2::ToBoltValue(v, request_router_, memgraph::storage::v3::View::NEW);
|
||||
decoded_values.emplace_back(std::move(bolt_value));
|
||||
auto maybe_value = memgraph::glue::v2::ToBoltValue(v, shard_request_manager_, memgraph::storage::v3::View::NEW);
|
||||
if (maybe_value.HasError()) {
|
||||
switch (maybe_value.GetError()) {
|
||||
case memgraph::storage::v3::Error::DELETED_OBJECT:
|
||||
throw memgraph::communication::bolt::ClientError("Returning a deleted object as a result.");
|
||||
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
throw memgraph::communication::bolt::ClientError("Returning a nonexistent object as a result.");
|
||||
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
|
||||
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
|
||||
case memgraph::storage::v3::Error::VERTEX_ALREADY_INSERTED:
|
||||
throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming results.");
|
||||
}
|
||||
}
|
||||
decoded_values.emplace_back(std::move(*maybe_value));
|
||||
}
|
||||
encoder_->MessageRecord(decoded_values);
|
||||
}
|
||||
|
||||
private:
|
||||
TEncoder *encoder_;
|
||||
const memgraph::query::v2::RequestRouterInterface *request_router_{nullptr};
|
||||
const memgraph::msgs::ShardRequestManagerInterface *shard_request_manager_{nullptr};
|
||||
};
|
||||
memgraph::query::v2::Interpreter interpreter_;
|
||||
memgraph::communication::v2::ServerEndpoint endpoint_;
|
||||
@@ -640,8 +663,6 @@ int main(int argc, char **argv) {
|
||||
memgraph::machine_manager::MachineManager<memgraph::io::local_transport::LocalTransport> mm{io, config, coordinator};
|
||||
std::jthread mm_thread([&mm] { mm.Run(); });
|
||||
|
||||
auto rr_factory = std::make_unique<memgraph::query::v2::LocalRequestRouterFactory>(io);
|
||||
|
||||
memgraph::query::v2::InterpreterContext interpreter_context{
|
||||
(memgraph::storage::v3::Shard *)(nullptr),
|
||||
{.query = {.allow_load_csv = FLAGS_allow_load_csv},
|
||||
@@ -652,7 +673,7 @@ int main(int argc, char **argv) {
|
||||
.stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries,
|
||||
.stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)},
|
||||
FLAGS_data_directory,
|
||||
std::move(rr_factory),
|
||||
std::move(io),
|
||||
mm.CoordinatorAddress()};
|
||||
|
||||
SessionData session_data{&interpreter_context};
|
||||
|
||||
@@ -11,6 +11,7 @@ set(mg_query_v2_sources
|
||||
cypher_query_interpreter.cpp
|
||||
frontend/semantic/required_privileges.cpp
|
||||
frontend/stripped.cpp
|
||||
interpret/awesome_memgraph_functions.cpp
|
||||
interpreter.cpp
|
||||
metadata.cpp
|
||||
plan/operator.cpp
|
||||
@@ -23,8 +24,7 @@ set(mg_query_v2_sources
|
||||
plan/variable_start_planner.cpp
|
||||
serialization/property_value.cpp
|
||||
bindings/typed_value.cpp
|
||||
accessors.cpp
|
||||
multiframe.cpp)
|
||||
accessors.cpp)
|
||||
|
||||
find_package(Boost REQUIRED)
|
||||
|
||||
@@ -34,7 +34,7 @@ target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
|
||||
target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
|
||||
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory mg-coordinator)
|
||||
target_link_libraries(mg-query-v2 mg-expr mg-functions)
|
||||
target_link_libraries(mg-query-v2 mg-expr)
|
||||
|
||||
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
|
||||
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
|
||||
|
||||
@@ -10,25 +10,24 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
|
||||
namespace memgraph::query::v2::accessors {
|
||||
EdgeAccessor::EdgeAccessor(Edge edge, const RequestRouterInterface *request_router)
|
||||
: edge(std::move(edge)), request_router_(request_router) {}
|
||||
EdgeAccessor::EdgeAccessor(Edge edge, const msgs::ShardRequestManagerInterface *manager)
|
||||
: edge(std::move(edge)), manager_(manager) {}
|
||||
|
||||
EdgeTypeId EdgeAccessor::EdgeType() const { return edge.type.id; }
|
||||
|
||||
const std::vector<std::pair<PropertyId, Value>> &EdgeAccessor::Properties() const { return edge.properties; }
|
||||
|
||||
Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
|
||||
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
|
||||
if (!maybe_prop) {
|
||||
auto prop_id = manager_->NameToProperty(prop_name);
|
||||
auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
|
||||
if (it == edge.properties.end()) {
|
||||
return {};
|
||||
}
|
||||
const auto prop_id = *maybe_prop;
|
||||
auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
|
||||
return it->second;
|
||||
}
|
||||
|
||||
@@ -36,23 +35,21 @@ const Edge &EdgeAccessor::GetEdge() const { return edge; }
|
||||
|
||||
bool EdgeAccessor::IsCycle() const { return edge.src == edge.dst; };
|
||||
|
||||
size_t EdgeAccessor::CypherId() const { return edge.id.gid; }
|
||||
|
||||
VertexAccessor EdgeAccessor::To() const {
|
||||
return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
|
||||
return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, manager_);
|
||||
}
|
||||
|
||||
VertexAccessor EdgeAccessor::From() const {
|
||||
return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
|
||||
return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, manager_);
|
||||
}
|
||||
|
||||
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
|
||||
const RequestRouterInterface *request_router)
|
||||
: vertex(std::move(v)), properties(std::move(props)), request_router_(request_router) {}
|
||||
const msgs::ShardRequestManagerInterface *manager)
|
||||
: vertex(std::move(v)), properties(std::move(props)), manager_(manager) {}
|
||||
|
||||
VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
|
||||
const RequestRouterInterface *request_router)
|
||||
: vertex(std::move(v)), request_router_(request_router) {
|
||||
const msgs::ShardRequestManagerInterface *manager)
|
||||
: vertex(std::move(v)), manager_(manager) {
|
||||
properties.reserve(props.size());
|
||||
for (auto &[id, value] : props) {
|
||||
properties.emplace_back(std::make_pair(id, std::move(value)));
|
||||
@@ -60,8 +57,8 @@ VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
|
||||
}
|
||||
|
||||
VertexAccessor::VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props,
|
||||
const RequestRouterInterface *request_router)
|
||||
: vertex(std::move(v)), request_router_(request_router) {
|
||||
const msgs::ShardRequestManagerInterface *manager)
|
||||
: vertex(std::move(v)), manager_(manager) {
|
||||
properties.reserve(props.size());
|
||||
for (const auto &[id, value] : props) {
|
||||
properties.emplace_back(std::make_pair(id, value));
|
||||
@@ -91,11 +88,7 @@ Value VertexAccessor::GetProperty(PropertyId prop_id) const {
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
Value VertexAccessor::GetProperty(const std::string &prop_name) const {
|
||||
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
|
||||
if (!maybe_prop) {
|
||||
return {};
|
||||
}
|
||||
return GetProperty(*maybe_prop);
|
||||
return GetProperty(manager_->NameToProperty(prop_name));
|
||||
}
|
||||
|
||||
msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }
|
||||
|
||||
@@ -24,24 +24,24 @@
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
class RequestRouterInterface;
|
||||
} // namespace memgraph::query::v2
|
||||
namespace memgraph::msgs {
|
||||
class ShardRequestManagerInterface;
|
||||
} // namespace memgraph::msgs
|
||||
|
||||
namespace memgraph::query::v2::accessors {
|
||||
|
||||
using Value = msgs::Value;
|
||||
using Edge = msgs::Edge;
|
||||
using Vertex = msgs::Vertex;
|
||||
using Label = msgs::Label;
|
||||
using PropertyId = msgs::PropertyId;
|
||||
using EdgeTypeId = msgs::EdgeTypeId;
|
||||
using Value = memgraph::msgs::Value;
|
||||
using Edge = memgraph::msgs::Edge;
|
||||
using Vertex = memgraph::msgs::Vertex;
|
||||
using Label = memgraph::msgs::Label;
|
||||
using PropertyId = memgraph::msgs::PropertyId;
|
||||
using EdgeTypeId = memgraph::msgs::EdgeTypeId;
|
||||
|
||||
class VertexAccessor;
|
||||
|
||||
class EdgeAccessor final {
|
||||
public:
|
||||
explicit EdgeAccessor(Edge edge, const RequestRouterInterface *request_router);
|
||||
explicit EdgeAccessor(Edge edge, const msgs::ShardRequestManagerInterface *manager);
|
||||
|
||||
[[nodiscard]] EdgeTypeId EdgeType() const;
|
||||
|
||||
@@ -53,7 +53,12 @@ class EdgeAccessor final {
|
||||
|
||||
[[nodiscard]] bool IsCycle() const;
|
||||
|
||||
[[nodiscard]] size_t CypherId() const;
|
||||
// Dummy function
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] size_t CypherId() const { return 10; }
|
||||
|
||||
// bool HasSrcAccessor const { return src == nullptr; }
|
||||
// bool HasDstAccessor const { return dst == nullptr; }
|
||||
|
||||
[[nodiscard]] VertexAccessor To() const;
|
||||
[[nodiscard]] VertexAccessor From() const;
|
||||
@@ -64,7 +69,7 @@ class EdgeAccessor final {
|
||||
|
||||
private:
|
||||
Edge edge;
|
||||
const RequestRouterInterface *request_router_;
|
||||
const msgs::ShardRequestManagerInterface *manager_;
|
||||
};
|
||||
|
||||
class VertexAccessor final {
|
||||
@@ -73,10 +78,10 @@ class VertexAccessor final {
|
||||
using Label = msgs::Label;
|
||||
using VertexId = msgs::VertexId;
|
||||
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
|
||||
const RequestRouterInterface *request_router);
|
||||
const msgs::ShardRequestManagerInterface *manager);
|
||||
|
||||
VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const RequestRouterInterface *request_router);
|
||||
VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const RequestRouterInterface *request_router);
|
||||
VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const msgs::ShardRequestManagerInterface *manager);
|
||||
VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const msgs::ShardRequestManagerInterface *manager);
|
||||
|
||||
[[nodiscard]] Label PrimaryLabel() const;
|
||||
|
||||
@@ -93,11 +98,48 @@ class VertexAccessor final {
|
||||
|
||||
[[nodiscard]] msgs::Vertex GetVertex() const;
|
||||
|
||||
// Dummy function
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] size_t InDegree() const { throw utils::NotYetImplemented("InDegree() not yet implemented"); }
|
||||
[[nodiscard]] size_t CypherId() const { return 10; }
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] size_t OutDegree() const { throw utils::NotYetImplemented("OutDegree() not yet implemented"); }
|
||||
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
// auto maybe_edges = impl_.InEdges(view, edge_types);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
//
|
||||
// auto InEdges(storage::View view) const { return InEdges(view, {}); }
|
||||
//
|
||||
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest)
|
||||
// const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
// auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
//
|
||||
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
// auto maybe_edges = impl_.OutEdges(view, edge_types);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
//
|
||||
// auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
|
||||
//
|
||||
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
|
||||
// const VertexAccessor &dest) const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
// auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
|
||||
// storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
|
||||
//
|
||||
// storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
|
||||
//
|
||||
|
||||
friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
|
||||
return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
|
||||
@@ -108,9 +150,13 @@ class VertexAccessor final {
|
||||
private:
|
||||
Vertex vertex;
|
||||
std::vector<std::pair<PropertyId, Value>> properties;
|
||||
const RequestRouterInterface *request_router_;
|
||||
const msgs::ShardRequestManagerInterface *manager_;
|
||||
};
|
||||
|
||||
// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
|
||||
|
||||
// inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
|
||||
|
||||
// Highly mocked interface. Won't work if used.
|
||||
class Path {
|
||||
public:
|
||||
@@ -151,14 +197,7 @@ class Path {
|
||||
friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
|
||||
utils::MemoryResource *GetMemoryResource() { return mem; }
|
||||
|
||||
auto &vertices() { return vertices_; }
|
||||
auto &edges() { return edges_; }
|
||||
const auto &vertices() const { return vertices_; }
|
||||
const auto &edges() const { return edges_; }
|
||||
|
||||
private:
|
||||
std::vector<VertexAccessor> vertices_;
|
||||
std::vector<EdgeAccessor> edges_;
|
||||
utils::MemoryResource *mem = utils::NewDeleteResource();
|
||||
};
|
||||
} // namespace memgraph::query::v2::accessors
|
||||
|
||||
@@ -21,27 +21,29 @@
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/conversions.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
|
||||
namespace memgraph::msgs {
|
||||
class ShardRequestManagerInterface;
|
||||
} // namespace memgraph::msgs
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
class RequestRouterInterface;
|
||||
|
||||
inline const auto lam = [](const auto &val) { return ValueToTypedValue(val); };
|
||||
namespace detail {
|
||||
class Callable {
|
||||
public:
|
||||
auto operator()(const storage::v3::PropertyValue &val) const {
|
||||
return storage::v3::PropertyToTypedValue<TypedValue>(val);
|
||||
auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
|
||||
return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
|
||||
};
|
||||
auto operator()(const msgs::Value &val, RequestRouterInterface *request_router) const {
|
||||
return ValueToTypedValue(val, request_router);
|
||||
auto operator()(const msgs::Value &val, memgraph::msgs::ShardRequestManagerInterface *manager) const {
|
||||
return ValueToTypedValue(val, manager);
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
using ExpressionEvaluator = expr::ExpressionEvaluator<TypedValue, query::v2::EvaluationContext, RequestRouterInterface,
|
||||
storage::v3::View, storage::v3::LabelId, msgs::Value,
|
||||
detail::Callable, common::ErrorCode, expr::QueryEngineTag>;
|
||||
using ExpressionEvaluator = memgraph::expr::ExpressionEvaluator<
|
||||
TypedValue, memgraph::query::v2::EvaluationContext, memgraph::msgs::ShardRequestManagerInterface, storage::v3::View,
|
||||
storage::v3::LabelId, msgs::Value, detail::Callable, memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -13,10 +13,9 @@
|
||||
|
||||
#include "query/v2/bindings/bindings.hpp"
|
||||
|
||||
#include "expr/interpret/frame.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "expr/interpret/frame.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
using Frame = memgraph::expr::Frame;
|
||||
using FrameWithValidity = memgraph::expr::FrameWithValidity;
|
||||
} // namespace memgraph::query::v2
|
||||
using Frame = memgraph::expr::Frame<TypedValue>;
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
#include "query/v2/bindings/symbol.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/exceptions.hpp"
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/path.hpp"
|
||||
@@ -27,6 +28,7 @@
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/shard_operation_result.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -80,5 +82,39 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
|
||||
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue new_value) {
|
||||
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue new_value) {
|
||||
{
|
||||
accessor.SetPropertyAndValidate(key, new_value)
|
||||
} -> std::same_as<storage::v3::ShardOperationResult<storage::v3::PropertyValue>>;
|
||||
};
|
||||
|
||||
template <typename TRecordAccessor>
|
||||
concept RecordAccessor =
|
||||
AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
|
||||
|
||||
inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
|
||||
switch (error) {
|
||||
case storage::v3::Error::SERIALIZATION_ERROR:
|
||||
throw TransactionSerializationException();
|
||||
case storage::v3::Error::DELETED_OBJECT:
|
||||
throw QueryRuntimeException("Trying to set properties on a deleted object.");
|
||||
case storage::v3::Error::PROPERTIES_DISABLED:
|
||||
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
|
||||
case storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
case storage::v3::Error::VERTEX_ALREADY_INSERTED:
|
||||
|
||||
throw QueryRuntimeException("Unexpected error when setting a property.");
|
||||
}
|
||||
}
|
||||
|
||||
int64_t QueryTimestamp();
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -20,7 +20,7 @@
|
||||
#include "query/v2/parameters.hpp"
|
||||
#include "query/v2/plan/profile.hpp"
|
||||
//#include "query/v2/trigger.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "utils/async_timer.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
@@ -60,27 +60,27 @@ struct EvaluationContext {
|
||||
mutable std::unordered_map<std::string, int64_t> counters;
|
||||
};
|
||||
|
||||
inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
|
||||
RequestRouterInterface *request_router) {
|
||||
inline std::vector<storage::v3::PropertyId> NamesToProperties(
|
||||
const std::vector<std::string> &property_names, msgs::ShardRequestManagerInterface *shard_request_manager) {
|
||||
std::vector<storage::v3::PropertyId> properties;
|
||||
// TODO Fix by using reference
|
||||
properties.reserve(property_names.size());
|
||||
if (request_router != nullptr) {
|
||||
if (shard_request_manager != nullptr) {
|
||||
for (const auto &name : property_names) {
|
||||
properties.push_back(request_router->NameToProperty(name));
|
||||
properties.push_back(shard_request_manager->NameToProperty(name));
|
||||
}
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names,
|
||||
RequestRouterInterface *request_router) {
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager) {
|
||||
std::vector<storage::v3::LabelId> labels;
|
||||
labels.reserve(label_names.size());
|
||||
// TODO Fix by using reference
|
||||
if (request_router != nullptr) {
|
||||
if (shard_request_manager != nullptr) {
|
||||
for (const auto &name : label_names) {
|
||||
labels.push_back(request_router->NameToLabel(name));
|
||||
labels.push_back(shard_request_manager->NameToLabel(name));
|
||||
}
|
||||
}
|
||||
return labels;
|
||||
@@ -97,7 +97,7 @@ struct ExecutionContext {
|
||||
plan::ProfilingStats *stats_root{nullptr};
|
||||
ExecutionStats execution_stats;
|
||||
utils::AsyncTimer timer;
|
||||
RequestRouterInterface *request_router{nullptr};
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager{nullptr};
|
||||
IdAllocator *edge_ids_alloc;
|
||||
};
|
||||
|
||||
|
||||
@@ -12,12 +12,12 @@
|
||||
#pragma once
|
||||
#include "bindings/typed_value.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInterface *request_router) {
|
||||
inline TypedValue ValueToTypedValue(const msgs::Value &value, msgs::ShardRequestManagerInterface *manager) {
|
||||
using Value = msgs::Value;
|
||||
switch (value.type) {
|
||||
case Value::Type::Null:
|
||||
@@ -35,7 +35,7 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
|
||||
std::vector<TypedValue> dst;
|
||||
dst.reserve(lst.size());
|
||||
for (const auto &elem : lst) {
|
||||
dst.push_back(ValueToTypedValue(elem, request_router));
|
||||
dst.push_back(ValueToTypedValue(elem, manager));
|
||||
}
|
||||
return TypedValue(std::move(dst));
|
||||
}
|
||||
@@ -43,23 +43,19 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
|
||||
const auto &value_map = value.map_v;
|
||||
std::map<std::string, TypedValue> dst;
|
||||
for (const auto &[key, val] : value_map) {
|
||||
dst[key] = ValueToTypedValue(val, request_router);
|
||||
dst[key] = ValueToTypedValue(val, manager);
|
||||
}
|
||||
return TypedValue(std::move(dst));
|
||||
}
|
||||
case Value::Type::Vertex:
|
||||
return TypedValue(accessors::VertexAccessor(
|
||||
value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, request_router));
|
||||
value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, manager));
|
||||
case Value::Type::Edge:
|
||||
return TypedValue(accessors::EdgeAccessor(value.edge_v, request_router));
|
||||
return TypedValue(accessors::EdgeAccessor(value.edge_v, manager));
|
||||
}
|
||||
throw std::runtime_error("Incorrect type in conversion");
|
||||
}
|
||||
|
||||
inline const auto ValueToTypedValueFunctor = [](const msgs::Value &value, RequestRouterInterface *request_router) {
|
||||
return ValueToTypedValue(value, request_router);
|
||||
};
|
||||
|
||||
inline msgs::Value TypedValueToValue(const TypedValue &value) {
|
||||
using Value = msgs::Value;
|
||||
switch (value.type()) {
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
|
||||
#include "query/v2/cypher_query_interpreter.hpp"
|
||||
#include "query/v2/bindings/symbol_generator.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
|
||||
@@ -118,9 +118,9 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
|
||||
}
|
||||
|
||||
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router,
|
||||
msgs::ShardRequestManagerInterface *shard_manager,
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
auto vertex_counts = plan::MakeVertexCountCache(request_router);
|
||||
auto vertex_counts = plan::MakeVertexCountCache(shard_manager);
|
||||
auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
|
||||
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
|
||||
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
|
||||
@@ -130,7 +130,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
|
||||
|
||||
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, utils::SkipList<PlanCacheEntry> *plan_cache,
|
||||
RequestRouterInterface *request_router,
|
||||
msgs::ShardRequestManagerInterface *shard_manager,
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
|
||||
if (plan_cache) {
|
||||
@@ -146,7 +146,7 @@ std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_stor
|
||||
}
|
||||
|
||||
auto plan = std::make_shared<CachedPlan>(
|
||||
MakeLogicalPlan(std::move(ast_storage), query, parameters, request_router, predefined_identifiers));
|
||||
MakeLogicalPlan(std::move(ast_storage), query, parameters, shard_manager, predefined_identifiers));
|
||||
if (plan_cache_access) {
|
||||
plan_cache_access->insert({hash, plan});
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
|
||||
};
|
||||
|
||||
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router,
|
||||
msgs::ShardRequestManagerInterface *shard_manager,
|
||||
const std::vector<Identifier *> &predefined_identifiers);
|
||||
|
||||
/**
|
||||
@@ -145,7 +145,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
|
||||
*/
|
||||
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, utils::SkipList<PlanCacheEntry> *plan_cache,
|
||||
RequestRouterInterface *request_router,
|
||||
msgs::ShardRequestManagerInterface *shard_manager,
|
||||
const std::vector<Identifier *> &predefined_identifiers = {});
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/shard_operation_result.hpp"
|
||||
|
||||
///////////////////////////////////////////////////////////
|
||||
// Our communication layer and query engine don't mix
|
||||
@@ -36,7 +37,7 @@
|
||||
// This cannot be avoided by simple include orderings so we
|
||||
// simply undefine those macros as we're sure that libkrb5
|
||||
// won't and can't be used anywhere in the query engine.
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/storage.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
@@ -64,11 +65,24 @@ class EdgeAccessor final {
|
||||
|
||||
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
|
||||
|
||||
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
return impl_.GetProperty(key, view);
|
||||
}
|
||||
|
||||
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue &value) {
|
||||
return impl_.SetProperty(key, value);
|
||||
}
|
||||
|
||||
storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
|
||||
return SetProperty(key, storage::v3::PropertyValue());
|
||||
}
|
||||
|
||||
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
|
||||
return impl_.ClearProperties();
|
||||
}
|
||||
|
||||
VertexAccessor To() const;
|
||||
|
||||
VertexAccessor From() const;
|
||||
@@ -100,19 +114,53 @@ class VertexAccessor final {
|
||||
|
||||
auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
|
||||
|
||||
storage::v3::ShardResult<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
|
||||
storage::v3::ShardOperationResult<bool> AddLabel(storage::v3::LabelId label) {
|
||||
return impl_.AddLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::ShardOperationResult<bool> AddLabelAndValidate(storage::v3::LabelId label) {
|
||||
return impl_.AddLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::ShardOperationResult<bool> RemoveLabel(storage::v3::LabelId label) {
|
||||
return impl_.RemoveLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::ShardOperationResult<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
|
||||
return impl_.RemoveLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
|
||||
return impl_.HasLabel(label, view);
|
||||
}
|
||||
|
||||
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
|
||||
|
||||
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
return impl_.GetProperty(key, view);
|
||||
}
|
||||
|
||||
storage::v3::ShardOperationResult<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue &value) {
|
||||
return impl_.SetPropertyAndValidate(key, value);
|
||||
}
|
||||
|
||||
storage::v3::ShardOperationResult<storage::v3::PropertyValue> SetPropertyAndValidate(
|
||||
storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
|
||||
return impl_.SetPropertyAndValidate(key, value);
|
||||
}
|
||||
|
||||
storage::v3::ShardOperationResult<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
|
||||
return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
|
||||
}
|
||||
|
||||
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
|
||||
return impl_.ClearProperties();
|
||||
}
|
||||
|
||||
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
auto maybe_edges = impl_.InEdges(view, edge_types);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
@@ -122,7 +170,7 @@ class VertexAccessor final {
|
||||
|
||||
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
|
||||
const VertexAccessor &dest) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
const auto dest_id = dest.impl_.Id(view).GetValue();
|
||||
auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
@@ -130,7 +178,7 @@ class VertexAccessor final {
|
||||
}
|
||||
|
||||
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
auto maybe_edges = impl_.OutEdges(view, edge_types);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
@@ -140,16 +188,16 @@ class VertexAccessor final {
|
||||
|
||||
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
|
||||
const VertexAccessor &dest) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
const auto dest_id = dest.impl_.Id(view).GetValue();
|
||||
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
}
|
||||
|
||||
storage::v3::ShardResult<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
|
||||
storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
|
||||
|
||||
storage::v3::ShardResult<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
|
||||
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
|
||||
|
||||
// TODO(jbajic) Fix Remove Gid
|
||||
static int64_t CypherId() { return 1; }
|
||||
|
||||
@@ -224,4 +224,12 @@ class VersionInfoInMulticommandTxException : public QueryException {
|
||||
: QueryException("Version info query not allowed in multicommand transactions.") {}
|
||||
};
|
||||
|
||||
/**
|
||||
* An exception for an illegal operation that violates schema
|
||||
*/
|
||||
class SchemaViolationException : public QueryRuntimeException {
|
||||
public:
|
||||
using QueryRuntimeException::QueryRuntimeException;
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -20,13 +20,11 @@
|
||||
#include "query/v2/bindings/ast_visitor.hpp"
|
||||
#include "common/types.hpp"
|
||||
#include "query/v2/bindings/symbol.hpp"
|
||||
#include "functions/awesome_memgraph_functions.hpp"
|
||||
#include "query/v2/interpret/awesome_memgraph_functions.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/path.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
#include "query/v2/conversions.hpp"
|
||||
|
||||
cpp<#
|
||||
|
||||
@@ -838,15 +836,13 @@ cpp<#
|
||||
:slk-load (slk-load-ast-vector "Expression"))
|
||||
(function-name "std::string" :scope :public)
|
||||
(function "std::function<TypedValue(const TypedValue *, int64_t,
|
||||
const functions::FunctionContext<RequestRouterInterface> &)>"
|
||||
const FunctionContext &)>"
|
||||
:scope :public
|
||||
:dont-save t
|
||||
:clone :copy
|
||||
:slk-load (lambda (member)
|
||||
#>cpp
|
||||
self->${member} = functions::NameToFunction<TypedValue,
|
||||
functions::FunctionContext<RequestRouterInterface>,
|
||||
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(self->function_name_);
|
||||
self->${member} = query::v2::NameToFunction(self->function_name_);
|
||||
cpp<#)))
|
||||
(:public
|
||||
#>cpp
|
||||
@@ -869,8 +865,7 @@ cpp<#
|
||||
const std::vector<Expression *> &arguments)
|
||||
: arguments_(arguments),
|
||||
function_name_(function_name),
|
||||
function_(functions::NameToFunction<TypedValue, functions::FunctionContext<RequestRouterInterface>,
|
||||
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(function_name_)) {
|
||||
function_(NameToFunction(function_name_)) {
|
||||
if (!function_) {
|
||||
throw SemanticException("Function '{}' doesn't exist.", function_name);
|
||||
}
|
||||
|
||||
1108
src/query/v2/interpret/awesome_memgraph_functions.cpp
Normal file
1108
src/query/v2/interpret/awesome_memgraph_functions.cpp
Normal file
File diff suppressed because it is too large
Load Diff
@@ -20,9 +20,11 @@
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
namespace memgraph::msgs {
|
||||
class ShardRequestManagerInterface;
|
||||
} // namespace memgraph::msgs
|
||||
|
||||
class RequestRouterInterface;
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
namespace {
|
||||
const char kStartsWith[] = "STARTSWITH";
|
||||
@@ -32,9 +34,9 @@ const char kId[] = "ID";
|
||||
} // namespace
|
||||
|
||||
struct FunctionContext {
|
||||
// TODO(kostasrim) consider optional here. RequestRouter does not exist on the storage.
|
||||
// TODO(kostasrim) consider optional here. ShardRequestManager does not exist on the storage.
|
||||
// DbAccessor *db_accessor;
|
||||
RequestRouterInterface *request_router;
|
||||
msgs::ShardRequestManagerInterface *manager;
|
||||
utils::MemoryResource *memory;
|
||||
int64_t timestamp;
|
||||
std::unordered_map<std::string, int64_t> *counters;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -41,13 +41,13 @@
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/frontend/semantic/required_privileges.hpp"
|
||||
#include "query/v2/metadata.hpp"
|
||||
#include "query/v2/multiframe.hpp"
|
||||
#include "query/v2/plan/planner.hpp"
|
||||
#include "query/v2/plan/profile.hpp"
|
||||
#include "query/v2/plan/vertex_count_cache.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/storage.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/csv_parsing.hpp"
|
||||
#include "utils/event_counter.hpp"
|
||||
@@ -64,11 +64,7 @@
|
||||
#include "utils/tsc.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
|
||||
|
||||
namespace EventCounter {
|
||||
|
||||
extern Event ReadQuery;
|
||||
extern Event WriteQuery;
|
||||
extern Event ReadWriteQuery;
|
||||
@@ -78,7 +74,6 @@ extern const Event LabelPropertyIndexCreated;
|
||||
|
||||
extern const Event StreamsCreated;
|
||||
extern const Event TriggersCreated;
|
||||
|
||||
} // namespace EventCounter
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
@@ -149,18 +144,18 @@ class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
|
||||
Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router) {
|
||||
msgs::ShardRequestManagerInterface *manager) {
|
||||
// Empty frame for evaluation of password expression. This is OK since
|
||||
// password should be either null or string literal and it's evaluation
|
||||
// should not depend on frame.
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
// the argument to Callback.
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
|
||||
|
||||
std::string username = auth_query->user_;
|
||||
std::string rolename = auth_query->role_;
|
||||
@@ -318,16 +313,16 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
|
||||
}
|
||||
|
||||
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters,
|
||||
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
|
||||
InterpreterContext *interpreter_context, msgs::ShardRequestManagerInterface *manager,
|
||||
std::vector<Notification> *notifications) {
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
// the argument to Callback.
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
|
||||
|
||||
Callback callback;
|
||||
switch (repl_query->action_) {
|
||||
@@ -454,8 +449,8 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
}
|
||||
|
||||
Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router) {
|
||||
expr::Frame frame(0);
|
||||
msgs::ShardRequestManagerInterface *manager) {
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
@@ -464,7 +459,7 @@ Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶m
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
evaluation_context.parameters = parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
|
||||
|
||||
Callback callback;
|
||||
switch (setting_query->action_) {
|
||||
@@ -655,21 +650,17 @@ struct PullPlanVector {
|
||||
struct PullPlan {
|
||||
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router = nullptr,
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager = nullptr,
|
||||
// TriggerContextCollector *trigger_context_collector = nullptr,
|
||||
std::optional<size_t> memory_limit = {});
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> PullMultiple(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
|
||||
private:
|
||||
std::shared_ptr<CachedPlan> plan_ = nullptr;
|
||||
plan::UniqueCursorPtr cursor_ = nullptr;
|
||||
expr::FrameWithValidity frame_;
|
||||
MultiFrame multi_frame_;
|
||||
expr::Frame<TypedValue> frame_;
|
||||
ExecutionContext ctx_;
|
||||
std::optional<size_t> memory_limit_;
|
||||
|
||||
@@ -689,137 +680,29 @@ struct PullPlan {
|
||||
|
||||
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router, const std::optional<size_t> memory_limit)
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager, const std::optional<size_t> memory_limit)
|
||||
: plan_(plan),
|
||||
cursor_(plan->plan().MakeCursor(execution_memory)),
|
||||
frame_(plan->symbol_table().max_position(), execution_memory),
|
||||
multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
|
||||
memory_limit_(memory_limit) {
|
||||
ctx_.db_accessor = dba;
|
||||
ctx_.symbol_table = plan->symbol_table();
|
||||
ctx_.evaluation_context.timestamp = QueryTimestamp();
|
||||
ctx_.evaluation_context.parameters = parameters;
|
||||
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, request_router);
|
||||
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, request_router);
|
||||
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, shard_request_manager);
|
||||
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, shard_request_manager);
|
||||
if (interpreter_context->config.execution_timeout_sec > 0) {
|
||||
ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec};
|
||||
}
|
||||
ctx_.is_shutting_down = &interpreter_context->is_shutting_down;
|
||||
ctx_.is_profile_query = is_profile_query;
|
||||
ctx_.request_router = request_router;
|
||||
ctx_.shard_request_manager = shard_request_manager;
|
||||
ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc;
|
||||
}
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary) {
|
||||
// Set up temporary memory for a single Pull. Initial memory comes from the
|
||||
// stack. 256 KiB should fit on the stack and should be more than enough for a
|
||||
// single `Pull`.
|
||||
MG_ASSERT(!n.has_value(), "should pull all!");
|
||||
static constexpr size_t stack_size = 256UL * 1024UL;
|
||||
char stack_data[stack_size];
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
|
||||
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
|
||||
|
||||
if (memory_limit_) {
|
||||
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
|
||||
ctx_.evaluation_context.memory = &*maybe_limited_resource;
|
||||
} else {
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
}
|
||||
|
||||
// Returns true if a result was pulled.
|
||||
const auto pull_result = [&]() -> bool { return cursor_->PullMultiple(multi_frame_, ctx_); };
|
||||
|
||||
const auto stream_values = [&output_symbols, &stream](const Frame &frame) {
|
||||
// TODO: The streamed values should also probably use the above memory.
|
||||
std::vector<TypedValue> values;
|
||||
values.reserve(output_symbols.size());
|
||||
|
||||
for (const auto &symbol : output_symbols) {
|
||||
values.emplace_back(frame[symbol]);
|
||||
}
|
||||
|
||||
stream->Result(values);
|
||||
};
|
||||
|
||||
// Get the execution time of all possible result pulls and streams.
|
||||
utils::Timer timer;
|
||||
|
||||
int i = 0;
|
||||
if (has_unsent_results_ && !output_symbols.empty()) {
|
||||
// stream unsent results from previous pull
|
||||
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
|
||||
stream_values(frame);
|
||||
frame.MakeInvalid();
|
||||
++i;
|
||||
if (i == n) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (; !n || i < n;) {
|
||||
if (!pull_result()) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (!output_symbols.empty()) {
|
||||
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
|
||||
stream_values(frame);
|
||||
frame.MakeInvalid();
|
||||
++i;
|
||||
if (i == n) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
multi_frame_.MakeAllFramesInvalid();
|
||||
}
|
||||
}
|
||||
|
||||
// If we finished because we streamed the requested n results,
|
||||
// we try to pull the next result to see if there is more.
|
||||
// If there is additional result, we leave the pulled result in the frame
|
||||
// and set the flag to true.
|
||||
has_unsent_results_ = i == n && pull_result();
|
||||
|
||||
execution_time_ += timer.Elapsed();
|
||||
|
||||
if (has_unsent_results_) {
|
||||
return std::nullopt;
|
||||
}
|
||||
summary->insert_or_assign("plan_execution_time", execution_time_.count());
|
||||
// We are finished with pulling all the data, therefore we can send any
|
||||
// metadata about the results i.e. notifications and statistics
|
||||
const bool is_any_counter_set =
|
||||
std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(),
|
||||
[](const auto &counter) { return counter > 0; });
|
||||
if (is_any_counter_set) {
|
||||
std::map<std::string, TypedValue> stats;
|
||||
for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) {
|
||||
stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]);
|
||||
}
|
||||
summary->insert_or_assign("stats", std::move(stats));
|
||||
}
|
||||
cursor_->Shutdown();
|
||||
ctx_.profile_execution_time = execution_time_;
|
||||
return GetStatsWithTotalTime(ctx_);
|
||||
}
|
||||
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary) {
|
||||
if (FLAGS_use_multi_frame) {
|
||||
return PullMultiple(stream, n, output_symbols, summary);
|
||||
}
|
||||
// Set up temporary memory for a single Pull. Initial memory comes from the
|
||||
// stack. 256 KiB should fit on the stack and should be more than enough for a
|
||||
// single `Pull`.
|
||||
@@ -911,24 +794,30 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
|
||||
|
||||
InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config,
|
||||
const std::filesystem::path & /*data_directory*/,
|
||||
std::unique_ptr<RequestRouterFactory> request_router_factory,
|
||||
io::Io<io::local_transport::LocalTransport> io,
|
||||
coordinator::Address coordinator_addr)
|
||||
: db(db),
|
||||
config(config),
|
||||
coordinator_address{coordinator_addr},
|
||||
request_router_factory_{std::move(request_router_factory)} {}
|
||||
: db(db), config(config), io{std::move(io)}, coordinator_address{coordinator_addr} {}
|
||||
|
||||
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
|
||||
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
|
||||
|
||||
request_router_ =
|
||||
interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address);
|
||||
|
||||
auto query_io = interpreter_context_->io.ForkLocal();
|
||||
shard_request_manager_ = std::make_unique<msgs::ShardRequestManager<io::local_transport::LocalTransport>>(
|
||||
coordinator::CoordinatorClient<io::local_transport::LocalTransport>(
|
||||
query_io, interpreter_context_->coordinator_address, std::vector{interpreter_context_->coordinator_address}),
|
||||
std::move(query_io));
|
||||
// Get edge ids
|
||||
const auto edge_ids_alloc_min_max_pair =
|
||||
request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address);
|
||||
if (edge_ids_alloc_min_max_pair) {
|
||||
interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second};
|
||||
coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
|
||||
io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
|
||||
ww.operation = requests;
|
||||
auto resp = interpreter_context_->io
|
||||
.Request<io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>,
|
||||
io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>>(
|
||||
interpreter_context_->coordinator_address, ww)
|
||||
.Wait();
|
||||
if (resp.HasValue()) {
|
||||
const auto alloc_edge_id_reps =
|
||||
std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);
|
||||
interpreter_context_->edge_ids_alloc = {alloc_edge_id_reps.low, alloc_edge_id_reps.high};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -989,16 +878,17 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
|
||||
RequestRouterInterface *request_router) {
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager) {
|
||||
// TriggerContextCollector *trigger_context_collector = nullptr) {
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
|
||||
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parsed_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, shard_request_manager,
|
||||
storage::v3::View::OLD);
|
||||
const auto memory_limit =
|
||||
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
if (memory_limit) {
|
||||
@@ -1013,9 +903,9 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
"convert the parsed row values to the appropriate type. This can be done using the built-in "
|
||||
"conversion functions such as ToInteger, ToFloat, ToBoolean etc.");
|
||||
}
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
|
||||
parsed_query.parameters,
|
||||
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
|
||||
auto plan = CypherQueryToPlan(
|
||||
parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query, parsed_query.parameters,
|
||||
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, shard_request_manager);
|
||||
|
||||
summary->insert_or_assign("cost_estimate", plan->cost());
|
||||
auto rw_type_checker = plan::ReadWriteTypeChecker();
|
||||
@@ -1034,7 +924,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
|
||||
}
|
||||
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
|
||||
execution_memory, request_router, memory_limit);
|
||||
execution_memory, shard_request_manager, memory_limit);
|
||||
// execution_memory, trigger_context_collector, memory_limit);
|
||||
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
|
||||
@@ -1048,7 +938,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
}
|
||||
|
||||
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
|
||||
InterpreterContext *interpreter_context,
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager,
|
||||
utils::MemoryResource *execution_memory) {
|
||||
const std::string kExplainQueryStart = "explain ";
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
|
||||
@@ -1067,20 +958,20 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
|
||||
|
||||
auto cypher_query_plan =
|
||||
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
|
||||
cypher_query, parsed_inner_query.parameters,
|
||||
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
|
||||
auto cypher_query_plan = CypherQueryToPlan(
|
||||
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
|
||||
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr,
|
||||
shard_request_manager);
|
||||
|
||||
std::stringstream printed_plan;
|
||||
plan::PrettyPrint(*request_router, &cypher_query_plan->plan(), &printed_plan);
|
||||
plan::PrettyPrint(*shard_request_manager, &cypher_query_plan->plan(), &printed_plan);
|
||||
|
||||
std::vector<std::vector<TypedValue>> printed_plan_rows;
|
||||
for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) {
|
||||
printed_plan_rows.push_back(std::vector<TypedValue>{TypedValue(row)});
|
||||
}
|
||||
|
||||
summary->insert_or_assign("explain", plan::PlanToJson(*request_router, &cypher_query_plan->plan()).dump());
|
||||
summary->insert_or_assign("explain", plan::PlanToJson(*shard_request_manager, &cypher_query_plan->plan()).dump());
|
||||
|
||||
return PreparedQuery{{"QUERY PLAN"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
@@ -1097,7 +988,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
|
||||
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router = nullptr) {
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager = nullptr) {
|
||||
const std::string kProfileQueryStart = "profile ";
|
||||
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
|
||||
@@ -1135,26 +1026,27 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parsed_inner_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, shard_request_manager,
|
||||
storage::v3::View::OLD);
|
||||
const auto memory_limit =
|
||||
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
|
||||
auto cypher_query_plan =
|
||||
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
|
||||
cypher_query, parsed_inner_query.parameters,
|
||||
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
|
||||
auto cypher_query_plan = CypherQueryToPlan(
|
||||
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
|
||||
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr,
|
||||
shard_request_manager);
|
||||
auto rw_type_checker = plan::ReadWriteTypeChecker();
|
||||
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
|
||||
|
||||
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME", "CUSTOM DATA"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
|
||||
summary, dba, interpreter_context, execution_memory, memory_limit, request_router,
|
||||
summary, dba, interpreter_context, execution_memory, memory_limit, shard_request_manager,
|
||||
// We want to execute the query we are profiling lazily, so we delay
|
||||
// the construction of the corresponding context.
|
||||
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
|
||||
@@ -1163,7 +1055,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
// No output symbols are given so that nothing is streamed.
|
||||
if (!stats_and_total_time) {
|
||||
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
|
||||
execution_memory, request_router, memory_limit)
|
||||
execution_memory, shard_request_manager, memory_limit)
|
||||
.Pull(stream, {}, {}, summary);
|
||||
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
|
||||
}
|
||||
@@ -1290,14 +1182,14 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
|
||||
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router) {
|
||||
msgs::ShardRequestManagerInterface *manager) {
|
||||
if (in_explicit_transaction) {
|
||||
throw UserModificationInMulticommandTxException();
|
||||
}
|
||||
|
||||
auto *auth_query = utils::Downcast<AuthQuery>(parsed_query.query);
|
||||
|
||||
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, request_router);
|
||||
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, manager);
|
||||
|
||||
SymbolTable symbol_table;
|
||||
std::vector<Symbol> output_symbols;
|
||||
@@ -1326,14 +1218,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
|
||||
|
||||
PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
std::vector<Notification> *notifications, InterpreterContext *interpreter_context,
|
||||
RequestRouterInterface *request_router) {
|
||||
msgs::ShardRequestManagerInterface *manager) {
|
||||
if (in_explicit_transaction) {
|
||||
throw ReplicationModificationInMulticommandTxException();
|
||||
}
|
||||
|
||||
auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
|
||||
auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context,
|
||||
request_router, notifications);
|
||||
auto callback =
|
||||
HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context, manager, notifications);
|
||||
|
||||
return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
|
||||
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
|
||||
@@ -1422,14 +1314,14 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
|
||||
}
|
||||
|
||||
PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
RequestRouterInterface *request_router) {
|
||||
msgs::ShardRequestManagerInterface *manager) {
|
||||
if (in_explicit_transaction) {
|
||||
throw SettingConfigInMulticommandTxException{};
|
||||
}
|
||||
|
||||
auto *setting_query = utils::Downcast<SettingQuery>(parsed_query.query);
|
||||
MG_ASSERT(setting_query);
|
||||
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, request_router);
|
||||
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, manager);
|
||||
|
||||
return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges),
|
||||
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
|
||||
@@ -1626,7 +1518,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
if (!in_explicit_transaction_ &&
|
||||
(utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ExplainQuery>(parsed_query.query) ||
|
||||
utils::Downcast<ProfileQuery>(parsed_query.query))) {
|
||||
request_router_->StartTransaction();
|
||||
shard_request_manager_->StartTransaction();
|
||||
}
|
||||
|
||||
utils::Timer planning_timer;
|
||||
@@ -1635,14 +1527,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory,
|
||||
&query_execution->notifications, request_router_.get());
|
||||
&query_execution->notifications, shard_request_manager_.get());
|
||||
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*request_router_, &query_execution->execution_memory_with_exception);
|
||||
&*shard_request_manager_, &query_execution->execution_memory_with_exception);
|
||||
} else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
|
||||
interpreter_context_, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory_with_exception, request_router_.get());
|
||||
prepared_query = PrepareProfileQuery(
|
||||
std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get());
|
||||
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory);
|
||||
@@ -1650,9 +1542,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
prepared_query = PrepareIndexQuery(std::move(parsed_query), in_explicit_transaction_,
|
||||
&query_execution->notifications, interpreter_context_);
|
||||
} else if (utils::Downcast<AuthQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
|
||||
interpreter_context_, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory_with_exception, request_router_.get());
|
||||
prepared_query = PrepareAuthQuery(
|
||||
std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get());
|
||||
} else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
|
||||
interpreter_context_, interpreter_context_->db,
|
||||
@@ -1663,7 +1555,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
} else if (utils::Downcast<ReplicationQuery>(parsed_query.query)) {
|
||||
prepared_query =
|
||||
PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
|
||||
interpreter_context_, request_router_.get());
|
||||
interpreter_context_, shard_request_manager_.get());
|
||||
} else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_,
|
||||
&*execution_db_accessor_);
|
||||
@@ -1680,7 +1572,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
prepared_query =
|
||||
PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
|
||||
} else if (utils::Downcast<SettingQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, request_router_.get());
|
||||
prepared_query =
|
||||
PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, shard_request_manager_.get());
|
||||
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
|
||||
} else if (utils::Downcast<SchemaQuery>(parsed_query.query)) {
|
||||
@@ -1721,7 +1614,7 @@ void Interpreter::Commit() {
|
||||
// For now, we will not check if there are some unfinished queries.
|
||||
// We should document clearly that all results should be pulled to complete
|
||||
// a query.
|
||||
request_router_->Commit();
|
||||
shard_request_manager_->Commit();
|
||||
if (!db_accessor_) return;
|
||||
|
||||
const auto reset_necessary_members = [this]() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/coordinator_client.hpp"
|
||||
#include "io/local_transport/local_transport.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "query/v2/auth_checker.hpp"
|
||||
#include "query/v2/bindings/cypher_main_visitor.hpp"
|
||||
@@ -171,8 +172,7 @@ struct PreparedQuery {
|
||||
struct InterpreterContext {
|
||||
explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
|
||||
const std::filesystem::path &data_directory,
|
||||
std::unique_ptr<RequestRouterFactory> request_router_factory,
|
||||
coordinator::Address coordinator_addr);
|
||||
io::Io<io::local_transport::LocalTransport> io, coordinator::Address coordinator_addr);
|
||||
|
||||
storage::v3::Shard *db;
|
||||
|
||||
@@ -188,24 +188,26 @@ struct InterpreterContext {
|
||||
const InterpreterConfig config;
|
||||
IdAllocator edge_ids_alloc;
|
||||
|
||||
// TODO (antaljanosbenjamin) Figure out an abstraction for io::Io to make it possible to construct an interpreter
|
||||
// context with a simulator transport without templatizing it.
|
||||
io::Io<io::local_transport::LocalTransport> io;
|
||||
coordinator::Address coordinator_address;
|
||||
std::unique_ptr<RequestRouterFactory> request_router_factory_;
|
||||
|
||||
storage::v3::LabelId NameToLabelId(std::string_view label_name) {
|
||||
return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name));
|
||||
return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
|
||||
}
|
||||
|
||||
storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
|
||||
return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name));
|
||||
return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
|
||||
}
|
||||
|
||||
storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
|
||||
return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name));
|
||||
return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
|
||||
}
|
||||
|
||||
private:
|
||||
// TODO Replace with local map of labels, properties and edge type ids
|
||||
storage::v3::NameIdMapper query_id_mapper_;
|
||||
storage::v3::NameIdMapper query_id_mapper;
|
||||
};
|
||||
|
||||
/// Function that is used to tell all active interpreters that they should stop
|
||||
@@ -294,16 +296,13 @@ class Interpreter final {
|
||||
*/
|
||||
void Abort();
|
||||
|
||||
const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
|
||||
void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
|
||||
request_router_->InstallSimulatorTicker(tick_simulator);
|
||||
}
|
||||
const msgs::ShardRequestManagerInterface *GetShardRequestManager() const { return shard_request_manager_.get(); }
|
||||
|
||||
private:
|
||||
struct QueryExecution {
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
|
||||
utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
|
||||
std::map<std::string, TypedValue> summary;
|
||||
std::vector<Notification> notifications;
|
||||
@@ -343,7 +342,7 @@ class Interpreter final {
|
||||
// move this unique_ptr into a shared_ptr.
|
||||
std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
|
||||
std::optional<DbAccessor> execution_db_accessor_;
|
||||
std::unique_ptr<RequestRouterInterface> request_router_;
|
||||
std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager_;
|
||||
bool in_explicit_transaction_{false};
|
||||
bool expect_rollback_{false};
|
||||
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "query/v2/multiframe.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
|
||||
#include "query/v2/bindings/frame.hpp"
|
||||
#include "utils/pmr/vector.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
|
||||
static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
|
||||
static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
|
||||
static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
|
||||
|
||||
MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
|
||||
: frames_(utils::pmr::vector<FrameWithValidity>(
|
||||
number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
|
||||
MG_ASSERT(number_of_frames > 0);
|
||||
}
|
||||
|
||||
MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
|
||||
|
||||
// NOLINTNEXTLINE (bugprone-exception-escape)
|
||||
MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
|
||||
|
||||
FrameWithValidity &MultiFrame::GetFirstFrame() {
|
||||
MG_ASSERT(!frames_.empty());
|
||||
return frames_.front();
|
||||
}
|
||||
|
||||
void MultiFrame::MakeAllFramesInvalid() noexcept {
|
||||
std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
|
||||
}
|
||||
|
||||
bool MultiFrame::HasValidFrame() const noexcept {
|
||||
return std::any_of(frames_.begin(), frames_.end(), [](const auto &frame) { return frame.IsValid(); });
|
||||
}
|
||||
|
||||
bool MultiFrame::HasInvalidFrame() const noexcept {
|
||||
return std::any_of(frames_.rbegin(), frames_.rend(), [](const auto &frame) { return !frame.IsValid(); });
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE (bugprone-exception-escape)
|
||||
void MultiFrame::DefragmentValidFrames() noexcept {
|
||||
static constexpr auto kIsValid = [](const FrameWithValidity &frame) { return frame.IsValid(); };
|
||||
static constexpr auto kIsInvalid = [](const FrameWithValidity &frame) { return !frame.IsValid(); };
|
||||
auto first_invalid_frame = std::find_if(frames_.begin(), frames_.end(), kIsInvalid);
|
||||
auto following_first_valid = std::find_if(first_invalid_frame, frames_.end(), kIsValid);
|
||||
while (first_invalid_frame != frames_.end() && following_first_valid != frames_.end()) {
|
||||
std::swap(*first_invalid_frame, *following_first_valid);
|
||||
first_invalid_frame++;
|
||||
first_invalid_frame = std::find_if(first_invalid_frame, frames_.end(), kIsInvalid);
|
||||
following_first_valid++;
|
||||
following_first_valid = std::find_if(following_first_valid, frames_.end(), kIsValid);
|
||||
}
|
||||
}
|
||||
|
||||
ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
|
||||
|
||||
ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
|
||||
|
||||
ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
|
||||
|
||||
InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
|
||||
|
||||
ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
|
||||
/*
|
||||
From: https://en.cppreference.com/w/cpp/algorithm/find
|
||||
Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
|
||||
find_if searches for an element for which predicate p returns true
|
||||
Return value
|
||||
Iterator to the first element satisfying the condition or last if no such element is found.
|
||||
|
||||
-> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
|
||||
*/
|
||||
auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
|
||||
[](const auto &frame) { return !frame.IsValid(); });
|
||||
after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
|
||||
}
|
||||
|
||||
ValidFramesReader::Iterator ValidFramesReader::begin() {
|
||||
if (multiframe_->frames_[0].IsValid()) {
|
||||
return Iterator{&multiframe_->frames_[0]};
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
|
||||
|
||||
ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
|
||||
|
||||
ValidFramesModifier::Iterator ValidFramesModifier::begin() {
|
||||
if (multiframe_->frames_[0].IsValid()) {
|
||||
return Iterator{&multiframe_->frames_[0], *this};
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
ValidFramesModifier::Iterator ValidFramesModifier::end() {
|
||||
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
|
||||
}
|
||||
|
||||
ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
|
||||
|
||||
// NOLINTNEXTLINE (bugprone-exception-escape)
|
||||
ValidFramesConsumer::~ValidFramesConsumer() noexcept {
|
||||
// TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
|
||||
// cost too much to store it
|
||||
multiframe_->DefragmentValidFrames();
|
||||
}
|
||||
|
||||
ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
|
||||
if (multiframe_->frames_[0].IsValid()) {
|
||||
return Iterator{&multiframe_->frames_[0], *this};
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
|
||||
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
|
||||
}
|
||||
|
||||
InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
|
||||
|
||||
InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
|
||||
for (auto &frame : multiframe_->frames_) {
|
||||
if (!frame.IsValid()) {
|
||||
return Iterator{&frame};
|
||||
}
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
|
||||
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
|
||||
}
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
@@ -1,308 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iterator>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include "query/v2/bindings/frame.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_uint64(default_multi_frame_size);
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
class ValidFramesConsumer;
|
||||
class ValidFramesModifier;
|
||||
class ValidFramesReader;
|
||||
class InvalidFramesPopulator;
|
||||
|
||||
class MultiFrame {
|
||||
public:
|
||||
friend class ValidFramesConsumer;
|
||||
friend class ValidFramesModifier;
|
||||
friend class ValidFramesReader;
|
||||
friend class InvalidFramesPopulator;
|
||||
|
||||
MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
|
||||
~MultiFrame() = default;
|
||||
|
||||
// Assigning and moving the MultiFrame is not allowed if any accessor from the above ones are alive.
|
||||
MultiFrame(const MultiFrame &other);
|
||||
MultiFrame(MultiFrame &&other) noexcept;
|
||||
MultiFrame &operator=(const MultiFrame &other) = delete;
|
||||
MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
|
||||
* state in the MultiFrame.
|
||||
* Iteration goes in a deterministic order.
|
||||
* One can't modify the validity of the Frame nor its content with this implementation.
|
||||
*/
|
||||
ValidFramesReader GetValidFramesReader();
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
|
||||
* state in the MultiFrame.
|
||||
* Iteration goes in a deterministic order.
|
||||
* One can't modify the validity of the Frame with this implementation. One can modify its content.
|
||||
*/
|
||||
ValidFramesModifier GetValidFramesModifier();
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
|
||||
* state in the MultiFrame.
|
||||
* Iteration goes in a deterministic order.
|
||||
* One can modify the validity of the Frame with this implementation.
|
||||
* If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
|
||||
* this is faster.
|
||||
*/
|
||||
ValidFramesConsumer GetValidFramesConsumer();
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
|
||||
* invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
|
||||
* the Frame with this implementation.
|
||||
*/
|
||||
InvalidFramesPopulator GetInvalidFramesPopulator();
|
||||
|
||||
/**
|
||||
* Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
|
||||
* using the iterators instead.
|
||||
* The Frame can be valid or invalid.
|
||||
*/
|
||||
FrameWithValidity &GetFirstFrame();
|
||||
|
||||
void MakeAllFramesInvalid() noexcept;
|
||||
|
||||
bool HasValidFrame() const noexcept;
|
||||
bool HasInvalidFrame() const noexcept;
|
||||
|
||||
inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
|
||||
|
||||
private:
|
||||
void DefragmentValidFrames() noexcept;
|
||||
|
||||
utils::pmr::vector<FrameWithValidity> frames_;
|
||||
};
|
||||
|
||||
class ValidFramesReader {
|
||||
public:
|
||||
explicit ValidFramesReader(MultiFrame &multiframe);
|
||||
|
||||
~ValidFramesReader() = default;
|
||||
ValidFramesReader(const ValidFramesReader &other) = delete;
|
||||
ValidFramesReader(ValidFramesReader &&other) noexcept = default;
|
||||
ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
|
||||
ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = const Frame;
|
||||
using pointer = value_type *;
|
||||
using reference = const Frame &;
|
||||
|
||||
Iterator() = default;
|
||||
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
Iterator &operator++() {
|
||||
ptr_++;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
ptr_++;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
FrameWithValidity *after_last_valid_frame_;
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
class ValidFramesModifier {
|
||||
public:
|
||||
explicit ValidFramesModifier(MultiFrame &multiframe);
|
||||
|
||||
~ValidFramesModifier() = default;
|
||||
ValidFramesModifier(const ValidFramesModifier &other) = delete;
|
||||
ValidFramesModifier(ValidFramesModifier &&other) noexcept = default;
|
||||
ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
|
||||
ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = Frame;
|
||||
using pointer = value_type *;
|
||||
using reference = Frame &;
|
||||
|
||||
Iterator() = default;
|
||||
Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
|
||||
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
// Prefix increment
|
||||
Iterator &operator++() {
|
||||
do {
|
||||
ptr_++;
|
||||
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
++*this;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
ValidFramesModifier *iterator_wrapper_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
class ValidFramesConsumer {
|
||||
public:
|
||||
explicit ValidFramesConsumer(MultiFrame &multiframe);
|
||||
|
||||
~ValidFramesConsumer() noexcept;
|
||||
ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
|
||||
ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default;
|
||||
ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
|
||||
ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = FrameWithValidity;
|
||||
using pointer = value_type *;
|
||||
using reference = FrameWithValidity &;
|
||||
|
||||
Iterator() = default;
|
||||
Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
|
||||
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
Iterator &operator++() {
|
||||
do {
|
||||
ptr_++;
|
||||
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
++*this;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
ValidFramesConsumer *iterator_wrapper_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
class InvalidFramesPopulator {
|
||||
public:
|
||||
explicit InvalidFramesPopulator(MultiFrame &multiframe);
|
||||
~InvalidFramesPopulator() = default;
|
||||
|
||||
InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
|
||||
InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = default;
|
||||
InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
|
||||
InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = FrameWithValidity;
|
||||
using pointer = value_type *;
|
||||
using reference = FrameWithValidity &;
|
||||
|
||||
Iterator() = default;
|
||||
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
Iterator &operator++() {
|
||||
ptr_->MakeValid();
|
||||
ptr_++;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
++ptr_;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -149,6 +149,8 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
|
||||
return true;
|
||||
}
|
||||
|
||||
// TODO: Cost estimate ScanAllById?
|
||||
|
||||
// For the given op first increments the cardinality and then cost.
|
||||
#define POST_VISIT_CARD_FIRST(NAME) \
|
||||
bool PostVisit(NAME &) override { \
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -28,7 +28,6 @@
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/bindings/frame.hpp"
|
||||
#include "query/v2/bindings/symbol_table.hpp"
|
||||
#include "query/v2/multiframe.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "utils/bound.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
@@ -72,22 +71,6 @@ class Cursor {
|
||||
/// @throws QueryRuntimeException if something went wrong with execution
|
||||
virtual bool Pull(Frame &, ExecutionContext &) = 0;
|
||||
|
||||
/// Run an iteration of a @c LogicalOperator with MultiFrame.
|
||||
///
|
||||
/// Since operators may be chained, the iteration may pull results from
|
||||
/// multiple operators.
|
||||
///
|
||||
/// @param MultiFrame May be read from or written to while performing the
|
||||
/// iteration.
|
||||
/// @param ExecutionContext Used to get the position of symbols in frame and
|
||||
/// other information.
|
||||
/// @return True if the operator was able to populate at least one Frame on the MultiFrame,
|
||||
/// thus if an operator returns true, that means there is at least one valid Frame in the
|
||||
/// MultiFrame.
|
||||
///
|
||||
/// @throws QueryRuntimeException if something went wrong with execution
|
||||
virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
|
||||
|
||||
/// Resets the Cursor to its initial state.
|
||||
virtual void Reset() = 0;
|
||||
|
||||
@@ -127,7 +110,7 @@ class ScanAllByLabel;
|
||||
class ScanAllByLabelPropertyRange;
|
||||
class ScanAllByLabelPropertyValue;
|
||||
class ScanAllByLabelProperty;
|
||||
class ScanByPrimaryKey;
|
||||
class ScanAllById;
|
||||
class Expand;
|
||||
class ExpandVariable;
|
||||
class ConstructNamedPath;
|
||||
@@ -158,7 +141,7 @@ class Foreach;
|
||||
using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
|
||||
Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
|
||||
ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue,
|
||||
ScanAllByLabelProperty, ScanByPrimaryKey,
|
||||
ScanAllByLabelProperty, ScanAllById,
|
||||
Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
|
||||
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
|
||||
EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
|
||||
@@ -349,7 +332,6 @@ and false on every following Pull.")
|
||||
class OnceCursor : public Cursor {
|
||||
public:
|
||||
OnceCursor() {}
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
@@ -859,21 +841,19 @@ given label and property.
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
(lcp:define-class scan-by-primary-key (scan-all)
|
||||
((label "::storage::v3::LabelId" :scope :public)
|
||||
(primary-key "std::vector<Expression*>" :scope :public)
|
||||
(expression "Expression *" :scope :public
|
||||
|
||||
|
||||
(lcp:define-class scan-all-by-id (scan-all)
|
||||
((expression "Expression *" :scope :public
|
||||
:slk-save #'slk-save-ast-pointer
|
||||
:slk-load (slk-load-ast-pointer "Expression")))
|
||||
(:documentation
|
||||
"ScanAll producing a single node with specified by the label and primary key")
|
||||
"ScanAll producing a single node with ID equal to evaluated expression")
|
||||
(:public
|
||||
#>cpp
|
||||
ScanByPrimaryKey() {}
|
||||
ScanByPrimaryKey(const std::shared_ptr<LogicalOperator> &input,
|
||||
Symbol output_symbol,
|
||||
storage::v3::LabelId label,
|
||||
std::vector<query::v2::Expression*> primary_key,
|
||||
ScanAllById() {}
|
||||
ScanAllById(const std::shared_ptr<LogicalOperator> &input,
|
||||
Symbol output_symbol, Expression *expression,
|
||||
storage::v3::View view = storage::v3::View::OLD);
|
||||
|
||||
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
|
||||
@@ -1176,7 +1156,6 @@ a boolean value.")
|
||||
public:
|
||||
FilterCursor(const Filter &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1228,7 +1207,6 @@ RETURN clause) the Produce's pull succeeds exactly once.")
|
||||
public:
|
||||
ProduceCursor(const Produce &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1276,7 +1254,6 @@ Has a flag for using DETACH DELETE when deleting vertices.")
|
||||
public:
|
||||
DeleteCursor(const Delete &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1570,7 +1547,6 @@ edge lists).")
|
||||
EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
|
||||
utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
|
||||
@@ -398,7 +398,7 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
|
||||
auto add_id_equal = [&](auto *maybe_id_fun, auto *val_expr) -> bool {
|
||||
auto *id_fun = utils::Downcast<Function>(maybe_id_fun);
|
||||
if (!id_fun) return false;
|
||||
if (id_fun->function_name_ != functions::kId) return false;
|
||||
if (id_fun->function_name_ != kId) return false;
|
||||
if (id_fun->arguments_.size() != 1U) return false;
|
||||
auto *ident = utils::Downcast<Identifier>(id_fun->arguments_.front());
|
||||
if (!ident) return false;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,13 +14,13 @@
|
||||
|
||||
#include "query/v2/bindings/pretty_print.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::query::v2::plan {
|
||||
|
||||
PlanPrinter::PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out)
|
||||
: request_router_(request_router), out_(out) {}
|
||||
PlanPrinter::PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out)
|
||||
: request_manager_(request_manager), out_(out) {}
|
||||
|
||||
#define PRE_VISIT(TOp) \
|
||||
bool PlanPrinter::PreVisit(TOp &) { \
|
||||
@@ -34,7 +34,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* CreateExpand (" << op.input_symbol_.name() << ")"
|
||||
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
|
||||
<< op.edge_info_.symbol.name() << ":" << request_router_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
|
||||
<< op.edge_info_.symbol.name() << ":" << request_manager_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
|
||||
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
|
||||
<< op.node_info_.symbol.name() << ")";
|
||||
});
|
||||
@@ -54,7 +54,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabel"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << ")";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -62,8 +62,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabelPropertyValue"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
|
||||
<< request_router_->PropertyToName(op.property_) << "})";
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
|
||||
<< request_manager_->PropertyToName(op.property_) << "})";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -71,8 +71,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabelPropertyRange"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
|
||||
<< request_router_->PropertyToName(op.property_) << "})";
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
|
||||
<< request_manager_->PropertyToName(op.property_) << "})";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -80,16 +80,16 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabelProperty"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
|
||||
<< request_router_->PropertyToName(op.property_) << "})";
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
|
||||
<< request_manager_->PropertyToName(op.property_) << "})";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanByPrimaryKey &op) {
|
||||
bool PlanPrinter::PreVisit(ScanAllById &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanByPrimaryKey"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
|
||||
out << "* ScanAllById"
|
||||
<< " (" << op.output_symbol_.name() << ")";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -100,7 +100,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) {
|
||||
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
|
||||
<< op.common_.edge_symbol.name();
|
||||
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
|
||||
stream << ":" << request_router_->EdgeTypeToName(edge_type);
|
||||
stream << ":" << request_manager_->EdgeTypeToName(edge_type);
|
||||
});
|
||||
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
|
||||
<< op.common_.node_symbol.name() << ")";
|
||||
@@ -129,7 +129,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) {
|
||||
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
|
||||
<< op.common_.edge_symbol.name();
|
||||
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
|
||||
stream << ":" << request_router_->EdgeTypeToName(edge_type);
|
||||
stream << ":" << request_manager_->EdgeTypeToName(edge_type);
|
||||
});
|
||||
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
|
||||
<< op.common_.node_symbol.name() << ")";
|
||||
@@ -263,14 +263,15 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string
|
||||
--depth_;
|
||||
}
|
||||
|
||||
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out) {
|
||||
PlanPrinter printer(&request_router, out);
|
||||
void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root,
|
||||
std::ostream *out) {
|
||||
PlanPrinter printer(&request_manager, out);
|
||||
// FIXME(mtomic): We should make visitors that take const arguments.
|
||||
const_cast<LogicalOperator *>(plan_root)->Accept(printer);
|
||||
}
|
||||
|
||||
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
|
||||
impl::PlanToJsonVisitor visitor(&request_router);
|
||||
nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) {
|
||||
impl::PlanToJsonVisitor visitor(&request_manager);
|
||||
// FIXME(mtomic): We should make visitors that take const arguments.
|
||||
const_cast<LogicalOperator *>(plan_root)->Accept(visitor);
|
||||
return visitor.output();
|
||||
@@ -348,16 +349,16 @@ json ToJson(const utils::Bound<Expression *> &bound) {
|
||||
|
||||
json ToJson(const Symbol &symbol) { return symbol.name(); }
|
||||
|
||||
json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router) {
|
||||
return request_router.EdgeTypeToName(edge_type);
|
||||
json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager) {
|
||||
return request_manager.EdgeTypeToName(edge_type);
|
||||
}
|
||||
|
||||
json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router) {
|
||||
return request_router.LabelToName(label);
|
||||
json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager) {
|
||||
return request_manager.LabelToName(label);
|
||||
}
|
||||
|
||||
json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router) {
|
||||
return request_router.PropertyToName(property);
|
||||
json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager) {
|
||||
return request_manager.PropertyToName(property);
|
||||
}
|
||||
|
||||
json ToJson(NamedExpression *nexpr) {
|
||||
@@ -368,29 +369,29 @@ json ToJson(NamedExpression *nexpr) {
|
||||
}
|
||||
|
||||
json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
|
||||
const RequestRouterInterface &request_router) {
|
||||
const msgs::ShardRequestManagerInterface &request_manager) {
|
||||
json json;
|
||||
for (const auto &prop_pair : properties) {
|
||||
json.emplace(ToJson(prop_pair.first, request_router), ToJson(prop_pair.second));
|
||||
json.emplace(ToJson(prop_pair.first, request_manager), ToJson(prop_pair.second));
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router) {
|
||||
json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager) {
|
||||
json self;
|
||||
self["symbol"] = ToJson(node_info.symbol);
|
||||
self["labels"] = ToJson(node_info.labels, request_router);
|
||||
self["labels"] = ToJson(node_info.labels, request_manager);
|
||||
const auto *props = std::get_if<PropertiesMapList>(&node_info.properties);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager);
|
||||
return self;
|
||||
}
|
||||
|
||||
json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router) {
|
||||
json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager) {
|
||||
json self;
|
||||
self["symbol"] = ToJson(edge_info.symbol);
|
||||
const auto *props = std::get_if<PropertiesMapList>(&edge_info.properties);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
|
||||
self["edge_type"] = ToJson(edge_info.edge_type, request_router);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager);
|
||||
self["edge_type"] = ToJson(edge_info.edge_type, request_manager);
|
||||
self["direction"] = ToString(edge_info.direction);
|
||||
return self;
|
||||
}
|
||||
@@ -432,7 +433,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabel";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *request_manager_);
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -445,8 +446,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabelPropertyRange";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *request_manager_);
|
||||
self["property"] = ToJson(op.property_, *request_manager_);
|
||||
self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json();
|
||||
self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json();
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
@@ -461,8 +462,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabelPropertyValue";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *request_manager_);
|
||||
self["property"] = ToJson(op.property_, *request_manager_);
|
||||
self["expression"] = ToJson(op.expression_);
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
@@ -476,8 +477,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabelProperty";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *request_manager_);
|
||||
self["property"] = ToJson(op.property_, *request_manager_);
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -487,15 +488,12 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
|
||||
json self;
|
||||
self["name"] = "ScanByPrimaryKey";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["name"] = "ScanAllById";
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
|
||||
output_ = std::move(self);
|
||||
return false;
|
||||
}
|
||||
@@ -503,7 +501,7 @@ bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(CreateNode &op) {
|
||||
json self;
|
||||
self["name"] = "CreateNode";
|
||||
self["node_info"] = ToJson(op.node_info_, *request_router_);
|
||||
self["node_info"] = ToJson(op.node_info_, *request_manager_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
@@ -516,8 +514,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) {
|
||||
json self;
|
||||
self["name"] = "CreateExpand";
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["node_info"] = ToJson(op.node_info_, *request_router_);
|
||||
self["edge_info"] = ToJson(op.edge_info_, *request_router_);
|
||||
self["node_info"] = ToJson(op.node_info_, *request_manager_);
|
||||
self["edge_info"] = ToJson(op.edge_info_, *request_manager_);
|
||||
self["existing_node"] = op.existing_node_;
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -533,7 +531,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) {
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["node_symbol"] = ToJson(op.common_.node_symbol);
|
||||
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_);
|
||||
self["direction"] = ToString(op.common_.direction);
|
||||
self["existing_node"] = op.common_.existing_node;
|
||||
|
||||
@@ -550,7 +548,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) {
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["node_symbol"] = ToJson(op.common_.node_symbol);
|
||||
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_);
|
||||
self["direction"] = ToString(op.common_.direction);
|
||||
self["type"] = ToString(op.type_);
|
||||
self["is_reverse"] = op.is_reverse_;
|
||||
@@ -625,7 +623,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(SetProperty &op) {
|
||||
json self;
|
||||
self["name"] = "SetProperty";
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_manager_);
|
||||
self["lhs"] = ToJson(op.lhs_);
|
||||
self["rhs"] = ToJson(op.rhs_);
|
||||
|
||||
@@ -662,7 +660,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
|
||||
json self;
|
||||
self["name"] = "SetLabels";
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["labels"] = ToJson(op.labels_, *request_router_);
|
||||
self["labels"] = ToJson(op.labels_, *request_manager_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
@@ -674,7 +672,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) {
|
||||
json self;
|
||||
self["name"] = "RemoveProperty";
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_manager_);
|
||||
self["lhs"] = ToJson(op.lhs_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -688,7 +686,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) {
|
||||
json self;
|
||||
self["name"] = "RemoveLabels";
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["labels"] = ToJson(op.labels_, *request_router_);
|
||||
self["labels"] = ToJson(op.labels_, *request_manager_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/plan/operator.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
@@ -27,19 +27,20 @@ namespace plan {
|
||||
class LogicalOperator;
|
||||
|
||||
/// Pretty print a `LogicalOperator` plan to a `std::ostream`.
|
||||
/// RequestRouter is needed for resolving label and property names.
|
||||
/// ShardRequestManager is needed for resolving label and property names.
|
||||
/// Note that `plan_root` isn't modified, but we can't take it as a const
|
||||
/// because we don't have support for visiting a const LogicalOperator.
|
||||
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out);
|
||||
void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root,
|
||||
std::ostream *out);
|
||||
|
||||
/// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`.
|
||||
inline void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
|
||||
PrettyPrint(request_router, plan_root, &std::cout);
|
||||
inline void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) {
|
||||
PrettyPrint(request_manager, plan_root, &std::cout);
|
||||
}
|
||||
|
||||
/// Convert a `LogicalOperator` plan to a JSON representation.
|
||||
/// DbAccessor is needed for resolving label and property names.
|
||||
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root);
|
||||
nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root);
|
||||
|
||||
class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
public:
|
||||
@@ -47,7 +48,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
using HierarchicalLogicalOperatorVisitor::PreVisit;
|
||||
using HierarchicalLogicalOperatorVisitor::Visit;
|
||||
|
||||
PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out);
|
||||
PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out);
|
||||
|
||||
bool DefaultPreVisit() override;
|
||||
|
||||
@@ -67,7 +68,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(ScanAllByLabelPropertyValue &) override;
|
||||
bool PreVisit(ScanAllByLabelPropertyRange &) override;
|
||||
bool PreVisit(ScanAllByLabelProperty &) override;
|
||||
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
|
||||
bool PreVisit(ScanAllById &) override;
|
||||
|
||||
bool PreVisit(Expand &) override;
|
||||
bool PreVisit(ExpandVariable &) override;
|
||||
@@ -114,7 +115,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
void Branch(LogicalOperator &op, const std::string &branch_name = "");
|
||||
|
||||
int64_t depth_{0};
|
||||
const RequestRouterInterface *request_router_{nullptr};
|
||||
const msgs::ShardRequestManagerInterface *request_manager_{nullptr};
|
||||
std::ostream *out_{nullptr};
|
||||
};
|
||||
|
||||
@@ -132,20 +133,20 @@ nlohmann::json ToJson(const utils::Bound<Expression *> &bound);
|
||||
|
||||
nlohmann::json ToJson(const Symbol &symbol);
|
||||
|
||||
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager);
|
||||
|
||||
nlohmann::json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager);
|
||||
|
||||
nlohmann::json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager);
|
||||
|
||||
nlohmann::json ToJson(NamedExpression *nexpr);
|
||||
|
||||
nlohmann::json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
|
||||
const RequestRouterInterface &request_router);
|
||||
const msgs::ShardRequestManagerInterface &request_manager);
|
||||
|
||||
nlohmann::json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager);
|
||||
|
||||
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager);
|
||||
|
||||
nlohmann::json ToJson(const Aggregate::Element &elem);
|
||||
|
||||
@@ -160,7 +161,8 @@ nlohmann::json ToJson(const std::vector<T> &items, Args &&...args) {
|
||||
|
||||
class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
public:
|
||||
explicit PlanToJsonVisitor(const RequestRouterInterface *request_router) : request_router_(request_router) {}
|
||||
explicit PlanToJsonVisitor(const msgs::ShardRequestManagerInterface *request_manager)
|
||||
: request_manager_(request_manager) {}
|
||||
|
||||
using HierarchicalLogicalOperatorVisitor::PostVisit;
|
||||
using HierarchicalLogicalOperatorVisitor::PreVisit;
|
||||
@@ -194,7 +196,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(ScanAllByLabelPropertyRange &) override;
|
||||
bool PreVisit(ScanAllByLabelPropertyValue &) override;
|
||||
bool PreVisit(ScanAllByLabelProperty &) override;
|
||||
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
|
||||
bool PreVisit(ScanAllById &) override;
|
||||
|
||||
bool PreVisit(Produce &) override;
|
||||
bool PreVisit(Accumulate &) override;
|
||||
@@ -216,7 +218,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
|
||||
protected:
|
||||
nlohmann::json output_;
|
||||
const RequestRouterInterface *request_router_;
|
||||
const msgs::ShardRequestManagerInterface *request_manager_;
|
||||
|
||||
nlohmann::json PopOutput() {
|
||||
nlohmann::json tmp;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,12 +11,10 @@
|
||||
|
||||
#include "query/v2/plan/read_write_type_checker.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define PRE_VISIT(TOp, RWType, continue_visiting) \
|
||||
/*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \
|
||||
bool ReadWriteTypeChecker::PreVisit(TOp & /*op*/) { \
|
||||
UpdateType(RWType); \
|
||||
return continue_visiting; \
|
||||
#define PRE_VISIT(TOp, RWType, continue_visiting) \
|
||||
bool ReadWriteTypeChecker::PreVisit(TOp &op) { \
|
||||
UpdateType(RWType); \
|
||||
return continue_visiting; \
|
||||
}
|
||||
|
||||
namespace memgraph::query::v2::plan {
|
||||
@@ -37,7 +35,7 @@ PRE_VISIT(ScanAllByLabel, RWType::R, true)
|
||||
PRE_VISIT(ScanAllByLabelPropertyRange, RWType::R, true)
|
||||
PRE_VISIT(ScanAllByLabelPropertyValue, RWType::R, true)
|
||||
PRE_VISIT(ScanAllByLabelProperty, RWType::R, true)
|
||||
PRE_VISIT(ScanByPrimaryKey, RWType::R, true)
|
||||
PRE_VISIT(ScanAllById, RWType::R, true)
|
||||
|
||||
PRE_VISIT(Expand, RWType::R, true)
|
||||
PRE_VISIT(ExpandVariable, RWType::R, true)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -59,7 +59,7 @@ class ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(ScanAllByLabelPropertyValue &) override;
|
||||
bool PreVisit(ScanAllByLabelPropertyRange &) override;
|
||||
bool PreVisit(ScanAllByLabelProperty &) override;
|
||||
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
|
||||
bool PreVisit(ScanAllById &) override;
|
||||
|
||||
bool PreVisit(Expand &) override;
|
||||
bool PreVisit(ExpandVariable &) override;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -25,10 +25,8 @@
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/plan/operator.hpp"
|
||||
#include "query/v2/plan/preprocess.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
|
||||
DECLARE_int64(query_vertex_count_to_expand_existing);
|
||||
|
||||
@@ -273,12 +271,11 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(ScanByPrimaryKey &op) override {
|
||||
bool PreVisit(ScanAllById &op) override {
|
||||
prev_ops_.push_back(&op);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(ScanByPrimaryKey & /*unused*/) override {
|
||||
bool PostVisit(ScanAllById &) override {
|
||||
prev_ops_.pop_back();
|
||||
return true;
|
||||
}
|
||||
@@ -490,12 +487,6 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
|
||||
|
||||
storage::v3::PropertyId GetProperty(PropertyIx prop) { return db_->NameToProperty(prop.name); }
|
||||
|
||||
void EraseLabelFilters(const memgraph::query::v2::Symbol &node_symbol, memgraph::query::v2::LabelIx prim_label) {
|
||||
std::vector<query::v2::Expression *> removed_expressions;
|
||||
filters_.EraseLabelFilter(node_symbol, prim_label, &removed_expressions);
|
||||
filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
|
||||
}
|
||||
|
||||
std::optional<LabelIx> FindBestLabelIndex(const std::unordered_set<LabelIx> &labels) {
|
||||
MG_ASSERT(!labels.empty(), "Trying to find the best label without any labels.");
|
||||
std::optional<LabelIx> best_label;
|
||||
@@ -568,84 +559,31 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
|
||||
const auto &view = scan.view_;
|
||||
const auto &modified_symbols = scan.ModifiedSymbols(*symbol_table_);
|
||||
std::unordered_set<Symbol> bound_symbols(modified_symbols.begin(), modified_symbols.end());
|
||||
|
||||
// Try to see if we can use label + primary-key or label + property index.
|
||||
// If not, try to use just the label index.
|
||||
auto are_bound = [&bound_symbols](const auto &used_symbols) {
|
||||
for (const auto &used_symbol : used_symbols) {
|
||||
if (!utils::Contains(bound_symbols, used_symbol)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
// First, try to see if we can find a vertex by ID.
|
||||
if (!max_vertex_count || *max_vertex_count >= 1) {
|
||||
for (const auto &filter : filters_.IdFilters(node_symbol)) {
|
||||
if (filter.id_filter->is_symbol_in_value_ || !are_bound(filter.used_symbols)) continue;
|
||||
auto *value = filter.id_filter->value_;
|
||||
filter_exprs_for_removal_.insert(filter.expression);
|
||||
filters_.EraseFilter(filter);
|
||||
return std::make_unique<ScanAllById>(input, node_symbol, value, view);
|
||||
}
|
||||
}
|
||||
// Now try to see if we can use label+property index. If not, try to use
|
||||
// just the label index.
|
||||
const auto labels = filters_.FilteredLabels(node_symbol);
|
||||
if (labels.empty()) {
|
||||
// Without labels, we cannot generate any indexed ScanAll.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// First, try to see if we can find a vertex based on the possibly
|
||||
// supplied primary key.
|
||||
auto property_filters = filters_.PropertyFilters(node_symbol);
|
||||
query::v2::LabelIx prim_label;
|
||||
std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> primary_key;
|
||||
|
||||
auto extract_primary_key = [this](storage::v3::LabelId label,
|
||||
std::vector<query::v2::plan::FilterInfo> property_filters)
|
||||
-> std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> {
|
||||
std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> pk_temp;
|
||||
std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> pk;
|
||||
std::vector<memgraph::storage::v3::SchemaProperty> schema = db_->GetSchemaForLabel(label);
|
||||
|
||||
std::vector<storage::v3::PropertyId> schema_properties;
|
||||
schema_properties.reserve(schema.size());
|
||||
|
||||
std::transform(schema.begin(), schema.end(), std::back_inserter(schema_properties),
|
||||
[](const auto &schema_elem) { return schema_elem.property_id; });
|
||||
|
||||
for (const auto &property_filter : property_filters) {
|
||||
if (property_filter.property_filter->type_ != PropertyFilter::Type::EQUAL) {
|
||||
continue;
|
||||
}
|
||||
const auto &property_id = db_->NameToProperty(property_filter.property_filter->property_.name);
|
||||
if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) {
|
||||
pk_temp.emplace_back(std::make_pair(property_filter.expression, property_filter));
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure pk is in the same order as schema_properties.
|
||||
for (const auto &schema_prop : schema_properties) {
|
||||
for (auto &pk_temp_prop : pk_temp) {
|
||||
const auto &property_id = db_->NameToProperty(pk_temp_prop.second.property_filter->property_.name);
|
||||
if (schema_prop == property_id) {
|
||||
pk.push_back(pk_temp_prop);
|
||||
}
|
||||
}
|
||||
}
|
||||
MG_ASSERT(pk.size() == pk_temp.size(),
|
||||
"The two vectors should represent the same primary key with a possibly different order of contained "
|
||||
"elements.");
|
||||
|
||||
return pk.size() == schema_properties.size()
|
||||
? pk
|
||||
: std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>>{};
|
||||
};
|
||||
|
||||
if (!property_filters.empty()) {
|
||||
for (const auto &label : labels) {
|
||||
if (db_->PrimaryLabelExists(GetLabel(label))) {
|
||||
prim_label = label;
|
||||
primary_key = extract_primary_key(GetLabel(prim_label), property_filters);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!primary_key.empty()) {
|
||||
// Mark the expressions so they won't be used for an additional, unnecessary filter.
|
||||
for (const auto &primary_property : primary_key) {
|
||||
filter_exprs_for_removal_.insert(primary_property.first);
|
||||
filters_.EraseFilter(primary_property.second);
|
||||
}
|
||||
EraseLabelFilters(node_symbol, prim_label);
|
||||
std::vector<query::v2::Expression *> pk_expressions;
|
||||
std::transform(primary_key.begin(), primary_key.end(), std::back_inserter(pk_expressions),
|
||||
[](const auto &exp) { return exp.second.property_filter->value_; });
|
||||
return std::make_unique<ScanByPrimaryKey>(input, node_symbol, GetLabel(prim_label), pk_expressions);
|
||||
}
|
||||
}
|
||||
|
||||
auto found_index = FindBestLabelPropertyIndex(node_symbol, bound_symbols);
|
||||
if (found_index &&
|
||||
// Use label+property index if we satisfy max_vertex_count.
|
||||
@@ -659,7 +597,9 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
|
||||
filter_exprs_for_removal_.insert(found_index->filter.expression);
|
||||
}
|
||||
filters_.EraseFilter(found_index->filter);
|
||||
EraseLabelFilters(node_symbol, found_index->label);
|
||||
std::vector<Expression *> removed_expressions;
|
||||
filters_.EraseLabelFilter(node_symbol, found_index->label, &removed_expressions);
|
||||
filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
|
||||
if (prop_filter.lower_bound_ || prop_filter.upper_bound_) {
|
||||
return std::make_unique<ScanAllByLabelPropertyRange>(
|
||||
input, node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_),
|
||||
|
||||
@@ -272,7 +272,7 @@ class RuleBasedPlanner {
|
||||
PropertiesMapList vector_props;
|
||||
vector_props.reserve(node_properties->size());
|
||||
for (const auto &kv : *node_properties) {
|
||||
// TODO(kostasrim) GetProperty should be implemented in terms of RequestRouter NameToProperty
|
||||
// TODO(kostasrim) GetProperty should be implemented in terms of ShardRequestManager NameToProperty
|
||||
vector_props.push_back({GetProperty(kv.first), kv.second});
|
||||
}
|
||||
return std::move(vector_props);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,17 +12,14 @@
|
||||
/// @file
|
||||
#pragma once
|
||||
|
||||
#include <iterator>
|
||||
#include <optional>
|
||||
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/plan/preprocess.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "storage/v3/conversions.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "utils/bound.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
|
||||
namespace memgraph::query::v2::plan {
|
||||
@@ -32,11 +29,11 @@ namespace memgraph::query::v2::plan {
|
||||
template <class TDbAccessor>
|
||||
class VertexCountCache {
|
||||
public:
|
||||
explicit VertexCountCache(TDbAccessor *request_router) : request_router_{request_router} {}
|
||||
explicit VertexCountCache(TDbAccessor *shard_request_manager) : shard_request_manager_{shard_request_manager} {}
|
||||
|
||||
auto NameToLabel(const std::string &name) { return request_router_->NameToLabel(name); }
|
||||
auto NameToProperty(const std::string &name) { return request_router_->NameToProperty(name); }
|
||||
auto NameToEdgeType(const std::string &name) { return request_router_->NameToEdgeType(name); }
|
||||
auto NameToLabel(const std::string &name) { return shard_request_manager_->NameToLabel(name); }
|
||||
auto NameToProperty(const std::string &name) { return shard_request_manager_->NameToProperty(name); }
|
||||
auto NameToEdgeType(const std::string &name) { return shard_request_manager_->NameToEdgeType(name); }
|
||||
|
||||
int64_t VerticesCount() { return 1; }
|
||||
|
||||
@@ -55,17 +52,12 @@ class VertexCountCache {
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool LabelIndexExists(storage::v3::LabelId label) { return PrimaryLabelExists(label); }
|
||||
|
||||
bool PrimaryLabelExists(storage::v3::LabelId label) { return request_router_->IsPrimaryLabel(label); }
|
||||
// For now return true if label is primary label
|
||||
bool LabelIndexExists(storage::v3::LabelId label) { return shard_request_manager_->IsPrimaryLabel(label); }
|
||||
|
||||
bool LabelPropertyIndexExists(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return false; }
|
||||
|
||||
const std::vector<memgraph::storage::v3::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) {
|
||||
return request_router_->GetSchemaForLabel(label);
|
||||
}
|
||||
|
||||
RequestRouterInterface *request_router_;
|
||||
msgs::ShardRequestManagerInterface *shard_request_manager_;
|
||||
};
|
||||
|
||||
template <class TDbAccessor>
|
||||
|
||||
@@ -1,815 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <deque>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/coordinator_client.hpp"
|
||||
#include "coordinator/coordinator_rsm.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/local_transport/local_transport.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include "io/rsm/rsm_client.hpp"
|
||||
#include "io/rsm/shard_rsm.hpp"
|
||||
#include "io/simulator/simulator.hpp"
|
||||
#include "io/simulator/simulator_transport.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
template <typename TStorageClient>
|
||||
class RsmStorageClientManager {
|
||||
public:
|
||||
using CompoundKey = io::rsm::ShardRsmKey;
|
||||
using ShardMetadata = coordinator::ShardMetadata;
|
||||
RsmStorageClientManager() = default;
|
||||
RsmStorageClientManager(const RsmStorageClientManager &) = delete;
|
||||
RsmStorageClientManager(RsmStorageClientManager &&) = delete;
|
||||
RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
|
||||
RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
|
||||
~RsmStorageClientManager() = default;
|
||||
|
||||
void AddClient(ShardMetadata key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
|
||||
|
||||
bool Exists(const ShardMetadata &key) { return cli_cache_.contains(key); }
|
||||
|
||||
void PurgeCache() { cli_cache_.clear(); }
|
||||
|
||||
TStorageClient &GetClient(const ShardMetadata &key) {
|
||||
auto it = cli_cache_.find(key);
|
||||
MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
|
||||
return it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<ShardMetadata, TStorageClient> cli_cache_;
|
||||
};
|
||||
|
||||
template <typename TRequest>
|
||||
struct ShardRequestState {
|
||||
memgraph::coordinator::ShardMetadata shard;
|
||||
TRequest request;
|
||||
};
|
||||
|
||||
// maps from ReadinessToken's internal size_t to the associated state
|
||||
template <typename TRequest>
|
||||
using RunningRequests = std::unordered_map<size_t, ShardRequestState<TRequest>>;
|
||||
|
||||
class RequestRouterInterface {
|
||||
public:
|
||||
using VertexAccessor = query::v2::accessors::VertexAccessor;
|
||||
RequestRouterInterface() = default;
|
||||
RequestRouterInterface(const RequestRouterInterface &) = delete;
|
||||
RequestRouterInterface(RequestRouterInterface &&) = delete;
|
||||
RequestRouterInterface &operator=(const RequestRouterInterface &) = delete;
|
||||
RequestRouterInterface &&operator=(RequestRouterInterface &&) = delete;
|
||||
|
||||
virtual ~RequestRouterInterface() = default;
|
||||
|
||||
virtual void StartTransaction() = 0;
|
||||
virtual void Commit() = 0;
|
||||
virtual std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) = 0;
|
||||
virtual std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) = 0;
|
||||
virtual std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) = 0;
|
||||
virtual std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) = 0;
|
||||
virtual std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest request) = 0;
|
||||
|
||||
virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
|
||||
virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
|
||||
virtual storage::v3::LabelId NameToLabel(const std::string &name) const = 0;
|
||||
virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0;
|
||||
virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0;
|
||||
virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0;
|
||||
virtual std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const = 0;
|
||||
virtual std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const = 0;
|
||||
virtual std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const = 0;
|
||||
virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0;
|
||||
virtual bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0;
|
||||
|
||||
virtual std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) = 0;
|
||||
virtual void InstallSimulatorTicker(std::function<bool()> tick_simulator) = 0;
|
||||
virtual const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const = 0;
|
||||
};
|
||||
|
||||
// TODO(kostasrim)rename this class template
|
||||
template <typename TTransport>
|
||||
class RequestRouter : public RequestRouterInterface {
|
||||
public:
|
||||
using StorageClient = coordinator::RsmClient<TTransport, msgs::WriteRequests, msgs::WriteResponses,
|
||||
msgs::ReadRequests, msgs::ReadResponses>;
|
||||
using CoordinatorWriteRequests = coordinator::CoordinatorWriteRequests;
|
||||
using CoordinatorClient = coordinator::CoordinatorClient<TTransport>;
|
||||
using Address = io::Address;
|
||||
using ShardMetadata = coordinator::ShardMetadata;
|
||||
using ShardMap = coordinator::ShardMap;
|
||||
using CompoundKey = coordinator::PrimaryKey;
|
||||
using VertexAccessor = query::v2::accessors::VertexAccessor;
|
||||
RequestRouter(CoordinatorClient coord, io::Io<TTransport> &&io) : coord_cli_(std::move(coord)), io_(std::move(io)) {}
|
||||
|
||||
RequestRouter(const RequestRouter &) = delete;
|
||||
RequestRouter(RequestRouter &&) = delete;
|
||||
RequestRouter &operator=(const RequestRouter &) = delete;
|
||||
RequestRouter &operator=(RequestRouter &&) = delete;
|
||||
|
||||
~RequestRouter() override {}
|
||||
|
||||
void InstallSimulatorTicker(std::function<bool()> tick_simulator) override {
|
||||
notifier_.InstallSimulatorTicker(tick_simulator);
|
||||
}
|
||||
|
||||
void StartTransaction() override {
|
||||
coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
|
||||
CoordinatorWriteRequests write_req = req;
|
||||
spdlog::trace("sending hlc request to start transaction");
|
||||
auto write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
spdlog::trace("received hlc response to start transaction");
|
||||
if (write_res.HasError()) {
|
||||
throw std::runtime_error("HLC request failed");
|
||||
}
|
||||
auto coordinator_write_response = write_res.GetValue();
|
||||
auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
|
||||
|
||||
// Transaction ID to be used later...
|
||||
transaction_id_ = hlc_response.new_hlc;
|
||||
|
||||
if (hlc_response.fresher_shard_map) {
|
||||
shards_map_ = hlc_response.fresher_shard_map.value();
|
||||
SetUpNameIdMappers();
|
||||
}
|
||||
}
|
||||
|
||||
void Commit() override {
|
||||
coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
|
||||
CoordinatorWriteRequests write_req = req;
|
||||
spdlog::trace("sending hlc request before committing transaction");
|
||||
auto write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
spdlog::trace("received hlc response before committing transaction");
|
||||
if (write_res.HasError()) {
|
||||
throw std::runtime_error("HLC request for commit failed");
|
||||
}
|
||||
auto coordinator_write_response = write_res.GetValue();
|
||||
auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
|
||||
|
||||
if (hlc_response.fresher_shard_map) {
|
||||
shards_map_ = hlc_response.fresher_shard_map.value();
|
||||
SetUpNameIdMappers();
|
||||
}
|
||||
auto commit_timestamp = hlc_response.new_hlc;
|
||||
|
||||
msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
|
||||
|
||||
for (const auto &[label, space] : shards_map_.label_spaces) {
|
||||
for (const auto &[key, shard] : space.shards) {
|
||||
auto &storage_client = GetStorageClientForShard(shard);
|
||||
// TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
|
||||
// instead.
|
||||
auto commit_response = storage_client.SendWriteRequest(commit_req);
|
||||
// RETRY on timeouts?
|
||||
// Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
|
||||
if (commit_response.HasError()) {
|
||||
throw std::runtime_error("Commit request timed out");
|
||||
}
|
||||
msgs::WriteResponses write_response_variant = commit_response.GetValue();
|
||||
auto &response = std::get<msgs::CommitResponse>(write_response_variant);
|
||||
if (response.error) {
|
||||
throw std::runtime_error("Commit request did not succeed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
|
||||
return shards_map_.GetEdgeTypeId(name).value();
|
||||
}
|
||||
|
||||
storage::v3::PropertyId NameToProperty(const std::string &name) const override {
|
||||
return shards_map_.GetPropertyId(name).value();
|
||||
}
|
||||
|
||||
storage::v3::LabelId NameToLabel(const std::string &name) const override {
|
||||
return shards_map_.GetLabelId(name).value();
|
||||
}
|
||||
|
||||
const std::string &PropertyToName(storage::v3::PropertyId id) const override {
|
||||
return properties_.IdToName(id.AsUint());
|
||||
}
|
||||
const std::string &LabelToName(storage::v3::LabelId id) const override { return labels_.IdToName(id.AsUint()); }
|
||||
const std::string &EdgeTypeToName(storage::v3::EdgeTypeId id) const override {
|
||||
return edge_types_.IdToName(id.AsUint());
|
||||
}
|
||||
|
||||
bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const override {
|
||||
const auto schema_it = shards_map_.schemas.find(primary_label);
|
||||
MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
|
||||
|
||||
return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
|
||||
return schema_prop.property_id == property;
|
||||
}) != schema_it->second.end();
|
||||
}
|
||||
|
||||
const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const override {
|
||||
return shards_map_.schemas.at(label);
|
||||
}
|
||||
|
||||
bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); }
|
||||
|
||||
// TODO(kostasrim) Simplify return result
|
||||
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) override {
|
||||
// create requests
|
||||
auto requests_to_be_sent = RequestsForScanVertices(label);
|
||||
|
||||
spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::ScanVerticesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
spdlog::trace("sent {} ScanVertices requests in parallel", running_requests.size());
|
||||
|
||||
// drive requests to completion
|
||||
auto responses = DriveReadResponses<msgs::ScanVerticesRequest, msgs::ScanVerticesResponse>(running_requests);
|
||||
spdlog::trace("got back {} ScanVertices responses after driving to completion", responses.size());
|
||||
|
||||
// convert responses into VertexAccessor objects to return
|
||||
std::vector<VertexAccessor> accessors;
|
||||
accessors.reserve(responses.size());
|
||||
for (auto &response : responses) {
|
||||
for (auto &result_row : response.results) {
|
||||
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
|
||||
}
|
||||
}
|
||||
|
||||
return accessors;
|
||||
}
|
||||
|
||||
std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) override {
|
||||
MG_ASSERT(!new_vertices.empty());
|
||||
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests_to_be_sent =
|
||||
RequestsForCreateVertices(new_vertices);
|
||||
spdlog::trace("created {} CreateVertices requests", requests_to_be_sent.size());
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::CreateVerticesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
for (auto &new_vertex : request.request.new_vertices) {
|
||||
new_vertex.label_ids.erase(new_vertex.label_ids.begin());
|
||||
}
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
storage_client.SendAsyncWriteRequest(request.request, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
spdlog::trace("sent {} CreateVertices requests in parallel", running_requests.size());
|
||||
|
||||
// drive requests to completion
|
||||
return DriveWriteResponses<msgs::CreateVerticesRequest, msgs::CreateVerticesResponse>(running_requests);
|
||||
}
|
||||
|
||||
std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) override {
|
||||
MG_ASSERT(!new_edges.empty());
|
||||
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests_to_be_sent =
|
||||
RequestsForCreateExpand(std::move(new_edges));
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::CreateExpandRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::WriteRequests req = request.request;
|
||||
storage_client.SendAsyncWriteRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
return DriveWriteResponses<msgs::CreateExpandRequest, msgs::CreateExpandResponse>(running_requests);
|
||||
}
|
||||
|
||||
std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) override {
|
||||
// TODO(kostasrim)Update to limit the batch size here
|
||||
// Expansions of the destination must be handled by the caller. For example
|
||||
// match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
|
||||
// For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
|
||||
// must be fetched again with an ExpandOne(Edges.dst)
|
||||
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent = RequestsForExpandOne(request);
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::ExpandOneRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::ReadRequests req = request.request;
|
||||
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto responses = DriveReadResponses<msgs::ExpandOneRequest, msgs::ExpandOneResponse>(running_requests);
|
||||
|
||||
// post-process responses
|
||||
std::vector<msgs::ExpandOneResultRow> result_rows;
|
||||
const auto total_row_count = std::accumulate(responses.begin(), responses.end(), 0,
|
||||
[](const int64_t partial_count, const msgs::ExpandOneResponse &resp) {
|
||||
return partial_count + resp.result.size();
|
||||
});
|
||||
result_rows.reserve(total_row_count);
|
||||
|
||||
for (auto &response : responses) {
|
||||
result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
|
||||
std::make_move_iterator(response.result.end()));
|
||||
}
|
||||
|
||||
return result_rows;
|
||||
}
|
||||
|
||||
std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest requests) override {
|
||||
requests.transaction_id = transaction_id_;
|
||||
// create requests
|
||||
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests_to_be_sent =
|
||||
RequestsForGetProperties(std::move(requests));
|
||||
|
||||
// begin all requests in parallel
|
||||
RunningRequests<msgs::GetPropertiesRequest> running_requests = {};
|
||||
running_requests.reserve(requests_to_be_sent.size());
|
||||
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
|
||||
auto &request = requests_to_be_sent[i];
|
||||
io::ReadinessToken readiness_token{i};
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
msgs::ReadRequests req = request.request;
|
||||
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
|
||||
running_requests.emplace(readiness_token.GetId(), request);
|
||||
}
|
||||
|
||||
// drive requests to completion
|
||||
auto responses = DriveReadResponses<msgs::GetPropertiesRequest, msgs::GetPropertiesResponse>(running_requests);
|
||||
|
||||
// post-process responses
|
||||
std::vector<msgs::GetPropertiesResultRow> result_rows;
|
||||
|
||||
for (auto &&response : responses) {
|
||||
std::move(response.result_row.begin(), response.result_row.end(), std::back_inserter(result_rows));
|
||||
}
|
||||
|
||||
return result_rows;
|
||||
}
|
||||
|
||||
std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const override {
|
||||
return shards_map_.GetPropertyId(name);
|
||||
}
|
||||
|
||||
std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const override {
|
||||
return shards_map_.GetEdgeTypeId(name);
|
||||
}
|
||||
|
||||
std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const override {
|
||||
return shards_map_.GetLabelId(name);
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> RequestsForCreateVertices(
|
||||
const std::vector<msgs::NewVertex> &new_vertices) {
|
||||
std::map<ShardMetadata, msgs::CreateVerticesRequest> per_shard_request_table;
|
||||
|
||||
for (auto &new_vertex : new_vertices) {
|
||||
MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices");
|
||||
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
|
||||
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
msgs::CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
|
||||
per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
|
||||
}
|
||||
per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests = {};
|
||||
|
||||
for (auto &[shard, request] : per_shard_request_table) {
|
||||
ShardRequestState<msgs::CreateVerticesRequest> shard_request_state{
|
||||
.shard = shard,
|
||||
.request = request,
|
||||
};
|
||||
requests.emplace_back(std::move(shard_request_state));
|
||||
}
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::CreateExpandRequest>> RequestsForCreateExpand(
|
||||
std::vector<msgs::NewExpand> new_expands) {
|
||||
std::map<ShardMetadata, msgs::CreateExpandRequest> per_shard_request_table;
|
||||
auto ensure_shard_exists_in_table = [&per_shard_request_table,
|
||||
transaction_id = transaction_id_](const ShardMetadata &shard) {
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
msgs::CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
|
||||
per_shard_request_table.insert({shard, std::move(create_expand_request)});
|
||||
}
|
||||
};
|
||||
|
||||
for (auto &new_expand : new_expands) {
|
||||
const auto shard_src_vertex = shards_map_.GetShardForKey(
|
||||
new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
|
||||
const auto shard_dest_vertex = shards_map_.GetShardForKey(
|
||||
new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
|
||||
|
||||
ensure_shard_exists_in_table(shard_src_vertex);
|
||||
|
||||
if (shard_src_vertex != shard_dest_vertex) {
|
||||
ensure_shard_exists_in_table(shard_dest_vertex);
|
||||
per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
|
||||
}
|
||||
per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests = {};
|
||||
|
||||
for (auto &[shard, request] : per_shard_request_table) {
|
||||
ShardRequestState<msgs::CreateExpandRequest> shard_request_state{
|
||||
.shard = shard,
|
||||
.request = request,
|
||||
};
|
||||
requests.emplace_back(std::move(shard_request_state));
|
||||
}
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> RequestsForScanVertices(
|
||||
const std::optional<std::string> &label) {
|
||||
std::vector<coordinator::Shards> multi_shards;
|
||||
if (label) {
|
||||
const auto label_id = shards_map_.GetLabelId(*label);
|
||||
MG_ASSERT(label_id);
|
||||
MG_ASSERT(IsPrimaryLabel(*label_id));
|
||||
multi_shards = {shards_map_.GetShardsForLabel(*label)};
|
||||
} else {
|
||||
multi_shards = shards_map_.GetAllShards();
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests = {};
|
||||
|
||||
for (auto &shards : multi_shards) {
|
||||
for (auto &[key, shard] : shards) {
|
||||
MG_ASSERT(!shard.peers.empty());
|
||||
|
||||
msgs::ScanVerticesRequest request;
|
||||
request.transaction_id = transaction_id_;
|
||||
request.start_id.second = storage::conversions::ConvertValueVector(key);
|
||||
|
||||
ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
|
||||
.shard = shard,
|
||||
.request = std::move(request),
|
||||
};
|
||||
|
||||
requests.emplace_back(std::move(shard_request_state));
|
||||
}
|
||||
}
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(const msgs::ExpandOneRequest &request) {
|
||||
std::map<ShardMetadata, msgs::ExpandOneRequest> per_shard_request_table;
|
||||
msgs::ExpandOneRequest top_level_rqst_template = request;
|
||||
top_level_rqst_template.transaction_id = transaction_id_;
|
||||
top_level_rqst_template.src_vertices.clear();
|
||||
|
||||
for (auto &vertex : request.src_vertices) {
|
||||
auto shard =
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
}
|
||||
per_shard_request_table[shard].src_vertices.push_back(vertex);
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests = {};
|
||||
|
||||
for (auto &[shard, request] : per_shard_request_table) {
|
||||
ShardRequestState<msgs::ExpandOneRequest> shard_request_state{
|
||||
.shard = shard,
|
||||
.request = request,
|
||||
};
|
||||
|
||||
requests.emplace_back(std::move(shard_request_state));
|
||||
}
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> RequestsForGetProperties(
|
||||
msgs::GetPropertiesRequest &&request) {
|
||||
std::map<ShardMetadata, msgs::GetPropertiesRequest> per_shard_request_table;
|
||||
auto top_level_rqst_template = request;
|
||||
top_level_rqst_template.transaction_id = transaction_id_;
|
||||
top_level_rqst_template.vertex_ids.clear();
|
||||
top_level_rqst_template.vertices_and_edges.clear();
|
||||
|
||||
for (auto &&vertex : request.vertex_ids) {
|
||||
auto shard =
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
}
|
||||
per_shard_request_table[shard].vertex_ids.emplace_back(std::move(vertex));
|
||||
}
|
||||
|
||||
for (auto &[vertex, maybe_edge] : request.vertices_and_edges) {
|
||||
auto shard =
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
}
|
||||
per_shard_request_table[shard].vertices_and_edges.emplace_back(std::move(vertex), maybe_edge);
|
||||
}
|
||||
|
||||
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests;
|
||||
|
||||
for (auto &[shard, rqst] : per_shard_request_table) {
|
||||
ShardRequestState<msgs::GetPropertiesRequest> shard_request_state{
|
||||
.shard = shard,
|
||||
.request = std::move(rqst),
|
||||
};
|
||||
|
||||
requests.emplace_back(std::move(shard_request_state));
|
||||
}
|
||||
|
||||
return requests;
|
||||
}
|
||||
|
||||
StorageClient &GetStorageClientForShard(ShardMetadata shard) {
|
||||
if (!storage_cli_manager_.Exists(shard)) {
|
||||
AddStorageClientToManager(shard);
|
||||
}
|
||||
return storage_cli_manager_.GetClient(shard);
|
||||
}
|
||||
|
||||
StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
|
||||
auto shard = shards_map_.GetShardForKey(label, key);
|
||||
return GetStorageClientForShard(std::move(shard));
|
||||
}
|
||||
|
||||
void AddStorageClientToManager(ShardMetadata target_shard) {
|
||||
MG_ASSERT(!target_shard.peers.empty());
|
||||
auto leader_addr = target_shard.peers.front();
|
||||
std::vector<Address> addresses;
|
||||
addresses.reserve(target_shard.peers.size());
|
||||
for (auto &address : target_shard.peers) {
|
||||
addresses.push_back(std::move(address.address));
|
||||
}
|
||||
auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
|
||||
storage_cli_manager_.AddClient(target_shard, std::move(cli));
|
||||
}
|
||||
|
||||
template <typename RequestT, typename ResponseT>
|
||||
std::vector<ResponseT> DriveReadResponses(RunningRequests<RequestT> &running_requests) {
|
||||
// Store responses in a map based on the corresponding request
|
||||
// offset, so that they can be reassembled in the correct order
|
||||
// even if they came back in randomized orders.
|
||||
std::map<size_t, ResponseT> response_map;
|
||||
|
||||
spdlog::trace("waiting on readiness for token");
|
||||
while (response_map.size() < running_requests.size()) {
|
||||
auto ready = notifier_.Await();
|
||||
spdlog::trace("got readiness for token {}", ready.GetId());
|
||||
auto &request = running_requests.at(ready.GetId());
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
|
||||
std::optional<utils::BasicResult<io::TimedOut, msgs::ReadResponses>> poll_result =
|
||||
storage_client.PollAsyncReadRequest(ready);
|
||||
|
||||
if (!poll_result.has_value()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (poll_result->HasError()) {
|
||||
throw std::runtime_error("RequestRouter Read request timed out");
|
||||
}
|
||||
|
||||
msgs::ReadResponses response_variant = poll_result->GetValue();
|
||||
auto response = std::get<ResponseT>(response_variant);
|
||||
if (response.error) {
|
||||
throw std::runtime_error("RequestRouter Read request did not succeed");
|
||||
}
|
||||
|
||||
// the readiness token has an ID based on the request vector offset
|
||||
response_map.emplace(ready.GetId(), std::move(response));
|
||||
}
|
||||
|
||||
std::vector<ResponseT> responses;
|
||||
responses.reserve(running_requests.size());
|
||||
|
||||
int last = -1;
|
||||
for (auto &&[offset, response] : response_map) {
|
||||
MG_ASSERT(last + 1 == offset);
|
||||
responses.emplace_back(std::forward<ResponseT>(response));
|
||||
last = offset;
|
||||
}
|
||||
|
||||
return responses;
|
||||
}
|
||||
|
||||
template <typename RequestT, typename ResponseT>
|
||||
std::vector<ResponseT> DriveWriteResponses(RunningRequests<RequestT> &running_requests) {
|
||||
// Store responses in a map based on the corresponding request
|
||||
// offset, so that they can be reassembled in the correct order
|
||||
// even if they came back in randomized orders.
|
||||
std::map<size_t, ResponseT> response_map;
|
||||
|
||||
while (response_map.size() < running_requests.size()) {
|
||||
auto ready = notifier_.Await();
|
||||
auto &request = running_requests.at(ready.GetId());
|
||||
auto &storage_client = GetStorageClientForShard(request.shard);
|
||||
|
||||
std::optional<utils::BasicResult<io::TimedOut, msgs::WriteResponses>> poll_result =
|
||||
storage_client.PollAsyncWriteRequest(ready);
|
||||
|
||||
if (!poll_result.has_value()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (poll_result->HasError()) {
|
||||
throw std::runtime_error("RequestRouter Write request timed out");
|
||||
}
|
||||
|
||||
msgs::WriteResponses response_variant = poll_result->GetValue();
|
||||
auto response = std::get<ResponseT>(response_variant);
|
||||
if (response.error) {
|
||||
throw std::runtime_error("RequestRouter Write request did not succeed");
|
||||
}
|
||||
|
||||
// the readiness token has an ID based on the request vector offset
|
||||
response_map.emplace(ready.GetId(), std::move(response));
|
||||
}
|
||||
|
||||
std::vector<ResponseT> responses;
|
||||
responses.reserve(running_requests.size());
|
||||
|
||||
int last = -1;
|
||||
for (auto &&[offset, response] : response_map) {
|
||||
MG_ASSERT(last + 1 == offset);
|
||||
responses.emplace_back(std::forward<ResponseT>(response));
|
||||
last = offset;
|
||||
}
|
||||
|
||||
return responses;
|
||||
}
|
||||
|
||||
void SetUpNameIdMappers() {
|
||||
std::unordered_map<uint64_t, std::string> id_to_name;
|
||||
for (const auto &[name, id] : shards_map_.labels) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
labels_.StoreMapping(std::move(id_to_name));
|
||||
id_to_name.clear();
|
||||
for (const auto &[name, id] : shards_map_.properties) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
properties_.StoreMapping(std::move(id_to_name));
|
||||
id_to_name.clear();
|
||||
for (const auto &[name, id] : shards_map_.edge_types) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
edge_types_.StoreMapping(std::move(id_to_name));
|
||||
}
|
||||
|
||||
std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) override {
|
||||
coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
|
||||
|
||||
io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
|
||||
ww.operation = requests;
|
||||
auto resp =
|
||||
io_.template Request<io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>,
|
||||
io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>>(coordinator_address, ww)
|
||||
.Wait();
|
||||
if (resp.HasValue()) {
|
||||
const auto alloc_edge_id_reps =
|
||||
std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);
|
||||
return std::make_pair(alloc_edge_id_reps.low, alloc_edge_id_reps.high);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
ShardMap shards_map_;
|
||||
storage::v3::NameIdMapper properties_;
|
||||
storage::v3::NameIdMapper edge_types_;
|
||||
storage::v3::NameIdMapper labels_;
|
||||
CoordinatorClient coord_cli_;
|
||||
RsmStorageClientManager<StorageClient> storage_cli_manager_;
|
||||
io::Io<TTransport> io_;
|
||||
coordinator::Hlc transaction_id_;
|
||||
io::Notifier notifier_ = {};
|
||||
// TODO(kostasrim) Add batch prefetching
|
||||
};
|
||||
|
||||
class RequestRouterFactory {
|
||||
public:
|
||||
RequestRouterFactory() = default;
|
||||
RequestRouterFactory(const RequestRouterFactory &) = delete;
|
||||
RequestRouterFactory &operator=(const RequestRouterFactory &) = delete;
|
||||
RequestRouterFactory(RequestRouterFactory &&) = delete;
|
||||
RequestRouterFactory &operator=(RequestRouterFactory &&) = delete;
|
||||
|
||||
virtual ~RequestRouterFactory() = default;
|
||||
|
||||
virtual std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
|
||||
const coordinator::Address &coordinator_address) const = 0;
|
||||
};
|
||||
|
||||
class LocalRequestRouterFactory : public RequestRouterFactory {
|
||||
using LocalTransportIo = io::Io<io::local_transport::LocalTransport>;
|
||||
LocalTransportIo &io_;
|
||||
|
||||
public:
|
||||
explicit LocalRequestRouterFactory(LocalTransportIo &io) : io_(io) {}
|
||||
|
||||
std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
|
||||
const coordinator::Address &coordinator_address) const override {
|
||||
using TransportType = io::local_transport::LocalTransport;
|
||||
|
||||
auto query_io = io_.ForkLocal(boost::uuids::uuid{boost::uuids::random_generator()()});
|
||||
auto local_transport_io = io_.ForkLocal(boost::uuids::uuid{boost::uuids::random_generator()()});
|
||||
|
||||
return std::make_unique<RequestRouter<TransportType>>(
|
||||
coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
|
||||
std::move(local_transport_io));
|
||||
}
|
||||
};
|
||||
|
||||
class SimulatedRequestRouterFactory : public RequestRouterFactory {
|
||||
io::simulator::Simulator *simulator_;
|
||||
|
||||
public:
|
||||
explicit SimulatedRequestRouterFactory(io::simulator::Simulator &simulator) : simulator_(&simulator) {}
|
||||
|
||||
std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
|
||||
const coordinator::Address &coordinator_address) const override {
|
||||
using TransportType = io::simulator::SimulatorTransport;
|
||||
auto actual_transport_handle = simulator_->GetSimulatorHandle();
|
||||
|
||||
boost::uuids::uuid random_uuid;
|
||||
io::Address unique_local_addr_query;
|
||||
|
||||
// The simulated RR should not introduce stochastic behavior.
|
||||
random_uuid = boost::uuids::uuid{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
|
||||
unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
|
||||
|
||||
auto io = simulator_->Register(unique_local_addr_query);
|
||||
auto query_io = io.ForkLocal(random_uuid);
|
||||
|
||||
return std::make_unique<RequestRouter<TransportType>>(
|
||||
coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
|
||||
std::move(io));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -24,8 +24,6 @@
|
||||
#include "coordinator/hybrid_logical_clock.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
|
||||
namespace memgraph::msgs {
|
||||
|
||||
@@ -37,7 +35,6 @@ struct Value;
|
||||
struct Label {
|
||||
LabelId id;
|
||||
friend bool operator==(const Label &lhs, const Label &rhs) { return lhs.id == rhs.id; }
|
||||
friend bool operator==(const Label &lhs, const LabelId &rhs) { return lhs.id == rhs; }
|
||||
};
|
||||
|
||||
// TODO(kostasrim) update this with CompoundKey, same for the rest of the file.
|
||||
@@ -320,15 +317,14 @@ struct Value {
|
||||
}
|
||||
};
|
||||
|
||||
struct ShardError {
|
||||
common::ErrorCode code;
|
||||
std::string message;
|
||||
};
|
||||
|
||||
struct Expression {
|
||||
std::string expression;
|
||||
};
|
||||
|
||||
struct Filter {
|
||||
std::string filter_expression;
|
||||
};
|
||||
|
||||
enum class OrderingDirection { ASCENDING = 1, DESCENDING = 2 };
|
||||
|
||||
struct OrderBy {
|
||||
@@ -365,38 +361,27 @@ struct ScanResultRow {
|
||||
};
|
||||
|
||||
struct ScanVerticesResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
std::optional<VertexId> next_start_id;
|
||||
std::vector<ScanResultRow> results;
|
||||
};
|
||||
|
||||
using VertexOrEdgeIds = std::variant<VertexId, EdgeId>;
|
||||
|
||||
struct GetPropertiesRequest {
|
||||
Hlc transaction_id;
|
||||
std::vector<VertexId> vertex_ids;
|
||||
std::vector<std::pair<VertexId, EdgeId>> vertices_and_edges;
|
||||
|
||||
std::optional<std::vector<PropertyId>> property_ids;
|
||||
std::vector<std::string> expressions;
|
||||
|
||||
std::vector<OrderBy> order_by;
|
||||
// Shouldn't contain mixed vertex and edge ids
|
||||
VertexOrEdgeIds vertex_or_edge_ids;
|
||||
std::vector<PropertyId> property_ids;
|
||||
std::vector<Expression> expressions;
|
||||
bool only_unique = false;
|
||||
std::optional<std::vector<OrderBy>> order_by;
|
||||
std::optional<size_t> limit;
|
||||
|
||||
// Return only the properties of the vertices or edges that the filter predicate
|
||||
// evaluates to true
|
||||
std::optional<std::string> filter;
|
||||
};
|
||||
|
||||
struct GetPropertiesResultRow {
|
||||
VertexId vertex;
|
||||
std::optional<EdgeId> edge;
|
||||
|
||||
std::vector<std::pair<PropertyId, Value>> props;
|
||||
std::vector<Value> evaluated_expressions;
|
||||
std::optional<Filter> filter;
|
||||
};
|
||||
|
||||
struct GetPropertiesResponse {
|
||||
std::vector<GetPropertiesResultRow> result_row;
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
enum class EdgeDirection : uint8_t { OUT = 1, IN = 2, BOTH = 3 };
|
||||
@@ -418,9 +403,7 @@ struct ExpandOneRequest {
|
||||
std::vector<std::string> vertex_expressions;
|
||||
std::vector<std::string> edge_expressions;
|
||||
|
||||
std::vector<OrderBy> order_by_vertices;
|
||||
std::vector<OrderBy> order_by_edges;
|
||||
|
||||
std::optional<std::vector<OrderBy>> order_by;
|
||||
// Limit the edges or the vertices?
|
||||
std::optional<size_t> limit;
|
||||
std::vector<std::string> filters;
|
||||
@@ -463,16 +446,14 @@ struct ExpandOneResultRow {
|
||||
};
|
||||
|
||||
struct ExpandOneResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
std::vector<ExpandOneResultRow> result;
|
||||
};
|
||||
|
||||
struct UpdateVertex {
|
||||
struct UpdateVertexProp {
|
||||
PrimaryKey primary_key;
|
||||
// Labels are first added and then removed from vertices
|
||||
std::vector<LabelId> add_labels;
|
||||
std::vector<LabelId> remove_labels;
|
||||
std::map<PropertyId, Value> property_updates;
|
||||
// This should be a map
|
||||
std::vector<std::pair<PropertyId, Value>> property_updates;
|
||||
};
|
||||
|
||||
struct UpdateEdgeProp {
|
||||
@@ -499,7 +480,7 @@ struct CreateVerticesRequest {
|
||||
};
|
||||
|
||||
struct CreateVerticesResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct DeleteVerticesRequest {
|
||||
@@ -510,16 +491,16 @@ struct DeleteVerticesRequest {
|
||||
};
|
||||
|
||||
struct DeleteVerticesResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct UpdateVerticesRequest {
|
||||
Hlc transaction_id;
|
||||
std::vector<UpdateVertex> update_vertices;
|
||||
std::vector<UpdateVertexProp> new_properties;
|
||||
};
|
||||
|
||||
struct UpdateVerticesResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
/*
|
||||
@@ -541,7 +522,7 @@ struct CreateExpandRequest {
|
||||
};
|
||||
|
||||
struct CreateExpandResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct DeleteEdgesRequest {
|
||||
@@ -550,7 +531,7 @@ struct DeleteEdgesRequest {
|
||||
};
|
||||
|
||||
struct DeleteEdgesResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct UpdateEdgesRequest {
|
||||
@@ -559,7 +540,7 @@ struct UpdateEdgesRequest {
|
||||
};
|
||||
|
||||
struct UpdateEdgesResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
struct CommitRequest {
|
||||
@@ -568,7 +549,7 @@ struct CommitRequest {
|
||||
};
|
||||
|
||||
struct CommitResponse {
|
||||
std::optional<ShardError> error;
|
||||
bool success;
|
||||
};
|
||||
|
||||
using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
|
||||
@@ -580,48 +561,3 @@ using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesRespon
|
||||
CreateExpandResponse, DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse>;
|
||||
|
||||
} // namespace memgraph::msgs
|
||||
|
||||
namespace std {
|
||||
|
||||
template <>
|
||||
struct hash<memgraph::msgs::Value>;
|
||||
|
||||
template <>
|
||||
struct hash<memgraph::msgs::VertexId> {
|
||||
size_t operator()(const memgraph::msgs::VertexId &id) const {
|
||||
using LabelId = memgraph::storage::v3::LabelId;
|
||||
using Value = memgraph::msgs::Value;
|
||||
return memgraph::utils::HashCombine<LabelId, std::vector<Value>, std::hash<LabelId>,
|
||||
memgraph::utils::FnvCollection<std::vector<Value>, Value>>{}(id.first.id,
|
||||
id.second);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct hash<memgraph::msgs::Value> {
|
||||
size_t operator()(const memgraph::msgs::Value &value) const {
|
||||
using Type = memgraph::msgs::Value::Type;
|
||||
switch (value.type) {
|
||||
case Type::Null:
|
||||
return std::hash<size_t>{}(0U);
|
||||
case Type::Bool:
|
||||
return std::hash<bool>{}(value.bool_v);
|
||||
case Type::Int64:
|
||||
return std::hash<int64_t>{}(value.int_v);
|
||||
case Type::Double:
|
||||
return std::hash<double>{}(value.double_v);
|
||||
case Type::String:
|
||||
return std::hash<std::string>{}(value.string_v);
|
||||
case Type::List:
|
||||
LOG_FATAL("Add hash for lists");
|
||||
case Type::Map:
|
||||
LOG_FATAL("Add hash for maps");
|
||||
case Type::Vertex:
|
||||
LOG_FATAL("Add hash for vertices");
|
||||
case Type::Edge:
|
||||
LOG_FATAL("Add hash for edges");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace std
|
||||
|
||||
730
src/query/v2/shard_request_manager.hpp
Normal file
730
src/query/v2/shard_request_manager.hpp
Normal file
@@ -0,0 +1,730 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <deque>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <map>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <random>
|
||||
#include <set>
|
||||
#include <stdexcept>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/coordinator_client.hpp"
|
||||
#include "coordinator/coordinator_rsm.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include "io/rsm/rsm_client.hpp"
|
||||
#include "io/rsm/shard_rsm.hpp"
|
||||
#include "io/simulator/simulator.hpp"
|
||||
#include "io/simulator/simulator_transport.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::msgs {
|
||||
template <typename TStorageClient>
|
||||
class RsmStorageClientManager {
|
||||
public:
|
||||
using CompoundKey = memgraph::io::rsm::ShardRsmKey;
|
||||
using Shard = memgraph::coordinator::Shard;
|
||||
using LabelId = memgraph::storage::v3::LabelId;
|
||||
RsmStorageClientManager() = default;
|
||||
RsmStorageClientManager(const RsmStorageClientManager &) = delete;
|
||||
RsmStorageClientManager(RsmStorageClientManager &&) = delete;
|
||||
RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
|
||||
RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
|
||||
~RsmStorageClientManager() = default;
|
||||
|
||||
void AddClient(Shard key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
|
||||
|
||||
bool Exists(const Shard &key) { return cli_cache_.contains(key); }
|
||||
|
||||
void PurgeCache() { cli_cache_.clear(); }
|
||||
|
||||
TStorageClient &GetClient(const Shard &key) {
|
||||
auto it = cli_cache_.find(key);
|
||||
MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
|
||||
return it->second;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<Shard, TStorageClient> cli_cache_;
|
||||
};
|
||||
|
||||
template <typename TRequest>
|
||||
struct ExecutionState {
|
||||
using CompoundKey = memgraph::io::rsm::ShardRsmKey;
|
||||
using Shard = memgraph::coordinator::Shard;
|
||||
|
||||
enum State : int8_t { INITIALIZING, EXECUTING, COMPLETED };
|
||||
// label is optional because some operators can create/remove etc, vertices. These kind of requests contain the label
|
||||
// on the request itself.
|
||||
std::optional<std::string> label;
|
||||
// CompoundKey is optional because some operators require to iterate over all the available keys
|
||||
// of a shard. One example is ScanAll, where we only require the field label.
|
||||
std::optional<CompoundKey> key;
|
||||
// Transaction id to be filled by the ShardRequestManager implementation
|
||||
memgraph::coordinator::Hlc transaction_id;
|
||||
// Initialized by ShardRequestManager implementation. This vector is filled with the shards that
|
||||
// the ShardRequestManager impl will send requests to. When a request to a shard exhausts it, meaning that
|
||||
// it pulled all the requested data from the given Shard, it will be removed from the Vector. When the Vector becomes
|
||||
// empty, it means that all of the requests have completed succefully.
|
||||
// TODO(gvolfing)
|
||||
// Maybe make this into a more complex object to be able to keep track of paginated resutls. E.g. instead of a vector
|
||||
// of Shards make it into a std::vector<std::pair<Shard, PaginatedResultType>> (probably a struct instead of a pair)
|
||||
// where PaginatedResultType is an enum signaling the progress on the given request. This way we can easily check if
|
||||
// a partial response on a shard(if there is one) is finished and we can send off the request for the next batch.
|
||||
std::vector<Shard> shard_cache;
|
||||
// 1-1 mapping with `shard_cache`.
|
||||
// A vector that tracks request metadata for each shard (For example, next_id for a ScanAll on Shard A)
|
||||
std::vector<TRequest> requests;
|
||||
State state = INITIALIZING;
|
||||
};
|
||||
|
||||
class ShardRequestManagerInterface {
|
||||
public:
|
||||
using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
|
||||
ShardRequestManagerInterface() = default;
|
||||
ShardRequestManagerInterface(const ShardRequestManagerInterface &) = delete;
|
||||
ShardRequestManagerInterface(ShardRequestManagerInterface &&) = delete;
|
||||
ShardRequestManagerInterface &operator=(const ShardRequestManagerInterface &) = delete;
|
||||
ShardRequestManagerInterface &&operator=(ShardRequestManagerInterface &&) = delete;
|
||||
|
||||
virtual ~ShardRequestManagerInterface() = default;
|
||||
|
||||
virtual void StartTransaction() = 0;
|
||||
virtual void Commit() = 0;
|
||||
virtual std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) = 0;
|
||||
virtual std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
|
||||
std::vector<NewVertex> new_vertices) = 0;
|
||||
virtual std::vector<ExpandOneResultRow> Request(ExecutionState<ExpandOneRequest> &state,
|
||||
ExpandOneRequest request) = 0;
|
||||
virtual std::vector<CreateExpandResponse> Request(ExecutionState<CreateExpandRequest> &state,
|
||||
std::vector<NewExpand> new_edges) = 0;
|
||||
|
||||
virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
|
||||
virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
|
||||
virtual storage::v3::LabelId NameToLabel(const std::string &name) const = 0;
|
||||
virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0;
|
||||
virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0;
|
||||
virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0;
|
||||
virtual bool IsPrimaryLabel(LabelId label) const = 0;
|
||||
virtual bool IsPrimaryKey(LabelId primary_label, PropertyId property) const = 0;
|
||||
};
|
||||
|
||||
// TODO(kostasrim)rename this class template
|
||||
template <typename TTransport>
|
||||
class ShardRequestManager : public ShardRequestManagerInterface {
|
||||
public:
|
||||
using StorageClient =
|
||||
memgraph::coordinator::RsmClient<TTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
|
||||
using CoordinatorWriteRequests = memgraph::coordinator::CoordinatorWriteRequests;
|
||||
using CoordinatorClient = memgraph::coordinator::CoordinatorClient<TTransport>;
|
||||
using Address = memgraph::io::Address;
|
||||
using Shard = memgraph::coordinator::Shard;
|
||||
using ShardMap = memgraph::coordinator::ShardMap;
|
||||
using CompoundKey = memgraph::coordinator::PrimaryKey;
|
||||
using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
|
||||
ShardRequestManager(CoordinatorClient coord, memgraph::io::Io<TTransport> &&io)
|
||||
: coord_cli_(std::move(coord)), io_(std::move(io)) {}
|
||||
|
||||
ShardRequestManager(const ShardRequestManager &) = delete;
|
||||
ShardRequestManager(ShardRequestManager &&) = delete;
|
||||
ShardRequestManager &operator=(const ShardRequestManager &) = delete;
|
||||
ShardRequestManager &operator=(ShardRequestManager &&) = delete;
|
||||
|
||||
~ShardRequestManager() override {}
|
||||
|
||||
void StartTransaction() override {
|
||||
memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
|
||||
CoordinatorWriteRequests write_req = req;
|
||||
auto write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
if (write_res.HasError()) {
|
||||
throw std::runtime_error("HLC request failed");
|
||||
}
|
||||
auto coordinator_write_response = write_res.GetValue();
|
||||
auto hlc_response = std::get<memgraph::coordinator::HlcResponse>(coordinator_write_response);
|
||||
|
||||
// Transaction ID to be used later...
|
||||
transaction_id_ = hlc_response.new_hlc;
|
||||
|
||||
if (hlc_response.fresher_shard_map) {
|
||||
shards_map_ = hlc_response.fresher_shard_map.value();
|
||||
SetUpNameIdMappers();
|
||||
}
|
||||
}
|
||||
|
||||
void Commit() override {
|
||||
memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
|
||||
CoordinatorWriteRequests write_req = req;
|
||||
auto write_res = coord_cli_.SendWriteRequest(write_req);
|
||||
if (write_res.HasError()) {
|
||||
throw std::runtime_error("HLC request for commit failed");
|
||||
}
|
||||
auto coordinator_write_response = write_res.GetValue();
|
||||
auto hlc_response = std::get<memgraph::coordinator::HlcResponse>(coordinator_write_response);
|
||||
|
||||
if (hlc_response.fresher_shard_map) {
|
||||
shards_map_ = hlc_response.fresher_shard_map.value();
|
||||
SetUpNameIdMappers();
|
||||
}
|
||||
auto commit_timestamp = hlc_response.new_hlc;
|
||||
|
||||
msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
|
||||
|
||||
for (const auto &[label, space] : shards_map_.label_spaces) {
|
||||
for (const auto &[key, shard] : space.shards) {
|
||||
auto &storage_client = GetStorageClientForShard(shard);
|
||||
// TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
|
||||
// instead.
|
||||
auto commit_response = storage_client.SendWriteRequest(commit_req);
|
||||
// RETRY on timeouts?
|
||||
// Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
|
||||
if (commit_response.HasError()) {
|
||||
throw std::runtime_error("Commit request timed out");
|
||||
}
|
||||
WriteResponses write_response_variant = commit_response.GetValue();
|
||||
auto &response = std::get<CommitResponse>(write_response_variant);
|
||||
if (!response.success) {
|
||||
throw std::runtime_error("Commit request did not succeed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
|
||||
return shards_map_.GetEdgeTypeId(name).value();
|
||||
}
|
||||
|
||||
storage::v3::PropertyId NameToProperty(const std::string &name) const override {
|
||||
return shards_map_.GetPropertyId(name).value();
|
||||
}
|
||||
|
||||
storage::v3::LabelId NameToLabel(const std::string &name) const override {
|
||||
return shards_map_.GetLabelId(name).value();
|
||||
}
|
||||
|
||||
const std::string &PropertyToName(memgraph::storage::v3::PropertyId id) const override {
|
||||
return properties_.IdToName(id.AsUint());
|
||||
}
|
||||
const std::string &LabelToName(memgraph::storage::v3::LabelId id) const override {
|
||||
return labels_.IdToName(id.AsUint());
|
||||
}
|
||||
const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId id) const override {
|
||||
return edge_types_.IdToName(id.AsUint());
|
||||
}
|
||||
|
||||
bool IsPrimaryKey(LabelId primary_label, PropertyId property) const override {
|
||||
const auto schema_it = shards_map_.schemas.find(primary_label);
|
||||
MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
|
||||
|
||||
return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
|
||||
return schema_prop.property_id == property;
|
||||
}) != schema_it->second.end();
|
||||
}
|
||||
|
||||
bool IsPrimaryLabel(LabelId label) const override { return shards_map_.label_spaces.contains(label); }
|
||||
|
||||
// TODO(kostasrim) Simplify return result
|
||||
std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) override {
|
||||
MaybeInitializeExecutionState(state);
|
||||
std::vector<ScanVerticesResponse> responses;
|
||||
|
||||
SendAllRequests(state);
|
||||
auto all_requests_gathered = [](auto &paginated_rsp_tracker) {
|
||||
return std::ranges::all_of(paginated_rsp_tracker, [](const auto &state) {
|
||||
return state.second == PaginatedResponseState::PartiallyFinished;
|
||||
});
|
||||
};
|
||||
|
||||
std::map<Shard, PaginatedResponseState> paginated_response_tracker;
|
||||
for (const auto &shard : state.shard_cache) {
|
||||
paginated_response_tracker.insert(std::make_pair(shard, PaginatedResponseState::Pending));
|
||||
}
|
||||
|
||||
do {
|
||||
AwaitOnPaginatedRequests(state, responses, paginated_response_tracker);
|
||||
} while (!all_requests_gathered(paginated_response_tracker));
|
||||
|
||||
MaybeCompleteState(state);
|
||||
// TODO(kostasrim) Before returning start prefetching the batch (this shall be done once we get MgFuture as return
|
||||
// result of storage_client.SendReadRequest()).
|
||||
return PostProcess(std::move(responses));
|
||||
}
|
||||
|
||||
std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
|
||||
std::vector<NewVertex> new_vertices) override {
|
||||
MG_ASSERT(!new_vertices.empty());
|
||||
MaybeInitializeExecutionState(state, new_vertices);
|
||||
std::vector<CreateVerticesResponse> responses;
|
||||
auto &shard_cache_ref = state.shard_cache;
|
||||
|
||||
// 1. Send the requests.
|
||||
SendAllRequests(state, shard_cache_ref);
|
||||
|
||||
// 2. Block untill all the futures are exhausted
|
||||
do {
|
||||
AwaitOnResponses(state, responses);
|
||||
} while (!state.shard_cache.empty());
|
||||
|
||||
MaybeCompleteState(state);
|
||||
// TODO(kostasrim) Before returning start prefetching the batch (this shall be done once we get MgFuture as return
|
||||
// result of storage_client.SendReadRequest()).
|
||||
return responses;
|
||||
}
|
||||
|
||||
std::vector<CreateExpandResponse> Request(ExecutionState<CreateExpandRequest> &state,
|
||||
std::vector<NewExpand> new_edges) override {
|
||||
MG_ASSERT(!new_edges.empty());
|
||||
MaybeInitializeExecutionState(state, new_edges);
|
||||
std::vector<CreateExpandResponse> responses;
|
||||
auto &shard_cache_ref = state.shard_cache;
|
||||
size_t id{0};
|
||||
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++id) {
|
||||
auto &storage_client = GetStorageClientForShard(*shard_it);
|
||||
WriteRequests req = state.requests[id];
|
||||
auto write_response_result = storage_client.SendWriteRequest(std::move(req));
|
||||
if (write_response_result.HasError()) {
|
||||
throw std::runtime_error("CreateVertices request timedout");
|
||||
}
|
||||
WriteResponses response_variant = write_response_result.GetValue();
|
||||
CreateExpandResponse mapped_response = std::get<CreateExpandResponse>(response_variant);
|
||||
|
||||
if (!mapped_response.success) {
|
||||
throw std::runtime_error("CreateExpand request did not succeed");
|
||||
}
|
||||
responses.push_back(mapped_response);
|
||||
shard_it = shard_cache_ref.erase(shard_it);
|
||||
}
|
||||
// We are done with this state
|
||||
MaybeCompleteState(state);
|
||||
return responses;
|
||||
}
|
||||
|
||||
std::vector<ExpandOneResultRow> Request(ExecutionState<ExpandOneRequest> &state, ExpandOneRequest request) override {
|
||||
// TODO(kostasrim)Update to limit the batch size here
|
||||
// Expansions of the destination must be handled by the caller. For example
|
||||
// match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
|
||||
// For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
|
||||
// must be fetched again with an ExpandOne(Edges.dst)
|
||||
MaybeInitializeExecutionState(state, std::move(request));
|
||||
std::vector<ExpandOneResponse> responses;
|
||||
auto &shard_cache_ref = state.shard_cache;
|
||||
|
||||
// 1. Send the requests.
|
||||
SendAllRequests(state, shard_cache_ref);
|
||||
|
||||
// 2. Block untill all the futures are exhausted
|
||||
do {
|
||||
AwaitOnResponses(state, responses);
|
||||
} while (!state.shard_cache.empty());
|
||||
std::vector<ExpandOneResultRow> result_rows;
|
||||
const auto total_row_count = std::accumulate(
|
||||
responses.begin(), responses.end(), 0,
|
||||
[](const int64_t partial_count, const ExpandOneResponse &resp) { return partial_count + resp.result.size(); });
|
||||
result_rows.reserve(total_row_count);
|
||||
|
||||
for (auto &response : responses) {
|
||||
result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
|
||||
std::make_move_iterator(response.result.end()));
|
||||
}
|
||||
MaybeCompleteState(state);
|
||||
return result_rows;
|
||||
}
|
||||
|
||||
private:
|
||||
enum class PaginatedResponseState { Pending, PartiallyFinished };
|
||||
|
||||
std::vector<VertexAccessor> PostProcess(std::vector<ScanVerticesResponse> &&responses) const {
|
||||
std::vector<VertexAccessor> accessors;
|
||||
for (auto &response : responses) {
|
||||
for (auto &result_row : response.results) {
|
||||
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
|
||||
}
|
||||
}
|
||||
return accessors;
|
||||
}
|
||||
|
||||
template <typename ExecutionState>
|
||||
void ThrowIfStateCompleted(ExecutionState &state) const {
|
||||
if (state.state == ExecutionState::COMPLETED) [[unlikely]] {
|
||||
throw std::runtime_error("State is completed and must be reset");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ExecutionState>
|
||||
void MaybeCompleteState(ExecutionState &state) const {
|
||||
if (state.requests.empty()) {
|
||||
state.state = ExecutionState::COMPLETED;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ExecutionState>
|
||||
bool ShallNotInitializeState(ExecutionState &state) const {
|
||||
return state.state != ExecutionState::INITIALIZING;
|
||||
}
|
||||
|
||||
void MaybeInitializeExecutionState(ExecutionState<CreateVerticesRequest> &state,
|
||||
std::vector<NewVertex> new_vertices) {
|
||||
ThrowIfStateCompleted(state);
|
||||
if (ShallNotInitializeState(state)) {
|
||||
return;
|
||||
}
|
||||
state.transaction_id = transaction_id_;
|
||||
|
||||
std::map<Shard, CreateVerticesRequest> per_shard_request_table;
|
||||
|
||||
for (auto &new_vertex : new_vertices) {
|
||||
MG_ASSERT(!new_vertex.label_ids.empty(), "This is error!");
|
||||
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
|
||||
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
|
||||
per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
|
||||
state.shard_cache.push_back(shard);
|
||||
}
|
||||
per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
|
||||
}
|
||||
|
||||
for (auto &[shard, rqst] : per_shard_request_table) {
|
||||
state.requests.push_back(std::move(rqst));
|
||||
}
|
||||
state.state = ExecutionState<CreateVerticesRequest>::EXECUTING;
|
||||
}
|
||||
|
||||
void MaybeInitializeExecutionState(ExecutionState<CreateExpandRequest> &state, std::vector<NewExpand> new_expands) {
|
||||
ThrowIfStateCompleted(state);
|
||||
if (ShallNotInitializeState(state)) {
|
||||
return;
|
||||
}
|
||||
state.transaction_id = transaction_id_;
|
||||
|
||||
std::map<Shard, CreateExpandRequest> per_shard_request_table;
|
||||
auto ensure_shard_exists_in_table = [&per_shard_request_table,
|
||||
transaction_id = transaction_id_](const Shard &shard) {
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
|
||||
per_shard_request_table.insert({shard, std::move(create_expand_request)});
|
||||
}
|
||||
};
|
||||
|
||||
for (auto &new_expand : new_expands) {
|
||||
const auto shard_src_vertex = shards_map_.GetShardForKey(
|
||||
new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
|
||||
const auto shard_dest_vertex = shards_map_.GetShardForKey(
|
||||
new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
|
||||
|
||||
ensure_shard_exists_in_table(shard_src_vertex);
|
||||
|
||||
if (shard_src_vertex != shard_dest_vertex) {
|
||||
ensure_shard_exists_in_table(shard_dest_vertex);
|
||||
per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
|
||||
}
|
||||
per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
|
||||
}
|
||||
|
||||
for (auto &[shard, request] : per_shard_request_table) {
|
||||
state.shard_cache.push_back(shard);
|
||||
state.requests.push_back(std::move(request));
|
||||
}
|
||||
state.state = ExecutionState<CreateExpandRequest>::EXECUTING;
|
||||
}
|
||||
|
||||
void MaybeInitializeExecutionState(ExecutionState<ScanVerticesRequest> &state) {
|
||||
ThrowIfStateCompleted(state);
|
||||
if (ShallNotInitializeState(state)) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<coordinator::Shards> multi_shards;
|
||||
state.transaction_id = transaction_id_;
|
||||
if (!state.label) {
|
||||
multi_shards = shards_map_.GetAllShards();
|
||||
} else {
|
||||
const auto label_id = shards_map_.GetLabelId(*state.label);
|
||||
MG_ASSERT(label_id);
|
||||
MG_ASSERT(IsPrimaryLabel(*label_id));
|
||||
multi_shards = {shards_map_.GetShardsForLabel(*state.label)};
|
||||
}
|
||||
for (auto &shards : multi_shards) {
|
||||
for (auto &[key, shard] : shards) {
|
||||
MG_ASSERT(!shard.empty());
|
||||
state.shard_cache.push_back(std::move(shard));
|
||||
ScanVerticesRequest rqst;
|
||||
rqst.transaction_id = transaction_id_;
|
||||
rqst.start_id.second = storage::conversions::ConvertValueVector(key);
|
||||
state.requests.push_back(std::move(rqst));
|
||||
}
|
||||
}
|
||||
state.state = ExecutionState<ScanVerticesRequest>::EXECUTING;
|
||||
}
|
||||
|
||||
void MaybeInitializeExecutionState(ExecutionState<ExpandOneRequest> &state, ExpandOneRequest request) {
|
||||
ThrowIfStateCompleted(state);
|
||||
if (ShallNotInitializeState(state)) {
|
||||
return;
|
||||
}
|
||||
state.transaction_id = transaction_id_;
|
||||
|
||||
std::map<Shard, ExpandOneRequest> per_shard_request_table;
|
||||
auto top_level_rqst_template = request;
|
||||
top_level_rqst_template.transaction_id = transaction_id_;
|
||||
top_level_rqst_template.src_vertices.clear();
|
||||
state.requests.clear();
|
||||
for (auto &vertex : request.src_vertices) {
|
||||
auto shard =
|
||||
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
|
||||
if (!per_shard_request_table.contains(shard)) {
|
||||
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
|
||||
state.shard_cache.push_back(shard);
|
||||
}
|
||||
per_shard_request_table[shard].src_vertices.push_back(vertex);
|
||||
}
|
||||
|
||||
for (auto &[shard, rqst] : per_shard_request_table) {
|
||||
state.requests.push_back(std::move(rqst));
|
||||
}
|
||||
state.state = ExecutionState<ExpandOneRequest>::EXECUTING;
|
||||
}
|
||||
|
||||
StorageClient &GetStorageClientForShard(Shard shard) {
|
||||
if (!storage_cli_manager_.Exists(shard)) {
|
||||
AddStorageClientToManager(shard);
|
||||
}
|
||||
return storage_cli_manager_.GetClient(shard);
|
||||
}
|
||||
|
||||
StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
|
||||
auto shard = shards_map_.GetShardForKey(label, key);
|
||||
return GetStorageClientForShard(std::move(shard));
|
||||
}
|
||||
|
||||
void AddStorageClientToManager(Shard target_shard) {
|
||||
MG_ASSERT(!target_shard.empty());
|
||||
auto leader_addr = target_shard.front();
|
||||
std::vector<Address> addresses;
|
||||
addresses.reserve(target_shard.size());
|
||||
for (auto &address : target_shard) {
|
||||
addresses.push_back(std::move(address.address));
|
||||
}
|
||||
auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
|
||||
storage_cli_manager_.AddClient(target_shard, std::move(cli));
|
||||
}
|
||||
|
||||
void SendAllRequests(ExecutionState<ScanVerticesRequest> &state) {
|
||||
int64_t shard_idx = 0;
|
||||
for (const auto &request : state.requests) {
|
||||
const auto ¤t_shard = state.shard_cache[shard_idx];
|
||||
|
||||
auto &storage_client = GetStorageClientForShard(current_shard);
|
||||
ReadRequests req = request;
|
||||
storage_client.SendAsyncReadRequest(request);
|
||||
|
||||
++shard_idx;
|
||||
}
|
||||
}
|
||||
|
||||
void SendAllRequests(ExecutionState<CreateVerticesRequest> &state,
|
||||
std::vector<memgraph::coordinator::Shard> &shard_cache_ref) {
|
||||
size_t id = 0;
|
||||
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++shard_it) {
|
||||
// This is fine because all new_vertices of each request end up on the same shard
|
||||
const auto labels = state.requests[id].new_vertices[0].label_ids;
|
||||
auto req_deep_copy = state.requests[id];
|
||||
|
||||
for (auto &new_vertex : req_deep_copy.new_vertices) {
|
||||
new_vertex.label_ids.erase(new_vertex.label_ids.begin());
|
||||
}
|
||||
|
||||
auto &storage_client = GetStorageClientForShard(*shard_it);
|
||||
|
||||
WriteRequests req = req_deep_copy;
|
||||
storage_client.SendAsyncWriteRequest(req);
|
||||
++id;
|
||||
}
|
||||
}
|
||||
|
||||
void SendAllRequests(ExecutionState<ExpandOneRequest> &state,
|
||||
std::vector<memgraph::coordinator::Shard> &shard_cache_ref) {
|
||||
size_t id = 0;
|
||||
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++shard_it) {
|
||||
auto &storage_client = GetStorageClientForShard(*shard_it);
|
||||
ReadRequests req = state.requests[id];
|
||||
storage_client.SendAsyncReadRequest(req);
|
||||
++id;
|
||||
}
|
||||
}
|
||||
|
||||
void AwaitOnResponses(ExecutionState<CreateVerticesRequest> &state, std::vector<CreateVerticesResponse> &responses) {
|
||||
auto &shard_cache_ref = state.shard_cache;
|
||||
int64_t request_idx = 0;
|
||||
|
||||
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
|
||||
// This is fine because all new_vertices of each request end up on the same shard
|
||||
const auto labels = state.requests[request_idx].new_vertices[0].label_ids;
|
||||
|
||||
auto &storage_client = GetStorageClientForShard(*shard_it);
|
||||
|
||||
auto poll_result = storage_client.AwaitAsyncWriteRequest();
|
||||
if (!poll_result) {
|
||||
++shard_it;
|
||||
++request_idx;
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (poll_result->HasError()) {
|
||||
throw std::runtime_error("CreateVertices request timed out");
|
||||
}
|
||||
|
||||
WriteResponses response_variant = poll_result->GetValue();
|
||||
auto response = std::get<CreateVerticesResponse>(response_variant);
|
||||
|
||||
if (!response.success) {
|
||||
throw std::runtime_error("CreateVertices request did not succeed");
|
||||
}
|
||||
responses.push_back(response);
|
||||
|
||||
shard_it = shard_cache_ref.erase(shard_it);
|
||||
// Needed to maintain the 1-1 mapping between the ShardCache and the requests.
|
||||
auto it = state.requests.begin() + request_idx;
|
||||
state.requests.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
void AwaitOnResponses(ExecutionState<ExpandOneRequest> &state, std::vector<ExpandOneResponse> &responses) {
|
||||
auto &shard_cache_ref = state.shard_cache;
|
||||
int64_t request_idx = 0;
|
||||
|
||||
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
|
||||
auto &storage_client = GetStorageClientForShard(*shard_it);
|
||||
|
||||
auto poll_result = storage_client.PollAsyncReadRequest();
|
||||
if (!poll_result) {
|
||||
++shard_it;
|
||||
++request_idx;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (poll_result->HasError()) {
|
||||
throw std::runtime_error("ExpandOne request timed out");
|
||||
}
|
||||
|
||||
ReadResponses response_variant = poll_result->GetValue();
|
||||
auto response = std::get<ExpandOneResponse>(response_variant);
|
||||
// -NOTE-
|
||||
// Currently a boolean flag for signaling the overall success of the
|
||||
// ExpandOne request does not exist. But it should, so here we assume
|
||||
// that it is already in place.
|
||||
if (!response.success) {
|
||||
throw std::runtime_error("ExpandOne request did not succeed");
|
||||
}
|
||||
|
||||
responses.push_back(std::move(response));
|
||||
shard_it = shard_cache_ref.erase(shard_it);
|
||||
// Needed to maintain the 1-1 mapping between the ShardCache and the requests.
|
||||
auto it = state.requests.begin() + request_idx;
|
||||
state.requests.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
void AwaitOnPaginatedRequests(ExecutionState<ScanVerticesRequest> &state,
|
||||
std::vector<ScanVerticesResponse> &responses,
|
||||
std::map<Shard, PaginatedResponseState> &paginated_response_tracker) {
|
||||
auto &shard_cache_ref = state.shard_cache;
|
||||
|
||||
// Find the first request that is not holding a paginated response.
|
||||
int64_t request_idx = 0;
|
||||
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
|
||||
if (paginated_response_tracker.at(*shard_it) != PaginatedResponseState::Pending) {
|
||||
++shard_it;
|
||||
++request_idx;
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &storage_client = GetStorageClientForShard(*shard_it);
|
||||
|
||||
auto await_result = storage_client.AwaitAsyncReadRequest();
|
||||
|
||||
if (!await_result) {
|
||||
// Redirection has occured.
|
||||
++shard_it;
|
||||
++request_idx;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (await_result->HasError()) {
|
||||
throw std::runtime_error("ScanAll request timed out");
|
||||
}
|
||||
|
||||
ReadResponses read_response_variant = await_result->GetValue();
|
||||
auto response = std::get<ScanVerticesResponse>(read_response_variant);
|
||||
if (!response.success) {
|
||||
throw std::runtime_error("ScanAll request did not succeed");
|
||||
}
|
||||
|
||||
if (!response.next_start_id) {
|
||||
paginated_response_tracker.erase((*shard_it));
|
||||
shard_cache_ref.erase(shard_it);
|
||||
// Needed to maintain the 1-1 mapping between the ShardCache and the requests.
|
||||
auto it = state.requests.begin() + request_idx;
|
||||
state.requests.erase(it);
|
||||
|
||||
} else {
|
||||
state.requests[request_idx].start_id.second = response.next_start_id->second;
|
||||
paginated_response_tracker[*shard_it] = PaginatedResponseState::PartiallyFinished;
|
||||
}
|
||||
responses.push_back(std::move(response));
|
||||
}
|
||||
}
|
||||
|
||||
void SetUpNameIdMappers() {
|
||||
std::unordered_map<uint64_t, std::string> id_to_name;
|
||||
for (const auto &[name, id] : shards_map_.labels) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
labels_.StoreMapping(std::move(id_to_name));
|
||||
id_to_name.clear();
|
||||
for (const auto &[name, id] : shards_map_.properties) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
properties_.StoreMapping(std::move(id_to_name));
|
||||
id_to_name.clear();
|
||||
for (const auto &[name, id] : shards_map_.edge_types) {
|
||||
id_to_name.emplace(id.AsUint(), name);
|
||||
}
|
||||
edge_types_.StoreMapping(std::move(id_to_name));
|
||||
}
|
||||
|
||||
ShardMap shards_map_;
|
||||
storage::v3::NameIdMapper properties_;
|
||||
storage::v3::NameIdMapper edge_types_;
|
||||
storage::v3::NameIdMapper labels_;
|
||||
CoordinatorClient coord_cli_;
|
||||
RsmStorageClientManager<StorageClient> storage_cli_manager_;
|
||||
memgraph::io::Io<TTransport> io_;
|
||||
memgraph::coordinator::Hlc transaction_id_;
|
||||
// TODO(kostasrim) Add batch prefetching
|
||||
};
|
||||
} // namespace memgraph::msgs
|
||||
@@ -97,15 +97,15 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c
|
||||
if (delta->label == label) {
|
||||
MG_ASSERT(!has_label, "Invalid database state!");
|
||||
has_label = true;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
if (delta->label == label) {
|
||||
MG_ASSERT(has_label, "Invalid database state!");
|
||||
has_label = false;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_IN_EDGE:
|
||||
case Delta::Action::ADD_OUT_EDGE:
|
||||
|
||||
@@ -9,16 +9,19 @@ set(storage_v3_src_files
|
||||
temporal.cpp
|
||||
edge_accessor.cpp
|
||||
indices.cpp
|
||||
key_store.cpp
|
||||
lexicographically_ordered_vertex.cpp
|
||||
property_store.cpp
|
||||
vertex_accessor.cpp
|
||||
schemas.cpp
|
||||
schema_validator.cpp
|
||||
shard.cpp
|
||||
storage.cpp
|
||||
shard_rsm.cpp
|
||||
bindings/typed_value.cpp
|
||||
expr.cpp
|
||||
vertex.cpp
|
||||
request_helper.cpp)
|
||||
request_helper.cpp
|
||||
storage.cpp)
|
||||
|
||||
# ######################
|
||||
find_package(gflags REQUIRED)
|
||||
@@ -30,4 +33,4 @@ target_link_libraries(mg-storage-v3 Threads::Threads mg-utils gflags)
|
||||
target_include_directories(mg-storage-v3 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
|
||||
|
||||
add_dependencies(mg-storage-v3 generate_lcp_storage)
|
||||
target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io mg-functions)
|
||||
target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io)
|
||||
|
||||
@@ -24,8 +24,6 @@
|
||||
#include "storage/v3/conversions.hpp"
|
||||
#include "storage/v3/path.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "functions/awesome_memgraph_functions.hpp"
|
||||
|
||||
cpp<#
|
||||
|
||||
@@ -180,6 +178,14 @@ cpp<#
|
||||
(:serialize (:slk :load-args '((storage "storage::v3::AstStorage *")))))
|
||||
|
||||
#>cpp
|
||||
struct FunctionContext {
|
||||
DbAccessor *db_accessor;
|
||||
utils::MemoryResource *memory;
|
||||
int64_t timestamp;
|
||||
std::unordered_map<std::string, int64_t> *counters;
|
||||
View view;
|
||||
};
|
||||
|
||||
inline bool operator==(const LabelIx &a, const LabelIx &b) {
|
||||
return a.ix == b.ix && a.name == b.name;
|
||||
}
|
||||
@@ -839,24 +845,16 @@ cpp<#
|
||||
:slk-load (slk-load-ast-vector "Expression"))
|
||||
(function-name "std::string" :scope :public)
|
||||
(function "std::function<TypedValue(const TypedValue *, int64_t,
|
||||
const functions::FunctionContext<memgraph::storage::v3::DbAccessor> &)>"
|
||||
const FunctionContext &)>"
|
||||
:scope :public
|
||||
:dont-save t
|
||||
:clone :copy
|
||||
:slk-load (lambda (member)
|
||||
#>cpp
|
||||
self->${member} = functions::NameToFunction<memgraph::storage::v3::TypedValue,
|
||||
functions::FunctionContext<memgraph::storage::v3::DbAccessor>,
|
||||
functions::StorageEngineTag, Conv>(self->function_name_);
|
||||
cpp<#)))
|
||||
(:public
|
||||
#>cpp
|
||||
Function() = default;
|
||||
using Conv = decltype(PropertyToTypedValueFunctor<TypedValue>);
|
||||
class SemanticException : public memgraph::utils::BasicException {
|
||||
using utils::BasicException::BasicException;
|
||||
};
|
||||
|
||||
|
||||
DEFVISITABLE(ExpressionVisitor<TypedValue>);
|
||||
DEFVISITABLE(ExpressionVisitor<void>);
|
||||
@@ -874,13 +872,7 @@ cpp<#
|
||||
Function(const std::string &function_name,
|
||||
const std::vector<Expression *> &arguments)
|
||||
: arguments_(arguments),
|
||||
function_name_(function_name),
|
||||
function_(functions::NameToFunction<memgraph::storage::v3::TypedValue,
|
||||
functions::FunctionContext<memgraph::storage::v3::DbAccessor>,
|
||||
functions::StorageEngineTag, Conv>(function_name_)) {
|
||||
if (!function_) {
|
||||
throw SemanticException("Function '{}' doesn't exist.", function_name);
|
||||
}
|
||||
function_name_(function_name) {
|
||||
}
|
||||
cpp<#)
|
||||
(:private
|
||||
|
||||
@@ -78,8 +78,8 @@ class DbAccessor final {
|
||||
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
|
||||
}
|
||||
|
||||
storage::v3::ShardResult<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
const storage::v3::EdgeTypeId &edge_type) {
|
||||
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
const storage::v3::EdgeTypeId &edge_type) {
|
||||
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
|
||||
auto maybe_edge = accessor_->CreateEdge(from->Id(storage::v3::View::NEW).GetValue(),
|
||||
to->Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
|
||||
@@ -87,8 +87,8 @@ class DbAccessor final {
|
||||
return EdgeAccessor(*maybe_edge);
|
||||
}
|
||||
|
||||
storage::v3::ShardResult<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
|
||||
auto res = accessor_->DeleteEdge(edge->From(), edge->To(), edge->Gid());
|
||||
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
|
||||
auto res = accessor_->DeleteEdge(edge->FromVertex(), edge->ToVertex(), edge->Gid());
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
@@ -101,7 +101,7 @@ class DbAccessor final {
|
||||
return std::make_optional<EdgeAccessor>(*value);
|
||||
}
|
||||
|
||||
storage::v3::ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
|
||||
storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
|
||||
VertexAccessor *vertex_accessor) {
|
||||
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
|
||||
|
||||
@@ -125,7 +125,7 @@ class DbAccessor final {
|
||||
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
|
||||
}
|
||||
|
||||
storage::v3::ShardResult<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
|
||||
storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
|
||||
auto res = accessor_->DeleteVertex(vertex_accessor);
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_store.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
|
||||
@@ -88,6 +87,6 @@ struct EvaluationContext {
|
||||
using ExpressionEvaluator =
|
||||
memgraph::expr::ExpressionEvaluator<TypedValue, EvaluationContext, DbAccessor, storage::v3::View,
|
||||
storage::v3::LabelId, storage::v3::PropertyStore, PropertyToTypedValueConverter,
|
||||
common::ErrorCode>;
|
||||
memgraph::storage::v3::Error>;
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -17,5 +17,5 @@
|
||||
#include "storage/v3/bindings/typed_value.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
using Frame = memgraph::expr::Frame;
|
||||
using Frame = memgraph::expr::Frame<TypedValue>;
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -69,10 +69,6 @@ TTypedValue PropertyToTypedValue(const PropertyValue &value) {
|
||||
LOG_FATAL("Unsupported type");
|
||||
}
|
||||
|
||||
template <typename TypedValueT>
|
||||
inline const auto PropertyToTypedValueFunctor =
|
||||
[](const PropertyValue &value) { return PropertyToTypedValue<TypedValueT>(value); };
|
||||
|
||||
template <typename TTypedValue>
|
||||
TTypedValue PropertyToTypedValue(const PropertyValue &value, utils::MemoryResource *mem) {
|
||||
switch (value.type()) {
|
||||
|
||||
@@ -11,18 +11,17 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include "storage/v3/edge_ref.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_id.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
// Forward declarations because we only store pointers here.
|
||||
struct Vertex;
|
||||
struct Edge;
|
||||
struct Delta;
|
||||
struct CommitInfo;
|
||||
@@ -130,7 +129,7 @@ inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer
|
||||
inline bool operator!=(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer &b) { return !(a == b); }
|
||||
|
||||
struct Delta {
|
||||
enum class Action : uint8_t {
|
||||
enum class Action {
|
||||
// Used for both Vertex and Edge
|
||||
DELETE_OBJECT,
|
||||
RECREATE_OBJECT,
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
using EdgeContainer = std::map<Gid, Edge>;
|
||||
struct Vertex;
|
||||
|
||||
struct Edge {
|
||||
Edge(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
|
||||
#include "storage/v3/mvcc.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
@@ -51,17 +50,17 @@ bool EdgeAccessor::IsVisible(const View view) const {
|
||||
return exists && (for_deleted_ || !deleted);
|
||||
}
|
||||
|
||||
const VertexId &EdgeAccessor::From() const { return from_vertex_; }
|
||||
const VertexId &EdgeAccessor::FromVertex() const { return from_vertex_; }
|
||||
|
||||
const VertexId &EdgeAccessor::To() const { return to_vertex_; }
|
||||
const VertexId &EdgeAccessor::ToVertex() const { return to_vertex_; }
|
||||
|
||||
ShardResult<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
|
||||
Result<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
if (!config_.properties_on_edges) return SHARD_ERROR(ErrorCode::PROPERTIES_DISABLED);
|
||||
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
|
||||
|
||||
if (!PrepareForWrite(transaction_, edge_.ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
if (edge_.ptr->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
|
||||
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
|
||||
|
||||
auto current_value = edge_.ptr->properties.GetProperty(property);
|
||||
// We could skip setting the value if the previous one is the same to the new
|
||||
@@ -76,12 +75,12 @@ ShardResult<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const
|
||||
return std::move(current_value);
|
||||
}
|
||||
|
||||
ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
|
||||
if (!config_.properties_on_edges) return SHARD_ERROR(ErrorCode::PROPERTIES_DISABLED);
|
||||
Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
|
||||
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
|
||||
|
||||
if (!PrepareForWrite(transaction_, edge_.ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
if (edge_.ptr->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
|
||||
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
|
||||
|
||||
auto properties = edge_.ptr->properties.Properties();
|
||||
for (const auto &property : properties) {
|
||||
@@ -93,11 +92,11 @@ ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties()
|
||||
return std::move(properties);
|
||||
}
|
||||
|
||||
ShardResult<PropertyValue> EdgeAccessor::GetProperty(View view, PropertyId property) const {
|
||||
Result<PropertyValue> EdgeAccessor::GetProperty(View view, PropertyId property) const {
|
||||
return GetProperty(property, view);
|
||||
}
|
||||
|
||||
ShardResult<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
|
||||
Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
|
||||
if (!config_.properties_on_edges) return PropertyValue();
|
||||
auto exists = true;
|
||||
auto deleted = edge_.ptr->deleted;
|
||||
@@ -129,12 +128,12 @@ ShardResult<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View v
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
|
||||
if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
return std::move(value);
|
||||
}
|
||||
|
||||
ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
|
||||
Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
|
||||
if (!config_.properties_on_edges) return std::map<PropertyId, PropertyValue>{};
|
||||
auto exists = true;
|
||||
auto deleted = edge_.ptr->deleted;
|
||||
@@ -175,12 +174,9 @@ ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View v
|
||||
break;
|
||||
}
|
||||
});
|
||||
if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
|
||||
if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
|
||||
if (!exists) return Error::NONEXISTENT_OBJECT;
|
||||
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
|
||||
return std::move(properties);
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
size_t EdgeAccessor::CypherId() const { return Gid().AsUint(); }
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
struct Vertex;
|
||||
class VertexAccessor;
|
||||
struct Indices;
|
||||
|
||||
@@ -47,27 +48,27 @@ class EdgeAccessor final {
|
||||
/// @return true if the object is visible from the current transaction
|
||||
bool IsVisible(View view) const;
|
||||
|
||||
const VertexId &From() const;
|
||||
const VertexId &FromVertex() const;
|
||||
|
||||
const VertexId &To() const;
|
||||
const VertexId &ToVertex() const;
|
||||
|
||||
EdgeTypeId EdgeType() const { return edge_type_; }
|
||||
|
||||
/// Set a property value and return the old value.
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
|
||||
Result<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
|
||||
|
||||
/// Remove all properties and return old values for each removed property.
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<std::map<PropertyId, PropertyValue>> ClearProperties();
|
||||
Result<std::map<PropertyId, PropertyValue>> ClearProperties();
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<PropertyValue> GetProperty(PropertyId property, View view) const;
|
||||
Result<PropertyValue> GetProperty(PropertyId property, View view) const;
|
||||
|
||||
ShardResult<PropertyValue> GetProperty(View view, PropertyId property) const;
|
||||
Result<PropertyValue> GetProperty(View view, PropertyId property) const;
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<std::map<PropertyId, PropertyValue>> Properties(View view) const;
|
||||
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
|
||||
|
||||
Gid Gid() const noexcept {
|
||||
if (config_.properties_on_edges) {
|
||||
@@ -83,9 +84,6 @@ class EdgeAccessor final {
|
||||
}
|
||||
bool operator!=(const EdgeAccessor &other) const noexcept { return !(*this == other); }
|
||||
|
||||
// Dummy function
|
||||
size_t CypherId() const;
|
||||
|
||||
private:
|
||||
EdgeRef edge_;
|
||||
EdgeTypeId edge_type_;
|
||||
|
||||
@@ -52,12 +52,13 @@ msgs::Value ConstructValueEdge(const EdgeAccessor &acc, View view) {
|
||||
msgs::EdgeType type = {.id = acc.EdgeType()};
|
||||
msgs::EdgeId gid = {.gid = acc.Gid().AsUint()};
|
||||
|
||||
msgs::Label src_prim_label = {.id = acc.From().primary_label};
|
||||
msgs::Label src_prim_label = {.id = acc.FromVertex().primary_label};
|
||||
memgraph::msgs::VertexId src_vertex =
|
||||
std::make_pair(src_prim_label, conversions::ConvertValueVector(acc.From().primary_key));
|
||||
std::make_pair(src_prim_label, conversions::ConvertValueVector(acc.FromVertex().primary_key));
|
||||
|
||||
msgs::Label dst_prim_label = {.id = acc.To().primary_label};
|
||||
msgs::VertexId dst_vertex = std::make_pair(dst_prim_label, conversions::ConvertValueVector(acc.To().primary_key));
|
||||
msgs::Label dst_prim_label = {.id = acc.ToVertex().primary_label};
|
||||
msgs::VertexId dst_vertex =
|
||||
std::make_pair(dst_prim_label, conversions::ConvertValueVector(acc.ToVertex().primary_key));
|
||||
|
||||
auto properties = acc.Properties(view);
|
||||
|
||||
@@ -164,7 +165,7 @@ std::any ParseExpression(const std::string &expr, memgraph::expr::AstStorage &st
|
||||
return visitor.visit(ast);
|
||||
}
|
||||
|
||||
TypedValue ComputeExpression(DbAccessor &dba, const memgraph::storage::v3::VertexAccessor &v_acc,
|
||||
TypedValue ComputeExpression(DbAccessor &dba, const std::optional<memgraph::storage::v3::VertexAccessor> &v_acc,
|
||||
const std::optional<memgraph::storage::v3::EdgeAccessor> &e_acc,
|
||||
const std::string &expression, std::string_view node_name, std::string_view edge_name) {
|
||||
AstStorage storage;
|
||||
@@ -191,11 +192,10 @@ TypedValue ComputeExpression(DbAccessor &dba, const memgraph::storage::v3::Verte
|
||||
return position_symbol_pair.second.name() == node_name;
|
||||
}) != symbol_table.table().end());
|
||||
|
||||
frame[symbol_table.at(node_identifier)] = v_acc;
|
||||
frame[symbol_table.at(node_identifier)] = *v_acc;
|
||||
}
|
||||
|
||||
if (edge_identifier.symbol_pos_ != -1) {
|
||||
MG_ASSERT(e_acc.has_value());
|
||||
MG_ASSERT(std::find_if(symbol_table.table().begin(), symbol_table.table().end(),
|
||||
[&edge_name](const std::pair<int32_t, Symbol> &position_symbol_pair) {
|
||||
return position_symbol_pair.second.name() == edge_name;
|
||||
|
||||
@@ -9,8 +9,6 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "db_accessor.hpp"
|
||||
@@ -50,7 +48,8 @@ auto Eval(TExpression *expr, EvaluationContext &ctx, AstStorage &storage, Expres
|
||||
|
||||
std::any ParseExpression(const std::string &expr, AstStorage &storage);
|
||||
|
||||
TypedValue ComputeExpression(DbAccessor &dba, const VertexAccessor &v_acc, const std::optional<EdgeAccessor> &e_acc,
|
||||
const std::string &expression, std::string_view node_name, std::string_view edge_name);
|
||||
TypedValue ComputeExpression(DbAccessor &dba, const std::optional<VertexAccessor> &v_acc,
|
||||
const std::optional<EdgeAccessor> &e_acc, const std::string &expression,
|
||||
std::string_view node_name, std::string_view edge_name);
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -11,16 +11,12 @@
|
||||
|
||||
#include "indices.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
|
||||
#include "storage/v3/delta.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/mvcc.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "utils/bound.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
@@ -57,9 +53,9 @@ bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t timestamp)
|
||||
bool deleted{false};
|
||||
const Delta *delta{nullptr};
|
||||
{
|
||||
has_label = utils::Contains(vertex.second.labels, label);
|
||||
deleted = vertex.second.deleted;
|
||||
delta = vertex.second.delta;
|
||||
has_label = utils::Contains(vertex.labels, label);
|
||||
deleted = vertex.deleted;
|
||||
delta = vertex.delta;
|
||||
}
|
||||
if (!deleted && has_label) {
|
||||
return true;
|
||||
@@ -109,10 +105,10 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, PropertyId
|
||||
bool deleted{false};
|
||||
const Delta *delta{nullptr};
|
||||
{
|
||||
has_label = utils::Contains(vertex.second.labels, label);
|
||||
current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
|
||||
deleted = vertex.second.deleted;
|
||||
delta = vertex.second.delta;
|
||||
has_label = utils::Contains(vertex.labels, label);
|
||||
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
|
||||
deleted = vertex.deleted;
|
||||
delta = vertex.delta;
|
||||
}
|
||||
|
||||
if (!deleted && has_label && current_value_equal_to_value) {
|
||||
@@ -167,9 +163,9 @@ bool CurrentVersionHasLabel(const Vertex &vertex, LabelId label, Transaction *tr
|
||||
bool has_label{false};
|
||||
const Delta *delta{nullptr};
|
||||
{
|
||||
deleted = vertex.second.deleted;
|
||||
has_label = utils::Contains(vertex.second.labels, label);
|
||||
delta = vertex.second.delta;
|
||||
deleted = vertex.deleted;
|
||||
has_label = utils::Contains(vertex.labels, label);
|
||||
delta = vertex.delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction, delta, view, [&deleted, &has_label, label](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
@@ -218,10 +214,10 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
|
||||
bool current_value_equal_to_value = value.IsNull();
|
||||
const Delta *delta{nullptr};
|
||||
{
|
||||
deleted = vertex.second.deleted;
|
||||
has_label = utils::Contains(vertex.second.labels, label);
|
||||
current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
|
||||
delta = vertex.second.delta;
|
||||
deleted = vertex.deleted;
|
||||
has_label = utils::Contains(vertex.labels, label);
|
||||
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
|
||||
delta = vertex.delta;
|
||||
}
|
||||
ApplyDeltasForRead(transaction, delta, view,
|
||||
[&deleted, &has_label, ¤t_value_equal_to_value, key, label, &value](const Delta &delta) {
|
||||
@@ -269,10 +265,11 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
|
||||
void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
|
||||
auto it = index_.find(label);
|
||||
if (it == index_.end()) return;
|
||||
it->second.insert(Entry{vertex, tx.start_timestamp.logical_id});
|
||||
auto acc = it->second.access();
|
||||
acc.insert(Entry{vertex, tx.start_timestamp.logical_id});
|
||||
}
|
||||
|
||||
bool LabelIndex::CreateIndex(LabelId label, VertexContainer &vertices) {
|
||||
bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
|
||||
if (!emplaced) {
|
||||
@@ -280,11 +277,13 @@ bool LabelIndex::CreateIndex(LabelId label, VertexContainer &vertices) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
for (auto &vertex : vertices) {
|
||||
if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
|
||||
auto acc = it->second.access();
|
||||
for (auto &lgo_vertex : vertices) {
|
||||
auto &vertex = lgo_vertex.vertex;
|
||||
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
|
||||
continue;
|
||||
}
|
||||
it->second.insert(Entry{&vertex, 0});
|
||||
acc.insert(Entry{&vertex, 0});
|
||||
}
|
||||
} catch (const utils::OutOfMemoryException &) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
|
||||
@@ -305,7 +304,7 @@ std::vector<LabelId> LabelIndex::ListIndices() const {
|
||||
|
||||
void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
|
||||
for (auto &label_storage : index_) {
|
||||
auto &vertices_acc = label_storage.second;
|
||||
auto vertices_acc = label_storage.second.access();
|
||||
for (auto it = vertices_acc.begin(); it != vertices_acc.end();) {
|
||||
auto next_it = it;
|
||||
++next_it;
|
||||
@@ -317,7 +316,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
|
||||
|
||||
if ((next_it != vertices_acc.end() && it->vertex == next_it->vertex) ||
|
||||
!AnyVersionHasLabel(*it->vertex, label_storage.first, clean_up_before_timestamp)) {
|
||||
vertices_acc.erase(*it);
|
||||
vertices_acc.remove(*it);
|
||||
}
|
||||
|
||||
it = next_it;
|
||||
@@ -325,7 +324,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator)
|
||||
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
|
||||
: self_(self),
|
||||
index_iterator_(index_iterator),
|
||||
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
|
||||
@@ -340,7 +339,7 @@ LabelIndex::Iterable::Iterator &LabelIndex::Iterable::Iterator::operator++() {
|
||||
}
|
||||
|
||||
void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
|
||||
for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
|
||||
if (index_iterator_->vertex == current_vertex_) {
|
||||
continue;
|
||||
}
|
||||
@@ -353,9 +352,10 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
}
|
||||
}
|
||||
|
||||
LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction,
|
||||
Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: index_container_(&index_container),
|
||||
LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view,
|
||||
Transaction *transaction, Indices *indices, Config::Items config,
|
||||
const VertexValidator &vertex_validator)
|
||||
: index_accessor_(std::move(index_accessor)),
|
||||
label_(label),
|
||||
view_(view),
|
||||
transaction_(transaction),
|
||||
@@ -363,6 +363,12 @@ LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId lab
|
||||
config_(config),
|
||||
vertex_validator_(&vertex_validator) {}
|
||||
|
||||
void LabelIndex::RunGC() {
|
||||
for (auto &index_entry : index_) {
|
||||
index_entry.second.run_gc();
|
||||
}
|
||||
}
|
||||
|
||||
bool LabelPropertyIndex::Entry::operator<(const Entry &rhs) const {
|
||||
if (value < rhs.value) {
|
||||
return true;
|
||||
@@ -382,13 +388,14 @@ bool LabelPropertyIndex::Entry::operator<(const PropertyValue &rhs) const { retu
|
||||
bool LabelPropertyIndex::Entry::operator==(const PropertyValue &rhs) const { return value == rhs; }
|
||||
|
||||
void LabelPropertyIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
|
||||
for (auto &[label_prop, index] : index_) {
|
||||
for (auto &[label_prop, storage] : index_) {
|
||||
if (label_prop.first != label) {
|
||||
continue;
|
||||
}
|
||||
auto prop_value = vertex->second.properties.GetProperty(label_prop.second);
|
||||
auto prop_value = vertex->properties.GetProperty(label_prop.second);
|
||||
if (!prop_value.IsNull()) {
|
||||
index.emplace(Entry{prop_value, vertex, tx.start_timestamp.logical_id});
|
||||
auto acc = storage.access();
|
||||
acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp.logical_id});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,17 +405,18 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
|
||||
if (value.IsNull()) {
|
||||
return;
|
||||
}
|
||||
for (auto &[label_prop, index] : index_) {
|
||||
for (auto &[label_prop, storage] : index_) {
|
||||
if (label_prop.second != property) {
|
||||
continue;
|
||||
}
|
||||
if (VertexHasLabel(*vertex, label_prop.first)) {
|
||||
index.emplace(Entry{value, vertex, tx.start_timestamp.logical_id});
|
||||
if (utils::Contains(vertex->labels, label_prop.first)) {
|
||||
auto acc = storage.access();
|
||||
acc.insert(Entry{value, vertex, tx.start_timestamp.logical_id});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices) {
|
||||
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
auto [it, emplaced] =
|
||||
index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
|
||||
@@ -417,15 +425,17 @@ bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VertexC
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
for (auto &vertex : vertices) {
|
||||
if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
|
||||
auto acc = it->second.access();
|
||||
for (auto &lgo_vertex : vertices) {
|
||||
auto &vertex = lgo_vertex.vertex;
|
||||
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
|
||||
continue;
|
||||
}
|
||||
auto value = vertex.second.properties.GetProperty(property);
|
||||
auto value = vertex.properties.GetProperty(property);
|
||||
if (value.IsNull()) {
|
||||
continue;
|
||||
}
|
||||
it->second.emplace(Entry{value, &vertex, 0});
|
||||
acc.insert(Entry{std::move(value), &vertex, 0});
|
||||
}
|
||||
} catch (const utils::OutOfMemoryException &) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
|
||||
@@ -446,7 +456,8 @@ std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::ListIndices() co
|
||||
|
||||
void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
|
||||
for (auto &[label_property, index] : index_) {
|
||||
for (auto it = index.begin(); it != index.end();) {
|
||||
auto index_acc = index.access();
|
||||
for (auto it = index_acc.begin(); it != index_acc.end();) {
|
||||
auto next_it = it;
|
||||
++next_it;
|
||||
|
||||
@@ -455,17 +466,17 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((next_it != index.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
|
||||
if ((next_it != index_acc.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
|
||||
!AnyVersionHasLabelProperty(*it->vertex, label_property.first, label_property.second, it->value,
|
||||
clean_up_before_timestamp)) {
|
||||
index.erase(it);
|
||||
index_acc.remove(*it);
|
||||
}
|
||||
it = next_it;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator)
|
||||
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
|
||||
: self_(self),
|
||||
index_iterator_(index_iterator),
|
||||
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
|
||||
@@ -480,7 +491,7 @@ LabelPropertyIndex::Iterable::Iterator &LabelPropertyIndex::Iterable::Iterator::
|
||||
}
|
||||
|
||||
void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
|
||||
for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
|
||||
if (index_iterator_->vertex == current_vertex_) {
|
||||
continue;
|
||||
}
|
||||
@@ -495,11 +506,11 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
}
|
||||
if (self_->upper_bound_) {
|
||||
if (self_->upper_bound_->value() < index_iterator_->value) {
|
||||
index_iterator_ = self_->index_container_->end();
|
||||
index_iterator_ = self_->index_accessor_.end();
|
||||
break;
|
||||
}
|
||||
if (!self_->upper_bound_->IsInclusive() && index_iterator_->value == self_->upper_bound_->value()) {
|
||||
index_iterator_ = self_->index_container_->end();
|
||||
index_iterator_ = self_->index_accessor_.end();
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -526,12 +537,13 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
|
||||
const PropertyValue kSmallestTemporalData =
|
||||
PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
|
||||
|
||||
LabelPropertyIndex::Iterable::Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
|
||||
LabelPropertyIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label,
|
||||
PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
|
||||
Transaction *transaction, Indices *indices, Config::Items config,
|
||||
const VertexValidator &vertex_validator)
|
||||
: index_container_(&index_container),
|
||||
: index_accessor_(std::move(index_accessor)),
|
||||
label_(label),
|
||||
property_(property),
|
||||
lower_bound_(lower_bound),
|
||||
@@ -639,55 +651,49 @@ LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::begin() {
|
||||
// If the bounds are set and don't have comparable types we don't yield any
|
||||
// items from the index.
|
||||
if (!bounds_valid_) {
|
||||
return {this, index_container_->end()};
|
||||
return {this, index_accessor_.end()};
|
||||
}
|
||||
auto index_iterator = index_accessor_.begin();
|
||||
if (lower_bound_) {
|
||||
return {this, std::ranges::lower_bound(*index_container_, lower_bound_->value(), std::less{}, &Entry::value)};
|
||||
index_iterator = index_accessor_.find_equal_or_greater(lower_bound_->value());
|
||||
}
|
||||
return {this, index_container_->begin()};
|
||||
return {this, index_iterator};
|
||||
}
|
||||
|
||||
LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_container_->end()}; }
|
||||
LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_accessor_.end()}; }
|
||||
|
||||
int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
|
||||
int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
|
||||
const PropertyValue &value) const {
|
||||
auto it = index_.find({label, property});
|
||||
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
|
||||
MG_ASSERT(!value.IsNull(), "Null is not supported!");
|
||||
auto acc = it->second.access();
|
||||
if (!value.IsNull()) {
|
||||
return static_cast<int64_t>(
|
||||
acc.estimate_count(value, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
|
||||
}
|
||||
// The value `Null` won't ever appear in the index because it indicates that
|
||||
// the property shouldn't exist. Instead, this value is used as an indicator
|
||||
// to estimate the average number of equal elements in the list (for any
|
||||
// given value).
|
||||
return static_cast<int64_t>(acc.estimate_average_number_of_equals(
|
||||
[](const auto &first, const auto &second) { return first.value == second.value; },
|
||||
static_cast<int>(utils::SkipListLayerForAverageEqualsEstimation(acc.size()))));
|
||||
}
|
||||
|
||||
// TODO(jbajic) This can be improved by exiting early
|
||||
auto start_it = std::ranges::lower_bound(it->second, value, std::less{}, &Entry::value);
|
||||
int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper) const {
|
||||
auto it = index_.find({label, property});
|
||||
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
|
||||
auto acc = it->second.access();
|
||||
return static_cast<int64_t>(
|
||||
std::ranges::count_if(start_it, it->second.end(), [&value](const auto &elem) { return elem.value == value; }));
|
||||
acc.estimate_range_count(lower, upper, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
|
||||
}
|
||||
|
||||
int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper) const {
|
||||
auto it = index_.find({label, property});
|
||||
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
|
||||
const auto lower_it = std::invoke(
|
||||
[&index = it->second](const auto value, const auto def) {
|
||||
if (value) {
|
||||
if (value->IsInclusive()) {
|
||||
return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
|
||||
}
|
||||
return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
|
||||
}
|
||||
return def;
|
||||
},
|
||||
lower, it->second.begin());
|
||||
const auto upper_it = std::invoke(
|
||||
[&index = it->second](const auto value, const auto def) {
|
||||
if (value) {
|
||||
if (value->IsInclusive()) {
|
||||
return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
|
||||
}
|
||||
return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
|
||||
}
|
||||
return def;
|
||||
},
|
||||
upper, it->second.end());
|
||||
return static_cast<int64_t>(std::distance(lower_it, upper_it));
|
||||
void LabelPropertyIndex::RunGC() {
|
||||
for (auto &index_entry : index_) {
|
||||
index_entry.second.run_gc();
|
||||
}
|
||||
}
|
||||
|
||||
void RemoveObsoleteEntries(Indices *indices, const uint64_t clean_up_before_timestamp) {
|
||||
|
||||
@@ -13,7 +13,6 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
|
||||
@@ -21,6 +20,7 @@
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/vertices_skip_list.hpp"
|
||||
#include "utils/bound.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/skip_list.hpp"
|
||||
@@ -30,6 +30,7 @@ namespace memgraph::storage::v3 {
|
||||
struct Indices;
|
||||
|
||||
class LabelIndex {
|
||||
private:
|
||||
struct Entry {
|
||||
Vertex *vertex;
|
||||
uint64_t timestamp;
|
||||
@@ -40,9 +41,17 @@ class LabelIndex {
|
||||
bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
|
||||
};
|
||||
|
||||
public:
|
||||
using LabelIndexContainer = std::set<Entry>;
|
||||
struct LabelStorage {
|
||||
LabelId label;
|
||||
utils::SkipList<Entry> vertices;
|
||||
|
||||
bool operator<(const LabelStorage &rhs) const { return label < rhs.label; }
|
||||
bool operator<(LabelId rhs) const { return label < rhs; }
|
||||
bool operator==(const LabelStorage &rhs) const { return label == rhs.label; }
|
||||
bool operator==(LabelId rhs) const { return label == rhs; }
|
||||
};
|
||||
|
||||
public:
|
||||
LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
|
||||
|
||||
@@ -50,7 +59,7 @@ class LabelIndex {
|
||||
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
bool CreateIndex(LabelId label, VertexContainer &vertices);
|
||||
bool CreateIndex(LabelId label, VerticesSkipList::Accessor vertices);
|
||||
|
||||
/// Returns false if there was no index to drop
|
||||
bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
|
||||
@@ -63,12 +72,12 @@ class LabelIndex {
|
||||
|
||||
class Iterable {
|
||||
public:
|
||||
Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
|
||||
Config::Items config, const VertexValidator &vertex_validator);
|
||||
Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view, Transaction *transaction,
|
||||
Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
|
||||
|
||||
class Iterator {
|
||||
public:
|
||||
Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator);
|
||||
Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
|
||||
|
||||
VertexAccessor operator*() const { return current_vertex_accessor_; }
|
||||
|
||||
@@ -81,16 +90,16 @@ class LabelIndex {
|
||||
void AdvanceUntilValid();
|
||||
|
||||
Iterable *self_;
|
||||
LabelIndexContainer::iterator index_iterator_;
|
||||
utils::SkipList<Entry>::Iterator index_iterator_;
|
||||
VertexAccessor current_vertex_accessor_;
|
||||
Vertex *current_vertex_;
|
||||
};
|
||||
|
||||
Iterator begin() { return {this, index_container_->begin()}; }
|
||||
Iterator end() { return {this, index_container_->end()}; }
|
||||
Iterator begin() { return {this, index_accessor_.begin()}; }
|
||||
Iterator end() { return {this, index_accessor_.end()}; }
|
||||
|
||||
private:
|
||||
LabelIndexContainer *index_container_;
|
||||
utils::SkipList<Entry>::Accessor index_accessor_;
|
||||
LabelId label_;
|
||||
View view_;
|
||||
Transaction *transaction_;
|
||||
@@ -103,7 +112,7 @@ class LabelIndex {
|
||||
Iterable Vertices(LabelId label, View view, Transaction *transaction) {
|
||||
auto it = index_.find(label);
|
||||
MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint());
|
||||
return {it->second, label, view, transaction, indices_, config_, *vertex_validator_};
|
||||
return {it->second.access(), label, view, transaction, indices_, config_, *vertex_validator_};
|
||||
}
|
||||
|
||||
int64_t ApproximateVertexCount(LabelId label) {
|
||||
@@ -114,14 +123,17 @@ class LabelIndex {
|
||||
|
||||
void Clear() { index_.clear(); }
|
||||
|
||||
void RunGC();
|
||||
|
||||
private:
|
||||
std::map<LabelId, LabelIndexContainer> index_;
|
||||
std::map<LabelId, utils::SkipList<Entry>> index_;
|
||||
Indices *indices_;
|
||||
Config::Items config_;
|
||||
const VertexValidator *vertex_validator_;
|
||||
};
|
||||
|
||||
class LabelPropertyIndex {
|
||||
private:
|
||||
struct Entry {
|
||||
PropertyValue value;
|
||||
Vertex *vertex;
|
||||
@@ -135,8 +147,6 @@ class LabelPropertyIndex {
|
||||
};
|
||||
|
||||
public:
|
||||
using LabelPropertyIndexContainer = std::set<Entry>;
|
||||
|
||||
LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
|
||||
|
||||
@@ -147,7 +157,7 @@ class LabelPropertyIndex {
|
||||
void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
bool CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices);
|
||||
bool CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices);
|
||||
|
||||
bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }
|
||||
|
||||
@@ -159,14 +169,14 @@ class LabelPropertyIndex {
|
||||
|
||||
class Iterable {
|
||||
public:
|
||||
Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
|
||||
Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
|
||||
Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
|
||||
|
||||
class Iterator {
|
||||
public:
|
||||
Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator);
|
||||
Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
|
||||
|
||||
VertexAccessor operator*() const { return current_vertex_accessor_; }
|
||||
|
||||
@@ -179,7 +189,7 @@ class LabelPropertyIndex {
|
||||
void AdvanceUntilValid();
|
||||
|
||||
Iterable *self_;
|
||||
LabelPropertyIndexContainer::iterator index_iterator_;
|
||||
utils::SkipList<Entry>::Iterator index_iterator_;
|
||||
VertexAccessor current_vertex_accessor_;
|
||||
Vertex *current_vertex_;
|
||||
};
|
||||
@@ -188,7 +198,7 @@ class LabelPropertyIndex {
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
LabelPropertyIndexContainer *index_container_;
|
||||
utils::SkipList<Entry>::Accessor index_accessor_;
|
||||
LabelId label_;
|
||||
PropertyId property_;
|
||||
std::optional<utils::Bound<PropertyValue>> lower_bound_;
|
||||
@@ -207,11 +217,11 @@ class LabelPropertyIndex {
|
||||
auto it = index_.find({label, property});
|
||||
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
|
||||
property.AsUint());
|
||||
return {it->second, label, property, lower_bound, upper_bound,
|
||||
view, transaction, indices_, config_, *vertex_validator_};
|
||||
return {it->second.access(), label, property, lower_bound, upper_bound, view,
|
||||
transaction, indices_, config_, *vertex_validator_};
|
||||
}
|
||||
|
||||
int64_t VertexCount(LabelId label, PropertyId property) const {
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
|
||||
auto it = index_.find({label, property});
|
||||
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
|
||||
property.AsUint());
|
||||
@@ -222,15 +232,18 @@ class LabelPropertyIndex {
|
||||
/// an estimated count of nodes which have their property's value set to
|
||||
/// `value`. If the `value` specified is `Null`, then an average number of
|
||||
/// equal elements is returned.
|
||||
int64_t VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
|
||||
|
||||
int64_t VertexCount(LabelId label, PropertyId property, const std::optional<utils::Bound<PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper) const;
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper) const;
|
||||
|
||||
void Clear() { index_.clear(); }
|
||||
|
||||
void RunGC();
|
||||
|
||||
private:
|
||||
std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndexContainer> index_;
|
||||
std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>> index_;
|
||||
Indices *indices_;
|
||||
Config::Items config_;
|
||||
const VertexValidator *vertex_validator_;
|
||||
|
||||
43
src/storage/v3/key_store.cpp
Normal file
43
src/storage/v3/key_store.cpp
Normal file
@@ -0,0 +1,43 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
#include <ranges>
|
||||
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
KeyStore::KeyStore(const PrimaryKey &key_values) {
|
||||
for (auto i = 0; i < key_values.size(); ++i) {
|
||||
MG_ASSERT(!key_values[i].IsNull());
|
||||
store_.SetProperty(PropertyId::FromInt(i), key_values[i]);
|
||||
}
|
||||
}
|
||||
|
||||
PropertyValue KeyStore::GetKey(const size_t index) const { return store_.GetProperty(PropertyId::FromUint(index)); }
|
||||
|
||||
PropertyValue KeyStore::GetKey(const PropertyId property_id) const { return store_.GetProperty(property_id); }
|
||||
|
||||
PrimaryKey KeyStore::Keys() const {
|
||||
auto keys_map = store_.Properties();
|
||||
PrimaryKey keys;
|
||||
keys.reserve(keys_map.size());
|
||||
std::ranges::transform(
|
||||
keys_map, std::back_inserter(keys),
|
||||
[](std::pair<const PropertyId, PropertyValue> &id_and_value) { return std::move(id_and_value.second); });
|
||||
return keys;
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
@@ -11,6 +11,12 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <compare>
|
||||
#include <functional>
|
||||
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_store.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
@@ -18,4 +24,39 @@ namespace memgraph::storage::v3 {
|
||||
// Primary key is a collection of primary properties.
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
|
||||
class KeyStore {
|
||||
public:
|
||||
explicit KeyStore(const PrimaryKey &key_values);
|
||||
|
||||
KeyStore(const KeyStore &) = delete;
|
||||
KeyStore(KeyStore &&other) noexcept = default;
|
||||
KeyStore &operator=(const KeyStore &) = delete;
|
||||
KeyStore &operator=(KeyStore &&other) noexcept = default;
|
||||
|
||||
~KeyStore() = default;
|
||||
|
||||
PropertyValue GetKey(size_t index) const;
|
||||
|
||||
PropertyValue GetKey(PropertyId property) const;
|
||||
|
||||
PrimaryKey Keys() const;
|
||||
|
||||
friend bool operator<(const KeyStore &lhs, const KeyStore &rhs) {
|
||||
return std::ranges::lexicographical_compare(lhs.Keys(), rhs.Keys(), std::less<PropertyValue>{});
|
||||
}
|
||||
|
||||
friend bool operator==(const KeyStore &lhs, const KeyStore &rhs) {
|
||||
return std::ranges::equal(lhs.Keys(), rhs.Keys());
|
||||
}
|
||||
|
||||
friend bool operator<(const KeyStore &lhs, const PrimaryKey &rhs) {
|
||||
return std::ranges::lexicographical_compare(lhs.Keys(), rhs, std::less<PropertyValue>{});
|
||||
}
|
||||
|
||||
friend bool operator==(const KeyStore &lhs, const PrimaryKey &rhs) { return std::ranges::equal(lhs.Keys(), rhs); }
|
||||
|
||||
private:
|
||||
PropertyStore store_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -9,11 +9,6 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
#include "storage/v3/lexicographically_ordered_vertex.hpp"
|
||||
|
||||
namespace memgraph::utils {
|
||||
|
||||
template <typename>
|
||||
constexpr auto kAlwaysFalse{false};
|
||||
|
||||
} // namespace memgraph::utils
|
||||
namespace memgraph::storage::v3 {} // namespace memgraph::storage::v3
|
||||
42
src/storage/v3/lexicographically_ordered_vertex.hpp
Normal file
42
src/storage/v3/lexicographically_ordered_vertex.hpp
Normal file
@@ -0,0 +1,42 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <concepts>
|
||||
#include <type_traits>
|
||||
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
struct LexicographicallyOrderedVertex {
|
||||
Vertex vertex;
|
||||
|
||||
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
|
||||
return lhs.vertex.keys == rhs.vertex.keys;
|
||||
}
|
||||
|
||||
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
|
||||
return lhs.vertex.keys < rhs.vertex.keys;
|
||||
}
|
||||
|
||||
// TODO(antaljanosbenjamin): maybe it worth to overload this for std::array to avoid heap construction of the vector
|
||||
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
|
||||
return lhs.vertex.keys == rhs;
|
||||
}
|
||||
|
||||
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
|
||||
return lhs.vertex.keys < rhs;
|
||||
}
|
||||
};
|
||||
} // namespace memgraph::storage::v3
|
||||
@@ -11,21 +11,12 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "storage/v3/edge.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
inline VertexData *GetDeltaHolder(Vertex *vertex) { return &vertex->second; }
|
||||
|
||||
inline Edge *GetDeltaHolder(Edge *edge) { return edge; }
|
||||
|
||||
/// This function iterates through the undo buffers from an object (starting
|
||||
/// from the supplied delta) and determines what deltas should be applied to get
|
||||
/// the currently visible version of the object. When the function finds a delta
|
||||
@@ -86,12 +77,10 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie
|
||||
/// transaction) and returns a `bool` value indicating whether the caller can
|
||||
/// proceed with a write operation.
|
||||
template <typename TObj>
|
||||
requires utils::SameAsAnyOf<TObj, Edge, Vertex>
|
||||
inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
|
||||
auto *delta_holder = GetDeltaHolder(object);
|
||||
if (delta_holder->delta == nullptr) return true;
|
||||
if (object->delta == nullptr) return true;
|
||||
|
||||
const auto &delta_commit_info = *delta_holder->delta->commit_info;
|
||||
const auto &delta_commit_info = *object->delta->commit_info;
|
||||
if (delta_commit_info.start_or_commit_timestamp == transaction->commit_info->start_or_commit_timestamp ||
|
||||
(delta_commit_info.is_locally_committed &&
|
||||
delta_commit_info.start_or_commit_timestamp < transaction->start_timestamp)) {
|
||||
@@ -116,11 +105,9 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
|
||||
/// the delta into the object's delta list.
|
||||
/// @throw std::bad_alloc
|
||||
template <typename TObj, class... Args>
|
||||
requires utils::SameAsAnyOf<TObj, Edge, Vertex>
|
||||
inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
|
||||
auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
|
||||
transaction->command_id);
|
||||
auto *delta_holder = GetDeltaHolder(object);
|
||||
|
||||
// The operations are written in such order so that both `next` and `prev`
|
||||
// chains are valid at all times. The chains must be valid at all times
|
||||
@@ -131,21 +118,21 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
|
||||
// TODO(antaljanosbenjamin): clang-tidy detects (in my opinion a false positive) issue in
|
||||
// `Shard::Accessor::CreateEdge`.
|
||||
// NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
|
||||
delta->next = delta_holder->delta;
|
||||
delta->next = object->delta;
|
||||
// 2. We need to set the previous delta of the new delta to the object.
|
||||
delta->prev.Set(object);
|
||||
// 3. We need to set the previous delta of the existing delta to the new
|
||||
// delta. After this point the garbage collector will be able to see the new
|
||||
// delta but won't modify it until we are done with all of our modifications.
|
||||
if (delta_holder->delta) {
|
||||
delta_holder->delta->prev.Set(delta);
|
||||
if (object->delta) {
|
||||
object->delta->prev.Set(delta);
|
||||
}
|
||||
// 4. Finally, we need to set the object's delta to the new delta. The garbage
|
||||
// collector and other transactions will acquire the object lock to read the
|
||||
// delta from the object. Because the lock is held during the whole time this
|
||||
// modification is being done, everybody else will wait until we are fully
|
||||
// done with our modification before they read the object's delta value.
|
||||
delta_holder->delta = delta;
|
||||
object->delta = delta;
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,463 +11,56 @@
|
||||
|
||||
#include "storage/v3/request_helper.hpp"
|
||||
|
||||
#include <iterator>
|
||||
#include <vector>
|
||||
|
||||
#include "pretty_print_ast_to_original_expression.hpp"
|
||||
#include "storage/v3/bindings/db_accessor.hpp"
|
||||
#include "storage/v3/bindings/pretty_print_ast_to_original_expression.hpp"
|
||||
#include "storage/v3/expr.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
using msgs::Label;
|
||||
using msgs::PropertyId;
|
||||
|
||||
using conversions::ConvertPropertyVector;
|
||||
using conversions::FromPropertyValueToValue;
|
||||
using conversions::ToMsgsVertexId;
|
||||
|
||||
namespace {
|
||||
|
||||
using AllEdgePropertyDataStucture = std::map<PropertyId, msgs::Value>;
|
||||
using SpecificEdgePropertyDataStucture = std::vector<msgs::Value>;
|
||||
|
||||
using AllEdgeProperties = std::tuple<msgs::VertexId, msgs::Gid, AllEdgePropertyDataStucture>;
|
||||
using SpecificEdgeProperties = std::tuple<msgs::VertexId, msgs::Gid, SpecificEdgePropertyDataStucture>;
|
||||
|
||||
using SpecificEdgePropertiesVector = std::vector<SpecificEdgeProperties>;
|
||||
using AllEdgePropertiesVector = std::vector<AllEdgeProperties>;
|
||||
|
||||
struct VertexIdCmpr {
|
||||
bool operator()(const storage::v3::VertexId *lhs, const storage::v3::VertexId *rhs) const { return *lhs < *rhs; }
|
||||
};
|
||||
|
||||
std::optional<std::map<PropertyId, Value>> PrimaryKeysFromAccessor(const VertexAccessor &acc, View view,
|
||||
const Schemas::Schema &schema) {
|
||||
std::map<PropertyId, Value> ret;
|
||||
auto props = acc.Properties(view);
|
||||
auto maybe_pk = acc.PrimaryKey(view);
|
||||
if (maybe_pk.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get vertex primary key.");
|
||||
return std::nullopt;
|
||||
}
|
||||
auto &pk = maybe_pk.GetValue();
|
||||
MG_ASSERT(schema.second.size() == pk.size(), "PrimaryKey size does not match schema!");
|
||||
for (size_t i{0}; i < schema.second.size(); ++i) {
|
||||
ret.emplace(schema.second[i].property_id, FromPropertyValueToValue(std::move(pk[i])));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ShardResult<std::vector<msgs::Label>> FillUpSourceVertexSecondaryLabels(const std::optional<VertexAccessor> &v_acc,
|
||||
const msgs::ExpandOneRequest &req) {
|
||||
auto secondary_labels = v_acc->Labels(View::NEW);
|
||||
if (secondary_labels.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get the secondary labels of a vertex. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return secondary_labels.GetError();
|
||||
}
|
||||
|
||||
auto &sec_labels = secondary_labels.GetValue();
|
||||
std::vector<msgs::Label> msgs_secondary_labels;
|
||||
msgs_secondary_labels.reserve(sec_labels.size());
|
||||
|
||||
std::transform(sec_labels.begin(), sec_labels.end(), std::back_inserter(msgs_secondary_labels),
|
||||
[](auto label_id) { return msgs::Label{.id = label_id}; });
|
||||
|
||||
return msgs_secondary_labels;
|
||||
}
|
||||
|
||||
ShardResult<std::map<PropertyId, Value>> FillUpSourceVertexProperties(const std::optional<VertexAccessor> &v_acc,
|
||||
const msgs::ExpandOneRequest &req,
|
||||
storage::v3::View view,
|
||||
const Schemas::Schema &schema) {
|
||||
std::map<PropertyId, Value> src_vertex_properties;
|
||||
|
||||
if (!req.src_vertex_properties) {
|
||||
auto props = v_acc->Properties(View::NEW);
|
||||
if (props.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to access vertex properties. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return props.GetError();
|
||||
}
|
||||
|
||||
for (auto &[key, val] : props.GetValue()) {
|
||||
src_vertex_properties.insert(std::make_pair(key, FromPropertyValueToValue(std::move(val))));
|
||||
}
|
||||
auto pks = PrimaryKeysFromAccessor(*v_acc, view, schema);
|
||||
if (pks) {
|
||||
src_vertex_properties.merge(*pks);
|
||||
}
|
||||
|
||||
} else if (req.src_vertex_properties.value().empty()) {
|
||||
// NOOP
|
||||
} else {
|
||||
for (const auto &prop : req.src_vertex_properties.value()) {
|
||||
auto prop_val = v_acc->GetProperty(prop, View::OLD);
|
||||
if (prop_val.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to access vertex properties. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return prop_val.GetError();
|
||||
std::vector<Element> OrderByElements(Shard::Accessor &acc, DbAccessor &dba, VerticesIterable &vertices_iterable,
|
||||
std::vector<msgs::OrderBy> &order_bys) {
|
||||
std::vector<Element> ordered;
|
||||
ordered.reserve(acc.ApproximateVertexCount());
|
||||
std::vector<Ordering> ordering;
|
||||
ordering.reserve(order_bys.size());
|
||||
for (const auto &order : order_bys) {
|
||||
switch (order.direction) {
|
||||
case memgraph::msgs::OrderingDirection::ASCENDING: {
|
||||
ordering.push_back(Ordering::ASC);
|
||||
break;
|
||||
}
|
||||
case memgraph::msgs::OrderingDirection::DESCENDING: {
|
||||
ordering.push_back(Ordering::DESC);
|
||||
break;
|
||||
}
|
||||
src_vertex_properties.insert(std::make_pair(prop, FromPropertyValueToValue(std::move(prop_val.GetValue()))));
|
||||
}
|
||||
}
|
||||
auto compare_typed_values = TypedValueVectorCompare(ordering);
|
||||
for (auto it = vertices_iterable.begin(); it != vertices_iterable.end(); ++it) {
|
||||
std::vector<TypedValue> properties_order_by;
|
||||
properties_order_by.reserve(order_bys.size());
|
||||
|
||||
return src_vertex_properties;
|
||||
}
|
||||
|
||||
ShardResult<std::array<std::vector<EdgeAccessor>, 2>> FillUpConnectingEdges(
|
||||
const std::optional<VertexAccessor> &v_acc, const msgs::ExpandOneRequest &req,
|
||||
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness) {
|
||||
std::vector<EdgeTypeId> edge_types{};
|
||||
edge_types.reserve(req.edge_types.size());
|
||||
std::transform(req.edge_types.begin(), req.edge_types.end(), std::back_inserter(edge_types),
|
||||
[](const msgs::EdgeType &edge_type) { return edge_type.id; });
|
||||
|
||||
std::vector<EdgeAccessor> in_edges;
|
||||
std::vector<EdgeAccessor> out_edges;
|
||||
|
||||
switch (req.direction) {
|
||||
case msgs::EdgeDirection::OUT: {
|
||||
auto out_edges_result = v_acc->OutEdges(View::NEW, edge_types);
|
||||
if (out_edges_result.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get out-going EdgeAccessors. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return out_edges_result.GetError();
|
||||
}
|
||||
out_edges =
|
||||
maybe_filter_based_on_edge_uniqueness(std::move(out_edges_result.GetValue()), msgs::EdgeDirection::OUT);
|
||||
break;
|
||||
}
|
||||
case msgs::EdgeDirection::IN: {
|
||||
auto in_edges_result = v_acc->InEdges(View::NEW, edge_types);
|
||||
if (in_edges_result.HasError()) {
|
||||
spdlog::debug(
|
||||
"Encountered an error while trying to get in-going EdgeAccessors. Transaction id: {}"[req.transaction_id
|
||||
.logical_id]);
|
||||
return in_edges_result.GetError();
|
||||
}
|
||||
in_edges = maybe_filter_based_on_edge_uniqueness(std::move(in_edges_result.GetValue()), msgs::EdgeDirection::IN);
|
||||
break;
|
||||
}
|
||||
case msgs::EdgeDirection::BOTH: {
|
||||
auto in_edges_result = v_acc->InEdges(View::NEW, edge_types);
|
||||
if (in_edges_result.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get in-going EdgeAccessors. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return in_edges_result.GetError();
|
||||
}
|
||||
in_edges = maybe_filter_based_on_edge_uniqueness(std::move(in_edges_result.GetValue()), msgs::EdgeDirection::IN);
|
||||
auto out_edges_result = v_acc->OutEdges(View::NEW, edge_types);
|
||||
if (out_edges_result.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get out-going EdgeAccessors. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return out_edges_result.GetError();
|
||||
}
|
||||
out_edges =
|
||||
maybe_filter_based_on_edge_uniqueness(std::move(out_edges_result.GetValue()), msgs::EdgeDirection::OUT);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return std::array<std::vector<EdgeAccessor>, 2>{std::move(in_edges), std::move(out_edges)};
|
||||
}
|
||||
|
||||
template <bool are_in_edges>
|
||||
ShardResult<void> FillEdges(const std::vector<EdgeAccessor> &edges, msgs::ExpandOneResultRow &row,
|
||||
const EdgeFiller &edge_filler) {
|
||||
for (const auto &edge : edges) {
|
||||
if (const auto res = edge_filler(edge, are_in_edges, row); res.HasError()) {
|
||||
return res.GetError();
|
||||
for (const auto &order_by : order_bys) {
|
||||
const auto val =
|
||||
ComputeExpression(dba, *it, std::nullopt, order_by.expression.expression, expr::identifier_node_symbol, "");
|
||||
properties_order_by.push_back(val);
|
||||
}
|
||||
ordered.push_back({std::move(properties_order_by), *it});
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
}; // namespace
|
||||
|
||||
ShardResult<std::map<PropertyId, Value>> CollectSpecificPropertiesFromAccessor(const VertexAccessor &acc,
|
||||
const std::vector<PropertyId> &props,
|
||||
View view) {
|
||||
std::map<PropertyId, Value> ret;
|
||||
|
||||
for (const auto &prop : props) {
|
||||
auto result = acc.GetProperty(prop, view);
|
||||
if (result.HasError()) {
|
||||
spdlog::debug("Encountered an Error while trying to get a vertex property.");
|
||||
return result.GetError();
|
||||
}
|
||||
auto &value = result.GetValue();
|
||||
ret.emplace(std::make_pair(prop, FromPropertyValueToValue(std::move(value))));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::vector<TypedValue> EvaluateVertexExpressions(DbAccessor &dba, const VertexAccessor &v_acc,
|
||||
const std::vector<std::string> &expressions,
|
||||
std::string_view node_name) {
|
||||
std::vector<TypedValue> evaluated_expressions;
|
||||
evaluated_expressions.reserve(expressions.size());
|
||||
|
||||
std::transform(expressions.begin(), expressions.end(), std::back_inserter(evaluated_expressions),
|
||||
[&dba, &v_acc, &node_name](const auto &expression) {
|
||||
return ComputeExpression(dba, v_acc, std::nullopt, expression, node_name, "");
|
||||
});
|
||||
|
||||
return evaluated_expressions;
|
||||
}
|
||||
|
||||
std::vector<TypedValue> EvaluateEdgeExpressions(DbAccessor &dba, const VertexAccessor &v_acc, const EdgeAccessor &e_acc,
|
||||
const std::vector<std::string> &expressions) {
|
||||
std::vector<TypedValue> evaluated_expressions;
|
||||
evaluated_expressions.reserve(expressions.size());
|
||||
|
||||
std::transform(expressions.begin(), expressions.end(), std::back_inserter(evaluated_expressions),
|
||||
[&dba, &v_acc, &e_acc](const auto &expression) {
|
||||
return ComputeExpression(dba, v_acc, e_acc, expression, expr::identifier_node_symbol,
|
||||
expr::identifier_edge_symbol);
|
||||
});
|
||||
|
||||
return evaluated_expressions;
|
||||
}
|
||||
|
||||
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view,
|
||||
const Schemas::Schema &schema) {
|
||||
auto ret = impl::CollectAllPropertiesImpl<VertexAccessor>(acc, view);
|
||||
if (ret.HasError()) {
|
||||
return ret.GetError();
|
||||
}
|
||||
|
||||
auto pks = PrimaryKeysFromAccessor(acc, view, schema);
|
||||
if (pks) {
|
||||
ret.GetValue().merge(std::move(*pks));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view) {
|
||||
return impl::CollectAllPropertiesImpl(acc, view);
|
||||
}
|
||||
|
||||
EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbor_rows) {
|
||||
// Functions to select connecting edges based on uniquness
|
||||
EdgeUniquenessFunction maybe_filter_based_on_edge_uniquness;
|
||||
|
||||
if (only_unique_neighbor_rows) {
|
||||
maybe_filter_based_on_edge_uniquness = [](EdgeAccessors &&edges,
|
||||
msgs::EdgeDirection edge_direction) -> EdgeAccessors {
|
||||
std::function<bool(std::set<const storage::v3::VertexId *, VertexIdCmpr> &, const storage::v3::EdgeAccessor &)>
|
||||
is_edge_unique;
|
||||
switch (edge_direction) {
|
||||
case msgs::EdgeDirection::OUT: {
|
||||
is_edge_unique = [](std::set<const storage::v3::VertexId *, VertexIdCmpr> &other_vertex_set,
|
||||
const storage::v3::EdgeAccessor &edge_acc) {
|
||||
auto [it, insertion_happened] = other_vertex_set.insert(&edge_acc.To());
|
||||
return insertion_happened;
|
||||
};
|
||||
break;
|
||||
}
|
||||
case msgs::EdgeDirection::IN: {
|
||||
is_edge_unique = [](std::set<const storage::v3::VertexId *, VertexIdCmpr> &other_vertex_set,
|
||||
const storage::v3::EdgeAccessor &edge_acc) {
|
||||
auto [it, insertion_happened] = other_vertex_set.insert(&edge_acc.From());
|
||||
return insertion_happened;
|
||||
};
|
||||
break;
|
||||
}
|
||||
case msgs::EdgeDirection::BOTH:
|
||||
MG_ASSERT(false, "This is should never happen, msgs::EdgeDirection::BOTH should not be passed here.");
|
||||
}
|
||||
|
||||
EdgeAccessors ret;
|
||||
std::set<const storage::v3::VertexId *, VertexIdCmpr> other_vertex_set;
|
||||
|
||||
for (const auto &edge : edges) {
|
||||
if (is_edge_unique(other_vertex_set, edge)) {
|
||||
ret.emplace_back(edge);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
};
|
||||
} else {
|
||||
maybe_filter_based_on_edge_uniquness =
|
||||
[](EdgeAccessors &&edges, msgs::EdgeDirection /*edge_direction*/) -> EdgeAccessors { return std::move(edges); };
|
||||
}
|
||||
|
||||
return maybe_filter_based_on_edge_uniquness;
|
||||
}
|
||||
|
||||
EdgeFiller InitializeEdgeFillerFunction(const msgs::ExpandOneRequest &req) {
|
||||
EdgeFiller edge_filler;
|
||||
|
||||
if (!req.edge_properties) {
|
||||
edge_filler = [transaction_id = req.transaction_id.logical_id](
|
||||
const EdgeAccessor &edge, const bool is_in_edge,
|
||||
msgs::ExpandOneResultRow &result_row) -> ShardResult<void> {
|
||||
auto properties_results = edge.Properties(View::NEW);
|
||||
if (properties_results.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get edge properties. Transaction id: {}", transaction_id);
|
||||
return properties_results.GetError();
|
||||
}
|
||||
|
||||
std::map<PropertyId, msgs::Value> value_properties;
|
||||
for (auto &[prop_key, prop_val] : properties_results.GetValue()) {
|
||||
value_properties.insert(std::make_pair(prop_key, FromPropertyValueToValue(std::move(prop_val))));
|
||||
}
|
||||
using EdgeWithAllProperties = msgs::ExpandOneResultRow::EdgeWithAllProperties;
|
||||
|
||||
if (is_in_edge) {
|
||||
result_row.in_edges_with_all_properties.push_back(
|
||||
EdgeWithAllProperties{ToMsgsVertexId(edge.From()), msgs::EdgeType{edge.EdgeType()}, edge.Gid().AsUint(),
|
||||
std::move(value_properties)});
|
||||
} else {
|
||||
result_row.out_edges_with_all_properties.push_back(
|
||||
EdgeWithAllProperties{ToMsgsVertexId(edge.To()), msgs::EdgeType{edge.EdgeType()}, edge.Gid().AsUint(),
|
||||
std::move(value_properties)});
|
||||
}
|
||||
return {};
|
||||
};
|
||||
} else {
|
||||
// TODO(gvolfing) - do we want to set the action_successful here?
|
||||
edge_filler = [&req](const EdgeAccessor &edge, const bool is_in_edge,
|
||||
msgs::ExpandOneResultRow &result_row) -> ShardResult<void> {
|
||||
std::vector<msgs::Value> value_properties;
|
||||
value_properties.reserve(req.edge_properties.value().size());
|
||||
for (const auto &edge_prop : req.edge_properties.value()) {
|
||||
auto property_result = edge.GetProperty(edge_prop, View::NEW);
|
||||
if (property_result.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get edge properties. Transaction id: {}",
|
||||
req.transaction_id.logical_id);
|
||||
return property_result.GetError();
|
||||
}
|
||||
value_properties.emplace_back(FromPropertyValueToValue(std::move(property_result.GetValue())));
|
||||
}
|
||||
using EdgeWithSpecificProperties = msgs::ExpandOneResultRow::EdgeWithSpecificProperties;
|
||||
|
||||
if (is_in_edge) {
|
||||
result_row.in_edges_with_specific_properties.push_back(
|
||||
EdgeWithSpecificProperties{ToMsgsVertexId(edge.From()), msgs::EdgeType{edge.EdgeType()},
|
||||
edge.Gid().AsUint(), std::move(value_properties)});
|
||||
} else {
|
||||
result_row.out_edges_with_specific_properties.push_back(
|
||||
EdgeWithSpecificProperties{ToMsgsVertexId(edge.To()), msgs::EdgeType{edge.EdgeType()}, edge.Gid().AsUint(),
|
||||
std::move(value_properties)});
|
||||
}
|
||||
return {};
|
||||
};
|
||||
}
|
||||
|
||||
return edge_filler;
|
||||
}
|
||||
|
||||
bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc,
|
||||
const std::vector<std::string> &filters) {
|
||||
return std::ranges::all_of(filters, [&dba, &v_acc](const auto &filter_expr) {
|
||||
const auto result = ComputeExpression(dba, v_acc, std::nullopt, filter_expr, expr::identifier_node_symbol, "");
|
||||
return result.IsBool() && result.ValueBool();
|
||||
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
|
||||
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
|
||||
});
|
||||
}
|
||||
|
||||
bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const EdgeAccessor &e_acc,
|
||||
const std::vector<std::string> &filters) {
|
||||
return std::ranges::all_of(filters, [&dba, &v_acc, &e_acc](const auto &filter_expr) {
|
||||
const auto result =
|
||||
ComputeExpression(dba, v_acc, e_acc, filter_expr, expr::identifier_node_symbol, expr::identifier_edge_symbol);
|
||||
return result.IsBool() && result.ValueBool();
|
||||
});
|
||||
}
|
||||
|
||||
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
|
||||
Shard::Accessor &acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
|
||||
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
|
||||
const Schemas::Schema &schema) {
|
||||
/// Fill up source vertex
|
||||
const auto primary_key = ConvertPropertyVector(src_vertex.second);
|
||||
auto v_acc = acc.FindVertex(primary_key, View::NEW);
|
||||
|
||||
msgs::Vertex source_vertex = {.id = src_vertex};
|
||||
auto maybe_secondary_labels = FillUpSourceVertexSecondaryLabels(v_acc, req);
|
||||
if (maybe_secondary_labels.HasError()) {
|
||||
return maybe_secondary_labels.GetError();
|
||||
}
|
||||
source_vertex.labels = std::move(*maybe_secondary_labels);
|
||||
|
||||
auto src_vertex_properties = FillUpSourceVertexProperties(v_acc, req, storage::v3::View::NEW, schema);
|
||||
if (src_vertex_properties.HasError()) {
|
||||
return src_vertex_properties.GetError();
|
||||
}
|
||||
|
||||
/// Fill up connecting edges
|
||||
auto fill_up_connecting_edges = FillUpConnectingEdges(v_acc, req, maybe_filter_based_on_edge_uniqueness);
|
||||
if (fill_up_connecting_edges.HasError()) {
|
||||
return fill_up_connecting_edges.GetError();
|
||||
}
|
||||
auto [in_edges, out_edges] = fill_up_connecting_edges.GetValue();
|
||||
|
||||
msgs::ExpandOneResultRow result_row;
|
||||
result_row.src_vertex = std::move(source_vertex);
|
||||
result_row.src_vertex_properties = std::move(*src_vertex_properties);
|
||||
static constexpr bool kInEdges = true;
|
||||
static constexpr bool kOutEdges = false;
|
||||
if (const auto fill_edges_res = FillEdges<kInEdges>(in_edges, result_row, edge_filler); fill_edges_res.HasError()) {
|
||||
return fill_edges_res.GetError();
|
||||
}
|
||||
if (const auto fill_edges_res = FillEdges<kOutEdges>(out_edges, result_row, edge_filler); fill_edges_res.HasError()) {
|
||||
return fill_edges_res.GetError();
|
||||
}
|
||||
|
||||
return result_row;
|
||||
}
|
||||
|
||||
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
|
||||
VertexAccessor v_acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
|
||||
std::vector<EdgeAccessor> in_edge_accessors, std::vector<EdgeAccessor> out_edge_accessors,
|
||||
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
|
||||
const Schemas::Schema &schema) {
|
||||
/// Fill up source vertex
|
||||
msgs::Vertex source_vertex = {.id = src_vertex};
|
||||
auto maybe_secondary_labels = FillUpSourceVertexSecondaryLabels(v_acc, req);
|
||||
if (maybe_secondary_labels.HasError()) {
|
||||
return maybe_secondary_labels.GetError();
|
||||
}
|
||||
source_vertex.labels = std::move(*maybe_secondary_labels);
|
||||
|
||||
/// Fill up source vertex properties
|
||||
auto src_vertex_properties = FillUpSourceVertexProperties(v_acc, req, storage::v3::View::NEW, schema);
|
||||
if (src_vertex_properties.HasError()) {
|
||||
return src_vertex_properties.GetError();
|
||||
}
|
||||
|
||||
/// Fill up connecting edges
|
||||
auto in_edges = maybe_filter_based_on_edge_uniqueness(std::move(in_edge_accessors), msgs::EdgeDirection::IN);
|
||||
auto out_edges = maybe_filter_based_on_edge_uniqueness(std::move(out_edge_accessors), msgs::EdgeDirection::OUT);
|
||||
|
||||
msgs::ExpandOneResultRow result_row;
|
||||
result_row.src_vertex = std::move(source_vertex);
|
||||
result_row.src_vertex_properties = std::move(*src_vertex_properties);
|
||||
static constexpr bool kInEdges = true;
|
||||
static constexpr bool kOutEdges = false;
|
||||
if (const auto fill_edges_res = FillEdges<kInEdges>(in_edges, result_row, edge_filler); fill_edges_res.HasError()) {
|
||||
return fill_edges_res.GetError();
|
||||
}
|
||||
if (const auto fill_edges_res = FillEdges<kOutEdges>(out_edges, result_row, edge_filler); fill_edges_res.HasError()) {
|
||||
return fill_edges_res.GetError();
|
||||
}
|
||||
|
||||
return result_row;
|
||||
return ordered;
|
||||
}
|
||||
|
||||
VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_iterable,
|
||||
const std::vector<PropertyValue> &primary_key, const View view) {
|
||||
const std::vector<PropertyValue> &start_ids, const View view) {
|
||||
auto it = vertex_iterable.begin();
|
||||
while (it != vertex_iterable.end()) {
|
||||
if (const auto &vertex = *it; primary_key <= vertex.PrimaryKey(view).GetValue()) {
|
||||
if (const auto &vertex = *it; start_ids <= vertex.PrimaryKey(view).GetValue()) {
|
||||
break;
|
||||
}
|
||||
++it;
|
||||
@@ -475,112 +68,15 @@ VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_itera
|
||||
return it;
|
||||
}
|
||||
|
||||
std::vector<Element<VertexAccessor>>::const_iterator GetStartOrderedElementsIterator(
|
||||
const std::vector<Element<VertexAccessor>> &ordered_elements, const std::vector<PropertyValue> &primary_key,
|
||||
const View view) {
|
||||
std::vector<Element>::const_iterator GetStartOrderedElementsIterator(const std::vector<Element> &ordered_elements,
|
||||
const std::vector<PropertyValue> &start_ids,
|
||||
const View view) {
|
||||
for (auto it = ordered_elements.begin(); it != ordered_elements.end(); ++it) {
|
||||
if (const auto &vertex = it->object_acc; primary_key <= vertex.PrimaryKey(view).GetValue()) {
|
||||
if (const auto &vertex = it->vertex_acc; start_ids <= vertex.PrimaryKey(view).GetValue()) {
|
||||
return it;
|
||||
}
|
||||
}
|
||||
return ordered_elements.end();
|
||||
}
|
||||
|
||||
std::array<std::vector<EdgeAccessor>, 2> GetEdgesFromVertex(const VertexAccessor &vertex_accessor,
|
||||
const msgs::EdgeDirection direction) {
|
||||
std::vector<EdgeAccessor> in_edges;
|
||||
std::vector<EdgeAccessor> out_edges;
|
||||
|
||||
switch (direction) {
|
||||
case memgraph::msgs::EdgeDirection::IN: {
|
||||
auto edges = vertex_accessor.InEdges(View::OLD);
|
||||
if (edges.HasValue()) {
|
||||
in_edges = edges.GetValue();
|
||||
}
|
||||
break;
|
||||
}
|
||||
case memgraph::msgs::EdgeDirection::OUT: {
|
||||
auto edges = vertex_accessor.OutEdges(View::OLD);
|
||||
if (edges.HasValue()) {
|
||||
out_edges = edges.GetValue();
|
||||
}
|
||||
break;
|
||||
}
|
||||
case memgraph::msgs::EdgeDirection::BOTH: {
|
||||
auto maybe_in_edges = vertex_accessor.InEdges(View::OLD);
|
||||
auto maybe_out_edges = vertex_accessor.OutEdges(View::OLD);
|
||||
std::vector<EdgeAccessor> edges;
|
||||
if (maybe_in_edges.HasValue()) {
|
||||
in_edges = maybe_in_edges.GetValue();
|
||||
}
|
||||
if (maybe_out_edges.HasValue()) {
|
||||
out_edges = maybe_out_edges.GetValue();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return std::array<std::vector<EdgeAccessor>, 2>{std::move(in_edges), std::move(out_edges)};
|
||||
}
|
||||
|
||||
std::vector<Element<EdgeAccessor>> OrderByEdges(DbAccessor &dba, std::vector<EdgeAccessor> &iterable,
|
||||
std::vector<msgs::OrderBy> &order_by_edges,
|
||||
const VertexAccessor &vertex_acc) {
|
||||
std::vector<Ordering> ordering;
|
||||
ordering.reserve(order_by_edges.size());
|
||||
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(ordering),
|
||||
[](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
|
||||
|
||||
std::vector<Element<EdgeAccessor>> ordered;
|
||||
for (auto it = iterable.begin(); it != iterable.end(); ++it) {
|
||||
std::vector<TypedValue> properties_order_by;
|
||||
properties_order_by.reserve(order_by_edges.size());
|
||||
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(properties_order_by),
|
||||
[&dba, &vertex_acc, &it](const auto &order_by) {
|
||||
return ComputeExpression(dba, vertex_acc, *it, order_by.expression.expression,
|
||||
expr::identifier_node_symbol, expr::identifier_edge_symbol);
|
||||
});
|
||||
|
||||
ordered.push_back({std::move(properties_order_by), *it});
|
||||
}
|
||||
|
||||
auto compare_typed_values = TypedValueVectorCompare(ordering);
|
||||
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
|
||||
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
|
||||
});
|
||||
return ordered;
|
||||
}
|
||||
|
||||
std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> OrderByEdges(
|
||||
DbAccessor &dba, std::vector<EdgeAccessor> &iterable, std::vector<msgs::OrderBy> &order_by_edges,
|
||||
const std::vector<VertexAccessor> &vertex_acc) {
|
||||
MG_ASSERT(vertex_acc.size() == iterable.size());
|
||||
std::vector<Ordering> ordering;
|
||||
ordering.reserve(order_by_edges.size());
|
||||
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(ordering),
|
||||
[](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
|
||||
|
||||
std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> ordered;
|
||||
VertexAccessor current = vertex_acc.front();
|
||||
size_t id = 0;
|
||||
for (auto it = iterable.begin(); it != iterable.end(); it++, id++) {
|
||||
current = vertex_acc[id];
|
||||
std::vector<TypedValue> properties_order_by;
|
||||
properties_order_by.reserve(order_by_edges.size());
|
||||
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(properties_order_by),
|
||||
[&dba, it, current](const auto &order_by) {
|
||||
return ComputeExpression(dba, current, *it, order_by.expression.expression,
|
||||
expr::identifier_node_symbol, expr::identifier_edge_symbol);
|
||||
});
|
||||
|
||||
ordered.push_back({std::move(properties_order_by), {current, *it}});
|
||||
}
|
||||
|
||||
auto compare_typed_values = TypedValueVectorCompare(ordering);
|
||||
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
|
||||
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
|
||||
});
|
||||
return ordered;
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -9,30 +9,14 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/bindings/ast/ast.hpp"
|
||||
#include "storage/v3/bindings/pretty_print_ast_to_original_expression.hpp"
|
||||
#include "ast/ast.hpp"
|
||||
#include "storage/v3/bindings/typed_value.hpp"
|
||||
#include "storage/v3/edge_accessor.hpp"
|
||||
#include "storage/v3/expr.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/value_conversions.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "utils/template_utils.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
using EdgeAccessors = std::vector<storage::v3::EdgeAccessor>;
|
||||
using EdgeUniquenessFunction = std::function<EdgeAccessors(EdgeAccessors &&, msgs::EdgeDirection)>;
|
||||
using EdgeFiller =
|
||||
std::function<ShardResult<void>(const EdgeAccessor &edge, bool is_in_edge, msgs::ExpandOneResultRow &result_row)>;
|
||||
using msgs::Value;
|
||||
|
||||
template <typename T>
|
||||
concept OrderableObject = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor, std::pair<VertexAccessor, EdgeAccessor>>;
|
||||
|
||||
inline bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
|
||||
// in ordering null comes after everything else
|
||||
@@ -88,17 +72,6 @@ inline bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
|
||||
}
|
||||
}
|
||||
|
||||
inline Ordering ConvertMsgsOrderByToOrdering(msgs::OrderingDirection ordering) {
|
||||
switch (ordering) {
|
||||
case memgraph::msgs::OrderingDirection::ASCENDING:
|
||||
return memgraph::storage::v3::Ordering::ASC;
|
||||
case memgraph::msgs::OrderingDirection::DESCENDING:
|
||||
return memgraph::storage::v3::Ordering::DESC;
|
||||
default:
|
||||
LOG_FATAL("Unknown ordering direction");
|
||||
}
|
||||
}
|
||||
|
||||
class TypedValueVectorCompare final {
|
||||
public:
|
||||
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
|
||||
@@ -126,134 +99,18 @@ class TypedValueVectorCompare final {
|
||||
std::vector<Ordering> ordering_;
|
||||
};
|
||||
|
||||
template <OrderableObject TObjectAccessor>
|
||||
struct Element {
|
||||
std::vector<TypedValue> properties_order_by;
|
||||
TObjectAccessor object_acc;
|
||||
VertexAccessor vertex_acc;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
concept VerticesIt = utils::SameAsAnyOf<T, VerticesIterable, std::vector<VertexAccessor>>;
|
||||
|
||||
template <VerticesIt TIterable>
|
||||
std::vector<Element<VertexAccessor>> OrderByVertices(DbAccessor &dba, TIterable &iterable,
|
||||
std::vector<msgs::OrderBy> &order_by_vertices) {
|
||||
std::vector<Ordering> ordering;
|
||||
ordering.reserve(order_by_vertices.size());
|
||||
std::transform(order_by_vertices.begin(), order_by_vertices.end(), std::back_inserter(ordering),
|
||||
[](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
|
||||
|
||||
std::vector<Element<VertexAccessor>> ordered;
|
||||
for (auto it = iterable.begin(); it != iterable.end(); ++it) {
|
||||
std::vector<TypedValue> properties_order_by;
|
||||
properties_order_by.reserve(order_by_vertices.size());
|
||||
|
||||
std::transform(order_by_vertices.begin(), order_by_vertices.end(), std::back_inserter(properties_order_by),
|
||||
[&dba, &it](const auto &order_by) {
|
||||
return ComputeExpression(dba, *it, std::nullopt /*e_acc*/, order_by.expression.expression,
|
||||
expr::identifier_node_symbol, expr::identifier_edge_symbol);
|
||||
});
|
||||
|
||||
ordered.push_back({std::move(properties_order_by), *it});
|
||||
}
|
||||
|
||||
auto compare_typed_values = TypedValueVectorCompare(ordering);
|
||||
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
|
||||
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
|
||||
});
|
||||
return ordered;
|
||||
}
|
||||
|
||||
std::vector<Element<EdgeAccessor>> OrderByEdges(DbAccessor &dba, std::vector<EdgeAccessor> &iterable,
|
||||
std::vector<msgs::OrderBy> &order_by_edges,
|
||||
const VertexAccessor &vertex_acc);
|
||||
|
||||
std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> OrderByEdges(
|
||||
DbAccessor &dba, std::vector<EdgeAccessor> &iterable, std::vector<msgs::OrderBy> &order_by_edges,
|
||||
const std::vector<VertexAccessor> &vertex_acc);
|
||||
std::vector<Element> OrderByElements(Shard::Accessor &acc, DbAccessor &dba, VerticesIterable &vertices_iterable,
|
||||
std::vector<msgs::OrderBy> &order_bys);
|
||||
|
||||
VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_iterable,
|
||||
const std::vector<PropertyValue> &primary_key, View view);
|
||||
const std::vector<PropertyValue> &start_ids, View view);
|
||||
|
||||
std::vector<Element<VertexAccessor>>::const_iterator GetStartOrderedElementsIterator(
|
||||
const std::vector<Element<VertexAccessor>> &ordered_elements, const std::vector<PropertyValue> &primary_key,
|
||||
View view);
|
||||
|
||||
std::array<std::vector<EdgeAccessor>, 2> GetEdgesFromVertex(const VertexAccessor &vertex_accessor,
|
||||
msgs::EdgeDirection direction);
|
||||
|
||||
bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const std::vector<std::string> &filters);
|
||||
|
||||
bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const EdgeAccessor &e_acc,
|
||||
const std::vector<std::string> &filters);
|
||||
|
||||
std::vector<TypedValue> EvaluateVertexExpressions(DbAccessor &dba, const VertexAccessor &v_acc,
|
||||
const std::vector<std::string> &expressions,
|
||||
std::string_view node_name);
|
||||
|
||||
std::vector<TypedValue> EvaluateEdgeExpressions(DbAccessor &dba, const VertexAccessor &v_acc, const EdgeAccessor &e_acc,
|
||||
const std::vector<std::string> &expressions);
|
||||
|
||||
template <typename T>
|
||||
concept PropertiesAccessor = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor>;
|
||||
|
||||
template <PropertiesAccessor TAccessor>
|
||||
ShardResult<std::map<PropertyId, Value>> CollectSpecificPropertiesFromAccessor(const TAccessor &acc,
|
||||
const std::vector<PropertyId> &props,
|
||||
View view) {
|
||||
std::map<PropertyId, Value> ret;
|
||||
|
||||
for (const auto &prop : props) {
|
||||
auto result = acc.GetProperty(prop, view);
|
||||
if (result.HasError()) {
|
||||
spdlog::debug("Encountered an Error while trying to get a vertex property.");
|
||||
return result.GetError();
|
||||
}
|
||||
auto &value = result.GetValue();
|
||||
ret.emplace(std::make_pair(prop, FromPropertyValueToValue(std::move(value))));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view,
|
||||
const Schemas::Schema &schema);
|
||||
namespace impl {
|
||||
template <PropertiesAccessor TAccessor>
|
||||
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesImpl(const TAccessor &acc, View view) {
|
||||
std::map<PropertyId, Value> ret;
|
||||
auto props = acc.Properties(view);
|
||||
if (props.HasError()) {
|
||||
spdlog::debug("Encountered an error while trying to get vertex properties.");
|
||||
return props.GetError();
|
||||
}
|
||||
|
||||
auto &properties = props.GetValue();
|
||||
std::transform(properties.begin(), properties.end(), std::inserter(ret, ret.begin()),
|
||||
[](std::pair<const PropertyId, PropertyValue> &pair) {
|
||||
return std::make_pair(pair.first, conversions::FromPropertyValueToValue(std::move(pair.second)));
|
||||
});
|
||||
return ret;
|
||||
}
|
||||
} // namespace impl
|
||||
|
||||
template <PropertiesAccessor TAccessor>
|
||||
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const TAccessor &acc, View view) {
|
||||
return impl::CollectAllPropertiesImpl<TAccessor>(acc, view);
|
||||
}
|
||||
|
||||
EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbor_rows);
|
||||
|
||||
EdgeFiller InitializeEdgeFillerFunction(const msgs::ExpandOneRequest &req);
|
||||
|
||||
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
|
||||
Shard::Accessor &acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
|
||||
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
|
||||
const Schemas::Schema &schema);
|
||||
|
||||
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
|
||||
VertexAccessor v_acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
|
||||
std::vector<EdgeAccessor> in_edge_accessors, std::vector<EdgeAccessor> out_edge_accessors,
|
||||
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
|
||||
const Schemas::Schema &schema);
|
||||
std::vector<Element>::const_iterator GetStartOrderedElementsIterator(const std::vector<Element> &ordered_elements,
|
||||
const std::vector<PropertyValue> &start_ids,
|
||||
View view);
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -11,43 +11,24 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <experimental/source_location>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
#include <type_traits>
|
||||
|
||||
#include "common/errors.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
static_assert(std::is_same_v<uint8_t, unsigned char>);
|
||||
|
||||
struct ShardError {
|
||||
ShardError(common::ErrorCode code, std::string message, const std::experimental::source_location location)
|
||||
: code{code}, message{std::move(message)}, source{fmt::format("{}:{}", location.file_name(), location.line())} {}
|
||||
|
||||
ShardError(common::ErrorCode code, const std::experimental::source_location location)
|
||||
: code{code}, source{fmt::format("{}:{}", location.file_name(), location.line())} {}
|
||||
|
||||
common::ErrorCode code;
|
||||
std::string message;
|
||||
std::string source;
|
||||
|
||||
inline friend bool operator==(const ShardError &lhs, const ShardError &rhs) { return lhs.code == rhs.code; }
|
||||
|
||||
inline friend bool operator==(const ShardError &lhs, const common::ErrorCode rhs) { return lhs.code == rhs; }
|
||||
enum class Error : uint8_t {
|
||||
SERIALIZATION_ERROR,
|
||||
NONEXISTENT_OBJECT,
|
||||
DELETED_OBJECT,
|
||||
VERTEX_HAS_EDGES,
|
||||
PROPERTIES_DISABLED,
|
||||
VERTEX_ALREADY_INSERTED
|
||||
};
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define SHARD_ERROR(error, ...) \
|
||||
({ \
|
||||
using ErrorCode = memgraph::common::ErrorCode; \
|
||||
memgraph::storage::v3::ShardError(error, GET_MESSAGE(__VA_ARGS__), std::experimental::source_location::current()); \
|
||||
})
|
||||
|
||||
template <class TValue>
|
||||
using ShardResult = utils::BasicResult<ShardError, TValue>;
|
||||
using Result = utils::BasicResult<Error, TValue>;
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
|
||||
@@ -16,58 +16,67 @@
|
||||
#include <ranges>
|
||||
|
||||
#include "common/types.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
SchemaValidator::SchemaValidator(Schemas &schemas, const NameIdMapper &name_id_mapper)
|
||||
: schemas_{&schemas}, name_id_mapper_{&name_id_mapper} {}
|
||||
bool operator==(const SchemaViolation &lhs, const SchemaViolation &rhs) {
|
||||
return lhs.status == rhs.status && lhs.label == rhs.label &&
|
||||
lhs.violated_schema_property == rhs.violated_schema_property &&
|
||||
lhs.violated_property_value == rhs.violated_property_value;
|
||||
}
|
||||
|
||||
ShardResult<void> SchemaValidator::ValidateVertexCreate(LabelId primary_label, const std::vector<LabelId> &labels,
|
||||
const std::vector<PropertyValue> &primary_properties) const {
|
||||
SchemaViolation::SchemaViolation(ValidationStatus status, LabelId label) : status{status}, label{label} {}
|
||||
|
||||
SchemaViolation::SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property)
|
||||
: status{status}, label{label}, violated_schema_property{violated_schema_property} {}
|
||||
|
||||
SchemaViolation::SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property,
|
||||
PropertyValue violated_property_value)
|
||||
: status{status},
|
||||
label{label},
|
||||
violated_schema_property{violated_schema_property},
|
||||
violated_property_value{violated_property_value} {}
|
||||
|
||||
SchemaValidator::SchemaValidator(Schemas &schemas) : schemas_{schemas} {}
|
||||
|
||||
std::optional<SchemaViolation> SchemaValidator::ValidateVertexCreate(
|
||||
LabelId primary_label, const std::vector<LabelId> &labels,
|
||||
const std::vector<PropertyValue> &primary_properties) const {
|
||||
// Schema on primary label
|
||||
const auto *schema = schemas_->GetSchema(primary_label);
|
||||
const auto *schema = schemas_.GetSchema(primary_label);
|
||||
if (schema == nullptr) {
|
||||
return SHARD_ERROR(ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL, "Schema not defined for label :{}",
|
||||
name_id_mapper_->IdToName(primary_label.AsInt()));
|
||||
return SchemaViolation(SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL, primary_label);
|
||||
}
|
||||
|
||||
// Is there another primary label among secondary labels
|
||||
for (const auto &secondary_label : labels) {
|
||||
if (schemas_->GetSchema(secondary_label)) {
|
||||
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
|
||||
"Cannot add label :{}, since it is defined as a primary label",
|
||||
name_id_mapper_->IdToName(secondary_label.AsInt()));
|
||||
if (schemas_.GetSchema(secondary_label)) {
|
||||
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY, secondary_label);
|
||||
}
|
||||
}
|
||||
|
||||
// Quick size check
|
||||
if (schema->second.size() != primary_properties.size()) {
|
||||
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
|
||||
"Not all primary properties have been specified for :{} vertex",
|
||||
name_id_mapper_->IdToName(primary_label.AsInt()));
|
||||
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_PRIMARY_PROPERTIES_UNDEFINED, primary_label);
|
||||
}
|
||||
// Check only properties defined by schema
|
||||
for (size_t i{0}; i < schema->second.size(); ++i) {
|
||||
// Check schema property type
|
||||
if (auto property_schema_type = PropertyTypeToSchemaType(primary_properties[i]);
|
||||
property_schema_type && *property_schema_type != schema->second[i].type) {
|
||||
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
|
||||
"Property {} is of wrong type, expected {}, actual {}",
|
||||
name_id_mapper_->IdToName(schema->second[i].property_id.AsInt()),
|
||||
SchemaTypeToString(schema->second[i].type), SchemaTypeToString(*property_schema_type));
|
||||
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE, primary_label,
|
||||
schema->second[i], primary_properties[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
ShardResult<void> SchemaValidator::ValidatePropertyUpdate(const LabelId primary_label,
|
||||
const PropertyId property_id) const {
|
||||
std::optional<SchemaViolation> SchemaValidator::ValidatePropertyUpdate(const LabelId primary_label,
|
||||
const PropertyId property_id) const {
|
||||
// Verify existence of schema on primary label
|
||||
const auto *schema = schemas_->GetSchema(primary_label);
|
||||
const auto *schema = schemas_.GetSchema(primary_label);
|
||||
MG_ASSERT(schema, "Cannot validate against non existing schema!");
|
||||
|
||||
// Verify that updating property is not part of schema
|
||||
@@ -75,37 +84,34 @@ ShardResult<void> SchemaValidator::ValidatePropertyUpdate(const LabelId primary_
|
||||
schema->second,
|
||||
[property_id](const auto &schema_property) { return property_id == schema_property.property_id; });
|
||||
schema_property != schema->second.end()) {
|
||||
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
|
||||
"Cannot update primary property {} of schema on label :{}",
|
||||
name_id_mapper_->IdToName(schema_property->property_id.AsInt()),
|
||||
name_id_mapper_->IdToName(primary_label.AsInt()));
|
||||
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY, primary_label,
|
||||
*schema_property);
|
||||
}
|
||||
return {};
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
ShardResult<void> SchemaValidator::ValidateLabelUpdate(const LabelId label) const {
|
||||
const auto *schema = schemas_->GetSchema(label);
|
||||
std::optional<SchemaViolation> SchemaValidator::ValidateLabelUpdate(const LabelId label) const {
|
||||
const auto *schema = schemas_.GetSchema(label);
|
||||
if (schema) {
|
||||
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL, "Cannot add/remove primary label :{}",
|
||||
name_id_mapper_->IdToName(label.AsInt()));
|
||||
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, label);
|
||||
}
|
||||
return {};
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const Schemas::Schema *SchemaValidator::GetSchema(LabelId label) const { return schemas_->GetSchema(label); }
|
||||
const Schemas::Schema *SchemaValidator::GetSchema(LabelId label) const { return schemas_.GetSchema(label); }
|
||||
|
||||
VertexValidator::VertexValidator(const SchemaValidator &schema_validator, const LabelId primary_label)
|
||||
: schema_validator{&schema_validator}, primary_label_{primary_label} {}
|
||||
|
||||
ShardResult<void> VertexValidator::ValidatePropertyUpdate(PropertyId property_id) const {
|
||||
std::optional<SchemaViolation> VertexValidator::ValidatePropertyUpdate(PropertyId property_id) const {
|
||||
return schema_validator->ValidatePropertyUpdate(primary_label_, property_id);
|
||||
};
|
||||
|
||||
ShardResult<void> VertexValidator::ValidateAddLabel(LabelId label) const {
|
||||
std::optional<SchemaViolation> VertexValidator::ValidateAddLabel(LabelId label) const {
|
||||
return schema_validator->ValidateLabelUpdate(label);
|
||||
}
|
||||
|
||||
ShardResult<void> VertexValidator::ValidateRemoveLabel(LabelId label) const {
|
||||
std::optional<SchemaViolation> VertexValidator::ValidateRemoveLabel(LabelId label) const {
|
||||
return schema_validator->ValidateLabelUpdate(label);
|
||||
}
|
||||
|
||||
|
||||
@@ -11,43 +11,68 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <variant>
|
||||
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
struct SchemaViolation {
|
||||
enum class ValidationStatus : uint8_t {
|
||||
NO_SCHEMA_DEFINED_FOR_LABEL,
|
||||
VERTEX_PROPERTY_WRONG_TYPE,
|
||||
VERTEX_UPDATE_PRIMARY_KEY,
|
||||
VERTEX_UPDATE_PRIMARY_LABEL,
|
||||
VERTEX_SECONDARY_LABEL_IS_PRIMARY,
|
||||
VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
|
||||
};
|
||||
|
||||
SchemaViolation(ValidationStatus status, LabelId label);
|
||||
|
||||
SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property);
|
||||
|
||||
SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property,
|
||||
PropertyValue violated_property_value);
|
||||
|
||||
friend bool operator==(const SchemaViolation &lhs, const SchemaViolation &rhs);
|
||||
|
||||
ValidationStatus status;
|
||||
LabelId label;
|
||||
std::optional<SchemaProperty> violated_schema_property;
|
||||
std::optional<PropertyValue> violated_property_value;
|
||||
};
|
||||
|
||||
class SchemaValidator {
|
||||
public:
|
||||
explicit SchemaValidator(Schemas &schemas, const NameIdMapper &name_id_mapper);
|
||||
explicit SchemaValidator(Schemas &schemas);
|
||||
|
||||
[[nodiscard]] ShardResult<void> ValidateVertexCreate(LabelId primary_label, const std::vector<LabelId> &labels,
|
||||
const std::vector<PropertyValue> &primary_properties) const;
|
||||
[[nodiscard]] std::optional<SchemaViolation> ValidateVertexCreate(
|
||||
LabelId primary_label, const std::vector<LabelId> &labels,
|
||||
const std::vector<PropertyValue> &primary_properties) const;
|
||||
|
||||
[[nodiscard]] ShardResult<void> ValidatePropertyUpdate(LabelId primary_label, PropertyId property_id) const;
|
||||
[[nodiscard]] std::optional<SchemaViolation> ValidatePropertyUpdate(LabelId primary_label,
|
||||
PropertyId property_id) const;
|
||||
|
||||
[[nodiscard]] ShardResult<void> ValidateLabelUpdate(LabelId label) const;
|
||||
[[nodiscard]] std::optional<SchemaViolation> ValidateLabelUpdate(LabelId label) const;
|
||||
|
||||
const Schemas::Schema *GetSchema(LabelId label) const;
|
||||
|
||||
private:
|
||||
Schemas *schemas_;
|
||||
const NameIdMapper *name_id_mapper_;
|
||||
Schemas &schemas_;
|
||||
};
|
||||
|
||||
struct VertexValidator {
|
||||
explicit VertexValidator(const SchemaValidator &schema_validator, LabelId primary_label);
|
||||
|
||||
[[nodiscard]] ShardResult<void> ValidatePropertyUpdate(PropertyId property_id) const;
|
||||
[[nodiscard]] std::optional<SchemaViolation> ValidatePropertyUpdate(PropertyId property_id) const;
|
||||
|
||||
[[nodiscard]] ShardResult<void> ValidateAddLabel(LabelId label) const;
|
||||
[[nodiscard]] std::optional<SchemaViolation> ValidateAddLabel(LabelId label) const;
|
||||
|
||||
[[nodiscard]] ShardResult<void> ValidateRemoveLabel(LabelId label) const;
|
||||
[[nodiscard]] std::optional<SchemaViolation> ValidateRemoveLabel(LabelId label) const;
|
||||
|
||||
const SchemaValidator *schema_validator;
|
||||
|
||||
|
||||
@@ -31,14 +31,13 @@
|
||||
#include "storage/v3/indices.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/mvcc.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
#include "storage/v3/shard_operation_result.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "storage/v3/vertices_skip_list.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -75,11 +74,11 @@ uint64_t GetCleanupBeforeTimestamp(const std::map<uint64_t, std::unique_ptr<Tran
|
||||
|
||||
} // namespace
|
||||
|
||||
auto AdvanceToVisibleVertex(VertexContainer::iterator it, VertexContainer::iterator end,
|
||||
auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Iterator end,
|
||||
std::optional<VertexAccessor> *vertex, Transaction *tx, View view, Indices *indices,
|
||||
Config::Items config, const VertexValidator &vertex_validator) {
|
||||
while (it != end) {
|
||||
*vertex = VertexAccessor::Create(&*it, tx, indices, config, vertex_validator, view);
|
||||
*vertex = VertexAccessor::Create(&it->vertex, tx, indices, config, vertex_validator, view);
|
||||
if (!*vertex) {
|
||||
++it;
|
||||
continue;
|
||||
@@ -89,17 +88,17 @@ auto AdvanceToVisibleVertex(VertexContainer::iterator it, VertexContainer::itera
|
||||
return it;
|
||||
}
|
||||
|
||||
AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VertexContainer::iterator it)
|
||||
AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it)
|
||||
: self_(self),
|
||||
it_(AdvanceToVisibleVertex(it, self->vertices_accessor_->end(), &self->vertex_, self->transaction_, self->view_,
|
||||
it_(AdvanceToVisibleVertex(it, self->vertices_accessor_.end(), &self->vertex_, self->transaction_, self->view_,
|
||||
self->indices_, self->config_, *self_->vertex_validator_)) {}
|
||||
|
||||
VertexAccessor AllVerticesIterable::Iterator::operator*() const { return *self_->vertex_; }
|
||||
|
||||
AllVerticesIterable::Iterator &AllVerticesIterable::Iterator::operator++() {
|
||||
++it_;
|
||||
it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_->end(), &self_->vertex_, self_->transaction_,
|
||||
self_->view_, self_->indices_, self_->config_, *self_->vertex_validator_);
|
||||
it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_.end(), &self_->vertex_, self_->transaction_, self_->view_,
|
||||
self_->indices_, self_->config_, *self_->vertex_validator_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -328,7 +327,7 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key,
|
||||
: primary_label_{primary_label},
|
||||
min_primary_key_{min_primary_key},
|
||||
max_primary_key_{max_primary_key},
|
||||
schema_validator_{schemas_, name_id_mapper_},
|
||||
schema_validator_{schemas_},
|
||||
vertex_validator_{schema_validator_, primary_label},
|
||||
indices_{config.items, vertex_validator_},
|
||||
isolation_level_{config.transaction.isolation_level},
|
||||
@@ -345,27 +344,28 @@ Shard::~Shard() {}
|
||||
Shard::Accessor::Accessor(Shard &shard, Transaction &transaction)
|
||||
: shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {}
|
||||
|
||||
ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
|
||||
const std::vector<LabelId> &labels, const PrimaryKey &primary_properties,
|
||||
ShardOperationResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
|
||||
const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
|
||||
const std::vector<std::pair<PropertyId, PropertyValue>> &properties) {
|
||||
OOMExceptionEnabler oom_exception;
|
||||
const auto schema = shard_->GetSchema(shard_->primary_label_)->second;
|
||||
|
||||
auto maybe_schema_violation =
|
||||
GetSchemaValidator().ValidateVertexCreate(shard_->primary_label_, labels, primary_properties);
|
||||
if (maybe_schema_violation.HasError()) {
|
||||
return {std::move(maybe_schema_violation.GetError())};
|
||||
if (maybe_schema_violation) {
|
||||
return {std::move(*maybe_schema_violation)};
|
||||
}
|
||||
|
||||
auto acc = shard_->vertices_.access();
|
||||
auto *delta = CreateDeleteObjectDelta(transaction_);
|
||||
auto [it, inserted] = shard_->vertices_.emplace(primary_properties, VertexData{delta});
|
||||
delta->prev.Set(&*it);
|
||||
auto [it, inserted] = acc.insert({Vertex{delta, primary_properties}});
|
||||
delta->prev.Set(&it->vertex);
|
||||
|
||||
VertexAccessor vertex_acc{&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
|
||||
VertexAccessor vertex_acc{&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
|
||||
if (!inserted) {
|
||||
return SHARD_ERROR(ErrorCode::VERTEX_ALREADY_INSERTED);
|
||||
return {Error::VERTEX_ALREADY_INSERTED};
|
||||
}
|
||||
MG_ASSERT(it != shard_->vertices_.end(), "Invalid Vertex accessor!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
|
||||
|
||||
// TODO(jbajic) Improve, maybe delay index update
|
||||
for (const auto &[property_id, property_value] : properties) {
|
||||
@@ -385,36 +385,37 @@ ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
|
||||
}
|
||||
|
||||
std::optional<VertexAccessor> Shard::Accessor::FindVertex(std::vector<PropertyValue> primary_key, View view) {
|
||||
auto it = shard_->vertices_.find(primary_key);
|
||||
if (it == shard_->vertices_.end()) {
|
||||
auto acc = shard_->vertices_.access();
|
||||
// Later on use label space
|
||||
auto it = acc.find(primary_key);
|
||||
if (it == acc.end()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
return VertexAccessor::Create(&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view);
|
||||
return VertexAccessor::Create(&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view);
|
||||
}
|
||||
|
||||
ShardResult<std::optional<VertexAccessor>> Shard::Accessor::DeleteVertex(VertexAccessor *vertex) {
|
||||
Result<std::optional<VertexAccessor>> Shard::Accessor::DeleteVertex(VertexAccessor *vertex) {
|
||||
MG_ASSERT(vertex->transaction_ == transaction_,
|
||||
"VertexAccessor must be from the same transaction as the storage "
|
||||
"accessor when deleting a vertex!");
|
||||
auto *vertex_ptr = vertex->vertex_;
|
||||
|
||||
if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
if (vertex_ptr->second.deleted) {
|
||||
if (vertex_ptr->deleted) {
|
||||
return std::optional<VertexAccessor>{};
|
||||
}
|
||||
|
||||
if (!vertex_ptr->second.in_edges.empty() || !vertex_ptr->second.out_edges.empty())
|
||||
return SHARD_ERROR(ErrorCode::VERTEX_HAS_EDGES);
|
||||
if (!vertex_ptr->in_edges.empty() || !vertex_ptr->out_edges.empty()) return Error::VERTEX_HAS_EDGES;
|
||||
|
||||
CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag());
|
||||
vertex_ptr->second.deleted = true;
|
||||
vertex_ptr->deleted = true;
|
||||
|
||||
return std::make_optional<VertexAccessor>(vertex_ptr, transaction_, &shard_->indices_, config_,
|
||||
shard_->vertex_validator_, true);
|
||||
}
|
||||
|
||||
ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> Shard::Accessor::DetachDeleteVertex(
|
||||
Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> Shard::Accessor::DetachDeleteVertex(
|
||||
VertexAccessor *vertex) {
|
||||
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
|
||||
|
||||
@@ -423,26 +424,26 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
|
||||
"accessor when deleting a vertex!");
|
||||
auto *vertex_ptr = vertex->vertex_;
|
||||
|
||||
std::vector<VertexData::EdgeLink> in_edges;
|
||||
std::vector<VertexData::EdgeLink> out_edges;
|
||||
std::vector<Vertex::EdgeLink> in_edges;
|
||||
std::vector<Vertex::EdgeLink> out_edges;
|
||||
|
||||
{
|
||||
if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
if (vertex_ptr->second.deleted) return std::optional<ReturnType>{};
|
||||
if (vertex_ptr->deleted) return std::optional<ReturnType>{};
|
||||
|
||||
in_edges = vertex_ptr->second.in_edges;
|
||||
out_edges = vertex_ptr->second.out_edges;
|
||||
in_edges = vertex_ptr->in_edges;
|
||||
out_edges = vertex_ptr->out_edges;
|
||||
}
|
||||
|
||||
std::vector<EdgeAccessor> deleted_edges;
|
||||
const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)}; // TODO Replace
|
||||
const VertexId vertex_id{shard_->primary_label_, vertex_ptr->keys.Keys()};
|
||||
for (const auto &item : in_edges) {
|
||||
auto [edge_type, from_vertex, edge] = item;
|
||||
EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_);
|
||||
auto ret = DeleteEdge(e.From(), e.To(), e.Gid());
|
||||
auto ret = DeleteEdge(e.FromVertex(), e.ToVertex(), e.Gid());
|
||||
if (ret.HasError()) {
|
||||
MG_ASSERT(ret.GetError() == common::ErrorCode::SERIALIZATION_ERROR, "Invalid database state!");
|
||||
MG_ASSERT(ret.GetError() == Error::SERIALIZATION_ERROR, "Invalid database state!");
|
||||
return ret.GetError();
|
||||
}
|
||||
|
||||
@@ -453,9 +454,9 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
|
||||
for (const auto &item : out_edges) {
|
||||
auto [edge_type, to_vertex, edge] = item;
|
||||
EdgeAccessor e(edge, edge_type, vertex_id, to_vertex, transaction_, &shard_->indices_, config_);
|
||||
auto ret = DeleteEdge(e.From(), e.To(), e.Gid());
|
||||
auto ret = DeleteEdge(e.FromVertex(), e.ToVertex(), e.Gid());
|
||||
if (ret.HasError()) {
|
||||
MG_ASSERT(ret.GetError() == common::ErrorCode::SERIALIZATION_ERROR, "Invalid database state!");
|
||||
MG_ASSERT(ret.GetError() == Error::SERIALIZATION_ERROR, "Invalid database state!");
|
||||
return ret.GetError();
|
||||
}
|
||||
|
||||
@@ -468,68 +469,69 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
|
||||
// vertex. Some other transaction could have modified the vertex in the
|
||||
// meantime if we didn't have any edges to delete.
|
||||
|
||||
if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
|
||||
|
||||
MG_ASSERT(!vertex_ptr->second.deleted, "Invalid database state!");
|
||||
MG_ASSERT(!vertex_ptr->deleted, "Invalid database state!");
|
||||
|
||||
CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag());
|
||||
vertex_ptr->second.deleted = true;
|
||||
vertex_ptr->deleted = true;
|
||||
|
||||
return std::make_optional<ReturnType>(
|
||||
VertexAccessor{vertex_ptr, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, true},
|
||||
std::move(deleted_edges));
|
||||
}
|
||||
|
||||
ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id,
|
||||
const EdgeTypeId edge_type, const Gid gid) {
|
||||
Result<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id,
|
||||
const EdgeTypeId edge_type, const Gid gid) {
|
||||
OOMExceptionEnabler oom_exception;
|
||||
Vertex *from_vertex{nullptr};
|
||||
Vertex *to_vertex{nullptr};
|
||||
|
||||
auto &vertices = shard_->vertices_;
|
||||
auto acc = shard_->vertices_.access();
|
||||
|
||||
const auto from_is_local = shard_->IsVertexBelongToShard(from_vertex_id);
|
||||
const auto to_is_local = shard_->IsVertexBelongToShard(to_vertex_id);
|
||||
MG_ASSERT(from_is_local || to_is_local, "Trying to create an edge without having a local vertex");
|
||||
|
||||
if (from_is_local) {
|
||||
auto it = vertices.find(from_vertex_id.primary_key);
|
||||
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
|
||||
from_vertex = &*it;
|
||||
auto it = acc.find(from_vertex_id.primary_key);
|
||||
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
|
||||
from_vertex = &it->vertex;
|
||||
}
|
||||
|
||||
if (to_is_local) {
|
||||
auto it = vertices.find(to_vertex_id.primary_key);
|
||||
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
|
||||
to_vertex = &*it;
|
||||
auto it = acc.find(to_vertex_id.primary_key);
|
||||
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
|
||||
to_vertex = &it->vertex;
|
||||
}
|
||||
|
||||
if (from_is_local) {
|
||||
if (!PrepareForWrite(transaction_, from_vertex)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (from_vertex->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
|
||||
if (!PrepareForWrite(transaction_, from_vertex)) return Error::SERIALIZATION_ERROR;
|
||||
if (from_vertex->deleted) return Error::DELETED_OBJECT;
|
||||
}
|
||||
if (to_is_local && to_vertex != from_vertex) {
|
||||
if (!PrepareForWrite(transaction_, to_vertex)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
if (to_vertex->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
|
||||
if (!PrepareForWrite(transaction_, to_vertex)) return Error::SERIALIZATION_ERROR;
|
||||
if (to_vertex->deleted) return Error::DELETED_OBJECT;
|
||||
}
|
||||
|
||||
EdgeRef edge(gid);
|
||||
if (config_.properties_on_edges) {
|
||||
auto acc = shard_->edges_.access();
|
||||
auto *delta = CreateDeleteObjectDelta(transaction_);
|
||||
auto [it, inserted] = shard_->edges_.emplace(gid, Edge{gid, delta});
|
||||
auto [it, inserted] = acc.insert(Edge(gid, delta));
|
||||
MG_ASSERT(inserted, "The edge must be inserted here!");
|
||||
MG_ASSERT(it != shard_->edges_.end(), "Invalid Edge accessor!");
|
||||
edge = EdgeRef(&it->second);
|
||||
delta->prev.Set(&it->second);
|
||||
MG_ASSERT(it != acc.end(), "Invalid Edge accessor!");
|
||||
edge = EdgeRef(&*it);
|
||||
delta->prev.Set(&*it);
|
||||
}
|
||||
|
||||
if (from_is_local) {
|
||||
CreateAndLinkDelta(transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex_id, edge);
|
||||
from_vertex->second.out_edges.emplace_back(edge_type, to_vertex_id, edge);
|
||||
from_vertex->out_edges.emplace_back(edge_type, to_vertex_id, edge);
|
||||
}
|
||||
if (to_is_local) {
|
||||
CreateAndLinkDelta(transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex_id, edge);
|
||||
to_vertex->second.in_edges.emplace_back(edge_type, from_vertex_id, edge);
|
||||
to_vertex->in_edges.emplace_back(edge_type, from_vertex_id, edge);
|
||||
}
|
||||
// Increment edge count.
|
||||
++shard_->edge_count_;
|
||||
@@ -538,56 +540,57 @@ ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, V
|
||||
&shard_->indices_, config_);
|
||||
}
|
||||
|
||||
ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id,
|
||||
const Gid edge_id) {
|
||||
VertexContainer::value_type *from_vertex{nullptr};
|
||||
VertexContainer::value_type *to_vertex{nullptr};
|
||||
Result<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id,
|
||||
const Gid edge_id) {
|
||||
Vertex *from_vertex{nullptr};
|
||||
Vertex *to_vertex{nullptr};
|
||||
|
||||
auto &vertices = shard_->vertices_;
|
||||
auto acc = shard_->vertices_.access();
|
||||
|
||||
const auto from_is_local = shard_->IsVertexBelongToShard(from_vertex_id);
|
||||
const auto to_is_local = shard_->IsVertexBelongToShard(to_vertex_id);
|
||||
|
||||
if (from_is_local) {
|
||||
auto it = vertices.find(from_vertex_id.primary_key);
|
||||
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
|
||||
from_vertex = &*it;
|
||||
auto it = acc.find(from_vertex_id.primary_key);
|
||||
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
|
||||
from_vertex = &it->vertex;
|
||||
}
|
||||
|
||||
if (to_is_local) {
|
||||
auto it = vertices.find(to_vertex_id.primary_key);
|
||||
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
|
||||
to_vertex = &*it;
|
||||
auto it = acc.find(to_vertex_id.primary_key);
|
||||
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
|
||||
to_vertex = &it->vertex;
|
||||
}
|
||||
|
||||
MG_ASSERT(from_is_local || to_is_local, "Trying to delete an edge without having a local vertex");
|
||||
|
||||
if (from_is_local) {
|
||||
if (!PrepareForWrite(transaction_, from_vertex)) {
|
||||
return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
return Error::SERIALIZATION_ERROR;
|
||||
}
|
||||
MG_ASSERT(!from_vertex->second.deleted, "Invalid database state!");
|
||||
MG_ASSERT(!from_vertex->deleted, "Invalid database state!");
|
||||
}
|
||||
if (to_is_local && to_vertex != from_vertex) {
|
||||
if (!PrepareForWrite(transaction_, to_vertex)) {
|
||||
return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
|
||||
return Error::SERIALIZATION_ERROR;
|
||||
}
|
||||
MG_ASSERT(!to_vertex->second.deleted, "Invalid database state!");
|
||||
MG_ASSERT(!to_vertex->deleted, "Invalid database state!");
|
||||
}
|
||||
|
||||
const auto edge_ref = std::invoke([edge_id, this]() -> EdgeRef {
|
||||
if (!config_.properties_on_edges) {
|
||||
return EdgeRef(edge_id);
|
||||
}
|
||||
auto res = shard_->edges_.find(edge_id);
|
||||
MG_ASSERT(res != shard_->edges_.end(), "Cannot find edge");
|
||||
return EdgeRef(&res->second);
|
||||
auto edge_acc = shard_->edges_.access();
|
||||
auto res = edge_acc.find(edge_id);
|
||||
MG_ASSERT(res != edge_acc.end(), "Cannot find edge");
|
||||
return EdgeRef(&*res);
|
||||
});
|
||||
|
||||
std::optional<EdgeTypeId> edge_type{};
|
||||
auto delete_edge_from_storage = [&edge_type, &edge_ref, this](std::vector<VertexData::EdgeLink> &edges) mutable {
|
||||
auto delete_edge_from_storage = [&edge_type, &edge_ref, this](std::vector<Vertex::EdgeLink> &edges) mutable {
|
||||
auto it = std::find_if(edges.begin(), edges.end(),
|
||||
[&edge_ref](const VertexData::EdgeLink &link) { return std::get<2>(link) == edge_ref; });
|
||||
[&edge_ref](const Vertex::EdgeLink &link) { return std::get<2>(link) == edge_ref; });
|
||||
if (config_.properties_on_edges) {
|
||||
MG_ASSERT(it != edges.end(), "Invalid database state!");
|
||||
} else if (it == edges.end()) {
|
||||
@@ -599,8 +602,8 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
|
||||
return true;
|
||||
};
|
||||
// NOLINTNEXTLINE(clang-analyzer-core.NonNullParamChecker)
|
||||
auto success_on_to = to_is_local ? delete_edge_from_storage(to_vertex->second.in_edges) : false;
|
||||
auto success_on_from = from_is_local ? delete_edge_from_storage(from_vertex->second.out_edges) : false;
|
||||
auto success_on_to = to_is_local ? delete_edge_from_storage(to_vertex->in_edges) : false;
|
||||
auto success_on_from = from_is_local ? delete_edge_from_storage(from_vertex->out_edges) : false;
|
||||
|
||||
if (config_.properties_on_edges) {
|
||||
// Because of the check above, we are sure that the vertex exists.
|
||||
@@ -626,10 +629,10 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
|
||||
MG_ASSERT(edge_type.has_value(), "Edge type is not determined");
|
||||
|
||||
if (from_is_local) {
|
||||
CreateAndLinkDelta(transaction_, &*from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref);
|
||||
CreateAndLinkDelta(transaction_, from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref);
|
||||
}
|
||||
if (to_is_local) {
|
||||
CreateAndLinkDelta(transaction_, &*to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref);
|
||||
CreateAndLinkDelta(transaction_, to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref);
|
||||
}
|
||||
|
||||
// Decrement edge count.
|
||||
@@ -691,41 +694,41 @@ void Shard::Accessor::Abort() {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
auto &[pk, vertex] = *prev.vertex;
|
||||
Delta *current = vertex.delta;
|
||||
auto *vertex = prev.vertex;
|
||||
Delta *current = vertex->delta;
|
||||
while (current != nullptr && current->commit_info->start_or_commit_timestamp == transaction_->start_timestamp) {
|
||||
switch (current->action) {
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
auto it = std::find(vertex.labels.begin(), vertex.labels.end(), current->label);
|
||||
MG_ASSERT(it != vertex.labels.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex.labels.rbegin());
|
||||
vertex.labels.pop_back();
|
||||
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
|
||||
MG_ASSERT(it != vertex->labels.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->labels.rbegin());
|
||||
vertex->labels.pop_back();
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_LABEL: {
|
||||
auto it = std::find(vertex.labels.begin(), vertex.labels.end(), current->label);
|
||||
MG_ASSERT(it == vertex.labels.end(), "Invalid database state!");
|
||||
vertex.labels.push_back(current->label);
|
||||
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
|
||||
MG_ASSERT(it == vertex->labels.end(), "Invalid database state!");
|
||||
vertex->labels.push_back(current->label);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::SET_PROPERTY: {
|
||||
vertex.properties.SetProperty(current->property.key, current->property.value);
|
||||
vertex->properties.SetProperty(current->property.key, current->property.value);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_IN_EDGE: {
|
||||
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex.in_edges.begin(), vertex.in_edges.end(), link);
|
||||
MG_ASSERT(it == vertex.in_edges.end(), "Invalid database state!");
|
||||
vertex.in_edges.push_back(link);
|
||||
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
|
||||
MG_ASSERT(it == vertex->in_edges.end(), "Invalid database state!");
|
||||
vertex->in_edges.push_back(link);
|
||||
break;
|
||||
}
|
||||
case Delta::Action::ADD_OUT_EDGE: {
|
||||
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex.out_edges.begin(), vertex.out_edges.end(), link);
|
||||
MG_ASSERT(it == vertex.out_edges.end(), "Invalid database state!");
|
||||
vertex.out_edges.push_back(link);
|
||||
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
|
||||
MG_ASSERT(it == vertex->out_edges.end(), "Invalid database state!");
|
||||
vertex->out_edges.push_back(link);
|
||||
// Increment edge count. We only increment the count here because
|
||||
// the information in `ADD_IN_EDGE` and `Edge/RECREATE_OBJECT` is
|
||||
// redundant. Also, `Edge/RECREATE_OBJECT` isn't available when
|
||||
@@ -734,21 +737,21 @@ void Shard::Accessor::Abort() {
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_IN_EDGE: {
|
||||
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex.in_edges.begin(), vertex.in_edges.end(), link);
|
||||
MG_ASSERT(it != vertex.in_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex.in_edges.rbegin());
|
||||
vertex.in_edges.pop_back();
|
||||
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
|
||||
MG_ASSERT(it != vertex->in_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->in_edges.rbegin());
|
||||
vertex->in_edges.pop_back();
|
||||
break;
|
||||
}
|
||||
case Delta::Action::REMOVE_OUT_EDGE: {
|
||||
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex.out_edges.begin(), vertex.out_edges.end(), link);
|
||||
MG_ASSERT(it != vertex.out_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex.out_edges.rbegin());
|
||||
vertex.out_edges.pop_back();
|
||||
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
|
||||
current->vertex_edge.edge};
|
||||
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
|
||||
MG_ASSERT(it != vertex->out_edges.end(), "Invalid database state!");
|
||||
std::swap(*it, *vertex->out_edges.rbegin());
|
||||
vertex->out_edges.pop_back();
|
||||
// Decrement edge count. We only decrement the count here because
|
||||
// the information in `REMOVE_IN_EDGE` and `Edge/DELETE_OBJECT` is
|
||||
// redundant. Also, `Edge/DELETE_OBJECT` isn't available when edge
|
||||
@@ -757,20 +760,20 @@ void Shard::Accessor::Abort() {
|
||||
break;
|
||||
}
|
||||
case Delta::Action::DELETE_OBJECT: {
|
||||
vertex.deleted = true;
|
||||
shard_->deleted_vertices_.push_back(&pk);
|
||||
vertex->deleted = true;
|
||||
shard_->deleted_vertices_.push_back(vertex->keys.Keys());
|
||||
break;
|
||||
}
|
||||
case Delta::Action::RECREATE_OBJECT: {
|
||||
vertex.deleted = false;
|
||||
vertex->deleted = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
current = current->next;
|
||||
}
|
||||
vertex.delta = current;
|
||||
vertex->delta = current;
|
||||
if (current != nullptr) {
|
||||
current->prev.Set(prev.vertex);
|
||||
current->prev.Set(vertex);
|
||||
}
|
||||
|
||||
break;
|
||||
@@ -847,7 +850,10 @@ const std::string &Shard::EdgeTypeToName(EdgeTypeId edge_type) const {
|
||||
|
||||
bool Shard::CreateIndex(LabelId label, const std::optional<uint64_t> /*desired_commit_timestamp*/) {
|
||||
// TODO(jbajic) response should be different when label == primary_label
|
||||
return !(label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_));
|
||||
if (label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_.access())) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Shard::CreateIndex(LabelId label, PropertyId property,
|
||||
@@ -858,7 +864,7 @@ bool Shard::CreateIndex(LabelId label, PropertyId property,
|
||||
// Index already exists on primary key
|
||||
return false;
|
||||
}
|
||||
if (!indices_.label_property_index.CreateIndex(label, property, vertices_)) {
|
||||
if (!indices_.label_property_index.CreateIndex(label, property, vertices_.access())) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
@@ -965,12 +971,10 @@ void Shard::CollectGarbage(const io::Time current_time) {
|
||||
auto prev = delta.prev.Get();
|
||||
switch (prev.type) {
|
||||
case PreviousPtr::Type::VERTEX: {
|
||||
// Here we need to get pk from prev pointer, and that is why change
|
||||
// to the PrevPtr so it points to pair of pk and vertex
|
||||
auto &[pk, vertex] = *prev.vertex;
|
||||
vertex.delta = nullptr;
|
||||
if (vertex.deleted) {
|
||||
deleted_vertices_.push_back(&pk);
|
||||
Vertex *vertex = prev.vertex;
|
||||
vertex->delta = nullptr;
|
||||
if (vertex->deleted) {
|
||||
deleted_vertices_.push_back(vertex->keys.Keys());
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1017,13 +1021,15 @@ void Shard::CollectGarbage(const io::Time current_time) {
|
||||
RemoveObsoleteEntries(&indices_, clean_up_before_timestamp);
|
||||
}
|
||||
|
||||
for (const auto *vertex : deleted_vertices_) {
|
||||
MG_ASSERT(vertices_.erase(*vertex), "Invalid database state!");
|
||||
auto vertex_acc = vertices_.access();
|
||||
for (const auto &vertex : deleted_vertices_) {
|
||||
MG_ASSERT(vertex_acc.remove(vertex), "Invalid database state!");
|
||||
}
|
||||
deleted_vertices_.clear();
|
||||
{
|
||||
auto edge_acc = edges_.access();
|
||||
for (auto edge : deleted_edges_) {
|
||||
MG_ASSERT(edges_.erase(edge), "Invalid database state!");
|
||||
MG_ASSERT(edge_acc.remove(edge), "Invalid database state!");
|
||||
}
|
||||
}
|
||||
deleted_edges_.clear();
|
||||
|
||||
@@ -31,16 +31,19 @@
|
||||
#include "storage/v3/indices.hpp"
|
||||
#include "storage/v3/isolation_level.hpp"
|
||||
#include "storage/v3/key_store.hpp"
|
||||
#include "storage/v3/lexicographically_ordered_vertex.hpp"
|
||||
#include "storage/v3/mvcc.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/shard_operation_result.hpp"
|
||||
#include "storage/v3/transaction.hpp"
|
||||
#include "storage/v3/vertex.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/vertex_id.hpp"
|
||||
#include "storage/v3/vertices_skip_list.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/file_locker.hpp"
|
||||
@@ -63,7 +66,7 @@ namespace memgraph::storage::v3 {
|
||||
/// An instance of this will be usually be wrapped inside VerticesIterable for
|
||||
/// generic, public use.
|
||||
class AllVerticesIterable final {
|
||||
VertexContainer *vertices_accessor_;
|
||||
VerticesSkipList::Accessor vertices_accessor_;
|
||||
Transaction *transaction_;
|
||||
View view_;
|
||||
Indices *indices_;
|
||||
@@ -74,10 +77,10 @@ class AllVerticesIterable final {
|
||||
public:
|
||||
class Iterator final {
|
||||
AllVerticesIterable *self_;
|
||||
VertexContainer::iterator it_;
|
||||
VerticesSkipList::Iterator it_;
|
||||
|
||||
public:
|
||||
Iterator(AllVerticesIterable *self, VertexContainer::iterator it);
|
||||
Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it);
|
||||
|
||||
VertexAccessor operator*() const;
|
||||
|
||||
@@ -88,17 +91,17 @@ class AllVerticesIterable final {
|
||||
bool operator!=(const Iterator &other) const { return !(*this == other); }
|
||||
};
|
||||
|
||||
AllVerticesIterable(VertexContainer &vertices_accessor, Transaction *transaction, View view, Indices *indices,
|
||||
Config::Items config, const VertexValidator &vertex_validator)
|
||||
: vertices_accessor_(&vertices_accessor),
|
||||
AllVerticesIterable(VerticesSkipList::Accessor vertices_accessor, Transaction *transaction, View view,
|
||||
Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
|
||||
: vertices_accessor_(std::move(vertices_accessor)),
|
||||
transaction_(transaction),
|
||||
view_(view),
|
||||
indices_(indices),
|
||||
config_(config),
|
||||
vertex_validator_{&vertex_validator} {}
|
||||
|
||||
Iterator begin() { return {this, vertices_accessor_->begin()}; }
|
||||
Iterator end() { return {this, vertices_accessor_->end()}; }
|
||||
Iterator begin() { return {this, vertices_accessor_.begin()}; }
|
||||
Iterator end() { return {this, vertices_accessor_.end()}; }
|
||||
};
|
||||
|
||||
/// Generic access to different kinds of vertex iterations.
|
||||
@@ -204,14 +207,14 @@ class Shard final {
|
||||
|
||||
public:
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<VertexAccessor> CreateVertexAndValidate(
|
||||
ShardOperationResult<VertexAccessor> CreateVertexAndValidate(
|
||||
const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
|
||||
const std::vector<std::pair<PropertyId, PropertyValue>> &properties);
|
||||
|
||||
std::optional<VertexAccessor> FindVertex(std::vector<PropertyValue> primary_key, View view);
|
||||
|
||||
VerticesIterable Vertices(View view) {
|
||||
return VerticesIterable(AllVerticesIterable(shard_->vertices_, transaction_, view, &shard_->indices_,
|
||||
return VerticesIterable(AllVerticesIterable(shard_->vertices_.access(), transaction_, view, &shard_->indices_,
|
||||
shard_->config_.items, shard_->vertex_validator_));
|
||||
}
|
||||
|
||||
@@ -234,17 +237,18 @@ class Shard final {
|
||||
int64_t ApproximateVertexCount(LabelId label) const {
|
||||
return shard_->indices_.label_index.ApproximateVertexCount(label);
|
||||
}
|
||||
|
||||
/// Return approximate number of vertices with the given label and property.
|
||||
/// Note that this is always an over-estimate and never an under-estimate.
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
|
||||
return shard_->indices_.label_property_index.VertexCount(label, property);
|
||||
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property);
|
||||
}
|
||||
|
||||
/// Return approximate number of vertices with the given label and the given
|
||||
/// value for the given property.
|
||||
/// Note that this is always an over-estimate and never an under-estimate.
|
||||
/// value for the given property. Note that this is always an over-estimate
|
||||
/// and never an under-estimate.
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
|
||||
return shard_->indices_.label_property_index.VertexCount(label, property, value);
|
||||
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, value);
|
||||
}
|
||||
|
||||
/// Return approximate number of vertices with the given label and value for
|
||||
@@ -253,24 +257,24 @@ class Shard final {
|
||||
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
|
||||
const std::optional<utils::Bound<PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<PropertyValue>> &upper) const {
|
||||
return shard_->indices_.label_property_index.VertexCount(label, property, lower, upper);
|
||||
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
|
||||
}
|
||||
|
||||
/// @return Accessor to the deleted vertex if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<std::optional<VertexAccessor>> DeleteVertex(VertexAccessor *vertex);
|
||||
Result<std::optional<VertexAccessor>> DeleteVertex(VertexAccessor *vertex);
|
||||
|
||||
/// @return Accessor to the deleted vertex and deleted edges if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachDeleteVertex(
|
||||
Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachDeleteVertex(
|
||||
VertexAccessor *vertex);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid);
|
||||
Result<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid);
|
||||
|
||||
/// Accessor to the deleted edge if a deletion took place, std::nullopt otherwise
|
||||
/// @throw std::bad_alloc
|
||||
ShardResult<std::optional<EdgeAccessor>> DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id, Gid edge_id);
|
||||
Result<std::optional<EdgeAccessor>> DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id, Gid edge_id);
|
||||
|
||||
LabelId NameToLabel(std::string_view name) const;
|
||||
|
||||
@@ -306,6 +310,9 @@ class Shard final {
|
||||
void Abort();
|
||||
|
||||
private:
|
||||
/// @throw std::bad_alloc
|
||||
VertexAccessor CreateVertex(Gid gid, LabelId primary_label);
|
||||
|
||||
Shard *shard_;
|
||||
Transaction *transaction_;
|
||||
Config::Items config_;
|
||||
@@ -371,8 +378,8 @@ class Shard final {
|
||||
// The shard's range is [min, max)
|
||||
PrimaryKey min_primary_key_;
|
||||
std::optional<PrimaryKey> max_primary_key_;
|
||||
VertexContainer vertices_;
|
||||
EdgeContainer edges_;
|
||||
VerticesSkipList vertices_;
|
||||
utils::SkipList<Edge> edges_;
|
||||
// Even though the edge count is already kept in the `edges_` SkipList, the
|
||||
// list is used only when properties are enabled for edges. Because of that we
|
||||
// keep a separate count of edges that is always updated.
|
||||
@@ -390,7 +397,7 @@ class Shard final {
|
||||
|
||||
// Vertices that are logically deleted but still have to be removed from
|
||||
// indices before removing them from the main storage.
|
||||
std::list<const PrimaryKey *> deleted_vertices_;
|
||||
std::list<PrimaryKey> deleted_vertices_;
|
||||
|
||||
// Edges that are logically deleted and wait to be removed from the main
|
||||
// storage.
|
||||
|
||||
@@ -190,12 +190,6 @@ class ShardManager {
|
||||
});
|
||||
}
|
||||
|
||||
void BlockOnQuiescence() {
|
||||
for (const auto &worker : workers_) {
|
||||
worker.BlockOnQuiescence();
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
io::Io<IoImpl> io_;
|
||||
std::vector<shard_worker::Queue> workers_;
|
||||
|
||||
26
src/storage/v3/shard_operation_result.hpp
Normal file
26
src/storage/v3/shard_operation_result.hpp
Normal file
@@ -0,0 +1,26 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <variant>
|
||||
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
using ResultErrorType = std::variant<SchemaViolation, Error>;
|
||||
|
||||
template <typename TValue>
|
||||
using ShardOperationResult = utils::BasicResult<ResultErrorType, TValue>;
|
||||
|
||||
} // namespace memgraph::storage::v3
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,9 @@
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
|
||||
template <typename>
|
||||
constexpr auto kAlwaysFalse = false;
|
||||
|
||||
class ShardRsm {
|
||||
std::unique_ptr<Shard> shard_;
|
||||
|
||||
|
||||
@@ -80,9 +80,6 @@ struct QueueInner {
|
||||
// starvation by sometimes randomizing priorities, rather than following a strict
|
||||
// prioritization.
|
||||
std::deque<Message> queue;
|
||||
|
||||
uint64_t submitted = 0;
|
||||
uint64_t calls_to_pop = 0;
|
||||
};
|
||||
|
||||
/// There are two reasons to implement our own Queue instead of using
|
||||
@@ -98,8 +95,6 @@ class Queue {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->submitted++;
|
||||
|
||||
inner_->queue.emplace_back(std::forward<Message>(message));
|
||||
} // lock dropped before notifying condition variable
|
||||
|
||||
@@ -110,9 +105,6 @@ class Queue {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->calls_to_pop++;
|
||||
inner_->cv.notify_all();
|
||||
|
||||
while (inner_->queue.empty()) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
@@ -122,15 +114,6 @@ class Queue {
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
void BlockOnQuiescence() const {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
while (inner_->calls_to_pop <= inner_->submitted) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// A ShardWorker owns Raft<ShardRsm> instances. receives messages from the ShardManager.
|
||||
@@ -139,6 +122,7 @@ class ShardWorker {
|
||||
io::Io<IoImpl> io_;
|
||||
Queue queue_;
|
||||
std::priority_queue<std::pair<Time, uuid>, std::vector<std::pair<Time, uuid>>, std::greater<>> cron_schedule_;
|
||||
Time next_cron_ = Time::min();
|
||||
std::map<uuid, ShardRaft<IoImpl>> rsm_map_;
|
||||
|
||||
bool Process(ShutDown && /* shut_down */) { return false; }
|
||||
@@ -191,7 +175,10 @@ class ShardWorker {
|
||||
return;
|
||||
}
|
||||
|
||||
auto rsm_io = io_.ForkLocal(to_init.uuid);
|
||||
auto rsm_io = io_.ForkLocal();
|
||||
auto io_addr = rsm_io.GetAddress();
|
||||
io_addr.unique_id = to_init.uuid;
|
||||
rsm_io.SetAddress(io_addr);
|
||||
|
||||
// TODO(tyler) get peers from Coordinator in HeartbeatResponse
|
||||
std::vector<Address> rsm_peers = {};
|
||||
@@ -221,12 +208,15 @@ class ShardWorker {
|
||||
~ShardWorker() = default;
|
||||
|
||||
void Run() {
|
||||
bool should_continue = true;
|
||||
while (should_continue) {
|
||||
while (true) {
|
||||
Message message = queue_.Pop();
|
||||
|
||||
should_continue =
|
||||
const bool should_continue =
|
||||
std::visit([&](auto &&msg) { return Process(std::forward<decltype(msg)>(msg)); }, std::move(message));
|
||||
|
||||
if (!should_continue) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user