Compare commits

..

2 Commits

Author SHA1 Message Date
János Benjamin Antal
18edf17496 Some changes 2022-11-16 23:55:24 +01:00
gvolfing
eeb7ab6272 Initial sketch for making the simulator deterministic. 2022-11-16 18:40:12 +01:00
178 changed files with 9334 additions and 12842 deletions

View File

@@ -99,7 +99,7 @@ jobs:
echo ${file}
if [[ ${file} == *.py ]]; then
python3 -m black --check --diff ${file}
python3 -m isort --check-only --profile "black" --diff ${file}
python3 -m isort --check-only --diff ${file}
fi
done

View File

@@ -14,7 +14,6 @@ repos:
hooks:
- id: isort
name: isort (python)
args: ["--profile", "black"]
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: v13.0.0
hooks:

View File

@@ -4,7 +4,6 @@ cmake_minimum_required(VERSION 3.8)
# !! IMPORTANT !! run ./project_root/init.sh before cmake command
# to download dependencies
if(NOT UNIX)
message(FATAL_ERROR "Unsupported operating system.")
endif()
@@ -19,6 +18,7 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM TRUE)
find_program(CCACHE_FOUND ccache)
option(USE_CCACHE "ccache:" ON)
message(STATUS "CCache: ${USE_CCACHE}")
if(CCACHE_FOUND AND USE_CCACHE)
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
@@ -28,7 +28,8 @@ endif(CCACHE_FOUND AND USE_CCACHE)
# NOTE: must be choosen before use of project() or enable_language()
find_program(CLANG_FOUND clang)
find_program(CLANGXX_FOUND clang++)
if (CLANG_FOUND AND CLANGXX_FOUND)
if(CLANG_FOUND AND CLANGXX_FOUND)
set(CMAKE_C_COMPILER ${CLANG_FOUND})
set(CMAKE_CXX_COMPILER ${CLANGXX_FOUND})
else()
@@ -36,12 +37,11 @@ else()
endif()
# -----------------------------------------------------------------------------
project(memgraph)
# Install licenses.
install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/licenses/
DESTINATION share/doc/memgraph)
DESTINATION share/doc/memgraph)
# For more information about how to release a new version of Memgraph, see
# `release/README.md`.
@@ -61,61 +61,65 @@ set(MEMGRAPH_OVERRIDE_VERSION "")
set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "")
# Variables used to generate the versions.
if (MG_ENTERPRISE)
if(MG_ENTERPRISE)
set(get_version_offering "")
else()
set(get_version_offering "--open-source")
endif()
set(get_version_script "${CMAKE_CURRENT_SOURCE_DIR}/release/get_version.py")
# Get version that should be used in the binary.
execute_process(
OUTPUT_VARIABLE MEMGRAPH_VERSION
RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
COMMAND "${get_version_script}" ${get_version_offering}
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE MEMGRAPH_VERSION
RESULT_VARIABLE MEMGRAPH_VERSION_RESULT
COMMAND "${get_version_script}" ${get_version_offering}
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_RESULT AND NOT MEMGRAPH_VERSION_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph version.")
message(FATAL_ERROR "Unable to get Memgraph version.")
else()
MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
MESSAGE(STATUS "Memgraph version: ${MEMGRAPH_VERSION}")
endif()
# Get version that should be used in the DEB package.
execute_process(
OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
COMMAND "${get_version_script}" ${get_version_offering}
--variant deb
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE MEMGRAPH_VERSION_DEB
RESULT_VARIABLE MEMGRAPH_VERSION_DEB_RESULT
COMMAND "${get_version_script}" ${get_version_offering}
--variant deb
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_DEB_RESULT AND NOT MEMGRAPH_VERSION_DEB_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
message(FATAL_ERROR "Unable to get Memgraph DEB version.")
else()
MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
MESSAGE(STATUS "Memgraph DEB version: ${MEMGRAPH_VERSION_DEB}")
endif()
# Get version that should be used in the RPM package.
execute_process(
OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
COMMAND "${get_version_script}" ${get_version_offering}
--variant rpm
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_CURRENT_SOURCE_DIR}"
OUTPUT_VARIABLE MEMGRAPH_VERSION_RPM
RESULT_VARIABLE MEMGRAPH_VERSION_RPM_RESULT
COMMAND "${get_version_script}" ${get_version_offering}
--variant rpm
"${MEMGRAPH_OVERRIDE_VERSION}"
"${MEMGRAPH_OVERRIDE_VERSION_SUFFIX}"
"--memgraph-root-dir"
"${CMAKE_CURRENT_SOURCE_DIR}"
)
if(MEMGRAPH_VERSION_RPM_RESULT AND NOT MEMGRAPH_VERSION_RPM_RESULT EQUAL 0)
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
message(FATAL_ERROR "Unable to get Memgraph RPM version.")
else()
MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
MESSAGE(STATUS "Memgraph RPM version: ${MEMGRAPH_VERSION_RPM}")
endif()
# We want the above variables to be updated each time something is committed to
@@ -135,22 +139,24 @@ endif()
# unnecessary recalculations of the release version. The release version only
# changes on every `git commit` or `git checkout`. That is why we watch the
# following files for changes:
# - `.git/HEAD` -> changes each time a `git checkout` is issued
# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
# you are on a branch) and you have to monitor the file of the specific
# branch to detect when a `git commit` was issued
# - `.git/HEAD` -> changes each time a `git checkout` is issued
# - `.git/refs/heads/...` -> the value in `.git/HEAD` is a branch name (when
# you are on a branch) and you have to monitor the file of the specific
# branch to detect when a `git commit` was issued
# More details about the contents of the `.git` directory and the specific
# files used can be seen here:
# https://git-scm.com/book/en/v2/Git-Internals-Git-References
set(git_directory "${CMAKE_SOURCE_DIR}/.git")
if (EXISTS "${git_directory}")
if(EXISTS "${git_directory}")
set_property(DIRECTORY APPEND PROPERTY
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
CMAKE_CONFIGURE_DEPENDS "${git_directory}/HEAD")
file(STRINGS "${git_directory}/HEAD" git_head_data)
if (git_head_data MATCHES "^ref: ")
if(git_head_data MATCHES "^ref: ")
string(SUBSTRING "${git_head_data}" 5 -1 git_head_ref)
set_property(DIRECTORY APPEND PROPERTY
CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
CMAKE_CONFIGURE_DEPENDS "${git_directory}/${git_head_ref}")
endif()
endif()
@@ -159,16 +165,19 @@ endif()
# setup CMake module path, defines path for include() and find_package()
# https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cmake)
# custom function definitions
include(functions)
# -----------------------------------------------------------------------------
# We want out of source builds, so that cmake generated files don't get mixed
# with source files. This allows for easier clean up.
disallow_in_source_build()
add_custom_target(clean_all
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/clean_all.cmake
COMMENT "Removing all files in ${CMAKE_BINARY_DIR}")
# -----------------------------------------------------------------------------
# build flags -----------------------------------------------------------------
@@ -179,18 +188,19 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
# c99-designator is disabled because of required mixture of designated and
# non-designated initializers in Python Query Module code (`py_module.cpp`).
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-Werror=switch -Werror=switch-bool -Werror=implicit-fallthrough \
-Werror=return-type \
-Werror=switch -Werror=switch-bool -Werror=return-type \
-Werror=return-stack-address \
-Wno-c99-designator \
-DBOOST_STACKTRACE_USE_ADDR2LINE \
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
# Don't omit frame pointer in RelWithDebInfo, for additional callchain debug.
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
"${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
# Statically link libgcc and libstdc++, the GCC allows this according to:
# https://gcc.gnu.org/onlinedocs/gcc-10.2.0/libstdc++/manual/manual/license.html
@@ -208,42 +218,47 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
SET(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -pthread")
#debug flags
# debug flags
set(PREFERRED_DEBUGGER "gdb" CACHE STRING
"Tunes the debug output for your preferred debugger (gdb or lldb).")
if ("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
"${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
elseif ("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
"${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
"Tunes the debug output for your preferred debugger (gdb or lldb).")
if("${PREFERRED_DEBUGGER}" STREQUAL "gdb" AND
"${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|GNU")
set(CMAKE_CXX_FLAGS_DEBUG "-ggdb")
elseif("${PREFERRED_DEBUGGER}" STREQUAL "lldb" AND
"${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
set(CMAKE_CXX_FLAGS_DEBUG "-glldb")
else()
message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
"'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
set(CMAKE_CXX_FLAGS_DEBUG "-g")
message(WARNING "Unable to tune for PREFERRED_DEBUGGER: "
"'${PREFERRED_DEBUGGER}' with compiler: '${CMAKE_CXX_COMPILER_ID}'")
set(CMAKE_CXX_FLAGS_DEBUG "-g")
endif()
# -----------------------------------------------------------------------------
# default build type is debug
if (NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Debug")
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Debug")
endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
set(MG_ARCH "x86_64" CACHE STRING "Host architecture to build Memgraph on. Supported values are x86_64 (default), ARM64.")
# setup external dependencies -------------------------------------------------
# threading
find_package(Threads REQUIRED)
# optional readline
option(USE_READLINE "Use GNU Readline library if available (default ON). \
Set this to OFF to prevent linking with Readline even if it is available." ON)
if (USE_READLINE)
if(USE_READLINE)
find_package(Readline)
if (READLINE_FOUND)
if(READLINE_FOUND)
add_definitions(-DHAS_READLINE)
endif()
endif()
@@ -259,24 +274,27 @@ option(ASAN "Build with Address Sanitizer. To get a reasonable performance optio
option(TSAN "Build with Thread Sanitizer. To get a reasonable performance option should be used only in Release or RelWithDebInfo build " OFF)
option(UBSAN "Build with Undefined Behaviour Sanitizer" OFF)
if (TEST_COVERAGE)
if(TEST_COVERAGE)
string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
if (NOT lower_build_type STREQUAL "debug")
if(NOT lower_build_type STREQUAL "debug")
message(FATAL_ERROR "Generating test coverage unsupported in non Debug builds. Current build type is '${CMAKE_BUILD_TYPE}'")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
endif()
if (MG_ENTERPRISE)
if(MG_ENTERPRISE)
add_definitions(-DMG_ENTERPRISE)
endif()
set(ENABLE_JEMALLOC ON)
if (ASAN)
if(ASAN)
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
set(ENABLE_JEMALLOC OFF)
# Enable Addres sanitizer and get nicer stack traces in error messages.
# NOTE: AddressSanitizer uses llvm-symbolizer binary from the Clang
# distribution to symbolize the stack traces (note that ideally the
@@ -285,19 +303,20 @@ if (ASAN)
# provide it in separate ASAN_SYMBOLIZER_PATH environment variable.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
# To detect Stack-use-after-return bugs set run-time flag:
# ASAN_OPTIONS=detect_stack_use_after_return=1
# ASAN_OPTIONS=detect_stack_use_after_return=1
# To check initialization order bugs set run-time flag:
# ASAN_OPTIONS=check_initialization_order=true
# This mode reports an error if initializer for a global variable accesses
# dynamically initialized global from another translation unit, which is
# not yet initialized
# ASAN_OPTIONS=strict_init_order=true
# This mode reports an error if initializer for a global variable accesses
# any dynamically initialized global from another translation unit.
# ASAN_OPTIONS=check_initialization_order=true
# This mode reports an error if initializer for a global variable accesses
# dynamically initialized global from another translation unit, which is
# not yet initialized
# ASAN_OPTIONS=strict_init_order=true
# This mode reports an error if initializer for a global variable accesses
# any dynamically initialized global from another translation unit.
endif()
if (TSAN)
if(TSAN)
# ThreadSanitizer generally requires all code to be compiled with -fsanitize=thread.
# If some code (e.g. dynamic libraries) is not compiled with the flag, it can
# lead to false positive race reports, false negative race reports and/or
@@ -310,19 +329,21 @@ if (TSAN)
# (unwinding stack on each memory access is too expensive).
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread")
# By default ThreadSanitizer uses addr2line utility to symbolize reports.
# llvm-symbolizer is faster, consumes less memory and produces much better
# reports. To use it set runtime flag:
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
# TSAN_OPTIONS="extern-symbolizer-path=~/llvm-symbolizer"
# For more runtime flags see: https://github.com/google/sanitizers/wiki/ThreadSanitizerFlags
endif()
if (UBSAN)
if(UBSAN)
# Compile with UBSAN but disable vptr check. This is disabled because it
# requires linking with clang++ to make sure C++ specific parts of the
# runtime library and c++ standard libraries are present.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fno-omit-frame-pointer -fno-sanitize=vptr")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined -fno-sanitize=vptr")
# Run program with environment variable UBSAN_OPTIONS=print_stacktrace=1.
# Make sure llvm-symbolizer binary is in path.
# To make the program abort on undefined behavior, use UBSAN_OPTIONS=halt_on_error=1.
@@ -341,7 +362,7 @@ add_subdirectory(release)
option(MG_ENABLE_TESTING "Set this to OFF to disable building test binaries" ON)
message(STATUS "MG_ENABLE_TESTING: ${MG_ENABLE_TESTING}")
if (MG_ENABLE_TESTING)
if(MG_ENABLE_TESTING)
enable_testing()
add_subdirectory(tests)
endif()

View File

@@ -171,7 +171,7 @@ benchmark_tag="v1.6.0"
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
# google test
googletest_tag="release-1.12.1"
googletest_tag="release-1.8.0"
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
# libbcrypt

View File

@@ -21,7 +21,6 @@ add_subdirectory(auth)
add_subdirectory(parser)
add_subdirectory(expr)
add_subdirectory(coordinator)
add_subdirectory(functions)
if (MG_ENTERPRISE)
add_subdirectory(audit)

View File

@@ -1,68 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
#include <string_view>
namespace memgraph::common {
enum class ErrorCode : uint8_t {
SERIALIZATION_ERROR,
NONEXISTENT_OBJECT,
DELETED_OBJECT,
VERTEX_HAS_EDGES,
PROPERTIES_DISABLED,
VERTEX_ALREADY_INSERTED,
// Schema Violations
SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL,
SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
OBJECT_NOT_FOUND,
};
constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
switch (code) {
case ErrorCode::SERIALIZATION_ERROR:
return "SERIALIZATION_ERROR";
case ErrorCode::NONEXISTENT_OBJECT:
return "NONEXISTENT_OBJECT";
case ErrorCode::DELETED_OBJECT:
return "DELETED_OBJECT";
case ErrorCode::VERTEX_HAS_EDGES:
return "VERTEX_HAS_EDGES";
case ErrorCode::PROPERTIES_DISABLED:
return "PROPERTIES_DISABLED";
case ErrorCode::VERTEX_ALREADY_INSERTED:
return "VERTEX_ALREADY_INSERTED";
case ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
return "SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL";
case ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
return "SCHEMA_VERTEX_PROPERTY_WRONG_TYPE";
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
return "SCHEMA_VERTEX_UPDATE_PRIMARY_KEY";
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
return "SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL";
case ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
return "SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY";
case ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
case ErrorCode::OBJECT_NOT_FOUND:
return "OBJECT_NOT_FOUND";
}
}
} // namespace memgraph::common

View File

@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
}
case Signature::Route: {
if constexpr (bolt_minor >= 3) {
return HandleRoute<TSession>(session);
if (signature == Signature::Route) return HandleRoute<TSession>(session);
} else {
spdlog::trace("Supported only in bolt v4.3");
return State::Close;

View File

@@ -71,9 +71,6 @@ struct QueueInner {
// starvation by sometimes randomizing priorities, rather than following a strict
// prioritization.
std::deque<Message> queue;
uint64_t submitted = 0;
uint64_t calls_to_pop = 0;
};
/// There are two reasons to implement our own Queue instead of using
@@ -89,8 +86,6 @@ class Queue {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
inner_->submitted++;
inner_->queue.emplace_back(std::move(message));
} // lock dropped before notifying condition variable
@@ -101,9 +96,6 @@ class Queue {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
inner_->calls_to_pop++;
inner_->cv.notify_all();
while (inner_->queue.empty()) {
inner_->cv.wait(lock);
}
@@ -113,15 +105,6 @@ class Queue {
return message;
}
void BlockOnQuiescence() const {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
while (inner_->calls_to_pop <= inner_->submitted) {
inner_->cv.wait(lock);
}
}
};
/// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
@@ -146,7 +129,9 @@ class CoordinatorWorker {
public:
CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
: io_(std::move(io)), queue_(std::move(queue)), coordinator_{std::move(io_), {}, std::move(coordinator)} {}
: io_(std::move(io)),
queue_(std::move(queue)),
coordinator_{std::move(io_.ForkLocal()), {}, std::move(coordinator)} {}
CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
@@ -155,12 +140,15 @@ class CoordinatorWorker {
~CoordinatorWorker() = default;
void Run() {
bool should_continue = true;
while (should_continue) {
while (true) {
Message message = queue_.Pop();
should_continue = std::visit([this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); },
std::move(message));
const bool should_continue = std::visit(
[this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); }, std::move(message));
if (!should_continue) {
return;
}
}
}
};

View File

@@ -228,7 +228,7 @@ Hlc ShardMap::IncrementShardMapVersion() noexcept {
return shard_map_version;
}
// TODO(antaljanosbenjamin) use a single map for all name id
// TODO(antaljanosbenjamin) use a single map for all name id
// mapping and a single counter to maintain the next id
std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
std::unordered_map<uint64_t, std::string> id_to_names;
@@ -248,25 +248,6 @@ std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
return boost::uuids::uuid{0,
0,
0,
0,
0,
0,
0,
0,
static_cast<unsigned char>(shard_id >> 56U),
static_cast<unsigned char>(shard_id >> 48U),
static_cast<unsigned char>(shard_id >> 40U),
static_cast<unsigned char>(shard_id >> 32U),
static_cast<unsigned char>(shard_id >> 24U),
static_cast<unsigned char>(shard_id >> 16U),
static_cast<unsigned char>(shard_id >> 8U),
static_cast<unsigned char>(shard_id)};
}
std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
std::set<boost::uuids::uuid> initialized) {
std::vector<ShardToInitialize> ret{};
@@ -283,11 +264,10 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
bool machine_contains_shard = false;
for (auto &aas : shard.peers) {
for (auto &aas : shard) {
if (initialized.contains(aas.address.unique_id)) {
machine_contains_shard = true;
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
mutated = true;
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
aas.status = Status::CONSENSUS_PARTICIPANT;
}
@@ -311,14 +291,11 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
}
}
if (!machine_contains_shard && shard.peers.size() < label_space.replication_factor) {
// increment version for each new uuid for deterministic creation
IncrementShardMapVersion();
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
Address address = storage_manager;
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
address.unique_id = NewShardUuid(shard_map_version.logical_id);
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
spdlog::info("assigning shard manager to shard");
@@ -337,7 +314,7 @@ std::vector<ShardToInitialize> ShardMap::AssignShards(Address storage_manager,
.status = Status::INITIALIZING,
};
shard.peers.emplace_back(aas);
shard.emplace_back(aas);
}
}
}
@@ -360,9 +337,9 @@ bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, cons
MG_ASSERT(!shards_in_map.contains(key));
MG_ASSERT(label_spaces.contains(label_id));
// Finding the ShardMetadata that the new PrimaryKey should map to.
// Finding the Shard that the new PrimaryKey should map to.
auto prev = std::prev(shards_in_map.upper_bound(key));
ShardMetadata duplicated_shard = prev->second;
Shard duplicated_shard = prev->second;
// Apply the split
shards_in_map[key] = duplicated_shard;
@@ -383,7 +360,7 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
labels.emplace(std::move(label_name), label_id);
PrimaryKey initial_key = SchemaToMinKey(schema);
ShardMetadata empty_shard = {};
Shard empty_shard = {};
Shards shards = {
{initial_key, empty_shard},
@@ -406,7 +383,6 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
void ShardMap::AddServer(Address server_address) {
// Find a random place for the server to plug in
}
std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
if (const auto it = labels.find(label); it != labels.end()) {
return it->second;
@@ -479,7 +455,7 @@ Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey
return shards;
}
ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
Shard ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
@@ -492,7 +468,7 @@ ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const Primar
return std::prev(label_space.shards.upper_bound(key))->second;
}
ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
Shard ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
MG_ASSERT(label_spaces.contains(label_id));
const auto &label_space = label_spaces.at(label_id);
@@ -556,12 +532,12 @@ EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new
bool ShardMap::ClusterInitialized() const {
for (const auto &[label_id, label_space] : label_spaces) {
for (const auto &[low_key, shard] : label_space.shards) {
if (shard.peers.size() < label_space.replication_factor) {
if (shard.size() < label_space.replication_factor) {
spdlog::info("label_space below desired replication factor");
return false;
}
for (const auto &aas : shard.peers) {
for (const auto &aas : shard) {
if (aas.status != Status::CONSENSUS_PARTICIPANT) {
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
return false;

View File

@@ -76,35 +76,8 @@ struct AddressAndStatus {
};
using PrimaryKey = std::vector<PropertyValue>;
struct ShardMetadata {
std::vector<AddressAndStatus> peers;
uint64_t version;
friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
using utils::print_helpers::operator<<;
in << "ShardMetadata { peers: ";
in << shard.peers;
in << " version: ";
in << shard.version;
in << " }";
return in;
}
friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
if (lhs.peers != rhs.peers) {
return lhs.peers < rhs.peers;
}
return lhs.version < rhs.version;
}
};
using Shards = std::map<PrimaryKey, ShardMetadata>;
using Shard = std::vector<AddressAndStatus>;
using Shards = std::map<PrimaryKey, Shard>;
using LabelName = std::string;
using PropertyName = std::string;
using EdgeTypeName = std::string;
@@ -126,7 +99,7 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
struct LabelSpace {
std::vector<SchemaProperty> schema;
// Maps between the smallest primary key stored in the shard and the shard
std::map<PrimaryKey, ShardMetadata> shards;
std::map<PrimaryKey, Shard> shards;
size_t replication_factor;
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
@@ -187,9 +160,9 @@ struct ShardMap {
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);

View File

@@ -17,4 +17,4 @@ target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser mg-functions)
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)

View File

@@ -24,7 +24,6 @@
#include "expr/exceptions.hpp"
#include "expr/interpret/frame.hpp"
#include "expr/semantic/symbol_table.hpp"
#include "functions/awesome_memgraph_functions.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::expr {
@@ -36,8 +35,8 @@ template <typename TypedValue, typename EvaluationContext, typename DbAccessor,
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
public:
ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
StorageView view)
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
DbAccessor *dba, StorageView view)
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
using ExpressionVisitor<TypedValue>::Visit;
@@ -101,28 +100,6 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
#undef BINARY_OPERATOR_VISITOR
#undef UNARY_OPERATOR_VISITOR
void HandleObjectAccessError(Error &shard_error, const std::string_view accessed_object) {
switch (shard_error) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to access {} on a deleted object.", accessed_object);
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to access {} from a node object doesn't exist.", accessed_object);
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
case Error::VERTEX_ALREADY_INSERTED:
case Error::OBJECT_NOT_FOUND:
throw ExpressionRuntimeException("Unexpected error when accessing {}.", accessed_object);
case Error::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
case Error::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
}
}
TypedValue Visit(AndOperator &op) override {
auto value1 = op.expression1_->Accept(*this);
if (value1.IsBool() && !value1.ValueBool()) {
@@ -419,7 +396,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
}
if (has_label.HasError()) {
HandleObjectAccessError(has_label.GetError().code, "labels");
switch (has_label.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
case Error::VERTEX_ALREADY_INSERTED:
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
}
}
return *has_label;
}
@@ -428,7 +415,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
return vertex.HasLabel(label);
auto has_label = vertex.HasLabel(label);
return !has_label;
}
TypedValue Visit(LabelsTest &labels_test) override {
@@ -491,7 +479,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(Function &function) override {
functions::FunctionContext<DbAccessor> function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
// Stack allocate evaluated arguments when there's a small number of them.
if (function.arguments_.size() <= 8) {
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
@@ -756,7 +744,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
}
if (maybe_prop.HasError()) {
HandleObjectAccessError(maybe_prop.GetError().code, "property");
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
case Error::VERTEX_ALREADY_INSERTED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
@@ -775,14 +773,24 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
}
if (maybe_prop.HasError()) {
HandleObjectAccessError(maybe_prop.GetError().code, "property");
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
case Error::VERTEX_ALREADY_INSERTED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
Frame *frame_;
Frame<TypedValue> *frame_;
const SymbolTable *symbol_table_;
const EvaluationContext *ctx_;
DbAccessor *dba_;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,12 +20,13 @@
namespace memgraph::expr {
template <typename TypedValue>
class Frame {
public:
/// Create a Frame of given size backed by a utils::NewDeleteResource()
explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
@@ -34,7 +35,6 @@ class Frame {
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
auto &elems() { return elems_; }
const auto &elems() const { return elems_; }
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
@@ -42,18 +42,4 @@ class Frame {
utils::pmr::vector<TypedValue> elems_;
};
class FrameWithValidity final : public Frame {
public:
explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {}
FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
bool IsValid() const noexcept { return is_valid_; }
void MakeValid() noexcept { is_valid_ = true; }
void MakeInvalid() noexcept { is_valid_ = false; }
private:
bool is_valid_;
};
} // namespace memgraph::expr

View File

@@ -1 +0,0 @@
add_library(mg-functions INTERFACE)

File diff suppressed because it is too large Load Diff

View File

@@ -15,13 +15,13 @@
#include <string>
#include <vector>
#include "common/errors.hpp"
#include "coordinator/shard_map.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/requests.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/view.hpp"
@@ -71,101 +71,106 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
}
}
communication::bolt::Vertex ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex,
const query::v2::RequestRouterInterface *request_router,
storage::v3::View /*view*/) {
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(
const query::v2::accessors::VertexAccessor &vertex, const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View /*view*/) {
auto id = communication::bolt::Id::FromUint(0);
auto labels = vertex.Labels();
std::vector<std::string> new_labels;
new_labels.reserve(labels.size());
for (const auto &label : labels) {
new_labels.push_back(request_router->LabelToName(label.id));
new_labels.push_back(shard_request_manager->LabelToName(label.id));
}
auto properties = vertex.Properties();
std::map<std::string, Value> new_properties;
for (const auto &[prop, property_value] : properties) {
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
new_properties[shard_request_manager->PropertyToName(prop)] = ToBoltValue(property_value);
}
return communication::bolt::Vertex{id, new_labels, new_properties};
}
communication::bolt::Edge ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge,
const query::v2::RequestRouterInterface *request_router,
storage::v3::View /*view*/) {
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(
const query::v2::accessors::EdgeAccessor &edge, const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View /*view*/) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto from = communication::bolt::Id::FromUint(0);
auto to = communication::bolt::Id::FromUint(0);
const auto &type = request_router->EdgeTypeToName(edge.EdgeType());
const auto &type = shard_request_manager->EdgeTypeToName(edge.EdgeType());
auto properties = edge.Properties();
std::map<std::string, Value> new_properties;
for (const auto &[prop, property_value] : properties) {
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
new_properties[shard_request_manager->PropertyToName(prop)] = ToBoltValue(property_value);
}
return communication::bolt::Edge{id, from, to, type, new_properties};
}
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path & /*edge*/,
const query::v2::RequestRouterInterface * /*request_router*/,
storage::v3::View /*view*/) {
storage::v3::Result<communication::bolt::Path> ToBoltPath(
const query::v2::accessors::Path & /*edge*/, const msgs::ShardRequestManagerInterface * /*shard_request_manager*/,
storage::v3::View /*view*/) {
// TODO(jbajic) Fix bolt communication
MG_ASSERT(false, "Path is unimplemented!");
return {};
return {storage::v3::Error::DELETED_OBJECT};
}
Value ToBoltValue(const query::v2::TypedValue &value, const query::v2::RequestRouterInterface *request_router,
storage::v3::View view) {
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value,
const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
return {};
return Value();
case query::v2::TypedValue::Type::Bool:
return {value.ValueBool()};
return Value(value.ValueBool());
case query::v2::TypedValue::Type::Int:
return {value.ValueInt()};
return Value(value.ValueInt());
case query::v2::TypedValue::Type::Double:
return {value.ValueDouble()};
return Value(value.ValueDouble());
case query::v2::TypedValue::Type::String:
return {std::string(value.ValueString())};
return Value(std::string(value.ValueString()));
case query::v2::TypedValue::Type::List: {
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
auto value = ToBoltValue(v, request_router, view);
values.emplace_back(std::move(value));
auto maybe_value = ToBoltValue(v, shard_request_manager, view);
if (maybe_value.HasError()) return maybe_value.GetError();
values.emplace_back(std::move(*maybe_value));
}
return {std::move(values)};
return Value(std::move(values));
}
case query::v2::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto value = ToBoltValue(kv.second, request_router, view);
map.emplace(kv.first, std::move(value));
auto maybe_value = ToBoltValue(kv.second, shard_request_manager, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return {std::move(map)};
return Value(std::move(map));
}
case query::v2::TypedValue::Type::Vertex: {
auto vertex = ToBoltVertex(value.ValueVertex(), request_router, view);
return {std::move(vertex)};
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), shard_request_manager, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::v2::TypedValue::Type::Edge: {
auto edge = ToBoltEdge(value.ValueEdge(), request_router, view);
return {std::move(edge)};
auto maybe_edge = ToBoltEdge(value.ValueEdge(), shard_request_manager, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::v2::TypedValue::Type::Path: {
auto path = ToBoltPath(value.ValuePath(), request_router, view);
return {std::move(path)};
auto maybe_path = ToBoltPath(value.ValuePath(), shard_request_manager, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::v2::TypedValue::Type::Date:
return {value.ValueDate()};
return Value(value.ValueDate());
case query::v2::TypedValue::Type::LocalTime:
return {value.ValueLocalTime()};
return Value(value.ValueLocalTime());
case query::v2::TypedValue::Type::LocalDateTime:
return {value.ValueLocalDateTime()};
return Value(value.ValueLocalDateTime());
case query::v2::TypedValue::Type::Duration:
return {value.ValueDuration()};
return Value(value.ValueDuration());
}
}

View File

@@ -15,12 +15,11 @@
#include "communication/bolt/v1/value.hpp"
#include "coordinator/shard_map.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/view.hpp"
#include "utils/result.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
@@ -32,37 +31,40 @@ namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param query::v2::RequestRouterInterface *request_router getting label and property names.
/// @param msgs::ShardRequestManagerInterface *shard_request_manager getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
communication::bolt::Vertex ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const query::v2::RequestRouterInterface *request_router,
storage::v3::View view);
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(
const storage::v3::VertexAccessor &vertex, const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param query::v2::RequestRouterInterface *request_router getting edge type and property names.
/// @param msgs::ShardRequestManagerInterface *shard_request_manager getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
communication::bolt::Edge ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(
const storage::v3::EdgeAccessor &edge, const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
/// @param msgs::ShardRequestManagerInterface *shard_request_manager ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path &path,
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
storage::v3::Result<communication::bolt::Path> ToBoltPath(
const query::v2::accessors::Path &path, const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
/// @param msgs::ShardRequestManagerInterface *shard_request_manager ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
communication::bolt::Value ToBoltValue(const query::v2::TypedValue &value,
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
storage::v3::Result<communication::bolt::Value> ToBoltValue(
const query::v2::TypedValue &value, const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
@@ -72,7 +74,8 @@ storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &val
communication::bolt::Value ToBoltValue(msgs::Value value);
communication::bolt::Value ToBoltValue(msgs::Value value, const query::v2::RequestRouterInterface *request_router,
communication::bolt::Value ToBoltValue(msgs::Value value,
const msgs::ShardRequestManagerInterface *shard_request_manager,
storage::v3::View view);
} // namespace memgraph::glue::v2

View File

@@ -20,8 +20,6 @@
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "utils/logging.hpp"
namespace memgraph::io {
struct PartialAddress {
@@ -60,39 +58,18 @@ struct Address {
uint16_t last_known_port;
static Address TestAddress(uint16_t port) {
MG_ASSERT(port <= 255);
return Address{
.unique_id = boost::uuids::uuid{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static_cast<unsigned char>(port)},
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
.last_known_port = port,
};
}
// NB: don't use this in test code because it is non-deterministic
static Address UniqueLocalAddress() {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
};
}
/// `Coordinator`s have constant UUIDs because there is at most one per ip/port pair.
Address ForkLocalCoordinator() {
return Address{
.unique_id = boost::uuids::uuid{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
.last_known_ip = last_known_ip,
.last_known_port = last_known_port,
};
}
/// `ShardManager`s have constant UUIDs because there is at most one per ip/port pair.
Address ForkLocalShardManager() {
return Address{
.unique_id = boost::uuids::uuid{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
.last_known_ip = last_known_ip,
.last_known_port = last_known_port,
};
}
/// Returns a new ID with the same IP and port but a unique UUID.
Address ForkUniqueAddress() {
return Address{

View File

@@ -35,13 +35,10 @@ class Shared {
std::optional<T> item_;
bool consumed_ = false;
bool waiting_ = false;
bool filled_ = false;
std::function<bool()> wait_notifier_ = nullptr;
std::function<void()> fill_notifier_ = nullptr;
std::function<bool()> simulator_notifier_ = nullptr;
public:
explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
: wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
Shared() = default;
Shared(Shared &&) = delete;
Shared &operator=(Shared &&) = delete;
@@ -67,7 +64,8 @@ class Shared {
waiting_ = true;
while (!item_) {
if (wait_notifier_) [[unlikely]] {
bool simulator_progressed = false;
if (simulator_notifier_) [[unlikely]] {
// We can't hold our own lock while notifying
// the simulator because notifying the simulator
// involves acquiring the simulator's mutex
@@ -79,7 +77,7 @@ class Shared {
// so we have to get out of its way to avoid
// a cyclical deadlock.
lock.unlock();
std::invoke(wait_notifier_);
simulator_progressed = std::invoke(simulator_notifier_);
lock.lock();
if (item_) {
// item may have been filled while we
@@ -87,7 +85,8 @@ class Shared {
// the simulator of our waiting_ status.
break;
}
} else {
}
if (!simulator_progressed) [[likely]] {
cv_.wait(lock);
}
MG_ASSERT(!consumed_, "Future consumed twice!");
@@ -118,19 +117,11 @@ class Shared {
std::unique_lock<std::mutex> lock(mu_);
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!filled_, "Promise filled twice!");
MG_ASSERT(!item_, "Promise filled twice!");
item_ = item;
filled_ = true;
} // lock released before condition variable notification
if (fill_notifier_) {
spdlog::trace("calling fill notifier");
std::invoke(fill_notifier_);
} else {
spdlog::trace("not calling fill notifier");
}
cv_.notify_all();
}
@@ -262,9 +253,8 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
}
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
std::function<void()> fill_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -31,10 +31,9 @@ class LocalTransport {
: local_transport_handle_(std::move(local_transport_handle)) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
std::function<void()> fill_notifier, Duration timeout) {
return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(
to_address, from_address, std::move(request), timeout, fill_notifier);
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
return local_transport_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address,
std::move(request), timeout);
}
template <Message... Ms>

View File

@@ -140,12 +140,8 @@ class LocalTransportHandle {
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RequestT &&request,
Duration timeout, std::function<void()> fill_notifier) {
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
// set null notifier for when the Future::Wait is called
nullptr,
// set notifier for when Promise::Fill is called
std::forward<std::function<void()>>(fill_notifier));
Duration timeout) {
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;

View File

@@ -13,7 +13,6 @@
#include <chrono>
#include <cmath>
#include <compare>
#include <unordered_map>
#include <boost/core/demangle.hpp>
@@ -40,8 +39,6 @@ struct LatencyHistogramSummary {
Duration p100;
Duration sum;
friend bool operator==(const LatencyHistogramSummary &lhs, const LatencyHistogramSummary &rhs) = default;
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummary &histo) {
in << "{ \"count\": " << histo.count;
in << ", \"p0\": " << histo.p0.count();
@@ -83,8 +80,6 @@ struct LatencyHistogramSummaries {
return output;
}
friend bool operator==(const LatencyHistogramSummaries &lhs, const LatencyHistogramSummaries &rhs) = default;
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummaries &histo) {
using memgraph::utils::print_helpers::operator<<;
in << histo.latencies;

View File

@@ -1,93 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <condition_variable>
#include <functional>
#include <mutex>
#include <optional>
#include <vector>
namespace memgraph::io {
class ReadinessToken {
size_t id_;
public:
explicit ReadinessToken(size_t id) : id_(id) {}
size_t GetId() const { return id_; }
};
class Inner {
std::condition_variable cv_;
std::mutex mu_;
std::vector<ReadinessToken> ready_;
std::optional<std::function<bool()>> tick_simulator_;
public:
void Notify(ReadinessToken readiness_token) {
{
std::unique_lock<std::mutex> lock(mu_);
ready_.emplace_back(readiness_token);
} // mutex dropped
cv_.notify_all();
}
ReadinessToken Await() {
std::unique_lock<std::mutex> lock(mu_);
while (ready_.empty()) {
if (tick_simulator_) [[unlikely]] {
// This avoids a deadlock in a similar way that
// Future::Wait will release its mutex while
// interacting with the simulator, due to
// the fact that the simulator may cause
// notifications that we are interested in.
lock.unlock();
std::invoke(tick_simulator_.value());
lock.lock();
} else {
cv_.wait(lock);
}
}
ReadinessToken ret = ready_.back();
ready_.pop_back();
return ret;
}
void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
std::unique_lock<std::mutex> lock(mu_);
tick_simulator_ = tick_simulator;
}
};
class Notifier {
std::shared_ptr<Inner> inner_;
public:
Notifier() : inner_(std::make_shared<Inner>()) {}
Notifier(const Notifier &) = default;
Notifier &operator=(const Notifier &) = default;
Notifier(Notifier &&old) = default;
Notifier &operator=(Notifier &&old) = default;
~Notifier() = default;
void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
ReadinessToken Await() const { return inner_->Await(); }
void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
};
} // namespace memgraph::io

View File

@@ -19,6 +19,7 @@
#include <map>
#include <set>
#include <thread>
#include <unordered_map>
#include <vector>
#include <boost/core/demangle.hpp>
@@ -91,43 +92,33 @@ struct ReadResponse {
};
template <class... ReadReturn>
utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
return TypeInfoForVariant(response.read_return);
utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &read_response) {
return TypeInfoForVariant(read_response.read_return);
}
template <class ReadReturn>
utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* read_response */) {
return typeid(ReadReturn);
}
template <class ReadOperation>
utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
return typeid(ReadOperation);
}
template <class... ReadOperations>
utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
return TypeInfoForVariant(request.operation);
}
template <class... WriteReturn>
utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
return TypeInfoForVariant(response.write_return);
utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &write_response) {
return TypeInfoForVariant(write_response.write_return);
}
template <class WriteReturn>
utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* write_response */) {
return typeid(WriteReturn);
}
template <class WriteOperation>
utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* write_request */) {
return typeid(WriteOperation);
}
template <class... WriteOperations>
utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
return TypeInfoForVariant(request.operation);
utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &write_request) {
return TypeInfoForVariant(write_request.operation);
}
/// AppendRequest is a raft-level message that the Leader
@@ -191,7 +182,7 @@ struct PendingClientRequest {
struct Leader {
std::map<Address, FollowerTracker> followers;
std::map<LogIndex, PendingClientRequest> pending_client_requests;
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
Time last_broadcast = Time::min();
std::string static ToString() { return "\tLeader \t"; }
@@ -321,12 +312,19 @@ class Raft {
}
void Run() {
// debug - gvolfing
uint64_t tick_count = 0;
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
if (now >= next_cron_) {
next_cron_ = Cron();
}
// debug - gvolfing
spdlog::info("Raft::Run() awakened on thread {}, Cron tick: {}", std::this_thread::get_id(), tick_count);
tick_count++;
const Duration receive_timeout = RandomTimeout(kMinimumReceiveTimeout, kMaximumReceiveTimeout);
auto request_result =
@@ -692,7 +690,7 @@ class Raft {
return Leader{
.followers = std::move(followers),
.pending_client_requests = std::map<LogIndex, PendingClientRequest>(),
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
};
}
@@ -856,9 +854,7 @@ class Raft {
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
Address from_address) {
auto type_info = TypeInfoFor(req);
std::string demangled_name = boost::core::demangle(type_info.get().name());
Log("handling ReadOperation<" + demangled_name + ">");
Log("handling ReadOperation");
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(read_operation);

View File

@@ -14,12 +14,10 @@
#include <iostream>
#include <optional>
#include <type_traits>
#include <unordered_map>
#include <vector>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/notifier.hpp"
#include "io/rsm/raft.hpp"
#include "utils/result.hpp"
@@ -38,14 +36,6 @@ using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::utils::BasicResult;
template <typename RequestT, typename ResponseT>
struct AsyncRequest {
Time start_time;
RequestT request;
Notifier notifier;
ResponseFuture<ResponseT> future;
};
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
typename ReadResponseT>
class RsmClient {
@@ -57,15 +47,23 @@ class RsmClient {
/// State for single async read/write operations. In the future this could become a map
/// of async operations that can be accessed via an ID etc...
std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
std::optional<Time> async_read_before_;
std::optional<ResponseFuture<ReadResponse<ReadResponseT>>> async_read_;
ReadRequestT current_read_request_;
std::optional<Time> async_write_before_;
std::optional<ResponseFuture<WriteResponse<WriteResponseT>>> async_write_;
WriteRequestT current_write_request_;
void SelectRandomLeader() {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = io_.Rand(addr_distrib);
leader_ = server_addrs_[addr_index];
spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
spdlog::debug(
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
"a RSM Candidate) trying a random one at index {} with address {}",
addr_index, leader_.ToString());
}
template <typename ResponseT>
@@ -93,76 +91,107 @@ class RsmClient {
~RsmClient() = default;
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
Notifier notifier;
const ReadinessToken readiness_token{0};
SendAsyncWriteRequest(req, notifier, readiness_token);
auto poll_result = AwaitAsyncWriteRequest(readiness_token);
while (!poll_result) {
poll_result = AwaitAsyncWriteRequest(readiness_token);
}
return poll_result.value();
WriteRequest<WriteRequestT> client_req;
client_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
if (response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return response_result.GetError();
}
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
if (write_response.success) {
return std::move(write_response.write_return);
}
PossiblyRedirectLeader(write_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
Notifier notifier;
const ReadinessToken readiness_token{0};
SendAsyncReadRequest(req, notifier, readiness_token);
auto poll_result = AwaitAsyncReadRequest(readiness_token);
while (!poll_result) {
poll_result = AwaitAsyncReadRequest(readiness_token);
}
return poll_result.value();
ReadRequest<ReadRequestT> read_req;
read_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
// receive response
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
if (get_response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return get_response_result.GetError();
}
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
if (read_get_response.success) {
return std::move(read_get_response.read_return);
}
PossiblyRedirectLeader(read_get_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
/// AsyncRead methods
void SendAsyncReadRequest(const ReadRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
void SendAsyncReadRequest(const ReadRequestT &req) {
MG_ASSERT(!async_read_);
ReadRequest<ReadRequestT> read_req = {.operation = req};
AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
.start_time = io_.Now(),
.request = std::move(req),
.notifier = notifier,
.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
leader_, read_req, notifier, readiness_token),
};
async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
if (!async_read_before_) {
async_read_before_ = io_.Now();
}
current_read_request_ = std::move(req);
async_read_ = io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
}
void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_reads_.at(readiness_token.GetId());
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest() {
MG_ASSERT(async_read_);
ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
async_request.future = io_.template RequestWithNotification<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(
leader_, read_req, async_request.notifier, readiness_token);
}
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_reads_.at(readiness_token.GetId());
if (!async_request.future.IsReady()) {
if (!async_read_->IsReady()) {
return std::nullopt;
}
return AwaitAsyncReadRequest(readiness_token);
return AwaitAsyncReadRequest();
}
std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_reads_.at(readiness_token.GetId());
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest() {
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(*async_read_).Wait();
async_read_.reset();
const Duration overall_timeout = io_.GetDefaultTimeout();
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
const bool past_time_out = io_.Now() < *async_read_before_ + overall_timeout;
const bool result_has_error = get_response_result.HasError();
if (result_has_error && past_time_out) {
// TODO static assert the exact type of error.
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
async_reads_.erase(readiness_token.GetId());
async_read_before_ = std::nullopt;
return TimedOut{};
}
if (!result_has_error) {
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
@@ -170,69 +199,54 @@ class RsmClient {
PossiblyRedirectLeader(read_get_response);
if (read_get_response.success) {
async_reads_.erase(readiness_token.GetId());
spdlog::debug("returning read_return for RSM request");
async_read_before_ = std::nullopt;
return std::move(read_get_response.read_return);
}
} else {
SendAsyncReadRequest(current_read_request_);
} else if (result_has_error) {
SelectRandomLeader();
SendAsyncReadRequest(current_read_request_);
}
ResendAsyncReadRequest(readiness_token);
return std::nullopt;
}
/// AsyncWrite methods
void SendAsyncWriteRequest(const WriteRequestT &req, Notifier notifier, ReadinessToken readiness_token) {
void SendAsyncWriteRequest(const WriteRequestT &req) {
MG_ASSERT(!async_write_);
WriteRequest<WriteRequestT> write_req = {.operation = req};
AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
.start_time = io_.Now(),
.request = std::move(req),
.notifier = notifier,
.future = io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
leader_, write_req, notifier, readiness_token),
};
async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
if (!async_write_before_) {
async_write_before_ = io_.Now();
}
current_write_request_ = std::move(req);
async_write_ = io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, write_req);
}
void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_writes_.at(readiness_token.GetId());
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest() {
MG_ASSERT(async_write_);
WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
async_request.future =
io_.template RequestWithNotification<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(
leader_, write_req, async_request.notifier, readiness_token);
}
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_writes_.at(readiness_token.GetId());
if (!async_request.future.IsReady()) {
if (!async_write_->IsReady()) {
return std::nullopt;
}
return AwaitAsyncWriteRequest(readiness_token);
return AwaitAsyncWriteRequest();
}
std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_writes_.at(readiness_token.GetId());
ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest() {
ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(*async_write_).Wait();
async_write_.reset();
const Duration overall_timeout = io_.GetDefaultTimeout();
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
const bool past_time_out = io_.Now() < *async_write_before_ + overall_timeout;
const bool result_has_error = get_response_result.HasError();
if (result_has_error && past_time_out) {
// TODO static assert the exact type of error.
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
async_writes_.erase(readiness_token.GetId());
async_write_before_ = std::nullopt;
return TimedOut{};
}
if (!result_has_error) {
ResponseEnvelope<WriteResponse<WriteResponseT>> &&get_response_envelope =
std::move(get_response_result.GetValue());
@@ -241,15 +255,14 @@ class RsmClient {
PossiblyRedirectLeader(write_get_response);
if (write_get_response.success) {
async_writes_.erase(readiness_token.GetId());
async_write_before_ = std::nullopt;
return std::move(write_get_response.write_return);
}
} else {
SendAsyncWriteRequest(current_write_request_);
} else if (result_has_error) {
SelectRandomLeader();
SendAsyncWriteRequest(current_write_request_);
}
ResendAsyncWriteRequest(readiness_token);
return std::nullopt;
}
};

View File

@@ -41,7 +41,7 @@ class Simulator {
Io<SimulatorTransport> Register(Address address) {
std::uniform_int_distribution<uint64_t> seed_distrib;
uint64_t seed = seed_distrib(rng_);
return Io{SimulatorTransport(simulator_handle_, address, seed), address};
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
}
void IncrementServerCountAndWaitForQuiescentState(Address address) {
@@ -49,14 +49,5 @@ class Simulator {
}
SimulatorStats Stats() { return simulator_handle_->Stats(); }
std::shared_ptr<SimulatorHandle> GetSimulatorHandle() const { return simulator_handle_; }
std::function<bool()> GetSimulatorTickClosure() {
std::function<bool()> tick_closure = [handle_copy = simulator_handle_] {
return handle_copy->MaybeTickSimulator();
};
return tick_closure;
}
};
}; // namespace memgraph::io::simulator

View File

@@ -23,12 +23,6 @@ namespace memgraph::io::simulator {
void SimulatorHandle::ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
std::move(dop).promise.TimeOut();
it = promises_.erase(it);
}
can_receive_.clear();
cv_.notify_all();
}
@@ -52,94 +46,92 @@ void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address addre
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
if (all_servers_blocked) {
spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
server_addresses_.size());
return;
}
spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
server_addresses_.size());
cv_.wait(lock);
}
}
bool SortInFlight(const std::pair<Address, OpaqueMessage> &lhs, const std::pair<Address, OpaqueMessage> &rhs) {
// NB: never sort based on the request ID etc...
// This should only be used from std::stable_sort
// because by comparing on the from_address alone,
// we expect the sender ordering to remain
// deterministic.
const auto &[addr_1, om_1] = lhs;
const auto &[addr_2, om_2] = rhs;
return om_1.from_address < om_2.from_address;
}
bool SimulatorHandle::MaybeTickSimulator() {
std::unique_lock<std::mutex> lock(mu_);
const size_t blocked_servers = blocked_on_receive_.size();
if (should_shut_down_ || blocked_servers < server_addresses_.size()) {
// Don't judge, this is a workaround for poc. gvolfing
server_count_++;
if (server_count_ == 3) {
is_quiescent_state_achieved_ = true;
}
// if (!is_quiescent_state_achieved_) {
if (!is_quiescent_state_achieved_) {
// return SimulatorProgression::DidSetUpOneServer;
cv_.notify_all(); // -> deadlock
// return false;// -> deadlock
return true; // -> deadlock
}
// if (blocked_servers < server_addresses_.size() && is_quiescent_state_achieved_) {
if (blocked_servers < server_addresses_.size()) {
// we only need to advance the simulator when all
// servers have reached a quiescent state, blocked
// on their own futures or receive methods.
return false;
}
// We allow the simulator to progress the state of the system only
// after all servers are blocked on receive.
spdlog::trace("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ simulator tick ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
stats_.simulator_ticks++;
blocked_on_receive_.clear();
cv_.notify_all();
bool timed_anything_out = TimeoutPromisesPastDeadline();
TimeoutPromisesPastDeadline();
if (timed_anything_out) {
spdlog::trace("simulator progressing: timed out a request");
}
auto sort_based_on_message_id = [](const std::pair<Address, OpaqueMessage> &lhs,
const std::pair<Address, OpaqueMessage> &rhs) {
// return a.second.request_id < b.second.request_id;
if (lhs.second.from_address != rhs.second.from_address) {
return lhs.second.from_address < rhs.second.from_address;
}
const Duration clock_advance = std::chrono::microseconds{time_distrib_(rng_)};
// We don't always want to advance the clock with every message that we deliver because
// when we advance it for every message, it causes timeouts to occur for any "request path"
// over a certain length. Alternatively, we don't want to simply deliver multiple messages
// in a single simulator tick because that would reduce the amount of concurrent message
// mixing that may naturally occur in production. This approach is to mod the random clock
// advance by a prime number (hopefully avoiding most harmonic effects that would be introduced
// by only advancing the clock by an even amount etc...) and only advancing the clock close to
// half of the time.
if (clock_advance.count() % 97 > 49) {
spdlog::trace("simulator progressing: clock advanced by {}", clock_advance.count());
cluster_wide_time_microseconds_ += clock_advance;
stats_.elapsed_time = cluster_wide_time_microseconds_ - config_.start_time;
}
if (cluster_wide_time_microseconds_ >= config_.abort_time) {
spdlog::error(
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time. The SimulatorConfig.rng_seed for this run is {}",
config_.rng_seed);
throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
}
return lhs.second.request_id < rhs.second.request_id;
};
std::sort(in_flight_.begin(), in_flight_.end(), sort_based_on_message_id);
if (in_flight_.empty()) {
// return early here because there are no messages to schedule
// We tick the clock forward when all servers are blocked but
// there are no in-flight messages to schedule delivery of.
// const Duration clock_advance = std::chrono::microseconds{Rand(time_distrib_)};
// cluster_wide_time_microseconds_ += clock_advance;
if (cluster_wide_time_microseconds_ >= config_.abort_time) {
if (should_shut_down_) {
return false;
}
spdlog::error(
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time.");
throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
}
// spdlog::info(
// "Time increased with {} to {}", clock_advance.count(),
// std::chrono::duration_cast<std::chrono::milliseconds>(cluster_wide_time_microseconds_.time_since_epoch())
// .count());
return true;
}
std::stable_sort(in_flight_.begin(), in_flight_.end(), SortInFlight);
if (config_.scramble_messages && in_flight_.size() > 1) {
if (config_.scramble_messages) {
// scramble messages
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
const size_t swap_index = swap_distrib(rng_);
const size_t swap_index = RandLocked(swap_distrib);
std::swap(in_flight_[swap_index], in_flight_.back());
}
auto [to_address, opaque_message] = std::move(in_flight_.back());
in_flight_.pop_back();
event_log_.AddAndLog(opaque_message, "Handle ");
const int drop_threshold = drop_distrib_(rng_);
const int drop_threshold = RandLocked(drop_distrib_);
const bool should_drop = drop_threshold < config_.drop_percent;
if (should_drop) {
@@ -149,6 +141,7 @@ bool SimulatorHandle::MaybeTickSimulator() {
PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
if (promises_.contains(promise_key)) {
event_log_.AddAndLog(opaque_message, "Promise contains");
// complete waiting promise if it's there
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
@@ -158,22 +151,24 @@ bool SimulatorHandle::MaybeTickSimulator() {
if (should_drop || normal_timeout) {
stats_.timed_out_requests++;
dop.promise.TimeOut();
spdlog::trace("simulator timing out request ");
} else {
stats_.total_responses++;
Duration response_latency = cluster_wide_time_microseconds_ - dop.requested_at;
auto type_info = opaque_message.type_info;
dop.promise.Fill(std::move(opaque_message), response_latency);
histograms_.Measure(type_info, response_latency);
spdlog::trace("simulator replying to request");
}
} else if (should_drop) {
event_log_.AddAndLog(opaque_message, "Dropped");
// don't add it anywhere, let it drop
spdlog::trace("simulator silently dropping request");
} else {
event_log_.AddAndLog(opaque_message, "To can_receive");
// add to can_receive_ if not
spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
opaque_message.to_address.last_known_port);
// This might be needed I am not sure, it can cause deadlock if you uncomment. Didn't find the issue yet.
// MG_ASSERT(server_addresses_.contains(to_address));
// MG_ASSERT(blocked_on_receive_.contains(to_address));
// blocked_on_receive_.erase(to_address);
const auto &[om_vec, inserted] =
can_receive_.try_emplace(to_address.ToPartialAddress(), std::vector<OpaqueMessage>());
om_vec->second.emplace_back(std::move(opaque_message));
@@ -187,6 +182,11 @@ Time SimulatorHandle::Now() const {
return cluster_wide_time_microseconds_;
}
Time SimulatorHandle::NowLocked() const {
// std::unique_lock<std::mutex> lock(mu_);
return cluster_wide_time_microseconds_;
}
SimulatorStats SimulatorHandle::Stats() {
std::unique_lock<std::mutex> lock(mu_);
return stats_;

View File

@@ -17,13 +17,20 @@
#include <map>
#include <memory>
#include <optional>
#include <queue>
#include <set>
#include <string>
#include <utility>
#include <variant>
#include <vector>
#include <boost/core/demangle.hpp>
#include <sstream>
#include <boost/lexical_cast.hpp>
#include <boost/stacktrace.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "fmt/format.h"
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/message_conversion.hpp"
@@ -32,9 +39,50 @@
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
#include "spdlog/spdlog.h"
namespace memgraph::io::simulator {
struct EventDescriptor {
boost::asio::ip::address from_address_ip;
uint16_t from_address_port;
boost::asio::ip::address to_address_ip;
uint16_t to_address_port;
uint64_t request_id;
std::string_view caller;
utils::TypeInfoRef type_info;
uint64_t event_id;
Time time;
};
class SimulatorHandle;
class EventLog {
std::vector<EventDescriptor> event_log_;
static constexpr const char *eventlog_signal_string_ = "EVENTLOG_UPDATE";
void LogEvent(const EventDescriptor &event) const {
spdlog::info(
fmt::format("{} ({})-> caller: {}, from_address: {}:{}, to_address: {}:{}, request_id: {}, type: {}, event_id: "
"{}, thread_id: {}",
eventlog_signal_string_,
std::chrono::duration_cast<std::chrono::milliseconds>(event.time.time_since_epoch()).count(),
event.caller, event.from_address_ip.to_string(), event.from_address_port,
event.to_address_ip.to_string(), event.to_address_port, event.request_id,
event.type_info.get().name(), event.event_id, std::this_thread::get_id()));
}
public:
SimulatorHandle *handle{nullptr};
void AddAndLog(const OpaqueMessage &message, std::string_view caller);
void LogAllEvents() const {
for (const auto &event : event_log_) {
LogEvent(event);
}
}
};
class SimulatorHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
@@ -48,40 +96,48 @@ class SimulatorHandle {
// messages that are sent to servers that may later receive them
std::map<PartialAddress, std::vector<OpaqueMessage>> can_receive_;
// maybe a boolean here
bool is_quiescent_state_achieved_ = false;
std::atomic<int> server_count_;
Time cluster_wide_time_microseconds_;
bool should_shut_down_ = false;
SimulatorStats stats_;
std::set<Address> blocked_on_receive_;
std::set<Address> server_addresses_;
std::mt19937 rng_;
std::uniform_int_distribution<int> time_distrib_{0, 30000};
std::uniform_int_distribution<int> time_distrib_{5, 50};
std::uniform_int_distribution<int> drop_distrib_{0, 99};
SimulatorConfig config_;
MessageHistogramCollector histograms_;
RequestId request_id_counter_{0};
EventLog event_log_;
bool TimeoutPromisesPastDeadline() {
bool timed_anything_out = false;
void TimeoutPromisesPastDeadline() {
const Time now = cluster_wide_time_microseconds_;
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
if (dop.deadline < now && config_.perform_timeouts) {
spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
const bool timed_out = dop.deadline < now;
if (timed_out && config_.perform_timeouts) {
// spdlog::info("timing out request from requester {}.", promise_key.requester_address.ToString());
spdlog::info("timing out request from requester {}. bool timed_out: {}",
promise_key.requester_address.ToString(), timed_out);
std::move(dop).promise.TimeOut();
it = promises_.erase(it);
stats_.timed_out_requests++;
timed_anything_out = true;
} else {
++it;
}
}
return timed_anything_out;
}
public:
explicit SimulatorHandle(SimulatorConfig config)
: cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
: server_count_(0), cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {
spdlog::info("SimulatorHandle constructed.");
event_log_.handle = this;
}
LatencyHistogramSummaries ResponseLatencies();
@@ -107,48 +163,41 @@ class SimulatorHandle {
template <Message Request, Message Response>
ResponseFuture<Response> SubmitRequest(Address to_address, Address from_address, Request &&request, Duration timeout,
std::function<bool()> &&maybe_tick_simulator,
std::function<void()> &&fill_notifier) {
std::function<bool()> &&maybe_tick_simulator) {
auto type_info = TypeInfoFor(request);
std::string demangled_name = boost::core::demangle(type_info.get().name());
spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<Response>>(
// set notifier for when the Future::Wait is called
std::forward<std::function<bool()>>(maybe_tick_simulator),
// set notifier for when Promise::Fill is called
std::forward<std::function<void()>>(fill_notifier));
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifier<ResponseResult<Response>>(
std::forward<std::function<bool()>>(maybe_tick_simulator));
{
std::unique_lock<std::mutex> lock(mu_);
std::unique_lock<std::mutex> lock(mu_);
RequestId request_id = ++request_id_counter_;
RequestId request_id = ++request_id_counter_;
const Time deadline = cluster_wide_time_microseconds_ + timeout;
const Time deadline = cluster_wide_time_microseconds_ + timeout;
std::any message(request);
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message),
.type_info = type_info};
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
std::any message(request);
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message),
.type_info = type_info};
event_log_.AddAndLog(om, "SubmitRequest");
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{
.requested_at = cluster_wide_time_microseconds_,
.deadline = deadline,
.promise = std::move(opaque_promise),
};
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{
.requested_at = cluster_wide_time_microseconds_,
.deadline = deadline,
.promise = std::move(opaque_promise),
};
MG_ASSERT(!promises_.contains(promise_key));
MG_ASSERT(!promises_.contains(promise_key));
promises_.emplace(std::move(promise_key), std::move(dop));
promises_.emplace(std::move(promise_key), std::move(dop));
stats_.total_messages++;
stats_.total_requests++;
} // lock dropped here
stats_.total_messages++;
stats_.total_requests++;
cv_.notify_all();
@@ -159,6 +208,8 @@ class SimulatorHandle {
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
std::unique_lock<std::mutex> lock(mu_);
blocked_on_receive_.emplace(receiver);
const Time deadline = cluster_wide_time_microseconds_ + timeout;
auto partial_address = receiver.ToPartialAddress();
@@ -170,57 +221,83 @@ class SimulatorHandle {
OpaqueMessage message = std::move(can_rx.back());
can_rx.pop_back();
event_log_.AddAndLog(message, "Receivexxxxxx");
// TODO(tyler) search for item in can_receive_ that matches the desired types, rather
// than asserting that the last item in can_rx matches.
auto m_opt = std::move(message).Take<Ms...>();
MG_ASSERT(m_opt.has_value(), "Wrong message type received compared to the expected type");
blocked_on_receive_.erase(receiver);
return std::move(m_opt).value();
}
}
if (!should_shut_down_) {
if (!blocked_on_receive_.contains(receiver)) {
blocked_on_receive_.emplace(receiver);
spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
cv_.notify_all();
}
lock.unlock();
auto simulator_progress = MaybeTickSimulator();
lock.lock();
if (!should_shut_down_ && !simulator_progress) {
cv_.wait(lock);
}
}
spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
blocked_on_receive_.erase(receiver);
return TimedOut{};
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
auto type_info = TypeInfoFor(message);
{
std::unique_lock<std::mutex> lock(mu_);
std::any message_any(std::move(message));
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any),
.type_info = type_info};
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
std::unique_lock<std::mutex> lock(mu_);
std::any message_any(std::move(message));
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any),
.type_info = type_info};
event_log_.AddAndLog(om, "Sendxxxxxxxxx");
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
stats_.total_messages++;
} // lock dropped before cv notification
stats_.total_messages++;
cv_.notify_all();
}
Time Now() const;
Time NowLocked() const;
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::unique_lock<std::mutex> lock(mu_);
return distrib(rng_);
return RandLocked<D, Return>(std::forward<D>(distrib));
}
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return RandLocked(D distrib) {
Return res = distrib(rng_);
spdlog::info("Getting random value from thread {}: {}. Stacktrace:\n{}", std::this_thread::get_id(), res,
boost::stacktrace::stacktrace());
return res;
}
SimulatorStats Stats();
};
inline void EventLog::AddAndLog(const OpaqueMessage &message, std::string_view caller) {
EventDescriptor event{.from_address_ip = message.from_address.last_known_ip,
.from_address_port = message.from_address.last_known_port,
.to_address_ip = message.to_address.last_known_ip,
.to_address_port = message.to_address.last_known_port,
.request_id = message.request_id,
.caller = caller,
.type_info = message.type_info,
.event_id = event_log_.size(),
.time = handle->NowLocked()};
event_log_.push_back(event);
LogEvent(event);
}
}; // namespace memgraph::io::simulator

View File

@@ -13,10 +13,6 @@
#include <cstdint>
#include <fmt/format.h>
#include "io/time.hpp"
namespace memgraph::io::simulator {
struct SimulatorStats {
uint64_t total_messages = 0;
@@ -25,22 +21,5 @@ struct SimulatorStats {
uint64_t total_requests = 0;
uint64_t total_responses = 0;
uint64_t simulator_ticks = 0;
Duration elapsed_time;
friend bool operator==(const SimulatorStats & /* lhs */, const SimulatorStats & /* rhs */) = default;
friend std::ostream &operator<<(std::ostream &in, const SimulatorStats &stats) {
auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(stats.elapsed_time).count();
std::string formated = fmt::format(
"SimulatorStats {{ total_messages: {}, dropped_messages: {}, timed_out_requests: {}, total_requests: {}, "
"total_responses: {}, simulator_ticks: {}, elapsed_time: {}ms }}",
stats.total_messages, stats.dropped_messages, stats.timed_out_requests, stats.total_requests,
stats.total_responses, stats.simulator_ticks, elapsed_ms);
in << formated;
return in;
}
};
}; // namespace memgraph::io::simulator

View File

@@ -15,7 +15,6 @@
#include <utility>
#include "io/address.hpp"
#include "io/notifier.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/time.hpp"
@@ -26,7 +25,7 @@ using memgraph::io::Time;
class SimulatorTransport {
std::shared_ptr<SimulatorHandle> simulator_handle_;
Address address_;
const Address address_;
std::mt19937 rng_;
public:
@@ -34,14 +33,11 @@ class SimulatorTransport {
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request,
std::function<void()> notification, Duration timeout) {
std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
return handle_copy->MaybeTickSimulator();
};
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestT request, Duration timeout) {
std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(
to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
return simulator_handle_->template SubmitRequest<RequestT, ResponseT>(to_address, from_address, std::move(request),
timeout, std::move(maybe_tick_simulator));
}
template <Message... Ms>
@@ -58,9 +54,11 @@ class SimulatorTransport {
bool ShouldShutDown() const { return simulator_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
template <class D = std::poisson_distribution<uint64_t>, class Return = uint64_t>
Return Rand(D distrib) {
return distrib(rng_);
// debug - gvolfing
auto ret = distrib(rng_);
return ret;
}
LatencyHistogramSummaries ResponseLatencies() { return simulator_handle_->ResponseLatencies(); }

View File

@@ -20,7 +20,6 @@
#include "io/errors.hpp"
#include "io/future.hpp"
#include "io/message_histogram_collector.hpp"
#include "io/notifier.hpp"
#include "io/time.hpp"
#include "utils/result.hpp"
@@ -85,9 +84,7 @@ class Io {
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
const Address from_address = address_;
std::function<void()> fill_notifier = nullptr;
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, fill_notifier,
timeout);
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request, timeout);
}
/// Issue a request that times out after the default timeout. This tends
@@ -96,30 +93,7 @@ class Io {
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
const Duration timeout = default_timeout_;
const Address from_address = address_;
std::function<void()> fill_notifier = nullptr;
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Issue a request that will notify a Notifier when it is filled or times out.
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RequestT request, Notifier notifier,
ReadinessToken readiness_token) {
const Duration timeout = default_timeout_;
const Address from_address = address_;
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Issue a request that will notify a Notifier when it is filled or times out.
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT request, Notifier notifier,
ReadinessToken readiness_token, Duration timeout) {
const Address from_address = address_;
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, std::move(request), timeout);
}
/// Wait for an explicit number of microseconds for a request of one of the
@@ -163,14 +137,7 @@ class Io {
Address GetAddress() { return address_; }
void SetAddress(Address address) { address_ = address; }
Io<I> ForkLocal(boost::uuids::uuid uuid) {
Address new_address{
.unique_id = uuid,
.last_known_ip = address_.last_known_ip,
.last_known_port = address_.last_known_port,
};
return Io(implementation_, new_address);
}
Io<I> ForkLocal() { return Io(implementation_, address_.ForkUniqueAddress()); }
LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
};

View File

@@ -42,7 +42,6 @@ struct MachineConfig {
boost::asio::ip::address listen_ip;
uint16_t listen_port;
size_t shard_worker_threads = std::max(static_cast<unsigned int>(1), std::thread::hardware_concurrency());
bool sync_message_handling = false;
};
} // namespace memgraph::machine_manager

View File

@@ -78,10 +78,10 @@ class MachineManager {
MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
: io_(io),
config_(config),
coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
coordinator_address_} {
auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
coordinator_address_(io.GetAddress().ForkUniqueAddress()),
shard_manager_{io.ForkLocal(), config.shard_worker_threads, coordinator_address_} {
auto coordinator_io = io.ForkLocal();
coordinator_io.SetAddress(coordinator_address_);
CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
}
@@ -101,23 +101,11 @@ class MachineManager {
Address CoordinatorAddress() { return coordinator_address_; }
void Run() {
while (true) {
MaybeBlockOnSyncHandling();
if (io_.ShouldShutDown()) {
break;
}
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
uint64_t now_us = now.time_since_epoch().count();
uint64_t next_us = next_cron_.time_since_epoch().count();
if (now >= next_cron_) {
spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
next_cron_ = Cron();
} else {
spdlog::info("now {} < next_cron_ {}", now_us, next_us);
}
Duration receive_timeout = std::max(next_cron_, now) - now;
@@ -206,27 +194,10 @@ class MachineManager {
}
private:
// This method exists for controlling concurrency
// during deterministic simulation testing.
void MaybeBlockOnSyncHandling() {
if (!config_.sync_message_handling) {
return;
}
// block on coordinator
coordinator_queue_.BlockOnQuiescence();
// block on shards
shard_manager_.BlockOnQuiescence();
}
Time Cron() {
spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
MaybeBlockOnSyncHandling();
Time ret = shard_manager_.Cron();
MaybeBlockOnSyncHandling();
return ret;
return shard_manager_.Cron();
}
};

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,7 +33,6 @@
#include <spdlog/sinks/dist_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include "common/errors.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
@@ -454,7 +453,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
std::map<std::string, memgraph::communication::bolt::Value> Pull(TEncoder *encoder, std::optional<int> n,
std::optional<int> qid) override {
TypedValueResultStream stream(encoder, interpreter_.GetRequestRouter());
TypedValueResultStream stream(encoder, interpreter_.GetShardRequestManager());
return PullResults(stream, n, qid);
}
@@ -481,9 +480,20 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
const auto &summary = interpreter_.Pull(&stream, n, qid);
std::map<std::string, memgraph::communication::bolt::Value> decoded_summary;
for (const auto &kv : summary) {
auto bolt_value = memgraph::glue::v2::ToBoltValue(kv.second, interpreter_.GetRequestRouter(),
memgraph::storage::v3::View::NEW);
decoded_summary.emplace(kv.first, std::move(bolt_value));
auto maybe_value = memgraph::glue::v2::ToBoltValue(kv.second, interpreter_.GetShardRequestManager(),
memgraph::storage::v3::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::v3::Error::DELETED_OBJECT:
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
case memgraph::storage::v3::Error::VERTEX_ALREADY_INSERTED:
throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming summary.");
}
}
decoded_summary.emplace(kv.first, std::move(*maybe_value));
}
return decoded_summary;
} catch (const memgraph::query::v2::QueryException &e) {
@@ -497,22 +507,35 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
/// before forwarding the calls to original TEncoder.
class TypedValueResultStream {
public:
TypedValueResultStream(TEncoder *encoder, const memgraph::query::v2::RequestRouterInterface *request_router)
: encoder_(encoder), request_router_(request_router) {}
TypedValueResultStream(TEncoder *encoder, const memgraph::msgs::ShardRequestManagerInterface *shard_request_manager)
: encoder_(encoder), shard_request_manager_(shard_request_manager) {}
void Result(const std::vector<memgraph::query::v2::TypedValue> &values) {
std::vector<memgraph::communication::bolt::Value> decoded_values;
decoded_values.reserve(values.size());
for (const auto &v : values) {
auto bolt_value = memgraph::glue::v2::ToBoltValue(v, request_router_, memgraph::storage::v3::View::NEW);
decoded_values.emplace_back(std::move(bolt_value));
auto maybe_value = memgraph::glue::v2::ToBoltValue(v, shard_request_manager_, memgraph::storage::v3::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::v3::Error::DELETED_OBJECT:
throw memgraph::communication::bolt::ClientError("Returning a deleted object as a result.");
case memgraph::storage::v3::Error::NONEXISTENT_OBJECT:
throw memgraph::communication::bolt::ClientError("Returning a nonexistent object as a result.");
case memgraph::storage::v3::Error::VERTEX_HAS_EDGES:
case memgraph::storage::v3::Error::SERIALIZATION_ERROR:
case memgraph::storage::v3::Error::PROPERTIES_DISABLED:
case memgraph::storage::v3::Error::VERTEX_ALREADY_INSERTED:
throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming results.");
}
}
decoded_values.emplace_back(std::move(*maybe_value));
}
encoder_->MessageRecord(decoded_values);
}
private:
TEncoder *encoder_;
const memgraph::query::v2::RequestRouterInterface *request_router_{nullptr};
const memgraph::msgs::ShardRequestManagerInterface *shard_request_manager_{nullptr};
};
memgraph::query::v2::Interpreter interpreter_;
memgraph::communication::v2::ServerEndpoint endpoint_;
@@ -640,8 +663,6 @@ int main(int argc, char **argv) {
memgraph::machine_manager::MachineManager<memgraph::io::local_transport::LocalTransport> mm{io, config, coordinator};
std::jthread mm_thread([&mm] { mm.Run(); });
auto rr_factory = std::make_unique<memgraph::query::v2::LocalRequestRouterFactory>(io);
memgraph::query::v2::InterpreterContext interpreter_context{
(memgraph::storage::v3::Shard *)(nullptr),
{.query = {.allow_load_csv = FLAGS_allow_load_csv},
@@ -652,7 +673,7 @@ int main(int argc, char **argv) {
.stream_transaction_conflict_retries = FLAGS_stream_transaction_conflict_retries,
.stream_transaction_retry_interval = std::chrono::milliseconds(FLAGS_stream_transaction_retry_interval)},
FLAGS_data_directory,
std::move(rr_factory),
std::move(io),
mm.CoordinatorAddress()};
SessionData session_data{&interpreter_context};

View File

@@ -11,6 +11,7 @@ set(mg_query_v2_sources
cypher_query_interpreter.cpp
frontend/semantic/required_privileges.cpp
frontend/stripped.cpp
interpret/awesome_memgraph_functions.cpp
interpreter.cpp
metadata.cpp
plan/operator.cpp
@@ -23,8 +24,7 @@ set(mg_query_v2_sources
plan/variable_start_planner.cpp
serialization/property_value.cpp
bindings/typed_value.cpp
accessors.cpp
multiframe.cpp)
accessors.cpp)
find_package(Boost REQUIRED)
@@ -34,7 +34,7 @@ target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory mg-coordinator)
target_link_libraries(mg-query-v2 mg-expr mg-functions)
target_link_libraries(mg-query-v2 mg-expr)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")

View File

@@ -10,25 +10,24 @@
// licenses/APL.txt.
#include "query/v2/accessors.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/requests.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "storage/v3/id_types.hpp"
namespace memgraph::query::v2::accessors {
EdgeAccessor::EdgeAccessor(Edge edge, const RequestRouterInterface *request_router)
: edge(std::move(edge)), request_router_(request_router) {}
EdgeAccessor::EdgeAccessor(Edge edge, const msgs::ShardRequestManagerInterface *manager)
: edge(std::move(edge)), manager_(manager) {}
EdgeTypeId EdgeAccessor::EdgeType() const { return edge.type.id; }
const std::vector<std::pair<PropertyId, Value>> &EdgeAccessor::Properties() const { return edge.properties; }
Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
if (!maybe_prop) {
auto prop_id = manager_->NameToProperty(prop_name);
auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
if (it == edge.properties.end()) {
return {};
}
const auto prop_id = *maybe_prop;
auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
return it->second;
}
@@ -36,23 +35,21 @@ const Edge &EdgeAccessor::GetEdge() const { return edge; }
bool EdgeAccessor::IsCycle() const { return edge.src == edge.dst; };
size_t EdgeAccessor::CypherId() const { return edge.id.gid; }
VertexAccessor EdgeAccessor::To() const {
return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, manager_);
}
VertexAccessor EdgeAccessor::From() const {
return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, manager_);
}
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
const RequestRouterInterface *request_router)
: vertex(std::move(v)), properties(std::move(props)), request_router_(request_router) {}
const msgs::ShardRequestManagerInterface *manager)
: vertex(std::move(v)), properties(std::move(props)), manager_(manager) {}
VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
const RequestRouterInterface *request_router)
: vertex(std::move(v)), request_router_(request_router) {
const msgs::ShardRequestManagerInterface *manager)
: vertex(std::move(v)), manager_(manager) {
properties.reserve(props.size());
for (auto &[id, value] : props) {
properties.emplace_back(std::make_pair(id, std::move(value)));
@@ -60,8 +57,8 @@ VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
}
VertexAccessor::VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props,
const RequestRouterInterface *request_router)
: vertex(std::move(v)), request_router_(request_router) {
const msgs::ShardRequestManagerInterface *manager)
: vertex(std::move(v)), manager_(manager) {
properties.reserve(props.size());
for (const auto &[id, value] : props) {
properties.emplace_back(std::make_pair(id, value));
@@ -91,11 +88,7 @@ Value VertexAccessor::GetProperty(PropertyId prop_id) const {
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value VertexAccessor::GetProperty(const std::string &prop_name) const {
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
if (!maybe_prop) {
return {};
}
return GetProperty(*maybe_prop);
return GetProperty(manager_->NameToProperty(prop_name));
}
msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }

View File

@@ -24,24 +24,24 @@
#include "utils/memory.hpp"
#include "utils/memory_tracker.hpp"
namespace memgraph::query::v2 {
class RequestRouterInterface;
} // namespace memgraph::query::v2
namespace memgraph::msgs {
class ShardRequestManagerInterface;
} // namespace memgraph::msgs
namespace memgraph::query::v2::accessors {
using Value = msgs::Value;
using Edge = msgs::Edge;
using Vertex = msgs::Vertex;
using Label = msgs::Label;
using PropertyId = msgs::PropertyId;
using EdgeTypeId = msgs::EdgeTypeId;
using Value = memgraph::msgs::Value;
using Edge = memgraph::msgs::Edge;
using Vertex = memgraph::msgs::Vertex;
using Label = memgraph::msgs::Label;
using PropertyId = memgraph::msgs::PropertyId;
using EdgeTypeId = memgraph::msgs::EdgeTypeId;
class VertexAccessor;
class EdgeAccessor final {
public:
explicit EdgeAccessor(Edge edge, const RequestRouterInterface *request_router);
explicit EdgeAccessor(Edge edge, const msgs::ShardRequestManagerInterface *manager);
[[nodiscard]] EdgeTypeId EdgeType() const;
@@ -53,7 +53,12 @@ class EdgeAccessor final {
[[nodiscard]] bool IsCycle() const;
[[nodiscard]] size_t CypherId() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
[[nodiscard]] size_t CypherId() const { return 10; }
// bool HasSrcAccessor const { return src == nullptr; }
// bool HasDstAccessor const { return dst == nullptr; }
[[nodiscard]] VertexAccessor To() const;
[[nodiscard]] VertexAccessor From() const;
@@ -64,7 +69,7 @@ class EdgeAccessor final {
private:
Edge edge;
const RequestRouterInterface *request_router_;
const msgs::ShardRequestManagerInterface *manager_;
};
class VertexAccessor final {
@@ -73,10 +78,10 @@ class VertexAccessor final {
using Label = msgs::Label;
using VertexId = msgs::VertexId;
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
const RequestRouterInterface *request_router);
const msgs::ShardRequestManagerInterface *manager);
VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const RequestRouterInterface *request_router);
VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const RequestRouterInterface *request_router);
VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const msgs::ShardRequestManagerInterface *manager);
VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const msgs::ShardRequestManagerInterface *manager);
[[nodiscard]] Label PrimaryLabel() const;
@@ -93,11 +98,48 @@ class VertexAccessor final {
[[nodiscard]] msgs::Vertex GetVertex() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
[[nodiscard]] size_t InDegree() const { throw utils::NotYetImplemented("InDegree() not yet implemented"); }
[[nodiscard]] size_t CypherId() const { return 10; }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
[[nodiscard]] size_t OutDegree() const { throw utils::NotYetImplemented("OutDegree() not yet implemented"); }
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
// auto maybe_edges = impl_.InEdges(view, edge_types);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto InEdges(storage::View view) const { return InEdges(view, {}); }
//
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest)
// const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
// auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
// auto maybe_edges = impl_.OutEdges(view, edge_types);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
//
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
// const VertexAccessor &dest) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
// auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
// storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
//
// storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
//
friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
@@ -108,9 +150,13 @@ class VertexAccessor final {
private:
Vertex vertex;
std::vector<std::pair<PropertyId, Value>> properties;
const RequestRouterInterface *request_router_;
const msgs::ShardRequestManagerInterface *manager_;
};
// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
// inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
// Highly mocked interface. Won't work if used.
class Path {
public:
@@ -151,14 +197,7 @@ class Path {
friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
utils::MemoryResource *GetMemoryResource() { return mem; }
auto &vertices() { return vertices_; }
auto &edges() { return edges_; }
const auto &vertices() const { return vertices_; }
const auto &edges() const { return edges_; }
private:
std::vector<VertexAccessor> vertices_;
std::vector<EdgeAccessor> edges_;
utils::MemoryResource *mem = utils::NewDeleteResource();
};
} // namespace memgraph::query::v2::accessors

View File

@@ -21,27 +21,29 @@
#include "query/v2/requests.hpp"
#include "storage/v3/conversions.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::msgs {
class ShardRequestManagerInterface;
} // namespace memgraph::msgs
namespace memgraph::query::v2 {
class RequestRouterInterface;
inline const auto lam = [](const auto &val) { return ValueToTypedValue(val); };
namespace detail {
class Callable {
public:
auto operator()(const storage::v3::PropertyValue &val) const {
return storage::v3::PropertyToTypedValue<TypedValue>(val);
auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
};
auto operator()(const msgs::Value &val, RequestRouterInterface *request_router) const {
return ValueToTypedValue(val, request_router);
auto operator()(const msgs::Value &val, memgraph::msgs::ShardRequestManagerInterface *manager) const {
return ValueToTypedValue(val, manager);
};
};
} // namespace detail
using ExpressionEvaluator = expr::ExpressionEvaluator<TypedValue, query::v2::EvaluationContext, RequestRouterInterface,
storage::v3::View, storage::v3::LabelId, msgs::Value,
detail::Callable, common::ErrorCode, expr::QueryEngineTag>;
using ExpressionEvaluator = memgraph::expr::ExpressionEvaluator<
TypedValue, memgraph::query::v2::EvaluationContext, memgraph::msgs::ShardRequestManagerInterface, storage::v3::View,
storage::v3::LabelId, msgs::Value, detail::Callable, memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
} // namespace memgraph::query::v2

View File

@@ -13,10 +13,9 @@
#include "query/v2/bindings/bindings.hpp"
#include "expr/interpret/frame.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "expr/interpret/frame.hpp"
namespace memgraph::query::v2 {
using Frame = memgraph::expr::Frame;
using FrameWithValidity = memgraph::expr::FrameWithValidity;
} // namespace memgraph::query::v2
using Frame = memgraph::expr::Frame<TypedValue>;
} // namespace memgraph::query::v2

View File

@@ -20,6 +20,7 @@
#include "query/v2/bindings/symbol.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/path.hpp"
@@ -27,6 +28,7 @@
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/shard_operation_result.hpp"
#include "storage/v3/view.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
@@ -80,5 +82,39 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
template <typename T>
concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
};
template <typename T>
concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{
accessor.SetPropertyAndValidate(key, new_value)
} -> std::same_as<storage::v3::ShardOperationResult<storage::v3::PropertyValue>>;
};
template <typename TRecordAccessor>
concept RecordAccessor =
AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
switch (error) {
case storage::v3::Error::SERIALIZATION_ERROR:
throw TransactionSerializationException();
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted object.");
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::NONEXISTENT_OBJECT:
case storage::v3::Error::VERTEX_ALREADY_INSERTED:
throw QueryRuntimeException("Unexpected error when setting a property.");
}
}
int64_t QueryTimestamp();
} // namespace memgraph::query::v2

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,7 +20,7 @@
#include "query/v2/parameters.hpp"
#include "query/v2/plan/profile.hpp"
//#include "query/v2/trigger.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "utils/async_timer.hpp"
namespace memgraph::query::v2 {
@@ -60,27 +60,27 @@ struct EvaluationContext {
mutable std::unordered_map<std::string, int64_t> counters;
};
inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
RequestRouterInterface *request_router) {
inline std::vector<storage::v3::PropertyId> NamesToProperties(
const std::vector<std::string> &property_names, msgs::ShardRequestManagerInterface *shard_request_manager) {
std::vector<storage::v3::PropertyId> properties;
// TODO Fix by using reference
properties.reserve(property_names.size());
if (request_router != nullptr) {
if (shard_request_manager != nullptr) {
for (const auto &name : property_names) {
properties.push_back(request_router->NameToProperty(name));
properties.push_back(shard_request_manager->NameToProperty(name));
}
}
return properties;
}
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names,
RequestRouterInterface *request_router) {
msgs::ShardRequestManagerInterface *shard_request_manager) {
std::vector<storage::v3::LabelId> labels;
labels.reserve(label_names.size());
// TODO Fix by using reference
if (request_router != nullptr) {
if (shard_request_manager != nullptr) {
for (const auto &name : label_names) {
labels.push_back(request_router->NameToLabel(name));
labels.push_back(shard_request_manager->NameToLabel(name));
}
}
return labels;
@@ -97,7 +97,7 @@ struct ExecutionContext {
plan::ProfilingStats *stats_root{nullptr};
ExecutionStats execution_stats;
utils::AsyncTimer timer;
RequestRouterInterface *request_router{nullptr};
msgs::ShardRequestManagerInterface *shard_request_manager{nullptr};
IdAllocator *edge_ids_alloc;
};

View File

@@ -12,12 +12,12 @@
#pragma once
#include "bindings/typed_value.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/requests.hpp"
#include "query/v2/shard_request_manager.hpp"
namespace memgraph::query::v2 {
inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInterface *request_router) {
inline TypedValue ValueToTypedValue(const msgs::Value &value, msgs::ShardRequestManagerInterface *manager) {
using Value = msgs::Value;
switch (value.type) {
case Value::Type::Null:
@@ -35,7 +35,7 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
std::vector<TypedValue> dst;
dst.reserve(lst.size());
for (const auto &elem : lst) {
dst.push_back(ValueToTypedValue(elem, request_router));
dst.push_back(ValueToTypedValue(elem, manager));
}
return TypedValue(std::move(dst));
}
@@ -43,23 +43,19 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
const auto &value_map = value.map_v;
std::map<std::string, TypedValue> dst;
for (const auto &[key, val] : value_map) {
dst[key] = ValueToTypedValue(val, request_router);
dst[key] = ValueToTypedValue(val, manager);
}
return TypedValue(std::move(dst));
}
case Value::Type::Vertex:
return TypedValue(accessors::VertexAccessor(
value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, request_router));
value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, manager));
case Value::Type::Edge:
return TypedValue(accessors::EdgeAccessor(value.edge_v, request_router));
return TypedValue(accessors::EdgeAccessor(value.edge_v, manager));
}
throw std::runtime_error("Incorrect type in conversion");
}
inline const auto ValueToTypedValueFunctor = [](const msgs::Value &value, RequestRouterInterface *request_router) {
return ValueToTypedValue(value, request_router);
};
inline msgs::Value TypedValueToValue(const TypedValue &value) {
using Value = msgs::Value;
switch (value.type()) {

View File

@@ -11,7 +11,7 @@
#include "query/v2/cypher_query_interpreter.hpp"
#include "query/v2/bindings/symbol_generator.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
@@ -118,9 +118,9 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
}
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
RequestRouterInterface *request_router,
msgs::ShardRequestManagerInterface *shard_manager,
const std::vector<Identifier *> &predefined_identifiers) {
auto vertex_counts = plan::MakeVertexCountCache(request_router);
auto vertex_counts = plan::MakeVertexCountCache(shard_manager);
auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
@@ -130,7 +130,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
RequestRouterInterface *request_router,
msgs::ShardRequestManagerInterface *shard_manager,
const std::vector<Identifier *> &predefined_identifiers) {
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
if (plan_cache) {
@@ -146,7 +146,7 @@ std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_stor
}
auto plan = std::make_shared<CachedPlan>(
MakeLogicalPlan(std::move(ast_storage), query, parameters, request_router, predefined_identifiers));
MakeLogicalPlan(std::move(ast_storage), query, parameters, shard_manager, predefined_identifiers));
if (plan_cache_access) {
plan_cache_access->insert({hash, plan});
}

View File

@@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
};
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
RequestRouterInterface *request_router,
msgs::ShardRequestManagerInterface *shard_manager,
const std::vector<Identifier *> &predefined_identifiers);
/**
@@ -145,7 +145,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
*/
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
RequestRouterInterface *request_router,
msgs::ShardRequestManagerInterface *shard_manager,
const std::vector<Identifier *> &predefined_identifiers = {});
} // namespace memgraph::query::v2

View File

@@ -23,6 +23,7 @@
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/shard_operation_result.hpp"
///////////////////////////////////////////////////////////
// Our communication layer and query engine don't mix
@@ -36,7 +37,7 @@
// This cannot be avoided by simple include orderings so we
// simply undefine those macros as we're sure that libkrb5
// won't and can't be used anywhere in the query engine.
#include "storage/v3/shard.hpp"
#include "storage/v3/storage.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
@@ -64,11 +65,24 @@ class EdgeAccessor final {
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
return SetProperty(key, storage::v3::PropertyValue());
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
VertexAccessor To() const;
VertexAccessor From() const;
@@ -100,19 +114,53 @@ class VertexAccessor final {
auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
storage::v3::ShardResult<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
storage::v3::ShardOperationResult<bool> AddLabel(storage::v3::LabelId label) {
return impl_.AddLabelAndValidate(label);
}
storage::v3::ShardOperationResult<bool> AddLabelAndValidate(storage::v3::LabelId label) {
return impl_.AddLabelAndValidate(label);
}
storage::v3::ShardOperationResult<bool> RemoveLabel(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::ShardOperationResult<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
return impl_.HasLabel(label, view);
}
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::ShardOperationResult<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ShardOperationResult<storage::v3::PropertyValue> SetPropertyAndValidate(
storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ShardOperationResult<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -122,7 +170,7 @@ class VertexAccessor final {
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
@@ -130,7 +178,7 @@ class VertexAccessor final {
}
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -140,16 +188,16 @@ class VertexAccessor final {
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
storage::v3::ShardResult<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
storage::v3::ShardResult<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
// TODO(jbajic) Fix Remove Gid
static int64_t CypherId() { return 1; }

View File

@@ -224,4 +224,12 @@ class VersionInfoInMulticommandTxException : public QueryException {
: QueryException("Version info query not allowed in multicommand transactions.") {}
};
/**
* An exception for an illegal operation that violates schema
*/
class SchemaViolationException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
};
} // namespace memgraph::query::v2

View File

@@ -20,13 +20,11 @@
#include "query/v2/bindings/ast_visitor.hpp"
#include "common/types.hpp"
#include "query/v2/bindings/symbol.hpp"
#include "functions/awesome_memgraph_functions.hpp"
#include "query/v2/interpret/awesome_memgraph_functions.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/path.hpp"
#include "query/v2/request_router.hpp"
#include "utils/typeinfo.hpp"
#include "query/v2/conversions.hpp"
cpp<#
@@ -838,15 +836,13 @@ cpp<#
:slk-load (slk-load-ast-vector "Expression"))
(function-name "std::string" :scope :public)
(function "std::function<TypedValue(const TypedValue *, int64_t,
const functions::FunctionContext<RequestRouterInterface> &)>"
const FunctionContext &)>"
:scope :public
:dont-save t
:clone :copy
:slk-load (lambda (member)
#>cpp
self->${member} = functions::NameToFunction<TypedValue,
functions::FunctionContext<RequestRouterInterface>,
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(self->function_name_);
self->${member} = query::v2::NameToFunction(self->function_name_);
cpp<#)))
(:public
#>cpp
@@ -869,8 +865,7 @@ cpp<#
const std::vector<Expression *> &arguments)
: arguments_(arguments),
function_name_(function_name),
function_(functions::NameToFunction<TypedValue, functions::FunctionContext<RequestRouterInterface>,
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(function_name_)) {
function_(NameToFunction(function_name_)) {
if (!function_) {
throw SemanticException("Function '{}' doesn't exist.", function_name);
}

File diff suppressed because it is too large Load Diff

View File

@@ -20,9 +20,11 @@
#include "storage/v3/view.hpp"
#include "utils/memory.hpp"
namespace memgraph::query::v2 {
namespace memgraph::msgs {
class ShardRequestManagerInterface;
} // namespace memgraph::msgs
class RequestRouterInterface;
namespace memgraph::query::v2 {
namespace {
const char kStartsWith[] = "STARTSWITH";
@@ -32,9 +34,9 @@ const char kId[] = "ID";
} // namespace
struct FunctionContext {
// TODO(kostasrim) consider optional here. RequestRouter does not exist on the storage.
// TODO(kostasrim) consider optional here. ShardRequestManager does not exist on the storage.
// DbAccessor *db_accessor;
RequestRouterInterface *request_router;
msgs::ShardRequestManagerInterface *manager;
utils::MemoryResource *memory;
int64_t timestamp;
std::unordered_map<std::string, int64_t> *counters;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -41,13 +41,13 @@
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/frontend/semantic/required_privileges.hpp"
#include "query/v2/metadata.hpp"
#include "query/v2/multiframe.hpp"
#include "query/v2/plan/planner.hpp"
#include "query/v2/plan/profile.hpp"
#include "query/v2/plan/vertex_count_cache.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/storage.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
#include "utils/event_counter.hpp"
@@ -64,11 +64,7 @@
#include "utils/tsc.hpp"
#include "utils/variant_helpers.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
namespace EventCounter {
extern Event ReadQuery;
extern Event WriteQuery;
extern Event ReadWriteQuery;
@@ -78,7 +74,6 @@ extern const Event LabelPropertyIndexCreated;
extern const Event StreamsCreated;
extern const Event TriggersCreated;
} // namespace EventCounter
namespace memgraph::query::v2 {
@@ -149,18 +144,18 @@ class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Parameters &parameters,
RequestRouterInterface *request_router) {
msgs::ShardRequestManagerInterface *manager) {
// Empty frame for evaluation of password expression. This is OK since
// password should be either null or string literal and it's evaluation
// should not depend on frame.
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
// the argument to Callback.
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
std::string username = auth_query->user_;
std::string rolename = auth_query->role_;
@@ -318,16 +313,16 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
}
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &parameters,
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
InterpreterContext *interpreter_context, msgs::ShardRequestManagerInterface *manager,
std::vector<Notification> *notifications) {
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
// the argument to Callback.
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
Callback callback;
switch (repl_query->action_) {
@@ -454,8 +449,8 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
}
Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &parameters,
RequestRouterInterface *request_router) {
expr::Frame frame(0);
msgs::ShardRequestManagerInterface *manager) {
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
@@ -464,7 +459,7 @@ Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &param
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
evaluation_context.parameters = parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, manager, storage::v3::View::OLD);
Callback callback;
switch (setting_query->action_) {
@@ -655,21 +650,17 @@ struct PullPlanVector {
struct PullPlan {
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router = nullptr,
msgs::ShardRequestManagerInterface *shard_request_manager = nullptr,
// TriggerContextCollector *trigger_context_collector = nullptr,
std::optional<size_t> memory_limit = {});
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary);
std::optional<plan::ProfilingStatsWithTotalTime> PullMultiple(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary);
private:
std::shared_ptr<CachedPlan> plan_ = nullptr;
plan::UniqueCursorPtr cursor_ = nullptr;
expr::FrameWithValidity frame_;
MultiFrame multi_frame_;
expr::Frame<TypedValue> frame_;
ExecutionContext ctx_;
std::optional<size_t> memory_limit_;
@@ -689,137 +680,29 @@ struct PullPlan {
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router, const std::optional<size_t> memory_limit)
msgs::ShardRequestManagerInterface *shard_request_manager, const std::optional<size_t> memory_limit)
: plan_(plan),
cursor_(plan->plan().MakeCursor(execution_memory)),
frame_(plan->symbol_table().max_position(), execution_memory),
multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
memory_limit_(memory_limit) {
ctx_.db_accessor = dba;
ctx_.symbol_table = plan->symbol_table();
ctx_.evaluation_context.timestamp = QueryTimestamp();
ctx_.evaluation_context.parameters = parameters;
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, request_router);
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, request_router);
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, shard_request_manager);
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, shard_request_manager);
if (interpreter_context->config.execution_timeout_sec > 0) {
ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec};
}
ctx_.is_shutting_down = &interpreter_context->is_shutting_down;
ctx_.is_profile_query = is_profile_query;
ctx_.request_router = request_router;
ctx_.shard_request_manager = shard_request_manager;
ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc;
}
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary) {
// Set up temporary memory for a single Pull. Initial memory comes from the
// stack. 256 KiB should fit on the stack and should be more than enough for a
// single `Pull`.
MG_ASSERT(!n.has_value(), "should pull all!");
static constexpr size_t stack_size = 256UL * 1024UL;
char stack_data[stack_size];
utils::ResourceWithOutOfMemoryException resource_with_exception;
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
// We can throw on every query because a simple queries for deleting will use only
// the stack allocated buffer.
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
if (memory_limit_) {
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
ctx_.evaluation_context.memory = &*maybe_limited_resource;
} else {
ctx_.evaluation_context.memory = &pool_memory;
}
// Returns true if a result was pulled.
const auto pull_result = [&]() -> bool { return cursor_->PullMultiple(multi_frame_, ctx_); };
const auto stream_values = [&output_symbols, &stream](const Frame &frame) {
// TODO: The streamed values should also probably use the above memory.
std::vector<TypedValue> values;
values.reserve(output_symbols.size());
for (const auto &symbol : output_symbols) {
values.emplace_back(frame[symbol]);
}
stream->Result(values);
};
// Get the execution time of all possible result pulls and streams.
utils::Timer timer;
int i = 0;
if (has_unsent_results_ && !output_symbols.empty()) {
// stream unsent results from previous pull
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
stream_values(frame);
frame.MakeInvalid();
++i;
if (i == n) {
break;
}
}
}
for (; !n || i < n;) {
if (!pull_result()) {
break;
}
if (!output_symbols.empty()) {
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
stream_values(frame);
frame.MakeInvalid();
++i;
if (i == n) {
break;
}
}
} else {
multi_frame_.MakeAllFramesInvalid();
}
}
// If we finished because we streamed the requested n results,
// we try to pull the next result to see if there is more.
// If there is additional result, we leave the pulled result in the frame
// and set the flag to true.
has_unsent_results_ = i == n && pull_result();
execution_time_ += timer.Elapsed();
if (has_unsent_results_) {
return std::nullopt;
}
summary->insert_or_assign("plan_execution_time", execution_time_.count());
// We are finished with pulling all the data, therefore we can send any
// metadata about the results i.e. notifications and statistics
const bool is_any_counter_set =
std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(),
[](const auto &counter) { return counter > 0; });
if (is_any_counter_set) {
std::map<std::string, TypedValue> stats;
for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) {
stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]);
}
summary->insert_or_assign("stats", std::move(stats));
}
cursor_->Shutdown();
ctx_.profile_execution_time = execution_time_;
return GetStatsWithTotalTime(ctx_);
}
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary) {
if (FLAGS_use_multi_frame) {
return PullMultiple(stream, n, output_symbols, summary);
}
// Set up temporary memory for a single Pull. Initial memory comes from the
// stack. 256 KiB should fit on the stack and should be more than enough for a
// single `Pull`.
@@ -911,24 +794,30 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config,
const std::filesystem::path & /*data_directory*/,
std::unique_ptr<RequestRouterFactory> request_router_factory,
io::Io<io::local_transport::LocalTransport> io,
coordinator::Address coordinator_addr)
: db(db),
config(config),
coordinator_address{coordinator_addr},
request_router_factory_{std::move(request_router_factory)} {}
: db(db), config(config), io{std::move(io)}, coordinator_address{coordinator_addr} {}
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
request_router_ =
interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address);
auto query_io = interpreter_context_->io.ForkLocal();
shard_request_manager_ = std::make_unique<msgs::ShardRequestManager<io::local_transport::LocalTransport>>(
coordinator::CoordinatorClient<io::local_transport::LocalTransport>(
query_io, interpreter_context_->coordinator_address, std::vector{interpreter_context_->coordinator_address}),
std::move(query_io));
// Get edge ids
const auto edge_ids_alloc_min_max_pair =
request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address);
if (edge_ids_alloc_min_max_pair) {
interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second};
coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
ww.operation = requests;
auto resp = interpreter_context_->io
.Request<io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>,
io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>>(
interpreter_context_->coordinator_address, ww)
.Wait();
if (resp.HasValue()) {
const auto alloc_edge_id_reps =
std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);
interpreter_context_->edge_ids_alloc = {alloc_edge_id_reps.low, alloc_edge_id_reps.high};
}
}
@@ -989,16 +878,17 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
RequestRouterInterface *request_router) {
msgs::ShardRequestManagerInterface *shard_request_manager) {
// TriggerContextCollector *trigger_context_collector = nullptr) {
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parsed_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, shard_request_manager,
storage::v3::View::OLD);
const auto memory_limit =
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
if (memory_limit) {
@@ -1013,9 +903,9 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
"convert the parsed row values to the appropriate type. This can be done using the built-in "
"conversion functions such as ToInteger, ToFloat, ToBoolean etc.");
}
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
parsed_query.parameters,
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
auto plan = CypherQueryToPlan(
parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query, parsed_query.parameters,
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, shard_request_manager);
summary->insert_or_assign("cost_estimate", plan->cost());
auto rw_type_checker = plan::ReadWriteTypeChecker();
@@ -1034,7 +924,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
}
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, request_router, memory_limit);
execution_memory, shard_request_manager, memory_limit);
// execution_memory, trigger_context_collector, memory_limit);
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
@@ -1048,7 +938,8 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
}
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
InterpreterContext *interpreter_context,
msgs::ShardRequestManagerInterface *shard_request_manager,
utils::MemoryResource *execution_memory) {
const std::string kExplainQueryStart = "explain ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
@@ -1067,20 +958,20 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
auto cypher_query_plan =
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
cypher_query, parsed_inner_query.parameters,
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr,
shard_request_manager);
std::stringstream printed_plan;
plan::PrettyPrint(*request_router, &cypher_query_plan->plan(), &printed_plan);
plan::PrettyPrint(*shard_request_manager, &cypher_query_plan->plan(), &printed_plan);
std::vector<std::vector<TypedValue>> printed_plan_rows;
for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) {
printed_plan_rows.push_back(std::vector<TypedValue>{TypedValue(row)});
}
summary->insert_or_assign("explain", plan::PlanToJson(*request_router, &cypher_query_plan->plan()).dump());
summary->insert_or_assign("explain", plan::PlanToJson(*shard_request_manager, &cypher_query_plan->plan()).dump());
return PreparedQuery{{"QUERY PLAN"},
std::move(parsed_query.required_privileges),
@@ -1097,7 +988,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router = nullptr) {
msgs::ShardRequestManagerInterface *shard_request_manager = nullptr) {
const std::string kProfileQueryStart = "profile ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -1135,26 +1026,27 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parsed_inner_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, shard_request_manager,
storage::v3::View::OLD);
const auto memory_limit =
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
auto cypher_query_plan =
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
cypher_query, parsed_inner_query.parameters,
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr,
shard_request_manager);
auto rw_type_checker = plan::ReadWriteTypeChecker();
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME", "CUSTOM DATA"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
summary, dba, interpreter_context, execution_memory, memory_limit, request_router,
summary, dba, interpreter_context, execution_memory, memory_limit, shard_request_manager,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
@@ -1163,7 +1055,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
// No output symbols are given so that nothing is streamed.
if (!stats_and_total_time) {
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
execution_memory, request_router, memory_limit)
execution_memory, shard_request_manager, memory_limit)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
}
@@ -1290,14 +1182,14 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router) {
msgs::ShardRequestManagerInterface *manager) {
if (in_explicit_transaction) {
throw UserModificationInMulticommandTxException();
}
auto *auth_query = utils::Downcast<AuthQuery>(parsed_query.query);
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, request_router);
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, manager);
SymbolTable symbol_table;
std::vector<Symbol> output_symbols;
@@ -1326,14 +1218,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
std::vector<Notification> *notifications, InterpreterContext *interpreter_context,
RequestRouterInterface *request_router) {
msgs::ShardRequestManagerInterface *manager) {
if (in_explicit_transaction) {
throw ReplicationModificationInMulticommandTxException();
}
auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context,
request_router, notifications);
auto callback =
HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context, manager, notifications);
return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -1422,14 +1314,14 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
}
PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
RequestRouterInterface *request_router) {
msgs::ShardRequestManagerInterface *manager) {
if (in_explicit_transaction) {
throw SettingConfigInMulticommandTxException{};
}
auto *setting_query = utils::Downcast<SettingQuery>(parsed_query.query);
MG_ASSERT(setting_query);
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, request_router);
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, manager);
return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges),
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -1626,7 +1518,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
if (!in_explicit_transaction_ &&
(utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ExplainQuery>(parsed_query.query) ||
utils::Downcast<ProfileQuery>(parsed_query.query))) {
request_router_->StartTransaction();
shard_request_manager_->StartTransaction();
}
utils::Timer planning_timer;
@@ -1635,14 +1527,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory,
&query_execution->notifications, request_router_.get());
&query_execution->notifications, shard_request_manager_.get());
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*request_router_, &query_execution->execution_memory_with_exception);
&*shard_request_manager_, &query_execution->execution_memory_with_exception);
} else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, request_router_.get());
prepared_query = PrepareProfileQuery(
std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get());
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
&query_execution->execution_memory);
@@ -1650,9 +1542,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
prepared_query = PrepareIndexQuery(std::move(parsed_query), in_explicit_transaction_,
&query_execution->notifications, interpreter_context_);
} else if (utils::Downcast<AuthQuery>(parsed_query.query)) {
prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, request_router_.get());
prepared_query = PrepareAuthQuery(
std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory_with_exception, shard_request_manager_.get());
} else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, interpreter_context_->db,
@@ -1663,7 +1555,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<ReplicationQuery>(parsed_query.query)) {
prepared_query =
PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
interpreter_context_, request_router_.get());
interpreter_context_, shard_request_manager_.get());
} else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_,
&*execution_db_accessor_);
@@ -1680,7 +1572,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
prepared_query =
PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
} else if (utils::Downcast<SettingQuery>(parsed_query.query)) {
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, request_router_.get());
prepared_query =
PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, shard_request_manager_.get());
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
} else if (utils::Downcast<SchemaQuery>(parsed_query.query)) {
@@ -1721,7 +1614,7 @@ void Interpreter::Commit() {
// For now, we will not check if there are some unfinished queries.
// We should document clearly that all results should be pulled to complete
// a query.
request_router_->Commit();
shard_request_manager_->Commit();
if (!db_accessor_) return;
const auto reset_necessary_members = [this]() {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -16,6 +16,7 @@
#include "coordinator/coordinator.hpp"
#include "coordinator/coordinator_client.hpp"
#include "io/local_transport/local_transport.hpp"
#include "io/transport.hpp"
#include "query/v2/auth_checker.hpp"
#include "query/v2/bindings/cypher_main_visitor.hpp"
@@ -171,8 +172,7 @@ struct PreparedQuery {
struct InterpreterContext {
explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
const std::filesystem::path &data_directory,
std::unique_ptr<RequestRouterFactory> request_router_factory,
coordinator::Address coordinator_addr);
io::Io<io::local_transport::LocalTransport> io, coordinator::Address coordinator_addr);
storage::v3::Shard *db;
@@ -188,24 +188,26 @@ struct InterpreterContext {
const InterpreterConfig config;
IdAllocator edge_ids_alloc;
// TODO (antaljanosbenjamin) Figure out an abstraction for io::Io to make it possible to construct an interpreter
// context with a simulator transport without templatizing it.
io::Io<io::local_transport::LocalTransport> io;
coordinator::Address coordinator_address;
std::unique_ptr<RequestRouterFactory> request_router_factory_;
storage::v3::LabelId NameToLabelId(std::string_view label_name) {
return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name));
return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
}
storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name));
return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
}
storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name));
return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
}
private:
// TODO Replace with local map of labels, properties and edge type ids
storage::v3::NameIdMapper query_id_mapper_;
storage::v3::NameIdMapper query_id_mapper;
};
/// Function that is used to tell all active interpreters that they should stop
@@ -294,16 +296,13 @@ class Interpreter final {
*/
void Abort();
const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
request_router_->InstallSimulatorTicker(tick_simulator);
}
const msgs::ShardRequestManagerInterface *GetShardRequestManager() const { return shard_request_manager_.get(); }
private:
struct QueryExecution {
std::optional<PreparedQuery> prepared_query;
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
std::optional<PreparedQuery> prepared_query;
std::map<std::string, TypedValue> summary;
std::vector<Notification> notifications;
@@ -343,7 +342,7 @@ class Interpreter final {
// move this unique_ptr into a shared_ptr.
std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
std::optional<DbAccessor> execution_db_accessor_;
std::unique_ptr<RequestRouterInterface> request_router_;
std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager_;
bool in_explicit_transaction_{false};
bool expect_rollback_{false};

View File

@@ -1,153 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/multiframe.hpp"
#include <algorithm>
#include <iterator>
#include "query/v2/bindings/frame.hpp"
#include "utils/pmr/vector.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
namespace memgraph::query::v2 {
static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
: frames_(utils::pmr::vector<FrameWithValidity>(
number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
MG_ASSERT(number_of_frames > 0);
}
MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
// NOLINTNEXTLINE (bugprone-exception-escape)
MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
FrameWithValidity &MultiFrame::GetFirstFrame() {
MG_ASSERT(!frames_.empty());
return frames_.front();
}
void MultiFrame::MakeAllFramesInvalid() noexcept {
std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
}
bool MultiFrame::HasValidFrame() const noexcept {
return std::any_of(frames_.begin(), frames_.end(), [](const auto &frame) { return frame.IsValid(); });
}
bool MultiFrame::HasInvalidFrame() const noexcept {
return std::any_of(frames_.rbegin(), frames_.rend(), [](const auto &frame) { return !frame.IsValid(); });
}
// NOLINTNEXTLINE (bugprone-exception-escape)
void MultiFrame::DefragmentValidFrames() noexcept {
static constexpr auto kIsValid = [](const FrameWithValidity &frame) { return frame.IsValid(); };
static constexpr auto kIsInvalid = [](const FrameWithValidity &frame) { return !frame.IsValid(); };
auto first_invalid_frame = std::find_if(frames_.begin(), frames_.end(), kIsInvalid);
auto following_first_valid = std::find_if(first_invalid_frame, frames_.end(), kIsValid);
while (first_invalid_frame != frames_.end() && following_first_valid != frames_.end()) {
std::swap(*first_invalid_frame, *following_first_valid);
first_invalid_frame++;
first_invalid_frame = std::find_if(first_invalid_frame, frames_.end(), kIsInvalid);
following_first_valid++;
following_first_valid = std::find_if(following_first_valid, frames_.end(), kIsValid);
}
}
ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
/*
From: https://en.cppreference.com/w/cpp/algorithm/find
Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
find_if searches for an element for which predicate p returns true
Return value
Iterator to the first element satisfying the condition or last if no such element is found.
-> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
*/
auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
[](const auto &frame) { return !frame.IsValid(); });
after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
}
ValidFramesReader::Iterator ValidFramesReader::begin() {
if (multiframe_->frames_[0].IsValid()) {
return Iterator{&multiframe_->frames_[0]};
}
return end();
}
ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
ValidFramesModifier::Iterator ValidFramesModifier::begin() {
if (multiframe_->frames_[0].IsValid()) {
return Iterator{&multiframe_->frames_[0], *this};
}
return end();
}
ValidFramesModifier::Iterator ValidFramesModifier::end() {
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
}
ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
// NOLINTNEXTLINE (bugprone-exception-escape)
ValidFramesConsumer::~ValidFramesConsumer() noexcept {
// TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
// cost too much to store it
multiframe_->DefragmentValidFrames();
}
ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
if (multiframe_->frames_[0].IsValid()) {
return Iterator{&multiframe_->frames_[0], *this};
}
return end();
}
ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
}
InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
for (auto &frame : multiframe_->frames_) {
if (!frame.IsValid()) {
return Iterator{&frame};
}
}
return end();
}
InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
}
} // namespace memgraph::query::v2

View File

@@ -1,308 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iterator>
#include <gflags/gflags.h>
#include "query/v2/bindings/frame.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint64(default_multi_frame_size);
namespace memgraph::query::v2 {
class ValidFramesConsumer;
class ValidFramesModifier;
class ValidFramesReader;
class InvalidFramesPopulator;
class MultiFrame {
public:
friend class ValidFramesConsumer;
friend class ValidFramesModifier;
friend class ValidFramesReader;
friend class InvalidFramesPopulator;
MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
~MultiFrame() = default;
// Assigning and moving the MultiFrame is not allowed if any accessor from the above ones are alive.
MultiFrame(const MultiFrame &other);
MultiFrame(MultiFrame &&other) noexcept;
MultiFrame &operator=(const MultiFrame &other) = delete;
MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
* state in the MultiFrame.
* Iteration goes in a deterministic order.
* One can't modify the validity of the Frame nor its content with this implementation.
*/
ValidFramesReader GetValidFramesReader();
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
* state in the MultiFrame.
* Iteration goes in a deterministic order.
* One can't modify the validity of the Frame with this implementation. One can modify its content.
*/
ValidFramesModifier GetValidFramesModifier();
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
* state in the MultiFrame.
* Iteration goes in a deterministic order.
* One can modify the validity of the Frame with this implementation.
* If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
* this is faster.
*/
ValidFramesConsumer GetValidFramesConsumer();
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
* invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
* the Frame with this implementation.
*/
InvalidFramesPopulator GetInvalidFramesPopulator();
/**
* Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
* using the iterators instead.
* The Frame can be valid or invalid.
*/
FrameWithValidity &GetFirstFrame();
void MakeAllFramesInvalid() noexcept;
bool HasValidFrame() const noexcept;
bool HasInvalidFrame() const noexcept;
inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
private:
void DefragmentValidFrames() noexcept;
utils::pmr::vector<FrameWithValidity> frames_;
};
class ValidFramesReader {
public:
explicit ValidFramesReader(MultiFrame &multiframe);
~ValidFramesReader() = default;
ValidFramesReader(const ValidFramesReader &other) = delete;
ValidFramesReader(ValidFramesReader &&other) noexcept = default;
ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = const Frame;
using pointer = value_type *;
using reference = const Frame &;
Iterator() = default;
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
Iterator &operator++() {
ptr_++;
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
ptr_++;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
};
Iterator begin();
Iterator end();
private:
FrameWithValidity *after_last_valid_frame_;
MultiFrame *multiframe_;
};
class ValidFramesModifier {
public:
explicit ValidFramesModifier(MultiFrame &multiframe);
~ValidFramesModifier() = default;
ValidFramesModifier(const ValidFramesModifier &other) = delete;
ValidFramesModifier(ValidFramesModifier &&other) noexcept = default;
ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = Frame;
using pointer = value_type *;
using reference = Frame &;
Iterator() = default;
Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
// Prefix increment
Iterator &operator++() {
do {
ptr_++;
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
++*this;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
ValidFramesModifier *iterator_wrapper_{nullptr};
};
Iterator begin();
Iterator end();
private:
MultiFrame *multiframe_;
};
class ValidFramesConsumer {
public:
explicit ValidFramesConsumer(MultiFrame &multiframe);
~ValidFramesConsumer() noexcept;
ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default;
ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = FrameWithValidity;
using pointer = value_type *;
using reference = FrameWithValidity &;
Iterator() = default;
Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
Iterator &operator++() {
do {
ptr_++;
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
++*this;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
ValidFramesConsumer *iterator_wrapper_{nullptr};
};
Iterator begin();
Iterator end();
private:
MultiFrame *multiframe_;
};
class InvalidFramesPopulator {
public:
explicit InvalidFramesPopulator(MultiFrame &multiframe);
~InvalidFramesPopulator() = default;
InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = default;
InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = FrameWithValidity;
using pointer = value_type *;
using reference = FrameWithValidity &;
Iterator() = default;
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
Iterator &operator++() {
ptr_->MakeValid();
ptr_++;
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
++ptr_;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
};
Iterator begin();
Iterator end();
private:
MultiFrame *multiframe_;
};
} // namespace memgraph::query::v2

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -149,6 +149,8 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
return true;
}
// TODO: Cost estimate ScanAllById?
// For the given op first increments the cardinality and then cost.
#define POST_VISIT_CARD_FIRST(NAME) \
bool PostVisit(NAME &) override { \

File diff suppressed because it is too large Load Diff

View File

@@ -28,7 +28,6 @@
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/bindings/frame.hpp"
#include "query/v2/bindings/symbol_table.hpp"
#include "query/v2/multiframe.hpp"
#include "storage/v3/id_types.hpp"
#include "utils/bound.hpp"
#include "utils/fnv.hpp"
@@ -72,22 +71,6 @@ class Cursor {
/// @throws QueryRuntimeException if something went wrong with execution
virtual bool Pull(Frame &, ExecutionContext &) = 0;
/// Run an iteration of a @c LogicalOperator with MultiFrame.
///
/// Since operators may be chained, the iteration may pull results from
/// multiple operators.
///
/// @param MultiFrame May be read from or written to while performing the
/// iteration.
/// @param ExecutionContext Used to get the position of symbols in frame and
/// other information.
/// @return True if the operator was able to populate at least one Frame on the MultiFrame,
/// thus if an operator returns true, that means there is at least one valid Frame in the
/// MultiFrame.
///
/// @throws QueryRuntimeException if something went wrong with execution
virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
/// Resets the Cursor to its initial state.
virtual void Reset() = 0;
@@ -127,7 +110,7 @@ class ScanAllByLabel;
class ScanAllByLabelPropertyRange;
class ScanAllByLabelPropertyValue;
class ScanAllByLabelProperty;
class ScanByPrimaryKey;
class ScanAllById;
class Expand;
class ExpandVariable;
class ConstructNamedPath;
@@ -158,7 +141,7 @@ class Foreach;
using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue,
ScanAllByLabelProperty, ScanByPrimaryKey,
ScanAllByLabelProperty, ScanAllById,
Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
@@ -349,7 +332,6 @@ and false on every following Pull.")
class OnceCursor : public Cursor {
public:
OnceCursor() {}
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
bool Pull(Frame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -859,21 +841,19 @@ given label and property.
(:serialize (:slk))
(:clone))
(lcp:define-class scan-by-primary-key (scan-all)
((label "::storage::v3::LabelId" :scope :public)
(primary-key "std::vector<Expression*>" :scope :public)
(expression "Expression *" :scope :public
(lcp:define-class scan-all-by-id (scan-all)
((expression "Expression *" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression")))
(:documentation
"ScanAll producing a single node with specified by the label and primary key")
"ScanAll producing a single node with ID equal to evaluated expression")
(:public
#>cpp
ScanByPrimaryKey() {}
ScanByPrimaryKey(const std::shared_ptr<LogicalOperator> &input,
Symbol output_symbol,
storage::v3::LabelId label,
std::vector<query::v2::Expression*> primary_key,
ScanAllById() {}
ScanAllById(const std::shared_ptr<LogicalOperator> &input,
Symbol output_symbol, Expression *expression,
storage::v3::View view = storage::v3::View::OLD);
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
@@ -1176,7 +1156,6 @@ a boolean value.")
public:
FilterCursor(const Filter &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1228,7 +1207,6 @@ RETURN clause) the Produce's pull succeeds exactly once.")
public:
ProduceCursor(const Produce &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1276,7 +1254,6 @@ Has a flag for using DETACH DELETE when deleting vertices.")
public:
DeleteCursor(const Delete &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1570,7 +1547,6 @@ edge lists).")
EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;

View File

@@ -398,7 +398,7 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
auto add_id_equal = [&](auto *maybe_id_fun, auto *val_expr) -> bool {
auto *id_fun = utils::Downcast<Function>(maybe_id_fun);
if (!id_fun) return false;
if (id_fun->function_name_ != functions::kId) return false;
if (id_fun->function_name_ != kId) return false;
if (id_fun->arguments_.size() != 1U) return false;
auto *ident = utils::Downcast<Identifier>(id_fun->arguments_.front());
if (!ident) return false;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,13 +14,13 @@
#include "query/v2/bindings/pretty_print.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "utils/string.hpp"
namespace memgraph::query::v2::plan {
PlanPrinter::PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out)
: request_router_(request_router), out_(out) {}
PlanPrinter::PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out)
: request_manager_(request_manager), out_(out) {}
#define PRE_VISIT(TOp) \
bool PlanPrinter::PreVisit(TOp &) { \
@@ -34,7 +34,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) {
WithPrintLn([&](auto &out) {
out << "* CreateExpand (" << op.input_symbol_.name() << ")"
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
<< op.edge_info_.symbol.name() << ":" << request_router_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
<< op.edge_info_.symbol.name() << ":" << request_manager_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
<< op.node_info_.symbol.name() << ")";
});
@@ -54,7 +54,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabel"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << ")";
});
return true;
}
@@ -62,8 +62,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabelPropertyValue"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
<< request_router_->PropertyToName(op.property_) << "})";
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
<< request_manager_->PropertyToName(op.property_) << "})";
});
return true;
}
@@ -71,8 +71,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabelPropertyRange"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
<< request_router_->PropertyToName(op.property_) << "})";
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
<< request_manager_->PropertyToName(op.property_) << "})";
});
return true;
}
@@ -80,16 +80,16 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabelProperty"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
<< request_router_->PropertyToName(op.property_) << "})";
<< " (" << op.output_symbol_.name() << " :" << request_manager_->LabelToName(op.label_) << " {"
<< request_manager_->PropertyToName(op.property_) << "})";
});
return true;
}
bool PlanPrinter::PreVisit(query::v2::plan::ScanByPrimaryKey &op) {
bool PlanPrinter::PreVisit(ScanAllById &op) {
WithPrintLn([&](auto &out) {
out << "* ScanByPrimaryKey"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
out << "* ScanAllById"
<< " (" << op.output_symbol_.name() << ")";
});
return true;
}
@@ -100,7 +100,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) {
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
<< op.common_.edge_symbol.name();
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
stream << ":" << request_router_->EdgeTypeToName(edge_type);
stream << ":" << request_manager_->EdgeTypeToName(edge_type);
});
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
<< op.common_.node_symbol.name() << ")";
@@ -129,7 +129,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) {
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
<< op.common_.edge_symbol.name();
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
stream << ":" << request_router_->EdgeTypeToName(edge_type);
stream << ":" << request_manager_->EdgeTypeToName(edge_type);
});
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
<< op.common_.node_symbol.name() << ")";
@@ -263,14 +263,15 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string
--depth_;
}
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out) {
PlanPrinter printer(&request_router, out);
void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root,
std::ostream *out) {
PlanPrinter printer(&request_manager, out);
// FIXME(mtomic): We should make visitors that take const arguments.
const_cast<LogicalOperator *>(plan_root)->Accept(printer);
}
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
impl::PlanToJsonVisitor visitor(&request_router);
nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) {
impl::PlanToJsonVisitor visitor(&request_manager);
// FIXME(mtomic): We should make visitors that take const arguments.
const_cast<LogicalOperator *>(plan_root)->Accept(visitor);
return visitor.output();
@@ -348,16 +349,16 @@ json ToJson(const utils::Bound<Expression *> &bound) {
json ToJson(const Symbol &symbol) { return symbol.name(); }
json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router) {
return request_router.EdgeTypeToName(edge_type);
json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager) {
return request_manager.EdgeTypeToName(edge_type);
}
json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router) {
return request_router.LabelToName(label);
json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager) {
return request_manager.LabelToName(label);
}
json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router) {
return request_router.PropertyToName(property);
json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager) {
return request_manager.PropertyToName(property);
}
json ToJson(NamedExpression *nexpr) {
@@ -368,29 +369,29 @@ json ToJson(NamedExpression *nexpr) {
}
json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
const RequestRouterInterface &request_router) {
const msgs::ShardRequestManagerInterface &request_manager) {
json json;
for (const auto &prop_pair : properties) {
json.emplace(ToJson(prop_pair.first, request_router), ToJson(prop_pair.second));
json.emplace(ToJson(prop_pair.first, request_manager), ToJson(prop_pair.second));
}
return json;
}
json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router) {
json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager) {
json self;
self["symbol"] = ToJson(node_info.symbol);
self["labels"] = ToJson(node_info.labels, request_router);
self["labels"] = ToJson(node_info.labels, request_manager);
const auto *props = std::get_if<PropertiesMapList>(&node_info.properties);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager);
return self;
}
json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router) {
json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager) {
json self;
self["symbol"] = ToJson(edge_info.symbol);
const auto *props = std::get_if<PropertiesMapList>(&edge_info.properties);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
self["edge_type"] = ToJson(edge_info.edge_type, request_router);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_manager);
self["edge_type"] = ToJson(edge_info.edge_type, request_manager);
self["direction"] = ToString(edge_info.direction);
return self;
}
@@ -432,7 +433,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
json self;
self["name"] = "ScanAllByLabel";
self["label"] = ToJson(op.label_, *request_router_);
self["label"] = ToJson(op.label_, *request_manager_);
self["output_symbol"] = ToJson(op.output_symbol_);
op.input_->Accept(*this);
@@ -445,8 +446,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
json self;
self["name"] = "ScanAllByLabelPropertyRange";
self["label"] = ToJson(op.label_, *request_router_);
self["property"] = ToJson(op.property_, *request_router_);
self["label"] = ToJson(op.label_, *request_manager_);
self["property"] = ToJson(op.property_, *request_manager_);
self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json();
self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json();
self["output_symbol"] = ToJson(op.output_symbol_);
@@ -461,8 +462,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
json self;
self["name"] = "ScanAllByLabelPropertyValue";
self["label"] = ToJson(op.label_, *request_router_);
self["property"] = ToJson(op.property_, *request_router_);
self["label"] = ToJson(op.label_, *request_manager_);
self["property"] = ToJson(op.property_, *request_manager_);
self["expression"] = ToJson(op.expression_);
self["output_symbol"] = ToJson(op.output_symbol_);
@@ -476,8 +477,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
json self;
self["name"] = "ScanAllByLabelProperty";
self["label"] = ToJson(op.label_, *request_router_);
self["property"] = ToJson(op.property_, *request_router_);
self["label"] = ToJson(op.label_, *request_manager_);
self["property"] = ToJson(op.property_, *request_manager_);
self["output_symbol"] = ToJson(op.output_symbol_);
op.input_->Accept(*this);
@@ -487,15 +488,12 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
return false;
}
bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
json self;
self["name"] = "ScanByPrimaryKey";
self["label"] = ToJson(op.label_, *request_router_);
self["name"] = "ScanAllById";
self["output_symbol"] = ToJson(op.output_symbol_);
op.input_->Accept(*this);
self["input"] = PopOutput();
output_ = std::move(self);
return false;
}
@@ -503,7 +501,7 @@ bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
bool PlanToJsonVisitor::PreVisit(CreateNode &op) {
json self;
self["name"] = "CreateNode";
self["node_info"] = ToJson(op.node_info_, *request_router_);
self["node_info"] = ToJson(op.node_info_, *request_manager_);
op.input_->Accept(*this);
self["input"] = PopOutput();
@@ -516,8 +514,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) {
json self;
self["name"] = "CreateExpand";
self["input_symbol"] = ToJson(op.input_symbol_);
self["node_info"] = ToJson(op.node_info_, *request_router_);
self["edge_info"] = ToJson(op.edge_info_, *request_router_);
self["node_info"] = ToJson(op.node_info_, *request_manager_);
self["edge_info"] = ToJson(op.edge_info_, *request_manager_);
self["existing_node"] = op.existing_node_;
op.input_->Accept(*this);
@@ -533,7 +531,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) {
self["input_symbol"] = ToJson(op.input_symbol_);
self["node_symbol"] = ToJson(op.common_.node_symbol);
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_);
self["direction"] = ToString(op.common_.direction);
self["existing_node"] = op.common_.existing_node;
@@ -550,7 +548,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) {
self["input_symbol"] = ToJson(op.input_symbol_);
self["node_symbol"] = ToJson(op.common_.node_symbol);
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
self["edge_types"] = ToJson(op.common_.edge_types, *request_manager_);
self["direction"] = ToString(op.common_.direction);
self["type"] = ToString(op.type_);
self["is_reverse"] = op.is_reverse_;
@@ -625,7 +623,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) {
bool PlanToJsonVisitor::PreVisit(SetProperty &op) {
json self;
self["name"] = "SetProperty";
self["property"] = ToJson(op.property_, *request_router_);
self["property"] = ToJson(op.property_, *request_manager_);
self["lhs"] = ToJson(op.lhs_);
self["rhs"] = ToJson(op.rhs_);
@@ -662,7 +660,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
json self;
self["name"] = "SetLabels";
self["input_symbol"] = ToJson(op.input_symbol_);
self["labels"] = ToJson(op.labels_, *request_router_);
self["labels"] = ToJson(op.labels_, *request_manager_);
op.input_->Accept(*this);
self["input"] = PopOutput();
@@ -674,7 +672,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) {
json self;
self["name"] = "RemoveProperty";
self["property"] = ToJson(op.property_, *request_router_);
self["property"] = ToJson(op.property_, *request_manager_);
self["lhs"] = ToJson(op.lhs_);
op.input_->Accept(*this);
@@ -688,7 +686,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) {
json self;
self["name"] = "RemoveLabels";
self["input_symbol"] = ToJson(op.input_symbol_);
self["labels"] = ToJson(op.labels_, *request_router_);
self["labels"] = ToJson(op.labels_, *request_manager_);
op.input_->Accept(*this);
self["input"] = PopOutput();

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,7 +18,7 @@
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/plan/operator.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
namespace memgraph::query::v2 {
@@ -27,19 +27,20 @@ namespace plan {
class LogicalOperator;
/// Pretty print a `LogicalOperator` plan to a `std::ostream`.
/// RequestRouter is needed for resolving label and property names.
/// ShardRequestManager is needed for resolving label and property names.
/// Note that `plan_root` isn't modified, but we can't take it as a const
/// because we don't have support for visiting a const LogicalOperator.
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out);
void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root,
std::ostream *out);
/// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`.
inline void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
PrettyPrint(request_router, plan_root, &std::cout);
inline void PrettyPrint(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root) {
PrettyPrint(request_manager, plan_root, &std::cout);
}
/// Convert a `LogicalOperator` plan to a JSON representation.
/// DbAccessor is needed for resolving label and property names.
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root);
nlohmann::json PlanToJson(const msgs::ShardRequestManagerInterface &request_manager, const LogicalOperator *plan_root);
class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
public:
@@ -47,7 +48,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
using HierarchicalLogicalOperatorVisitor::PreVisit;
using HierarchicalLogicalOperatorVisitor::Visit;
PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out);
PlanPrinter(const msgs::ShardRequestManagerInterface *request_manager, std::ostream *out);
bool DefaultPreVisit() override;
@@ -67,7 +68,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(ScanAllByLabelPropertyValue &) override;
bool PreVisit(ScanAllByLabelPropertyRange &) override;
bool PreVisit(ScanAllByLabelProperty &) override;
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
bool PreVisit(ScanAllById &) override;
bool PreVisit(Expand &) override;
bool PreVisit(ExpandVariable &) override;
@@ -114,7 +115,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
void Branch(LogicalOperator &op, const std::string &branch_name = "");
int64_t depth_{0};
const RequestRouterInterface *request_router_{nullptr};
const msgs::ShardRequestManagerInterface *request_manager_{nullptr};
std::ostream *out_{nullptr};
};
@@ -132,20 +133,20 @@ nlohmann::json ToJson(const utils::Bound<Expression *> &bound);
nlohmann::json ToJson(const Symbol &symbol);
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router);
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const msgs::ShardRequestManagerInterface &request_manager);
nlohmann::json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router);
nlohmann::json ToJson(storage::v3::LabelId label, const msgs::ShardRequestManagerInterface &request_manager);
nlohmann::json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router);
nlohmann::json ToJson(storage::v3::PropertyId property, const msgs::ShardRequestManagerInterface &request_manager);
nlohmann::json ToJson(NamedExpression *nexpr);
nlohmann::json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
const RequestRouterInterface &request_router);
const msgs::ShardRequestManagerInterface &request_manager);
nlohmann::json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router);
nlohmann::json ToJson(const NodeCreationInfo &node_info, const msgs::ShardRequestManagerInterface &request_manager);
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router);
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const msgs::ShardRequestManagerInterface &request_manager);
nlohmann::json ToJson(const Aggregate::Element &elem);
@@ -160,7 +161,8 @@ nlohmann::json ToJson(const std::vector<T> &items, Args &&...args) {
class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
public:
explicit PlanToJsonVisitor(const RequestRouterInterface *request_router) : request_router_(request_router) {}
explicit PlanToJsonVisitor(const msgs::ShardRequestManagerInterface *request_manager)
: request_manager_(request_manager) {}
using HierarchicalLogicalOperatorVisitor::PostVisit;
using HierarchicalLogicalOperatorVisitor::PreVisit;
@@ -194,7 +196,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(ScanAllByLabelPropertyRange &) override;
bool PreVisit(ScanAllByLabelPropertyValue &) override;
bool PreVisit(ScanAllByLabelProperty &) override;
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
bool PreVisit(ScanAllById &) override;
bool PreVisit(Produce &) override;
bool PreVisit(Accumulate &) override;
@@ -216,7 +218,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
protected:
nlohmann::json output_;
const RequestRouterInterface *request_router_;
const msgs::ShardRequestManagerInterface *request_manager_;
nlohmann::json PopOutput() {
nlohmann::json tmp;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,12 +11,10 @@
#include "query/v2/plan/read_write_type_checker.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define PRE_VISIT(TOp, RWType, continue_visiting) \
/*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \
bool ReadWriteTypeChecker::PreVisit(TOp & /*op*/) { \
UpdateType(RWType); \
return continue_visiting; \
#define PRE_VISIT(TOp, RWType, continue_visiting) \
bool ReadWriteTypeChecker::PreVisit(TOp &op) { \
UpdateType(RWType); \
return continue_visiting; \
}
namespace memgraph::query::v2::plan {
@@ -37,7 +35,7 @@ PRE_VISIT(ScanAllByLabel, RWType::R, true)
PRE_VISIT(ScanAllByLabelPropertyRange, RWType::R, true)
PRE_VISIT(ScanAllByLabelPropertyValue, RWType::R, true)
PRE_VISIT(ScanAllByLabelProperty, RWType::R, true)
PRE_VISIT(ScanByPrimaryKey, RWType::R, true)
PRE_VISIT(ScanAllById, RWType::R, true)
PRE_VISIT(Expand, RWType::R, true)
PRE_VISIT(ExpandVariable, RWType::R, true)

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -59,7 +59,7 @@ class ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(ScanAllByLabelPropertyValue &) override;
bool PreVisit(ScanAllByLabelPropertyRange &) override;
bool PreVisit(ScanAllByLabelProperty &) override;
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
bool PreVisit(ScanAllById &) override;
bool PreVisit(Expand &) override;
bool PreVisit(ExpandVariable &) override;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -25,10 +25,8 @@
#include <gflags/gflags.h>
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/plan/operator.hpp"
#include "query/v2/plan/preprocess.hpp"
#include "storage/v3/id_types.hpp"
DECLARE_int64(query_vertex_count_to_expand_existing);
@@ -273,12 +271,11 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
return true;
}
bool PreVisit(ScanByPrimaryKey &op) override {
bool PreVisit(ScanAllById &op) override {
prev_ops_.push_back(&op);
return true;
}
bool PostVisit(ScanByPrimaryKey & /*unused*/) override {
bool PostVisit(ScanAllById &) override {
prev_ops_.pop_back();
return true;
}
@@ -490,12 +487,6 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
storage::v3::PropertyId GetProperty(PropertyIx prop) { return db_->NameToProperty(prop.name); }
void EraseLabelFilters(const memgraph::query::v2::Symbol &node_symbol, memgraph::query::v2::LabelIx prim_label) {
std::vector<query::v2::Expression *> removed_expressions;
filters_.EraseLabelFilter(node_symbol, prim_label, &removed_expressions);
filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
}
std::optional<LabelIx> FindBestLabelIndex(const std::unordered_set<LabelIx> &labels) {
MG_ASSERT(!labels.empty(), "Trying to find the best label without any labels.");
std::optional<LabelIx> best_label;
@@ -568,84 +559,31 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
const auto &view = scan.view_;
const auto &modified_symbols = scan.ModifiedSymbols(*symbol_table_);
std::unordered_set<Symbol> bound_symbols(modified_symbols.begin(), modified_symbols.end());
// Try to see if we can use label + primary-key or label + property index.
// If not, try to use just the label index.
auto are_bound = [&bound_symbols](const auto &used_symbols) {
for (const auto &used_symbol : used_symbols) {
if (!utils::Contains(bound_symbols, used_symbol)) {
return false;
}
}
return true;
};
// First, try to see if we can find a vertex by ID.
if (!max_vertex_count || *max_vertex_count >= 1) {
for (const auto &filter : filters_.IdFilters(node_symbol)) {
if (filter.id_filter->is_symbol_in_value_ || !are_bound(filter.used_symbols)) continue;
auto *value = filter.id_filter->value_;
filter_exprs_for_removal_.insert(filter.expression);
filters_.EraseFilter(filter);
return std::make_unique<ScanAllById>(input, node_symbol, value, view);
}
}
// Now try to see if we can use label+property index. If not, try to use
// just the label index.
const auto labels = filters_.FilteredLabels(node_symbol);
if (labels.empty()) {
// Without labels, we cannot generate any indexed ScanAll.
return nullptr;
}
// First, try to see if we can find a vertex based on the possibly
// supplied primary key.
auto property_filters = filters_.PropertyFilters(node_symbol);
query::v2::LabelIx prim_label;
std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> primary_key;
auto extract_primary_key = [this](storage::v3::LabelId label,
std::vector<query::v2::plan::FilterInfo> property_filters)
-> std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> {
std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> pk_temp;
std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>> pk;
std::vector<memgraph::storage::v3::SchemaProperty> schema = db_->GetSchemaForLabel(label);
std::vector<storage::v3::PropertyId> schema_properties;
schema_properties.reserve(schema.size());
std::transform(schema.begin(), schema.end(), std::back_inserter(schema_properties),
[](const auto &schema_elem) { return schema_elem.property_id; });
for (const auto &property_filter : property_filters) {
if (property_filter.property_filter->type_ != PropertyFilter::Type::EQUAL) {
continue;
}
const auto &property_id = db_->NameToProperty(property_filter.property_filter->property_.name);
if (std::find(schema_properties.begin(), schema_properties.end(), property_id) != schema_properties.end()) {
pk_temp.emplace_back(std::make_pair(property_filter.expression, property_filter));
}
}
// Make sure pk is in the same order as schema_properties.
for (const auto &schema_prop : schema_properties) {
for (auto &pk_temp_prop : pk_temp) {
const auto &property_id = db_->NameToProperty(pk_temp_prop.second.property_filter->property_.name);
if (schema_prop == property_id) {
pk.push_back(pk_temp_prop);
}
}
}
MG_ASSERT(pk.size() == pk_temp.size(),
"The two vectors should represent the same primary key with a possibly different order of contained "
"elements.");
return pk.size() == schema_properties.size()
? pk
: std::vector<std::pair<query::v2::Expression *, query::v2::plan::FilterInfo>>{};
};
if (!property_filters.empty()) {
for (const auto &label : labels) {
if (db_->PrimaryLabelExists(GetLabel(label))) {
prim_label = label;
primary_key = extract_primary_key(GetLabel(prim_label), property_filters);
break;
}
}
if (!primary_key.empty()) {
// Mark the expressions so they won't be used for an additional, unnecessary filter.
for (const auto &primary_property : primary_key) {
filter_exprs_for_removal_.insert(primary_property.first);
filters_.EraseFilter(primary_property.second);
}
EraseLabelFilters(node_symbol, prim_label);
std::vector<query::v2::Expression *> pk_expressions;
std::transform(primary_key.begin(), primary_key.end(), std::back_inserter(pk_expressions),
[](const auto &exp) { return exp.second.property_filter->value_; });
return std::make_unique<ScanByPrimaryKey>(input, node_symbol, GetLabel(prim_label), pk_expressions);
}
}
auto found_index = FindBestLabelPropertyIndex(node_symbol, bound_symbols);
if (found_index &&
// Use label+property index if we satisfy max_vertex_count.
@@ -659,7 +597,9 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
filter_exprs_for_removal_.insert(found_index->filter.expression);
}
filters_.EraseFilter(found_index->filter);
EraseLabelFilters(node_symbol, found_index->label);
std::vector<Expression *> removed_expressions;
filters_.EraseLabelFilter(node_symbol, found_index->label, &removed_expressions);
filter_exprs_for_removal_.insert(removed_expressions.begin(), removed_expressions.end());
if (prop_filter.lower_bound_ || prop_filter.upper_bound_) {
return std::make_unique<ScanAllByLabelPropertyRange>(
input, node_symbol, GetLabel(found_index->label), GetProperty(prop_filter.property_),

View File

@@ -272,7 +272,7 @@ class RuleBasedPlanner {
PropertiesMapList vector_props;
vector_props.reserve(node_properties->size());
for (const auto &kv : *node_properties) {
// TODO(kostasrim) GetProperty should be implemented in terms of RequestRouter NameToProperty
// TODO(kostasrim) GetProperty should be implemented in terms of ShardRequestManager NameToProperty
vector_props.push_back({GetProperty(kv.first), kv.second});
}
return std::move(vector_props);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,17 +12,14 @@
/// @file
#pragma once
#include <iterator>
#include <optional>
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/plan/preprocess.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "storage/v3/conversions.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "utils/bound.hpp"
#include "utils/exceptions.hpp"
#include "utils/fnv.hpp"
namespace memgraph::query::v2::plan {
@@ -32,11 +29,11 @@ namespace memgraph::query::v2::plan {
template <class TDbAccessor>
class VertexCountCache {
public:
explicit VertexCountCache(TDbAccessor *request_router) : request_router_{request_router} {}
explicit VertexCountCache(TDbAccessor *shard_request_manager) : shard_request_manager_{shard_request_manager} {}
auto NameToLabel(const std::string &name) { return request_router_->NameToLabel(name); }
auto NameToProperty(const std::string &name) { return request_router_->NameToProperty(name); }
auto NameToEdgeType(const std::string &name) { return request_router_->NameToEdgeType(name); }
auto NameToLabel(const std::string &name) { return shard_request_manager_->NameToLabel(name); }
auto NameToProperty(const std::string &name) { return shard_request_manager_->NameToProperty(name); }
auto NameToEdgeType(const std::string &name) { return shard_request_manager_->NameToEdgeType(name); }
int64_t VerticesCount() { return 1; }
@@ -55,17 +52,12 @@ class VertexCountCache {
return 1;
}
bool LabelIndexExists(storage::v3::LabelId label) { return PrimaryLabelExists(label); }
bool PrimaryLabelExists(storage::v3::LabelId label) { return request_router_->IsPrimaryLabel(label); }
// For now return true if label is primary label
bool LabelIndexExists(storage::v3::LabelId label) { return shard_request_manager_->IsPrimaryLabel(label); }
bool LabelPropertyIndexExists(storage::v3::LabelId /*label*/, storage::v3::PropertyId /*property*/) { return false; }
const std::vector<memgraph::storage::v3::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) {
return request_router_->GetSchemaForLabel(label);
}
RequestRouterInterface *request_router_;
msgs::ShardRequestManagerInterface *shard_request_manager_;
};
template <class TDbAccessor>

View File

@@ -1,815 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <algorithm>
#include <chrono>
#include <deque>
#include <iostream>
#include <iterator>
#include <map>
#include <numeric>
#include <optional>
#include <random>
#include <set>
#include <stdexcept>
#include <thread>
#include <unordered_map>
#include <variant>
#include <vector>
#include <boost/uuid/uuid.hpp>
#include "coordinator/coordinator.hpp"
#include "coordinator/coordinator_client.hpp"
#include "coordinator/coordinator_rsm.hpp"
#include "coordinator/shard_map.hpp"
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/local_transport/local_transport.hpp"
#include "io/notifier.hpp"
#include "io/rsm/raft.hpp"
#include "io/rsm/rsm_client.hpp"
#include "io/rsm/shard_rsm.hpp"
#include "io/simulator/simulator.hpp"
#include "io/simulator/simulator_transport.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/value_conversions.hpp"
#include "utils/result.hpp"
namespace memgraph::query::v2 {
template <typename TStorageClient>
class RsmStorageClientManager {
public:
using CompoundKey = io::rsm::ShardRsmKey;
using ShardMetadata = coordinator::ShardMetadata;
RsmStorageClientManager() = default;
RsmStorageClientManager(const RsmStorageClientManager &) = delete;
RsmStorageClientManager(RsmStorageClientManager &&) = delete;
RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
~RsmStorageClientManager() = default;
void AddClient(ShardMetadata key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
bool Exists(const ShardMetadata &key) { return cli_cache_.contains(key); }
void PurgeCache() { cli_cache_.clear(); }
TStorageClient &GetClient(const ShardMetadata &key) {
auto it = cli_cache_.find(key);
MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
return it->second;
}
private:
std::map<ShardMetadata, TStorageClient> cli_cache_;
};
template <typename TRequest>
struct ShardRequestState {
memgraph::coordinator::ShardMetadata shard;
TRequest request;
};
// maps from ReadinessToken's internal size_t to the associated state
template <typename TRequest>
using RunningRequests = std::unordered_map<size_t, ShardRequestState<TRequest>>;
class RequestRouterInterface {
public:
using VertexAccessor = query::v2::accessors::VertexAccessor;
RequestRouterInterface() = default;
RequestRouterInterface(const RequestRouterInterface &) = delete;
RequestRouterInterface(RequestRouterInterface &&) = delete;
RequestRouterInterface &operator=(const RequestRouterInterface &) = delete;
RequestRouterInterface &&operator=(RequestRouterInterface &&) = delete;
virtual ~RequestRouterInterface() = default;
virtual void StartTransaction() = 0;
virtual void Commit() = 0;
virtual std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) = 0;
virtual std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) = 0;
virtual std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) = 0;
virtual std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) = 0;
virtual std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest request) = 0;
virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
virtual storage::v3::LabelId NameToLabel(const std::string &name) const = 0;
virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0;
virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0;
virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0;
virtual std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const = 0;
virtual std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const = 0;
virtual std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const = 0;
virtual bool IsPrimaryLabel(storage::v3::LabelId label) const = 0;
virtual bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const = 0;
virtual std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) = 0;
virtual void InstallSimulatorTicker(std::function<bool()> tick_simulator) = 0;
virtual const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const = 0;
};
// TODO(kostasrim)rename this class template
template <typename TTransport>
class RequestRouter : public RequestRouterInterface {
public:
using StorageClient = coordinator::RsmClient<TTransport, msgs::WriteRequests, msgs::WriteResponses,
msgs::ReadRequests, msgs::ReadResponses>;
using CoordinatorWriteRequests = coordinator::CoordinatorWriteRequests;
using CoordinatorClient = coordinator::CoordinatorClient<TTransport>;
using Address = io::Address;
using ShardMetadata = coordinator::ShardMetadata;
using ShardMap = coordinator::ShardMap;
using CompoundKey = coordinator::PrimaryKey;
using VertexAccessor = query::v2::accessors::VertexAccessor;
RequestRouter(CoordinatorClient coord, io::Io<TTransport> &&io) : coord_cli_(std::move(coord)), io_(std::move(io)) {}
RequestRouter(const RequestRouter &) = delete;
RequestRouter(RequestRouter &&) = delete;
RequestRouter &operator=(const RequestRouter &) = delete;
RequestRouter &operator=(RequestRouter &&) = delete;
~RequestRouter() override {}
void InstallSimulatorTicker(std::function<bool()> tick_simulator) override {
notifier_.InstallSimulatorTicker(tick_simulator);
}
void StartTransaction() override {
coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
CoordinatorWriteRequests write_req = req;
spdlog::trace("sending hlc request to start transaction");
auto write_res = coord_cli_.SendWriteRequest(write_req);
spdlog::trace("received hlc response to start transaction");
if (write_res.HasError()) {
throw std::runtime_error("HLC request failed");
}
auto coordinator_write_response = write_res.GetValue();
auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
// Transaction ID to be used later...
transaction_id_ = hlc_response.new_hlc;
if (hlc_response.fresher_shard_map) {
shards_map_ = hlc_response.fresher_shard_map.value();
SetUpNameIdMappers();
}
}
void Commit() override {
coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
CoordinatorWriteRequests write_req = req;
spdlog::trace("sending hlc request before committing transaction");
auto write_res = coord_cli_.SendWriteRequest(write_req);
spdlog::trace("received hlc response before committing transaction");
if (write_res.HasError()) {
throw std::runtime_error("HLC request for commit failed");
}
auto coordinator_write_response = write_res.GetValue();
auto hlc_response = std::get<coordinator::HlcResponse>(coordinator_write_response);
if (hlc_response.fresher_shard_map) {
shards_map_ = hlc_response.fresher_shard_map.value();
SetUpNameIdMappers();
}
auto commit_timestamp = hlc_response.new_hlc;
msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
for (const auto &[label, space] : shards_map_.label_spaces) {
for (const auto &[key, shard] : space.shards) {
auto &storage_client = GetStorageClientForShard(shard);
// TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
// instead.
auto commit_response = storage_client.SendWriteRequest(commit_req);
// RETRY on timeouts?
// Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
if (commit_response.HasError()) {
throw std::runtime_error("Commit request timed out");
}
msgs::WriteResponses write_response_variant = commit_response.GetValue();
auto &response = std::get<msgs::CommitResponse>(write_response_variant);
if (response.error) {
throw std::runtime_error("Commit request did not succeed");
}
}
}
}
storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
return shards_map_.GetEdgeTypeId(name).value();
}
storage::v3::PropertyId NameToProperty(const std::string &name) const override {
return shards_map_.GetPropertyId(name).value();
}
storage::v3::LabelId NameToLabel(const std::string &name) const override {
return shards_map_.GetLabelId(name).value();
}
const std::string &PropertyToName(storage::v3::PropertyId id) const override {
return properties_.IdToName(id.AsUint());
}
const std::string &LabelToName(storage::v3::LabelId id) const override { return labels_.IdToName(id.AsUint()); }
const std::string &EdgeTypeToName(storage::v3::EdgeTypeId id) const override {
return edge_types_.IdToName(id.AsUint());
}
bool IsPrimaryProperty(storage::v3::LabelId primary_label, storage::v3::PropertyId property) const override {
const auto schema_it = shards_map_.schemas.find(primary_label);
MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
return schema_prop.property_id == property;
}) != schema_it->second.end();
}
const std::vector<coordinator::SchemaProperty> &GetSchemaForLabel(storage::v3::LabelId label) const override {
return shards_map_.schemas.at(label);
}
bool IsPrimaryLabel(storage::v3::LabelId label) const override { return shards_map_.label_spaces.contains(label); }
// TODO(kostasrim) Simplify return result
std::vector<VertexAccessor> ScanVertices(std::optional<std::string> label) override {
// create requests
auto requests_to_be_sent = RequestsForScanVertices(label);
spdlog::trace("created {} ScanVertices requests", requests_to_be_sent.size());
// begin all requests in parallel
RunningRequests<msgs::ScanVerticesRequest> running_requests = {};
running_requests.reserve(requests_to_be_sent.size());
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
auto &request = requests_to_be_sent[i];
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
spdlog::trace("sent {} ScanVertices requests in parallel", running_requests.size());
// drive requests to completion
auto responses = DriveReadResponses<msgs::ScanVerticesRequest, msgs::ScanVerticesResponse>(running_requests);
spdlog::trace("got back {} ScanVertices responses after driving to completion", responses.size());
// convert responses into VertexAccessor objects to return
std::vector<VertexAccessor> accessors;
accessors.reserve(responses.size());
for (auto &response : responses) {
for (auto &result_row : response.results) {
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
}
}
return accessors;
}
std::vector<msgs::CreateVerticesResponse> CreateVertices(std::vector<msgs::NewVertex> new_vertices) override {
MG_ASSERT(!new_vertices.empty());
// create requests
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests_to_be_sent =
RequestsForCreateVertices(new_vertices);
spdlog::trace("created {} CreateVertices requests", requests_to_be_sent.size());
// begin all requests in parallel
RunningRequests<msgs::CreateVerticesRequest> running_requests = {};
running_requests.reserve(requests_to_be_sent.size());
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
auto &request = requests_to_be_sent[i];
io::ReadinessToken readiness_token{i};
for (auto &new_vertex : request.request.new_vertices) {
new_vertex.label_ids.erase(new_vertex.label_ids.begin());
}
auto &storage_client = GetStorageClientForShard(request.shard);
storage_client.SendAsyncWriteRequest(request.request, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
spdlog::trace("sent {} CreateVertices requests in parallel", running_requests.size());
// drive requests to completion
return DriveWriteResponses<msgs::CreateVerticesRequest, msgs::CreateVerticesResponse>(running_requests);
}
std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) override {
MG_ASSERT(!new_edges.empty());
// create requests
std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests_to_be_sent =
RequestsForCreateExpand(std::move(new_edges));
// begin all requests in parallel
RunningRequests<msgs::CreateExpandRequest> running_requests = {};
running_requests.reserve(requests_to_be_sent.size());
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
auto &request = requests_to_be_sent[i];
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
msgs::WriteRequests req = request.request;
storage_client.SendAsyncWriteRequest(req, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
// drive requests to completion
return DriveWriteResponses<msgs::CreateExpandRequest, msgs::CreateExpandResponse>(running_requests);
}
std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) override {
// TODO(kostasrim)Update to limit the batch size here
// Expansions of the destination must be handled by the caller. For example
// match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
// For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
// must be fetched again with an ExpandOne(Edges.dst)
// create requests
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests_to_be_sent = RequestsForExpandOne(request);
// begin all requests in parallel
RunningRequests<msgs::ExpandOneRequest> running_requests = {};
running_requests.reserve(requests_to_be_sent.size());
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
auto &request = requests_to_be_sent[i];
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
msgs::ReadRequests req = request.request;
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
// drive requests to completion
auto responses = DriveReadResponses<msgs::ExpandOneRequest, msgs::ExpandOneResponse>(running_requests);
// post-process responses
std::vector<msgs::ExpandOneResultRow> result_rows;
const auto total_row_count = std::accumulate(responses.begin(), responses.end(), 0,
[](const int64_t partial_count, const msgs::ExpandOneResponse &resp) {
return partial_count + resp.result.size();
});
result_rows.reserve(total_row_count);
for (auto &response : responses) {
result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
std::make_move_iterator(response.result.end()));
}
return result_rows;
}
std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest requests) override {
requests.transaction_id = transaction_id_;
// create requests
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests_to_be_sent =
RequestsForGetProperties(std::move(requests));
// begin all requests in parallel
RunningRequests<msgs::GetPropertiesRequest> running_requests = {};
running_requests.reserve(requests_to_be_sent.size());
for (size_t i = 0; i < requests_to_be_sent.size(); i++) {
auto &request = requests_to_be_sent[i];
io::ReadinessToken readiness_token{i};
auto &storage_client = GetStorageClientForShard(request.shard);
msgs::ReadRequests req = request.request;
storage_client.SendAsyncReadRequest(req, notifier_, readiness_token);
running_requests.emplace(readiness_token.GetId(), request);
}
// drive requests to completion
auto responses = DriveReadResponses<msgs::GetPropertiesRequest, msgs::GetPropertiesResponse>(running_requests);
// post-process responses
std::vector<msgs::GetPropertiesResultRow> result_rows;
for (auto &&response : responses) {
std::move(response.result_row.begin(), response.result_row.end(), std::back_inserter(result_rows));
}
return result_rows;
}
std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const override {
return shards_map_.GetPropertyId(name);
}
std::optional<storage::v3::EdgeTypeId> MaybeNameToEdgeType(const std::string &name) const override {
return shards_map_.GetEdgeTypeId(name);
}
std::optional<storage::v3::LabelId> MaybeNameToLabel(const std::string &name) const override {
return shards_map_.GetLabelId(name);
}
private:
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> RequestsForCreateVertices(
const std::vector<msgs::NewVertex> &new_vertices) {
std::map<ShardMetadata, msgs::CreateVerticesRequest> per_shard_request_table;
for (auto &new_vertex : new_vertices) {
MG_ASSERT(!new_vertex.label_ids.empty(), "No label_ids provided for new vertex in RequestRouter::CreateVertices");
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
if (!per_shard_request_table.contains(shard)) {
msgs::CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
}
per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
}
std::vector<ShardRequestState<msgs::CreateVerticesRequest>> requests = {};
for (auto &[shard, request] : per_shard_request_table) {
ShardRequestState<msgs::CreateVerticesRequest> shard_request_state{
.shard = shard,
.request = request,
};
requests.emplace_back(std::move(shard_request_state));
}
return requests;
}
std::vector<ShardRequestState<msgs::CreateExpandRequest>> RequestsForCreateExpand(
std::vector<msgs::NewExpand> new_expands) {
std::map<ShardMetadata, msgs::CreateExpandRequest> per_shard_request_table;
auto ensure_shard_exists_in_table = [&per_shard_request_table,
transaction_id = transaction_id_](const ShardMetadata &shard) {
if (!per_shard_request_table.contains(shard)) {
msgs::CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
per_shard_request_table.insert({shard, std::move(create_expand_request)});
}
};
for (auto &new_expand : new_expands) {
const auto shard_src_vertex = shards_map_.GetShardForKey(
new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
const auto shard_dest_vertex = shards_map_.GetShardForKey(
new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
ensure_shard_exists_in_table(shard_src_vertex);
if (shard_src_vertex != shard_dest_vertex) {
ensure_shard_exists_in_table(shard_dest_vertex);
per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
}
per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
}
std::vector<ShardRequestState<msgs::CreateExpandRequest>> requests = {};
for (auto &[shard, request] : per_shard_request_table) {
ShardRequestState<msgs::CreateExpandRequest> shard_request_state{
.shard = shard,
.request = request,
};
requests.emplace_back(std::move(shard_request_state));
}
return requests;
}
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> RequestsForScanVertices(
const std::optional<std::string> &label) {
std::vector<coordinator::Shards> multi_shards;
if (label) {
const auto label_id = shards_map_.GetLabelId(*label);
MG_ASSERT(label_id);
MG_ASSERT(IsPrimaryLabel(*label_id));
multi_shards = {shards_map_.GetShardsForLabel(*label)};
} else {
multi_shards = shards_map_.GetAllShards();
}
std::vector<ShardRequestState<msgs::ScanVerticesRequest>> requests = {};
for (auto &shards : multi_shards) {
for (auto &[key, shard] : shards) {
MG_ASSERT(!shard.peers.empty());
msgs::ScanVerticesRequest request;
request.transaction_id = transaction_id_;
request.start_id.second = storage::conversions::ConvertValueVector(key);
ShardRequestState<msgs::ScanVerticesRequest> shard_request_state{
.shard = shard,
.request = std::move(request),
};
requests.emplace_back(std::move(shard_request_state));
}
}
return requests;
}
std::vector<ShardRequestState<msgs::ExpandOneRequest>> RequestsForExpandOne(const msgs::ExpandOneRequest &request) {
std::map<ShardMetadata, msgs::ExpandOneRequest> per_shard_request_table;
msgs::ExpandOneRequest top_level_rqst_template = request;
top_level_rqst_template.transaction_id = transaction_id_;
top_level_rqst_template.src_vertices.clear();
for (auto &vertex : request.src_vertices) {
auto shard =
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
if (!per_shard_request_table.contains(shard)) {
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
}
per_shard_request_table[shard].src_vertices.push_back(vertex);
}
std::vector<ShardRequestState<msgs::ExpandOneRequest>> requests = {};
for (auto &[shard, request] : per_shard_request_table) {
ShardRequestState<msgs::ExpandOneRequest> shard_request_state{
.shard = shard,
.request = request,
};
requests.emplace_back(std::move(shard_request_state));
}
return requests;
}
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> RequestsForGetProperties(
msgs::GetPropertiesRequest &&request) {
std::map<ShardMetadata, msgs::GetPropertiesRequest> per_shard_request_table;
auto top_level_rqst_template = request;
top_level_rqst_template.transaction_id = transaction_id_;
top_level_rqst_template.vertex_ids.clear();
top_level_rqst_template.vertices_and_edges.clear();
for (auto &&vertex : request.vertex_ids) {
auto shard =
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
if (!per_shard_request_table.contains(shard)) {
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
}
per_shard_request_table[shard].vertex_ids.emplace_back(std::move(vertex));
}
for (auto &[vertex, maybe_edge] : request.vertices_and_edges) {
auto shard =
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
if (!per_shard_request_table.contains(shard)) {
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
}
per_shard_request_table[shard].vertices_and_edges.emplace_back(std::move(vertex), maybe_edge);
}
std::vector<ShardRequestState<msgs::GetPropertiesRequest>> requests;
for (auto &[shard, rqst] : per_shard_request_table) {
ShardRequestState<msgs::GetPropertiesRequest> shard_request_state{
.shard = shard,
.request = std::move(rqst),
};
requests.emplace_back(std::move(shard_request_state));
}
return requests;
}
StorageClient &GetStorageClientForShard(ShardMetadata shard) {
if (!storage_cli_manager_.Exists(shard)) {
AddStorageClientToManager(shard);
}
return storage_cli_manager_.GetClient(shard);
}
StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
auto shard = shards_map_.GetShardForKey(label, key);
return GetStorageClientForShard(std::move(shard));
}
void AddStorageClientToManager(ShardMetadata target_shard) {
MG_ASSERT(!target_shard.peers.empty());
auto leader_addr = target_shard.peers.front();
std::vector<Address> addresses;
addresses.reserve(target_shard.peers.size());
for (auto &address : target_shard.peers) {
addresses.push_back(std::move(address.address));
}
auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
storage_cli_manager_.AddClient(target_shard, std::move(cli));
}
template <typename RequestT, typename ResponseT>
std::vector<ResponseT> DriveReadResponses(RunningRequests<RequestT> &running_requests) {
// Store responses in a map based on the corresponding request
// offset, so that they can be reassembled in the correct order
// even if they came back in randomized orders.
std::map<size_t, ResponseT> response_map;
spdlog::trace("waiting on readiness for token");
while (response_map.size() < running_requests.size()) {
auto ready = notifier_.Await();
spdlog::trace("got readiness for token {}", ready.GetId());
auto &request = running_requests.at(ready.GetId());
auto &storage_client = GetStorageClientForShard(request.shard);
std::optional<utils::BasicResult<io::TimedOut, msgs::ReadResponses>> poll_result =
storage_client.PollAsyncReadRequest(ready);
if (!poll_result.has_value()) {
continue;
}
if (poll_result->HasError()) {
throw std::runtime_error("RequestRouter Read request timed out");
}
msgs::ReadResponses response_variant = poll_result->GetValue();
auto response = std::get<ResponseT>(response_variant);
if (response.error) {
throw std::runtime_error("RequestRouter Read request did not succeed");
}
// the readiness token has an ID based on the request vector offset
response_map.emplace(ready.GetId(), std::move(response));
}
std::vector<ResponseT> responses;
responses.reserve(running_requests.size());
int last = -1;
for (auto &&[offset, response] : response_map) {
MG_ASSERT(last + 1 == offset);
responses.emplace_back(std::forward<ResponseT>(response));
last = offset;
}
return responses;
}
template <typename RequestT, typename ResponseT>
std::vector<ResponseT> DriveWriteResponses(RunningRequests<RequestT> &running_requests) {
// Store responses in a map based on the corresponding request
// offset, so that they can be reassembled in the correct order
// even if they came back in randomized orders.
std::map<size_t, ResponseT> response_map;
while (response_map.size() < running_requests.size()) {
auto ready = notifier_.Await();
auto &request = running_requests.at(ready.GetId());
auto &storage_client = GetStorageClientForShard(request.shard);
std::optional<utils::BasicResult<io::TimedOut, msgs::WriteResponses>> poll_result =
storage_client.PollAsyncWriteRequest(ready);
if (!poll_result.has_value()) {
continue;
}
if (poll_result->HasError()) {
throw std::runtime_error("RequestRouter Write request timed out");
}
msgs::WriteResponses response_variant = poll_result->GetValue();
auto response = std::get<ResponseT>(response_variant);
if (response.error) {
throw std::runtime_error("RequestRouter Write request did not succeed");
}
// the readiness token has an ID based on the request vector offset
response_map.emplace(ready.GetId(), std::move(response));
}
std::vector<ResponseT> responses;
responses.reserve(running_requests.size());
int last = -1;
for (auto &&[offset, response] : response_map) {
MG_ASSERT(last + 1 == offset);
responses.emplace_back(std::forward<ResponseT>(response));
last = offset;
}
return responses;
}
void SetUpNameIdMappers() {
std::unordered_map<uint64_t, std::string> id_to_name;
for (const auto &[name, id] : shards_map_.labels) {
id_to_name.emplace(id.AsUint(), name);
}
labels_.StoreMapping(std::move(id_to_name));
id_to_name.clear();
for (const auto &[name, id] : shards_map_.properties) {
id_to_name.emplace(id.AsUint(), name);
}
properties_.StoreMapping(std::move(id_to_name));
id_to_name.clear();
for (const auto &[name, id] : shards_map_.edge_types) {
id_to_name.emplace(id.AsUint(), name);
}
edge_types_.StoreMapping(std::move(id_to_name));
}
std::optional<std::pair<uint64_t, uint64_t>> AllocateInitialEdgeIds(io::Address coordinator_address) override {
coordinator::CoordinatorWriteRequests requests{coordinator::AllocateEdgeIdBatchRequest{.batch_size = 1000000}};
io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests> ww;
ww.operation = requests;
auto resp =
io_.template Request<io::rsm::WriteRequest<coordinator::CoordinatorWriteRequests>,
io::rsm::WriteResponse<coordinator::CoordinatorWriteResponses>>(coordinator_address, ww)
.Wait();
if (resp.HasValue()) {
const auto alloc_edge_id_reps =
std::get<coordinator::AllocateEdgeIdBatchResponse>(resp.GetValue().message.write_return);
return std::make_pair(alloc_edge_id_reps.low, alloc_edge_id_reps.high);
}
return {};
}
ShardMap shards_map_;
storage::v3::NameIdMapper properties_;
storage::v3::NameIdMapper edge_types_;
storage::v3::NameIdMapper labels_;
CoordinatorClient coord_cli_;
RsmStorageClientManager<StorageClient> storage_cli_manager_;
io::Io<TTransport> io_;
coordinator::Hlc transaction_id_;
io::Notifier notifier_ = {};
// TODO(kostasrim) Add batch prefetching
};
class RequestRouterFactory {
public:
RequestRouterFactory() = default;
RequestRouterFactory(const RequestRouterFactory &) = delete;
RequestRouterFactory &operator=(const RequestRouterFactory &) = delete;
RequestRouterFactory(RequestRouterFactory &&) = delete;
RequestRouterFactory &operator=(RequestRouterFactory &&) = delete;
virtual ~RequestRouterFactory() = default;
virtual std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
const coordinator::Address &coordinator_address) const = 0;
};
class LocalRequestRouterFactory : public RequestRouterFactory {
using LocalTransportIo = io::Io<io::local_transport::LocalTransport>;
LocalTransportIo &io_;
public:
explicit LocalRequestRouterFactory(LocalTransportIo &io) : io_(io) {}
std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
const coordinator::Address &coordinator_address) const override {
using TransportType = io::local_transport::LocalTransport;
auto query_io = io_.ForkLocal(boost::uuids::uuid{boost::uuids::random_generator()()});
auto local_transport_io = io_.ForkLocal(boost::uuids::uuid{boost::uuids::random_generator()()});
return std::make_unique<RequestRouter<TransportType>>(
coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
std::move(local_transport_io));
}
};
class SimulatedRequestRouterFactory : public RequestRouterFactory {
io::simulator::Simulator *simulator_;
public:
explicit SimulatedRequestRouterFactory(io::simulator::Simulator &simulator) : simulator_(&simulator) {}
std::unique_ptr<RequestRouterInterface> CreateRequestRouter(
const coordinator::Address &coordinator_address) const override {
using TransportType = io::simulator::SimulatorTransport;
auto actual_transport_handle = simulator_->GetSimulatorHandle();
boost::uuids::uuid random_uuid;
io::Address unique_local_addr_query;
// The simulated RR should not introduce stochastic behavior.
random_uuid = boost::uuids::uuid{3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
unique_local_addr_query = {.unique_id = boost::uuids::uuid{4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}};
auto io = simulator_->Register(unique_local_addr_query);
auto query_io = io.ForkLocal(random_uuid);
return std::make_unique<RequestRouter<TransportType>>(
coordinator::CoordinatorClient<TransportType>(query_io, coordinator_address, {coordinator_address}),
std::move(io));
}
};
} // namespace memgraph::query::v2

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,8 +24,6 @@
#include "coordinator/hybrid_logical_clock.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "utils/fnv.hpp"
namespace memgraph::msgs {
@@ -37,7 +35,6 @@ struct Value;
struct Label {
LabelId id;
friend bool operator==(const Label &lhs, const Label &rhs) { return lhs.id == rhs.id; }
friend bool operator==(const Label &lhs, const LabelId &rhs) { return lhs.id == rhs; }
};
// TODO(kostasrim) update this with CompoundKey, same for the rest of the file.
@@ -320,15 +317,14 @@ struct Value {
}
};
struct ShardError {
common::ErrorCode code;
std::string message;
};
struct Expression {
std::string expression;
};
struct Filter {
std::string filter_expression;
};
enum class OrderingDirection { ASCENDING = 1, DESCENDING = 2 };
struct OrderBy {
@@ -365,38 +361,27 @@ struct ScanResultRow {
};
struct ScanVerticesResponse {
std::optional<ShardError> error;
bool success;
std::optional<VertexId> next_start_id;
std::vector<ScanResultRow> results;
};
using VertexOrEdgeIds = std::variant<VertexId, EdgeId>;
struct GetPropertiesRequest {
Hlc transaction_id;
std::vector<VertexId> vertex_ids;
std::vector<std::pair<VertexId, EdgeId>> vertices_and_edges;
std::optional<std::vector<PropertyId>> property_ids;
std::vector<std::string> expressions;
std::vector<OrderBy> order_by;
// Shouldn't contain mixed vertex and edge ids
VertexOrEdgeIds vertex_or_edge_ids;
std::vector<PropertyId> property_ids;
std::vector<Expression> expressions;
bool only_unique = false;
std::optional<std::vector<OrderBy>> order_by;
std::optional<size_t> limit;
// Return only the properties of the vertices or edges that the filter predicate
// evaluates to true
std::optional<std::string> filter;
};
struct GetPropertiesResultRow {
VertexId vertex;
std::optional<EdgeId> edge;
std::vector<std::pair<PropertyId, Value>> props;
std::vector<Value> evaluated_expressions;
std::optional<Filter> filter;
};
struct GetPropertiesResponse {
std::vector<GetPropertiesResultRow> result_row;
std::optional<ShardError> error;
bool success;
};
enum class EdgeDirection : uint8_t { OUT = 1, IN = 2, BOTH = 3 };
@@ -418,9 +403,7 @@ struct ExpandOneRequest {
std::vector<std::string> vertex_expressions;
std::vector<std::string> edge_expressions;
std::vector<OrderBy> order_by_vertices;
std::vector<OrderBy> order_by_edges;
std::optional<std::vector<OrderBy>> order_by;
// Limit the edges or the vertices?
std::optional<size_t> limit;
std::vector<std::string> filters;
@@ -463,16 +446,14 @@ struct ExpandOneResultRow {
};
struct ExpandOneResponse {
std::optional<ShardError> error;
bool success;
std::vector<ExpandOneResultRow> result;
};
struct UpdateVertex {
struct UpdateVertexProp {
PrimaryKey primary_key;
// Labels are first added and then removed from vertices
std::vector<LabelId> add_labels;
std::vector<LabelId> remove_labels;
std::map<PropertyId, Value> property_updates;
// This should be a map
std::vector<std::pair<PropertyId, Value>> property_updates;
};
struct UpdateEdgeProp {
@@ -499,7 +480,7 @@ struct CreateVerticesRequest {
};
struct CreateVerticesResponse {
std::optional<ShardError> error;
bool success;
};
struct DeleteVerticesRequest {
@@ -510,16 +491,16 @@ struct DeleteVerticesRequest {
};
struct DeleteVerticesResponse {
std::optional<ShardError> error;
bool success;
};
struct UpdateVerticesRequest {
Hlc transaction_id;
std::vector<UpdateVertex> update_vertices;
std::vector<UpdateVertexProp> new_properties;
};
struct UpdateVerticesResponse {
std::optional<ShardError> error;
bool success;
};
/*
@@ -541,7 +522,7 @@ struct CreateExpandRequest {
};
struct CreateExpandResponse {
std::optional<ShardError> error;
bool success;
};
struct DeleteEdgesRequest {
@@ -550,7 +531,7 @@ struct DeleteEdgesRequest {
};
struct DeleteEdgesResponse {
std::optional<ShardError> error;
bool success;
};
struct UpdateEdgesRequest {
@@ -559,7 +540,7 @@ struct UpdateEdgesRequest {
};
struct UpdateEdgesResponse {
std::optional<ShardError> error;
bool success;
};
struct CommitRequest {
@@ -568,7 +549,7 @@ struct CommitRequest {
};
struct CommitResponse {
std::optional<ShardError> error;
bool success;
};
using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
@@ -580,48 +561,3 @@ using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesRespon
CreateExpandResponse, DeleteEdgesResponse, UpdateEdgesResponse, CommitResponse>;
} // namespace memgraph::msgs
namespace std {
template <>
struct hash<memgraph::msgs::Value>;
template <>
struct hash<memgraph::msgs::VertexId> {
size_t operator()(const memgraph::msgs::VertexId &id) const {
using LabelId = memgraph::storage::v3::LabelId;
using Value = memgraph::msgs::Value;
return memgraph::utils::HashCombine<LabelId, std::vector<Value>, std::hash<LabelId>,
memgraph::utils::FnvCollection<std::vector<Value>, Value>>{}(id.first.id,
id.second);
}
};
template <>
struct hash<memgraph::msgs::Value> {
size_t operator()(const memgraph::msgs::Value &value) const {
using Type = memgraph::msgs::Value::Type;
switch (value.type) {
case Type::Null:
return std::hash<size_t>{}(0U);
case Type::Bool:
return std::hash<bool>{}(value.bool_v);
case Type::Int64:
return std::hash<int64_t>{}(value.int_v);
case Type::Double:
return std::hash<double>{}(value.double_v);
case Type::String:
return std::hash<std::string>{}(value.string_v);
case Type::List:
LOG_FATAL("Add hash for lists");
case Type::Map:
LOG_FATAL("Add hash for maps");
case Type::Vertex:
LOG_FATAL("Add hash for vertices");
case Type::Edge:
LOG_FATAL("Add hash for edges");
}
}
};
} // namespace std

View File

@@ -0,0 +1,730 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <deque>
#include <iostream>
#include <iterator>
#include <map>
#include <numeric>
#include <optional>
#include <random>
#include <set>
#include <stdexcept>
#include <thread>
#include <unordered_map>
#include <vector>
#include "coordinator/coordinator.hpp"
#include "coordinator/coordinator_client.hpp"
#include "coordinator/coordinator_rsm.hpp"
#include "coordinator/shard_map.hpp"
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/rsm/raft.hpp"
#include "io/rsm/rsm_client.hpp"
#include "io/rsm/shard_rsm.hpp"
#include "io/simulator/simulator.hpp"
#include "io/simulator/simulator_transport.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/value_conversions.hpp"
#include "utils/result.hpp"
namespace memgraph::msgs {
template <typename TStorageClient>
class RsmStorageClientManager {
public:
using CompoundKey = memgraph::io::rsm::ShardRsmKey;
using Shard = memgraph::coordinator::Shard;
using LabelId = memgraph::storage::v3::LabelId;
RsmStorageClientManager() = default;
RsmStorageClientManager(const RsmStorageClientManager &) = delete;
RsmStorageClientManager(RsmStorageClientManager &&) = delete;
RsmStorageClientManager &operator=(const RsmStorageClientManager &) = delete;
RsmStorageClientManager &operator=(RsmStorageClientManager &&) = delete;
~RsmStorageClientManager() = default;
void AddClient(Shard key, TStorageClient client) { cli_cache_.emplace(std::move(key), std::move(client)); }
bool Exists(const Shard &key) { return cli_cache_.contains(key); }
void PurgeCache() { cli_cache_.clear(); }
TStorageClient &GetClient(const Shard &key) {
auto it = cli_cache_.find(key);
MG_ASSERT(it != cli_cache_.end(), "Non-existing shard client");
return it->second;
}
private:
std::map<Shard, TStorageClient> cli_cache_;
};
template <typename TRequest>
struct ExecutionState {
using CompoundKey = memgraph::io::rsm::ShardRsmKey;
using Shard = memgraph::coordinator::Shard;
enum State : int8_t { INITIALIZING, EXECUTING, COMPLETED };
// label is optional because some operators can create/remove etc, vertices. These kind of requests contain the label
// on the request itself.
std::optional<std::string> label;
// CompoundKey is optional because some operators require to iterate over all the available keys
// of a shard. One example is ScanAll, where we only require the field label.
std::optional<CompoundKey> key;
// Transaction id to be filled by the ShardRequestManager implementation
memgraph::coordinator::Hlc transaction_id;
// Initialized by ShardRequestManager implementation. This vector is filled with the shards that
// the ShardRequestManager impl will send requests to. When a request to a shard exhausts it, meaning that
// it pulled all the requested data from the given Shard, it will be removed from the Vector. When the Vector becomes
// empty, it means that all of the requests have completed succefully.
// TODO(gvolfing)
// Maybe make this into a more complex object to be able to keep track of paginated resutls. E.g. instead of a vector
// of Shards make it into a std::vector<std::pair<Shard, PaginatedResultType>> (probably a struct instead of a pair)
// where PaginatedResultType is an enum signaling the progress on the given request. This way we can easily check if
// a partial response on a shard(if there is one) is finished and we can send off the request for the next batch.
std::vector<Shard> shard_cache;
// 1-1 mapping with `shard_cache`.
// A vector that tracks request metadata for each shard (For example, next_id for a ScanAll on Shard A)
std::vector<TRequest> requests;
State state = INITIALIZING;
};
class ShardRequestManagerInterface {
public:
using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
ShardRequestManagerInterface() = default;
ShardRequestManagerInterface(const ShardRequestManagerInterface &) = delete;
ShardRequestManagerInterface(ShardRequestManagerInterface &&) = delete;
ShardRequestManagerInterface &operator=(const ShardRequestManagerInterface &) = delete;
ShardRequestManagerInterface &&operator=(ShardRequestManagerInterface &&) = delete;
virtual ~ShardRequestManagerInterface() = default;
virtual void StartTransaction() = 0;
virtual void Commit() = 0;
virtual std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) = 0;
virtual std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
std::vector<NewVertex> new_vertices) = 0;
virtual std::vector<ExpandOneResultRow> Request(ExecutionState<ExpandOneRequest> &state,
ExpandOneRequest request) = 0;
virtual std::vector<CreateExpandResponse> Request(ExecutionState<CreateExpandRequest> &state,
std::vector<NewExpand> new_edges) = 0;
virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
virtual storage::v3::LabelId NameToLabel(const std::string &name) const = 0;
virtual const std::string &PropertyToName(memgraph::storage::v3::PropertyId prop) const = 0;
virtual const std::string &LabelToName(memgraph::storage::v3::LabelId label) const = 0;
virtual const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId type) const = 0;
virtual bool IsPrimaryLabel(LabelId label) const = 0;
virtual bool IsPrimaryKey(LabelId primary_label, PropertyId property) const = 0;
};
// TODO(kostasrim)rename this class template
template <typename TTransport>
class ShardRequestManager : public ShardRequestManagerInterface {
public:
using StorageClient =
memgraph::coordinator::RsmClient<TTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
using CoordinatorWriteRequests = memgraph::coordinator::CoordinatorWriteRequests;
using CoordinatorClient = memgraph::coordinator::CoordinatorClient<TTransport>;
using Address = memgraph::io::Address;
using Shard = memgraph::coordinator::Shard;
using ShardMap = memgraph::coordinator::ShardMap;
using CompoundKey = memgraph::coordinator::PrimaryKey;
using VertexAccessor = memgraph::query::v2::accessors::VertexAccessor;
ShardRequestManager(CoordinatorClient coord, memgraph::io::Io<TTransport> &&io)
: coord_cli_(std::move(coord)), io_(std::move(io)) {}
ShardRequestManager(const ShardRequestManager &) = delete;
ShardRequestManager(ShardRequestManager &&) = delete;
ShardRequestManager &operator=(const ShardRequestManager &) = delete;
ShardRequestManager &operator=(ShardRequestManager &&) = delete;
~ShardRequestManager() override {}
void StartTransaction() override {
memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
CoordinatorWriteRequests write_req = req;
auto write_res = coord_cli_.SendWriteRequest(write_req);
if (write_res.HasError()) {
throw std::runtime_error("HLC request failed");
}
auto coordinator_write_response = write_res.GetValue();
auto hlc_response = std::get<memgraph::coordinator::HlcResponse>(coordinator_write_response);
// Transaction ID to be used later...
transaction_id_ = hlc_response.new_hlc;
if (hlc_response.fresher_shard_map) {
shards_map_ = hlc_response.fresher_shard_map.value();
SetUpNameIdMappers();
}
}
void Commit() override {
memgraph::coordinator::HlcRequest req{.last_shard_map_version = shards_map_.GetHlc()};
CoordinatorWriteRequests write_req = req;
auto write_res = coord_cli_.SendWriteRequest(write_req);
if (write_res.HasError()) {
throw std::runtime_error("HLC request for commit failed");
}
auto coordinator_write_response = write_res.GetValue();
auto hlc_response = std::get<memgraph::coordinator::HlcResponse>(coordinator_write_response);
if (hlc_response.fresher_shard_map) {
shards_map_ = hlc_response.fresher_shard_map.value();
SetUpNameIdMappers();
}
auto commit_timestamp = hlc_response.new_hlc;
msgs::CommitRequest commit_req{.transaction_id = transaction_id_, .commit_timestamp = commit_timestamp};
for (const auto &[label, space] : shards_map_.label_spaces) {
for (const auto &[key, shard] : space.shards) {
auto &storage_client = GetStorageClientForShard(shard);
// TODO(kostasrim) Currently requests return the result directly. Adjust this when the API works MgFuture
// instead.
auto commit_response = storage_client.SendWriteRequest(commit_req);
// RETRY on timeouts?
// Sometimes this produces a timeout. Temporary solution is to use a while(true) as was done in shard_map test
if (commit_response.HasError()) {
throw std::runtime_error("Commit request timed out");
}
WriteResponses write_response_variant = commit_response.GetValue();
auto &response = std::get<CommitResponse>(write_response_variant);
if (!response.success) {
throw std::runtime_error("Commit request did not succeed");
}
}
}
}
storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const override {
return shards_map_.GetEdgeTypeId(name).value();
}
storage::v3::PropertyId NameToProperty(const std::string &name) const override {
return shards_map_.GetPropertyId(name).value();
}
storage::v3::LabelId NameToLabel(const std::string &name) const override {
return shards_map_.GetLabelId(name).value();
}
const std::string &PropertyToName(memgraph::storage::v3::PropertyId id) const override {
return properties_.IdToName(id.AsUint());
}
const std::string &LabelToName(memgraph::storage::v3::LabelId id) const override {
return labels_.IdToName(id.AsUint());
}
const std::string &EdgeTypeToName(memgraph::storage::v3::EdgeTypeId id) const override {
return edge_types_.IdToName(id.AsUint());
}
bool IsPrimaryKey(LabelId primary_label, PropertyId property) const override {
const auto schema_it = shards_map_.schemas.find(primary_label);
MG_ASSERT(schema_it != shards_map_.schemas.end(), "Invalid primary label id: {}", primary_label.AsUint());
return std::find_if(schema_it->second.begin(), schema_it->second.end(), [property](const auto &schema_prop) {
return schema_prop.property_id == property;
}) != schema_it->second.end();
}
bool IsPrimaryLabel(LabelId label) const override { return shards_map_.label_spaces.contains(label); }
// TODO(kostasrim) Simplify return result
std::vector<VertexAccessor> Request(ExecutionState<ScanVerticesRequest> &state) override {
MaybeInitializeExecutionState(state);
std::vector<ScanVerticesResponse> responses;
SendAllRequests(state);
auto all_requests_gathered = [](auto &paginated_rsp_tracker) {
return std::ranges::all_of(paginated_rsp_tracker, [](const auto &state) {
return state.second == PaginatedResponseState::PartiallyFinished;
});
};
std::map<Shard, PaginatedResponseState> paginated_response_tracker;
for (const auto &shard : state.shard_cache) {
paginated_response_tracker.insert(std::make_pair(shard, PaginatedResponseState::Pending));
}
do {
AwaitOnPaginatedRequests(state, responses, paginated_response_tracker);
} while (!all_requests_gathered(paginated_response_tracker));
MaybeCompleteState(state);
// TODO(kostasrim) Before returning start prefetching the batch (this shall be done once we get MgFuture as return
// result of storage_client.SendReadRequest()).
return PostProcess(std::move(responses));
}
std::vector<CreateVerticesResponse> Request(ExecutionState<CreateVerticesRequest> &state,
std::vector<NewVertex> new_vertices) override {
MG_ASSERT(!new_vertices.empty());
MaybeInitializeExecutionState(state, new_vertices);
std::vector<CreateVerticesResponse> responses;
auto &shard_cache_ref = state.shard_cache;
// 1. Send the requests.
SendAllRequests(state, shard_cache_ref);
// 2. Block untill all the futures are exhausted
do {
AwaitOnResponses(state, responses);
} while (!state.shard_cache.empty());
MaybeCompleteState(state);
// TODO(kostasrim) Before returning start prefetching the batch (this shall be done once we get MgFuture as return
// result of storage_client.SendReadRequest()).
return responses;
}
std::vector<CreateExpandResponse> Request(ExecutionState<CreateExpandRequest> &state,
std::vector<NewExpand> new_edges) override {
MG_ASSERT(!new_edges.empty());
MaybeInitializeExecutionState(state, new_edges);
std::vector<CreateExpandResponse> responses;
auto &shard_cache_ref = state.shard_cache;
size_t id{0};
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++id) {
auto &storage_client = GetStorageClientForShard(*shard_it);
WriteRequests req = state.requests[id];
auto write_response_result = storage_client.SendWriteRequest(std::move(req));
if (write_response_result.HasError()) {
throw std::runtime_error("CreateVertices request timedout");
}
WriteResponses response_variant = write_response_result.GetValue();
CreateExpandResponse mapped_response = std::get<CreateExpandResponse>(response_variant);
if (!mapped_response.success) {
throw std::runtime_error("CreateExpand request did not succeed");
}
responses.push_back(mapped_response);
shard_it = shard_cache_ref.erase(shard_it);
}
// We are done with this state
MaybeCompleteState(state);
return responses;
}
std::vector<ExpandOneResultRow> Request(ExecutionState<ExpandOneRequest> &state, ExpandOneRequest request) override {
// TODO(kostasrim)Update to limit the batch size here
// Expansions of the destination must be handled by the caller. For example
// match (u:L1 { prop : 1 })-[:Friend]-(v:L1)
// For each vertex U, the ExpandOne will result in <U, Edges>. The destination vertex and its properties
// must be fetched again with an ExpandOne(Edges.dst)
MaybeInitializeExecutionState(state, std::move(request));
std::vector<ExpandOneResponse> responses;
auto &shard_cache_ref = state.shard_cache;
// 1. Send the requests.
SendAllRequests(state, shard_cache_ref);
// 2. Block untill all the futures are exhausted
do {
AwaitOnResponses(state, responses);
} while (!state.shard_cache.empty());
std::vector<ExpandOneResultRow> result_rows;
const auto total_row_count = std::accumulate(
responses.begin(), responses.end(), 0,
[](const int64_t partial_count, const ExpandOneResponse &resp) { return partial_count + resp.result.size(); });
result_rows.reserve(total_row_count);
for (auto &response : responses) {
result_rows.insert(result_rows.end(), std::make_move_iterator(response.result.begin()),
std::make_move_iterator(response.result.end()));
}
MaybeCompleteState(state);
return result_rows;
}
private:
enum class PaginatedResponseState { Pending, PartiallyFinished };
std::vector<VertexAccessor> PostProcess(std::vector<ScanVerticesResponse> &&responses) const {
std::vector<VertexAccessor> accessors;
for (auto &response : responses) {
for (auto &result_row : response.results) {
accessors.emplace_back(VertexAccessor(std::move(result_row.vertex), std::move(result_row.props), this));
}
}
return accessors;
}
template <typename ExecutionState>
void ThrowIfStateCompleted(ExecutionState &state) const {
if (state.state == ExecutionState::COMPLETED) [[unlikely]] {
throw std::runtime_error("State is completed and must be reset");
}
}
template <typename ExecutionState>
void MaybeCompleteState(ExecutionState &state) const {
if (state.requests.empty()) {
state.state = ExecutionState::COMPLETED;
}
}
template <typename ExecutionState>
bool ShallNotInitializeState(ExecutionState &state) const {
return state.state != ExecutionState::INITIALIZING;
}
void MaybeInitializeExecutionState(ExecutionState<CreateVerticesRequest> &state,
std::vector<NewVertex> new_vertices) {
ThrowIfStateCompleted(state);
if (ShallNotInitializeState(state)) {
return;
}
state.transaction_id = transaction_id_;
std::map<Shard, CreateVerticesRequest> per_shard_request_table;
for (auto &new_vertex : new_vertices) {
MG_ASSERT(!new_vertex.label_ids.empty(), "This is error!");
auto shard = shards_map_.GetShardForKey(new_vertex.label_ids[0].id,
storage::conversions::ConvertPropertyVector(new_vertex.primary_key));
if (!per_shard_request_table.contains(shard)) {
CreateVerticesRequest create_v_rqst{.transaction_id = transaction_id_};
per_shard_request_table.insert(std::pair(shard, std::move(create_v_rqst)));
state.shard_cache.push_back(shard);
}
per_shard_request_table[shard].new_vertices.push_back(std::move(new_vertex));
}
for (auto &[shard, rqst] : per_shard_request_table) {
state.requests.push_back(std::move(rqst));
}
state.state = ExecutionState<CreateVerticesRequest>::EXECUTING;
}
void MaybeInitializeExecutionState(ExecutionState<CreateExpandRequest> &state, std::vector<NewExpand> new_expands) {
ThrowIfStateCompleted(state);
if (ShallNotInitializeState(state)) {
return;
}
state.transaction_id = transaction_id_;
std::map<Shard, CreateExpandRequest> per_shard_request_table;
auto ensure_shard_exists_in_table = [&per_shard_request_table,
transaction_id = transaction_id_](const Shard &shard) {
if (!per_shard_request_table.contains(shard)) {
CreateExpandRequest create_expand_request{.transaction_id = transaction_id};
per_shard_request_table.insert({shard, std::move(create_expand_request)});
}
};
for (auto &new_expand : new_expands) {
const auto shard_src_vertex = shards_map_.GetShardForKey(
new_expand.src_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.src_vertex.second));
const auto shard_dest_vertex = shards_map_.GetShardForKey(
new_expand.dest_vertex.first.id, storage::conversions::ConvertPropertyVector(new_expand.dest_vertex.second));
ensure_shard_exists_in_table(shard_src_vertex);
if (shard_src_vertex != shard_dest_vertex) {
ensure_shard_exists_in_table(shard_dest_vertex);
per_shard_request_table[shard_dest_vertex].new_expands.push_back(new_expand);
}
per_shard_request_table[shard_src_vertex].new_expands.push_back(std::move(new_expand));
}
for (auto &[shard, request] : per_shard_request_table) {
state.shard_cache.push_back(shard);
state.requests.push_back(std::move(request));
}
state.state = ExecutionState<CreateExpandRequest>::EXECUTING;
}
void MaybeInitializeExecutionState(ExecutionState<ScanVerticesRequest> &state) {
ThrowIfStateCompleted(state);
if (ShallNotInitializeState(state)) {
return;
}
std::vector<coordinator::Shards> multi_shards;
state.transaction_id = transaction_id_;
if (!state.label) {
multi_shards = shards_map_.GetAllShards();
} else {
const auto label_id = shards_map_.GetLabelId(*state.label);
MG_ASSERT(label_id);
MG_ASSERT(IsPrimaryLabel(*label_id));
multi_shards = {shards_map_.GetShardsForLabel(*state.label)};
}
for (auto &shards : multi_shards) {
for (auto &[key, shard] : shards) {
MG_ASSERT(!shard.empty());
state.shard_cache.push_back(std::move(shard));
ScanVerticesRequest rqst;
rqst.transaction_id = transaction_id_;
rqst.start_id.second = storage::conversions::ConvertValueVector(key);
state.requests.push_back(std::move(rqst));
}
}
state.state = ExecutionState<ScanVerticesRequest>::EXECUTING;
}
void MaybeInitializeExecutionState(ExecutionState<ExpandOneRequest> &state, ExpandOneRequest request) {
ThrowIfStateCompleted(state);
if (ShallNotInitializeState(state)) {
return;
}
state.transaction_id = transaction_id_;
std::map<Shard, ExpandOneRequest> per_shard_request_table;
auto top_level_rqst_template = request;
top_level_rqst_template.transaction_id = transaction_id_;
top_level_rqst_template.src_vertices.clear();
state.requests.clear();
for (auto &vertex : request.src_vertices) {
auto shard =
shards_map_.GetShardForKey(vertex.first.id, storage::conversions::ConvertPropertyVector(vertex.second));
if (!per_shard_request_table.contains(shard)) {
per_shard_request_table.insert(std::pair(shard, top_level_rqst_template));
state.shard_cache.push_back(shard);
}
per_shard_request_table[shard].src_vertices.push_back(vertex);
}
for (auto &[shard, rqst] : per_shard_request_table) {
state.requests.push_back(std::move(rqst));
}
state.state = ExecutionState<ExpandOneRequest>::EXECUTING;
}
StorageClient &GetStorageClientForShard(Shard shard) {
if (!storage_cli_manager_.Exists(shard)) {
AddStorageClientToManager(shard);
}
return storage_cli_manager_.GetClient(shard);
}
StorageClient &GetStorageClientForShard(const std::string &label, const CompoundKey &key) {
auto shard = shards_map_.GetShardForKey(label, key);
return GetStorageClientForShard(std::move(shard));
}
void AddStorageClientToManager(Shard target_shard) {
MG_ASSERT(!target_shard.empty());
auto leader_addr = target_shard.front();
std::vector<Address> addresses;
addresses.reserve(target_shard.size());
for (auto &address : target_shard) {
addresses.push_back(std::move(address.address));
}
auto cli = StorageClient(io_, std::move(leader_addr.address), std::move(addresses));
storage_cli_manager_.AddClient(target_shard, std::move(cli));
}
void SendAllRequests(ExecutionState<ScanVerticesRequest> &state) {
int64_t shard_idx = 0;
for (const auto &request : state.requests) {
const auto &current_shard = state.shard_cache[shard_idx];
auto &storage_client = GetStorageClientForShard(current_shard);
ReadRequests req = request;
storage_client.SendAsyncReadRequest(request);
++shard_idx;
}
}
void SendAllRequests(ExecutionState<CreateVerticesRequest> &state,
std::vector<memgraph::coordinator::Shard> &shard_cache_ref) {
size_t id = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++shard_it) {
// This is fine because all new_vertices of each request end up on the same shard
const auto labels = state.requests[id].new_vertices[0].label_ids;
auto req_deep_copy = state.requests[id];
for (auto &new_vertex : req_deep_copy.new_vertices) {
new_vertex.label_ids.erase(new_vertex.label_ids.begin());
}
auto &storage_client = GetStorageClientForShard(*shard_it);
WriteRequests req = req_deep_copy;
storage_client.SendAsyncWriteRequest(req);
++id;
}
}
void SendAllRequests(ExecutionState<ExpandOneRequest> &state,
std::vector<memgraph::coordinator::Shard> &shard_cache_ref) {
size_t id = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end(); ++shard_it) {
auto &storage_client = GetStorageClientForShard(*shard_it);
ReadRequests req = state.requests[id];
storage_client.SendAsyncReadRequest(req);
++id;
}
}
void AwaitOnResponses(ExecutionState<CreateVerticesRequest> &state, std::vector<CreateVerticesResponse> &responses) {
auto &shard_cache_ref = state.shard_cache;
int64_t request_idx = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
// This is fine because all new_vertices of each request end up on the same shard
const auto labels = state.requests[request_idx].new_vertices[0].label_ids;
auto &storage_client = GetStorageClientForShard(*shard_it);
auto poll_result = storage_client.AwaitAsyncWriteRequest();
if (!poll_result) {
++shard_it;
++request_idx;
continue;
}
if (poll_result->HasError()) {
throw std::runtime_error("CreateVertices request timed out");
}
WriteResponses response_variant = poll_result->GetValue();
auto response = std::get<CreateVerticesResponse>(response_variant);
if (!response.success) {
throw std::runtime_error("CreateVertices request did not succeed");
}
responses.push_back(response);
shard_it = shard_cache_ref.erase(shard_it);
// Needed to maintain the 1-1 mapping between the ShardCache and the requests.
auto it = state.requests.begin() + request_idx;
state.requests.erase(it);
}
}
void AwaitOnResponses(ExecutionState<ExpandOneRequest> &state, std::vector<ExpandOneResponse> &responses) {
auto &shard_cache_ref = state.shard_cache;
int64_t request_idx = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
auto &storage_client = GetStorageClientForShard(*shard_it);
auto poll_result = storage_client.PollAsyncReadRequest();
if (!poll_result) {
++shard_it;
++request_idx;
continue;
}
if (poll_result->HasError()) {
throw std::runtime_error("ExpandOne request timed out");
}
ReadResponses response_variant = poll_result->GetValue();
auto response = std::get<ExpandOneResponse>(response_variant);
// -NOTE-
// Currently a boolean flag for signaling the overall success of the
// ExpandOne request does not exist. But it should, so here we assume
// that it is already in place.
if (!response.success) {
throw std::runtime_error("ExpandOne request did not succeed");
}
responses.push_back(std::move(response));
shard_it = shard_cache_ref.erase(shard_it);
// Needed to maintain the 1-1 mapping between the ShardCache and the requests.
auto it = state.requests.begin() + request_idx;
state.requests.erase(it);
}
}
void AwaitOnPaginatedRequests(ExecutionState<ScanVerticesRequest> &state,
std::vector<ScanVerticesResponse> &responses,
std::map<Shard, PaginatedResponseState> &paginated_response_tracker) {
auto &shard_cache_ref = state.shard_cache;
// Find the first request that is not holding a paginated response.
int64_t request_idx = 0;
for (auto shard_it = shard_cache_ref.begin(); shard_it != shard_cache_ref.end();) {
if (paginated_response_tracker.at(*shard_it) != PaginatedResponseState::Pending) {
++shard_it;
++request_idx;
continue;
}
auto &storage_client = GetStorageClientForShard(*shard_it);
auto await_result = storage_client.AwaitAsyncReadRequest();
if (!await_result) {
// Redirection has occured.
++shard_it;
++request_idx;
continue;
}
if (await_result->HasError()) {
throw std::runtime_error("ScanAll request timed out");
}
ReadResponses read_response_variant = await_result->GetValue();
auto response = std::get<ScanVerticesResponse>(read_response_variant);
if (!response.success) {
throw std::runtime_error("ScanAll request did not succeed");
}
if (!response.next_start_id) {
paginated_response_tracker.erase((*shard_it));
shard_cache_ref.erase(shard_it);
// Needed to maintain the 1-1 mapping between the ShardCache and the requests.
auto it = state.requests.begin() + request_idx;
state.requests.erase(it);
} else {
state.requests[request_idx].start_id.second = response.next_start_id->second;
paginated_response_tracker[*shard_it] = PaginatedResponseState::PartiallyFinished;
}
responses.push_back(std::move(response));
}
}
void SetUpNameIdMappers() {
std::unordered_map<uint64_t, std::string> id_to_name;
for (const auto &[name, id] : shards_map_.labels) {
id_to_name.emplace(id.AsUint(), name);
}
labels_.StoreMapping(std::move(id_to_name));
id_to_name.clear();
for (const auto &[name, id] : shards_map_.properties) {
id_to_name.emplace(id.AsUint(), name);
}
properties_.StoreMapping(std::move(id_to_name));
id_to_name.clear();
for (const auto &[name, id] : shards_map_.edge_types) {
id_to_name.emplace(id.AsUint(), name);
}
edge_types_.StoreMapping(std::move(id_to_name));
}
ShardMap shards_map_;
storage::v3::NameIdMapper properties_;
storage::v3::NameIdMapper edge_types_;
storage::v3::NameIdMapper labels_;
CoordinatorClient coord_cli_;
RsmStorageClientManager<StorageClient> storage_cli_manager_;
memgraph::io::Io<TTransport> io_;
memgraph::coordinator::Hlc transaction_id_;
// TODO(kostasrim) Add batch prefetching
};
} // namespace memgraph::msgs

View File

@@ -97,15 +97,15 @@ bool LastCommittedVersionHasLabelProperty(const Vertex &vertex, LabelId label, c
if (delta->label == label) {
MG_ASSERT(!has_label, "Invalid database state!");
has_label = true;
break;
}
break;
}
case Delta::Action::REMOVE_LABEL: {
if (delta->label == label) {
MG_ASSERT(has_label, "Invalid database state!");
has_label = false;
break;
}
break;
}
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::ADD_OUT_EDGE:

View File

@@ -9,16 +9,19 @@ set(storage_v3_src_files
temporal.cpp
edge_accessor.cpp
indices.cpp
key_store.cpp
lexicographically_ordered_vertex.cpp
property_store.cpp
vertex_accessor.cpp
schemas.cpp
schema_validator.cpp
shard.cpp
storage.cpp
shard_rsm.cpp
bindings/typed_value.cpp
expr.cpp
vertex.cpp
request_helper.cpp)
request_helper.cpp
storage.cpp)
# ######################
find_package(gflags REQUIRED)
@@ -30,4 +33,4 @@ target_link_libraries(mg-storage-v3 Threads::Threads mg-utils gflags)
target_include_directories(mg-storage-v3 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
add_dependencies(mg-storage-v3 generate_lcp_storage)
target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io mg-functions)
target_link_libraries(mg-storage-v3 mg-slk mg-expr mg-io)

View File

@@ -24,8 +24,6 @@
#include "storage/v3/conversions.hpp"
#include "storage/v3/path.hpp"
#include "utils/typeinfo.hpp"
#include "utils/exceptions.hpp"
#include "functions/awesome_memgraph_functions.hpp"
cpp<#
@@ -180,6 +178,14 @@ cpp<#
(:serialize (:slk :load-args '((storage "storage::v3::AstStorage *")))))
#>cpp
struct FunctionContext {
DbAccessor *db_accessor;
utils::MemoryResource *memory;
int64_t timestamp;
std::unordered_map<std::string, int64_t> *counters;
View view;
};
inline bool operator==(const LabelIx &a, const LabelIx &b) {
return a.ix == b.ix && a.name == b.name;
}
@@ -839,24 +845,16 @@ cpp<#
:slk-load (slk-load-ast-vector "Expression"))
(function-name "std::string" :scope :public)
(function "std::function<TypedValue(const TypedValue *, int64_t,
const functions::FunctionContext<memgraph::storage::v3::DbAccessor> &)>"
const FunctionContext &)>"
:scope :public
:dont-save t
:clone :copy
:slk-load (lambda (member)
#>cpp
self->${member} = functions::NameToFunction<memgraph::storage::v3::TypedValue,
functions::FunctionContext<memgraph::storage::v3::DbAccessor>,
functions::StorageEngineTag, Conv>(self->function_name_);
cpp<#)))
(:public
#>cpp
Function() = default;
using Conv = decltype(PropertyToTypedValueFunctor<TypedValue>);
class SemanticException : public memgraph::utils::BasicException {
using utils::BasicException::BasicException;
};
DEFVISITABLE(ExpressionVisitor<TypedValue>);
DEFVISITABLE(ExpressionVisitor<void>);
@@ -874,13 +872,7 @@ cpp<#
Function(const std::string &function_name,
const std::vector<Expression *> &arguments)
: arguments_(arguments),
function_name_(function_name),
function_(functions::NameToFunction<memgraph::storage::v3::TypedValue,
functions::FunctionContext<memgraph::storage::v3::DbAccessor>,
functions::StorageEngineTag, Conv>(function_name_)) {
if (!function_) {
throw SemanticException("Function '{}' doesn't exist.", function_name);
}
function_name_(function_name) {
}
cpp<#)
(:private

View File

@@ -78,8 +78,8 @@ class DbAccessor final {
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
}
storage::v3::ShardResult<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::v3::EdgeTypeId &edge_type) {
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::v3::EdgeTypeId &edge_type) {
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
auto maybe_edge = accessor_->CreateEdge(from->Id(storage::v3::View::NEW).GetValue(),
to->Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
@@ -87,8 +87,8 @@ class DbAccessor final {
return EdgeAccessor(*maybe_edge);
}
storage::v3::ShardResult<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(edge->From(), edge->To(), edge->Gid());
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(edge->FromVertex(), edge->ToVertex(), edge->Gid());
if (res.HasError()) {
return res.GetError();
}
@@ -101,7 +101,7 @@ class DbAccessor final {
return std::make_optional<EdgeAccessor>(*value);
}
storage::v3::ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
@@ -125,7 +125,7 @@ class DbAccessor final {
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
}
storage::v3::ShardResult<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
auto res = accessor_->DeleteVertex(vertex_accessor);
if (res.HasError()) {
return res.GetError();

View File

@@ -21,7 +21,6 @@
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
#include "utils/memory.hpp"
@@ -88,6 +87,6 @@ struct EvaluationContext {
using ExpressionEvaluator =
memgraph::expr::ExpressionEvaluator<TypedValue, EvaluationContext, DbAccessor, storage::v3::View,
storage::v3::LabelId, storage::v3::PropertyStore, PropertyToTypedValueConverter,
common::ErrorCode>;
memgraph::storage::v3::Error>;
} // namespace memgraph::storage::v3

View File

@@ -17,5 +17,5 @@
#include "storage/v3/bindings/typed_value.hpp"
namespace memgraph::storage::v3 {
using Frame = memgraph::expr::Frame;
using Frame = memgraph::expr::Frame<TypedValue>;
} // namespace memgraph::storage::v3

View File

@@ -69,10 +69,6 @@ TTypedValue PropertyToTypedValue(const PropertyValue &value) {
LOG_FATAL("Unsupported type");
}
template <typename TypedValueT>
inline const auto PropertyToTypedValueFunctor =
[](const PropertyValue &value) { return PropertyToTypedValue<TypedValueT>(value); };
template <typename TTypedValue>
TTypedValue PropertyToTypedValue(const PropertyValue &value, utils::MemoryResource *mem) {
switch (value.type()) {

View File

@@ -11,18 +11,17 @@
#pragma once
#include <cstdint>
#include <memory>
#include "storage/v3/edge_ref.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_id.hpp"
#include "utils/logging.hpp"
namespace memgraph::storage::v3 {
// Forward declarations because we only store pointers here.
struct Vertex;
struct Edge;
struct Delta;
struct CommitInfo;
@@ -130,7 +129,7 @@ inline bool operator==(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer
inline bool operator!=(const PreviousPtr::Pointer &a, const PreviousPtr::Pointer &b) { return !(a == b); }
struct Delta {
enum class Action : uint8_t {
enum class Action {
// Used for both Vertex and Edge
DELETE_OBJECT,
RECREATE_OBJECT,

View File

@@ -21,7 +21,7 @@
namespace memgraph::storage::v3 {
using EdgeContainer = std::map<Gid, Edge>;
struct Vertex;
struct Edge {
Edge(Gid gid, Delta *delta) : gid(gid), deleted(false), delta(delta) {

View File

@@ -15,7 +15,6 @@
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/memory_tracker.hpp"
@@ -51,17 +50,17 @@ bool EdgeAccessor::IsVisible(const View view) const {
return exists && (for_deleted_ || !deleted);
}
const VertexId &EdgeAccessor::From() const { return from_vertex_; }
const VertexId &EdgeAccessor::FromVertex() const { return from_vertex_; }
const VertexId &EdgeAccessor::To() const { return to_vertex_; }
const VertexId &EdgeAccessor::ToVertex() const { return to_vertex_; }
ShardResult<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
Result<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const PropertyValue &value) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
if (!config_.properties_on_edges) return SHARD_ERROR(ErrorCode::PROPERTIES_DISABLED);
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
if (!PrepareForWrite(transaction_, edge_.ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
if (edge_.ptr->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
auto current_value = edge_.ptr->properties.GetProperty(property);
// We could skip setting the value if the previous one is the same to the new
@@ -76,12 +75,12 @@ ShardResult<PropertyValue> EdgeAccessor::SetProperty(PropertyId property, const
return std::move(current_value);
}
ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
if (!config_.properties_on_edges) return SHARD_ERROR(ErrorCode::PROPERTIES_DISABLED);
Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties() {
if (!config_.properties_on_edges) return Error::PROPERTIES_DISABLED;
if (!PrepareForWrite(transaction_, edge_.ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (!PrepareForWrite(transaction_, edge_.ptr)) return Error::SERIALIZATION_ERROR;
if (edge_.ptr->deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
if (edge_.ptr->deleted) return Error::DELETED_OBJECT;
auto properties = edge_.ptr->properties.Properties();
for (const auto &property : properties) {
@@ -93,11 +92,11 @@ ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::ClearProperties()
return std::move(properties);
}
ShardResult<PropertyValue> EdgeAccessor::GetProperty(View view, PropertyId property) const {
Result<PropertyValue> EdgeAccessor::GetProperty(View view, PropertyId property) const {
return GetProperty(property, view);
}
ShardResult<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
Result<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View view) const {
if (!config_.properties_on_edges) return PropertyValue();
auto exists = true;
auto deleted = edge_.ptr->deleted;
@@ -129,12 +128,12 @@ ShardResult<PropertyValue> EdgeAccessor::GetProperty(PropertyId property, View v
break;
}
});
if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
if (!exists) return Error::NONEXISTENT_OBJECT;
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
return std::move(value);
}
ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
Result<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View view) const {
if (!config_.properties_on_edges) return std::map<PropertyId, PropertyValue>{};
auto exists = true;
auto deleted = edge_.ptr->deleted;
@@ -175,12 +174,9 @@ ShardResult<std::map<PropertyId, PropertyValue>> EdgeAccessor::Properties(View v
break;
}
});
if (!exists) return SHARD_ERROR(ErrorCode::NONEXISTENT_OBJECT);
if (!for_deleted_ && deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
if (!exists) return Error::NONEXISTENT_OBJECT;
if (!for_deleted_ && deleted) return Error::DELETED_OBJECT;
return std::move(properties);
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
size_t EdgeAccessor::CypherId() const { return Gid().AsUint(); }
} // namespace memgraph::storage::v3

View File

@@ -25,6 +25,7 @@
namespace memgraph::storage::v3 {
struct Vertex;
class VertexAccessor;
struct Indices;
@@ -47,27 +48,27 @@ class EdgeAccessor final {
/// @return true if the object is visible from the current transaction
bool IsVisible(View view) const;
const VertexId &From() const;
const VertexId &FromVertex() const;
const VertexId &To() const;
const VertexId &ToVertex() const;
EdgeTypeId EdgeType() const { return edge_type_; }
/// Set a property value and return the old value.
/// @throw std::bad_alloc
ShardResult<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
Result<PropertyValue> SetProperty(PropertyId property, const PropertyValue &value);
/// Remove all properties and return old values for each removed property.
/// @throw std::bad_alloc
ShardResult<std::map<PropertyId, PropertyValue>> ClearProperties();
Result<std::map<PropertyId, PropertyValue>> ClearProperties();
/// @throw std::bad_alloc
ShardResult<PropertyValue> GetProperty(PropertyId property, View view) const;
Result<PropertyValue> GetProperty(PropertyId property, View view) const;
ShardResult<PropertyValue> GetProperty(View view, PropertyId property) const;
Result<PropertyValue> GetProperty(View view, PropertyId property) const;
/// @throw std::bad_alloc
ShardResult<std::map<PropertyId, PropertyValue>> Properties(View view) const;
Result<std::map<PropertyId, PropertyValue>> Properties(View view) const;
Gid Gid() const noexcept {
if (config_.properties_on_edges) {
@@ -83,9 +84,6 @@ class EdgeAccessor final {
}
bool operator!=(const EdgeAccessor &other) const noexcept { return !(*this == other); }
// Dummy function
size_t CypherId() const;
private:
EdgeRef edge_;
EdgeTypeId edge_type_;

View File

@@ -52,12 +52,13 @@ msgs::Value ConstructValueEdge(const EdgeAccessor &acc, View view) {
msgs::EdgeType type = {.id = acc.EdgeType()};
msgs::EdgeId gid = {.gid = acc.Gid().AsUint()};
msgs::Label src_prim_label = {.id = acc.From().primary_label};
msgs::Label src_prim_label = {.id = acc.FromVertex().primary_label};
memgraph::msgs::VertexId src_vertex =
std::make_pair(src_prim_label, conversions::ConvertValueVector(acc.From().primary_key));
std::make_pair(src_prim_label, conversions::ConvertValueVector(acc.FromVertex().primary_key));
msgs::Label dst_prim_label = {.id = acc.To().primary_label};
msgs::VertexId dst_vertex = std::make_pair(dst_prim_label, conversions::ConvertValueVector(acc.To().primary_key));
msgs::Label dst_prim_label = {.id = acc.ToVertex().primary_label};
msgs::VertexId dst_vertex =
std::make_pair(dst_prim_label, conversions::ConvertValueVector(acc.ToVertex().primary_key));
auto properties = acc.Properties(view);
@@ -164,7 +165,7 @@ std::any ParseExpression(const std::string &expr, memgraph::expr::AstStorage &st
return visitor.visit(ast);
}
TypedValue ComputeExpression(DbAccessor &dba, const memgraph::storage::v3::VertexAccessor &v_acc,
TypedValue ComputeExpression(DbAccessor &dba, const std::optional<memgraph::storage::v3::VertexAccessor> &v_acc,
const std::optional<memgraph::storage::v3::EdgeAccessor> &e_acc,
const std::string &expression, std::string_view node_name, std::string_view edge_name) {
AstStorage storage;
@@ -191,11 +192,10 @@ TypedValue ComputeExpression(DbAccessor &dba, const memgraph::storage::v3::Verte
return position_symbol_pair.second.name() == node_name;
}) != symbol_table.table().end());
frame[symbol_table.at(node_identifier)] = v_acc;
frame[symbol_table.at(node_identifier)] = *v_acc;
}
if (edge_identifier.symbol_pos_ != -1) {
MG_ASSERT(e_acc.has_value());
MG_ASSERT(std::find_if(symbol_table.table().begin(), symbol_table.table().end(),
[&edge_name](const std::pair<int32_t, Symbol> &position_symbol_pair) {
return position_symbol_pair.second.name() == edge_name;

View File

@@ -9,8 +9,6 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include "db_accessor.hpp"
@@ -50,7 +48,8 @@ auto Eval(TExpression *expr, EvaluationContext &ctx, AstStorage &storage, Expres
std::any ParseExpression(const std::string &expr, AstStorage &storage);
TypedValue ComputeExpression(DbAccessor &dba, const VertexAccessor &v_acc, const std::optional<EdgeAccessor> &e_acc,
const std::string &expression, std::string_view node_name, std::string_view edge_name);
TypedValue ComputeExpression(DbAccessor &dba, const std::optional<VertexAccessor> &v_acc,
const std::optional<EdgeAccessor> &e_acc, const std::string &expression,
std::string_view node_name, std::string_view edge_name);
} // namespace memgraph::storage::v3

View File

@@ -11,16 +11,12 @@
#include "indices.hpp"
#include <algorithm>
#include <functional>
#include <limits>
#include "storage/v3/delta.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/vertex.hpp"
#include "utils/bound.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
@@ -57,9 +53,9 @@ bool AnyVersionHasLabel(const Vertex &vertex, LabelId label, uint64_t timestamp)
bool deleted{false};
const Delta *delta{nullptr};
{
has_label = utils::Contains(vertex.second.labels, label);
deleted = vertex.second.deleted;
delta = vertex.second.delta;
has_label = utils::Contains(vertex.labels, label);
deleted = vertex.deleted;
delta = vertex.delta;
}
if (!deleted && has_label) {
return true;
@@ -109,10 +105,10 @@ bool AnyVersionHasLabelProperty(const Vertex &vertex, LabelId label, PropertyId
bool deleted{false};
const Delta *delta{nullptr};
{
has_label = utils::Contains(vertex.second.labels, label);
current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
deleted = vertex.second.deleted;
delta = vertex.second.delta;
has_label = utils::Contains(vertex.labels, label);
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
deleted = vertex.deleted;
delta = vertex.delta;
}
if (!deleted && has_label && current_value_equal_to_value) {
@@ -167,9 +163,9 @@ bool CurrentVersionHasLabel(const Vertex &vertex, LabelId label, Transaction *tr
bool has_label{false};
const Delta *delta{nullptr};
{
deleted = vertex.second.deleted;
has_label = utils::Contains(vertex.second.labels, label);
delta = vertex.second.delta;
deleted = vertex.deleted;
has_label = utils::Contains(vertex.labels, label);
delta = vertex.delta;
}
ApplyDeltasForRead(transaction, delta, view, [&deleted, &has_label, label](const Delta &delta) {
switch (delta.action) {
@@ -218,10 +214,10 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
bool current_value_equal_to_value = value.IsNull();
const Delta *delta{nullptr};
{
deleted = vertex.second.deleted;
has_label = utils::Contains(vertex.second.labels, label);
current_value_equal_to_value = vertex.second.properties.IsPropertyEqual(key, value);
delta = vertex.second.delta;
deleted = vertex.deleted;
has_label = utils::Contains(vertex.labels, label);
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
delta = vertex.delta;
}
ApplyDeltasForRead(transaction, delta, view,
[&deleted, &has_label, &current_value_equal_to_value, key, label, &value](const Delta &delta) {
@@ -269,10 +265,11 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
auto it = index_.find(label);
if (it == index_.end()) return;
it->second.insert(Entry{vertex, tx.start_timestamp.logical_id});
auto acc = it->second.access();
acc.insert(Entry{vertex, tx.start_timestamp.logical_id});
}
bool LabelIndex::CreateIndex(LabelId label, VertexContainer &vertices) {
bool LabelIndex::CreateIndex(LabelId label, VerticesSkipList::Accessor vertices) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
if (!emplaced) {
@@ -280,11 +277,13 @@ bool LabelIndex::CreateIndex(LabelId label, VertexContainer &vertices) {
return false;
}
try {
for (auto &vertex : vertices) {
if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
auto acc = it->second.access();
for (auto &lgo_vertex : vertices) {
auto &vertex = lgo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}
it->second.insert(Entry{&vertex, 0});
acc.insert(Entry{&vertex, 0});
}
} catch (const utils::OutOfMemoryException &) {
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
@@ -305,7 +304,7 @@ std::vector<LabelId> LabelIndex::ListIndices() const {
void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
for (auto &label_storage : index_) {
auto &vertices_acc = label_storage.second;
auto vertices_acc = label_storage.second.access();
for (auto it = vertices_acc.begin(); it != vertices_acc.end();) {
auto next_it = it;
++next_it;
@@ -317,7 +316,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
if ((next_it != vertices_acc.end() && it->vertex == next_it->vertex) ||
!AnyVersionHasLabel(*it->vertex, label_storage.first, clean_up_before_timestamp)) {
vertices_acc.erase(*it);
vertices_acc.remove(*it);
}
it = next_it;
@@ -325,7 +324,7 @@ void LabelIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp)
}
}
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator)
LabelIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
: self_(self),
index_iterator_(index_iterator),
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -340,7 +339,7 @@ LabelIndex::Iterable::Iterator &LabelIndex::Iterable::Iterator::operator++() {
}
void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
if (index_iterator_->vertex == current_vertex_) {
continue;
}
@@ -353,9 +352,10 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
}
}
LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction,
Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: index_container_(&index_container),
LabelIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view,
Transaction *transaction, Indices *indices, Config::Items config,
const VertexValidator &vertex_validator)
: index_accessor_(std::move(index_accessor)),
label_(label),
view_(view),
transaction_(transaction),
@@ -363,6 +363,12 @@ LabelIndex::Iterable::Iterable(LabelIndexContainer &index_container, LabelId lab
config_(config),
vertex_validator_(&vertex_validator) {}
void LabelIndex::RunGC() {
for (auto &index_entry : index_) {
index_entry.second.run_gc();
}
}
bool LabelPropertyIndex::Entry::operator<(const Entry &rhs) const {
if (value < rhs.value) {
return true;
@@ -382,13 +388,14 @@ bool LabelPropertyIndex::Entry::operator<(const PropertyValue &rhs) const { retu
bool LabelPropertyIndex::Entry::operator==(const PropertyValue &rhs) const { return value == rhs; }
void LabelPropertyIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
for (auto &[label_prop, index] : index_) {
for (auto &[label_prop, storage] : index_) {
if (label_prop.first != label) {
continue;
}
auto prop_value = vertex->second.properties.GetProperty(label_prop.second);
auto prop_value = vertex->properties.GetProperty(label_prop.second);
if (!prop_value.IsNull()) {
index.emplace(Entry{prop_value, vertex, tx.start_timestamp.logical_id});
auto acc = storage.access();
acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp.logical_id});
}
}
}
@@ -398,17 +405,18 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
if (value.IsNull()) {
return;
}
for (auto &[label_prop, index] : index_) {
for (auto &[label_prop, storage] : index_) {
if (label_prop.second != property) {
continue;
}
if (VertexHasLabel(*vertex, label_prop.first)) {
index.emplace(Entry{value, vertex, tx.start_timestamp.logical_id});
if (utils::Contains(vertex->labels, label_prop.first)) {
auto acc = storage.access();
acc.insert(Entry{value, vertex, tx.start_timestamp.logical_id});
}
}
}
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices) {
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices) {
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
auto [it, emplaced] =
index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
@@ -417,15 +425,17 @@ bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, VertexC
return false;
}
try {
for (auto &vertex : vertices) {
if (vertex.second.deleted || !VertexHasLabel(vertex, label)) {
auto acc = it->second.access();
for (auto &lgo_vertex : vertices) {
auto &vertex = lgo_vertex.vertex;
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
continue;
}
auto value = vertex.second.properties.GetProperty(property);
auto value = vertex.properties.GetProperty(property);
if (value.IsNull()) {
continue;
}
it->second.emplace(Entry{value, &vertex, 0});
acc.insert(Entry{std::move(value), &vertex, 0});
}
} catch (const utils::OutOfMemoryException &) {
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
@@ -446,7 +456,8 @@ std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::ListIndices() co
void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_timestamp) {
for (auto &[label_property, index] : index_) {
for (auto it = index.begin(); it != index.end();) {
auto index_acc = index.access();
for (auto it = index_acc.begin(); it != index_acc.end();) {
auto next_it = it;
++next_it;
@@ -455,17 +466,17 @@ void LabelPropertyIndex::RemoveObsoleteEntries(const uint64_t clean_up_before_ti
continue;
}
if ((next_it != index.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
if ((next_it != index_acc.end() && it->vertex == next_it->vertex && it->value == next_it->value) ||
!AnyVersionHasLabelProperty(*it->vertex, label_property.first, label_property.second, it->value,
clean_up_before_timestamp)) {
index.erase(it);
index_acc.remove(*it);
}
it = next_it;
}
}
}
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator)
LabelPropertyIndex::Iterable::Iterator::Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator)
: self_(self),
index_iterator_(index_iterator),
current_vertex_accessor_(nullptr, nullptr, nullptr, self_->config_, *self_->vertex_validator_),
@@ -480,7 +491,7 @@ LabelPropertyIndex::Iterable::Iterator &LabelPropertyIndex::Iterable::Iterator::
}
void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
for (; index_iterator_ != self_->index_container_->end(); ++index_iterator_) {
for (; index_iterator_ != self_->index_accessor_.end(); ++index_iterator_) {
if (index_iterator_->vertex == current_vertex_) {
continue;
}
@@ -495,11 +506,11 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
}
if (self_->upper_bound_) {
if (self_->upper_bound_->value() < index_iterator_->value) {
index_iterator_ = self_->index_container_->end();
index_iterator_ = self_->index_accessor_.end();
break;
}
if (!self_->upper_bound_->IsInclusive() && index_iterator_->value == self_->upper_bound_->value()) {
index_iterator_ = self_->index_container_->end();
index_iterator_ = self_->index_accessor_.end();
break;
}
}
@@ -526,12 +537,13 @@ const PropertyValue kSmallestMap = PropertyValue(std::map<std::string, PropertyV
const PropertyValue kSmallestTemporalData =
PropertyValue(TemporalData{static_cast<TemporalType>(0), std::numeric_limits<int64_t>::min()});
LabelPropertyIndex::Iterable::Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
LabelPropertyIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label,
PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view,
Transaction *transaction, Indices *indices, Config::Items config,
const VertexValidator &vertex_validator)
: index_container_(&index_container),
: index_accessor_(std::move(index_accessor)),
label_(label),
property_(property),
lower_bound_(lower_bound),
@@ -639,55 +651,49 @@ LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::begin() {
// If the bounds are set and don't have comparable types we don't yield any
// items from the index.
if (!bounds_valid_) {
return {this, index_container_->end()};
return {this, index_accessor_.end()};
}
auto index_iterator = index_accessor_.begin();
if (lower_bound_) {
return {this, std::ranges::lower_bound(*index_container_, lower_bound_->value(), std::less{}, &Entry::value)};
index_iterator = index_accessor_.find_equal_or_greater(lower_bound_->value());
}
return {this, index_container_->begin()};
return {this, index_iterator};
}
LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_container_->end()}; }
LabelPropertyIndex::Iterable::Iterator LabelPropertyIndex::Iterable::end() { return {this, index_accessor_.end()}; }
int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
const PropertyValue &value) const {
auto it = index_.find({label, property});
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
MG_ASSERT(!value.IsNull(), "Null is not supported!");
auto acc = it->second.access();
if (!value.IsNull()) {
return static_cast<int64_t>(
acc.estimate_count(value, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
}
// The value `Null` won't ever appear in the index because it indicates that
// the property shouldn't exist. Instead, this value is used as an indicator
// to estimate the average number of equal elements in the list (for any
// given value).
return static_cast<int64_t>(acc.estimate_average_number_of_equals(
[](const auto &first, const auto &second) { return first.value == second.value; },
static_cast<int>(utils::SkipListLayerForAverageEqualsEstimation(acc.size()))));
}
// TODO(jbajic) This can be improved by exiting early
auto start_it = std::ranges::lower_bound(it->second, value, std::less{}, &Entry::value);
int64_t LabelPropertyIndex::ApproximateVertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const {
auto it = index_.find({label, property});
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
auto acc = it->second.access();
return static_cast<int64_t>(
std::ranges::count_if(start_it, it->second.end(), [&value](const auto &elem) { return elem.value == value; }));
acc.estimate_range_count(lower, upper, static_cast<int>(utils::SkipListLayerForCountEstimation(acc.size()))));
}
int64_t LabelPropertyIndex::VertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const {
auto it = index_.find({label, property});
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(), property.AsUint());
const auto lower_it = std::invoke(
[&index = it->second](const auto value, const auto def) {
if (value) {
if (value->IsInclusive()) {
return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
}
return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
}
return def;
},
lower, it->second.begin());
const auto upper_it = std::invoke(
[&index = it->second](const auto value, const auto def) {
if (value) {
if (value->IsInclusive()) {
return std::ranges::upper_bound(index, value->value(), std::less{}, &Entry::value);
}
return std::ranges::lower_bound(index, value->value(), std::less{}, &Entry::value);
}
return def;
},
upper, it->second.end());
return static_cast<int64_t>(std::distance(lower_it, upper_it));
void LabelPropertyIndex::RunGC() {
for (auto &index_entry : index_) {
index_entry.second.run_gc();
}
}
void RemoveObsoleteEntries(Indices *indices, const uint64_t clean_up_before_timestamp) {

View File

@@ -13,7 +13,6 @@
#include <cstdint>
#include <optional>
#include <set>
#include <tuple>
#include <utility>
@@ -21,6 +20,7 @@
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/bound.hpp"
#include "utils/logging.hpp"
#include "utils/skip_list.hpp"
@@ -30,6 +30,7 @@ namespace memgraph::storage::v3 {
struct Indices;
class LabelIndex {
private:
struct Entry {
Vertex *vertex;
uint64_t timestamp;
@@ -40,9 +41,17 @@ class LabelIndex {
bool operator==(const Entry &rhs) const { return vertex == rhs.vertex && timestamp == rhs.timestamp; }
};
public:
using LabelIndexContainer = std::set<Entry>;
struct LabelStorage {
LabelId label;
utils::SkipList<Entry> vertices;
bool operator<(const LabelStorage &rhs) const { return label < rhs.label; }
bool operator<(LabelId rhs) const { return label < rhs; }
bool operator==(const LabelStorage &rhs) const { return label == rhs.label; }
bool operator==(LabelId rhs) const { return label == rhs; }
};
public:
LabelIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
@@ -50,7 +59,7 @@ class LabelIndex {
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, VertexContainer &vertices);
bool CreateIndex(LabelId label, VerticesSkipList::Accessor vertices);
/// Returns false if there was no index to drop
bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
@@ -63,12 +72,12 @@ class LabelIndex {
class Iterable {
public:
Iterable(LabelIndexContainer &index_container, LabelId label, View view, Transaction *transaction, Indices *indices,
Config::Items config, const VertexValidator &vertex_validator);
Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, View view, Transaction *transaction,
Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
class Iterator {
public:
Iterator(Iterable *self, LabelIndexContainer::iterator index_iterator);
Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
VertexAccessor operator*() const { return current_vertex_accessor_; }
@@ -81,16 +90,16 @@ class LabelIndex {
void AdvanceUntilValid();
Iterable *self_;
LabelIndexContainer::iterator index_iterator_;
utils::SkipList<Entry>::Iterator index_iterator_;
VertexAccessor current_vertex_accessor_;
Vertex *current_vertex_;
};
Iterator begin() { return {this, index_container_->begin()}; }
Iterator end() { return {this, index_container_->end()}; }
Iterator begin() { return {this, index_accessor_.begin()}; }
Iterator end() { return {this, index_accessor_.end()}; }
private:
LabelIndexContainer *index_container_;
utils::SkipList<Entry>::Accessor index_accessor_;
LabelId label_;
View view_;
Transaction *transaction_;
@@ -103,7 +112,7 @@ class LabelIndex {
Iterable Vertices(LabelId label, View view, Transaction *transaction) {
auto it = index_.find(label);
MG_ASSERT(it != index_.end(), "Index for label {} doesn't exist", label.AsUint());
return {it->second, label, view, transaction, indices_, config_, *vertex_validator_};
return {it->second.access(), label, view, transaction, indices_, config_, *vertex_validator_};
}
int64_t ApproximateVertexCount(LabelId label) {
@@ -114,14 +123,17 @@ class LabelIndex {
void Clear() { index_.clear(); }
void RunGC();
private:
std::map<LabelId, LabelIndexContainer> index_;
std::map<LabelId, utils::SkipList<Entry>> index_;
Indices *indices_;
Config::Items config_;
const VertexValidator *vertex_validator_;
};
class LabelPropertyIndex {
private:
struct Entry {
PropertyValue value;
Vertex *vertex;
@@ -135,8 +147,6 @@ class LabelPropertyIndex {
};
public:
using LabelPropertyIndexContainer = std::set<Entry>;
LabelPropertyIndex(Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: indices_(indices), config_(config), vertex_validator_{&vertex_validator} {}
@@ -147,7 +157,7 @@ class LabelPropertyIndex {
void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
/// @throw std::bad_alloc
bool CreateIndex(LabelId label, PropertyId property, VertexContainer &vertices);
bool CreateIndex(LabelId label, PropertyId property, VerticesSkipList::Accessor vertices);
bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }
@@ -159,14 +169,14 @@ class LabelPropertyIndex {
class Iterable {
public:
Iterable(LabelPropertyIndexContainer &index_container, LabelId label, PropertyId property,
Iterable(utils::SkipList<Entry>::Accessor index_accessor, LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower_bound,
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Transaction *transaction,
Indices *indices, Config::Items config, const VertexValidator &vertex_validator);
class Iterator {
public:
Iterator(Iterable *self, LabelPropertyIndexContainer::iterator index_iterator);
Iterator(Iterable *self, utils::SkipList<Entry>::Iterator index_iterator);
VertexAccessor operator*() const { return current_vertex_accessor_; }
@@ -179,7 +189,7 @@ class LabelPropertyIndex {
void AdvanceUntilValid();
Iterable *self_;
LabelPropertyIndexContainer::iterator index_iterator_;
utils::SkipList<Entry>::Iterator index_iterator_;
VertexAccessor current_vertex_accessor_;
Vertex *current_vertex_;
};
@@ -188,7 +198,7 @@ class LabelPropertyIndex {
Iterator end();
private:
LabelPropertyIndexContainer *index_container_;
utils::SkipList<Entry>::Accessor index_accessor_;
LabelId label_;
PropertyId property_;
std::optional<utils::Bound<PropertyValue>> lower_bound_;
@@ -207,11 +217,11 @@ class LabelPropertyIndex {
auto it = index_.find({label, property});
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
property.AsUint());
return {it->second, label, property, lower_bound, upper_bound,
view, transaction, indices_, config_, *vertex_validator_};
return {it->second.access(), label, property, lower_bound, upper_bound, view,
transaction, indices_, config_, *vertex_validator_};
}
int64_t VertexCount(LabelId label, PropertyId property) const {
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
auto it = index_.find({label, property});
MG_ASSERT(it != index_.end(), "Index for label {} and property {} doesn't exist", label.AsUint(),
property.AsUint());
@@ -222,15 +232,18 @@ class LabelPropertyIndex {
/// an estimated count of nodes which have their property's value set to
/// `value`. If the `value` specified is `Null`, then an average number of
/// equal elements is returned.
int64_t VertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const;
int64_t VertexCount(LabelId label, PropertyId property, const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const;
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const;
void Clear() { index_.clear(); }
void RunGC();
private:
std::map<std::pair<LabelId, PropertyId>, LabelPropertyIndexContainer> index_;
std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>> index_;
Indices *indices_;
Config::Items config_;
const VertexValidator *vertex_validator_;

View File

@@ -0,0 +1,43 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <algorithm>
#include <iterator>
#include <ranges>
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3 {
KeyStore::KeyStore(const PrimaryKey &key_values) {
for (auto i = 0; i < key_values.size(); ++i) {
MG_ASSERT(!key_values[i].IsNull());
store_.SetProperty(PropertyId::FromInt(i), key_values[i]);
}
}
PropertyValue KeyStore::GetKey(const size_t index) const { return store_.GetProperty(PropertyId::FromUint(index)); }
PropertyValue KeyStore::GetKey(const PropertyId property_id) const { return store_.GetProperty(property_id); }
PrimaryKey KeyStore::Keys() const {
auto keys_map = store_.Properties();
PrimaryKey keys;
keys.reserve(keys_map.size());
std::ranges::transform(
keys_map, std::back_inserter(keys),
[](std::pair<const PropertyId, PropertyValue> &id_and_value) { return std::move(id_and_value.second); });
return keys;
}
} // namespace memgraph::storage::v3

View File

@@ -11,6 +11,12 @@
#pragma once
#include <algorithm>
#include <compare>
#include <functional>
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_store.hpp"
#include "storage/v3/property_value.hpp"
namespace memgraph::storage::v3 {
@@ -18,4 +24,39 @@ namespace memgraph::storage::v3 {
// Primary key is a collection of primary properties.
using PrimaryKey = std::vector<PropertyValue>;
class KeyStore {
public:
explicit KeyStore(const PrimaryKey &key_values);
KeyStore(const KeyStore &) = delete;
KeyStore(KeyStore &&other) noexcept = default;
KeyStore &operator=(const KeyStore &) = delete;
KeyStore &operator=(KeyStore &&other) noexcept = default;
~KeyStore() = default;
PropertyValue GetKey(size_t index) const;
PropertyValue GetKey(PropertyId property) const;
PrimaryKey Keys() const;
friend bool operator<(const KeyStore &lhs, const KeyStore &rhs) {
return std::ranges::lexicographical_compare(lhs.Keys(), rhs.Keys(), std::less<PropertyValue>{});
}
friend bool operator==(const KeyStore &lhs, const KeyStore &rhs) {
return std::ranges::equal(lhs.Keys(), rhs.Keys());
}
friend bool operator<(const KeyStore &lhs, const PrimaryKey &rhs) {
return std::ranges::lexicographical_compare(lhs.Keys(), rhs, std::less<PropertyValue>{});
}
friend bool operator==(const KeyStore &lhs, const PrimaryKey &rhs) { return std::ranges::equal(lhs.Keys(), rhs); }
private:
PropertyStore store_;
};
} // namespace memgraph::storage::v3

View File

@@ -9,11 +9,6 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "storage/v3/lexicographically_ordered_vertex.hpp"
namespace memgraph::utils {
template <typename>
constexpr auto kAlwaysFalse{false};
} // namespace memgraph::utils
namespace memgraph::storage::v3 {} // namespace memgraph::storage::v3

View File

@@ -0,0 +1,42 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <concepts>
#include <type_traits>
#include "storage/v3/vertex.hpp"
#include "utils/concepts.hpp"
namespace memgraph::storage::v3 {
struct LexicographicallyOrderedVertex {
Vertex vertex;
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
return lhs.vertex.keys == rhs.vertex.keys;
}
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const LexicographicallyOrderedVertex &rhs) {
return lhs.vertex.keys < rhs.vertex.keys;
}
// TODO(antaljanosbenjamin): maybe it worth to overload this for std::array to avoid heap construction of the vector
friend bool operator==(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
return lhs.vertex.keys == rhs;
}
friend bool operator<(const LexicographicallyOrderedVertex &lhs, const std::vector<PropertyValue> &rhs) {
return lhs.vertex.keys < rhs;
}
};
} // namespace memgraph::storage::v3

View File

@@ -11,21 +11,12 @@
#pragma once
#include <type_traits>
#include "storage/v3/edge.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/view.hpp"
#include "utils/concepts.hpp"
namespace memgraph::storage::v3 {
inline VertexData *GetDeltaHolder(Vertex *vertex) { return &vertex->second; }
inline Edge *GetDeltaHolder(Edge *edge) { return edge; }
/// This function iterates through the undo buffers from an object (starting
/// from the supplied delta) and determines what deltas should be applied to get
/// the currently visible version of the object. When the function finds a delta
@@ -86,12 +77,10 @@ inline void ApplyDeltasForRead(Transaction *transaction, const Delta *delta, Vie
/// transaction) and returns a `bool` value indicating whether the caller can
/// proceed with a write operation.
template <typename TObj>
requires utils::SameAsAnyOf<TObj, Edge, Vertex>
inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
auto *delta_holder = GetDeltaHolder(object);
if (delta_holder->delta == nullptr) return true;
if (object->delta == nullptr) return true;
const auto &delta_commit_info = *delta_holder->delta->commit_info;
const auto &delta_commit_info = *object->delta->commit_info;
if (delta_commit_info.start_or_commit_timestamp == transaction->commit_info->start_or_commit_timestamp ||
(delta_commit_info.is_locally_committed &&
delta_commit_info.start_or_commit_timestamp < transaction->start_timestamp)) {
@@ -116,11 +105,9 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
/// the delta into the object's delta list.
/// @throw std::bad_alloc
template <typename TObj, class... Args>
requires utils::SameAsAnyOf<TObj, Edge, Vertex>
inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_info.get(),
transaction->command_id);
auto *delta_holder = GetDeltaHolder(object);
// The operations are written in such order so that both `next` and `prev`
// chains are valid at all times. The chains must be valid at all times
@@ -131,21 +118,21 @@ inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&..
// TODO(antaljanosbenjamin): clang-tidy detects (in my opinion a false positive) issue in
// `Shard::Accessor::CreateEdge`.
// NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
delta->next = delta_holder->delta;
delta->next = object->delta;
// 2. We need to set the previous delta of the new delta to the object.
delta->prev.Set(object);
// 3. We need to set the previous delta of the existing delta to the new
// delta. After this point the garbage collector will be able to see the new
// delta but won't modify it until we are done with all of our modifications.
if (delta_holder->delta) {
delta_holder->delta->prev.Set(delta);
if (object->delta) {
object->delta->prev.Set(delta);
}
// 4. Finally, we need to set the object's delta to the new delta. The garbage
// collector and other transactions will acquire the object lock to read the
// delta from the object. Because the lock is held during the whole time this
// modification is being done, everybody else will wait until we are fully
// done with our modification before they read the object's delta value.
delta_holder->delta = delta;
object->delta = delta;
}
} // namespace memgraph::storage::v3

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,463 +11,56 @@
#include "storage/v3/request_helper.hpp"
#include <iterator>
#include <vector>
#include "pretty_print_ast_to_original_expression.hpp"
#include "storage/v3/bindings/db_accessor.hpp"
#include "storage/v3/bindings/pretty_print_ast_to_original_expression.hpp"
#include "storage/v3/expr.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/value_conversions.hpp"
namespace memgraph::storage::v3 {
using msgs::Label;
using msgs::PropertyId;
using conversions::ConvertPropertyVector;
using conversions::FromPropertyValueToValue;
using conversions::ToMsgsVertexId;
namespace {
using AllEdgePropertyDataStucture = std::map<PropertyId, msgs::Value>;
using SpecificEdgePropertyDataStucture = std::vector<msgs::Value>;
using AllEdgeProperties = std::tuple<msgs::VertexId, msgs::Gid, AllEdgePropertyDataStucture>;
using SpecificEdgeProperties = std::tuple<msgs::VertexId, msgs::Gid, SpecificEdgePropertyDataStucture>;
using SpecificEdgePropertiesVector = std::vector<SpecificEdgeProperties>;
using AllEdgePropertiesVector = std::vector<AllEdgeProperties>;
struct VertexIdCmpr {
bool operator()(const storage::v3::VertexId *lhs, const storage::v3::VertexId *rhs) const { return *lhs < *rhs; }
};
std::optional<std::map<PropertyId, Value>> PrimaryKeysFromAccessor(const VertexAccessor &acc, View view,
const Schemas::Schema &schema) {
std::map<PropertyId, Value> ret;
auto props = acc.Properties(view);
auto maybe_pk = acc.PrimaryKey(view);
if (maybe_pk.HasError()) {
spdlog::debug("Encountered an error while trying to get vertex primary key.");
return std::nullopt;
}
auto &pk = maybe_pk.GetValue();
MG_ASSERT(schema.second.size() == pk.size(), "PrimaryKey size does not match schema!");
for (size_t i{0}; i < schema.second.size(); ++i) {
ret.emplace(schema.second[i].property_id, FromPropertyValueToValue(std::move(pk[i])));
}
return ret;
}
ShardResult<std::vector<msgs::Label>> FillUpSourceVertexSecondaryLabels(const std::optional<VertexAccessor> &v_acc,
const msgs::ExpandOneRequest &req) {
auto secondary_labels = v_acc->Labels(View::NEW);
if (secondary_labels.HasError()) {
spdlog::debug("Encountered an error while trying to get the secondary labels of a vertex. Transaction id: {}",
req.transaction_id.logical_id);
return secondary_labels.GetError();
}
auto &sec_labels = secondary_labels.GetValue();
std::vector<msgs::Label> msgs_secondary_labels;
msgs_secondary_labels.reserve(sec_labels.size());
std::transform(sec_labels.begin(), sec_labels.end(), std::back_inserter(msgs_secondary_labels),
[](auto label_id) { return msgs::Label{.id = label_id}; });
return msgs_secondary_labels;
}
ShardResult<std::map<PropertyId, Value>> FillUpSourceVertexProperties(const std::optional<VertexAccessor> &v_acc,
const msgs::ExpandOneRequest &req,
storage::v3::View view,
const Schemas::Schema &schema) {
std::map<PropertyId, Value> src_vertex_properties;
if (!req.src_vertex_properties) {
auto props = v_acc->Properties(View::NEW);
if (props.HasError()) {
spdlog::debug("Encountered an error while trying to access vertex properties. Transaction id: {}",
req.transaction_id.logical_id);
return props.GetError();
}
for (auto &[key, val] : props.GetValue()) {
src_vertex_properties.insert(std::make_pair(key, FromPropertyValueToValue(std::move(val))));
}
auto pks = PrimaryKeysFromAccessor(*v_acc, view, schema);
if (pks) {
src_vertex_properties.merge(*pks);
}
} else if (req.src_vertex_properties.value().empty()) {
// NOOP
} else {
for (const auto &prop : req.src_vertex_properties.value()) {
auto prop_val = v_acc->GetProperty(prop, View::OLD);
if (prop_val.HasError()) {
spdlog::debug("Encountered an error while trying to access vertex properties. Transaction id: {}",
req.transaction_id.logical_id);
return prop_val.GetError();
std::vector<Element> OrderByElements(Shard::Accessor &acc, DbAccessor &dba, VerticesIterable &vertices_iterable,
std::vector<msgs::OrderBy> &order_bys) {
std::vector<Element> ordered;
ordered.reserve(acc.ApproximateVertexCount());
std::vector<Ordering> ordering;
ordering.reserve(order_bys.size());
for (const auto &order : order_bys) {
switch (order.direction) {
case memgraph::msgs::OrderingDirection::ASCENDING: {
ordering.push_back(Ordering::ASC);
break;
}
case memgraph::msgs::OrderingDirection::DESCENDING: {
ordering.push_back(Ordering::DESC);
break;
}
src_vertex_properties.insert(std::make_pair(prop, FromPropertyValueToValue(std::move(prop_val.GetValue()))));
}
}
auto compare_typed_values = TypedValueVectorCompare(ordering);
for (auto it = vertices_iterable.begin(); it != vertices_iterable.end(); ++it) {
std::vector<TypedValue> properties_order_by;
properties_order_by.reserve(order_bys.size());
return src_vertex_properties;
}
ShardResult<std::array<std::vector<EdgeAccessor>, 2>> FillUpConnectingEdges(
const std::optional<VertexAccessor> &v_acc, const msgs::ExpandOneRequest &req,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness) {
std::vector<EdgeTypeId> edge_types{};
edge_types.reserve(req.edge_types.size());
std::transform(req.edge_types.begin(), req.edge_types.end(), std::back_inserter(edge_types),
[](const msgs::EdgeType &edge_type) { return edge_type.id; });
std::vector<EdgeAccessor> in_edges;
std::vector<EdgeAccessor> out_edges;
switch (req.direction) {
case msgs::EdgeDirection::OUT: {
auto out_edges_result = v_acc->OutEdges(View::NEW, edge_types);
if (out_edges_result.HasError()) {
spdlog::debug("Encountered an error while trying to get out-going EdgeAccessors. Transaction id: {}",
req.transaction_id.logical_id);
return out_edges_result.GetError();
}
out_edges =
maybe_filter_based_on_edge_uniqueness(std::move(out_edges_result.GetValue()), msgs::EdgeDirection::OUT);
break;
}
case msgs::EdgeDirection::IN: {
auto in_edges_result = v_acc->InEdges(View::NEW, edge_types);
if (in_edges_result.HasError()) {
spdlog::debug(
"Encountered an error while trying to get in-going EdgeAccessors. Transaction id: {}"[req.transaction_id
.logical_id]);
return in_edges_result.GetError();
}
in_edges = maybe_filter_based_on_edge_uniqueness(std::move(in_edges_result.GetValue()), msgs::EdgeDirection::IN);
break;
}
case msgs::EdgeDirection::BOTH: {
auto in_edges_result = v_acc->InEdges(View::NEW, edge_types);
if (in_edges_result.HasError()) {
spdlog::debug("Encountered an error while trying to get in-going EdgeAccessors. Transaction id: {}",
req.transaction_id.logical_id);
return in_edges_result.GetError();
}
in_edges = maybe_filter_based_on_edge_uniqueness(std::move(in_edges_result.GetValue()), msgs::EdgeDirection::IN);
auto out_edges_result = v_acc->OutEdges(View::NEW, edge_types);
if (out_edges_result.HasError()) {
spdlog::debug("Encountered an error while trying to get out-going EdgeAccessors. Transaction id: {}",
req.transaction_id.logical_id);
return out_edges_result.GetError();
}
out_edges =
maybe_filter_based_on_edge_uniqueness(std::move(out_edges_result.GetValue()), msgs::EdgeDirection::OUT);
break;
}
}
return std::array<std::vector<EdgeAccessor>, 2>{std::move(in_edges), std::move(out_edges)};
}
template <bool are_in_edges>
ShardResult<void> FillEdges(const std::vector<EdgeAccessor> &edges, msgs::ExpandOneResultRow &row,
const EdgeFiller &edge_filler) {
for (const auto &edge : edges) {
if (const auto res = edge_filler(edge, are_in_edges, row); res.HasError()) {
return res.GetError();
for (const auto &order_by : order_bys) {
const auto val =
ComputeExpression(dba, *it, std::nullopt, order_by.expression.expression, expr::identifier_node_symbol, "");
properties_order_by.push_back(val);
}
ordered.push_back({std::move(properties_order_by), *it});
}
return {};
}
}; // namespace
ShardResult<std::map<PropertyId, Value>> CollectSpecificPropertiesFromAccessor(const VertexAccessor &acc,
const std::vector<PropertyId> &props,
View view) {
std::map<PropertyId, Value> ret;
for (const auto &prop : props) {
auto result = acc.GetProperty(prop, view);
if (result.HasError()) {
spdlog::debug("Encountered an Error while trying to get a vertex property.");
return result.GetError();
}
auto &value = result.GetValue();
ret.emplace(std::make_pair(prop, FromPropertyValueToValue(std::move(value))));
}
return ret;
}
std::vector<TypedValue> EvaluateVertexExpressions(DbAccessor &dba, const VertexAccessor &v_acc,
const std::vector<std::string> &expressions,
std::string_view node_name) {
std::vector<TypedValue> evaluated_expressions;
evaluated_expressions.reserve(expressions.size());
std::transform(expressions.begin(), expressions.end(), std::back_inserter(evaluated_expressions),
[&dba, &v_acc, &node_name](const auto &expression) {
return ComputeExpression(dba, v_acc, std::nullopt, expression, node_name, "");
});
return evaluated_expressions;
}
std::vector<TypedValue> EvaluateEdgeExpressions(DbAccessor &dba, const VertexAccessor &v_acc, const EdgeAccessor &e_acc,
const std::vector<std::string> &expressions) {
std::vector<TypedValue> evaluated_expressions;
evaluated_expressions.reserve(expressions.size());
std::transform(expressions.begin(), expressions.end(), std::back_inserter(evaluated_expressions),
[&dba, &v_acc, &e_acc](const auto &expression) {
return ComputeExpression(dba, v_acc, e_acc, expression, expr::identifier_node_symbol,
expr::identifier_edge_symbol);
});
return evaluated_expressions;
}
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view,
const Schemas::Schema &schema) {
auto ret = impl::CollectAllPropertiesImpl<VertexAccessor>(acc, view);
if (ret.HasError()) {
return ret.GetError();
}
auto pks = PrimaryKeysFromAccessor(acc, view, schema);
if (pks) {
ret.GetValue().merge(std::move(*pks));
}
return ret;
}
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view) {
return impl::CollectAllPropertiesImpl(acc, view);
}
EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbor_rows) {
// Functions to select connecting edges based on uniquness
EdgeUniquenessFunction maybe_filter_based_on_edge_uniquness;
if (only_unique_neighbor_rows) {
maybe_filter_based_on_edge_uniquness = [](EdgeAccessors &&edges,
msgs::EdgeDirection edge_direction) -> EdgeAccessors {
std::function<bool(std::set<const storage::v3::VertexId *, VertexIdCmpr> &, const storage::v3::EdgeAccessor &)>
is_edge_unique;
switch (edge_direction) {
case msgs::EdgeDirection::OUT: {
is_edge_unique = [](std::set<const storage::v3::VertexId *, VertexIdCmpr> &other_vertex_set,
const storage::v3::EdgeAccessor &edge_acc) {
auto [it, insertion_happened] = other_vertex_set.insert(&edge_acc.To());
return insertion_happened;
};
break;
}
case msgs::EdgeDirection::IN: {
is_edge_unique = [](std::set<const storage::v3::VertexId *, VertexIdCmpr> &other_vertex_set,
const storage::v3::EdgeAccessor &edge_acc) {
auto [it, insertion_happened] = other_vertex_set.insert(&edge_acc.From());
return insertion_happened;
};
break;
}
case msgs::EdgeDirection::BOTH:
MG_ASSERT(false, "This is should never happen, msgs::EdgeDirection::BOTH should not be passed here.");
}
EdgeAccessors ret;
std::set<const storage::v3::VertexId *, VertexIdCmpr> other_vertex_set;
for (const auto &edge : edges) {
if (is_edge_unique(other_vertex_set, edge)) {
ret.emplace_back(edge);
}
}
return ret;
};
} else {
maybe_filter_based_on_edge_uniquness =
[](EdgeAccessors &&edges, msgs::EdgeDirection /*edge_direction*/) -> EdgeAccessors { return std::move(edges); };
}
return maybe_filter_based_on_edge_uniquness;
}
EdgeFiller InitializeEdgeFillerFunction(const msgs::ExpandOneRequest &req) {
EdgeFiller edge_filler;
if (!req.edge_properties) {
edge_filler = [transaction_id = req.transaction_id.logical_id](
const EdgeAccessor &edge, const bool is_in_edge,
msgs::ExpandOneResultRow &result_row) -> ShardResult<void> {
auto properties_results = edge.Properties(View::NEW);
if (properties_results.HasError()) {
spdlog::debug("Encountered an error while trying to get edge properties. Transaction id: {}", transaction_id);
return properties_results.GetError();
}
std::map<PropertyId, msgs::Value> value_properties;
for (auto &[prop_key, prop_val] : properties_results.GetValue()) {
value_properties.insert(std::make_pair(prop_key, FromPropertyValueToValue(std::move(prop_val))));
}
using EdgeWithAllProperties = msgs::ExpandOneResultRow::EdgeWithAllProperties;
if (is_in_edge) {
result_row.in_edges_with_all_properties.push_back(
EdgeWithAllProperties{ToMsgsVertexId(edge.From()), msgs::EdgeType{edge.EdgeType()}, edge.Gid().AsUint(),
std::move(value_properties)});
} else {
result_row.out_edges_with_all_properties.push_back(
EdgeWithAllProperties{ToMsgsVertexId(edge.To()), msgs::EdgeType{edge.EdgeType()}, edge.Gid().AsUint(),
std::move(value_properties)});
}
return {};
};
} else {
// TODO(gvolfing) - do we want to set the action_successful here?
edge_filler = [&req](const EdgeAccessor &edge, const bool is_in_edge,
msgs::ExpandOneResultRow &result_row) -> ShardResult<void> {
std::vector<msgs::Value> value_properties;
value_properties.reserve(req.edge_properties.value().size());
for (const auto &edge_prop : req.edge_properties.value()) {
auto property_result = edge.GetProperty(edge_prop, View::NEW);
if (property_result.HasError()) {
spdlog::debug("Encountered an error while trying to get edge properties. Transaction id: {}",
req.transaction_id.logical_id);
return property_result.GetError();
}
value_properties.emplace_back(FromPropertyValueToValue(std::move(property_result.GetValue())));
}
using EdgeWithSpecificProperties = msgs::ExpandOneResultRow::EdgeWithSpecificProperties;
if (is_in_edge) {
result_row.in_edges_with_specific_properties.push_back(
EdgeWithSpecificProperties{ToMsgsVertexId(edge.From()), msgs::EdgeType{edge.EdgeType()},
edge.Gid().AsUint(), std::move(value_properties)});
} else {
result_row.out_edges_with_specific_properties.push_back(
EdgeWithSpecificProperties{ToMsgsVertexId(edge.To()), msgs::EdgeType{edge.EdgeType()}, edge.Gid().AsUint(),
std::move(value_properties)});
}
return {};
};
}
return edge_filler;
}
bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc,
const std::vector<std::string> &filters) {
return std::ranges::all_of(filters, [&dba, &v_acc](const auto &filter_expr) {
const auto result = ComputeExpression(dba, v_acc, std::nullopt, filter_expr, expr::identifier_node_symbol, "");
return result.IsBool() && result.ValueBool();
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
});
}
bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const EdgeAccessor &e_acc,
const std::vector<std::string> &filters) {
return std::ranges::all_of(filters, [&dba, &v_acc, &e_acc](const auto &filter_expr) {
const auto result =
ComputeExpression(dba, v_acc, e_acc, filter_expr, expr::identifier_node_symbol, expr::identifier_edge_symbol);
return result.IsBool() && result.ValueBool();
});
}
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
Shard::Accessor &acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
const Schemas::Schema &schema) {
/// Fill up source vertex
const auto primary_key = ConvertPropertyVector(src_vertex.second);
auto v_acc = acc.FindVertex(primary_key, View::NEW);
msgs::Vertex source_vertex = {.id = src_vertex};
auto maybe_secondary_labels = FillUpSourceVertexSecondaryLabels(v_acc, req);
if (maybe_secondary_labels.HasError()) {
return maybe_secondary_labels.GetError();
}
source_vertex.labels = std::move(*maybe_secondary_labels);
auto src_vertex_properties = FillUpSourceVertexProperties(v_acc, req, storage::v3::View::NEW, schema);
if (src_vertex_properties.HasError()) {
return src_vertex_properties.GetError();
}
/// Fill up connecting edges
auto fill_up_connecting_edges = FillUpConnectingEdges(v_acc, req, maybe_filter_based_on_edge_uniqueness);
if (fill_up_connecting_edges.HasError()) {
return fill_up_connecting_edges.GetError();
}
auto [in_edges, out_edges] = fill_up_connecting_edges.GetValue();
msgs::ExpandOneResultRow result_row;
result_row.src_vertex = std::move(source_vertex);
result_row.src_vertex_properties = std::move(*src_vertex_properties);
static constexpr bool kInEdges = true;
static constexpr bool kOutEdges = false;
if (const auto fill_edges_res = FillEdges<kInEdges>(in_edges, result_row, edge_filler); fill_edges_res.HasError()) {
return fill_edges_res.GetError();
}
if (const auto fill_edges_res = FillEdges<kOutEdges>(out_edges, result_row, edge_filler); fill_edges_res.HasError()) {
return fill_edges_res.GetError();
}
return result_row;
}
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
VertexAccessor v_acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
std::vector<EdgeAccessor> in_edge_accessors, std::vector<EdgeAccessor> out_edge_accessors,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
const Schemas::Schema &schema) {
/// Fill up source vertex
msgs::Vertex source_vertex = {.id = src_vertex};
auto maybe_secondary_labels = FillUpSourceVertexSecondaryLabels(v_acc, req);
if (maybe_secondary_labels.HasError()) {
return maybe_secondary_labels.GetError();
}
source_vertex.labels = std::move(*maybe_secondary_labels);
/// Fill up source vertex properties
auto src_vertex_properties = FillUpSourceVertexProperties(v_acc, req, storage::v3::View::NEW, schema);
if (src_vertex_properties.HasError()) {
return src_vertex_properties.GetError();
}
/// Fill up connecting edges
auto in_edges = maybe_filter_based_on_edge_uniqueness(std::move(in_edge_accessors), msgs::EdgeDirection::IN);
auto out_edges = maybe_filter_based_on_edge_uniqueness(std::move(out_edge_accessors), msgs::EdgeDirection::OUT);
msgs::ExpandOneResultRow result_row;
result_row.src_vertex = std::move(source_vertex);
result_row.src_vertex_properties = std::move(*src_vertex_properties);
static constexpr bool kInEdges = true;
static constexpr bool kOutEdges = false;
if (const auto fill_edges_res = FillEdges<kInEdges>(in_edges, result_row, edge_filler); fill_edges_res.HasError()) {
return fill_edges_res.GetError();
}
if (const auto fill_edges_res = FillEdges<kOutEdges>(out_edges, result_row, edge_filler); fill_edges_res.HasError()) {
return fill_edges_res.GetError();
}
return result_row;
return ordered;
}
VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_iterable,
const std::vector<PropertyValue> &primary_key, const View view) {
const std::vector<PropertyValue> &start_ids, const View view) {
auto it = vertex_iterable.begin();
while (it != vertex_iterable.end()) {
if (const auto &vertex = *it; primary_key <= vertex.PrimaryKey(view).GetValue()) {
if (const auto &vertex = *it; start_ids <= vertex.PrimaryKey(view).GetValue()) {
break;
}
++it;
@@ -475,112 +68,15 @@ VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_itera
return it;
}
std::vector<Element<VertexAccessor>>::const_iterator GetStartOrderedElementsIterator(
const std::vector<Element<VertexAccessor>> &ordered_elements, const std::vector<PropertyValue> &primary_key,
const View view) {
std::vector<Element>::const_iterator GetStartOrderedElementsIterator(const std::vector<Element> &ordered_elements,
const std::vector<PropertyValue> &start_ids,
const View view) {
for (auto it = ordered_elements.begin(); it != ordered_elements.end(); ++it) {
if (const auto &vertex = it->object_acc; primary_key <= vertex.PrimaryKey(view).GetValue()) {
if (const auto &vertex = it->vertex_acc; start_ids <= vertex.PrimaryKey(view).GetValue()) {
return it;
}
}
return ordered_elements.end();
}
std::array<std::vector<EdgeAccessor>, 2> GetEdgesFromVertex(const VertexAccessor &vertex_accessor,
const msgs::EdgeDirection direction) {
std::vector<EdgeAccessor> in_edges;
std::vector<EdgeAccessor> out_edges;
switch (direction) {
case memgraph::msgs::EdgeDirection::IN: {
auto edges = vertex_accessor.InEdges(View::OLD);
if (edges.HasValue()) {
in_edges = edges.GetValue();
}
break;
}
case memgraph::msgs::EdgeDirection::OUT: {
auto edges = vertex_accessor.OutEdges(View::OLD);
if (edges.HasValue()) {
out_edges = edges.GetValue();
}
break;
}
case memgraph::msgs::EdgeDirection::BOTH: {
auto maybe_in_edges = vertex_accessor.InEdges(View::OLD);
auto maybe_out_edges = vertex_accessor.OutEdges(View::OLD);
std::vector<EdgeAccessor> edges;
if (maybe_in_edges.HasValue()) {
in_edges = maybe_in_edges.GetValue();
}
if (maybe_out_edges.HasValue()) {
out_edges = maybe_out_edges.GetValue();
}
break;
}
}
return std::array<std::vector<EdgeAccessor>, 2>{std::move(in_edges), std::move(out_edges)};
}
std::vector<Element<EdgeAccessor>> OrderByEdges(DbAccessor &dba, std::vector<EdgeAccessor> &iterable,
std::vector<msgs::OrderBy> &order_by_edges,
const VertexAccessor &vertex_acc) {
std::vector<Ordering> ordering;
ordering.reserve(order_by_edges.size());
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(ordering),
[](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
std::vector<Element<EdgeAccessor>> ordered;
for (auto it = iterable.begin(); it != iterable.end(); ++it) {
std::vector<TypedValue> properties_order_by;
properties_order_by.reserve(order_by_edges.size());
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(properties_order_by),
[&dba, &vertex_acc, &it](const auto &order_by) {
return ComputeExpression(dba, vertex_acc, *it, order_by.expression.expression,
expr::identifier_node_symbol, expr::identifier_edge_symbol);
});
ordered.push_back({std::move(properties_order_by), *it});
}
auto compare_typed_values = TypedValueVectorCompare(ordering);
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
});
return ordered;
}
std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> OrderByEdges(
DbAccessor &dba, std::vector<EdgeAccessor> &iterable, std::vector<msgs::OrderBy> &order_by_edges,
const std::vector<VertexAccessor> &vertex_acc) {
MG_ASSERT(vertex_acc.size() == iterable.size());
std::vector<Ordering> ordering;
ordering.reserve(order_by_edges.size());
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(ordering),
[](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> ordered;
VertexAccessor current = vertex_acc.front();
size_t id = 0;
for (auto it = iterable.begin(); it != iterable.end(); it++, id++) {
current = vertex_acc[id];
std::vector<TypedValue> properties_order_by;
properties_order_by.reserve(order_by_edges.size());
std::transform(order_by_edges.begin(), order_by_edges.end(), std::back_inserter(properties_order_by),
[&dba, it, current](const auto &order_by) {
return ComputeExpression(dba, current, *it, order_by.expression.expression,
expr::identifier_node_symbol, expr::identifier_edge_symbol);
});
ordered.push_back({std::move(properties_order_by), {current, *it}});
}
auto compare_typed_values = TypedValueVectorCompare(ordering);
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
});
return ordered;
}
} // namespace memgraph::storage::v3

View File

@@ -9,30 +9,14 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <vector>
#include "query/v2/requests.hpp"
#include "storage/v3/bindings/ast/ast.hpp"
#include "storage/v3/bindings/pretty_print_ast_to_original_expression.hpp"
#include "ast/ast.hpp"
#include "storage/v3/bindings/typed_value.hpp"
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/expr.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/value_conversions.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "utils/template_utils.hpp"
namespace memgraph::storage::v3 {
using EdgeAccessors = std::vector<storage::v3::EdgeAccessor>;
using EdgeUniquenessFunction = std::function<EdgeAccessors(EdgeAccessors &&, msgs::EdgeDirection)>;
using EdgeFiller =
std::function<ShardResult<void>(const EdgeAccessor &edge, bool is_in_edge, msgs::ExpandOneResultRow &result_row)>;
using msgs::Value;
template <typename T>
concept OrderableObject = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor, std::pair<VertexAccessor, EdgeAccessor>>;
inline bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
// in ordering null comes after everything else
@@ -88,17 +72,6 @@ inline bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
}
}
inline Ordering ConvertMsgsOrderByToOrdering(msgs::OrderingDirection ordering) {
switch (ordering) {
case memgraph::msgs::OrderingDirection::ASCENDING:
return memgraph::storage::v3::Ordering::ASC;
case memgraph::msgs::OrderingDirection::DESCENDING:
return memgraph::storage::v3::Ordering::DESC;
default:
LOG_FATAL("Unknown ordering direction");
}
}
class TypedValueVectorCompare final {
public:
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
@@ -126,134 +99,18 @@ class TypedValueVectorCompare final {
std::vector<Ordering> ordering_;
};
template <OrderableObject TObjectAccessor>
struct Element {
std::vector<TypedValue> properties_order_by;
TObjectAccessor object_acc;
VertexAccessor vertex_acc;
};
template <typename T>
concept VerticesIt = utils::SameAsAnyOf<T, VerticesIterable, std::vector<VertexAccessor>>;
template <VerticesIt TIterable>
std::vector<Element<VertexAccessor>> OrderByVertices(DbAccessor &dba, TIterable &iterable,
std::vector<msgs::OrderBy> &order_by_vertices) {
std::vector<Ordering> ordering;
ordering.reserve(order_by_vertices.size());
std::transform(order_by_vertices.begin(), order_by_vertices.end(), std::back_inserter(ordering),
[](const auto &order_by) { return ConvertMsgsOrderByToOrdering(order_by.direction); });
std::vector<Element<VertexAccessor>> ordered;
for (auto it = iterable.begin(); it != iterable.end(); ++it) {
std::vector<TypedValue> properties_order_by;
properties_order_by.reserve(order_by_vertices.size());
std::transform(order_by_vertices.begin(), order_by_vertices.end(), std::back_inserter(properties_order_by),
[&dba, &it](const auto &order_by) {
return ComputeExpression(dba, *it, std::nullopt /*e_acc*/, order_by.expression.expression,
expr::identifier_node_symbol, expr::identifier_edge_symbol);
});
ordered.push_back({std::move(properties_order_by), *it});
}
auto compare_typed_values = TypedValueVectorCompare(ordering);
std::sort(ordered.begin(), ordered.end(), [compare_typed_values](const auto &pair1, const auto &pair2) {
return compare_typed_values(pair1.properties_order_by, pair2.properties_order_by);
});
return ordered;
}
std::vector<Element<EdgeAccessor>> OrderByEdges(DbAccessor &dba, std::vector<EdgeAccessor> &iterable,
std::vector<msgs::OrderBy> &order_by_edges,
const VertexAccessor &vertex_acc);
std::vector<Element<std::pair<VertexAccessor, EdgeAccessor>>> OrderByEdges(
DbAccessor &dba, std::vector<EdgeAccessor> &iterable, std::vector<msgs::OrderBy> &order_by_edges,
const std::vector<VertexAccessor> &vertex_acc);
std::vector<Element> OrderByElements(Shard::Accessor &acc, DbAccessor &dba, VerticesIterable &vertices_iterable,
std::vector<msgs::OrderBy> &order_bys);
VerticesIterable::Iterator GetStartVertexIterator(VerticesIterable &vertex_iterable,
const std::vector<PropertyValue> &primary_key, View view);
const std::vector<PropertyValue> &start_ids, View view);
std::vector<Element<VertexAccessor>>::const_iterator GetStartOrderedElementsIterator(
const std::vector<Element<VertexAccessor>> &ordered_elements, const std::vector<PropertyValue> &primary_key,
View view);
std::array<std::vector<EdgeAccessor>, 2> GetEdgesFromVertex(const VertexAccessor &vertex_accessor,
msgs::EdgeDirection direction);
bool FilterOnVertex(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const std::vector<std::string> &filters);
bool FilterOnEdge(DbAccessor &dba, const storage::v3::VertexAccessor &v_acc, const EdgeAccessor &e_acc,
const std::vector<std::string> &filters);
std::vector<TypedValue> EvaluateVertexExpressions(DbAccessor &dba, const VertexAccessor &v_acc,
const std::vector<std::string> &expressions,
std::string_view node_name);
std::vector<TypedValue> EvaluateEdgeExpressions(DbAccessor &dba, const VertexAccessor &v_acc, const EdgeAccessor &e_acc,
const std::vector<std::string> &expressions);
template <typename T>
concept PropertiesAccessor = utils::SameAsAnyOf<T, VertexAccessor, EdgeAccessor>;
template <PropertiesAccessor TAccessor>
ShardResult<std::map<PropertyId, Value>> CollectSpecificPropertiesFromAccessor(const TAccessor &acc,
const std::vector<PropertyId> &props,
View view) {
std::map<PropertyId, Value> ret;
for (const auto &prop : props) {
auto result = acc.GetProperty(prop, view);
if (result.HasError()) {
spdlog::debug("Encountered an Error while trying to get a vertex property.");
return result.GetError();
}
auto &value = result.GetValue();
ret.emplace(std::make_pair(prop, FromPropertyValueToValue(std::move(value))));
}
return ret;
}
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const VertexAccessor &acc, View view,
const Schemas::Schema &schema);
namespace impl {
template <PropertiesAccessor TAccessor>
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesImpl(const TAccessor &acc, View view) {
std::map<PropertyId, Value> ret;
auto props = acc.Properties(view);
if (props.HasError()) {
spdlog::debug("Encountered an error while trying to get vertex properties.");
return props.GetError();
}
auto &properties = props.GetValue();
std::transform(properties.begin(), properties.end(), std::inserter(ret, ret.begin()),
[](std::pair<const PropertyId, PropertyValue> &pair) {
return std::make_pair(pair.first, conversions::FromPropertyValueToValue(std::move(pair.second)));
});
return ret;
}
} // namespace impl
template <PropertiesAccessor TAccessor>
ShardResult<std::map<PropertyId, Value>> CollectAllPropertiesFromAccessor(const TAccessor &acc, View view) {
return impl::CollectAllPropertiesImpl<TAccessor>(acc, view);
}
EdgeUniquenessFunction InitializeEdgeUniquenessFunction(bool only_unique_neighbor_rows);
EdgeFiller InitializeEdgeFillerFunction(const msgs::ExpandOneRequest &req);
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
Shard::Accessor &acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
const Schemas::Schema &schema);
ShardResult<msgs::ExpandOneResultRow> GetExpandOneResult(
VertexAccessor v_acc, msgs::VertexId src_vertex, const msgs::ExpandOneRequest &req,
std::vector<EdgeAccessor> in_edge_accessors, std::vector<EdgeAccessor> out_edge_accessors,
const EdgeUniquenessFunction &maybe_filter_based_on_edge_uniqueness, const EdgeFiller &edge_filler,
const Schemas::Schema &schema);
std::vector<Element>::const_iterator GetStartOrderedElementsIterator(const std::vector<Element> &ordered_elements,
const std::vector<PropertyValue> &start_ids,
View view);
} // namespace memgraph::storage::v3

View File

@@ -11,43 +11,24 @@
#pragma once
#include <cstdint>
#include <experimental/source_location>
#include <string>
#include <string_view>
#include <type_traits>
#include "common/errors.hpp"
#include "utils/result.hpp"
namespace memgraph::storage::v3 {
static_assert(std::is_same_v<uint8_t, unsigned char>);
struct ShardError {
ShardError(common::ErrorCode code, std::string message, const std::experimental::source_location location)
: code{code}, message{std::move(message)}, source{fmt::format("{}:{}", location.file_name(), location.line())} {}
ShardError(common::ErrorCode code, const std::experimental::source_location location)
: code{code}, source{fmt::format("{}:{}", location.file_name(), location.line())} {}
common::ErrorCode code;
std::string message;
std::string source;
inline friend bool operator==(const ShardError &lhs, const ShardError &rhs) { return lhs.code == rhs.code; }
inline friend bool operator==(const ShardError &lhs, const common::ErrorCode rhs) { return lhs.code == rhs; }
enum class Error : uint8_t {
SERIALIZATION_ERROR,
NONEXISTENT_OBJECT,
DELETED_OBJECT,
VERTEX_HAS_EDGES,
PROPERTIES_DISABLED,
VERTEX_ALREADY_INSERTED
};
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define SHARD_ERROR(error, ...) \
({ \
using ErrorCode = memgraph::common::ErrorCode; \
memgraph::storage::v3::ShardError(error, GET_MESSAGE(__VA_ARGS__), std::experimental::source_location::current()); \
})
template <class TValue>
using ShardResult = utils::BasicResult<ShardError, TValue>;
using Result = utils::BasicResult<Error, TValue>;
} // namespace memgraph::storage::v3

View File

@@ -16,58 +16,67 @@
#include <ranges>
#include "common/types.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schemas.hpp"
namespace memgraph::storage::v3 {
SchemaValidator::SchemaValidator(Schemas &schemas, const NameIdMapper &name_id_mapper)
: schemas_{&schemas}, name_id_mapper_{&name_id_mapper} {}
bool operator==(const SchemaViolation &lhs, const SchemaViolation &rhs) {
return lhs.status == rhs.status && lhs.label == rhs.label &&
lhs.violated_schema_property == rhs.violated_schema_property &&
lhs.violated_property_value == rhs.violated_property_value;
}
ShardResult<void> SchemaValidator::ValidateVertexCreate(LabelId primary_label, const std::vector<LabelId> &labels,
const std::vector<PropertyValue> &primary_properties) const {
SchemaViolation::SchemaViolation(ValidationStatus status, LabelId label) : status{status}, label{label} {}
SchemaViolation::SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property)
: status{status}, label{label}, violated_schema_property{violated_schema_property} {}
SchemaViolation::SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property,
PropertyValue violated_property_value)
: status{status},
label{label},
violated_schema_property{violated_schema_property},
violated_property_value{violated_property_value} {}
SchemaValidator::SchemaValidator(Schemas &schemas) : schemas_{schemas} {}
std::optional<SchemaViolation> SchemaValidator::ValidateVertexCreate(
LabelId primary_label, const std::vector<LabelId> &labels,
const std::vector<PropertyValue> &primary_properties) const {
// Schema on primary label
const auto *schema = schemas_->GetSchema(primary_label);
const auto *schema = schemas_.GetSchema(primary_label);
if (schema == nullptr) {
return SHARD_ERROR(ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL, "Schema not defined for label :{}",
name_id_mapper_->IdToName(primary_label.AsInt()));
return SchemaViolation(SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL, primary_label);
}
// Is there another primary label among secondary labels
for (const auto &secondary_label : labels) {
if (schemas_->GetSchema(secondary_label)) {
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
"Cannot add label :{}, since it is defined as a primary label",
name_id_mapper_->IdToName(secondary_label.AsInt()));
if (schemas_.GetSchema(secondary_label)) {
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY, secondary_label);
}
}
// Quick size check
if (schema->second.size() != primary_properties.size()) {
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
"Not all primary properties have been specified for :{} vertex",
name_id_mapper_->IdToName(primary_label.AsInt()));
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_PRIMARY_PROPERTIES_UNDEFINED, primary_label);
}
// Check only properties defined by schema
for (size_t i{0}; i < schema->second.size(); ++i) {
// Check schema property type
if (auto property_schema_type = PropertyTypeToSchemaType(primary_properties[i]);
property_schema_type && *property_schema_type != schema->second[i].type) {
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
"Property {} is of wrong type, expected {}, actual {}",
name_id_mapper_->IdToName(schema->second[i].property_id.AsInt()),
SchemaTypeToString(schema->second[i].type), SchemaTypeToString(*property_schema_type));
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE, primary_label,
schema->second[i], primary_properties[i]);
}
}
return {};
return std::nullopt;
}
ShardResult<void> SchemaValidator::ValidatePropertyUpdate(const LabelId primary_label,
const PropertyId property_id) const {
std::optional<SchemaViolation> SchemaValidator::ValidatePropertyUpdate(const LabelId primary_label,
const PropertyId property_id) const {
// Verify existence of schema on primary label
const auto *schema = schemas_->GetSchema(primary_label);
const auto *schema = schemas_.GetSchema(primary_label);
MG_ASSERT(schema, "Cannot validate against non existing schema!");
// Verify that updating property is not part of schema
@@ -75,37 +84,34 @@ ShardResult<void> SchemaValidator::ValidatePropertyUpdate(const LabelId primary_
schema->second,
[property_id](const auto &schema_property) { return property_id == schema_property.property_id; });
schema_property != schema->second.end()) {
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
"Cannot update primary property {} of schema on label :{}",
name_id_mapper_->IdToName(schema_property->property_id.AsInt()),
name_id_mapper_->IdToName(primary_label.AsInt()));
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY, primary_label,
*schema_property);
}
return {};
return std::nullopt;
}
ShardResult<void> SchemaValidator::ValidateLabelUpdate(const LabelId label) const {
const auto *schema = schemas_->GetSchema(label);
std::optional<SchemaViolation> SchemaValidator::ValidateLabelUpdate(const LabelId label) const {
const auto *schema = schemas_.GetSchema(label);
if (schema) {
return SHARD_ERROR(ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL, "Cannot add/remove primary label :{}",
name_id_mapper_->IdToName(label.AsInt()));
return SchemaViolation(SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL, label);
}
return {};
return std::nullopt;
}
const Schemas::Schema *SchemaValidator::GetSchema(LabelId label) const { return schemas_->GetSchema(label); }
const Schemas::Schema *SchemaValidator::GetSchema(LabelId label) const { return schemas_.GetSchema(label); }
VertexValidator::VertexValidator(const SchemaValidator &schema_validator, const LabelId primary_label)
: schema_validator{&schema_validator}, primary_label_{primary_label} {}
ShardResult<void> VertexValidator::ValidatePropertyUpdate(PropertyId property_id) const {
std::optional<SchemaViolation> VertexValidator::ValidatePropertyUpdate(PropertyId property_id) const {
return schema_validator->ValidatePropertyUpdate(primary_label_, property_id);
};
ShardResult<void> VertexValidator::ValidateAddLabel(LabelId label) const {
std::optional<SchemaViolation> VertexValidator::ValidateAddLabel(LabelId label) const {
return schema_validator->ValidateLabelUpdate(label);
}
ShardResult<void> VertexValidator::ValidateRemoveLabel(LabelId label) const {
std::optional<SchemaViolation> VertexValidator::ValidateRemoveLabel(LabelId label) const {
return schema_validator->ValidateLabelUpdate(label);
}

View File

@@ -11,43 +11,68 @@
#pragma once
#include <optional>
#include <variant>
#include "storage/v2/result.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schemas.hpp"
namespace memgraph::storage::v3 {
struct SchemaViolation {
enum class ValidationStatus : uint8_t {
NO_SCHEMA_DEFINED_FOR_LABEL,
VERTEX_PROPERTY_WRONG_TYPE,
VERTEX_UPDATE_PRIMARY_KEY,
VERTEX_UPDATE_PRIMARY_LABEL,
VERTEX_SECONDARY_LABEL_IS_PRIMARY,
VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
};
SchemaViolation(ValidationStatus status, LabelId label);
SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property);
SchemaViolation(ValidationStatus status, LabelId label, SchemaProperty violated_schema_property,
PropertyValue violated_property_value);
friend bool operator==(const SchemaViolation &lhs, const SchemaViolation &rhs);
ValidationStatus status;
LabelId label;
std::optional<SchemaProperty> violated_schema_property;
std::optional<PropertyValue> violated_property_value;
};
class SchemaValidator {
public:
explicit SchemaValidator(Schemas &schemas, const NameIdMapper &name_id_mapper);
explicit SchemaValidator(Schemas &schemas);
[[nodiscard]] ShardResult<void> ValidateVertexCreate(LabelId primary_label, const std::vector<LabelId> &labels,
const std::vector<PropertyValue> &primary_properties) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateVertexCreate(
LabelId primary_label, const std::vector<LabelId> &labels,
const std::vector<PropertyValue> &primary_properties) const;
[[nodiscard]] ShardResult<void> ValidatePropertyUpdate(LabelId primary_label, PropertyId property_id) const;
[[nodiscard]] std::optional<SchemaViolation> ValidatePropertyUpdate(LabelId primary_label,
PropertyId property_id) const;
[[nodiscard]] ShardResult<void> ValidateLabelUpdate(LabelId label) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateLabelUpdate(LabelId label) const;
const Schemas::Schema *GetSchema(LabelId label) const;
private:
Schemas *schemas_;
const NameIdMapper *name_id_mapper_;
Schemas &schemas_;
};
struct VertexValidator {
explicit VertexValidator(const SchemaValidator &schema_validator, LabelId primary_label);
[[nodiscard]] ShardResult<void> ValidatePropertyUpdate(PropertyId property_id) const;
[[nodiscard]] std::optional<SchemaViolation> ValidatePropertyUpdate(PropertyId property_id) const;
[[nodiscard]] ShardResult<void> ValidateAddLabel(LabelId label) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateAddLabel(LabelId label) const;
[[nodiscard]] ShardResult<void> ValidateRemoveLabel(LabelId label) const;
[[nodiscard]] std::optional<SchemaViolation> ValidateRemoveLabel(LabelId label) const;
const SchemaValidator *schema_validator;

View File

@@ -31,14 +31,13 @@
#include "storage/v3/indices.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/shard_operation_result.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/view.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "utils/exceptions.hpp"
#include "utils/file.hpp"
#include "utils/logging.hpp"
@@ -75,11 +74,11 @@ uint64_t GetCleanupBeforeTimestamp(const std::map<uint64_t, std::unique_ptr<Tran
} // namespace
auto AdvanceToVisibleVertex(VertexContainer::iterator it, VertexContainer::iterator end,
auto AdvanceToVisibleVertex(VerticesSkipList::Iterator it, VerticesSkipList::Iterator end,
std::optional<VertexAccessor> *vertex, Transaction *tx, View view, Indices *indices,
Config::Items config, const VertexValidator &vertex_validator) {
while (it != end) {
*vertex = VertexAccessor::Create(&*it, tx, indices, config, vertex_validator, view);
*vertex = VertexAccessor::Create(&it->vertex, tx, indices, config, vertex_validator, view);
if (!*vertex) {
++it;
continue;
@@ -89,17 +88,17 @@ auto AdvanceToVisibleVertex(VertexContainer::iterator it, VertexContainer::itera
return it;
}
AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VertexContainer::iterator it)
AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it)
: self_(self),
it_(AdvanceToVisibleVertex(it, self->vertices_accessor_->end(), &self->vertex_, self->transaction_, self->view_,
it_(AdvanceToVisibleVertex(it, self->vertices_accessor_.end(), &self->vertex_, self->transaction_, self->view_,
self->indices_, self->config_, *self_->vertex_validator_)) {}
VertexAccessor AllVerticesIterable::Iterator::operator*() const { return *self_->vertex_; }
AllVerticesIterable::Iterator &AllVerticesIterable::Iterator::operator++() {
++it_;
it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_->end(), &self_->vertex_, self_->transaction_,
self_->view_, self_->indices_, self_->config_, *self_->vertex_validator_);
it_ = AdvanceToVisibleVertex(it_, self_->vertices_accessor_.end(), &self_->vertex_, self_->transaction_, self_->view_,
self_->indices_, self_->config_, *self_->vertex_validator_);
return *this;
}
@@ -328,7 +327,7 @@ Shard::Shard(const LabelId primary_label, const PrimaryKey min_primary_key,
: primary_label_{primary_label},
min_primary_key_{min_primary_key},
max_primary_key_{max_primary_key},
schema_validator_{schemas_, name_id_mapper_},
schema_validator_{schemas_},
vertex_validator_{schema_validator_, primary_label},
indices_{config.items, vertex_validator_},
isolation_level_{config.transaction.isolation_level},
@@ -345,27 +344,28 @@ Shard::~Shard() {}
Shard::Accessor::Accessor(Shard &shard, Transaction &transaction)
: shard_(&shard), transaction_(&transaction), config_(shard_->config_.items) {}
ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
const std::vector<LabelId> &labels, const PrimaryKey &primary_properties,
ShardOperationResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
const std::vector<std::pair<PropertyId, PropertyValue>> &properties) {
OOMExceptionEnabler oom_exception;
const auto schema = shard_->GetSchema(shard_->primary_label_)->second;
auto maybe_schema_violation =
GetSchemaValidator().ValidateVertexCreate(shard_->primary_label_, labels, primary_properties);
if (maybe_schema_violation.HasError()) {
return {std::move(maybe_schema_violation.GetError())};
if (maybe_schema_violation) {
return {std::move(*maybe_schema_violation)};
}
auto acc = shard_->vertices_.access();
auto *delta = CreateDeleteObjectDelta(transaction_);
auto [it, inserted] = shard_->vertices_.emplace(primary_properties, VertexData{delta});
delta->prev.Set(&*it);
auto [it, inserted] = acc.insert({Vertex{delta, primary_properties}});
delta->prev.Set(&it->vertex);
VertexAccessor vertex_acc{&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
VertexAccessor vertex_acc{&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_};
if (!inserted) {
return SHARD_ERROR(ErrorCode::VERTEX_ALREADY_INSERTED);
return {Error::VERTEX_ALREADY_INSERTED};
}
MG_ASSERT(it != shard_->vertices_.end(), "Invalid Vertex accessor!");
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
// TODO(jbajic) Improve, maybe delay index update
for (const auto &[property_id, property_value] : properties) {
@@ -385,36 +385,37 @@ ShardResult<VertexAccessor> Shard::Accessor::CreateVertexAndValidate(
}
std::optional<VertexAccessor> Shard::Accessor::FindVertex(std::vector<PropertyValue> primary_key, View view) {
auto it = shard_->vertices_.find(primary_key);
if (it == shard_->vertices_.end()) {
auto acc = shard_->vertices_.access();
// Later on use label space
auto it = acc.find(primary_key);
if (it == acc.end()) {
return std::nullopt;
}
return VertexAccessor::Create(&*it, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view);
return VertexAccessor::Create(&it->vertex, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, view);
}
ShardResult<std::optional<VertexAccessor>> Shard::Accessor::DeleteVertex(VertexAccessor *vertex) {
Result<std::optional<VertexAccessor>> Shard::Accessor::DeleteVertex(VertexAccessor *vertex) {
MG_ASSERT(vertex->transaction_ == transaction_,
"VertexAccessor must be from the same transaction as the storage "
"accessor when deleting a vertex!");
auto *vertex_ptr = vertex->vertex_;
if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
if (vertex_ptr->second.deleted) {
if (vertex_ptr->deleted) {
return std::optional<VertexAccessor>{};
}
if (!vertex_ptr->second.in_edges.empty() || !vertex_ptr->second.out_edges.empty())
return SHARD_ERROR(ErrorCode::VERTEX_HAS_EDGES);
if (!vertex_ptr->in_edges.empty() || !vertex_ptr->out_edges.empty()) return Error::VERTEX_HAS_EDGES;
CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag());
vertex_ptr->second.deleted = true;
vertex_ptr->deleted = true;
return std::make_optional<VertexAccessor>(vertex_ptr, transaction_, &shard_->indices_, config_,
shard_->vertex_validator_, true);
}
ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> Shard::Accessor::DetachDeleteVertex(
Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> Shard::Accessor::DetachDeleteVertex(
VertexAccessor *vertex) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
@@ -423,26 +424,26 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
"accessor when deleting a vertex!");
auto *vertex_ptr = vertex->vertex_;
std::vector<VertexData::EdgeLink> in_edges;
std::vector<VertexData::EdgeLink> out_edges;
std::vector<Vertex::EdgeLink> in_edges;
std::vector<Vertex::EdgeLink> out_edges;
{
if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
if (vertex_ptr->second.deleted) return std::optional<ReturnType>{};
if (vertex_ptr->deleted) return std::optional<ReturnType>{};
in_edges = vertex_ptr->second.in_edges;
out_edges = vertex_ptr->second.out_edges;
in_edges = vertex_ptr->in_edges;
out_edges = vertex_ptr->out_edges;
}
std::vector<EdgeAccessor> deleted_edges;
const VertexId vertex_id{shard_->primary_label_, *vertex->PrimaryKey(View::OLD)}; // TODO Replace
const VertexId vertex_id{shard_->primary_label_, vertex_ptr->keys.Keys()};
for (const auto &item : in_edges) {
auto [edge_type, from_vertex, edge] = item;
EdgeAccessor e(edge, edge_type, from_vertex, vertex_id, transaction_, &shard_->indices_, config_);
auto ret = DeleteEdge(e.From(), e.To(), e.Gid());
auto ret = DeleteEdge(e.FromVertex(), e.ToVertex(), e.Gid());
if (ret.HasError()) {
MG_ASSERT(ret.GetError() == common::ErrorCode::SERIALIZATION_ERROR, "Invalid database state!");
MG_ASSERT(ret.GetError() == Error::SERIALIZATION_ERROR, "Invalid database state!");
return ret.GetError();
}
@@ -453,9 +454,9 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
for (const auto &item : out_edges) {
auto [edge_type, to_vertex, edge] = item;
EdgeAccessor e(edge, edge_type, vertex_id, to_vertex, transaction_, &shard_->indices_, config_);
auto ret = DeleteEdge(e.From(), e.To(), e.Gid());
auto ret = DeleteEdge(e.FromVertex(), e.ToVertex(), e.Gid());
if (ret.HasError()) {
MG_ASSERT(ret.GetError() == common::ErrorCode::SERIALIZATION_ERROR, "Invalid database state!");
MG_ASSERT(ret.GetError() == Error::SERIALIZATION_ERROR, "Invalid database state!");
return ret.GetError();
}
@@ -468,68 +469,69 @@ ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>>
// vertex. Some other transaction could have modified the vertex in the
// meantime if we didn't have any edges to delete.
if (!PrepareForWrite(transaction_, vertex_ptr)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (!PrepareForWrite(transaction_, vertex_ptr)) return Error::SERIALIZATION_ERROR;
MG_ASSERT(!vertex_ptr->second.deleted, "Invalid database state!");
MG_ASSERT(!vertex_ptr->deleted, "Invalid database state!");
CreateAndLinkDelta(transaction_, vertex_ptr, Delta::RecreateObjectTag());
vertex_ptr->second.deleted = true;
vertex_ptr->deleted = true;
return std::make_optional<ReturnType>(
VertexAccessor{vertex_ptr, transaction_, &shard_->indices_, config_, shard_->vertex_validator_, true},
std::move(deleted_edges));
}
ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id,
const EdgeTypeId edge_type, const Gid gid) {
Result<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id,
const EdgeTypeId edge_type, const Gid gid) {
OOMExceptionEnabler oom_exception;
Vertex *from_vertex{nullptr};
Vertex *to_vertex{nullptr};
auto &vertices = shard_->vertices_;
auto acc = shard_->vertices_.access();
const auto from_is_local = shard_->IsVertexBelongToShard(from_vertex_id);
const auto to_is_local = shard_->IsVertexBelongToShard(to_vertex_id);
MG_ASSERT(from_is_local || to_is_local, "Trying to create an edge without having a local vertex");
if (from_is_local) {
auto it = vertices.find(from_vertex_id.primary_key);
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
from_vertex = &*it;
auto it = acc.find(from_vertex_id.primary_key);
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
from_vertex = &it->vertex;
}
if (to_is_local) {
auto it = vertices.find(to_vertex_id.primary_key);
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
to_vertex = &*it;
auto it = acc.find(to_vertex_id.primary_key);
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
to_vertex = &it->vertex;
}
if (from_is_local) {
if (!PrepareForWrite(transaction_, from_vertex)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (from_vertex->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
if (!PrepareForWrite(transaction_, from_vertex)) return Error::SERIALIZATION_ERROR;
if (from_vertex->deleted) return Error::DELETED_OBJECT;
}
if (to_is_local && to_vertex != from_vertex) {
if (!PrepareForWrite(transaction_, to_vertex)) return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
if (to_vertex->second.deleted) return SHARD_ERROR(ErrorCode::DELETED_OBJECT);
if (!PrepareForWrite(transaction_, to_vertex)) return Error::SERIALIZATION_ERROR;
if (to_vertex->deleted) return Error::DELETED_OBJECT;
}
EdgeRef edge(gid);
if (config_.properties_on_edges) {
auto acc = shard_->edges_.access();
auto *delta = CreateDeleteObjectDelta(transaction_);
auto [it, inserted] = shard_->edges_.emplace(gid, Edge{gid, delta});
auto [it, inserted] = acc.insert(Edge(gid, delta));
MG_ASSERT(inserted, "The edge must be inserted here!");
MG_ASSERT(it != shard_->edges_.end(), "Invalid Edge accessor!");
edge = EdgeRef(&it->second);
delta->prev.Set(&it->second);
MG_ASSERT(it != acc.end(), "Invalid Edge accessor!");
edge = EdgeRef(&*it);
delta->prev.Set(&*it);
}
if (from_is_local) {
CreateAndLinkDelta(transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex_id, edge);
from_vertex->second.out_edges.emplace_back(edge_type, to_vertex_id, edge);
from_vertex->out_edges.emplace_back(edge_type, to_vertex_id, edge);
}
if (to_is_local) {
CreateAndLinkDelta(transaction_, to_vertex, Delta::RemoveInEdgeTag(), edge_type, from_vertex_id, edge);
to_vertex->second.in_edges.emplace_back(edge_type, from_vertex_id, edge);
to_vertex->in_edges.emplace_back(edge_type, from_vertex_id, edge);
}
// Increment edge count.
++shard_->edge_count_;
@@ -538,56 +540,57 @@ ShardResult<EdgeAccessor> Shard::Accessor::CreateEdge(VertexId from_vertex_id, V
&shard_->indices_, config_);
}
ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id,
const Gid edge_id) {
VertexContainer::value_type *from_vertex{nullptr};
VertexContainer::value_type *to_vertex{nullptr};
Result<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id,
const Gid edge_id) {
Vertex *from_vertex{nullptr};
Vertex *to_vertex{nullptr};
auto &vertices = shard_->vertices_;
auto acc = shard_->vertices_.access();
const auto from_is_local = shard_->IsVertexBelongToShard(from_vertex_id);
const auto to_is_local = shard_->IsVertexBelongToShard(to_vertex_id);
if (from_is_local) {
auto it = vertices.find(from_vertex_id.primary_key);
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
from_vertex = &*it;
auto it = acc.find(from_vertex_id.primary_key);
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
from_vertex = &it->vertex;
}
if (to_is_local) {
auto it = vertices.find(to_vertex_id.primary_key);
MG_ASSERT(it != vertices.end(), "Cannot find local vertex");
to_vertex = &*it;
auto it = acc.find(to_vertex_id.primary_key);
MG_ASSERT(it != acc.end(), "Cannot find local vertex");
to_vertex = &it->vertex;
}
MG_ASSERT(from_is_local || to_is_local, "Trying to delete an edge without having a local vertex");
if (from_is_local) {
if (!PrepareForWrite(transaction_, from_vertex)) {
return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
return Error::SERIALIZATION_ERROR;
}
MG_ASSERT(!from_vertex->second.deleted, "Invalid database state!");
MG_ASSERT(!from_vertex->deleted, "Invalid database state!");
}
if (to_is_local && to_vertex != from_vertex) {
if (!PrepareForWrite(transaction_, to_vertex)) {
return SHARD_ERROR(ErrorCode::SERIALIZATION_ERROR);
return Error::SERIALIZATION_ERROR;
}
MG_ASSERT(!to_vertex->second.deleted, "Invalid database state!");
MG_ASSERT(!to_vertex->deleted, "Invalid database state!");
}
const auto edge_ref = std::invoke([edge_id, this]() -> EdgeRef {
if (!config_.properties_on_edges) {
return EdgeRef(edge_id);
}
auto res = shard_->edges_.find(edge_id);
MG_ASSERT(res != shard_->edges_.end(), "Cannot find edge");
return EdgeRef(&res->second);
auto edge_acc = shard_->edges_.access();
auto res = edge_acc.find(edge_id);
MG_ASSERT(res != edge_acc.end(), "Cannot find edge");
return EdgeRef(&*res);
});
std::optional<EdgeTypeId> edge_type{};
auto delete_edge_from_storage = [&edge_type, &edge_ref, this](std::vector<VertexData::EdgeLink> &edges) mutable {
auto delete_edge_from_storage = [&edge_type, &edge_ref, this](std::vector<Vertex::EdgeLink> &edges) mutable {
auto it = std::find_if(edges.begin(), edges.end(),
[&edge_ref](const VertexData::EdgeLink &link) { return std::get<2>(link) == edge_ref; });
[&edge_ref](const Vertex::EdgeLink &link) { return std::get<2>(link) == edge_ref; });
if (config_.properties_on_edges) {
MG_ASSERT(it != edges.end(), "Invalid database state!");
} else if (it == edges.end()) {
@@ -599,8 +602,8 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
return true;
};
// NOLINTNEXTLINE(clang-analyzer-core.NonNullParamChecker)
auto success_on_to = to_is_local ? delete_edge_from_storage(to_vertex->second.in_edges) : false;
auto success_on_from = from_is_local ? delete_edge_from_storage(from_vertex->second.out_edges) : false;
auto success_on_to = to_is_local ? delete_edge_from_storage(to_vertex->in_edges) : false;
auto success_on_from = from_is_local ? delete_edge_from_storage(from_vertex->out_edges) : false;
if (config_.properties_on_edges) {
// Because of the check above, we are sure that the vertex exists.
@@ -626,10 +629,10 @@ ShardResult<std::optional<EdgeAccessor>> Shard::Accessor::DeleteEdge(VertexId fr
MG_ASSERT(edge_type.has_value(), "Edge type is not determined");
if (from_is_local) {
CreateAndLinkDelta(transaction_, &*from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref);
CreateAndLinkDelta(transaction_, from_vertex, Delta::AddOutEdgeTag(), *edge_type, to_vertex_id, edge_ref);
}
if (to_is_local) {
CreateAndLinkDelta(transaction_, &*to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref);
CreateAndLinkDelta(transaction_, to_vertex, Delta::AddInEdgeTag(), *edge_type, from_vertex_id, edge_ref);
}
// Decrement edge count.
@@ -691,41 +694,41 @@ void Shard::Accessor::Abort() {
auto prev = delta.prev.Get();
switch (prev.type) {
case PreviousPtr::Type::VERTEX: {
auto &[pk, vertex] = *prev.vertex;
Delta *current = vertex.delta;
auto *vertex = prev.vertex;
Delta *current = vertex->delta;
while (current != nullptr && current->commit_info->start_or_commit_timestamp == transaction_->start_timestamp) {
switch (current->action) {
case Delta::Action::REMOVE_LABEL: {
auto it = std::find(vertex.labels.begin(), vertex.labels.end(), current->label);
MG_ASSERT(it != vertex.labels.end(), "Invalid database state!");
std::swap(*it, *vertex.labels.rbegin());
vertex.labels.pop_back();
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
MG_ASSERT(it != vertex->labels.end(), "Invalid database state!");
std::swap(*it, *vertex->labels.rbegin());
vertex->labels.pop_back();
break;
}
case Delta::Action::ADD_LABEL: {
auto it = std::find(vertex.labels.begin(), vertex.labels.end(), current->label);
MG_ASSERT(it == vertex.labels.end(), "Invalid database state!");
vertex.labels.push_back(current->label);
auto it = std::find(vertex->labels.begin(), vertex->labels.end(), current->label);
MG_ASSERT(it == vertex->labels.end(), "Invalid database state!");
vertex->labels.push_back(current->label);
break;
}
case Delta::Action::SET_PROPERTY: {
vertex.properties.SetProperty(current->property.key, current->property.value);
vertex->properties.SetProperty(current->property.key, current->property.value);
break;
}
case Delta::Action::ADD_IN_EDGE: {
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex.in_edges.begin(), vertex.in_edges.end(), link);
MG_ASSERT(it == vertex.in_edges.end(), "Invalid database state!");
vertex.in_edges.push_back(link);
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
MG_ASSERT(it == vertex->in_edges.end(), "Invalid database state!");
vertex->in_edges.push_back(link);
break;
}
case Delta::Action::ADD_OUT_EDGE: {
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex.out_edges.begin(), vertex.out_edges.end(), link);
MG_ASSERT(it == vertex.out_edges.end(), "Invalid database state!");
vertex.out_edges.push_back(link);
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
MG_ASSERT(it == vertex->out_edges.end(), "Invalid database state!");
vertex->out_edges.push_back(link);
// Increment edge count. We only increment the count here because
// the information in `ADD_IN_EDGE` and `Edge/RECREATE_OBJECT` is
// redundant. Also, `Edge/RECREATE_OBJECT` isn't available when
@@ -734,21 +737,21 @@ void Shard::Accessor::Abort() {
break;
}
case Delta::Action::REMOVE_IN_EDGE: {
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex.in_edges.begin(), vertex.in_edges.end(), link);
MG_ASSERT(it != vertex.in_edges.end(), "Invalid database state!");
std::swap(*it, *vertex.in_edges.rbegin());
vertex.in_edges.pop_back();
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex->in_edges.begin(), vertex->in_edges.end(), link);
MG_ASSERT(it != vertex->in_edges.end(), "Invalid database state!");
std::swap(*it, *vertex->in_edges.rbegin());
vertex->in_edges.pop_back();
break;
}
case Delta::Action::REMOVE_OUT_EDGE: {
VertexData::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex.out_edges.begin(), vertex.out_edges.end(), link);
MG_ASSERT(it != vertex.out_edges.end(), "Invalid database state!");
std::swap(*it, *vertex.out_edges.rbegin());
vertex.out_edges.pop_back();
Vertex::EdgeLink link{current->vertex_edge.edge_type, current->vertex_edge.vertex_id,
current->vertex_edge.edge};
auto it = std::find(vertex->out_edges.begin(), vertex->out_edges.end(), link);
MG_ASSERT(it != vertex->out_edges.end(), "Invalid database state!");
std::swap(*it, *vertex->out_edges.rbegin());
vertex->out_edges.pop_back();
// Decrement edge count. We only decrement the count here because
// the information in `REMOVE_IN_EDGE` and `Edge/DELETE_OBJECT` is
// redundant. Also, `Edge/DELETE_OBJECT` isn't available when edge
@@ -757,20 +760,20 @@ void Shard::Accessor::Abort() {
break;
}
case Delta::Action::DELETE_OBJECT: {
vertex.deleted = true;
shard_->deleted_vertices_.push_back(&pk);
vertex->deleted = true;
shard_->deleted_vertices_.push_back(vertex->keys.Keys());
break;
}
case Delta::Action::RECREATE_OBJECT: {
vertex.deleted = false;
vertex->deleted = false;
break;
}
}
current = current->next;
}
vertex.delta = current;
vertex->delta = current;
if (current != nullptr) {
current->prev.Set(prev.vertex);
current->prev.Set(vertex);
}
break;
@@ -847,7 +850,10 @@ const std::string &Shard::EdgeTypeToName(EdgeTypeId edge_type) const {
bool Shard::CreateIndex(LabelId label, const std::optional<uint64_t> /*desired_commit_timestamp*/) {
// TODO(jbajic) response should be different when label == primary_label
return !(label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_));
if (label == primary_label_ || !indices_.label_index.CreateIndex(label, vertices_.access())) {
return false;
}
return true;
}
bool Shard::CreateIndex(LabelId label, PropertyId property,
@@ -858,7 +864,7 @@ bool Shard::CreateIndex(LabelId label, PropertyId property,
// Index already exists on primary key
return false;
}
if (!indices_.label_property_index.CreateIndex(label, property, vertices_)) {
if (!indices_.label_property_index.CreateIndex(label, property, vertices_.access())) {
return false;
}
return true;
@@ -965,12 +971,10 @@ void Shard::CollectGarbage(const io::Time current_time) {
auto prev = delta.prev.Get();
switch (prev.type) {
case PreviousPtr::Type::VERTEX: {
// Here we need to get pk from prev pointer, and that is why change
// to the PrevPtr so it points to pair of pk and vertex
auto &[pk, vertex] = *prev.vertex;
vertex.delta = nullptr;
if (vertex.deleted) {
deleted_vertices_.push_back(&pk);
Vertex *vertex = prev.vertex;
vertex->delta = nullptr;
if (vertex->deleted) {
deleted_vertices_.push_back(vertex->keys.Keys());
}
break;
}
@@ -1017,13 +1021,15 @@ void Shard::CollectGarbage(const io::Time current_time) {
RemoveObsoleteEntries(&indices_, clean_up_before_timestamp);
}
for (const auto *vertex : deleted_vertices_) {
MG_ASSERT(vertices_.erase(*vertex), "Invalid database state!");
auto vertex_acc = vertices_.access();
for (const auto &vertex : deleted_vertices_) {
MG_ASSERT(vertex_acc.remove(vertex), "Invalid database state!");
}
deleted_vertices_.clear();
{
auto edge_acc = edges_.access();
for (auto edge : deleted_edges_) {
MG_ASSERT(edges_.erase(edge), "Invalid database state!");
MG_ASSERT(edge_acc.remove(edge), "Invalid database state!");
}
}
deleted_edges_.clear();

View File

@@ -31,16 +31,19 @@
#include "storage/v3/indices.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/lexicographically_ordered_vertex.hpp"
#include "storage/v3/mvcc.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/shard_operation_result.hpp"
#include "storage/v3/transaction.hpp"
#include "storage/v3/vertex.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/vertex_id.hpp"
#include "storage/v3/vertices_skip_list.hpp"
#include "storage/v3/view.hpp"
#include "utils/exceptions.hpp"
#include "utils/file_locker.hpp"
@@ -63,7 +66,7 @@ namespace memgraph::storage::v3 {
/// An instance of this will be usually be wrapped inside VerticesIterable for
/// generic, public use.
class AllVerticesIterable final {
VertexContainer *vertices_accessor_;
VerticesSkipList::Accessor vertices_accessor_;
Transaction *transaction_;
View view_;
Indices *indices_;
@@ -74,10 +77,10 @@ class AllVerticesIterable final {
public:
class Iterator final {
AllVerticesIterable *self_;
VertexContainer::iterator it_;
VerticesSkipList::Iterator it_;
public:
Iterator(AllVerticesIterable *self, VertexContainer::iterator it);
Iterator(AllVerticesIterable *self, VerticesSkipList::Iterator it);
VertexAccessor operator*() const;
@@ -88,17 +91,17 @@ class AllVerticesIterable final {
bool operator!=(const Iterator &other) const { return !(*this == other); }
};
AllVerticesIterable(VertexContainer &vertices_accessor, Transaction *transaction, View view, Indices *indices,
Config::Items config, const VertexValidator &vertex_validator)
: vertices_accessor_(&vertices_accessor),
AllVerticesIterable(VerticesSkipList::Accessor vertices_accessor, Transaction *transaction, View view,
Indices *indices, Config::Items config, const VertexValidator &vertex_validator)
: vertices_accessor_(std::move(vertices_accessor)),
transaction_(transaction),
view_(view),
indices_(indices),
config_(config),
vertex_validator_{&vertex_validator} {}
Iterator begin() { return {this, vertices_accessor_->begin()}; }
Iterator end() { return {this, vertices_accessor_->end()}; }
Iterator begin() { return {this, vertices_accessor_.begin()}; }
Iterator end() { return {this, vertices_accessor_.end()}; }
};
/// Generic access to different kinds of vertex iterations.
@@ -204,14 +207,14 @@ class Shard final {
public:
/// @throw std::bad_alloc
ShardResult<VertexAccessor> CreateVertexAndValidate(
ShardOperationResult<VertexAccessor> CreateVertexAndValidate(
const std::vector<LabelId> &labels, const std::vector<PropertyValue> &primary_properties,
const std::vector<std::pair<PropertyId, PropertyValue>> &properties);
std::optional<VertexAccessor> FindVertex(std::vector<PropertyValue> primary_key, View view);
VerticesIterable Vertices(View view) {
return VerticesIterable(AllVerticesIterable(shard_->vertices_, transaction_, view, &shard_->indices_,
return VerticesIterable(AllVerticesIterable(shard_->vertices_.access(), transaction_, view, &shard_->indices_,
shard_->config_.items, shard_->vertex_validator_));
}
@@ -234,17 +237,18 @@ class Shard final {
int64_t ApproximateVertexCount(LabelId label) const {
return shard_->indices_.label_index.ApproximateVertexCount(label);
}
/// Return approximate number of vertices with the given label and property.
/// Note that this is always an over-estimate and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label, PropertyId property) const {
return shard_->indices_.label_property_index.VertexCount(label, property);
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property);
}
/// Return approximate number of vertices with the given label and the given
/// value for the given property.
/// Note that this is always an over-estimate and never an under-estimate.
/// value for the given property. Note that this is always an over-estimate
/// and never an under-estimate.
int64_t ApproximateVertexCount(LabelId label, PropertyId property, const PropertyValue &value) const {
return shard_->indices_.label_property_index.VertexCount(label, property, value);
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, value);
}
/// Return approximate number of vertices with the given label and value for
@@ -253,24 +257,24 @@ class Shard final {
int64_t ApproximateVertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const {
return shard_->indices_.label_property_index.VertexCount(label, property, lower, upper);
return shard_->indices_.label_property_index.ApproximateVertexCount(label, property, lower, upper);
}
/// @return Accessor to the deleted vertex if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
ShardResult<std::optional<VertexAccessor>> DeleteVertex(VertexAccessor *vertex);
Result<std::optional<VertexAccessor>> DeleteVertex(VertexAccessor *vertex);
/// @return Accessor to the deleted vertex and deleted edges if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
ShardResult<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachDeleteVertex(
Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachDeleteVertex(
VertexAccessor *vertex);
/// @throw std::bad_alloc
ShardResult<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid);
Result<EdgeAccessor> CreateEdge(VertexId from_vertex_id, VertexId to_vertex_id, EdgeTypeId edge_type, Gid gid);
/// Accessor to the deleted edge if a deletion took place, std::nullopt otherwise
/// @throw std::bad_alloc
ShardResult<std::optional<EdgeAccessor>> DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id, Gid edge_id);
Result<std::optional<EdgeAccessor>> DeleteEdge(VertexId from_vertex_id, VertexId to_vertex_id, Gid edge_id);
LabelId NameToLabel(std::string_view name) const;
@@ -306,6 +310,9 @@ class Shard final {
void Abort();
private:
/// @throw std::bad_alloc
VertexAccessor CreateVertex(Gid gid, LabelId primary_label);
Shard *shard_;
Transaction *transaction_;
Config::Items config_;
@@ -371,8 +378,8 @@ class Shard final {
// The shard's range is [min, max)
PrimaryKey min_primary_key_;
std::optional<PrimaryKey> max_primary_key_;
VertexContainer vertices_;
EdgeContainer edges_;
VerticesSkipList vertices_;
utils::SkipList<Edge> edges_;
// Even though the edge count is already kept in the `edges_` SkipList, the
// list is used only when properties are enabled for edges. Because of that we
// keep a separate count of edges that is always updated.
@@ -390,7 +397,7 @@ class Shard final {
// Vertices that are logically deleted but still have to be removed from
// indices before removing them from the main storage.
std::list<const PrimaryKey *> deleted_vertices_;
std::list<PrimaryKey> deleted_vertices_;
// Edges that are logically deleted and wait to be removed from the main
// storage.

View File

@@ -190,12 +190,6 @@ class ShardManager {
});
}
void BlockOnQuiescence() {
for (const auto &worker : workers_) {
worker.BlockOnQuiescence();
}
}
private:
io::Io<IoImpl> io_;
std::vector<shard_worker::Queue> workers_;

View File

@@ -0,0 +1,26 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <variant>
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
namespace memgraph::storage::v3 {
using ResultErrorType = std::variant<SchemaViolation, Error>;
template <typename TValue>
using ShardOperationResult = utils::BasicResult<ResultErrorType, TValue>;
} // namespace memgraph::storage::v3

File diff suppressed because it is too large Load Diff

View File

@@ -21,6 +21,9 @@
namespace memgraph::storage::v3 {
template <typename>
constexpr auto kAlwaysFalse = false;
class ShardRsm {
std::unique_ptr<Shard> shard_;

View File

@@ -80,9 +80,6 @@ struct QueueInner {
// starvation by sometimes randomizing priorities, rather than following a strict
// prioritization.
std::deque<Message> queue;
uint64_t submitted = 0;
uint64_t calls_to_pop = 0;
};
/// There are two reasons to implement our own Queue instead of using
@@ -98,8 +95,6 @@ class Queue {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
inner_->submitted++;
inner_->queue.emplace_back(std::forward<Message>(message));
} // lock dropped before notifying condition variable
@@ -110,9 +105,6 @@ class Queue {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
inner_->calls_to_pop++;
inner_->cv.notify_all();
while (inner_->queue.empty()) {
inner_->cv.wait(lock);
}
@@ -122,15 +114,6 @@ class Queue {
return message;
}
void BlockOnQuiescence() const {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
while (inner_->calls_to_pop <= inner_->submitted) {
inner_->cv.wait(lock);
}
}
};
/// A ShardWorker owns Raft<ShardRsm> instances. receives messages from the ShardManager.
@@ -139,6 +122,7 @@ class ShardWorker {
io::Io<IoImpl> io_;
Queue queue_;
std::priority_queue<std::pair<Time, uuid>, std::vector<std::pair<Time, uuid>>, std::greater<>> cron_schedule_;
Time next_cron_ = Time::min();
std::map<uuid, ShardRaft<IoImpl>> rsm_map_;
bool Process(ShutDown && /* shut_down */) { return false; }
@@ -191,7 +175,10 @@ class ShardWorker {
return;
}
auto rsm_io = io_.ForkLocal(to_init.uuid);
auto rsm_io = io_.ForkLocal();
auto io_addr = rsm_io.GetAddress();
io_addr.unique_id = to_init.uuid;
rsm_io.SetAddress(io_addr);
// TODO(tyler) get peers from Coordinator in HeartbeatResponse
std::vector<Address> rsm_peers = {};
@@ -221,12 +208,15 @@ class ShardWorker {
~ShardWorker() = default;
void Run() {
bool should_continue = true;
while (should_continue) {
while (true) {
Message message = queue_.Pop();
should_continue =
const bool should_continue =
std::visit([&](auto &&msg) { return Process(std::forward<decltype(msg)>(msg)); }, std::move(message));
if (!should_continue) {
return;
}
}
}
};

Some files were not shown because too many files have changed in this diff Show More