Compare commits

...

14 Commits

Author SHA1 Message Date
Marko Budiselic
15fee55fa1 Add debugging tools 2023-06-15 10:52:58 +00:00
Marko Budiselić
7fb3f62703 Upgrade to RocksDB 8.1.1 (#1013) 2023-06-15 11:54:24 +02:00
Marko Budiselić
cb4b71bdbd Update pull_request_template.md 2023-06-14 16:04:35 +02:00
andrejtonev
30ec570bb9 Add Bolt v5 support (#938) 2023-06-12 18:55:15 +02:00
Antonio Filipovic
d917c3f0fd Fix slow IN LIST evaluation (#901) 2023-05-29 17:52:20 +02:00
andrejtonev
d842adbed3 Handle user-defined metadata and expose it with SHOW TRANSACTIONS(#945) 2023-05-29 11:40:14 +02:00
Bruno Sačarić
cdfcbc106c Update license date (#941) 2023-05-18 11:42:12 +02:00
Josipmrden
651b6f3a5a Expose system metrics over HTTP Endpoint (#940) 2023-05-18 05:10:57 +00:00
Ante Pušić
0d9bd74a8a Add support for map projection (#892) 2023-05-16 20:05:35 +02:00
andrejtonev
802f8aceda Add data directory status and (un)lock query (#933) 2023-05-16 18:36:04 +02:00
gvolfing
7ddce539fa Add return build type command (#894) 2023-05-16 16:02:03 +02:00
gvolfing
c3e4f81026 Include additional info inside storage mode info query (#883) 2023-05-16 14:25:41 +02:00
Antonio Filipovic
208705f296 Reduce memory consumption on return from python procedures (#932) 2023-05-16 10:33:09 +02:00
Ante Javor
69634a5354 Fix typo in mgbench 2023-05-10 14:02:46 +02:00
129 changed files with 4711 additions and 429 deletions

View File

@@ -1,6 +1,7 @@
---
BasedOnStyle: Google
---
Language: Cpp
BasedOnStyle: Google
Standard: "c++20"
UseTab: Never
DerivePointerAlignment: false

View File

@@ -10,5 +10,5 @@
To keep docs changelog up to date, one more thing to do:
- [ ] Write a release note here
- [ ] Write a release note here, including added/changed clauses
- [ ] Tag someone from docs team in the comments

View File

@@ -3,6 +3,7 @@ repos:
rev: v4.4.0
hooks:
- id: check-yaml
args: [--allow-multiple-documents]
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black

View File

@@ -231,6 +231,8 @@ endif()
message(STATUS "CMake build type: ${CMAKE_BUILD_TYPE}")
# -----------------------------------------------------------------------------
add_definitions( -DCMAKE_BUILD_TYPE_NAME="${CMAKE_BUILD_TYPE}")
if (NOT MG_ARCH)
set(MG_ARCH_DESCR "Host architecture to build Memgraph on. Supported values are x86_64, ARM64.")
if (${CMAKE_HOST_SYSTEM_PROCESSOR} MATCHES "aarch64")

1
libs/.gitignore vendored
View File

@@ -6,3 +6,4 @@
!__main.cpp
!pulsar.patch
!antlr4.10.1.patch
!rocksdb8.1.1.patch

13
libs/rocksdb8.1.1.patch Normal file
View File

@@ -0,0 +1,13 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 598c728..816c705 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1242,7 +1242,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
if(ROCKSDB_BUILD_SHARED)
install(
- TARGETS ${ROCKSDB_SHARED_LIB}
+ TARGETS ${ROCKSDB_SHARED_LIB} OPTIONAL
EXPORT RocksDBTargets
COMPONENT runtime
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"

View File

@@ -192,10 +192,10 @@ cd json
file_get_try_double "${primary_urls[nlohmann]}" "${secondary_urls[nlohmann]}"
cd ..
rocksdb_tag="v6.14.6" # (2020-10-14)
rocksdb_tag="v8.1.1" # (2023-04-21)
repo_clone_try_double "${primary_urls[rocksdb]}" "${secondary_urls[rocksdb]}" "rocksdb" "$rocksdb_tag" true
pushd rocksdb
git apply ../rocksdb.patch
git apply ../rocksdb8.1.1.patch
popd
# mgclient

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2027-05-04
CHANGE DATE: 2027-18-05
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,6 +20,8 @@ inline constexpr uint8_t kPreamble[4] = {0x60, 0x60, 0xB0, 0x17};
enum class Signature : uint8_t {
Noop = 0x00,
Init = 0x01,
LogOn = 0x6A,
LogOff = 0x6B,
AckFailure = 0x0E, // only v1
Reset = 0x0F,
Goodbye = 0x02,

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -28,7 +28,7 @@ inline constexpr size_t kChunkWholeSize = kChunkHeaderSize + kChunkMaxDataSize;
*/
inline constexpr size_t kHandshakeSize = 20;
inline constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403};
inline constexpr uint16_t kSupportedVersions[] = {0x0100, 0x0400, 0x0401, 0x0403, 0x0502};
inline constexpr int kPullAll = -1;
inline constexpr int kPullLast = -1;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,7 +33,14 @@ namespace memgraph::communication::bolt {
template <typename Buffer>
class Decoder {
public:
explicit Decoder(Buffer &buffer) : buffer_(buffer) {}
explicit Decoder(Buffer &buffer) : buffer_(buffer), major_v_(0) {}
/**
* Lets the user update the version.
* This is all single thread for now. TODO: Update if ever multithreaded.
* @param major_v the major version of the Bolt protocol used.
*/
void UpdateVersion(int major_v) { major_v_ = major_v; }
/**
* Reads a Value from the available data in the buffer.
@@ -208,6 +215,10 @@ class Decoder {
protected:
Buffer &buffer_;
int major_v_; //!< Major version of the underlying Bolt protocol
// TODO: when refactoring
// Ideally the major_v would be a compile time constant. If the higher level (Bolt driver) ends up being separate
// classes, this could be just a template and each version of the driver would use the appropriate decoder.
private:
bool ReadNull(const Marker &marker, Value *data) {
@@ -370,11 +381,7 @@ class Decoder {
}
ret.emplace(std::move(dv_key.ValueString()), std::move(dv_val));
}
if (ret.size() != size) {
return false;
}
return true;
return ret.size() == size;
}
bool ReadVertex(Value *data) {
@@ -407,6 +414,14 @@ class Decoder {
}
vertex.properties = std::move(dv.ValueMap());
if (major_v_ > 4) {
// element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
vertex.element_id = std::move(dv.ValueString());
}
return true;
}
@@ -445,6 +460,23 @@ class Decoder {
}
edge.properties = std::move(dv.ValueMap());
if (major_v_ > 4) {
// element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.element_id = std::move(dv.ValueString());
// from_element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.from_element_id = std::move(dv.ValueString());
// to_element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.to_element_id = std::move(dv.ValueString());
}
return true;
}
@@ -471,6 +503,14 @@ class Decoder {
}
edge.properties = std::move(dv.ValueMap());
if (major_v_ > 4) {
// element_id introduced in v5.0
if (!ReadValue(&dv, Value::Type::String)) {
return false;
}
edge.element_id = std::move(dv.ValueString());
}
return true;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -36,7 +36,14 @@ namespace memgraph::communication::bolt {
template <typename Buffer>
class BaseEncoder {
public:
explicit BaseEncoder(Buffer &buffer) : buffer_(buffer) {}
explicit BaseEncoder(Buffer &buffer) : buffer_(buffer), major_v_(0) {}
/**
* Lets the user update the version.
* This is all single thread for now. TODO: Update if ever multithreaded.
* @param major_v the major version of the Bolt protocol used.
*/
void UpdateVersion(int major_v) { major_v_ = major_v; }
void WriteRAW(const uint8_t *data, uint64_t len) { buffer_.Write(data, len); }
@@ -116,7 +123,8 @@ class BaseEncoder {
}
void WriteVertex(const Vertex &vertex) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
int struct_n = 3 + 1 * int(major_v_ > 4); // element_id introduced from v5
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + struct_n);
WriteRAW(utils::UnderlyingCast(Signature::Node));
WriteInt(vertex.id.AsInt());
@@ -132,10 +140,16 @@ class BaseEncoder {
WriteString(prop.first);
WriteValue(prop.second);
}
if (major_v_ > 4) {
// element_id introduced in v5.0
WriteString(vertex.element_id);
}
}
void WriteEdge(const Edge &edge, bool unbound = false) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + (unbound ? 3 : 5));
int struct_n = (unbound ? 3 + 1 * int(major_v_ > 4) : 5 + 3 * int(major_v_ > 4)); // element_id introduced from v5
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + struct_n);
WriteRAW(utils::UnderlyingCast(unbound ? Signature::UnboundRelationship : Signature::Relationship));
WriteInt(edge.id.AsInt());
@@ -152,10 +166,22 @@ class BaseEncoder {
WriteString(prop.first);
WriteValue(prop.second);
}
if (major_v_ > 4) {
// element_id introduced in v5.0
WriteString(edge.element_id);
if (!unbound) {
// from_element_id introduced in v5.0
WriteString(edge.from_element_id);
// to_element_id introduced in v5.0
WriteString(edge.to_element_id);
}
}
}
void WriteEdge(const UnboundedEdge &edge) {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
const int struct_n = 3 + 1 * int(major_v_ > 4); // element_id introduced from v5
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + struct_n);
WriteRAW(utils::UnderlyingCast(Signature::UnboundRelationship));
WriteInt(edge.id.AsInt());
@@ -168,6 +194,11 @@ class BaseEncoder {
WriteString(prop.first);
WriteValue(prop.second);
}
if (major_v_ > 4) {
// element_id introduced in v5.0
WriteString(edge.element_id);
}
}
void WritePath(const Path &path) {
@@ -264,6 +295,7 @@ class BaseEncoder {
protected:
Buffer &buffer_;
int major_v_; //!< Major version of the underlying Bolt protocol (TODO: Think about reimplementing the versioning)
private:
template <class T>

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -41,6 +41,8 @@ class ClientEncoder : private BaseEncoder<Buffer> {
public:
ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
using BaseEncoder<Buffer>::UpdateVersion;
/**
* Writes a Init message.
*

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -34,6 +34,8 @@ class Encoder : private BaseEncoder<Buffer> {
public:
Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
using BaseEncoder<Buffer>::UpdateVersion;
/**
* Sends a Record message.
*

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -63,7 +63,8 @@ class Session {
* if an explicit transaction was started.
*/
virtual std::pair<std::vector<std::string>, std::optional<int>> Interpret(
const std::string &query, const std::map<std::string, Value> &params) = 0;
const std::string &query, const std::map<std::string, Value> &params,
const std::map<std::string, memgraph::communication::bolt::Value> &metadata) = 0;
/**
* Put results of the processed query in the `encoder`.
@@ -85,7 +86,7 @@ class Session {
*/
virtual std::map<std::string, Value> Discard(std::optional<int> n, std::optional<int> qid) = 0;
virtual void BeginTransaction() = 0;
virtual void BeginTransaction(const std::map<std::string, memgraph::communication::bolt::Value> &) = 0;
virtual void CommitTransaction() = 0;
virtual void RollbackTransaction() = 0;
@@ -120,6 +121,9 @@ class Session {
return;
}
handshake_done_ = true;
// Update the decoder's Bolt version (v5 has changed the undelying structure)
decoder_.UpdateVersion(version_.major);
encoder_.UpdateVersion(version_.major);
}
ChunkState chunk_state;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,10 +22,15 @@
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/states/handlers.hpp"
#include "communication/bolt/v1/value.hpp"
#include "utils/event_counter.hpp"
#include "utils/likely.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
namespace memgraph::metrics {
extern const Event BoltMessages;
} // namespace memgraph::metrics
namespace memgraph::communication::bolt {
template <typename TSession>
@@ -86,6 +91,37 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
}
}
template <typename TSession>
State RunHandlerV5(Signature signature, TSession &session, State state, Marker marker) {
switch (signature) {
case Signature::Run:
return HandleRunV5<TSession>(session, state, marker);
case Signature::Pull:
return HandlePullV5<TSession>(session, state, marker);
case Signature::Discard:
return HandleDiscardV5<TSession>(session, state, marker);
case Signature::Reset:
return HandleReset<TSession>(session, marker);
case Signature::Begin:
return HandleBegin<TSession>(session, state, marker);
case Signature::Commit:
return HandleCommit<TSession>(session, state, marker);
case Signature::Goodbye:
return HandleGoodbye<TSession>();
case Signature::Rollback:
return HandleRollback<TSession>(session, state, marker);
case Signature::Noop:
return HandleNoop<TSession>(state);
case Signature::Route:
return HandleRoute<TSession>(session, marker);
case Signature::LogOff:
return HandleLogOff<TSession>();
default:
spdlog::trace("Unrecognized signature received (0x{:02X})!", utils::UnderlyingCast(signature));
return State::Close;
}
}
/**
* Executor state run function
* This function executes an initialized Bolt session.
@@ -103,8 +139,10 @@ State StateExecutingRun(TSession &session, State state) {
switch (session.version_.major) {
case 1:
memgraph::metrics::IncrementCounter(memgraph::metrics::BoltMessages);
return RunHandlerV1(signature, session, state, marker);
case 4: {
memgraph::metrics::IncrementCounter(memgraph::metrics::BoltMessages);
if (session.version_.minor >= 3) {
return RunHandlerV4<TSession, 3>(signature, session, state, marker);
}
@@ -113,6 +151,8 @@ State StateExecutingRun(TSession &session, State state) {
}
return RunHandlerV4<TSession>(signature, session, state, marker);
}
case 5:
return RunHandlerV5<TSession>(signature, session, state, marker);
default:
spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
return State::Close;

View File

@@ -12,6 +12,7 @@
#pragma once
#include <map>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
@@ -22,6 +23,7 @@
#include "communication/bolt/v1/state.hpp"
#include "communication/bolt/v1/value.hpp"
#include "communication/exceptions.hpp"
#include "storage/v2/property_value.hpp"
#include "utils/logging.hpp"
#include "utils/message.hpp"
@@ -71,6 +73,23 @@ inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::ex
"should be in database logs."};
}
namespace helpers {
/** Extracts metadata from the extras field.
* NOTE: In order to avoid a copy, the metadata in moved.
* TODO: Update if extra field is used for anything else.
*/
inline std::map<std::string, Value> ConsumeMetadata(Value &extra) {
std::map<std::string, Value> md;
auto &md_tv = extra.ValueMap()["tx_metadata"];
if (md_tv.IsMap()) {
md = std::move(md_tv.ValueMap());
}
return md;
}
} // namespace helpers
namespace details {
template <bool is_pull, typename TSession>
@@ -209,7 +228,7 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap(), {});
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
@@ -250,6 +269,7 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
// Even though this part seems unnecessary it is needed to move the buffer
if (!session.decoder_.ReadValue(&extra, Value::Type::Map)) {
spdlog::trace("Couldn't read extra field!");
return State::Close;
}
if (state != State::Idle) {
@@ -266,7 +286,8 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
const auto [header, qid] =
session.Interpret(query.ValueString(), params.ValueMap(), helpers::ConsumeMetadata(extra));
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
@@ -288,6 +309,12 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
}
}
template <typename TSession>
State HandleRunV5(TSession &session, const State state, const Marker marker) {
// Using V4 on purpose
return HandleRunV4<TSession>(session, state, marker);
}
template <typename TSession>
State HandlePullV1(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV1<true>(session, state, marker);
@@ -298,6 +325,12 @@ State HandlePullV4(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV4<true>(session, state, marker);
}
template <typename TSession>
State HandlePullV5(TSession &session, const State state, const Marker marker) {
// Using V4 on purpose
return HandlePullV4<TSession>(session, state, marker);
}
template <typename TSession>
State HandleDiscardV1(TSession &session, const State state, const Marker marker) {
return details::HandlePullDiscardV1<false>(session, state, marker);
@@ -308,6 +341,12 @@ State HandleDiscardV4(TSession &session, const State state, const Marker marker)
return details::HandlePullDiscardV4<false>(session, state, marker);
}
template <typename TSession>
State HandleDiscardV5(TSession &session, const State state, const Marker marker) {
// Using V4 on purpose
return HandleDiscardV4<TSession>(session, state, marker);
}
template <typename TSession>
State HandleReset(TSession &session, const Marker marker) {
// IMPORTANT: This implementation of the Bolt RESET command isn't fully
@@ -360,7 +399,7 @@ State HandleBegin(TSession &session, const State state, const Marker marker) {
}
try {
session.BeginTransaction();
session.BeginTransaction(helpers::ConsumeMetadata(extra));
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
@@ -465,4 +504,10 @@ State HandleRoute(TSession &session, const Marker marker) {
}
return State::Error;
}
template <typename TSession>
State HandleLogOff() {
// Not arguments sent, the user just needs to reauthenticate
return State::Init;
}
} // namespace memgraph::communication::bolt

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -27,17 +27,25 @@ namespace details {
template <typename TSession>
std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
// Get authentication data.
// From neo4j driver v4.4, fields that have a default value are not sent.
// In order to have back-compatibility, the missing fields will be added.
auto &data = metadata.ValueMap();
if (!data.count("scheme")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
if (data.empty()) { // Special case auth=None
spdlog::warn("The client didn't supply the authentication scheme! Trying with \"none\"...");
data["scheme"] = "none";
}
std::string username;
std::string password;
if (data["scheme"].ValueString() == "basic") {
if (!data.count("principal") || !data.count("credentials")) {
spdlog::warn("The client didn't supply authentication information!");
return State::Close;
if (!data.count("principal")) { // Special case principal = ""
spdlog::warn("The client didn't supply the principal field! Trying with \"\"...");
data["principal"] = "";
}
if (!data.count("credentials")) { // Special case credentials = ""
spdlog::warn("The client didn't supply the credentials field! Trying with \"\"...");
data["credentials"] = "";
}
username = data["principal"].ValueString();
password = data["credentials"].ValueString();
@@ -106,6 +114,30 @@ std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
return std::nullopt;
}
auto &data = metadata.ValueMap();
if (!data.count("user_agent")) {
spdlog::warn("The client didn't supply the user agent!");
return std::nullopt;
}
spdlog::info("Client connected '{}'", data.at("user_agent").ValueString());
return metadata;
}
template <typename TSession>
std::optional<Value> GetInitDataV5(TSession &session, const Marker marker) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return std::nullopt;
}
Value metadata;
if (!session.decoder_.ReadValue(&metadata, Value::Type::Map)) {
spdlog::trace("Couldn't read metadata!");
return std::nullopt;
}
const auto &data = metadata.ValueMap();
if (!data.count("user_agent")) {
spdlog::warn("The client didn't supply the user agent!");
@@ -117,6 +149,22 @@ std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
return metadata;
}
template <typename TSession>
std::optional<Value> GetAuthDataV5(TSession &session, const Marker marker) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
return std::nullopt;
}
Value metadata;
if (!session.decoder_.ReadValue(&metadata, Value::Type::Map)) {
spdlog::trace("Couldn't read metadata!");
return std::nullopt;
}
return metadata;
}
template <typename TSession>
State SendSuccessMessage(TSession &session) {
// Neo4j's Java driver 4.1.1+ requires connection_id.
@@ -180,6 +228,57 @@ State StateInitRunV4(TSession &session, Marker marker, Signature signature) {
return SendSuccessMessage(session);
}
template <typename TSession>
State StateInitRunV5(TSession &session, Marker marker, Signature signature) {
if (signature == Signature::Noop) [[unlikely]] {
SPDLOG_DEBUG("Received NOOP message");
return State::Init;
}
if (signature == Signature::Init) {
auto maybeMetadata = GetInitDataV5(session, marker);
if (!maybeMetadata) {
return State::Close;
}
if (SendSuccessMessage(session) == State::Close) {
return State::Close;
}
// Stay in Init
return State::Init;
} else if (signature == Signature::LogOn) {
if (marker != Marker::TinyStruct1) [[unlikely]] {
spdlog::trace("Expected TinyStruct1 marker, but received 0x{:02X}!", utils::UnderlyingCast(marker));
spdlog::trace(
"The client sent malformed data, but we are continuing "
"because the official Neo4j Java driver sends malformed "
"data. D'oh!");
return State::Close;
}
auto maybeMetadata = GetAuthDataV5(session, marker);
if (!maybeMetadata) {
return State::Close;
}
auto result = AuthenticateUser(session, *maybeMetadata);
if (result) {
spdlog::trace("Failed to authenticate, closing connection...");
return State::Close;
}
if (SendSuccessMessage(session) == State::Close) {
return State::Close;
}
return State::Idle;
} else [[unlikely]] {
spdlog::trace("Expected Init signature, but received 0x{:02X}!", utils::UnderlyingCast(signature));
return State::Close;
}
}
} // namespace details
/**
@@ -208,6 +307,9 @@ State StateInitRun(TSession &session) {
}
return details::StateInitRunV4<TSession>(session, marker, signature);
}
case 5: {
return details::StateInitRunV5<TSession>(session, marker, signature);
}
}
spdlog::trace("Unsupported bolt version:{}.{})!", session.version_.major, session.version_.minor);
return State::Close;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -342,6 +342,9 @@ std::ostream &operator<<(std::ostream &os, const Vertex &vertex) {
[&](auto &stream, const auto &pair) { stream << pair.first << ": " << pair.second; });
os << "}";
}
if (!vertex.element_id.empty()) {
os << " element_id: " << vertex.element_id;
}
return os << ")";
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -57,6 +57,7 @@ struct Vertex {
Id id;
std::vector<std::string> labels;
std::map<std::string, Value> properties;
std::string element_id;
};
/**
@@ -69,6 +70,9 @@ struct Edge {
Id to;
std::string type;
std::map<std::string, Value> properties;
std::string element_id;
std::string from_element_id;
std::string to_element_id;
};
/**
@@ -79,6 +83,7 @@ struct UnboundedEdge {
Id id;
std::string type;
std::map<std::string, Value> properties;
std::string element_id;
};
/**

View File

@@ -0,0 +1,108 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <list>
#include <memory>
#include <spdlog/spdlog.h>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core.hpp>
#include "communication/context.hpp"
#include "communication/http/session.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
namespace memgraph::communication::http {
template <class TRequestHandler, typename TSessionData>
class Listener final : public std::enable_shared_from_this<Listener<TRequestHandler, TSessionData>> {
using tcp = boost::asio::ip::tcp;
using SessionHandler = Session<TRequestHandler, TSessionData>;
using std::enable_shared_from_this<Listener<TRequestHandler, TSessionData>>::shared_from_this;
public:
Listener(const Listener &) = delete;
Listener(Listener &&) = delete;
Listener &operator=(const Listener &) = delete;
Listener &operator=(Listener &&) = delete;
~Listener() {}
template <typename... Args>
static std::shared_ptr<Listener> Create(Args &&...args) {
return std::shared_ptr<Listener>{new Listener(std::forward<Args>(args)...)};
}
// Start accepting incoming connections
void Run() { DoAccept(); }
tcp::endpoint GetEndpoint() const { return acceptor_.local_endpoint(); }
private:
Listener(boost::asio::io_context &ioc, TSessionData *data, ServerContext *context, tcp::endpoint endpoint)
: ioc_(ioc), data_(data), context_(context), acceptor_(ioc) {
boost::beast::error_code ec;
// Open the acceptor
acceptor_.open(endpoint.protocol(), ec);
if (ec) {
LogError(ec, "open");
return;
}
// Allow address reuse
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
if (ec) {
LogError(ec, "set_option");
return;
}
// Bind to the server address
acceptor_.bind(endpoint, ec);
if (ec) {
LogError(ec, "bind");
return;
}
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
if (ec) {
LogError(ec, "listen");
return;
}
spdlog::info("HTTP server is listening on {}:{}", endpoint.address(), endpoint.port());
}
void DoAccept() {
acceptor_.async_accept(ioc_, [shared_this = shared_from_this()](auto ec, auto socket) {
shared_this->OnAccept(ec, std::move(socket));
});
}
void OnAccept(boost::beast::error_code ec, tcp::socket socket) {
if (ec) {
return LogError(ec, "accept");
}
SessionHandler::Create(std::move(socket), data_, *context_)->Run();
DoAccept();
}
boost::asio::io_context &ioc_;
TSessionData *data_;
ServerContext *context_;
tcp::acceptor acceptor_;
};
} // namespace memgraph::communication::http

View File

@@ -0,0 +1,65 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <thread>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include "communication/http/listener.hpp"
#include "io/network/endpoint.hpp"
namespace memgraph::communication::http {
template <class TRequestHandler, typename TSessionData>
class Server final {
using tcp = boost::asio::ip::tcp;
public:
explicit Server(io::network::Endpoint endpoint, TSessionData *data, ServerContext *context)
: listener_{Listener<TRequestHandler, TSessionData>::Create(
ioc_, data, context, tcp::endpoint{boost::asio::ip::make_address(endpoint.address), endpoint.port})} {}
Server(const Server &) = delete;
Server(Server &&) = delete;
Server &operator=(const Server &) = delete;
Server &operator=(Server &&) = delete;
~Server() {
MG_ASSERT(!background_thread_ || (ioc_.stopped() && !background_thread_->joinable()),
"Server wasn't shutdown properly");
}
void Start() {
MG_ASSERT(!background_thread_, "The server was already started!");
listener_->Run();
background_thread_.emplace([this] { ioc_.run(); });
}
void Shutdown() { ioc_.stop(); }
void AwaitShutdown() {
if (background_thread_ && background_thread_->joinable()) {
background_thread_->join();
}
}
bool IsRunning() const { return background_thread_ && !ioc_.stopped(); }
tcp::endpoint GetEndpoint() const { return listener_->GetEndpoint(); }
private:
boost::asio::io_context ioc_;
std::shared_ptr<Listener<TRequestHandler, TSessionData>> listener_;
std::optional<std::thread> background_thread_;
};
} // namespace memgraph::communication::http

View File

@@ -0,0 +1,193 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <deque>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <variant>
#include <spdlog/spdlog.h>
#include <boost/asio/bind_executor.hpp>
#include <boost/asio/dispatch.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core/buffers_to_string.hpp>
#include <boost/beast/core/stream_traits.hpp>
#include <boost/beast/core/tcp_stream.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/version.hpp>
#include <json/json.hpp>
#include "communication/context.hpp"
#include "utils/logging.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::communication::http {
inline constexpr uint16_t kSSLExpirySeconds = 30;
inline void LogError(boost::beast::error_code ec, const std::string_view what) {
spdlog::warn("HTTP session failed on {}: {}", what, ec.message());
}
template <class TRequestHandler, typename TSessionData>
class Session : public std::enable_shared_from_this<Session<TRequestHandler, TSessionData>> {
using tcp = boost::asio::ip::tcp;
using std::enable_shared_from_this<Session<TRequestHandler, TSessionData>>::shared_from_this;
public:
template <typename... Args>
static std::shared_ptr<Session> Create(Args &&...args) {
return std::shared_ptr<Session>{new Session{std::forward<Args>(args)...}};
}
void Run() {
if (auto *ssl = std::get_if<SSLSocket>(&stream_); ssl != nullptr) {
try {
boost::beast::get_lowest_layer(*ssl).expires_after(std::chrono::seconds(kSSLExpirySeconds));
ssl->handshake(boost::asio::ssl::stream_base::server);
} catch (const boost::system::system_error &e) {
spdlog::warn("Failed on SSL handshake: {}", e.what());
return;
}
}
// run on the strand
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
}
private:
using PlainSocket = boost::beast::tcp_stream;
using SSLSocket = boost::beast::ssl_stream<boost::beast::tcp_stream>;
explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &context)
: stream_(CreateSocket(std::move(socket), context)),
handler_(data),
strand_{boost::asio::make_strand(GetExecutor())} {}
std::variant<PlainSocket, SSLSocket> CreateSocket(tcp::socket &&socket, ServerContext &context) {
if (context.use_ssl()) {
ssl_context_.emplace(context.context_clone());
return Session::SSLSocket{std::move(socket), *ssl_context_};
}
return Session::PlainSocket{std::move(socket)};
}
void OnWrite(boost::beast::error_code ec, size_t bytes_transferred) {
boost::ignore_unused(bytes_transferred);
if (ec) {
close_ = true;
return LogError(ec, "write");
}
if (close_) {
DoClose();
return;
}
res_ = nullptr;
DoRead();
}
void DoRead() {
req_ = {};
ExecuteForStream([this](auto &&stream) {
boost::beast::get_lowest_layer(stream).expires_after(std::chrono::seconds(kSSLExpirySeconds));
boost::beast::http::async_read(
stream, buffer_, req_,
boost::asio::bind_executor(strand_, std::bind_front(&Session::OnRead, shared_from_this())));
});
}
void OnRead(boost::beast::error_code ec, size_t bytes_transferred) {
boost::ignore_unused(bytes_transferred);
if (ec == boost::beast::http::error::end_of_stream) {
DoClose();
return;
}
if (ec) {
return LogError(ec, "read");
}
auto async_write = [this](boost::beast::http::response<boost::beast::http::string_body> msg) {
ExecuteForStream([this, &msg](auto &&stream) {
// The lifetime of the message has to extend
// for the duration of the async operation so
// we use a shared_ptr to manage it.
auto sp = std::make_shared<boost::beast::http::response<boost::beast::http::string_body>>(std::move(msg));
// Store a type-erased version of the shared
// pointer in the class to keep it alive.
res_ = sp;
// Write the response
boost::beast::http::async_write(
stream, *sp, boost::asio::bind_executor(strand_, std::bind_front(&Session::OnWrite, shared_from_this())));
});
};
// handle request
handler_.HandleRequest(std::move(req_), async_write);
}
void DoClose() {
std::visit(utils::Overloaded{[this](SSLSocket &stream) {
boost::beast::get_lowest_layer(stream).expires_after(std::chrono::seconds(30));
// Perform the SSL shutdown
stream.async_shutdown(
boost::beast::bind_front_handler(&Session::OnClose, shared_from_this()));
},
[](PlainSocket &stream) {
boost::beast::error_code ec;
stream.socket().shutdown(tcp::socket::shutdown_send, ec);
}},
stream_);
}
void OnClose(boost::beast::error_code ec) {
if (ec) {
LogError(ec, "close");
}
// At this point the connection is closed gracefully
}
auto GetExecutor() {
return std::visit(utils::Overloaded{[](auto &&stream) { return stream.get_executor(); }}, stream_);
}
template <typename F>
decltype(auto) ExecuteForStream(F &&fn) {
return std::visit(utils::Overloaded{std::forward<F>(fn)}, stream_);
}
std::optional<std::reference_wrapper<boost::asio::ssl::context>> ssl_context_;
std::variant<PlainSocket, SSLSocket> stream_;
boost::beast::flat_buffer buffer_;
TRequestHandler handler_;
boost::beast::http::request<boost::beast::http::string_body> req_;
std::shared_ptr<void> res_;
boost::asio::strand<boost::beast::tcp_stream::executor_type> strand_;
bool close_{false};
};
} // namespace memgraph::communication::http

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,6 +30,7 @@
#include "communication/context.hpp"
#include "communication/v2/pool.hpp"
#include "communication/v2/session.hpp"
#include "utils/message.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -41,11 +41,21 @@
#include <boost/beast/websocket/rfc6455.hpp>
#include <boost/system/detail/error_code.hpp>
#include "communication/buffer.hpp"
#include "communication/context.hpp"
#include "communication/exceptions.hpp"
#include "utils/event_counter.hpp"
#include "utils/logging.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/variant_helpers.hpp"
namespace memgraph::metrics {
extern const Event ActiveSessions;
extern const Event ActiveTCPSessions;
extern const Event ActiveSSLSessions;
extern const Event ActiveWebSocketSessions;
} // namespace memgraph::metrics
namespace memgraph::communication::v2 {
/**
@@ -99,6 +109,8 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
// Start the asynchronous accept operation
template <class Body, class Allocator>
void DoAccept(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> req) {
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveWebSocketSessions);
execution_active_ = true;
// Set suggested timeout settings for the websocket
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(boost::beast::role_type::server));
@@ -213,6 +225,10 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
if (!IsConnected()) {
return;
}
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveSessions);
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveWebSocketSessions);
if (ec) {
return OnError(ec, "close");
}
@@ -259,12 +275,19 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
if (execution_active_) {
return false;
}
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveSessions);
execution_active_ = true;
timeout_timer_.async_wait(boost::asio::bind_executor(strand_, std::bind(&Session::OnTimeout, shared_from_this())));
if (std::holds_alternative<SSLSocket>(socket_)) {
utils::OnScopeExit increment_counter(
[] { memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveSSLSessions); });
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoHandshake(); });
} else {
utils::OnScopeExit increment_counter(
[] { memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveTCPSessions); });
boost::asio::dispatch(strand_, [shared_this = shared_from_this()] { shared_this->DoRead(); });
}
return true;
@@ -450,6 +473,14 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
}
void OnClose(const boost::system::error_code &ec) {
if (ssl_context_.has_value()) {
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveSSLSessions);
} else {
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTCPSessions);
}
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveSessions);
if (ec) {
return OnError(ec);
}
@@ -465,7 +496,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
// The deadline has passed. Stop the session. The other actors will
// terminate as soon as possible.
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
spdlog::info("Shutting down session after {} seconds of inactivity", timeout_seconds_.count());
DoShutdown();
} else {
// Put the actor back to sleep.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -150,7 +150,9 @@ storage::Result<communication::bolt::Vertex> ToBoltVertex(const storage::VertexA
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Vertex{id, labels, properties};
// Introduced in Bolt v5 (for now just send the ID)
const auto element_id = std::to_string(id.AsInt());
return communication::bolt::Vertex{id, labels, properties, element_id};
}
storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccessor &edge, const storage::Storage &db,
@@ -165,7 +167,11 @@ storage::Result<communication::bolt::Edge> ToBoltEdge(const storage::EdgeAccesso
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Edge{id, from, to, type, properties};
// Introduced in Bolt v5 (for now just send the ID)
const auto element_id = std::to_string(id.AsInt());
const auto from_element_id = std::to_string(from.AsInt());
const auto to_element_id = std::to_string(to.AsInt());
return communication::bolt::Edge{id, from, to, type, properties, element_id, from_element_id, to_element_id};
}
storage::Result<communication::bolt::Path> ToBoltPath(const query::Path &path, const storage::Storage &db,

View File

@@ -0,0 +1,4 @@
set(mg_http_handlers_sources)
add_library(mg-http-handlers STATIC ${mg_http_handlers_sources})
target_link_libraries(mg-http-handlers mg-query mg-storage-v2)

View File

@@ -0,0 +1,211 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <tuple>
#include <vector>
#include <spdlog/spdlog.h>
#include <boost/beast/http.hpp>
#include <boost/beast/version.hpp>
#include <json/json.hpp>
#include <utils/event_counter.hpp>
#include <utils/event_gauge.hpp>
#include "storage/v2/storage.hpp"
#include "utils/event_gauge.hpp"
#include "utils/event_histogram.hpp"
namespace memgraph::http {
struct MetricsResponse {
uint64_t vertex_count;
uint64_t edge_count;
double average_degree;
uint64_t memory_usage;
uint64_t disk_usage;
// Storage of all the counter values throughout the system
// e.g. number of active transactions
std::vector<std::tuple<std::string, std::string, uint64_t>> event_counters{};
// Storage of all the current values throughout the system
std::vector<std::tuple<std::string, std::string, uint64_t>> event_gauges{};
// Storage of all the percentile values across the histograms in the system
// e.g. query latency percentiles, snapshot recovery duration percentiles, etc.
std::vector<std::tuple<std::string, std::string, uint64_t>> event_histograms{};
};
template <typename TSessionData>
class MetricsService {
public:
explicit MetricsService(TSessionData *data) : db_(data->db) {}
nlohmann::json GetMetricsJSON() {
auto response = GetMetrics();
return AsJson(response);
}
private:
const storage::Storage *db_;
MetricsResponse GetMetrics() {
auto info = db_->GetInfo();
return MetricsResponse{.vertex_count = info.vertex_count,
.edge_count = info.edge_count,
.average_degree = info.average_degree,
.memory_usage = info.memory_usage,
.disk_usage = info.disk_usage,
.event_counters = GetEventCounters(),
.event_gauges = GetEventGauges(),
.event_histograms = GetEventHistograms()};
}
nlohmann::json AsJson(MetricsResponse response) {
auto metrics_response = nlohmann::json();
const auto *general_type = "General";
metrics_response[general_type]["vertex_count"] = response.vertex_count;
metrics_response[general_type]["edge_count"] = response.edge_count;
metrics_response[general_type]["average_degree"] = response.average_degree;
metrics_response[general_type]["memory_usage"] = response.memory_usage;
metrics_response[general_type]["disk_usage"] = response.disk_usage;
for (const auto &[name, type, value] : response.event_counters) {
metrics_response[type][name] = value;
}
for (const auto &[name, type, value] : response.event_gauges) {
metrics_response[type][name] = value;
}
for (const auto &[name, type, value] : response.event_histograms) {
metrics_response[type][name] = value;
}
return metrics_response;
}
auto GetEventCounters() {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
std::vector<std::tuple<std::string, std::string, uint64_t>> event_counters{};
for (auto i = 0; i < memgraph::metrics::CounterEnd(); i++) {
event_counters.emplace_back(memgraph::metrics::GetCounterName(i), memgraph::metrics::GetCounterType(i),
memgraph::metrics::global_counters[i].load(std::memory_order_acquire));
}
return event_counters;
}
auto GetEventGauges() {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
std::vector<std::tuple<std::string, std::string, uint64_t>> event_gauges{};
for (auto i = 0; i < memgraph::metrics::GaugeEnd(); i++) {
event_gauges.emplace_back(memgraph::metrics::GetGaugeName(i), memgraph::metrics::GetGaugeType(i),
memgraph::metrics::global_gauges[i].load(std::memory_order_acquire));
}
return event_gauges;
}
auto GetEventHistograms() {
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
std::vector<std::tuple<std::string, std::string, uint64_t>> event_histograms{};
for (auto i = 0; i < memgraph::metrics::HistogramEnd(); i++) {
const auto *name = memgraph::metrics::GetHistogramName(i);
auto &histogram = memgraph::metrics::global_histograms[i];
for (auto &[percentile, value] : histogram.YieldPercentiles()) {
auto metric_name = std::string(name) + "_" + std::to_string(percentile) + "p";
event_histograms.emplace_back(metric_name, memgraph::metrics::GetHistogramType(i), value);
}
}
return event_histograms;
}
};
template <typename TSessionData>
class MetricsRequestHandler final {
public:
explicit MetricsRequestHandler(TSessionData *data) : service_(data) {
spdlog::info("Basic request handler started!");
}
MetricsRequestHandler(const MetricsRequestHandler &) = delete;
MetricsRequestHandler(MetricsRequestHandler &&) = delete;
MetricsRequestHandler &operator=(const MetricsRequestHandler &) = delete;
MetricsRequestHandler &operator=(MetricsRequestHandler &&) = delete;
~MetricsRequestHandler() = default;
template <class Body, class Allocator>
// NOLINTNEXTLINE(misc-unused-parameters)
void HandleRequest(boost::beast::http::request<Body, boost::beast::http::basic_fields<Allocator>> &&req,
std::function<void(boost::beast::http::response<boost::beast::http::string_body>)> &&send) {
auto response_json = nlohmann::json();
// Returns a bad request response
auto const bad_request = [&req, &response_json](const auto why) {
response_json["error"] = std::string(why);
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
boost::beast::http::response<boost::beast::http::string_body> res{boost::beast::http::status::bad_request,
req.version()};
res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING);
res.set(boost::beast::http::field::content_type, "application/json");
res.keep_alive(req.keep_alive());
res.body() = response_json.dump();
res.prepare_payload();
return res;
};
// Make sure we can handle the method
if (req.method() != boost::beast::http::verb::get) {
return send(bad_request("Unknown HTTP-method"));
}
// Request path must be absolute and not contain "..".
if (req.target().empty() || req.target()[0] != '/' || req.target().find("..") != boost::beast::string_view::npos) {
return send(bad_request("Illegal request-target"));
}
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
boost::beast::http::string_body::value_type body;
auto service_response = service_.GetMetricsJSON();
body.append(service_response.dump());
// Cache the size since we need it after the move
const auto size = body.size();
// Respond to GET request
// NOLINTNEXTLINE(cppcoreguidelines-init-variables)
boost::beast::http::response<boost::beast::http::string_body> res{
std::piecewise_construct, std::make_tuple(std::move(body)),
std::make_tuple(boost::beast::http::status::ok, req.version())};
res.set(boost::beast::http::field::server, BOOST_BEAST_VERSION_STRING);
res.set(boost::beast::http::field::content_type, "application/json");
res.content_length(size);
res.keep_alive(req.keep_alive());
return send(std::move(res));
}
private:
MetricsService<TSessionData> service_;
};
} // namespace memgraph::http

View File

@@ -36,11 +36,13 @@
#include "auth/models.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/http/server.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "glue/auth_checker.hpp"
#include "glue/auth_handler.hpp"
#include "helpers.hpp"
#include "http_handlers/metrics.hpp"
#include "license/license.hpp"
#include "license/license_sender.hpp"
#include "py/py.hpp"
@@ -113,6 +115,9 @@ DEFINE_string(bolt_address, "0.0.0.0", "IP address on which the Bolt server shou
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(monitoring_address, "0.0.0.0",
"IP address on which the websocket server for Memgraph monitoring should listen.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(metrics_address, "0.0.0.0",
"IP address on which the Memgraph server for exposing metrics should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687, "Port on which the Bolt server should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
@@ -120,6 +125,9 @@ DEFINE_VALIDATED_int32(monitoring_port, 7444,
"Port on which the websocket server for Memgraph monitoring should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(metrics_port, 9091, "Port on which the Memgraph server for exposing metrics should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(bolt_num_workers, std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers used by the Bolt server. By default, this will be the "
"number of processing units available on the machine.",
@@ -492,6 +500,10 @@ void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, std::string c
}
}
namespace memgraph::metrics {
extern const Event ActiveBoltSessions;
} // namespace memgraph::metrics
class BoltSession final : public memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream> {
public:
@@ -509,26 +521,41 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
#endif
endpoint_(endpoint),
run_id_(data->run_id) {
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions);
interpreter_context_->interpreters.WithLock([this](auto &interpreters) { interpreters.insert(&interpreter_); });
}
~BoltSession() override {
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveBoltSessions);
interpreter_context_->interpreters.WithLock([this](auto &interpreters) { interpreters.erase(&interpreter_); });
}
using memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream>::TEncoder;
void BeginTransaction() override { interpreter_.BeginTransaction(); }
void BeginTransaction(const std::map<std::string, memgraph::communication::bolt::Value> &metadata) override {
std::map<std::string, memgraph::storage::PropertyValue> metadata_pv;
for (const auto &[key, bolt_value] : metadata) {
metadata_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_value));
}
interpreter_.BeginTransaction(metadata_pv);
}
void CommitTransaction() override { interpreter_.CommitTransaction(); }
void RollbackTransaction() override { interpreter_.RollbackTransaction(); }
std::pair<std::vector<std::string>, std::optional<int>> Interpret(
const std::string &query, const std::map<std::string, memgraph::communication::bolt::Value> &params) override {
const std::string &query, const std::map<std::string, memgraph::communication::bolt::Value> &params,
const std::map<std::string, memgraph::communication::bolt::Value> &metadata) override {
std::map<std::string, memgraph::storage::PropertyValue> params_pv;
for (const auto &kv : params) params_pv.emplace(kv.first, memgraph::glue::ToPropertyValue(kv.second));
std::map<std::string, memgraph::storage::PropertyValue> metadata_pv;
for (const auto &[key, bolt_param] : params) {
params_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_param));
}
for (const auto &[key, bolt_md] : metadata) {
metadata_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_md));
}
const std::string *username{nullptr};
if (user_) {
username = &user_->username();
@@ -540,7 +567,7 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
}
#endif
try {
auto result = interpreter_.Prepare(query, params_pv, username);
auto result = interpreter_.Prepare(query, params_pv, username, metadata_pv);
if (user_ && !memgraph::glue::AuthChecker::IsUserAuthorized(*user_, result.privileges)) {
interpreter_.Abort();
throw memgraph::communication::bolt::ClientError(
@@ -672,6 +699,8 @@ class BoltSession final : public memgraph::communication::bolt::Session<memgraph
};
using ServerT = memgraph::communication::v2::Server<BoltSession, SessionData>;
using MonitoringServerT =
memgraph::communication::http::Server<memgraph::http::MetricsRequestHandler<SessionData>, SessionData>;
using memgraph::communication::ServerContext;
// Needed to correctly handle memgraph destruction from a signal handler.
@@ -981,8 +1010,9 @@ int main(int argc, char **argv) {
});
telemetry->AddCollector("event_counters", []() -> nlohmann::json {
nlohmann::json ret;
for (size_t i = 0; i < EventCounter::End(); ++i) {
ret[EventCounter::GetName(i)] = EventCounter::global_counters[i].load(std::memory_order_relaxed);
for (size_t i = 0; i < memgraph::metrics::CounterEnd(); ++i) {
ret[memgraph::metrics::GetCounterName(i)] =
memgraph::metrics::global_counters[i].load(std::memory_order_relaxed);
}
return ret;
});
@@ -998,6 +1028,43 @@ int main(int argc, char **argv) {
{FLAGS_monitoring_address, static_cast<uint16_t>(FLAGS_monitoring_port)}, &context, websocket_auth};
AddLoggerSink(websocket_server.GetLoggingSink());
MonitoringServerT metrics_server{
{FLAGS_metrics_address, static_cast<uint16_t>(FLAGS_metrics_port)}, &session_data, &context};
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
// Handler for regular termination signals
auto shutdown = [&metrics_server, &websocket_server, &server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
// a race condition when a transaction is accepted during server shutdown.
server.Shutdown();
// After the server is notified to stop accepting and processing
// connections we tell the execution engine to stop processing all pending
// queries.
memgraph::query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
metrics_server.Shutdown();
};
InitSignalHandlers(shutdown);
} else {
// Handler for regular termination signals
auto shutdown = [&websocket_server, &server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
// a race condition when a transaction is accepted during server shutdown.
server.Shutdown();
// After the server is notified to stop accepting and processing
// connections we tell the execution engine to stop processing all pending
// queries.
memgraph::query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
};
InitSignalHandlers(shutdown);
}
#else
// Handler for regular termination signals
auto shutdown = [&websocket_server, &server, &interpreter_context] {
// Server needs to be shutdown first and then the database. This prevents
@@ -1007,14 +1074,22 @@ int main(int argc, char **argv) {
// connections we tell the execution engine to stop processing all pending
// queries.
memgraph::query::Shutdown(&interpreter_context);
websocket_server.Shutdown();
};
InitSignalHandlers(shutdown);
#endif
MG_ASSERT(server.Start(), "Couldn't start the Bolt server!");
websocket_server.Start();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
metrics_server.Start();
}
#endif
if (!FLAGS_init_data_file.empty()) {
spdlog::info("Running init data file.");
#ifdef MG_ENTERPRISE
@@ -1028,6 +1103,11 @@ int main(int argc, char **argv) {
server.AwaitShutdown();
websocket_server.AwaitShutdown();
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
metrics_server.AwaitShutdown();
}
#endif
memgraph::query::procedure::gModuleRegistry.UnloadAllModules();

View File

@@ -22,6 +22,8 @@
#include "query/trigger.hpp"
#include "utils/async_timer.hpp"
#include "query/frame_change.hpp"
namespace memgraph::query {
enum class TransactionStatus {
@@ -82,6 +84,7 @@ struct ExecutionContext {
plan::ProfilingStats *stats_root{nullptr};
ExecutionStats execution_stats;
TriggerContextCollector *trigger_context_collector{nullptr};
FrameChangeCollector *frame_change_collector{nullptr};
utils::AsyncTimer timer;
#ifdef MG_ENTERPRISE
std::unique_ptr<FineGrainedAuthChecker> auth_checker{nullptr};

122
src/query/frame_change.hpp Normal file
View File

@@ -0,0 +1,122 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/typed_value.hpp"
#include "utils/memory.hpp"
#include "utils/pmr/unordered_map.hpp"
#include "utils/pmr/vector.hpp"
namespace memgraph::query {
// Key is hash output, value is vector of unique elements
using CachedType = utils::pmr::unordered_map<size_t, std::vector<TypedValue>>;
struct CachedValue {
// Cached value, this can be probably templateized
CachedType cache_;
explicit CachedValue(utils::MemoryResource *mem) : cache_(mem) {}
CachedValue(CachedType &&cache, memgraph::utils::MemoryResource *memory) : cache_(std::move(cache), memory) {}
CachedValue(const CachedValue &other, memgraph::utils::MemoryResource *memory) : cache_(other.cache_, memory) {}
CachedValue(CachedValue &&other, memgraph::utils::MemoryResource *memory) : cache_(std::move(other.cache_), memory) {}
CachedValue(CachedValue &&other) noexcept = delete;
/// Copy construction without memgraph::utils::MemoryResource is not allowed.
CachedValue(const CachedValue &) = delete;
CachedValue &operator=(const CachedValue &) = delete;
CachedValue &operator=(CachedValue &&) = delete;
~CachedValue() = default;
memgraph::utils::MemoryResource *GetMemoryResource() const noexcept {
return cache_.get_allocator().GetMemoryResource();
}
// Func to check if cache_ contains value
bool CacheValue(const TypedValue &value) {
if (!value.IsList()) {
return false;
}
const auto &list = value.ValueList();
TypedValue::Hash hash{};
for (const TypedValue &element : list) {
const auto key = hash(element);
auto &vector_values = cache_[key];
if (!IsValueInVec(vector_values, element)) {
vector_values.push_back(element);
}
}
return true;
}
// Func to cache_value inside cache_
bool ContainsValue(const TypedValue &value) const {
TypedValue::Hash hash{};
const auto key = hash(value);
if (cache_.contains(key)) {
return IsValueInVec(cache_.at(key), value);
}
return false;
}
private:
bool IsValueInVec(const std::vector<TypedValue> &vec_values, const TypedValue &value) const {
return std::any_of(vec_values.begin(), vec_values.end(), [&value](auto &vec_value) {
const auto is_value_equal = vec_value == value;
if (is_value_equal.IsNull()) return false;
return is_value_equal.ValueBool();
});
}
};
// Class tracks keys for which user can cache values which help with faster search or faster retrieval
// in the future.
class FrameChangeCollector {
public:
explicit FrameChangeCollector(utils::MemoryResource *mem) : tracked_values_(mem){};
// Add tracking key to cache later value
CachedValue &AddTrackingKey(const std::string &key) {
const auto &[it, _] = tracked_values_.emplace(key, tracked_values_.get_allocator().GetMemoryResource());
return it->second;
}
// Is key tracked
bool IsKeyTracked(const std::string &key) const { return tracked_values_.contains(key); }
// Is value for given key cached
bool IsKeyValueCached(const std::string &key) const {
return tracked_values_.contains(key) && !tracked_values_.at(key).cache_.empty();
}
// Reset value for tracking key
bool ResetTrackingValue(const std::string &key) {
if (tracked_values_.contains(key)) {
tracked_values_.erase(key);
AddTrackingKey(key);
}
return true;
}
// Get value cached for tracking key, throws if key is not in tracked
CachedValue &GetCachedValue(const std::string &key) { return tracked_values_.at(key); }
// Checks for keys tracked
bool IsTrackingValues() const { return !tracked_values_.empty(); }
private:
// Key is output of utils::GetFrameChangeId, value is utils::pmr::unordered_map
memgraph::utils::pmr::unordered_map<std::string, CachedValue> tracked_values_;
};
} // namespace memgraph::query

View File

@@ -114,12 +114,18 @@ constexpr utils::TypeInfo query::ListLiteral::kType{utils::TypeId::AST_LIST_LITE
constexpr utils::TypeInfo query::MapLiteral::kType{utils::TypeId::AST_MAP_LITERAL, "MapLiteral",
&query::BaseLiteral::kType};
constexpr utils::TypeInfo query::MapProjectionLiteral::kType{utils::TypeId::AST_MAP_PROJECTION_LITERAL,
"MapProjectionLiteral", &query::BaseLiteral::kType};
constexpr utils::TypeInfo query::Identifier::kType{utils::TypeId::AST_IDENTIFIER, "Identifier",
&query::Expression::kType};
constexpr utils::TypeInfo query::PropertyLookup::kType{utils::TypeId::AST_PROPERTY_LOOKUP, "PropertyLookup",
&query::Expression::kType};
constexpr utils::TypeInfo query::AllPropertiesLookup::kType{utils::TypeId::AST_ALL_PROPERTIES_LOOKUP,
"AllPropertiesLookup", &query::Expression::kType};
constexpr utils::TypeInfo query::LabelsTest::kType{utils::TypeId::AST_LABELS_TEST, "LabelsTest",
&query::Expression::kType};

View File

@@ -1063,8 +1063,9 @@ class MapLiteral : public memgraph::query::BaseLiteral {
DEFVISITABLE(ExpressionVisitor<void>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
for (auto pair : elements_)
for (auto pair : elements_) {
if (!pair.second->Accept(visitor)) break;
}
}
return visitor.PostVisit(*this);
}
@@ -1087,6 +1088,60 @@ class MapLiteral : public memgraph::query::BaseLiteral {
friend class AstStorage;
};
struct MapProjectionData {
Expression *map_variable;
std::unordered_map<PropertyIx, Expression *> elements;
};
class MapProjectionLiteral : public memgraph::query::BaseLiteral {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
MapProjectionLiteral() = default;
DEFVISITABLE(ExpressionVisitor<TypedValue>);
DEFVISITABLE(ExpressionVisitor<TypedValue *>);
DEFVISITABLE(ExpressionVisitor<void>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
for (auto pair : elements_) {
if (!pair.second) continue;
if (!pair.second->Accept(visitor)) break;
}
}
return visitor.PostVisit(*this);
}
Expression *map_variable_;
std::unordered_map<PropertyIx, Expression *> elements_;
MapProjectionLiteral *Clone(AstStorage *storage) const override {
MapProjectionLiteral *object = storage->Create<MapProjectionLiteral>();
object->map_variable_ = map_variable_;
for (const auto &entry : elements_) {
auto key = storage->GetPropertyIx(entry.first.name);
if (!entry.second) {
object->elements_[key] = nullptr;
continue;
}
object->elements_[key] = entry.second->Clone(storage);
}
return object;
}
protected:
explicit MapProjectionLiteral(Expression *map_variable, std::unordered_map<PropertyIx, Expression *> &&elements)
: map_variable_(map_variable), elements_(std::move(elements)) {}
private:
friend class AstStorage;
};
class Identifier : public memgraph::query::Expression {
public:
static const utils::TypeInfo kType;
@@ -1158,6 +1213,38 @@ class PropertyLookup : public memgraph::query::Expression {
friend class AstStorage;
};
class AllPropertiesLookup : public memgraph::query::Expression {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
AllPropertiesLookup() = default;
DEFVISITABLE(ExpressionVisitor<TypedValue>);
DEFVISITABLE(ExpressionVisitor<TypedValue *>);
DEFVISITABLE(ExpressionVisitor<void>);
bool Accept(HierarchicalTreeVisitor &visitor) override {
if (visitor.PreVisit(*this)) {
expression_->Accept(visitor);
}
return visitor.PostVisit(*this);
}
memgraph::query::Expression *expression_{nullptr};
AllPropertiesLookup *Clone(AstStorage *storage) const override {
AllPropertiesLookup *object = storage->Create<AllPropertiesLookup>();
object->expression_ = expression_ ? expression_->Clone(storage) : nullptr;
return object;
}
protected:
explicit AllPropertiesLookup(Expression *expression) : expression_(expression) {}
private:
friend class AstStorage;
};
class LabelsTest : public memgraph::query::Expression {
public:
static const utils::TypeInfo kType;
@@ -2786,7 +2873,7 @@ class InfoQuery : public memgraph::query::Query {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class InfoType { STORAGE, INDEX, CONSTRAINT };
enum class InfoType { STORAGE, INDEX, CONSTRAINT, BUILD };
DEFVISITABLE(QueryVisitor<void>);
@@ -2898,7 +2985,7 @@ class LockPathQuery : public memgraph::query::Query {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
enum class Action { LOCK_PATH, UNLOCK_PATH };
enum class Action { LOCK_PATH, UNLOCK_PATH, STATUS };
LockPathQuery() = default;

View File

@@ -22,6 +22,7 @@ class CypherUnion;
class NamedExpression;
class Identifier;
class PropertyLookup;
class AllPropertiesLookup;
class LabelsTest;
class Aggregation;
class Function;
@@ -44,6 +45,7 @@ class EdgeAtom;
class PrimitiveLiteral;
class ListLiteral;
class MapLiteral;
class MapProjectionLiteral;
class OrOperator;
class XorOperator;
class AndOperator;
@@ -106,9 +108,10 @@ using TreeCompositeVisitor = utils::CompositeVisitor<
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral, MapLiteral,
PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None, CallProcedure,
Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete, Where, SetProperty, SetProperties, SetLabels,
RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv, Foreach, Exists, CallSubquery, CypherQuery>;
MapProjectionLiteral, PropertyLookup, AllPropertiesLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce,
Extract, All, Single, Any, None, CallProcedure, Create, Match, Return, With, Pattern, NodeAtom, EdgeAtom, Delete,
Where, SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels, Merge, Unwind, RegexMatch, LoadCsv,
Foreach, Exists, CallSubquery, CypherQuery>;
using TreeLeafVisitor = utils::LeafVisitor<Identifier, PrimitiveLiteral, ParameterLookup>;
@@ -122,13 +125,14 @@ class HierarchicalTreeVisitor : public TreeCompositeVisitor, public TreeLeafVisi
template <class TResult>
class ExpressionVisitor
: public utils::Visitor<
TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
SubtractionOperator, MultiplicationOperator, DivisionOperator, ModOperator, NotEqualOperator, EqualOperator,
LessOperator, GreaterOperator, LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator, ListLiteral,
MapLiteral, PropertyLookup, LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any,
None, ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch, Exists> {};
: public utils::Visitor<TResult, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator,
AdditionOperator, SubtractionOperator, MultiplicationOperator, DivisionOperator,
ModOperator, NotEqualOperator, EqualOperator, LessOperator, GreaterOperator,
LessEqualOperator, GreaterEqualOperator, InListOperator, SubscriptOperator,
ListSlicingOperator, IfOperator, UnaryPlusOperator, UnaryMinusOperator, IsNullOperator,
ListLiteral, MapLiteral, MapProjectionLiteral, PropertyLookup, AllPropertiesLookup,
LabelsTest, Aggregation, Function, Reduce, Coalesce, Extract, All, Single, Any, None,
ParameterLookup, Identifier, PrimitiveLiteral, RegexMatch, Exists> {};
template <class TResult>
class QueryVisitor

View File

@@ -124,6 +124,9 @@ antlrcpp::Any CypherMainVisitor::visitInfoQuery(MemgraphCypher::InfoQueryContext
} else if (ctx->constraintInfo()) {
info_query->info_type_ = InfoQuery::InfoType::CONSTRAINT;
return info_query;
} else if (ctx->buildInfo()) {
info_query->info_type_ = InfoQuery::InfoType::BUILD;
return info_query;
} else {
throw utils::NotYetImplemented("Info query: '{}'", ctx->getText());
}
@@ -325,7 +328,9 @@ antlrcpp::Any CypherMainVisitor::visitShowReplicas(MemgraphCypher::ShowReplicasC
antlrcpp::Any CypherMainVisitor::visitLockPathQuery(MemgraphCypher::LockPathQueryContext *ctx) {
auto *lock_query = storage_->Create<LockPathQuery>();
if (ctx->LOCK()) {
if (ctx->STATUS()) {
lock_query->action_ = LockPathQuery::Action::STATUS;
} else if (ctx->LOCK()) {
lock_query->action_ = LockPathQuery::Action::LOCK_PATH;
} else if (ctx->UNLOCK()) {
lock_query->action_ = LockPathQuery::Action::UNLOCK_PATH;
@@ -1696,6 +1701,38 @@ antlrcpp::Any CypherMainVisitor::visitMapLiteral(MemgraphCypher::MapLiteralConte
return map;
}
antlrcpp::Any CypherMainVisitor::visitMapProjectionLiteral(MemgraphCypher::MapProjectionLiteralContext *ctx) {
MapProjectionData map_projection_data;
map_projection_data.map_variable =
storage_->Create<Identifier>(std::any_cast<std::string>(ctx->variable()->accept(this)));
for (auto *map_el : ctx->mapElement()) {
if (map_el->propertyLookup()) {
auto key = std::any_cast<PropertyIx>(map_el->propertyLookup()->propertyKeyName()->accept(this));
auto property = std::any_cast<PropertyIx>(map_el->propertyLookup()->accept(this));
auto *property_lookup = storage_->Create<PropertyLookup>(map_projection_data.map_variable, property);
map_projection_data.elements.insert_or_assign(key, property_lookup);
}
if (map_el->allPropertiesLookup()) {
auto key = AddProperty("*");
auto *all_properties_lookup = storage_->Create<AllPropertiesLookup>(map_projection_data.map_variable);
map_projection_data.elements.insert_or_assign(key, all_properties_lookup);
}
if (map_el->variable()) {
auto key = AddProperty(std::any_cast<std::string>(map_el->variable()->accept(this)));
auto *variable = storage_->Create<Identifier>(std::any_cast<std::string>(map_el->variable()->accept(this)));
map_projection_data.elements.insert_or_assign(key, variable);
}
if (map_el->propertyKeyValuePair()) {
auto key = std::any_cast<PropertyIx>(map_el->propertyKeyValuePair()->propertyKeyName()->accept(this));
auto *value = std::any_cast<Expression *>(map_el->propertyKeyValuePair()->expression()->accept(this));
map_projection_data.elements.insert_or_assign(key, value);
}
}
return map_projection_data;
}
antlrcpp::Any CypherMainVisitor::visitListLiteral(MemgraphCypher::ListLiteralContext *ctx) {
std::vector<Expression *> expressions;
for (auto *expr_ctx : ctx->expression()) {
@@ -2276,6 +2313,10 @@ antlrcpp::Any CypherMainVisitor::visitLiteral(MemgraphCypher::LiteralContext *ct
} else if (ctx->listLiteral()) {
return static_cast<Expression *>(
storage_->Create<ListLiteral>(std::any_cast<std::vector<Expression *>>(ctx->listLiteral()->accept(this))));
} else if (ctx->mapProjectionLiteral()) {
auto map_projection_data = std::any_cast<MapProjectionData>(ctx->mapProjectionLiteral()->accept(this));
return static_cast<Expression *>(storage_->Create<MapProjectionLiteral>(map_projection_data.map_variable,
std::move(map_projection_data.elements)));
} else {
return static_cast<Expression *>(storage_->Create<MapLiteral>(
std::any_cast<std::unordered_map<PropertyIx, Expression *>>(ctx->mapLiteral()->accept(this))));

View File

@@ -15,8 +15,6 @@
#include <unordered_set>
#include <utility>
#include <antlr4-runtime.h>
#include "query/frontend/ast/ast.hpp"
#include "query/frontend/opencypher/generated/MemgraphCypherBaseVisitor.h"
#include "utils/exceptions.hpp"
@@ -608,6 +606,11 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitMapLiteral(MemgraphCypher::MapLiteralContext *ctx) override;
/**
* @return MapProjectionData
*/
antlrcpp::Any visitMapProjectionLiteral(MemgraphCypher::MapProjectionLiteralContext *ctx) override;
/**
* @return vector<Expression*>
*/

View File

@@ -54,6 +54,7 @@ class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
void Visit(IfOperator &op) override;
void Visit(ListLiteral &op) override;
void Visit(MapLiteral &op) override;
void Visit(MapProjectionLiteral &op) override;
void Visit(LabelsTest &op) override;
void Visit(Aggregation &op) override;
void Visit(Function &op) override;
@@ -68,6 +69,7 @@ class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
void Visit(Identifier &op) override;
void Visit(PrimitiveLiteral &op) override;
void Visit(PropertyLookup &op) override;
void Visit(AllPropertiesLookup &op) override;
void Visit(ParameterLookup &op) override;
void Visit(NamedExpression &op) override;
void Visit(RegexMatch &op) override;
@@ -89,6 +91,8 @@ void PrintObject(std::ostream *out, Aggregation::Op op);
void PrintObject(std::ostream *out, Expression *expr);
void PrintObject(std::ostream *out, AllPropertiesLookup *apl);
void PrintObject(std::ostream *out, Identifier *expr);
void PrintObject(std::ostream *out, const storage::PropertyValue &value);
@@ -122,6 +126,15 @@ void PrintObject(std::ostream *out, Expression *expr) {
}
}
void PrintObject(std::ostream *out, AllPropertiesLookup *apl) {
if (apl) {
ExpressionPrettyPrinter printer{out};
*out << ".*";
} else {
*out << "<null>";
}
}
void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
void PrintObject(std::ostream *out, const storage::PropertyValue &value) {
@@ -249,6 +262,17 @@ void ExpressionPrettyPrinter::Visit(MapLiteral &op) {
PrintObject(out_, map);
}
void ExpressionPrettyPrinter::Visit(MapProjectionLiteral &op) {
std::map<std::string, Expression *> map_projection_elements;
for (const auto &kv : op.elements_) {
map_projection_elements[kv.first.name] = kv.second;
}
PrintObject(out_, op.map_variable_);
PrintObject(out_, map_projection_elements);
}
void ExpressionPrettyPrinter::Visit(AllPropertiesLookup &op) { PrintObject(out_, &op); }
void ExpressionPrettyPrinter::Visit(LabelsTest &op) { PrintOperator(out_, "LabelsTest", op.expression_); }
void ExpressionPrettyPrinter::Visit(Aggregation &op) { PrintOperator(out_, "Aggregation", op.op_); }

View File

@@ -46,7 +46,9 @@ indexInfo : INDEX INFO ;
constraintInfo : CONSTRAINT INFO ;
infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo ) ;
buildInfo : BUILD INFO ;
infoQuery : SHOW ( storageInfo | indexInfo | constraintInfo | buildInfo) ;
explainQuery : EXPLAIN cypherQuery ;
@@ -248,6 +250,7 @@ literal : numberLiteral
| booleanLiteral
| CYPHERNULL
| mapLiteral
| mapProjectionLiteral
| listLiteral
;
@@ -290,6 +293,8 @@ patternComprehension : '[' ( variable '=' )? relationshipsPattern ( WHERE expres
propertyLookup : '.' ( propertyKeyName ) ;
allPropertiesLookup : '.' '*' ;
caseExpression : ( ( CASE ( caseAlternatives )+ ) | ( CASE test=expression ( caseAlternatives )+ ) ) ( ELSE else_expression=expression )? END ;
caseAlternatives : WHEN when_expression=expression THEN then_expression=expression ;
@@ -302,12 +307,22 @@ numberLiteral : doubleLiteral
mapLiteral : '{' ( propertyKeyName ':' expression ( ',' propertyKeyName ':' expression )* )? '}' ;
mapProjectionLiteral : variable '{' ( mapElement ( ',' mapElement )* )? '}' ;
mapElement : propertyLookup
| allPropertiesLookup
| variable
| propertyKeyValuePair
;
parameter : '$' ( symbolicName | DecimalLiteral ) ;
propertyExpression : atom ( propertyLookup )+ ;
propertyKeyName : symbolicName ;
propertyKeyValuePair : propertyKeyName ':' expression ;
integerLiteral : DecimalLiteral
| OctalLiteral
| HexadecimalLiteral

View File

@@ -31,6 +31,7 @@ memgraphCypherKeyword : cypherKeyword
| BATCH_SIZE
| BEFORE
| BOOTSTRAP_SERVERS
| BUILD
| CHECK
| CLEAR
| COMMIT
@@ -90,6 +91,7 @@ memgraphCypherKeyword : cypherKeyword
| SNAPSHOT
| START
| STATS
| STATUS
| STORAGE
| STREAM
| STREAMS
@@ -334,7 +336,7 @@ dropReplica : DROP REPLICA replicaName ;
showReplicas : SHOW REPLICAS ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY ;
lockPathQuery : ( LOCK | UNLOCK ) DATA DIRECTORY | DATA DIRECTORY LOCK STATUS;
freeMemoryQuery : FREE MEMORY ;

View File

@@ -35,6 +35,7 @@ BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ;
BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ;
BATCH_SIZE : B A T C H UNDERSCORE S I Z E ;
BEFORE : B E F O R E ;
BUILD : B U I L D ;
BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ;
CALL : C A L L ;
CHECK : C H E C K ;
@@ -106,6 +107,7 @@ SNAPSHOT : S N A P S H O T ;
START : S T A R T ;
STATISTICS : S T A T I S T I C S ;
STATS : S T A T S ;
STATUS : S T A T U S ;
STOP : S T O P ;
STORAGE : S T O R A G E;
STORAGE_MODE : S T O R A G E UNDERSCORE MODE;

View File

@@ -43,6 +43,7 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
AddPrivilege(AuthQuery::Privilege::INDEX);
break;
case InfoQuery::InfoType::STORAGE:
case InfoQuery::InfoType::BUILD:
AddPrivilege(AuthQuery::Privilege::STATS);
break;
case InfoQuery::InfoType::CONSTRAINT:

View File

@@ -145,6 +145,7 @@ const trie::Trie kKeywords = {"union",
"drop",
"show",
"stats",
"status",
"unique",
"explain",
"profile",
@@ -211,7 +212,12 @@ const trie::Trie kKeywords = {"union",
"edge_types",
"off",
"in_memory_transactional",
"in_memory_analytical"};
"in_memory_analytical",
"data",
"directory",
"lock",
"unlock"
"build"};
// Unicode codepoints that are allowed at the start of the unescaped name.
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

View File

@@ -13,10 +13,12 @@
#pragma once
#include <algorithm>
#include <cstddef>
#include <limits>
#include <map>
#include <optional>
#include <regex>
#include <string>
#include <vector>
#include "query/common.hpp"
@@ -27,7 +29,11 @@
#include "query/frontend/semantic/symbol_table.hpp"
#include "query/interpret/frame.hpp"
#include "query/typed_value.hpp"
#include "spdlog/spdlog.h"
#include "utils/exceptions.hpp"
#include "utils/frame_change_id.hpp"
#include "utils/logging.hpp"
#include "utils/pmr/unordered_map.hpp"
namespace memgraph::query {
@@ -73,11 +79,13 @@ class ReferenceExpressionEvaluator : public ExpressionVisitor<TypedValue *> {
UNSUCCESSFUL_VISIT(ListSlicingOperator);
UNSUCCESSFUL_VISIT(IsNullOperator);
UNSUCCESSFUL_VISIT(PropertyLookup);
UNSUCCESSFUL_VISIT(AllPropertiesLookup);
UNSUCCESSFUL_VISIT(LabelsTest);
UNSUCCESSFUL_VISIT(PrimitiveLiteral);
UNSUCCESSFUL_VISIT(ListLiteral);
UNSUCCESSFUL_VISIT(MapLiteral);
UNSUCCESSFUL_VISIT(MapProjectionLiteral);
UNSUCCESSFUL_VISIT(Aggregation);
UNSUCCESSFUL_VISIT(Coalesce);
UNSUCCESSFUL_VISIT(Function);
@@ -100,8 +108,13 @@ class ReferenceExpressionEvaluator : public ExpressionVisitor<TypedValue *> {
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
public:
ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
storage::View view)
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
storage::View view, FrameChangeCollector *frame_change_collector = nullptr)
: frame_(frame),
symbol_table_(&symbol_table),
ctx_(&ctx),
dba_(dba),
view_(view),
frame_change_collector_(frame_change_collector) {}
using ExpressionVisitor<TypedValue>::Visit;
@@ -190,25 +203,78 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(InListOperator &in_list) override {
TypedValue *_list_ptr = nullptr;
TypedValue _list;
auto literal = in_list.expression1_->Accept(*this);
auto _list = in_list.expression2_->Accept(*this);
if (_list.IsNull()) {
return TypedValue(ctx_->memory);
auto get_list_literal = [this, &in_list, &_list, &_list_ptr]() -> void {
ReferenceExpressionEvaluator reference_expression_evaluator{frame_, symbol_table_, ctx_};
_list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
if (nullptr == _list_ptr) {
_list = in_list.expression2_->Accept(*this);
_list_ptr = &_list;
}
};
auto do_list_literal_checks = [this, &literal, &_list_ptr]() -> std::optional<TypedValue> {
MG_ASSERT(_list_ptr, "List literal should have been defined");
if (_list_ptr->IsNull()) {
return TypedValue(ctx_->memory);
}
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list_ptr->type() != TypedValue::Type::List) {
throw QueryRuntimeException("IN expected a list, got {}.", _list_ptr->type());
}
const auto &list = _list_ptr->ValueList();
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
return {};
};
const auto cached_id = memgraph::utils::GetFrameChangeId(in_list);
const auto do_cache{frame_change_collector_ != nullptr && cached_id &&
frame_change_collector_->IsKeyTracked(*cached_id)};
if (do_cache) {
if (!frame_change_collector_->IsKeyValueCached(*cached_id)) {
// Check only first time if everything is okay, later when we use
// cache there is no need to check again as we did check first time
get_list_literal();
auto preoperational_checks = do_list_literal_checks();
if (preoperational_checks) {
return std::move(*preoperational_checks);
}
auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
cached_value.CacheValue(*_list_ptr);
spdlog::trace("Value cached {}", *cached_id);
}
const auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
if (cached_value.ContainsValue(literal)) {
return TypedValue(true, ctx_->memory);
}
// has null
if (cached_value.ContainsValue(TypedValue(ctx_->memory))) {
return TypedValue(ctx_->memory);
}
return TypedValue(false, ctx_->memory);
}
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list.type() != TypedValue::Type::List) {
throw QueryRuntimeException("IN expected a list, got {}.", _list.type());
// When caching is not an option, we need to evaluate list literal every time
// and do the checks
get_list_literal();
auto preoperational_checks = do_list_literal_checks();
if (preoperational_checks) {
return std::move(*preoperational_checks);
}
const auto &list = _list.ValueList();
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
spdlog::trace("Not using cache on IN LIST operator");
auto has_null = false;
for (const auto &element : list) {
auto result = literal == element;
@@ -466,7 +532,101 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
throw QueryRuntimeException("Invalid property name {} for Graph", prop_name);
}
default:
throw QueryRuntimeException("Only nodes, edges, maps and temporal types have properties to be looked-up.");
throw QueryRuntimeException(
"Only nodes, edges, maps, temporal types and graphs have properties to be looked up.");
}
}
TypedValue Visit(AllPropertiesLookup &all_properties_lookup) override {
TypedValue::TMap result(ctx_->memory);
auto expression_result = all_properties_lookup.expression_->Accept(*this);
switch (expression_result.type()) {
case TypedValue::Type::Null:
return TypedValue(ctx_->memory);
case TypedValue::Type::Vertex: {
for (const auto properties = *expression_result.ValueVertex().Properties(view_);
const auto &[property_id, value] : properties) {
result.emplace(dba_->PropertyToName(property_id), value);
}
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Edge: {
for (const auto properties = *expression_result.ValueEdge().Properties(view_);
const auto &[property_id, value] : properties) {
result.emplace(dba_->PropertyToName(property_id), value);
}
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Map: {
for (auto &[name, value] : expression_result.ValueMap()) {
result.emplace(name, value);
}
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Duration: {
const auto &dur = expression_result.ValueDuration();
result.emplace("day", TypedValue(dur.Days(), ctx_->memory));
result.emplace("hour", TypedValue(dur.SubDaysAsHours(), ctx_->memory));
result.emplace("minute", TypedValue(dur.SubDaysAsMinutes(), ctx_->memory));
result.emplace("second", TypedValue(dur.SubDaysAsSeconds(), ctx_->memory));
result.emplace("millisecond", TypedValue(dur.SubDaysAsMilliseconds(), ctx_->memory));
result.emplace("microseconds", TypedValue(dur.SubDaysAsMicroseconds(), ctx_->memory));
result.emplace("nanoseconds", TypedValue(dur.SubDaysAsNanoseconds(), ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Date: {
const auto &date = expression_result.ValueDate();
result.emplace("year", TypedValue(date.year, ctx_->memory));
result.emplace("month", TypedValue(date.month, ctx_->memory));
result.emplace("day", TypedValue(date.day, ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::LocalTime: {
const auto &lt = expression_result.ValueLocalTime();
result.emplace("hour", TypedValue(lt.hour, ctx_->memory));
result.emplace("minute", TypedValue(lt.minute, ctx_->memory));
result.emplace("second", TypedValue(lt.second, ctx_->memory));
result.emplace("millisecond", TypedValue(lt.millisecond, ctx_->memory));
result.emplace("microsecond", TypedValue(lt.microsecond, ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::LocalDateTime: {
const auto &ldt = expression_result.ValueLocalDateTime();
const auto &date = ldt.date;
const auto &lt = ldt.local_time;
result.emplace("year", TypedValue(date.year, ctx_->memory));
result.emplace("month", TypedValue(date.month, ctx_->memory));
result.emplace("day", TypedValue(date.day, ctx_->memory));
result.emplace("hour", TypedValue(lt.hour, ctx_->memory));
result.emplace("minute", TypedValue(lt.minute, ctx_->memory));
result.emplace("second", TypedValue(lt.second, ctx_->memory));
result.emplace("millisecond", TypedValue(lt.millisecond, ctx_->memory));
result.emplace("microsecond", TypedValue(lt.microsecond, ctx_->memory));
return TypedValue(result, ctx_->memory);
}
case TypedValue::Type::Graph: {
const auto &graph = expression_result.ValueGraph();
utils::pmr::vector<TypedValue> vertices(ctx_->memory);
vertices.reserve(graph.vertices().size());
for (const auto &v : graph.vertices()) {
vertices.emplace_back(TypedValue(v, ctx_->memory));
}
result.emplace("nodes", TypedValue(std::move(vertices), ctx_->memory));
utils::pmr::vector<TypedValue> edges(ctx_->memory);
edges.reserve(graph.edges().size());
for (const auto &e : graph.edges()) {
edges.emplace_back(TypedValue(e, ctx_->memory));
}
result.emplace("edges", TypedValue(std::move(edges), ctx_->memory));
return TypedValue(result, ctx_->memory);
}
default:
throw QueryRuntimeException(
"Only nodes, edges, maps, temporal types and graphs have properties to be looked up.");
}
}
@@ -531,6 +691,30 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(MapProjectionLiteral &literal) override {
constexpr std::string_view kAllPropertiesSelector{"*"};
TypedValue::TMap result(ctx_->memory);
TypedValue::TMap all_properties_lookup(ctx_->memory);
for (const auto &[property_key, property_value] : literal.elements_) {
if (property_key.name == kAllPropertiesSelector.data()) {
auto maybe_all_properties_lookup = property_value->Accept(*this);
if (maybe_all_properties_lookup.type() != TypedValue::Type::Map) {
throw QueryRuntimeException("Expected a map from AllPropertiesLookup, got {}.",
maybe_all_properties_lookup.type());
}
all_properties_lookup = std::move(maybe_all_properties_lookup.ValueMap());
continue;
}
result.emplace(property_key.name, property_value->Accept(*this));
}
if (!all_properties_lookup.empty()) result.merge(all_properties_lookup);
return TypedValue(result, ctx_->memory);
}
TypedValue Visit(Aggregation &aggregation) override {
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
}
@@ -852,7 +1036,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
DbAccessor *dba_;
// which switching approach should be used when evaluating
storage::View view_;
};
FrameChangeCollector *frame_change_collector_;
}; // namespace memgraph::query
/// A helper function for evaluating an expression that's an int.
///

View File

@@ -15,6 +15,7 @@
#include <algorithm>
#include <atomic>
#include <chrono>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <functional>
@@ -44,19 +45,26 @@
#include "query/frontend/semantic/required_privileges.hpp"
#include "query/frontend/semantic/symbol_generator.hpp"
#include "query/interpret/eval.hpp"
#include "query/interpret/frame.hpp"
#include "query/metadata.hpp"
#include "query/plan/planner.hpp"
#include "query/plan/profile.hpp"
#include "query/plan/vertex_count_cache.hpp"
#include "query/stream.hpp"
#include "query/stream/common.hpp"
#include "query/trigger.hpp"
#include "query/typed_value.hpp"
#include "spdlog/spdlog.h"
#include "storage/v2/edge.hpp"
#include "storage/v2/id_types.hpp"
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/property_value.hpp"
#include "storage/v2/storage_mode.hpp"
#include "utils/algorithm.hpp"
#include "utils/build_info.hpp"
#include "utils/csv_parsing.hpp"
#include "utils/event_counter.hpp"
#include "utils/event_histogram.hpp"
#include "utils/exceptions.hpp"
#include "utils/flag_validation.hpp"
#include "utils/likely.hpp"
@@ -71,17 +79,20 @@
#include "utils/typeinfo.hpp"
#include "utils/variant_helpers.hpp"
namespace EventCounter {
namespace memgraph::metrics {
extern Event ReadQuery;
extern Event WriteQuery;
extern Event ReadWriteQuery;
extern const Event LabelIndexCreated;
extern const Event LabelPropertyIndexCreated;
extern const Event StreamsCreated;
extern const Event TriggersCreated;
} // namespace EventCounter
extern const Event QueryExecutionLatency_us;
extern const Event CommitedTransactions;
extern const Event RollbackedTransactions;
extern const Event ActiveTransactions;
} // namespace memgraph::metrics
namespace memgraph::query {
@@ -92,19 +103,29 @@ namespace {
void UpdateTypeCount(const plan::ReadWriteTypeChecker::RWType type) {
switch (type) {
case plan::ReadWriteTypeChecker::RWType::R:
EventCounter::IncrementCounter(EventCounter::ReadQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::ReadQuery);
break;
case plan::ReadWriteTypeChecker::RWType::W:
EventCounter::IncrementCounter(EventCounter::WriteQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::WriteQuery);
break;
case plan::ReadWriteTypeChecker::RWType::RW:
EventCounter::IncrementCounter(EventCounter::ReadWriteQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::ReadWriteQuery);
break;
default:
break;
}
}
template <typename T>
concept HasEmpty = requires(T t) {
{ t.empty() } -> std::convertible_to<bool>;
};
template <typename T>
inline std::optional<T> GenOptional(const T &in) {
return in.empty() ? std::nullopt : std::make_optional<T>(in);
}
struct Callback {
std::vector<std::string> header;
using CallbackFunction = std::function<std::vector<std::vector<TypedValue>>()>;
@@ -663,6 +684,8 @@ Callback::CallbackFunction GetKafkaCreateCallback(StreamQuery *stream_query, Exp
return config_map;
};
memgraph::metrics::IncrementCounter(memgraph::metrics::StreamsCreated);
return [interpreter_context, stream_name = stream_query->stream_name_,
topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
consumer_group = std::move(consumer_group), common_stream_info = std::move(common_stream_info),
@@ -693,6 +716,8 @@ Callback::CallbackFunction GetPulsarCreateCallback(StreamQuery *stream_query, Ex
throw SemanticException("Service URL must not be an empty string!");
}
auto common_stream_info = GetCommonStreamInfo(stream_query, evaluator);
memgraph::metrics::IncrementCounter(memgraph::metrics::StreamsCreated);
return [interpreter_context, stream_name = stream_query->stream_name_,
topic_names = EvaluateTopicNames(evaluator, stream_query->topic_names_),
common_stream_info = std::move(common_stream_info), service_url = std::move(service_url),
@@ -723,7 +748,6 @@ Callback HandleStreamQuery(StreamQuery *stream_query, const Parameters &paramete
Callback callback;
switch (stream_query->action_) {
case StreamQuery::Action::CREATE_STREAM: {
EventCounter::IncrementCounter(EventCounter::StreamsCreated);
switch (stream_query->type_) {
case StreamQuery::Type::KAFKA:
callback.fn = GetKafkaCreateCallback(stream_query, evaluator, interpreter_context, username);
@@ -983,7 +1007,8 @@ struct PullPlan {
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
TriggerContextCollector *trigger_context_collector = nullptr,
std::optional<size_t> memory_limit = {}, bool use_monotonic_memory = true);
std::optional<size_t> memory_limit = {}, bool use_monotonic_memory = true,
FrameChangeCollector *frame_change_collector_ = nullptr);
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
@@ -1021,7 +1046,7 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
TriggerContextCollector *trigger_context_collector, const std::optional<size_t> memory_limit,
bool use_monotonic_memory)
bool use_monotonic_memory, FrameChangeCollector *frame_change_collector)
: plan_(plan),
cursor_(plan->plan().MakeCursor(execution_memory)),
frame_(plan->symbol_table().max_position(), execution_memory),
@@ -1052,6 +1077,7 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
ctx_.transaction_status = transaction_status;
ctx_.is_profile_query = is_profile_query;
ctx_.trigger_context_collector = trigger_context_collector;
ctx_.frame_change_collector = frame_change_collector;
}
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
@@ -1134,7 +1160,11 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
if (has_unsent_results_) {
return std::nullopt;
}
summary->insert_or_assign("plan_execution_time", execution_time_.count());
memgraph::metrics::Measure(memgraph::metrics::QueryExecutionLatency_us,
std::chrono::duration_cast<std::chrono::microseconds>(execution_time_).count());
// We are finished with pulling all the data, therefore we can send any
// metadata about the results i.e. notifications and statistics
const bool is_any_counter_set =
@@ -1163,16 +1193,23 @@ Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
}
PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) {
PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper,
const std::map<std::string, storage::PropertyValue> &metadata) {
std::function<void()> handler;
if (query_upper == "BEGIN") {
handler = [this] {
// TODO: Evaluate doing move(metadata). Currently the metadata is very small, but this will be important if it ever
// becomes large.
handler = [this, metadata] {
if (in_explicit_transaction_) {
throw ExplicitTransactionUsageException("Nested transactions are not supported.");
}
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveTransactions);
in_explicit_transaction_ = true;
expect_rollback_ = false;
metadata_ = GenOptional(metadata);
db_accessor_ =
std::make_unique<storage::Storage::Accessor>(interpreter_context_->db->Access(GetIsolationLevelOverride()));
@@ -1203,15 +1240,20 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
expect_rollback_ = false;
in_explicit_transaction_ = false;
metadata_ = std::nullopt;
};
} else if (query_upper == "ROLLBACK") {
handler = [this] {
if (!in_explicit_transaction_) {
throw ExplicitTransactionUsageException("No current transaction to rollback.");
}
memgraph::metrics::IncrementCounter(memgraph::metrics::RollbackedTransactions);
Abort();
expect_rollback_ = false;
in_explicit_transaction_ = false;
metadata_ = std::nullopt;
};
} else {
LOG_FATAL("Should not get here -- unknown transaction query!");
@@ -1226,11 +1268,28 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
RWType::NONE};
}
inline static void TryCaching(const AstStorage &ast_storage, FrameChangeCollector *frame_change_collector) {
if (!frame_change_collector) return;
for (const auto &tree : ast_storage.storage_) {
if (tree->GetTypeInfo() != memgraph::query::InListOperator::kType) {
continue;
}
auto *in_list_operator = utils::Downcast<InListOperator>(tree.get());
const auto cached_id = memgraph::utils::GetFrameChangeId(*in_list_operator);
if (!cached_id || cached_id->empty()) {
continue;
}
frame_change_collector->AddTrackingKey(*cached_id);
spdlog::trace("Tracking {} operator, by id: {}", InListOperator::kType.name, *cached_id);
}
}
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
const std::string *username, std::atomic<TransactionStatus> *transaction_status,
TriggerContextCollector *trigger_context_collector = nullptr) {
TriggerContextCollector *trigger_context_collector = nullptr,
FrameChangeCollector *frame_change_collector = nullptr) {
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
Frame frame(0);
@@ -1261,6 +1320,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
parsed_query.parameters,
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
TryCaching(plan->ast_storage(), frame_change_collector);
summary->insert_or_assign("cost_estimate", plan->cost());
auto rw_type_checker = plan::ReadWriteTypeChecker();
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(plan->plan()));
@@ -1277,9 +1337,10 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
header.push_back(
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
}
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, StringPointerToOptional(username), transaction_status,
trigger_context_collector, memory_limit, use_monotonic_memory);
auto pull_plan = std::make_shared<PullPlan>(
plan, parsed_query.parameters, false, dba, interpreter_context, execution_memory,
StringPointerToOptional(username), transaction_status, trigger_context_collector, memory_limit,
use_monotonic_memory, frame_change_collector->IsTrackingValues() ? frame_change_collector : nullptr);
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
@@ -1340,7 +1401,8 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory, const std::string *username,
std::atomic<TransactionStatus> *transaction_status) {
std::atomic<TransactionStatus> *transaction_status,
FrameChangeCollector *frame_change_collector) {
const std::string kProfileQueryStart = "profile ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -1399,39 +1461,42 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
TryCaching(cypher_query_plan->ast_storage(), frame_change_collector);
auto rw_type_checker = plan::ReadWriteTypeChecker();
auto optional_username = StringPointerToOptional(username);
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
summary, dba, interpreter_context, execution_memory, memory_limit, optional_username,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
pull_plan = std::shared_ptr<PullPlanVector>(nullptr), transaction_status, use_monotonic_memory](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
// No output symbols are given so that nothing is streamed.
if (!stats_and_total_time) {
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
execution_memory, optional_username, transaction_status,
nullptr, memory_limit, use_monotonic_memory)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
}
return PreparedQuery{
{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters), summary, dba,
interpreter_context, execution_memory, memory_limit, optional_username,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
pull_plan = std::shared_ptr<PullPlanVector>(nullptr), transaction_status, use_monotonic_memory,
frame_change_collector](AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
// No output symbols are given so that nothing is streamed.
if (!stats_and_total_time) {
stats_and_total_time =
PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory, optional_username,
transaction_status, nullptr, memory_limit, use_monotonic_memory,
frame_change_collector->IsTrackingValues() ? frame_change_collector : nullptr)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
}
MG_ASSERT(stats_and_total_time, "Failed to execute the query!");
MG_ASSERT(stats_and_total_time, "Failed to execute the query!");
if (pull_plan->Pull(stream, n)) {
summary->insert_or_assign("profile", ProfilingStatsToJson(*stats_and_total_time).dump());
return QueryHandlerResult::ABORT;
}
if (pull_plan->Pull(stream, n)) {
summary->insert_or_assign("profile", ProfilingStatsToJson(*stats_and_total_time).dump());
return QueryHandlerResult::ABORT;
}
return std::nullopt;
},
rw_type_checker.type};
return std::nullopt;
},
rw_type_checker.type};
}
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
@@ -1635,7 +1700,6 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
[&index_notification, &label_name, &properties_stringified]<typename T>(T &&) {
using ErrorType = std::remove_cvref_t<T>;
if constexpr (std::is_same_v<ErrorType, storage::ReplicationError>) {
EventCounter::IncrementCounter(EventCounter::LabelIndexCreated);
throw ReplicationException(
fmt::format("At least one SYNC replica has not confirmed the creation of the index on label {} "
"on properties {}.",
@@ -1649,8 +1713,6 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
}
},
error);
} else {
EventCounter::IncrementCounter(EventCounter::LabelIndexCreated);
}
};
break;
@@ -1777,25 +1839,49 @@ PreparedQuery PrepareLockPathQuery(ParsedQuery parsed_query, bool in_explicit_tr
auto *lock_path_query = utils::Downcast<LockPathQuery>(parsed_query.query);
return PreparedQuery{{},
std::move(parsed_query.required_privileges),
[interpreter_context, action = lock_path_query->action_](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
switch (action) {
case LockPathQuery::Action::LOCK_PATH:
if (!interpreter_context->db->LockPath()) {
throw QueryRuntimeException("Failed to lock the data directory");
}
break;
case LockPathQuery::Action::UNLOCK_PATH:
if (!interpreter_context->db->UnlockPath()) {
throw QueryRuntimeException("Failed to unlock the data directory");
}
break;
}
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
return PreparedQuery{
{"STATUS"},
std::move(parsed_query.required_privileges),
[interpreter_context, action = lock_path_query->action_](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
std::vector<std::vector<TypedValue>> status;
std::string res;
switch (action) {
case LockPathQuery::Action::LOCK_PATH: {
const auto lock_success = interpreter_context->db->LockPath();
if (lock_success.HasError()) [[unlikely]] {
throw QueryRuntimeException("Failed to lock the data directory");
}
res = lock_success.GetValue() ? "Data directory is now locked." : "Data directory is already locked.";
break;
}
case LockPathQuery::Action::UNLOCK_PATH: {
const auto unlock_success = interpreter_context->db->UnlockPath();
if (unlock_success.HasError()) [[unlikely]] {
throw QueryRuntimeException("Failed to unlock the data directory");
}
res = unlock_success.GetValue() ? "Data directory is now unlocked." : "Data directory is already unlocked.";
break;
}
case LockPathQuery::Action::STATUS: {
const auto locked_status = interpreter_context->db->IsPathLocked();
if (locked_status.HasError()) [[unlikely]] {
throw QueryRuntimeException("Failed to access the data directory");
}
res = locked_status.GetValue() ? "Data directory is locked." : "Data directory is unlocked.";
break;
}
}
status.emplace_back(std::vector<TypedValue>{TypedValue(res)});
auto pull_plan = std::make_shared<PullPlanVector>(std::move(status));
if (pull_plan->Pull(stream, n)) {
return QueryHandlerResult::COMMIT;
}
return std::nullopt;
},
RWType::NONE};
}
PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
@@ -1883,6 +1969,7 @@ Callback CreateTrigger(TriggerQuery *trigger_query,
std::move(trigger_name), trigger_statement, user_parameters, ToTriggerEventType(event_type),
before_commit ? TriggerPhase::BEFORE_COMMIT : TriggerPhase::AFTER_COMMIT, &interpreter_context->ast_cache,
dba, interpreter_context->config.query, std::move(owner), interpreter_context->auth_checker);
memgraph::metrics::IncrementCounter(memgraph::metrics::TriggersCreated);
return {};
}};
}
@@ -1937,7 +2024,6 @@ PreparedQuery PrepareTriggerQuery(ParsedQuery parsed_query, bool in_explicit_tra
case TriggerQuery::Action::CREATE_TRIGGER:
trigger_notification.emplace(SeverityLevel::INFO, NotificationCode::CREATE_TRIGGER,
fmt::format("Created trigger {}.", trigger_query->trigger_name_));
EventCounter::IncrementCounter(EventCounter::TriggersCreated);
return CreateTrigger(trigger_query, user_parameters, interpreter_context, dba, std::move(owner));
case TriggerQuery::Action::DROP_TRIGGER:
trigger_notification.emplace(SeverityLevel::INFO, NotificationCode::DROP_TRIGGER,
@@ -2171,6 +2257,14 @@ std::vector<std::vector<TypedValue>> TransactionQueueQueryHandler::ShowTransacti
const auto &typed_queries = interpreter->GetQueries();
results.push_back({TypedValue(interpreter->username_.value_or("")),
TypedValue(std::to_string(transaction_id.value())), TypedValue(typed_queries)});
// Handle user-defined metadata
std::map<std::string, TypedValue> metadata_tv;
if (interpreter->metadata_) {
for (const auto &md : *(interpreter->metadata_)) {
metadata_tv.emplace(md.first, TypedValue(md.second));
}
}
results.back().push_back(TypedValue(metadata_tv));
}
}
return results;
@@ -2240,7 +2334,7 @@ Callback HandleTransactionQueueQuery(TransactionQueueQuery *transaction_query,
Callback callback;
switch (transaction_query->action_) {
case TransactionQueueQuery::Action::SHOW_TRANSACTIONS: {
callback.header = {"username", "transaction_id", "query"};
callback.header = {"username", "transaction_id", "query", "metadata"};
callback.fn = [handler = TransactionQueueQueryHandler(), interpreter_context, username,
hasTransactionManagementPrivilege]() mutable {
std::vector<std::vector<TypedValue>> results;
@@ -2318,8 +2412,10 @@ PreparedQuery PrepareVersionQuery(ParsedQuery parsed_query, bool in_explicit_tra
}
PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
storage::Storage *db, utils::MemoryResource *execution_memory) {
std::map<std::string, TypedValue> * /*summary*/, InterpreterContext *interpreter_context,
storage::Storage *db, utils::MemoryResource * /*execution_memory*/,
std::optional<storage::IsolationLevel> interpreter_isolation_level,
std::optional<storage::IsolationLevel> next_transaction_isolation_level) {
if (in_explicit_transaction) {
throw InfoInMulticommandTxException();
}
@@ -2331,7 +2427,8 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
switch (info_query->info_type_) {
case InfoQuery::InfoType::STORAGE:
header = {"storage info", "value"};
handler = [db] {
handler = [db, interpreter_isolation_level, next_transaction_isolation_level] {
auto info = db->GetInfo();
std::vector<std::vector<TypedValue>> results{
{TypedValue("vertex_count"), TypedValue(static_cast<int64_t>(info.vertex_count))},
@@ -2340,8 +2437,12 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
{TypedValue("memory_usage"), TypedValue(static_cast<int64_t>(info.memory_usage))},
{TypedValue("disk_usage"), TypedValue(static_cast<int64_t>(info.disk_usage))},
{TypedValue("memory_allocated"), TypedValue(static_cast<int64_t>(utils::total_memory_tracker.Amount()))},
{TypedValue("allocation_limit"),
TypedValue(static_cast<int64_t>(utils::total_memory_tracker.HardLimit()))}};
{TypedValue("allocation_limit"), TypedValue(static_cast<int64_t>(utils::total_memory_tracker.HardLimit()))},
{TypedValue("global_isolation_level"), TypedValue(IsolationLevelToString(db->GetIsolationLevel()))},
{TypedValue("session_isolation_level"), TypedValue(IsolationLevelToString(interpreter_isolation_level))},
{TypedValue("next_session_isolation_level"),
TypedValue(IsolationLevelToString(next_transaction_isolation_level))},
{TypedValue("storage_mode"), TypedValue(StorageModeToString(db->GetStorageMode()))}};
return std::pair{results, QueryHandlerResult::COMMIT};
};
break;
@@ -2385,6 +2486,15 @@ PreparedQuery PrepareInfoQuery(ParsedQuery parsed_query, bool in_explicit_transa
return std::pair{results, QueryHandlerResult::NOTHING};
};
break;
case InfoQuery::InfoType::BUILD:
header = {"build info", "value"};
handler = [] {
std::vector<std::vector<TypedValue>> results{
{TypedValue("build_type"), TypedValue(utils::GetBuildInfo().build_name)}};
return std::pair{results, QueryHandlerResult::NOTHING};
};
break;
}
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
@@ -2660,8 +2770,8 @@ std::optional<uint64_t> Interpreter::GetTransactionId() const {
return {};
}
void Interpreter::BeginTransaction() {
const auto prepared_query = PrepareTransactionQuery("BEGIN");
void Interpreter::BeginTransaction(const std::map<std::string, storage::PropertyValue> &metadata) {
const auto prepared_query = PrepareTransactionQuery("BEGIN", metadata);
prepared_query.query_handler(nullptr, {});
}
@@ -2681,10 +2791,13 @@ void Interpreter::RollbackTransaction() {
Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
const std::map<std::string, storage::PropertyValue> &params,
const std::string *username) {
const std::string *username,
const std::map<std::string, storage::PropertyValue> &metadata) {
if (!in_explicit_transaction_) {
query_executions_.clear();
transaction_queries_->clear();
// Handle user-defined metadata in auto-transactions
metadata_ = GenOptional(metadata);
}
// This will be done in the handle transaction query. Our handler can save username and then send it to the kill and
@@ -2704,7 +2817,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
std::optional<int> qid =
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query, metadata));
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
}
@@ -2715,14 +2828,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
// an explicit transaction block.
if (in_explicit_transaction_) {
AdvanceCommand();
}
// If we're not in an explicit transaction block and we have an open
// transaction, abort it since we're about to prepare a new query.
else if (db_accessor_) {
} else if (db_accessor_) {
// If we're not in an explicit transaction block and we have an open
// transaction, abort it since we're about to prepare a new query.
query_executions_.emplace_back(
std::make_unique<QueryExecution>(utils::MonotonicBufferResource(kExecutionMemoryBlockSize)));
AbortCommand(&query_executions_.back());
}
std::unique_ptr<QueryExecution> *query_execution_ptr = nullptr;
try {
query_executions_.emplace_back(
@@ -2770,6 +2883,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
utils::Downcast<ProfileQuery>(parsed_query.query) || utils::Downcast<DumpQuery>(parsed_query.query) ||
utils::Downcast<TriggerQuery>(parsed_query.query) || utils::Downcast<AnalyzeGraphQuery>(parsed_query.query) ||
utils::Downcast<TransactionQueueQuery>(parsed_query.query))) {
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveTransactions);
db_accessor_ =
std::make_unique<storage::Storage::Accessor>(interpreter_context_->db->Access(GetIsolationLevelOverride()));
execution_db_accessor_.emplace(db_accessor_.get());
@@ -2785,18 +2899,21 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
utils::MemoryResource *memory_resource =
std::visit([](auto &execution_memory) -> utils::MemoryResource * { return &execution_memory; },
query_execution->execution_memory);
frame_change_collector_.reset();
frame_change_collector_.emplace(memory_resource);
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
prepared_query =
PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, memory_resource, &query_execution->notifications, username,
&transaction_status_, trigger_context_collector_ ? &*trigger_context_collector_ : nullptr);
prepared_query = PrepareCypherQuery(
std::move(parsed_query), &query_execution->summary, interpreter_context_, &*execution_db_accessor_,
memory_resource, &query_execution->notifications, username, &transaction_status_,
trigger_context_collector_ ? &*trigger_context_collector_ : nullptr, &*frame_change_collector_);
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory_with_exception);
} else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
prepared_query = PrepareProfileQuery(
std::move(parsed_query), in_explicit_transaction_, &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory_with_exception, username, &transaction_status_);
prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, username,
&transaction_status_, &*frame_change_collector_);
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
memory_resource);
@@ -2813,7 +2930,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, interpreter_context_->db,
&query_execution->execution_memory_with_exception);
&query_execution->execution_memory_with_exception, interpreter_isolation_level,
next_transaction_isolation_level);
} else if (utils::Downcast<ConstraintQuery>(parsed_query.query)) {
prepared_query = PrepareConstraintQuery(std::move(parsed_query), in_explicit_transaction_,
&query_execution->notifications, interpreter_context_);
@@ -2872,7 +2990,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
} catch (const utils::BasicException &) {
EventCounter::IncrementCounter(EventCounter::FailedQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::FailedQuery);
AbortCommand(query_execution_ptr);
throw;
}
@@ -2903,11 +3021,17 @@ void Interpreter::Abort() {
expect_rollback_ = false;
in_explicit_transaction_ = false;
metadata_ = std::nullopt;
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTransactions);
if (!db_accessor_) return;
db_accessor_->Abort();
execution_db_accessor_.reset();
db_accessor_.reset();
trigger_context_collector_.reset();
frame_change_collector_.reset();
}
namespace {
@@ -2999,12 +3123,21 @@ void Interpreter::Commit() {
utils::OnScopeExit clean_status(
[this]() { transaction_status_.store(TransactionStatus::IDLE, std::memory_order_release); });
utils::OnScopeExit update_metrics([]() {
memgraph::metrics::IncrementCounter(memgraph::metrics::CommitedTransactions);
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveTransactions);
});
std::optional<TriggerContext> trigger_context = std::nullopt;
if (trigger_context_collector_) {
trigger_context.emplace(std::move(*trigger_context_collector_).TransformToTriggerContext());
trigger_context_collector_.reset();
}
if (frame_change_collector_) {
frame_change_collector_.reset();
}
if (trigger_context) {
// Run the triggers
for (const auto &trigger : interpreter_context_->trigger_store.BeforeCommitTriggers().access()) {

View File

@@ -44,14 +44,14 @@
#include "utils/timer.hpp"
#include "utils/tsc.hpp"
namespace EventCounter {
namespace memgraph::metrics {
extern const Event FailedQuery;
} // namespace EventCounter
} // namespace memgraph::metrics
namespace memgraph::query {
inline constexpr size_t kExecutionMemoryBlockSize = 1UL * 1024UL * 1024UL;
inline constexpr size_t kExecutionPoolMaxBlockSize = 2048UL; // 2 ^ 11
inline constexpr size_t kExecutionPoolMaxBlockSize = 1024UL; // 2 ^ 10
class AuthQueryHandler {
public:
@@ -261,6 +261,7 @@ class Interpreter final {
std::optional<std::string> username_;
bool in_explicit_transaction_{false};
bool expect_rollback_{false};
std::optional<std::map<std::string, storage::PropertyValue>> metadata_{}; //!< User defined transaction metadata
/**
* Prepare a query for execution.
@@ -271,7 +272,8 @@ class Interpreter final {
* @throw query::QueryException
*/
PrepareResult Prepare(const std::string &query, const std::map<std::string, storage::PropertyValue> &params,
const std::string *username);
const std::string *username,
const std::map<std::string, storage::PropertyValue> &metadata = {});
/**
* Execute the last prepared query and stream *all* of the results into the
@@ -315,7 +317,7 @@ class Interpreter final {
std::map<std::string, TypedValue> Pull(TStream *result_stream, std::optional<int> n = {},
std::optional<int> qid = {});
void BeginTransaction();
void BeginTransaction(const std::map<std::string, storage::PropertyValue> &metadata = {});
/*
Returns transaction id or empty if the db_accessor is not initialized.
@@ -401,11 +403,13 @@ class Interpreter final {
std::unique_ptr<storage::Storage::Accessor> db_accessor_;
std::optional<DbAccessor> execution_db_accessor_;
std::optional<TriggerContextCollector> trigger_context_collector_;
std::optional<FrameChangeCollector> frame_change_collector_;
std::optional<storage::IsolationLevel> interpreter_isolation_level;
std::optional<storage::IsolationLevel> next_transaction_isolation_level;
PreparedQuery PrepareTransactionQuery(std::string_view query_upper);
PreparedQuery PrepareTransactionQuery(std::string_view query_upper,
const std::map<std::string, storage::PropertyValue> &metadata = {});
void Commit();
void AdvanceCommand();
void AbortCommand(std::unique_ptr<QueryExecution> *query_execution);
@@ -515,7 +519,7 @@ std::map<std::string, TypedValue> Interpreter::Pull(TStream *result_stream, std:
query_execution.reset(nullptr);
throw;
} catch (const utils::BasicException &) {
EventCounter::IncrementCounter(EventCounter::FailedQuery);
memgraph::metrics::IncrementCounter(memgraph::metrics::FailedQuery);
AbortCommand(&query_execution);
throw;
}

View File

@@ -61,6 +61,7 @@
#include "utils/readable_size.hpp"
#include "utils/string.hpp"
#include "utils/temporal.hpp"
#include "utils/typeinfo.hpp"
// macro for the default implementation of LogicalOperator::Accept
// that accepts the visitor and visits it's input_ operator
@@ -81,7 +82,7 @@
LOG_FATAL("Operator " #class_name " has no single input!"); \
}
namespace EventCounter {
namespace memgraph::metrics {
extern const Event OnceOperator;
extern const Event CreateNodeOperator;
extern const Event CreateExpandOperator;
@@ -119,7 +120,7 @@ extern const Event ForeachOperator;
extern const Event EmptyResultOperator;
extern const Event EvaluatePatternFilterOperator;
extern const Event ApplyOperator;
} // namespace EventCounter
} // namespace memgraph::metrics
namespace memgraph::query::plan {
@@ -170,7 +171,7 @@ bool Once::OnceCursor::Pull(Frame &, ExecutionContext &context) {
}
UniqueCursorPtr Once::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::OnceOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::OnceOperator);
return MakeUniqueCursorPtr<OnceCursor>(mem);
}
@@ -232,7 +233,7 @@ VertexAccessor &CreateLocalVertex(const NodeCreationInfo &node_info, Frame *fram
ACCEPT_WITH_INPUT(CreateNode)
UniqueCursorPtr CreateNode::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::CreateNodeOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::CreateNodeOperator);
return MakeUniqueCursorPtr<CreateNodeCursor>(mem, *this, mem);
}
@@ -282,7 +283,7 @@ CreateExpand::CreateExpand(const NodeCreationInfo &node_info, const EdgeCreation
ACCEPT_WITH_INPUT(CreateExpand)
UniqueCursorPtr CreateExpand::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::CreateNodeOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::CreateNodeOperator);
return MakeUniqueCursorPtr<CreateExpandCursor>(mem, *this, mem);
}
@@ -489,7 +490,7 @@ ScanAll::ScanAll(const std::shared_ptr<LogicalOperator> &input, Symbol output_sy
ACCEPT_WITH_INPUT(ScanAll)
UniqueCursorPtr ScanAll::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ScanAllOperator);
auto vertices = [this](Frame &, ExecutionContext &context) {
auto *db = context.db_accessor;
@@ -512,7 +513,7 @@ ScanAllByLabel::ScanAllByLabel(const std::shared_ptr<LogicalOperator> &input, Sy
ACCEPT_WITH_INPUT(ScanAllByLabel)
UniqueCursorPtr ScanAllByLabel::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllByLabelOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ScanAllByLabelOperator);
auto vertices = [this](Frame &, ExecutionContext &context) {
auto *db = context.db_accessor;
@@ -542,7 +543,7 @@ ScanAllByLabelPropertyRange::ScanAllByLabelPropertyRange(const std::shared_ptr<L
ACCEPT_WITH_INPUT(ScanAllByLabelPropertyRange)
UniqueCursorPtr ScanAllByLabelPropertyRange::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyRangeOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ScanAllByLabelPropertyRangeOperator);
auto vertices = [this](Frame &frame, ExecutionContext &context)
-> std::optional<decltype(context.db_accessor->Vertices(view_, label_, property_, std::nullopt, std::nullopt))> {
@@ -602,7 +603,7 @@ ScanAllByLabelPropertyValue::ScanAllByLabelPropertyValue(const std::shared_ptr<L
ACCEPT_WITH_INPUT(ScanAllByLabelPropertyValue)
UniqueCursorPtr ScanAllByLabelPropertyValue::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyValueOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ScanAllByLabelPropertyValueOperator);
auto vertices = [this](Frame &frame, ExecutionContext &context)
-> std::optional<decltype(context.db_accessor->Vertices(view_, label_, property_, storage::PropertyValue()))> {
@@ -627,7 +628,7 @@ ScanAllByLabelProperty::ScanAllByLabelProperty(const std::shared_ptr<LogicalOper
ACCEPT_WITH_INPUT(ScanAllByLabelProperty)
UniqueCursorPtr ScanAllByLabelProperty::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllByLabelPropertyOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ScanAllByLabelPropertyOperator);
auto vertices = [this](Frame &frame, ExecutionContext &context) {
auto *db = context.db_accessor;
@@ -646,7 +647,7 @@ ScanAllById::ScanAllById(const std::shared_ptr<LogicalOperator> &input, Symbol o
ACCEPT_WITH_INPUT(ScanAllById)
UniqueCursorPtr ScanAllById::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ScanAllByIdOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ScanAllByIdOperator);
auto vertices = [this](Frame &frame, ExecutionContext &context) -> std::optional<std::vector<VertexAccessor>> {
auto *db = context.db_accessor;
@@ -701,7 +702,7 @@ Expand::Expand(const std::shared_ptr<LogicalOperator> &input, Symbol input_symbo
ACCEPT_WITH_INPUT(Expand)
UniqueCursorPtr Expand::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ExpandOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ExpandOperator);
return MakeUniqueCursorPtr<ExpandCursor>(mem, *this, mem);
}
@@ -2171,7 +2172,7 @@ class ExpandAllShortestPathsCursor : public query::plan::Cursor {
};
UniqueCursorPtr ExpandVariable::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ExpandVariableOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ExpandVariableOperator);
switch (type_) {
case EdgeAtom::Type::BREADTH_FIRST:
@@ -2274,7 +2275,7 @@ class ConstructNamedPathCursor : public Cursor {
ACCEPT_WITH_INPUT(ConstructNamedPath)
UniqueCursorPtr ConstructNamedPath::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ConstructNamedPathOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ConstructNamedPathOperator);
return MakeUniqueCursorPtr<ConstructNamedPathCursor>(mem, *this, mem);
}
@@ -2300,7 +2301,7 @@ bool Filter::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
}
UniqueCursorPtr Filter::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::FilterOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::FilterOperator);
return MakeUniqueCursorPtr<FilterCursor>(mem, *this, mem);
}
@@ -2332,12 +2333,11 @@ bool Filter::FilterCursor::Pull(Frame &frame, ExecutionContext &context) {
// Like all filters, newly set values should not affect filtering of old
// nodes and edges.
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor,
storage::View::OLD);
storage::View::OLD, context.frame_change_collector);
while (input_cursor_->Pull(frame, context)) {
for (const auto &pattern_filter_cursor : pattern_filter_cursors_) {
pattern_filter_cursor->Pull(frame, context);
}
if (EvaluateFilter(evaluator, self_.expression_)) return true;
}
return false;
@@ -2353,7 +2353,7 @@ EvaluatePatternFilter::EvaluatePatternFilter(const std::shared_ptr<LogicalOperat
ACCEPT_WITH_INPUT(EvaluatePatternFilter);
UniqueCursorPtr EvaluatePatternFilter::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::EvaluatePatternFilterOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::EvaluatePatternFilterOperator);
return MakeUniqueCursorPtr<EvaluatePatternFilterCursor>(mem, *this, mem);
}
@@ -2386,7 +2386,7 @@ Produce::Produce(const std::shared_ptr<LogicalOperator> &input, const std::vecto
ACCEPT_WITH_INPUT(Produce)
UniqueCursorPtr Produce::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ProduceOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ProduceOperator);
return MakeUniqueCursorPtr<ProduceCursor>(mem, *this, mem);
}
@@ -2410,9 +2410,13 @@ bool Produce::ProduceCursor::Pull(Frame &frame, ExecutionContext &context) {
if (input_cursor_->Pull(frame, context)) {
// Produce should always yield the latest results.
ExpressionEvaluator evaluator(&frame, context.symbol_table, context.evaluation_context, context.db_accessor,
storage::View::NEW);
for (auto named_expr : self_.named_expressions_) named_expr->Accept(evaluator);
storage::View::NEW, context.frame_change_collector);
for (auto *named_expr : self_.named_expressions_) {
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(named_expr->name_)) {
context.frame_change_collector->ResetTrackingValue(named_expr->name_);
}
named_expr->Accept(evaluator);
}
return true;
}
return false;
@@ -2429,7 +2433,7 @@ Delete::Delete(const std::shared_ptr<LogicalOperator> &input_, const std::vector
ACCEPT_WITH_INPUT(Delete)
UniqueCursorPtr Delete::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::DeleteOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::DeleteOperator);
return MakeUniqueCursorPtr<DeleteCursor>(mem, *this, mem);
}
@@ -2581,7 +2585,7 @@ SetProperty::SetProperty(const std::shared_ptr<LogicalOperator> &input, storage:
ACCEPT_WITH_INPUT(SetProperty)
UniqueCursorPtr SetProperty::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::SetPropertyOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::SetPropertyOperator);
return MakeUniqueCursorPtr<SetPropertyCursor>(mem, *this, mem);
}
@@ -2664,7 +2668,7 @@ SetProperties::SetProperties(const std::shared_ptr<LogicalOperator> &input, Symb
ACCEPT_WITH_INPUT(SetProperties)
UniqueCursorPtr SetProperties::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::SetPropertiesOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::SetPropertiesOperator);
return MakeUniqueCursorPtr<SetPropertiesCursor>(mem, *this, mem);
}
@@ -2861,7 +2865,7 @@ SetLabels::SetLabels(const std::shared_ptr<LogicalOperator> &input, Symbol input
ACCEPT_WITH_INPUT(SetLabels)
UniqueCursorPtr SetLabels::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::SetLabelsOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::SetLabelsOperator);
return MakeUniqueCursorPtr<SetLabelsCursor>(mem, *this, mem);
}
@@ -2933,7 +2937,7 @@ RemoveProperty::RemoveProperty(const std::shared_ptr<LogicalOperator> &input, st
ACCEPT_WITH_INPUT(RemoveProperty)
UniqueCursorPtr RemoveProperty::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::RemovePropertyOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::RemovePropertyOperator);
return MakeUniqueCursorPtr<RemovePropertyCursor>(mem, *this, mem);
}
@@ -3019,7 +3023,7 @@ RemoveLabels::RemoveLabels(const std::shared_ptr<LogicalOperator> &input, Symbol
ACCEPT_WITH_INPUT(RemoveLabels)
UniqueCursorPtr RemoveLabels::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::RemoveLabelsOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::RemoveLabelsOperator);
return MakeUniqueCursorPtr<RemoveLabelsCursor>(mem, *this, mem);
}
@@ -3092,7 +3096,7 @@ EdgeUniquenessFilter::EdgeUniquenessFilter(const std::shared_ptr<LogicalOperator
ACCEPT_WITH_INPUT(EdgeUniquenessFilter)
UniqueCursorPtr EdgeUniquenessFilter::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::EdgeUniquenessFilterOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::EdgeUniquenessFilterOperator);
return MakeUniqueCursorPtr<EdgeUniquenessFilterCursor>(mem, *this, mem);
}
@@ -3194,7 +3198,7 @@ class EmptyResultCursor : public Cursor {
};
UniqueCursorPtr EmptyResult::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::EmptyResultOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::EmptyResultOperator);
return MakeUniqueCursorPtr<EmptyResultCursor>(mem, *this, mem);
}
@@ -3233,7 +3237,12 @@ class AccumulateCursor : public Cursor {
if (MustAbort(context)) throw HintedAbortError();
if (cache_it_ == cache_.end()) return false;
auto row_it = (cache_it_++)->begin();
for (const Symbol &symbol : self_.symbols_) frame[symbol] = *row_it++;
for (const Symbol &symbol : self_.symbols_) {
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(symbol.name())) {
context.frame_change_collector->ResetTrackingValue(symbol.name());
}
frame[symbol] = *row_it++;
}
return true;
}
@@ -3255,7 +3264,7 @@ class AccumulateCursor : public Cursor {
};
UniqueCursorPtr Accumulate::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::AccumulateOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::AccumulateOperator);
return MakeUniqueCursorPtr<AccumulateCursor>(mem, *this, mem);
}
@@ -3315,10 +3324,20 @@ class AggregateCursor : public Cursor {
if (aggregation_.empty()) {
auto *pull_memory = context.evaluation_context.memory;
// place default aggregation values on the frame
for (const auto &elem : self_.aggregations_)
for (const auto &elem : self_.aggregations_) {
frame[elem.output_sym] = DefaultAggregationOpValue(elem, pull_memory);
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(elem.output_sym.name())) {
context.frame_change_collector->ResetTrackingValue(elem.output_sym.name());
}
}
// place null as remember values on the frame
for (const Symbol &remember_sym : self_.remember_) frame[remember_sym] = TypedValue(pull_memory);
for (const Symbol &remember_sym : self_.remember_) {
frame[remember_sym] = TypedValue(pull_memory);
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(remember_sym.name())) {
context.frame_change_collector->ResetTrackingValue(remember_sym.name());
}
}
return true;
}
}
@@ -3617,7 +3636,7 @@ class AggregateCursor : public Cursor {
};
UniqueCursorPtr Aggregate::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::AggregateOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::AggregateOperator);
return MakeUniqueCursorPtr<AggregateCursor>(mem, *this, mem);
}
@@ -3628,7 +3647,7 @@ Skip::Skip(const std::shared_ptr<LogicalOperator> &input, Expression *expression
ACCEPT_WITH_INPUT(Skip)
UniqueCursorPtr Skip::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::SkipOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::SkipOperator);
return MakeUniqueCursorPtr<SkipCursor>(mem, *this, mem);
}
@@ -3681,7 +3700,7 @@ Limit::Limit(const std::shared_ptr<LogicalOperator> &input, Expression *expressi
ACCEPT_WITH_INPUT(Limit)
UniqueCursorPtr Limit::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::LimitOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::LimitOperator);
return MakeUniqueCursorPtr<LimitCursor>(mem, *this, mem);
}
@@ -3798,8 +3817,12 @@ class OrderByCursor : public Cursor {
"Number of values does not match the number of output symbols "
"in OrderBy");
auto output_sym_it = self_.output_symbols_.begin();
for (const TypedValue &output : cache_it_->remember) frame[*output_sym_it++] = output;
for (const TypedValue &output : cache_it_->remember) {
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(output_sym_it->name())) {
context.frame_change_collector->ResetTrackingValue(output_sym_it->name());
}
frame[*output_sym_it++] = output;
}
cache_it_++;
return true;
}
@@ -3829,7 +3852,7 @@ class OrderByCursor : public Cursor {
};
UniqueCursorPtr OrderBy::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::OrderByOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::OrderByOperator);
return MakeUniqueCursorPtr<OrderByCursor>(mem, *this, mem);
}
@@ -3846,7 +3869,7 @@ bool Merge::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
}
UniqueCursorPtr Merge::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::MergeOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::MergeOperator);
return MakeUniqueCursorPtr<MergeCursor>(mem, *this, mem);
}
@@ -3926,7 +3949,7 @@ bool Optional::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
}
UniqueCursorPtr Optional::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::OptionalOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::OptionalOperator);
return MakeUniqueCursorPtr<OptionalCursor>(mem, *this, mem);
}
@@ -4032,6 +4055,9 @@ class UnwindCursor : public Cursor {
if (input_value_it_ == input_value_.end()) continue;
frame[self_.output_symbol_] = *input_value_it_++;
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(self_.output_symbol_.name_)) {
context.frame_change_collector->ResetTrackingValue(self_.output_symbol_.name_);
}
return true;
}
}
@@ -4054,7 +4080,7 @@ class UnwindCursor : public Cursor {
};
UniqueCursorPtr Unwind::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::UnwindOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::UnwindOperator);
return MakeUniqueCursorPtr<UnwindCursor>(mem, *this, mem);
}
@@ -4108,7 +4134,7 @@ Distinct::Distinct(const std::shared_ptr<LogicalOperator> &input, const std::vec
ACCEPT_WITH_INPUT(Distinct)
UniqueCursorPtr Distinct::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::DistinctOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::DistinctOperator);
return MakeUniqueCursorPtr<DistinctCursor>(mem, *this, mem);
}
@@ -4130,7 +4156,7 @@ Union::Union(const std::shared_ptr<LogicalOperator> &left_op, const std::shared_
right_symbols_(right_symbols) {}
UniqueCursorPtr Union::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::UnionOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::UnionOperator);
return MakeUniqueCursorPtr<Union::UnionCursor>(mem, *this, mem);
}
@@ -4161,11 +4187,17 @@ bool Union::UnionCursor::Pull(Frame &frame, ExecutionContext &context) {
// collect values from the left child
for (const auto &output_symbol : self_.left_symbols_) {
results[output_symbol.name()] = frame[output_symbol];
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(output_symbol.name())) {
context.frame_change_collector->ResetTrackingValue(output_symbol.name());
}
}
} else if (right_cursor_->Pull(frame, context)) {
// collect values from the right child
for (const auto &output_symbol : self_.right_symbols_) {
results[output_symbol.name()] = frame[output_symbol];
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(output_symbol.name())) {
context.frame_change_collector->ResetTrackingValue(output_symbol.name());
}
}
} else {
return false;
@@ -4174,6 +4206,9 @@ bool Union::UnionCursor::Pull(Frame &frame, ExecutionContext &context) {
// put collected values on frame under union symbols
for (const auto &symbol : self_.union_symbols_) {
frame[symbol] = results[symbol.name()];
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(symbol.name())) {
context.frame_change_collector->ResetTrackingValue(symbol.name());
}
}
return true;
}
@@ -4238,9 +4273,12 @@ class CartesianCursor : public Cursor {
return false;
}
auto restore_frame = [&frame](const auto &symbols, const auto &restore_from) {
auto restore_frame = [&frame, &context](const auto &symbols, const auto &restore_from) {
for (const auto &symbol : symbols) {
frame[symbol] = restore_from[symbol.position()];
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(symbol.name())) {
context.frame_change_collector->ResetTrackingValue(symbol.name());
}
}
};
@@ -4289,7 +4327,7 @@ class CartesianCursor : public Cursor {
} // namespace
UniqueCursorPtr Cartesian::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::CartesianOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::CartesianOperator);
return MakeUniqueCursorPtr<CartesianCursor>(mem, *this, mem);
}
@@ -4318,6 +4356,10 @@ class OutputTableCursor : public Cursor {
if (current_row_ < rows_.size()) {
for (size_t i = 0; i < self_.output_symbols_.size(); ++i) {
frame[self_.output_symbols_[i]] = rows_[current_row_][i];
if (context.frame_change_collector &&
context.frame_change_collector->IsKeyTracked(self_.output_symbols_[i].name())) {
context.frame_change_collector->ResetTrackingValue(self_.output_symbols_[i].name());
}
}
current_row_++;
return true;
@@ -4361,6 +4403,10 @@ class OutputTableStreamCursor : public Cursor {
MG_ASSERT(row->size() == self_->output_symbols_.size(), "Wrong number of columns in row!");
for (size_t i = 0; i < self_->output_symbols_.size(); ++i) {
frame[self_->output_symbols_[i]] = row->at(i);
if (context.frame_change_collector &&
context.frame_change_collector->IsKeyTracked(self_->output_symbols_[i].name())) {
context.frame_change_collector->ResetTrackingValue(self_->output_symbols_[i].name());
}
}
return true;
}
@@ -4545,7 +4591,7 @@ class CallProcedureCursor : public Cursor {
result_row_it_ = result_.rows.begin();
}
const auto &values = result_row_it_->values;
auto &values = result_row_it_->values;
// Check that the row has all fields as required by the result signature.
// C API guarantees that it's impossible to set fields which are not part of
// the result record, but it does not gurantee that some may be missing. See
@@ -4563,7 +4609,11 @@ class CallProcedureCursor : public Cursor {
throw QueryRuntimeException("Procedure '{}' did not yield a record with '{}' field.", self_->procedure_name_,
field_name);
}
frame[self_->result_symbols_[i]] = result_it->second;
frame[self_->result_symbols_[i]] = std::move(result_it->second);
if (context.frame_change_collector &&
context.frame_change_collector->IsKeyTracked(self_->result_symbols_[i].name())) {
context.frame_change_collector->ResetTrackingValue(self_->result_symbols_[i].name());
}
}
++result_row_it_;
@@ -4580,7 +4630,7 @@ class CallProcedureCursor : public Cursor {
};
UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::CallProcedureOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::CallProcedureOperator);
CallProcedure::IncrementCounter(procedure_name_);
return MakeUniqueCursorPtr<CallProcedureCursor>(mem, this, mem);
@@ -4690,6 +4740,9 @@ class LoadCsvCursor : public Cursor {
frame[self_->row_var_] =
CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory));
}
if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(self_->row_var_.name())) {
context.frame_change_collector->ResetTrackingValue(self_->row_var_.name());
}
return true;
}
@@ -4786,7 +4839,7 @@ Foreach::Foreach(std::shared_ptr<LogicalOperator> input, std::shared_ptr<Logical
loop_variable_symbol_(loop_variable_symbol) {}
UniqueCursorPtr Foreach::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ForeachOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ForeachOperator);
return MakeUniqueCursorPtr<ForeachCursor>(mem, *this, mem);
}
@@ -4818,7 +4871,7 @@ bool Apply::Accept(HierarchicalLogicalOperatorVisitor &visitor) {
}
UniqueCursorPtr Apply::MakeCursor(utils::MemoryResource *mem) const {
EventCounter::IncrementCounter(EventCounter::ApplyOperator);
memgraph::metrics::IncrementCounter(memgraph::metrics::ApplyOperator);
return MakeUniqueCursorPtr<ApplyCursor>(mem, *this, mem);
}

View File

@@ -180,6 +180,7 @@ class PatternFilterVisitor : public ExpressionVisitor<void> {
void Visit(IfOperator &op) override{};
void Visit(ListLiteral &op) override{};
void Visit(MapLiteral &op) override{};
void Visit(MapProjectionLiteral &op) override{};
void Visit(LabelsTest &op) override{};
void Visit(Aggregation &op) override{};
void Visit(Function &op) override{};
@@ -194,6 +195,7 @@ class PatternFilterVisitor : public ExpressionVisitor<void> {
void Visit(Identifier &op) override{};
void Visit(PrimitiveLiteral &op) override{};
void Visit(PropertyLookup &op) override{};
void Visit(AllPropertiesLookup &op) override{};
void Visit(ParameterLookup &op) override{};
void Visit(NamedExpression &op) override{};
void Visit(RegexMatch &op) override{};

View File

@@ -124,11 +124,18 @@ class ReturnBodyContext : public HierarchicalTreeVisitor {
bool PostVisit(MapLiteral &map_literal) override {
MG_ASSERT(map_literal.elements_.size() <= has_aggregation_.size(),
"Expected has_aggregation_ flags as much as there are map elements.");
"Expected as many has_aggregation_ flags as there are map elements.");
PostVisitCollectionLiteral(map_literal, [](auto it) { return it->second; });
return true;
}
bool PostVisit(MapProjectionLiteral &map_projection_literal) override {
MG_ASSERT(map_projection_literal.elements_.size() <= has_aggregation_.size(),
"Expected as many has_aggregation_ flags as there are map elements.");
PostVisitCollectionLiteral(map_projection_literal, [](auto it) { return it->second; });
return true;
}
bool PostVisit(All &all) override {
// Remove the symbol which is bound by all, because we are only interested
// in free (unbound) symbols.

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,6 +14,7 @@
#include <datetime.h>
#include <pyerrors.h>
#include <array>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
@@ -860,7 +861,7 @@ py::Object MgpListToPyTuple(mgp_list *list, PyObject *py_graph) {
}
namespace {
std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Object py_record) {
std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Object py_record, mgp_memory *memory) {
py::Object py_mgp(PyImport_ImportModule("mgp"));
if (!py_mgp) return py::FetchError();
auto record_cls = py_mgp.GetAttr("Record");
@@ -902,8 +903,8 @@ std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Obj
if (!field_name) return py::FetchError();
auto *val = PyTuple_GetItem(item, 1);
if (!val) return py::FetchError();
mgp_memory memory{result->rows.get_allocator().GetMemoryResource()};
mgp_value *field_val = PyObjectToMgpValueWithPythonExceptions(val, &memory);
// This memory is one dedicated for mg_procedure.
mgp_value *field_val = PyObjectToMgpValueWithPythonExceptions(val, memory);
if (field_val == nullptr) {
return py::FetchError();
}
@@ -921,15 +922,26 @@ std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Obj
return std::nullopt;
}
std::optional<py::ExceptionInfo> AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq) {
std::optional<py::ExceptionInfo> AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq,
mgp_memory *memory) {
Py_ssize_t len = PySequence_Size(py_seq.Ptr());
if (len == -1) return py::FetchError();
for (Py_ssize_t i = 0; i < len; ++i) {
py::Object py_record(PySequence_GetItem(py_seq.Ptr(), i));
result->rows.reserve(len);
// This proved to be good enough constant not to lose performance on transformation
static constexpr auto del_cnt{100000};
for (Py_ssize_t i = 0, curr_item = 0; i < len; ++i, ++curr_item) {
py::Object py_record(PySequence_GetItem(py_seq.Ptr(), curr_item));
if (!py_record) return py::FetchError();
auto maybe_exc = AddRecordFromPython(result, py_record);
auto maybe_exc = AddRecordFromPython(result, py_record, memory);
if (maybe_exc) return maybe_exc;
// Once PySequence_DelSlice deletes "transformed" objects, starting index is 0 again.
if (i && i % del_cnt == 0) {
PySequence_DelSlice(py_seq.Ptr(), 0, del_cnt);
curr_item = -1;
}
}
// Clear at the end what left
PySequence_DelSlice(py_seq.Ptr(), 0, PySequence_Size(py_seq.Ptr()));
return std::nullopt;
}
@@ -962,6 +974,7 @@ std::function<void()> PyObjectCleanup(py::Object &py_object) {
void CallPythonProcedure(const py::Object &py_cb, mgp_list *args, mgp_graph *graph, mgp_result *result,
mgp_memory *memory) {
// *memory here is memory from `EvalContext`
auto gil = py::EnsureGIL();
auto error_to_msg = [](const std::optional<py::ExceptionInfo> &exc_info) -> std::optional<std::string> {
@@ -979,9 +992,9 @@ void CallPythonProcedure(const py::Object &py_cb, mgp_list *args, mgp_graph *gra
auto py_res = py_cb.Call(py_graph, py_args);
if (!py_res) return py::FetchError();
if (PySequence_Check(py_res.Ptr())) {
return AddMultipleRecordsFromPython(result, py_res);
return AddMultipleRecordsFromPython(result, py_res, memory);
} else {
return AddRecordFromPython(result, py_res);
return AddRecordFromPython(result, py_res, memory);
}
};
@@ -1027,9 +1040,9 @@ void CallPythonTransformation(const py::Object &py_cb, mgp_messages *msgs, mgp_g
auto py_res = py_cb.Call(py_graph, py_messages);
if (!py_res) return py::FetchError();
if (PySequence_Check(py_res.Ptr())) {
return AddMultipleRecordsFromPython(result, py_res);
return AddMultipleRecordsFromPython(result, py_res, memory);
}
return AddRecordFromPython(result, py_res);
return AddRecordFromPython(result, py_res, memory);
};
// It is *VERY IMPORTANT* to note that this code takes great care not to keep

View File

@@ -36,9 +36,9 @@
#include "utils/pmr/string.hpp"
#include "utils/variant_helpers.hpp"
namespace EventCounter {
namespace memgraph::metrics {
extern const Event MessagesConsumed;
} // namespace EventCounter
} // namespace memgraph::metrics
namespace memgraph::query::stream {
namespace {
@@ -495,7 +495,7 @@ Streams::StreamsMap::iterator Streams::CreateConsumer(StreamsMap &map, const std
utils::OnScopeExit interpreter_cleanup{
[interpreter_context, interpreter]() { interpreter_context->interpreters->erase(interpreter.get()); }};
EventCounter::IncrementCounter(EventCounter::MessagesConsumed, messages.size());
memgraph::metrics::IncrementCounter(memgraph::metrics::MessagesConsumed, messages.size());
CallCustomTransformation(transformation_name, messages, result, accessor, *memory_resource, stream_name);
DiscardValueResultStream stream;

View File

@@ -25,9 +25,9 @@
#include "utils/event_counter.hpp"
#include "utils/memory.hpp"
namespace EventCounter {
namespace memgraph::metrics {
extern const Event TriggersExecuted;
} // namespace EventCounter
} // namespace memgraph::metrics
namespace memgraph::query {
namespace {
@@ -248,7 +248,7 @@ void Trigger::Execute(DbAccessor *dba, utils::MonotonicBufferResource *execution
;
cursor->Shutdown();
EventCounter::IncrementCounter(EventCounter::TriggersExecuted);
memgraph::metrics::IncrementCounter(memgraph::metrics::TriggersExecuted);
}
namespace {

View File

@@ -10,7 +10,9 @@ set(storage_v2_src_files
indices.cpp
property_store.cpp
vertex_accessor.cpp
storage.cpp)
storage.cpp
storage_mode.cpp
isolation_level.cpp)
set(storage_v2_src_files

View File

@@ -27,9 +27,15 @@
#include "storage/v2/durability/paths.hpp"
#include "storage/v2/durability/snapshot.hpp"
#include "storage/v2/durability/wal.hpp"
#include "utils/event_histogram.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/message.hpp"
#include "utils/timer.hpp"
namespace memgraph::metrics {
extern const Event SnapshotRecoveryLatency_us;
} // namespace memgraph::metrics
namespace memgraph::storage::durability {
@@ -176,6 +182,8 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
return std::nullopt;
}
utils::Timer timer;
auto snapshot_files = GetSnapshotFiles(snapshot_directory);
RecoveryInfo recovery_info;
@@ -347,6 +355,10 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
}
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices);
memgraph::metrics::Measure(memgraph::metrics::SnapshotRecoveryLatency_us,
std::chrono::duration_cast<std::chrono::microseconds>(timer.Elapsed()).count());
return recovery_info;
}

View File

@@ -0,0 +1,34 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "isolation_level.hpp"
namespace memgraph::storage {
std::string_view IsolationLevelToString(IsolationLevel isolation_level) {
switch (isolation_level) {
case IsolationLevel::READ_COMMITTED:
return "READ_COMMITTED";
case IsolationLevel::READ_UNCOMMITTED:
return "READ_UNCOMMITTED";
case IsolationLevel::SNAPSHOT_ISOLATION:
return "SNAPSHOT_ISOLATION";
}
}
std::string_view IsolationLevelToString(std::optional<IsolationLevel> isolation_level) {
if (isolation_level) {
return IsolationLevelToString(*isolation_level);
}
return "";
}
} // namespace memgraph::storage

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,9 +12,14 @@
#pragma once
#include <cstdint>
#include <optional>
#include <string_view>
namespace memgraph::storage {
enum class IsolationLevel : std::uint8_t { SNAPSHOT_ISOLATION, READ_COMMITTED, READ_UNCOMMITTED };
std::string_view IsolationLevelToString(IsolationLevel isolation_level);
std::string_view IsolationLevelToString(std::optional<IsolationLevel> isolation_level);
} // namespace memgraph::storage

View File

@@ -399,7 +399,8 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
// we cannot know if the difference is only in the current WAL or we need
// to send the snapshot.
if (latest_snapshot) {
locker_acc.AddPath(latest_snapshot->path);
const auto lock_success = locker_acc.AddPath(latest_snapshot->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_steps.emplace_back(std::in_place_type_t<RecoverySnapshot>{}, std::move(latest_snapshot->path));
}
// if there are no finalized WAL files, snapshot left the current WAL
@@ -446,7 +447,8 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
// We need to lock these files and add them to the chain
for (auto result_wal_it = wal_files->begin() + distance_from_first; result_wal_it != wal_files->end();
++result_wal_it) {
locker_acc.AddPath(result_wal_it->path);
const auto lock_success = locker_acc.AddPath(result_wal_it->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
wal_chain.push_back(std::move(result_wal_it->path));
}
@@ -464,7 +466,8 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
MG_ASSERT(latest_snapshot, "Invalid durability state, missing snapshot");
// We didn't manage to find a WAL chain, we need to send the latest snapshot
// with its WALs
locker_acc.AddPath(latest_snapshot->path);
const auto lock_success = locker_acc.AddPath(latest_snapshot->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_steps.emplace_back(std::in_place_type_t<RecoverySnapshot>{}, std::move(latest_snapshot->path));
std::vector<std::filesystem::path> recovery_wal_files;
@@ -483,13 +486,15 @@ std::vector<Storage::ReplicationClient::RecoveryStep> Storage::ReplicationClient
}
for (; wal_it != wal_files->end(); ++wal_it) {
locker_acc.AddPath(wal_it->path);
const auto lock_success = locker_acc.AddPath(wal_it->path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_wal_files.push_back(std::move(wal_it->path));
}
// We only have a WAL before the snapshot
if (recovery_wal_files.empty()) {
locker_acc.AddPath(wal_files->back().path);
const auto lock_success = locker_acc.AddPath(wal_files->back().path);
MG_ASSERT(!lock_success.HasError(), "Tried to lock a nonexistant path.");
recovery_wal_files.push_back(std::move(wal_files->back().path));
}

View File

@@ -34,6 +34,8 @@
#include "storage/v2/storage_mode.hpp"
#include "storage/v2/transaction.hpp"
#include "storage/v2/vertex_accessor.hpp"
#include "utils/event_counter.hpp"
#include "utils/event_histogram.hpp"
#include "utils/file.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
@@ -41,6 +43,7 @@
#include "utils/rw_lock.hpp"
#include "utils/spin_lock.hpp"
#include "utils/stat.hpp"
#include "utils/timer.hpp"
#include "utils/uuid.hpp"
/// REPLICATION ///
@@ -49,6 +52,13 @@
#include "storage/v2/replication/rpc.hpp"
#include "storage/v2/storage_error.hpp"
namespace memgraph::metrics {
extern const Event SnapshotCreationLatency_us;
extern const Event ActiveLabelIndices;
extern const Event ActiveLabelPropertyIndices;
} // namespace memgraph::metrics
namespace memgraph::storage {
using OOMExceptionEnabler = utils::MemoryTracker::OutOfMemoryExceptionEnabler;
@@ -1213,6 +1223,9 @@ utils::BasicResult<StorageIndexDefinitionError, void> Storage::CreateIndex(
commit_log_->MarkFinished(commit_timestamp);
last_commit_timestamp_ = commit_timestamp;
// We don't care if there is a replication error because on main node the change will go through
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveLabelIndices);
if (success) {
return {};
}
@@ -1232,6 +1245,9 @@ utils::BasicResult<StorageIndexDefinitionError, void> Storage::CreateIndex(
commit_log_->MarkFinished(commit_timestamp);
last_commit_timestamp_ = commit_timestamp;
// We don't care if there is a replication error because on main node the change will go through
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveLabelPropertyIndices);
if (success) {
return {};
}
@@ -1251,6 +1267,9 @@ utils::BasicResult<StorageIndexDefinitionError, void> Storage::DropIndex(
commit_log_->MarkFinished(commit_timestamp);
last_commit_timestamp_ = commit_timestamp;
// We don't care if there is a replication error because on main node the change will go through
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveLabelIndices);
if (success) {
return {};
}
@@ -1272,6 +1291,9 @@ utils::BasicResult<StorageIndexDefinitionError, void> Storage::DropIndex(
commit_log_->MarkFinished(commit_timestamp);
last_commit_timestamp_ = commit_timestamp;
// We don't care if there is a replication error because on main node the change will go through
memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveLabelPropertyIndices);
if (success) {
return {};
}
@@ -1943,6 +1965,8 @@ utils::BasicResult<Storage::CreateSnapshotError> Storage::CreateSnapshot(std::op
}
auto snapshot_creator = [this]() {
utils::Timer timer;
auto transaction = CreateTransaction(IsolationLevel::SNAPSHOT_ISOLATION, storage_mode_);
// Create snapshot.
durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_,
@@ -1950,6 +1974,9 @@ utils::BasicResult<Storage::CreateSnapshotError> Storage::CreateSnapshot(std::op
&indices_, &constraints_, config_, uuid_, epoch_id_, epoch_history_, &file_retainer_);
// Finalize snapshot transaction.
commit_log_->MarkFinished(transaction.start_timestamp);
memgraph::metrics::Measure(memgraph::metrics::SnapshotCreationLatency_us,
std::chrono::duration_cast<std::chrono::microseconds>(timer.Elapsed()).count());
};
std::lock_guard snapshot_guard(snapshot_lock_);
@@ -1980,16 +2007,23 @@ utils::BasicResult<Storage::CreateSnapshotError> Storage::CreateSnapshot(std::op
return CreateSnapshotError::ReachedMaxNumTries;
}
bool Storage::LockPath() {
utils::FileRetainer::FileLockerAccessor::ret_type Storage::IsPathLocked() {
auto locker_accessor = global_locker_.Access();
return locker_accessor.IsPathLocked(config_.durability.storage_directory);
}
utils::FileRetainer::FileLockerAccessor::ret_type Storage::LockPath() {
auto locker_accessor = global_locker_.Access();
return locker_accessor.AddPath(config_.durability.storage_directory);
}
bool Storage::UnlockPath() {
utils::FileRetainer::FileLockerAccessor::ret_type Storage::UnlockPath() {
{
auto locker_accessor = global_locker_.Access();
if (!locker_accessor.RemovePath(config_.durability.storage_directory)) {
return false;
const auto ret = locker_accessor.RemovePath(config_.durability.storage_directory);
if (ret.HasError() || !ret.GetValue()) {
// Exit without cleaning the queue
return ret;
}
}
@@ -2173,6 +2207,8 @@ utils::BasicResult<Storage::SetIsolationLevelError> Storage::SetIsolationLevel(I
return {};
}
IsolationLevel Storage::GetIsolationLevel() const noexcept { return isolation_level_; }
void Storage::SetStorageMode(StorageMode storage_mode) {
std::unique_lock main_guard{main_lock_};
storage_mode_ = storage_mode;

View File

@@ -39,6 +39,7 @@
#include "storage/v2/vertex_accessor.hpp"
#include "utils/file_locker.hpp"
#include "utils/on_scope_exit.hpp"
#include "utils/result.hpp"
#include "utils/rw_lock.hpp"
#include "utils/scheduler.hpp"
#include "utils/skip_list.hpp"
@@ -467,8 +468,9 @@ class Storage final {
StorageInfo GetInfo() const;
bool LockPath();
bool UnlockPath();
utils::FileRetainer::FileLockerAccessor::ret_type IsPathLocked();
utils::FileRetainer::FileLockerAccessor::ret_type LockPath();
utils::FileRetainer::FileLockerAccessor::ret_type UnlockPath();
bool SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config = {});
@@ -513,6 +515,7 @@ class Storage final {
enum class SetIsolationLevelError : uint8_t { DisabledForAnalyticalMode };
utils::BasicResult<SetIsolationLevelError> SetIsolationLevel(IsolationLevel isolation_level);
IsolationLevel GetIsolationLevel() const noexcept;
void SetStorageMode(StorageMode storage_mode);

View File

@@ -0,0 +1,25 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage_mode.hpp"
namespace memgraph::storage {
std::string_view StorageModeToString(memgraph::storage::StorageMode storage_mode) {
switch (storage_mode) {
case memgraph::storage::StorageMode::IN_MEMORY_ANALYTICAL:
return "IN_MEMORY_ANALYTICAL";
case memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL:
return "IN_MEMORY_TRANSACTIONAL";
}
}
} // namespace memgraph::storage

View File

@@ -12,9 +12,12 @@
#pragma once
#include <cstdint>
#include <string_view>
namespace memgraph::storage {
enum class StorageMode : std::uint8_t { IN_MEMORY_ANALYTICAL, IN_MEMORY_TRANSACTIONAL };
std::string_view StorageModeToString(memgraph::storage::StorageMode storage_mode);
} // namespace memgraph::storage

View File

@@ -2,6 +2,8 @@ set(utils_src_files
async_timer.cpp
base64.cpp
event_counter.cpp
event_gauge.cpp
event_histogram.cpp
csv_parsing.cpp
file.cpp
file_locker.cpp
@@ -15,7 +17,8 @@ set(utils_src_files
thread_pool.cpp
tsc.cpp
system_info.cpp
uuid.cpp)
uuid.cpp
build_info.cpp)
find_package(Boost REQUIRED)
find_package(fmt REQUIRED)

26
src/utils/build_info.cpp Normal file
View File

@@ -0,0 +1,26 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "build_info.hpp"
namespace memgraph::utils {
BuildInfo GetBuildInfo() {
#ifdef CMAKE_BUILD_TYPE_NAME
constexpr const char *build_info_name = CMAKE_BUILD_TYPE_NAME;
#else
constexpr const char *build_info_name = "unkown";
#endif
BuildInfo info{build_info_name};
return info;
}
} // namespace memgraph::utils

24
src/utils/build_info.hpp Normal file
View File

@@ -0,0 +1,24 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <string>
namespace memgraph::utils {
struct BuildInfo {
std::string build_name;
};
BuildInfo GetBuildInfo();
} // namespace memgraph::utils

View File

@@ -11,69 +11,86 @@
#include "utils/event_counter.hpp"
#define APPLY_FOR_EVENTS(M) \
M(ReadQuery, "Number of read-only queries executed.") \
M(WriteQuery, "Number of write-only queries executed.") \
M(ReadWriteQuery, "Number of read-write queries executed.") \
\
M(OnceOperator, "Number of times Once operator was used.") \
M(CreateNodeOperator, "Number of times CreateNode operator was used.") \
M(CreateExpandOperator, "Number of times CreateExpand operator was used.") \
M(ScanAllOperator, "Number of times ScanAll operator was used.") \
M(ScanAllByLabelOperator, "Number of times ScanAllByLabel operator was used.") \
M(ScanAllByLabelPropertyRangeOperator, "Number of times ScanAllByLabelPropertyRange operator was used.") \
M(ScanAllByLabelPropertyValueOperator, "Number of times ScanAllByLabelPropertyValue operator was used.") \
M(ScanAllByLabelPropertyOperator, "Number of times ScanAllByLabelProperty operator was used.") \
M(ScanAllByIdOperator, "Number of times ScanAllById operator was used.") \
M(ExpandOperator, "Number of times Expand operator was used.") \
M(ExpandVariableOperator, "Number of times ExpandVariable operator was used.") \
M(ConstructNamedPathOperator, "Number of times ConstructNamedPath operator was used.") \
M(FilterOperator, "Number of times Filter operator was used.") \
M(ProduceOperator, "Number of times Produce operator was used.") \
M(DeleteOperator, "Number of times Delete operator was used.") \
M(SetPropertyOperator, "Number of times SetProperty operator was used.") \
M(SetPropertiesOperator, "Number of times SetProperties operator was used.") \
M(SetLabelsOperator, "Number of times SetLabels operator was used.") \
M(RemovePropertyOperator, "Number of times RemoveProperty operator was used.") \
M(RemoveLabelsOperator, "Number of times RemoveLabels operator was used.") \
M(EdgeUniquenessFilterOperator, "Number of times EdgeUniquenessFilter operator was used.") \
M(EmptyResultOperator, "Number of times EmptyResult operator was used.") \
M(AccumulateOperator, "Number of times Accumulate operator was used.") \
M(AggregateOperator, "Number of times Aggregate operator was used.") \
M(SkipOperator, "Number of times Skip operator was used.") \
M(LimitOperator, "Number of times Limit operator was used.") \
M(OrderByOperator, "Number of times OrderBy operator was used.") \
M(MergeOperator, "Number of times Merge operator was used.") \
M(OptionalOperator, "Number of times Optional operator was used.") \
M(UnwindOperator, "Number of times Unwind operator was used.") \
M(DistinctOperator, "Number of times Distinct operator was used.") \
M(UnionOperator, "Number of times Union operator was used.") \
M(CartesianOperator, "Number of times Cartesian operator was used.") \
M(CallProcedureOperator, "Number of times CallProcedure operator was used.") \
M(ForeachOperator, "Number of times Foreach operator was used.") \
M(EvaluatePatternFilterOperator, "Number of times EvaluatePatternFilter operator was used.") \
M(ApplyOperator, "Number of times ApplyOperator operator was used.") \
\
M(FailedQuery, "Number of times executing a query failed.") \
M(LabelIndexCreated, "Number of times a label index was created.") \
M(LabelPropertyIndexCreated, "Number of times a label property index was created.") \
M(StreamsCreated, "Number of Streams created.") \
M(MessagesConsumed, "Number of consumed streamed messages.") \
M(TriggersCreated, "Number of Triggers created.") \
M(TriggersExecuted, "Number of Triggers executed.")
namespace EventCounter {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define APPLY_FOR_COUNTERS(M) \
M(ReadQuery, QueryType, "Number of read-only queries executed.") \
M(WriteQuery, QueryType, "Number of write-only queries executed.") \
M(ReadWriteQuery, QueryType, "Number of read-write queries executed.") \
\
M(OnceOperator, Operator, "Number of times Once operator was used.") \
M(CreateNodeOperator, Operator, "Number of times CreateNode operator was used.") \
M(CreateExpandOperator, Operator, "Number of times CreateExpand operator was used.") \
M(ScanAllOperator, Operator, "Number of times ScanAll operator was used.") \
M(ScanAllByLabelOperator, Operator, "Number of times ScanAllByLabel operator was used.") \
M(ScanAllByLabelPropertyRangeOperator, Operator, "Number of times ScanAllByLabelPropertyRange operator was used.") \
M(ScanAllByLabelPropertyValueOperator, Operator, "Number of times ScanAllByLabelPropertyValue operator was used.") \
M(ScanAllByLabelPropertyOperator, Operator, "Number of times ScanAllByLabelProperty operator was used.") \
M(ScanAllByIdOperator, Operator, "Number of times ScanAllById operator was used.") \
M(ExpandOperator, Operator, "Number of times Expand operator was used.") \
M(ExpandVariableOperator, Operator, "Number of times ExpandVariable operator was used.") \
M(ConstructNamedPathOperator, Operator, "Number of times ConstructNamedPath operator was used.") \
M(FilterOperator, Operator, "Number of times Filter operator was used.") \
M(ProduceOperator, Operator, "Number of times Produce operator was used.") \
M(DeleteOperator, Operator, "Number of times Delete operator was used.") \
M(SetPropertyOperator, Operator, "Number of times SetProperty operator was used.") \
M(SetPropertiesOperator, Operator, "Number of times SetProperties operator was used.") \
M(SetLabelsOperator, Operator, "Number of times SetLabels operator was used.") \
M(RemovePropertyOperator, Operator, "Number of times RemoveProperty operator was used.") \
M(RemoveLabelsOperator, Operator, "Number of times RemoveLabels operator was used.") \
M(EdgeUniquenessFilterOperator, Operator, "Number of times EdgeUniquenessFilter operator was used.") \
M(EmptyResultOperator, Operator, "Number of times EmptyResult operator was used.") \
M(AccumulateOperator, Operator, "Number of times Accumulate operator was used.") \
M(AggregateOperator, Operator, "Number of times Aggregate operator was used.") \
M(SkipOperator, Operator, "Number of times Skip operator was used.") \
M(LimitOperator, Operator, "Number of times Limit operator was used.") \
M(OrderByOperator, Operator, "Number of times OrderBy operator was used.") \
M(MergeOperator, Operator, "Number of times Merge operator was used.") \
M(OptionalOperator, Operator, "Number of times Optional operator was used.") \
M(UnwindOperator, Operator, "Number of times Unwind operator was used.") \
M(DistinctOperator, Operator, "Number of times Distinct operator was used.") \
M(UnionOperator, Operator, "Number of times Union operator was used.") \
M(CartesianOperator, Operator, "Number of times Cartesian operator was used.") \
M(CallProcedureOperator, Operator, "Number of times CallProcedure operator was used.") \
M(ForeachOperator, Operator, "Number of times Foreach operator was used.") \
M(EvaluatePatternFilterOperator, Operator, "Number of times EvaluatePatternFilter operator was used.") \
M(ApplyOperator, Operator, "Number of times ApplyOperator operator was used.") \
\
M(ActiveLabelIndices, Index, "Number of active label indices in the system.") \
M(ActiveLabelPropertyIndices, Index, "Number of active label property indices in the system<.") \
\
M(StreamsCreated, Stream, "Number of Streams created.") \
M(MessagesConsumed, Stream, "Number of consumed streamed messages.") \
\
M(TriggersCreated, Trigger, "Number of Triggers created.") \
M(TriggersExecuted, Trigger, "Number of Triggers executed.") \
\
M(ActiveSessions, Session, "Number of active connections.") \
M(ActiveBoltSessions, Session, "Number of active Bolt connections.") \
M(ActiveTCPSessions, Session, "Number of active TCP connections.") \
M(ActiveSSLSessions, Session, "Number of active SSL connections.") \
M(ActiveWebSocketSessions, Session, "Number of active websocket connections.") \
M(BoltMessages, Session, "Number of Bolt messages sent.") \
\
M(ActiveTransactions, Transaction, "Number of active transactions.") \
M(CommitedTransactions, Transaction, "Number of committed transactions.") \
M(RollbackedTransactions, Transaction, "Number of rollbacked transactions.") \
M(FailedQuery, Transaction, "Number of times executing a query failed.")
namespace memgraph::metrics {
// define every Event as an index in the array of counters
#define M(NAME, DOCUMENTATION) extern const Event NAME = __COUNTER__;
APPLY_FOR_EVENTS(M)
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) extern const Event NAME = __COUNTER__;
APPLY_FOR_COUNTERS(M)
#undef M
inline constexpr Event END = __COUNTER__;
// Initialize array for the global counter with all values set to 0
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
Counter global_counters_array[END]{};
// Initialize global counters
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
EventCounters global_counters(global_counters_array);
const Event EventCounters::num_counters = END;
@@ -82,28 +99,45 @@ void EventCounters::Increment(const Event event, Count amount) {
counters_[event].fetch_add(amount, std::memory_order_relaxed);
}
void EventCounters::Decrement(const Event event, Count amount) {
counters_[event].fetch_sub(amount, std::memory_order_relaxed);
}
void IncrementCounter(const Event event, Count amount) { global_counters.Increment(event, amount); }
void DecrementCounter(const Event event, Count amount) { global_counters.Decrement(event, amount); }
const char *GetName(const Event event) {
const char *GetCounterName(const Event event) {
static const char *strings[] = {
#define M(NAME, DOCUMENTATION) #NAME,
APPLY_FOR_EVENTS(M)
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) #NAME,
APPLY_FOR_COUNTERS(M)
#undef M
};
return strings[event];
}
const char *GetDocumentation(const Event event) {
const char *GetCounterDocumentation(const Event event) {
static const char *strings[] = {
#define M(NAME, DOCUMENTATION) DOCUMENTATION,
APPLY_FOR_EVENTS(M)
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) DOCUMENTATION,
APPLY_FOR_COUNTERS(M)
#undef M
};
return strings[event];
}
Event End() { return END; }
const char *GetCounterType(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) #TYPE,
APPLY_FOR_COUNTERS(M)
#undef M
};
} // namespace EventCounter
return strings[event];
}
Event CounterEnd() { return END; }
} // namespace memgraph::metrics

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,11 +10,12 @@
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <cstdlib>
#include <memory>
namespace EventCounter {
namespace memgraph::metrics {
using Event = uint64_t;
using Count = uint64_t;
using Counter = std::atomic<Count>;
@@ -29,19 +30,23 @@ class EventCounters {
void Increment(Event event, Count amount = 1);
void Decrement(Event event, Count amount = 1);
static const Event num_counters;
private:
Counter *counters_;
};
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
extern EventCounters global_counters;
void IncrementCounter(Event event, Count amount = 1);
void DecrementCounter(Event event, Count amount = 1);
const char *GetName(Event event);
const char *GetDocumentation(Event event);
const char *GetCounterName(Event event);
const char *GetCounterDocumentation(Event event);
const char *GetCounterType(Event event);
Event End();
} // namespace EventCounter
Event CounterEnd();
} // namespace memgraph::metrics

76
src/utils/event_gauge.cpp Normal file
View File

@@ -0,0 +1,76 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "utils/event_gauge.hpp"
// We don't have any gauges for now
#define APPLY_FOR_GAUGES(M)
namespace memgraph::metrics {
// define every Event as an index in the array of gauges
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) extern const Event NAME = __COUNTER__;
APPLY_FOR_GAUGES(M)
#undef M
inline constexpr Event END = __COUNTER__;
// Initialize array for the global gauges with all values set to 0
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
Gauge global_gauges_array[END]{};
// Initialize global counters
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
EventGauges global_gauges(global_gauges_array);
const Event EventGauges::num_gauges = END;
void EventGauges::SetValue(const Event event, Value value) { gauges_[event].store(value, std::memory_order_seq_cst); }
void SetGaugeValue(const Event event, Value value) { global_gauges.SetValue(event, value); }
const char *GetGaugeName(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) #NAME,
APPLY_FOR_GAUGES(M)
#undef M
};
return strings[event];
}
const char *GetGaugeDocumentation(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) DOCUMENTATION,
APPLY_FOR_GAUGES(M)
#undef M
};
return strings[event];
}
const char *GetGaugeType(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION) #TYPE,
APPLY_FOR_GAUGES(M)
#undef M
};
return strings[event];
}
Event GaugeEnd() { return END; }
} // namespace memgraph::metrics

49
src/utils/event_gauge.hpp Normal file
View File

@@ -0,0 +1,49 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <atomic>
#include <cstdlib>
#include <memory>
namespace memgraph::metrics {
using Event = uint64_t;
using Value = uint64_t;
using Gauge = std::atomic<Value>;
class EventGauges {
public:
explicit EventGauges(Gauge *allocated_gauges) noexcept : gauges_(allocated_gauges) {}
auto &operator[](const Event event) { return gauges_[event]; }
const auto &operator[](const Event event) const { return gauges_[event]; }
void SetValue(Event event, Value value);
static const Event num_gauges;
private:
Gauge *gauges_;
};
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
extern EventGauges global_gauges;
void SetGaugeValue(Event event, Value value);
const char *GetGaugeName(Event event);
const char *GetGaugeDocumentation(Event event);
const char *GetGaugeType(Event event);
Event GaugeEnd();
} // namespace memgraph::metrics

View File

@@ -0,0 +1,84 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "utils/event_histogram.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define APPLY_FOR_HISTOGRAMS(M) \
M(QueryExecutionLatency_us, Query, "Query execution latency in microseconds", 50, 90, 99) \
M(SnapshotCreationLatency_us, Snapshot, "Snapshot creation latency in microseconds", 50, 90, 99) \
M(SnapshotRecoveryLatency_us, Snapshot, "Snapshot recovery latency in microseconds", 50, 90, 99)
namespace memgraph::metrics {
// define every Event as an index in the array of counters
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION, ...) extern const Event NAME = __COUNTER__;
APPLY_FOR_HISTOGRAMS(M)
#undef M
inline constexpr Event END = __COUNTER__;
// Initialize array for the global histogram with all named histograms and their percentiles
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
Histogram global_histograms_array[END]{
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION, ...) Histogram({__VA_ARGS__}),
APPLY_FOR_HISTOGRAMS(M)
#undef M
};
// Initialize global histograms
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
EventHistograms global_histograms(global_histograms_array);
const Event EventHistograms::num_histograms = END;
void Measure(const Event event, Value value) { global_histograms.Measure(event, value); }
void EventHistograms::Measure(const Event event, Value value) { histograms_[event].Measure(value); }
const char *GetHistogramName(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION, ...) #NAME,
APPLY_FOR_HISTOGRAMS(M)
#undef M
};
return strings[event];
}
const char *GetHistogramDocumentation(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION, ...) DOCUMENTATION,
APPLY_FOR_HISTOGRAMS(M)
#undef M
};
return strings[event];
}
const char *GetHistogramType(const Event event) {
static const char *strings[] = {
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define M(NAME, TYPE, DOCUMENTATION, ...) #TYPE,
APPLY_FOR_HISTOGRAMS(M)
#undef M
};
return strings[event];
}
Event HistogramEnd() { return END; }
} // namespace memgraph::metrics

View File

@@ -0,0 +1,171 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cmath>
#include "utils/logging.hpp"
namespace memgraph::metrics {
using Event = uint64_t;
using Value = uint64_t;
using Measurement = std::atomic<uint64_t>;
// This is a logarithmically bucketing histogram optimized
// for collecting network response latency distributions.
// It "compresses" values by mapping them to a point on a
// logarithmic curve, which serves as the bucket index. This
// compression technique allows for very accurate histograms
// (unlike what is the case for sampling or lossy probabilistic
// approaches) with the trade-off that we sacrifice around 1%
// precision.
//
// properties:
// * roughly 1% precision loss - can be higher for values
// less than 100, so if measuring latency, generally do
// so in microseconds.
// * ~32kb constant space, single allocation per Histogram.
// * Histogram::Percentile() will return 0 if there were no
// samples measured yet.
class Histogram {
// This is the number of buckets that observed values
// will be logarithmically compressed into.
constexpr static auto kSampleLimit = 4096;
// This is roughly 1/error rate, where 100.0 is roughly
// a 1% error bound for measurements. This is less true
// for tiny measurements, but because we tend to measure
// microseconds, it is usually over 100, which is where
// the error bound starts to stabilize a bit. This has
// been tuned to allow the maximum uint64_t to compress
// within 4096 samples while still achieving a high accuracy.
constexpr static auto kPrecision = 92.0;
// samples_ stores per-bucket counts for measurements
// that have been mapped to a specific uint64_t in
// the "compression" logic below.
std::vector<uint64_t> samples_ = {};
std::vector<uint8_t> percentiles_;
// count_ is the number of measurements that have been
// included in this Histogram.
Measurement count_ = 0;
// sum_ is the summed value of all measurements that
// have been included in this Histogram.
Measurement sum_ = 0;
std::mutex samples_mutex_;
public:
Histogram() {
samples_.resize(kSampleLimit, 0);
percentiles_ = {0, 25, 50, 75, 90, 100};
}
explicit Histogram(std::vector<uint8_t> percentiles) : percentiles_(percentiles) { samples_.resize(kSampleLimit, 0); }
uint64_t Count() const { return count_.load(std::memory_order_relaxed); }
uint64_t Sum() const { return sum_.load(std::memory_order_relaxed); }
std::vector<uint8_t> Percentiles() const { return percentiles_; }
void Measure(uint64_t value) {
// "compression" logic
double boosted = 1.0 + static_cast<double>(value);
double ln = std::log(boosted);
double compressed = (kPrecision * ln) + 0.5;
MG_ASSERT(compressed < kSampleLimit, "compressing value {} to {} is invalid", value, compressed);
auto sample_index = static_cast<uint16_t>(compressed);
count_.fetch_add(1, std::memory_order_relaxed);
sum_.fetch_add(value, std::memory_order_relaxed);
{
std::lock_guard<std::mutex> lock(samples_mutex_);
samples_[sample_index]++;
}
}
std::vector<std::pair<uint64_t, uint64_t>> YieldPercentiles() const {
std::vector<std::pair<uint64_t, uint64_t>> percentile_yield;
percentile_yield.reserve(percentiles_.size());
for (const auto percentile : percentiles_) {
percentile_yield.emplace_back(std::make_pair(percentile, Percentile(percentile)));
}
return percentile_yield;
}
uint64_t Percentile(double percentile) const {
MG_ASSERT(percentile <= 100.0, "percentiles must not exceed 100.0");
MG_ASSERT(percentile >= 0.0, "percentiles must be greater than or equal to 0.0");
auto count = Count();
if (count == 0) {
return 0;
}
const auto floated_count = static_cast<double>(count);
const auto target = std::max(floated_count * percentile / 100.0, 1.0);
auto scanned = 0.0;
for (int i = 0; i < kSampleLimit; i++) {
const auto samples_at_index = samples_[i];
scanned += static_cast<double>(samples_at_index);
if (scanned >= target) {
// "decompression" logic
auto floated = static_cast<double>(i);
auto unboosted = floated / kPrecision;
auto decompressed = std::exp(unboosted) - 1.0;
return static_cast<uint64_t>(decompressed);
}
}
LOG_FATAL("bug in Histogram::Percentile where it failed to return the {} percentile", percentile);
return 0;
}
};
class EventHistograms {
public:
explicit EventHistograms(Histogram *allocated_histograms) noexcept : histograms_(allocated_histograms) {}
auto &operator[](const Event event) { return histograms_[event]; }
const auto &operator[](const Event event) const { return histograms_[event]; }
void Measure(Event event, Value value);
static const Event num_histograms;
private:
Histogram *histograms_;
};
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
extern EventHistograms global_histograms;
void Measure(Event event, Value value);
const char *GetHistogramName(Event event);
const char *GetHistogramDocumentation(Event event);
const char *GetHistogramType(Event event);
Event HistogramEnd();
} // namespace memgraph::metrics

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -80,13 +80,14 @@ void FileRetainer::CleanQueue() {
}
////// LockerEntry //////
void FileRetainer::LockerEntry::LockPath(const std::filesystem::path &path) {
bool FileRetainer::LockerEntry::LockPath(const std::filesystem::path &path) {
auto absolute_path = std::filesystem::absolute(path);
if (std::filesystem::is_directory(absolute_path)) {
directories_.emplace(std::move(absolute_path));
return;
const auto [itr, success] = directories_.emplace(std::move(absolute_path));
return success;
}
files_.emplace(std::move(absolute_path));
const auto [itr, success] = files_.emplace(std::move(absolute_path));
return success;
}
bool FileRetainer::LockerEntry::RemovePath(const std::filesystem::path &path) {
@@ -140,13 +141,27 @@ FileRetainer::FileLockerAccessor::FileLockerAccessor(FileRetainer *retainer, siz
file_retainer_->active_accessors_.fetch_add(1);
}
bool FileRetainer::FileLockerAccessor::AddPath(const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) return false;
file_retainer_->lockers_.WithLock([&](auto &lockers) { lockers[locker_id_].LockPath(path); });
return true;
FileRetainer::FileLockerAccessor::ret_type FileRetainer::FileLockerAccessor::IsPathLocked(
const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) {
return Error::NonexistentPath;
}
return file_retainer_->FileLocked(std::filesystem::absolute(path));
}
bool FileRetainer::FileLockerAccessor::RemovePath(const std::filesystem::path &path) {
FileRetainer::FileLockerAccessor::ret_type FileRetainer::FileLockerAccessor::AddPath(
const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) {
return Error::NonexistentPath;
}
return file_retainer_->lockers_.WithLock([&](auto &lockers) { return lockers[locker_id_].LockPath(path); });
}
FileRetainer::FileLockerAccessor::ret_type FileRetainer::FileLockerAccessor::RemovePath(
const std::filesystem::path &path) {
if (!std::filesystem::exists(path)) {
return Error::NonexistentPath;
}
return file_retainer_->lockers_.WithLock([&](auto &lockers) { return lockers[locker_id_].RemovePath(path); });
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,6 +18,7 @@
#include <unordered_map>
#include "utils/file.hpp"
#include "utils/result.hpp"
#include "utils/rw_lock.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
@@ -114,15 +115,26 @@ class FileRetainer {
struct FileLockerAccessor {
friend FileLocker;
enum class Error : uint8_t {
NonexistentPath = 0,
};
using ret_type = utils::BasicResult<FileRetainer::FileLockerAccessor::Error, bool>;
/**
* Checks if a single path is in the current locker.
*/
ret_type IsPathLocked(const std::filesystem::path &path);
/**
* Add a single path to the current locker.
*/
bool AddPath(const std::filesystem::path &path);
ret_type AddPath(const std::filesystem::path &path);
/**
* Remove a single path form the current locker.
*/
bool RemovePath(const std::filesystem::path &path);
ret_type RemovePath(const std::filesystem::path &path);
FileLockerAccessor(const FileLockerAccessor &) = delete;
FileLockerAccessor(FileLockerAccessor &&) = default;
@@ -182,7 +194,7 @@ class FileRetainer {
class LockerEntry {
public:
void LockPath(const std::filesystem::path &path);
bool LockPath(const std::filesystem::path &path);
bool RemovePath(const std::filesystem::path &path);
[[nodiscard]] bool LocksFile(const std::filesystem::path &path) const;

View File

@@ -0,0 +1,22 @@
#include <string>
#include "query/frontend/ast/ast.hpp"
#include "spdlog/spdlog.h"
namespace memgraph::utils {
// Get ID by which FrameChangeCollector struct can cache in_list.expression2_
inline std::optional<std::string> GetFrameChangeId(memgraph::query::InListOperator &in_list) {
if (in_list.expression2_->GetTypeInfo() == memgraph::query::ListLiteral::kType) {
std::stringstream ss;
ss << static_cast<const void *>(in_list.expression2_);
return ss.str();
}
if (in_list.expression2_->GetTypeInfo() == memgraph::query::Identifier::kType) {
auto *identifier = utils::Downcast<memgraph::query::Identifier>(in_list.expression2_);
return identifier->name_;
}
return {};
};
} // namespace memgraph::utils

View File

@@ -118,8 +118,10 @@ enum class TypeId : uint64_t {
AST_PRIMITIVE_LITERAL,
AST_LIST_LITERAL,
AST_MAP_LITERAL,
AST_MAP_PROJECTION_LITERAL,
AST_IDENTIFIER,
AST_PROPERTY_LOOKUP,
AST_ALL_PROPERTIES_LOOKUP,
AST_LABELS_TEST,
AST_FUNCTION,
AST_REDUCE,

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -60,6 +60,8 @@ BENCHMARK_TEMPLATE(MapLiteral, NewDeleteResource)->Range(512, 1U << 15U)->Unit(b
BENCHMARK_TEMPLATE(MapLiteral, MonotonicBufferResource)->Range(512, 1U << 15U)->Unit(benchmark::kMicrosecond);
// TODO ante benchmark template for MapProjectionLiteral
template <class TMemory>
// NOLINTNEXTLINE(google-runtime-references)
static void AdditionOperator(benchmark::State &state) {

View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Neo4j.Driver.Simple" Version="4.1.1" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,75 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using Neo4j.Driver;
public class Transactions {
public static void Main(string[] args) {
var driver =
GraphDatabase.Driver("bolt://localhost:7687", AuthTokens.None,
(builder) => builder.WithEncryptionLevel(EncryptionLevel.None));
ClearDatabase(driver);
// Explicit transaction query.
using (var session = driver.Session()) {
Console.WriteLine("Checking explicit transaction metadata...");
var txMetadata = new Dictionary<string, object> {
{ "ver", "transaction" }, { "str", "oho" }, { "num", 456 }
};
using (var tx = session.BeginTransaction(txConfig => txConfig.WithMetadata(txMetadata))) {
tx.Run("MATCH (n) RETURN n LIMIT 1").Consume();
// Check transaction info from another thread
Thread show_tx = new Thread(() => ShowTx(ref driver));
show_tx.Start();
show_tx.Join();
// End current transaction
tx.Commit();
}
}
// Implicit transaction query
using (var session = driver.Session()) {
Console.WriteLine("Checking implicit transaction metadata...");
var txMetadata = new Dictionary<string, object> {
{ "ver", "session" }, { "str", "aha" }, { "num", 123 }
};
CheckMD(session.Run("SHOW TRANSACTIONS", txConfig => txConfig.WithMetadata(txMetadata)));
}
Console.WriteLine("All ok!");
}
private static void ClearDatabase(IDriver driver) {
using (var session = driver.Session()) session.Run("MATCH (n) DETACH DELETE n").Consume();
}
public static void ShowTx(ref IDriver driver) {
using (var session = driver.Session()) {
CheckMD(session.Run("SHOW TRANSACTIONS"));
}
}
public static void CheckMD(IResult tx_md) {
int n = 0;
try {
foreach (var res in tx_md) {
var md = res["metadata"].As<Dictionary<string, object>>();
if (md.Count != 0) {
if (md["ver"].As<string>() == "transaction" && md["str"].As<string>() == "oho" &&
md["num"].As<int>() == 456) {
n = n + 1;
} else if (md["ver"].As<string>() == "session" && md["str"].As<string>() == "aha" &&
md["num"].As<int>() == 123) {
n = n + 1;
}
}
}
} catch {
n = 0;
}
if (n == 0) {
Console.WriteLine("Metadata error!");
Environment.Exit(1);
}
}
}

View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Neo4j.Driver.Simple" Version="5.8.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,35 @@
using System;
using System.Linq;
using Neo4j.Driver;
public class Basic {
public static void Main(string[] args) {
using (var driver = GraphDatabase.Driver(
"bolt://localhost:7687", AuthTokens.None,
(ConfigBuilder builder) => builder.WithEncryptionLevel(
EncryptionLevel.None))) using (var session = driver.Session()) {
session.Run("MATCH (n) DETACH DELETE n;").Consume();
Console.WriteLine("Database cleared.");
session.Run("CREATE (alice:Person {name: \"Alice\", age: 22});").Consume();
Console.WriteLine("Record created.");
var node = (INode)session.Run("MATCH (n) RETURN n;").First()["n"];
Console.WriteLine("Record matched.");
var label = string.Join("", node.Labels);
var name = node["name"];
var age = (long)node["age"];
if (!label.Equals("Person") || !name.Equals("Alice") || !age.Equals(22)) {
Console.WriteLine("Data doesn't match!");
System.Environment.Exit(1);
}
Console.WriteLine("Label: " + label);
Console.WriteLine("name: " + name);
Console.WriteLine("age: " + age);
}
Console.WriteLine("All ok!");
}
}

View File

@@ -0,0 +1,64 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using Neo4j.Driver;
public class Transactions {
public static void Main(string[] args) {
using (var driver = GraphDatabase.Driver(
"bolt://localhost:7687", AuthTokens.None,
(builder) => builder.WithEncryptionLevel(EncryptionLevel.None))) {
ClearDatabase(driver);
// Wrong query.
try {
using (var session = driver.Session()) using (var tx = session.BeginTransaction()) {
CreatePerson(tx, "mirko");
// Incorrectly start CREATE
tx.Run("CREATE (").Consume();
CreatePerson(tx, "slavko");
tx.Commit();
}
} catch (ClientException) {
Console.WriteLine("Rolled back transaction");
}
Trace.Assert(CountNodes(driver) == 0, "Expected transaction was rolled back.");
// Correct query.
using (var session = driver.Session()) using (var tx = session.BeginTransaction()) {
CreatePerson(tx, "mirka");
CreatePerson(tx, "slavka");
tx.Commit();
}
Trace.Assert(CountNodes(driver) == 2, "Expected 2 created nodes.");
ClearDatabase(driver);
using (var session = driver.Session()) {
// Create a lot of nodes so that the next read takes a long time.
session.Run("UNWIND range(1, 100000) AS i CREATE ()").Consume();
try {
Console.WriteLine("Running a long read...");
session.Run("MATCH (a), (b), (c), (d), (e), (f) RETURN COUNT(*) AS cnt").Consume();
} catch (TransientException) {
Console.WriteLine("Transaction timed out");
}
}
}
Console.WriteLine("All ok!");
}
private static void CreatePerson(ITransaction tx, string name) {
var parameters = new Dictionary<string, Object> { { "name", name } };
var result = tx.Run("CREATE (person:Person {name: $name}) RETURN person", parameters);
Console.WriteLine("Created: " + ((INode)result.First()["person"])["name"]);
}
private static void ClearDatabase(IDriver driver) {
using (var session = driver.Session()) session.Run("MATCH (n) DETACH DELETE n").Consume();
}
private static int CountNodes(IDriver driver) {
using (var session = driver.Session()) {
var result = session.Run("MATCH (n) RETURN COUNT(*) AS cnt");
return Convert.ToInt32(result.First()["cnt"]);
}
}
}

View File

@@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Neo4j.Driver.Simple" Version="5.8.0" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,22 @@
#!/bin/bash -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR"
# check if dotnet-sdk-2.1 is installed
for i in dotnet; do
if ! which $i >/dev/null; then
echo "Please install $i!"
exit 1
fi
done
for i in *; do
if [ ! -d $i ]; then
continue
fi
pushd $i
dotnet publish -c release --self-contained --runtime linux-x64 --framework netcoreapp2.1 -o build/
./build/$i
popd
done;

View File

@@ -0,0 +1,90 @@
package main
import "github.com/neo4j/neo4j-go-driver/neo4j"
import "log"
import "fmt"
func handle_error(err error) {
log.Fatal("Error occured: %s", err)
}
func check_md(result neo4j.Result, err error) {
if err != nil {
handle_error(err)
}
n := 0
for result.Next() {
md, ok := result.Record().Get("metadata")
if !ok {
log.Fatal("Failed to read metadata!")
}
md_map, ok := md.(map[string]interface{})
if ok {
ver_val, ver_ok := md_map["ver"]
str_val, str_ok := md_map["str"]
num_val, num_ok := md_map["num"]
if (ver_ok && str_ok && num_ok) {
if ((ver_val.(string) == "session" && str_val.(string) == "aha" && num_val.(int64) == 123) ||
(ver_val.(string) == "transaction" && str_val.(string) == "oho" && num_val.(int64) == 456)) {
n++
}
}
}
}
if n == 0 {
log.Fatal("Wrong metadata values!")
}
_, err = result.Consume()
if err != nil {
handle_error(err)
}
}
func check_tx(driver neo4j.Driver) {
sessionConfig := neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}
session, err := driver.NewSession(sessionConfig)
if err != nil {
log.Fatal("An error occurred while creating a session: %s", err)
}
defer session.Close()
result, err := session.Run("SHOW TRANSACTIONS", nil)
check_md(result, err)
}
func main() {
configForNeo4j40 := func(conf *neo4j.Config) { conf.Encrypted = false }
driver, err := neo4j.NewDriver("bolt://localhost:7687", neo4j.BasicAuth("", "", ""), configForNeo4j40)
if err != nil {
log.Fatal("An error occurred opening conn: %s", err)
}
defer driver.Close()
sessionConfig := neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}
session, err := driver.NewSession(sessionConfig)
if err != nil {
log.Fatal("An error occurred while creating a session: %s", err)
}
defer session.Close()
// Implicit transaction
fmt.Println("Checking implicit transaction metadata...")
result, err := session.Run("SHOW TRANSACTIONS", nil, neo4j.WithTxMetadata(map[string]interface{}{"ver":"session", "str":"aha", "num":123}))
check_md(result, err)
// Explicit transaction
fmt.Println("Checking explicit transaction metadata...")
tx, err := session.BeginTransaction(neo4j.WithTxMetadata(map[string]interface{}{"ver":"transaction", "str":"oho", "num":456}))
if err != nil {
handle_error(err)
}
tx.Run("MATCH (n) RETURN n LIMIT 1", map[string]interface{}{})
go check_tx(driver)
tx.Commit()
fmt.Println("All ok!")
}

View File

@@ -12,3 +12,4 @@ go get github.com/neo4j/neo4j-go-driver/neo4j
go run docs_how_to_query.go
go run transactions.go
go run metadata.go

View File

@@ -0,0 +1,89 @@
package main
import (
"log"
"fmt"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
)
func handle_if_error(err error) {
if err != nil {
log.Fatal("Error occured: %s", err)
}
}
func main() {
dbUri := "bolt://localhost:7687"
driver, err := neo4j.NewDriver(dbUri, neo4j.BasicAuth("", "", ""))
if err != nil {
log.Fatal("An error occurred opening conn: %s", err)
}
defer driver.Close()
session := driver.NewSession(neo4j.SessionConfig{})
defer session.Close()
_, err = session.WriteTransaction(clearDatabase)
handle_if_error(err)
fmt.Println("Database cleared.")
_, err = session.WriteTransaction(createItemFn)
handle_if_error(err)
fmt.Println("Record created.")
_,err = session.WriteTransaction(testAll)
handle_if_error(err)
fmt.Println("All ok!")
}
func clearDatabase(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(
"MATCH (n) DETACH DELETE n;",
map[string]interface{}{})
handle_if_error(err)
return result.Consume()
}
func createItemFn(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(
`CREATE (alice:Person {name: "Alice", age: 22});`,
map[string]interface{}{})
handle_if_error(err)
return result.Consume()
}
func testAll(tx neo4j.Transaction) (interface{}, error) {
result, err := tx.Run(
"MATCH (n) RETURN n;",
map[string]interface{}{})
handle_if_error(err)
if !result.Next() {
log.Fatal("Missing result.")
}
node_record, found := result.Record().Get("n")
if !found {
return nil, fmt.Errorf("Wrong result returned.")
}
node_value := node_record.(neo4j.Node)
fmt.Println("Record matched.")
label := node_value.Labels[0]
name, err := neo4j.GetProperty[string](node_value, "name")
handle_if_error(err)
age, err := neo4j.GetProperty[int64](node_value, "age")
handle_if_error(err)
if label != "Person" && name != "Alice" && age != 22 {
return nil, fmt.Errorf("Data doesn't match.")
}
fmt.Println("Label", label)
fmt.Println("name", name)
fmt.Println("age", age)
return result.Consume()
}

View File

@@ -0,0 +1,8 @@
module bolt-test
go 1.18
require (
github.com/neo4j/neo4j-go-driver/v5 v5.9.0 // indirect
golang.org/dl v0.0.0-20230502172222-5216546bad51 // indirect
)

View File

@@ -0,0 +1,10 @@
github.com/neo4j/neo4j-go-driver/v5 v5.5.0 h1:KxufacDV+IqkzbzvjIAIGkBsa2i0lEB8/MhCgOQxrQo=
github.com/neo4j/neo4j-go-driver/v5 v5.5.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
github.com/neo4j/neo4j-go-driver/v5 v5.6.0 h1:+LxOHCyDWGjtD8qHhb20GUpvwCFcJm1wqSEyo2MiehE=
github.com/neo4j/neo4j-go-driver/v5 v5.6.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ=
github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
github.com/neo4j/neo4j-go-driver/v5 v5.9.0 h1:TYxT0RSiwnvVFia90V7TLnRXv8HkdQQ6rTUaPVoyZ+w=
github.com/neo4j/neo4j-go-driver/v5 v5.9.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
golang.org/dl v0.0.0-20230502172222-5216546bad51 h1:Bmo/kmR2hzyhGt3jjtl1ghkCqa5LINbB9D3QTkiLJIY=
golang.org/dl v0.0.0-20230502172222-5216546bad51/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ=

20
tests/drivers/go/v5/run.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/bin/bash -e
GO_VERSION="1.18.9"
GO_VERSION_DIR="/opt/go$GO_VERSION"
if [ -f "$GO_VERSION_DIR/go/bin/go" ]; then
export GOROOT="$GO_VERSION_DIR/go"
export GOPATH="$HOME/go$GO_VERSION"
export PATH="$GO_VERSION_DIR/go/bin:$PATH"
fi
# check if go is installed
for i in go; do
if ! which $i >/dev/null; then
echo "Please install $i!"
exit 1
fi
done
go get github.com/neo4j/neo4j-go-driver/v5
go run docs_quick_start.go

View File

@@ -1,38 +1,39 @@
import static org.neo4j.driver.v1.Values.parameters;
import java.util.*;
import org.neo4j.driver.v1.*;
import org.neo4j.driver.v1.types.*;
import static org.neo4j.driver.v1.Values.parameters;
import java.util.*;
public class Basic {
public static void main(String[] args) {
Config config = Config.build().withoutEncryption().toConfig();
Driver driver = GraphDatabase.driver( "bolt://localhost:7687", AuthTokens.basic( "neo4j", "1234" ), config );
public static void main(String[] args) {
Config config = Config.build().withoutEncryption().toConfig();
Driver driver =
GraphDatabase.driver("bolt://localhost:7687", AuthTokens.basic("neo4j", "1234"), config);
try ( Session session = driver.session() ) {
StatementResult rs1 = session.run( "MATCH (n) DETACH DELETE n" );
System.out.println( "Database cleared." );
try (Session session = driver.session()) {
StatementResult rs1 = session.run("MATCH (n) DETACH DELETE n");
System.out.println("Database cleared.");
StatementResult rs2 = session.run( "CREATE (alice: Person {name: 'Alice', age: 22})" );
System.out.println( "Record created." );
StatementResult rs2 = session.run("CREATE (alice: Person {name: 'Alice', age: 22})");
System.out.println("Record created.");
StatementResult rs3 = session.run( "MATCH (n) RETURN n" );
System.out.println( "Record matched." );
StatementResult rs3 = session.run("MATCH (n) RETURN n");
System.out.println("Record matched.");
List<Record> records = rs3.list();
Record record = records.get( 0 );
Node node = record.get( "n" ).asNode();
if ( !node.get("name").asString().equals( "Alice" ) || node.get("age").asInt() != 22 ) {
System.out.println( "Data doesn't match!" );
System.exit( 1 );
}
List<org.neo4j.driver.v1.Record> records = rs3.list();
org.neo4j.driver.v1.Record record = records.get(0);
Node node = record.get("n").asNode();
if (!node.get("name").asString().equals("Alice") || node.get("age").asInt() != 22) {
System.out.println("Data doesn't match!");
System.exit(1);
}
System.out.println( "All ok!" );
}
catch ( Exception e ) {
System.out.println( e );
System.exit( 1 );
}
driver.close();
System.out.println("All ok!");
} catch (Exception e) {
System.out.println(e);
System.exit(1);
}
driver.close();
}
}

View File

@@ -0,0 +1,111 @@
import static org.neo4j.driver.Values.parameters;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.TimeUnit;
import org.neo4j.driver.AuthTokens;
import org.neo4j.driver.Config;
import org.neo4j.driver.Driver;
import org.neo4j.driver.GraphDatabase;
import org.neo4j.driver.Record;
import org.neo4j.driver.Result;
import org.neo4j.driver.Session;
import org.neo4j.driver.Transaction;
import org.neo4j.driver.TransactionConfig;
import org.neo4j.driver.Value;
import org.neo4j.driver.exceptions.ClientException;
import org.neo4j.driver.exceptions.TransientException;
public class Metadata {
public static String createPerson(Transaction tx, String name) {
Result result =
tx.run("CREATE (a:Person {name: $name}) RETURN a.name", parameters("name", name));
return result.single().get(0).asString();
}
public static void checkMd(Result result) {
int n = 0;
while (result.hasNext()) {
Record r = result.next();
Value md = r.get("metadata");
if (md != null && Objects.equals(md.get("ver").asString(), "transaction")
&& Objects.equals(md.get("str").asString(), "oho")
&& Objects.equals(md.get("num").asInt(), 456)) {
n = n + 1;
} else if (md != null && Objects.equals(md.get("ver").asString(), "session")
&& Objects.equals(md.get("str").asString(), "aha")
&& Objects.equals(md.get("num").asInt(), 123)) {
n = n + 1;
}
}
if (n == 0) {
System.out.println("Error while reading metadata!");
System.exit(1);
}
}
public static void main(String[] args) {
Config config = Config.builder()
.withoutEncryption()
.withMaxTransactionRetryTime(0, TimeUnit.SECONDS)
.build();
Driver driver =
GraphDatabase.driver("bolt://localhost:7687", AuthTokens.basic("neo4j", "1234"), config);
try (Session session = driver.session()) {
// Explicit transaction
System.out.println("Checking explicit transaction metadata...");
try {
TransactionConfig tx_config =
TransactionConfig.builder()
.withTimeout(Duration.ofSeconds(2))
.withMetadata(Map.ofEntries(Map.entry("ver", "transaction"),
Map.entry("str", "oho"), Map.entry("num", 456)))
.build();
Transaction tx = session.beginTransaction(tx_config);
tx.run("MATCH (n) RETURN n LIMIT 1");
// Check the metadata from another thread
try {
Runnable checkTx = new Runnable() {
public void run() {
try (Session s = driver.session()) {
checkMd(s.run("SHOW TRANSACTIONS"));
} catch (ClientException e) {
System.out.println(e);
}
}
};
Thread thread = new Thread(checkTx);
thread.start();
thread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
tx.commit();
} catch (ClientException e) {
System.out.println(e);
}
// Implicit transaction
System.out.println("Checking implicit transaction metadata...");
try {
TransactionConfig tx_config = TransactionConfig.builder()
.withTimeout(Duration.ofSeconds(2))
.withMetadata(Map.ofEntries(Map.entry("ver", "session"),
Map.entry("str", "aha"), Map.entry("num", 123)))
.build();
checkMd(session.run("SHOW TRANSACTIONS", tx_config));
} catch (ClientException e) {
System.out.println(e);
}
System.out.println("All ok!");
} catch (Exception e) {
System.out.println(e);
System.exit(1);
}
driver.close();
}
}

View File

@@ -35,3 +35,6 @@ java -classpath .:$DRIVER:$REACTIVE_STREAM_DEP MaxQueryLength
javac -classpath .:$DRIVER:$REACTIVE_STREAM_DEP Transactions.java
java -classpath .:$DRIVER:$REACTIVE_STREAM_DEP Transactions
javac -classpath .:$DRIVER:$REACTIVE_STREAM_DEP Metadata.java
java -classpath .:$DRIVER:$REACTIVE_STREAM_DEP Metadata

1
tests/drivers/java/v5_8/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
target/

View File

@@ -0,0 +1,93 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>memgraph</groupId>
<artifactId>java-driver-tests</artifactId>
<version>1.0</version>
<properties>
<maven.compiler.source>17</maven.compiler.source>
<maven.compiler.target>17</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.neo4j.driver</groupId>
<artifactId>neo4j-java-driver</artifactId>
<version>5.8.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>build-a</id>
<configuration>
<archive>
<manifest>
<mainClass>memgraph.DocsHowToQuery</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<appendAssemblyId>false</appendAssemblyId>
<finalName>DocsHowToQuery</finalName>
</configuration>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
<execution>
<id>build-b</id>
<configuration>
<archive>
<manifest>
<mainClass>memgraph.MaxQueryLength</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<appendAssemblyId>false</appendAssemblyId>
<finalName>MaxQueryLength</finalName>
</configuration>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
<execution>
<id>build-c</id>
<configuration>
<archive>
<manifest>
<mainClass>memgraph.Transactions</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<appendAssemblyId>false</appendAssemblyId>
<finalName>Transactions</finalName>
</configuration>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

38
tests/drivers/java/v5_8/run.sh Executable file
View File

@@ -0,0 +1,38 @@
#!/bin/bash -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR"
if [ -d "/usr/lib/jvm/java-17-oracle" ]; then
export JAVA_HOME="/usr/lib/jvm/java-17-oracle"
fi
if [ -d "/usr/lib/jvm/java-17-openjdk-amd64" ]; then
export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
fi
if [ -d "/opt/apache-maven-3.9.2" ]; then
export M2_HOME="/opt/apache-maven-3.9.2"
fi
export PATH="$JAVA_HOME/bin:$M2_HOME/bin:$PATH"
for i in java mvn; do
if ! which $i >/dev/null; then
echo "Please install $i!"
exit 1
fi
done
JAVA_VER=$(java -version 2>&1 >/dev/null | grep 'version' | cut -d "\"" -f2 | cut -d "." -f1)
if [ $JAVA_VER -ne 17 ]
then
echo "neo4j-java-driver v5.8 requires Java 17. Please install it!"
exit 1
fi
# CentOS 7 doesn't have Java version that supports var keyword
source ../../../../environment/util.sh
mvn clean package
java -jar target/DocsHowToQuery.jar
java -jar target/MaxQueryLength.jar
java -jar target/Transactions.jar

View File

@@ -0,0 +1,48 @@
package memgraph;
import java.util.*;
import org.neo4j.driver.AuthTokens;
import org.neo4j.driver.Driver;
import org.neo4j.driver.GraphDatabase;
import org.neo4j.driver.Query;
import org.neo4j.driver.Config;
import static org.neo4j.driver.Values.parameters;
public class DocsHowToQuery {
public static void main(String[] args) {
var config = Config.builder().withoutEncryption().build();
var driver = GraphDatabase.driver("bolt://localhost:7687", AuthTokens.basic("", ""), config);
try (var session = driver.session()) {
session.run("MATCH (n) DETACH DELETE n;");
System.out.println("Database cleared.");
session.run("CREATE (alice:Person {name: 'Alice', age: 22});");
System.out.println("Record created.");
var node = session.run("MATCH (n) RETURN n;").list().get(0).get("n").asNode();
System.out.println("Record matched.");
var label = node.labels().iterator().next();
var name = node.get("name").asString();
var age = node.get("age").asInt();
if (!label.equals("Person") || !name.equals("Alice") || age != 22) {
System.out.println("Data doesn't match!");
System.exit(1);
}
System.out.println("Label: " + label);
System.out.println("name: " + name);
System.out.println("age: " + age);
System.out.println("All ok!");
} catch (Exception e) {
System.out.println(e);
System.exit(1);
}
driver.close();
}
}

View File

@@ -0,0 +1,54 @@
/**
* Determines how long could be a query executed
* from Java driver.
*
* Performs binary search until the maximum possible
* query size has found.
*/
package memgraph;
import static org.neo4j.driver.Values.parameters;
import java.util.*;
import org.neo4j.driver.*;
import org.neo4j.driver.types.*;
public class MaxQueryLength {
public static void main(String[] args) {
// init driver
Config config = Config.builder().withoutEncryption().build();
Driver driver = GraphDatabase.driver("bolt://localhost:7687", AuthTokens.basic("", ""), config);
// init query
int property_size = 0;
int min_len = 1;
int max_len = 100000;
String query_template = "CREATE (n {name:\"%s\"})";
int template_size = query_template.length() - 2; // because of %s
// binary search
while (true) {
property_size = (max_len + min_len) / 2;
try (Session session = driver.session()) {
String property_value = new String(new char[property_size]).replace('\0', 'a');
String query = String.format(query_template, property_value);
session.run(query).consume();
if (min_len == max_len || property_size + 1 > max_len) {
break;
}
min_len = property_size + 1;
} catch (Exception e) {
System.out.println(
String.format("Query length: %d; Error: %s", property_size + template_size, e));
max_len = property_size - 1;
}
}
// final result
System.out.println(String.format("\nThe max length of a query executed from "
+ "Java driver is: %s\n",
property_size + template_size));
// cleanup
driver.close();
}
}

View File

@@ -0,0 +1,83 @@
package memgraph;
import static org.neo4j.driver.Values.parameters;
import java.util.*;
import java.util.concurrent.TimeUnit;
import org.neo4j.driver.AuthTokens;
import org.neo4j.driver.Config;
import org.neo4j.driver.Driver;
import org.neo4j.driver.GraphDatabase;
import org.neo4j.driver.Result;
import org.neo4j.driver.Session;
import org.neo4j.driver.Transaction;
import org.neo4j.driver.TransactionWork;
import org.neo4j.driver.exceptions.ClientException;
import org.neo4j.driver.exceptions.TransientException;
public class Transactions {
public static String createPerson(Transaction tx, String name) {
Result result =
tx.run("CREATE (a:Person {name: $name}) RETURN a.name", parameters("name", name));
return result.single().get(0).asString();
}
public static void main(String[] args) {
Config config = Config.builder()
.withoutEncryption()
.withMaxTransactionRetryTime(0, TimeUnit.SECONDS)
.build();
Driver driver =
GraphDatabase.driver("bolt://localhost:7687", AuthTokens.basic("neo4j", "1234"), config);
try (Session session = driver.session()) {
try {
session.writeTransaction(new TransactionWork<String>() {
@Override
public String execute(Transaction tx) {
createPerson(tx, "mirko");
Result result = tx.run("CREATE (");
return result.single().get(0).asString();
}
});
} catch (ClientException e) {
System.out.println(e);
}
session.writeTransaction(new TransactionWork<String>() {
@Override
public String execute(Transaction tx) {
System.out.println(createPerson(tx, "mirko"));
System.out.println(createPerson(tx, "slavko"));
return "Done";
}
});
System.out.println("All ok!");
boolean timed_out = false;
try {
session.writeTransaction(new TransactionWork<String>() {
@Override
public String execute(Transaction tx) {
Result result = tx.run("MATCH (a), (b), (c), (d), (e), (f) RETURN COUNT(*) AS cnt");
return result.single().get(0).asString();
}
});
} catch (TransientException e) {
timed_out = true;
}
if (timed_out) {
System.out.println("The query timed out as was expected.");
} else {
throw new Exception("The query should have timed out, but it didn't!");
}
} catch (Exception e) {
System.out.println(e);
System.exit(1);
}
driver.close();
}
}

View File

@@ -0,0 +1,31 @@
var neo4j = require('neo4j-driver');
var driver = neo4j.driver("bolt://localhost:7687",
neo4j.auth.basic("", ""),
{ encrypted: 'ENCRYPTION_OFF' });
var session = driver.session();
function die() {
session.close();
driver.close();
process.exit(1);
}
function run_query(query, callback) {
var run = session.run(query, {}, {metadata:{"ver":"session", "str":"aha", "num":123}});
run.then(callback).catch(function (error) {
console.log(error);
die();
});
}
console.log("Checking implicit transaction metadata...");
run_query("SHOW TRANSACTIONS;", function (result) {
const md = result.records[0].get("metadata");
if (md["ver"] != "session" || md["str"] != "aha" || md["num"] != 123){
console.log("Error while reading metadata!");
die();
}
console.log("All ok!");
session.close();
driver.close();
});

View File

@@ -10,8 +10,9 @@ fi
if [ ! -d node_modules ]; then
# Driver generated with: `npm install neo4j-driver`
npm install --no-package-lock --no-save neo4j-driver @babel/runtime
npm install --no-package-lock --no-save neo4j-driver@4.1.1
fi
node docs_how_to_query.js
node max_query_length.js
node metadata.js

View File

@@ -0,0 +1,42 @@
var neo4j = require('neo4j-driver');
var driver = neo4j.driver("bolt://localhost:7687",
neo4j.auth.basic("", ""),
{ encrypted: 'ENCRYPTION_OFF' });
var session = driver.session();
function die() {
session.close();
driver.close();
process.exit(1);
}
function run_query(query, callback) {
var run = session.run(query, {});
run.then(callback).catch(function (error) {
console.log(error);
die();
});
}
run_query("MATCH (n) DETACH DELETE n;", function (result) {
console.log("Database cleared.");
run_query("CREATE (alice:Person {name: 'Alice', age: 22});", function (result) {
console.log("Record created.");
run_query("MATCH (n) RETURN n", function (result) {
console.log("Record matched.");
const alice = result.records[0].get("n");
const label = alice.labels[0];
const name = alice.properties["name"];
const age = alice.properties["age"];
if(label != "Person" || name != "Alice" || age != 22){
console.log("Data doesn't match!");
die();
}
console.log("Label: " + label);
console.log("name: " + name);
console.log("age: " + age);
console.log("All ok!");
driver.close();
});
});
});

View File

@@ -0,0 +1,51 @@
// Determines how long could be a query executed
// from JavaScript driver.
//
// Performs binary search until the maximum possible
// query size has found.
// init driver
var neo4j = require('neo4j-driver');
var driver = neo4j.driver("bolt://localhost:7687",
neo4j.auth.basic("", ""),
{ encrypted: 'ENCRYPTION_OFF' });
// init state
var property_size = 0;
var min_len = 1;
var max_len = 1000000;
// hacking with JS and callbacks concept
function serial_execution() {
var next_size = [Math.floor((min_len + max_len) / 2)];
setInterval(function() {
if (next_size.length > 0) {
property_size = next_size.pop();
var query = "CREATE (n {name:\"" +
(new Array(property_size)).join("a")+ "\"})";
var session = driver.session();
session.run(query, {}).then(function (result) {
console.log("Success with the query length " + query.length);
if (min_len == max_len || property_size + 1 > max_len) {
console.log("\nThe max length of a query from JS driver is: " +
query.length + "\n");
session.close();
driver.close();
process.exit(0);
}
min_len = property_size + 1;
next_size.push(Math.floor((min_len + max_len) / 2));
}).catch(function (error) {
console.log("Failure with the query length " + query.length);
max_len = property_size - 1;
next_size.push(Math.floor((min_len + max_len) / 2));
}).then(function(){
session.close();
});
}
}, 100);
}
// execution
console.log("\nDetermine how long can be a query sent from JavaScript driver.");
serial_execution(); // I don't like JavaScript

Some files were not shown because too many files have changed in this diff Show More