Compare commits

...

10 Commits

Author SHA1 Message Date
Jure Bajic
128a6cd522 Update license year (#739) 2023-01-19 12:31:59 +01:00
niko4299
d9eeedb9ee Adding qid in bolt (#721) 2023-01-18 16:33:03 +01:00
Andi
156e2cd095 On delete triggers invalid edge reference (#717)
* Added check if there is invalid reference to the underlying edge

* Added fix and e2e tests

* Isolation levels tracking based on from_vertex_

* Added explicit transaction test + edge accessor changes based on the vertex_edge

* Autocommit on tests, initialize deleted by checking out_edges

Co-authored-by: Marko Budiselić <marko.budiselic@memgraph.com>
2023-01-18 15:05:10 +01:00
Ante Javor
8b834c702c Update mgbench to run Diff workflow under 30mins (#730) 2023-01-14 16:11:49 +01:00
Katarina Supe
eda5213d95 Release pypi mgp 1.1.1 (#727) 2022-12-24 09:33:53 +01:00
Bruno Sačarić
1f2a15e7c8 Fix MATCH not allowed on replica (#709) 2022-12-23 14:47:12 +01:00
Ante Javor
d72e7fa38d Fix mgp.py create edge type hint and comment (#724) 2022-12-23 10:08:52 +01:00
Antonio Filipovic
e5e37bc14a Fix LOAD CSV large memory usage (#712) 2022-12-22 19:38:48 +01:00
Katarina Supe
3ee068bbf9 Update build badge (#722) 2022-12-20 21:56:05 +01:00
Jure Bajic
68e846b182 Update license year (#711) 2022-12-13 13:54:09 +01:00
24 changed files with 490 additions and 114 deletions

View File

@@ -22,7 +22,7 @@ Build modern, graph-based applications on top of your streaming data in minutes.
<p align="center">
<a href="https://github.com/memgraph/memgraph">
<img src="https://img.shields.io/github/workflow/status/memgraph/memgraph/Release%20Ubuntu%2020.04/master" alt="build" title="build"/>
<img src="https://img.shields.io/github/actions/workflow/status/memgraph/memgraph/release_debian10.yaml?branch=master&label=build%20and%20test&logo=github"/>
</a>
<a href="https://memgraph.com/docs/" alt="Documentation">
<img src="https://img.shields.io/badge/documentation-Memgraph-orange" />

View File

@@ -1283,7 +1283,7 @@ class Graph:
raise InvalidContextError()
self._graph.detach_delete_vertex(vertex._vertex)
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> None:
def create_edge(self, from_vertex: Vertex, to_vertex: Vertex, edge_type: EdgeType) -> Edge:
"""
Create an edge.
@@ -1292,13 +1292,16 @@ class Graph:
to_vertex: `Vertex' to where edge is directed.
edge_type: `EdgeType` defines the type of edge.
Returns:
Created `Edge`.
Raises:
ImmutableObjectError: If `graph` is immutable.
UnableToAllocateError: If unable to allocate an edge.
DeletedObjectError: If `from_vertex` or `to_vertex` has been deleted.
SerializationError: If `from_vertex` or `to_vertex` has been modified by another transaction.
Examples:
```graph.create_edge(from_vertex, vertex, edge_type)```
```edge = graph.create_edge(from_vertex, vertex, edge_type)```
"""
if not self.is_valid():
raise InvalidContextError()

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2026-07-11
CHANGE DATE: 2027-19-01
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -1,3 +1,4 @@
.venv
dist
mgp.py
poetry.lock

View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "mgp"
version = "1.1.0"
version = "1.1.1"
description = "Memgraph's module for developing MAGE modules. Used only for type hinting!"
authors = [
"katarinasupe <katarina.supe@memgraph.io>",

View File

@@ -146,9 +146,10 @@ QueryData Client::Execute(const std::string &query, const std::map<std::string,
throw ServerMalformedDataException();
}
auto &header = fields.ValueMap();
QueryData ret{{}, std::move(records), std::move(metadata.ValueMap())};
auto &header = fields.ValueMap();
if (header.find("fields") == header.end()) {
throw ServerMalformedDataException();
}
@@ -164,6 +165,10 @@ QueryData Client::Execute(const std::string &query, const std::map<std::string,
ret.fields.emplace_back(std::move(field_item.ValueString()));
}
if (header.contains("qid")) {
ret.metadata["qid"] = header["qid"];
}
return ret;
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -73,40 +73,6 @@ inline std::pair<std::string, std::string> ExceptionToErrorMessage(const std::ex
namespace details {
template <typename TSession>
State HandleRun(TSession &session, const State state, const Value &query, const Value &params) {
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <bool is_pull, typename TSession>
State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<int> qid) {
try {
@@ -229,7 +195,36 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
return State::Close;
}
return details::HandleRun(session, state, query, params);
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <typename TSession>
@@ -257,7 +252,40 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
spdlog::trace("Couldn't read extra field!");
}
return details::HandleRun(session, state, query, params);
if (state != State::Idle) {
// Client could potentially recover if we move to error state, but there is
// no legitimate situation in which well working client would end up in this
// situation.
spdlog::trace("Unexpected RUN command!");
return State::Close;
}
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
spdlog::debug("[Run] '{}'", query.ValueString());
try {
// Interpret can throw.
const auto [header, qid] = session.Interpret(query.ValueString(), params.ValueMap());
// Convert std::string to Value
std::vector<Value> vec;
std::map<std::string, Value> data;
vec.reserve(header.size());
for (auto &i : header) vec.emplace_back(std::move(i));
data.emplace("fields", std::move(vec));
if (qid.has_value()) {
data.emplace("qid", Value{*qid});
}
// Send the header.
if (!session.encoder_.MessageSuccess(data)) {
spdlog::trace("Couldn't send query header!");
return State::Close;
}
return State::Result;
} catch (const std::exception &e) {
return HandleFailure(session, e);
}
}
template <typename TSession>

View File

@@ -1044,7 +1044,7 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
utils::PoolResource pool_memory(128, 1024, &monotonic_memory, utils::NewDeleteResource());
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
if (memory_limit_) {

View File

@@ -4528,24 +4528,24 @@ auto ToOptionalString(ExpressionEvaluator *evaluator, Expression *expression) ->
return std::nullopt;
};
TypedValue CsvRowToTypedList(csv::Reader::Row row) {
TypedValue CsvRowToTypedList(csv::Reader::Row &row) {
auto *mem = row.get_allocator().GetMemoryResource();
auto typed_columns = utils::pmr::vector<TypedValue>(mem);
typed_columns.reserve(row.size());
for (auto &column : row) {
typed_columns.emplace_back(std::move(column));
}
return TypedValue(typed_columns, mem);
return {std::move(typed_columns), mem};
}
TypedValue CsvRowToTypedMap(csv::Reader::Row row, csv::Reader::Header header) {
TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header) {
// a valid row has the same number of elements as the header
auto *mem = row.get_allocator().GetMemoryResource();
utils::pmr::map<utils::pmr::string, TypedValue> m(mem);
for (auto i = 0; i < row.size(); ++i) {
m.emplace(std::move(header[i]), std::move(row[i]));
}
return TypedValue(m, mem);
return {std::move(m), mem};
}
} // namespace
@@ -4584,18 +4584,17 @@ class LoadCsvCursor : public Cursor {
// have to read at most cardinality(n) rows (but we can read less and stop
// pulling MATCH).
if (!input_is_once_ && !input_pulled) return false;
if (auto row = reader_->GetNextRow(context.evaluation_context.memory)) {
if (!reader_->HasHeader()) {
frame[self_->row_var_] = CsvRowToTypedList(std::move(*row));
} else {
frame[self_->row_var_] = CsvRowToTypedMap(
std::move(*row), csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory));
}
return true;
auto row = reader_->GetNextRow(context.evaluation_context.memory);
if (!row) {
return false;
}
return false;
if (!reader_->HasHeader()) {
frame[self_->row_var_] = CsvRowToTypedList(*row);
} else {
frame[self_->row_var_] =
CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory));
}
return true;
}
void Reset() override { input_cursor_->Reset(); }

View File

@@ -91,18 +91,25 @@ bool ReadWriteTypeChecker::PreVisit([[maybe_unused]] Foreach &op) {
bool ReadWriteTypeChecker::Visit(Once &) { return false; } // NOLINT(hicpp-named-parameter)
void ReadWriteTypeChecker::UpdateType(RWType op_type) {
// Update type only if it's not the NONE type and the current operator's type
// is different than the one that's currently inferred.
if (type != RWType::NONE && type != op_type) {
type = RWType::RW;
}
// Stop inference because RW is the most "dominant" type, i.e. it isn't
// affected by the type of nodes in the plan appearing after the node for
// which the type is set to RW.
if (type == RWType::RW) {
return;
}
if (type == RWType::NONE && op_type != RWType::NONE) {
// if op_type is NONE, type doesn't change.
if (op_type == RWType::NONE) {
return;
}
// Update type only if it's not the NONE type and the current operator's type
// is different than the one that's currently inferred.
if (type != RWType::NONE && type != op_type) {
type = RWType::RW;
}
if (type == RWType::NONE) {
type = op_type;
}
}

View File

@@ -15,7 +15,7 @@
namespace memgraph::query::plan {
class ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor {
struct ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor {
public:
ReadWriteTypeChecker() = default;
@@ -89,7 +89,6 @@ class ReadWriteTypeChecker : public virtual HierarchicalLogicalOperatorVisitor {
bool Visit(Once &) override;
private:
void UpdateType(RWType op_type);
};

View File

@@ -12,6 +12,7 @@
#include "storage/v2/edge_accessor.hpp"
#include <memory>
#include <tuple>
#include "storage/v2/mvcc.hpp"
#include "storage/v2/property_value.hpp"
@@ -21,8 +22,47 @@
namespace memgraph::storage {
bool EdgeAccessor::IsVisible(const View view) const {
bool deleted = true;
bool exists = true;
bool deleted = true;
// When edges don't have properties, their isolation level is still dictated by MVCC ->
// iterate over the deltas of the from_vertex_ and see which deltas can be applied on edges.
if (!config_.properties_on_edges) {
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(from_vertex_->lock);
// Initialize deleted by checking if out edges contain edge_
deleted = std::find_if(from_vertex_->out_edges.begin(), from_vertex_->out_edges.end(), [&](const auto &out_edge) {
return std::get<2>(out_edge) == edge_;
}) == from_vertex_->out_edges.end();
delta = from_vertex_->delta;
}
ApplyDeltasForRead(transaction_, delta, view, [&](const Delta &delta) {
switch (delta.action) {
case Delta::Action::ADD_LABEL:
case Delta::Action::REMOVE_LABEL:
case Delta::Action::SET_PROPERTY:
case Delta::Action::REMOVE_IN_EDGE:
case Delta::Action::ADD_IN_EDGE:
case Delta::Action::RECREATE_OBJECT:
case Delta::Action::DELETE_OBJECT:
break;
case Delta::Action::ADD_OUT_EDGE: { // relevant for the from_vertex_ -> we just deleted the edge
if (delta.vertex_edge.edge == edge_) {
deleted = false;
}
break;
}
case Delta::Action::REMOVE_OUT_EDGE: { // also relevant for the from_vertex_ -> we just added the edge
if (delta.vertex_edge.edge == edge_) {
exists = false;
}
break;
}
}
});
return exists && (for_deleted_ || !deleted);
}
Delta *delta = nullptr;
{
std::lock_guard<utils::SpinLock> guard(edge_.ptr->lock);
@@ -49,7 +89,6 @@ bool EdgeAccessor::IsVisible(const View view) const {
}
}
});
return exists && (for_deleted_ || !deleted);
}

View File

@@ -40,7 +40,7 @@ std::optional<utils::pmr::string> Reader::GetNextLine(utils::MemoryResource *mem
return std::nullopt;
}
++line_count_;
return line;
return std::move(line);
}
Reader::ParsingResult Reader::ParseHeader() {

View File

@@ -251,9 +251,10 @@ void Pool::Release() {
} // namespace impl
PoolResource::PoolResource(size_t max_blocks_per_chunk, size_t max_block_size, MemoryResource *memory)
: pools_(memory),
unpooled_(memory),
PoolResource::PoolResource(size_t max_blocks_per_chunk, size_t max_block_size, MemoryResource *memory_pools,
MemoryResource *memory_unpooled)
: pools_(memory_pools),
unpooled_(memory_unpooled),
max_blocks_per_chunk_(std::min(max_blocks_per_chunk, static_cast<size_t>(impl::Pool::MaxBlocksInChunk()))),
max_block_size_(max_block_size) {
MG_ASSERT(max_blocks_per_chunk_ > 0U, "Invalid number of blocks per chunk");
@@ -273,14 +274,14 @@ void *PoolResource::DoAllocate(size_t bytes, size_t alignment) {
if (block_size % alignment != 0) throw BadAlloc("Requested bytes must be a multiple of alignment");
if (block_size > max_block_size_) {
// Allocate a big block.
BigBlock big_block{bytes, alignment, GetUpstreamResource()->Allocate(bytes, alignment)};
BigBlock big_block{bytes, alignment, GetUpstreamResourceBlocks()->Allocate(bytes, alignment)};
// Insert the big block in the sorted position.
auto it = std::lower_bound(unpooled_.begin(), unpooled_.end(), big_block,
[](const auto &a, const auto &b) { return a.data < b.data; });
try {
unpooled_.insert(it, big_block);
} catch (...) {
GetUpstreamResource()->Deallocate(big_block.data, bytes, alignment);
GetUpstreamResourceBlocks()->Deallocate(big_block.data, bytes, alignment);
throw;
}
return big_block.data;
@@ -318,7 +319,7 @@ void PoolResource::DoDeallocate(void *p, size_t bytes, size_t alignment) {
MG_ASSERT(it != unpooled_.end(), "Failed deallocation");
MG_ASSERT(it->data == p && it->bytes == bytes && it->alignment == alignment, "Failed deallocation");
unpooled_.erase(it);
GetUpstreamResource()->Deallocate(p, bytes, alignment);
GetUpstreamResourceBlocks()->Deallocate(p, bytes, alignment);
return;
}
// Deallocate a regular block, first check if last_dealloc_pool_ is suitable.
@@ -339,7 +340,7 @@ void PoolResource::Release() {
for (auto &pool : pools_) pool.Release();
pools_.clear();
for (auto &big_block : unpooled_)
GetUpstreamResource()->Deallocate(big_block.data, big_block.bytes, big_block.alignment);
GetUpstreamResourceBlocks()->Deallocate(big_block.data, big_block.bytes, big_block.alignment);
unpooled_.clear();
last_alloc_pool_ = nullptr;
last_dealloc_pool_ = nullptr;

View File

@@ -469,7 +469,8 @@ class PoolResource final : public MemoryResource {
/// impl::Pool::MaxBlocksInChunk()) as the real maximum number of blocks per
/// chunk. Allocation requests exceeding max_block_size are simply forwarded
/// to upstream memory.
PoolResource(size_t max_blocks_per_chunk, size_t max_block_size, MemoryResource *memory = NewDeleteResource());
PoolResource(size_t max_blocks_per_chunk, size_t max_block_size, MemoryResource *memory_pools = NewDeleteResource(),
MemoryResource *memory_unpooled = NewDeleteResource());
PoolResource(const PoolResource &) = delete;
PoolResource &operator=(const PoolResource &) = delete;
@@ -480,6 +481,7 @@ class PoolResource final : public MemoryResource {
~PoolResource() override { Release(); }
MemoryResource *GetUpstreamResource() const { return pools_.get_allocator().GetMemoryResource(); }
MemoryResource *GetUpstreamResourceBlocks() const { return unpooled_.get_allocator().GetMemoryResource(); }
/// Release all allocated memory.
void Release();

View File

@@ -20,3 +20,10 @@ add_subdirectory(procedures)
add_dependencies(memgraph__e2e__triggers__on_create memgraph__e2e__triggers__write.py)
add_dependencies(memgraph__e2e__triggers__on_update memgraph__e2e__triggers__write.py)
add_dependencies(memgraph__e2e__triggers__on_delete memgraph__e2e__triggers__write.py)
function(copy_triggers_e2e_python_files FILE_NAME)
copy_e2e_python_files(triggers ${FILE_NAME})
endfunction()
copy_triggers_e2e_python_files(common.py)
copy_triggers_e2e_python_files(triggers_properties_false.py)

View File

@@ -0,0 +1,34 @@
# Copyright 2022 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import typing
import mgclient
import pytest
def execute_and_fetch_all(cursor: mgclient.Cursor, query: str, params: dict = {}) -> typing.List[tuple]:
cursor.execute(query, params)
return cursor.fetchall()
@pytest.fixture
def connect(**kwargs) -> mgclient.Connection:
connection = mgclient.connect(host="localhost", port=7687, **kwargs)
connection.autocommit = True
triggers_list = execute_and_fetch_all(connection.cursor(), "SHOW TRIGGERS;")
for trigger in triggers_list:
execute_and_fetch_all(connection.cursor(), f"DROP TRIGGER {trigger[0]}")
execute_and_fetch_all(connection.cursor(), "MATCH (n) DETACH DELETE n")
yield connection
for trigger in triggers_list:
execute_and_fetch_all(connection.cursor(), f"DROP TRIGGER {trigger[0]}")
execute_and_fetch_all(connection.cursor(), "MATCH (n) DETACH DELETE n")

View File

@@ -0,0 +1,170 @@
# Copyright 2022 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import sys
import mgclient
import pytest
from common import connect, execute_and_fetch_all
@pytest.mark.parametrize("ba_commit", ["BEFORE COMMIT", "AFTER COMMIT"])
def test_create_on_create(ba_commit, connect):
"""
Args:
ba_commit (str): BEFORE OR AFTER commit
"""
cursor = connect.cursor()
QUERY_TRIGGER_CREATE = f"""
CREATE TRIGGER CreateTriggerEdgesCount
ON --> CREATE
{ba_commit}
EXECUTE
CREATE (n:CreatedEdge {{count: size(createdEdges)}})
"""
execute_and_fetch_all(cursor, QUERY_TRIGGER_CREATE)
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 1})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 2})")
res = execute_and_fetch_all(cursor, "MATCH (n:Node) RETURN n")
assert len(res) == 2
res2 = execute_and_fetch_all(cursor, "MATCH (n:CreatedEdge) RETURN n")
assert len(res2) == 0
QUERY_CREATE_EDGE = """
MATCH (n:Node {id: 1}), (m:Node {id: 2})
CREATE (n)-[r:TYPE]->(m);
"""
execute_and_fetch_all(cursor, QUERY_CREATE_EDGE)
# See if trigger was triggered
nodes = execute_and_fetch_all(cursor, "MATCH (n:Node) RETURN n")
assert len(nodes) == 2
created_edges = execute_and_fetch_all(cursor, "MATCH (n:CreatedEdge) RETURN n")
assert len(created_edges) == 1
# execute_and_fetch_all(cursor, "DROP TRIGGER CreateTriggerEdgesCount")
# execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n;")
@pytest.mark.parametrize("ba_commit", ["AFTER COMMIT", "BEFORE COMMIT"])
def test_create_on_delete(ba_commit, connect):
"""
Args:
ba_commit (str): BEFORE OR AFTER commit
"""
cursor = connect.cursor()
QUERY_TRIGGER_CREATE = f"""
CREATE TRIGGER DeleteTriggerEdgesCount
ON --> DELETE
{ba_commit}
EXECUTE
CREATE (n:DeletedEdge {{count: size(deletedEdges)}})
"""
# Setup queries
execute_and_fetch_all(cursor, QUERY_TRIGGER_CREATE)
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 1})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 2})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 3})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 4})")
res = execute_and_fetch_all(cursor, "MATCH (n:Node) RETURN n")
assert len(res) == 4
res2 = execute_and_fetch_all(cursor, "MATCH (n:DeletedEdge) RETURN n")
assert len(res2) == 0
# create an edge that will be deleted
QUERY_CREATE_EDGE = """
MATCH (n:Node {id: 1}), (m:Node {id: 2})
CREATE (n)-[r:TYPE]->(m);
"""
execute_and_fetch_all(cursor, QUERY_CREATE_EDGE)
# create an edge that won't be deleted
QUERY_CREATE_EDGE_NO_DELETE = """
MATCH (n:Node {id: 3}), (m:Node {id: 4})
CREATE (n)-[r:NO_DELETE_EDGE]->(m);
"""
execute_and_fetch_all(cursor, QUERY_CREATE_EDGE_NO_DELETE)
# Delete only one type of the edger
QUERY_DELETE_EDGE = """
MATCH ()-[r:TYPE]->()
DELETE r;
"""
execute_and_fetch_all(cursor, QUERY_DELETE_EDGE)
# See if trigger was triggered
nodes = execute_and_fetch_all(cursor, "MATCH (n:Node) RETURN n")
assert len(nodes) == 4
# Check how many edges got deleted
deleted_edges = execute_and_fetch_all(cursor, "MATCH (n:DeletedEdge) RETURN n")
assert len(deleted_edges) == 1
# execute_and_fetch_all(cursor, "DROP TRIGGER DeleteTriggerEdgesCount")
# execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n")``
@pytest.mark.parametrize("ba_commit", ["BEFORE COMMIT", "AFTER COMMIT"])
def test_create_on_delete_explicit_transaction(ba_commit):
"""
Args:
ba_commit (str): BEFORE OR AFTER commit
"""
connection_with_autocommit = mgclient.connect(host="localhost", port=7687)
connection_with_autocommit.autocommit = True
cursor_autocommit = connection_with_autocommit.cursor()
QUERY_TRIGGER_CREATE = f"""
CREATE TRIGGER DeleteTriggerEdgesCountExplicit
ON --> DELETE
{ba_commit}
EXECUTE
CREATE (n:DeletedEdge {{count: size(deletedEdges)}})
"""
# Setup queries
execute_and_fetch_all(cursor_autocommit, QUERY_TRIGGER_CREATE)
# Start explicit transaction on the execution of the first command
connection_without_autocommit = mgclient.connect(host="localhost", port=7687)
connection_without_autocommit.autocommit = False
cursor = connection_without_autocommit.cursor()
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 1})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 2})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 3})")
execute_and_fetch_all(cursor, "CREATE (n:Node {id: 4})")
res = execute_and_fetch_all(cursor, "MATCH (n:Node) RETURN n")
assert len(res) == 4
res2 = execute_and_fetch_all(cursor, "MATCH (n:DeletedEdge) RETURN n;")
assert len(res2) == 0
QUERY_CREATE_EDGE = """
MATCH (n:Node {id: 1}), (m:Node {id: 2})
CREATE (n)-[r:TYPE]->(m);
"""
execute_and_fetch_all(cursor, QUERY_CREATE_EDGE)
# create an edge that won't be deleted
QUERY_CREATE_EDGE_NO_DELETE = """
MATCH (n:Node {id: 3}), (m:Node {id: 4})
CREATE (n)-[r:NO_DELETE_EDGE]->(m);
"""
execute_and_fetch_all(cursor, QUERY_CREATE_EDGE_NO_DELETE)
# Delete only one type of the edger
QUERY_DELETE_EDGE = """
MATCH ()-[r:TYPE]->()
DELETE r;
"""
execute_and_fetch_all(cursor, QUERY_DELETE_EDGE)
connection_without_autocommit.commit() # finish explicit transaction
nodes = execute_and_fetch_all(cursor, "MATCH (n:Node) RETURN n")
assert len(nodes) == 4
# Check how many edges got deleted
deleted_nodes_edges = execute_and_fetch_all(cursor, "MATCH (n:DeletedEdge) RETURN n")
assert len(deleted_nodes_edges) == 0
# Delete with the original cursor because triggers aren't allowed in multi-transaction environment
execute_and_fetch_all(cursor_autocommit, "DROP TRIGGER DeleteTriggerEdgesCountExplicit")
execute_and_fetch_all(cursor, "MATCH (n) DETACH DELETE n")
connection_without_autocommit.commit() # finish explicit transaction
nodes = execute_and_fetch_all(cursor_autocommit, "MATCH (n) RETURN n")
assert len(nodes) == 0
connection_with_autocommit.close()
connection_without_autocommit.close()
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-rA"]))

View File

@@ -2,7 +2,14 @@ bolt_port: &bolt_port "7687"
template_cluster: &template_cluster
cluster:
main:
args: ["--bolt-port", *bolt_port, "--log-level=TRACE"]
args: ["--bolt-port", *bolt_port, "--log-level=TRACE", "--storage-properties-on-edges=True"]
log_file: "triggers-e2e.log"
setup_queries: []
validation_queries: []
storage_properties_edges_false: &storage_properties_edges_false
cluster:
main:
args: ["--bolt-port", *bolt_port, "--log-level=TRACE", "--also-log-to-stderr", "--storage-properties-on-edges=False"]
log_file: "triggers-e2e.log"
setup_queries: []
validation_queries: []
@@ -18,7 +25,7 @@ workloads:
args: ["--bolt-port", *bolt_port]
proc: "tests/e2e/triggers/procedures/"
<<: *template_cluster
- name: "ON DELETE Triggers"
- name: "ON DELETE Triggers Storage Properties On Edges True"
binary: "tests/e2e/triggers/memgraph__e2e__triggers__on_delete"
args: ["--bolt-port", *bolt_port]
proc: "tests/e2e/triggers/procedures/"
@@ -27,5 +34,8 @@ workloads:
binary: "tests/e2e/triggers/memgraph__e2e__triggers__privileges"
args: ["--bolt-port", *bolt_port]
<<: *template_cluster
- name: "ON DELETE Triggers Storage Properties On Edges False" # should be the same as the python file
binary: "tests/e2e/pytest_runner.sh"
proc: "tests/e2e/triggers/procedures/"
args: ["triggers/triggers_properties_false.py"]
<<: *storage_properties_edges_false

View File

@@ -45,6 +45,21 @@ class BoltClient : public ::testing::Test {
return true;
}
bool ExecuteAndCheckQid(const std::string &query, int qid, const std::string &message = "") {
try {
auto ret = client_.Execute(query, {});
if (ret.metadata["qid"].ValueInt() != qid) {
return false;
}
} catch (const ClientQueryException &e) {
if (message != "") {
EXPECT_EQ(e.what(), message);
}
throw;
}
return true;
}
int64_t GetCount() {
auto ret = client_.Execute("match (n) return count(n)", {});
EXPECT_EQ(ret.records.size(), 1);
@@ -461,6 +476,18 @@ TEST_F(BoltClient, MixedCaseAndWhitespace) {
EXPECT_FALSE(TransactionActive());
}
TEST_F(BoltClient, TestQid) {
for (int i = 0; i < 3; ++i) {
EXPECT_TRUE(Execute("match (n) return count(n)"));
}
EXPECT_TRUE(Execute("begin"));
for (int i = 0; i < 3; ++i) {
EXPECT_TRUE(ExecuteAndCheckQid("match (n) return count(n)", i + 1));
}
EXPECT_TRUE(Execute("commit"));
EXPECT_FALSE(TransactionActive());
}
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
gflags::ParseCommandLineFlags(&argc, &argv, true);

View File

@@ -142,7 +142,7 @@ parser.add_argument(
with the presence of 300 write queries from write type or 30%""",
)
parser.add_argument("--tail-latency", type=int, default=100, help="Number of queries for the tail latency statistics")
parser.add_argument("--tail-latency", type=int, default=0, help="Number of queries for the tail latency statistics")
parser.add_argument(
"--performance-tracking",
@@ -223,8 +223,17 @@ def filter_benchmarks(generators, patterns):
patterns,
):
current[group].append((query_name, query_func))
if len(current) > 0:
filtered.append((generator(variant, args.vendor_name), dict(current)))
if len(current) == 0:
continue
# Ignore benchgraph "basic" queries in standard CI/CD run
for pattern in patterns:
res = pattern.count("*")
key = "basic"
if res >= 2 and key in current.keys():
current.pop(key)
filtered.append((generator(variant, args.vendor_name), dict(current)))
return filtered
@@ -241,30 +250,34 @@ def warmup(client):
def tail_latency(vendor, client, func):
vendor.start_benchmark("tail_latency")
if args.warmup_run:
warmup(client)
latency = []
iteration = args.tail_latency
query_list = get_queries(func, iteration)
for i in range(0, iteration):
ret = client.execute(queries=[query_list[i]], num_workers=1)
latency.append(ret[0]["duration"])
latency.sort()
query_stats = {
"iterations": iteration,
"min": latency[0],
"max": latency[iteration - 1],
"mean": statistics.mean(latency),
"p99": latency[math.floor(iteration * 0.99) - 1],
"p95": latency[math.floor(iteration * 0.95) - 1],
"p90": latency[math.floor(iteration * 0.90) - 1],
"p75": latency[math.floor(iteration * 0.75) - 1],
"p50": latency[math.floor(iteration * 0.50) - 1],
}
print("Query statistics for tail latency: ")
print(query_stats)
vendor.stop("tail_latency")
if iteration >= 10:
vendor.start_benchmark("tail_latency")
if args.warmup_run:
warmup(client)
latency = []
query_list = get_queries(func, iteration)
for i in range(0, iteration):
ret = client.execute(queries=[query_list[i]], num_workers=1)
latency.append(ret[0]["duration"])
latency.sort()
query_stats = {
"iterations": iteration,
"min": latency[0],
"max": latency[iteration - 1],
"mean": statistics.mean(latency),
"p99": latency[math.floor(iteration * 0.99) - 1],
"p95": latency[math.floor(iteration * 0.95) - 1],
"p90": latency[math.floor(iteration * 0.90) - 1],
"p75": latency[math.floor(iteration * 0.75) - 1],
"p50": latency[math.floor(iteration * 0.50) - 1],
}
print("Query statistics for tail latency: ")
print(query_stats)
vendor.stop("tail_latency")
else:
query_stats = {}
return query_stats

View File

@@ -147,6 +147,8 @@ def run_full_benchmarks(vendor, binary, dataset_size, dataset_group, realistic,
"12",
"--no-authorization",
"pokec/" + dataset_size + "/" + dataset_group + "/*",
"--tail-latency",
"100",
]
for config in configurations:

View File

@@ -253,3 +253,31 @@ TEST_F(ReadWriteTypeCheckTest, Foreach) {
std::shared_ptr<LogicalOperator> foreach = std::make_shared<plan::Foreach>(nullptr, nullptr, nullptr, x);
CheckPlanType(foreach.get(), RWType::RW);
}
TEST_F(ReadWriteTypeCheckTest, CheckUpdateType) {
std::array<std::array<RWType, 3>, 16> scenarios = {{
{RWType::NONE, RWType::NONE, RWType::NONE},
{RWType::NONE, RWType::R, RWType::R},
{RWType::NONE, RWType::W, RWType::W},
{RWType::NONE, RWType::RW, RWType::RW},
{RWType::R, RWType::NONE, RWType::R},
{RWType::R, RWType::R, RWType::R},
{RWType::R, RWType::W, RWType::RW},
{RWType::R, RWType::RW, RWType::RW},
{RWType::W, RWType::NONE, RWType::W},
{RWType::W, RWType::R, RWType::RW},
{RWType::W, RWType::W, RWType::W},
{RWType::W, RWType::RW, RWType::RW},
{RWType::RW, RWType::NONE, RWType::RW},
{RWType::RW, RWType::R, RWType::RW},
{RWType::RW, RWType::W, RWType::RW},
{RWType::RW, RWType::RW, RWType::RW},
}};
auto rw_type_checker = ReadWriteTypeChecker();
for (auto scenario : scenarios) {
rw_type_checker.type = scenario[0];
rw_type_checker.UpdateType(scenario[1]);
EXPECT_EQ(scenario[2], rw_type_checker.type);
}
}

View File

@@ -252,20 +252,21 @@ TEST(PoolResource, MultipleSmallBlockAllocations) {
// NOLINTNEXTLINE(hicpp-special-member-functions)
TEST(PoolResource, BigBlockAllocations) {
TestMemory test_mem;
TestMemory test_mem_unpooled;
const size_t max_blocks_per_chunk = 3U;
const size_t max_block_size = 64U;
memgraph::utils::PoolResource mem(max_blocks_per_chunk, max_block_size, &test_mem);
memgraph::utils::PoolResource mem(max_blocks_per_chunk, max_block_size, &test_mem, &test_mem_unpooled);
CheckAllocation(&mem, max_block_size + 1, 1U);
// May allocate more than once per block due to bookkeeping.
EXPECT_GE(test_mem.new_count_, 1U);
EXPECT_GE(test_mem_unpooled.new_count_, 1U);
CheckAllocation(&mem, max_block_size + 1, 1U);
EXPECT_GE(test_mem.new_count_, 2U);
EXPECT_GE(test_mem_unpooled.new_count_, 2U);
auto *ptr = CheckAllocation(&mem, max_block_size * 2, 1U);
EXPECT_GE(test_mem.new_count_, 3U);
EXPECT_GE(test_mem_unpooled.new_count_, 3U);
mem.Deallocate(ptr, max_block_size * 2, 1U);
EXPECT_GE(test_mem.delete_count_, 1U);
EXPECT_GE(test_mem_unpooled.delete_count_, 1U);
mem.Release();
EXPECT_GE(test_mem.delete_count_, 3U);
EXPECT_GE(test_mem_unpooled.delete_count_, 3U);
CheckAllocation(&mem, max_block_size + 1, 1U);
}