Compare commits

..

4 Commits

Author SHA1 Message Date
gvolfing
2faa58fe21 restructure test 2022-09-29 08:05:23 +02:00
János Benjamin Antal
40b2e12eaf Use include from bindings instead of directly from expr 2022-09-27 11:30:40 +02:00
gvolfing
0e613a1ff9 Fix expression test 2022-09-23 19:56:00 +02:00
jbajic
4b5cbb99b6 Test out expr in storage 2022-09-23 14:14:03 +02:00
11 changed files with 274 additions and 2674 deletions

View File

@@ -1,21 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
namespace memgraph::io {
/// Protocol:
/// crc32: 4 bytes
/// len: 8 bytes
/// buffer: <len bytes>
std::optional<std::pair<size_t, size_t>> make_frame(char *ptr, size_t len) { return std::nullopt; }
} // namespace memgraph::io

View File

@@ -1,61 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/address.hpp"
#include "io/message_conversion.hpp"
#include "io/protobuf_transport/protobuf_transport_handle.hpp"
#include "io/transport.hpp"
namespace memgraph::io::protobuf_transport {
class ProtobufTransport {
std::shared_ptr<ProtobufTransportHandle> protobuf_transport_handle_;
uint16_t listen_port_;
public:
explicit ProtobufTransport(uint16_t listen_port)
: protobuf_transport_handle_(std::make_unique<ProtobufTransportHandle>()), listen_port_(listen_port) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
Duration timeout) {
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
protobuf_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
return protobuf_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
return protobuf_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
}
Time Now() const { return protobuf_transport_handle_->Now(); }
bool ShouldShutDown() const { return protobuf_transport_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::random_device rng;
return distrib(rng);
}
};
}; // namespace memgraph::io::protobuf_transport

View File

@@ -1,161 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <map>
#include <mutex>
#include "google/protobuf/message.h"
#include "io/address.hpp"
#include "io/transport.hpp"
//#include "protobuf/messages.pb.cc"
#include "protobuf/messages.pb.h"
namespace memgraph::io::protobuf_transport {
using PbAddress = memgraph::protobuf::Address;
using memgraph::protobuf::UberMessage;
class ProtobufTransportHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
bool should_shut_down_ = false;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::vector<std::string> can_receive_;
// serialized outbound messages
std::vector<std::string> outbox_;
public:
~ProtobufTransportHandle() {
for (auto &&[pk, promise] : promises_) {
std::move(promise.promise).TimeOut();
}
promises_.clear();
}
void ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
static Time Now() {
auto nano_time = std::chrono::system_clock::now();
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
std::unique_lock lock(mu_);
Time before = Now();
spdlog::info("can_receive_ size: {}", can_receive_.size());
while (can_receive_.empty()) {
Time now = Now();
// protection against non-monotonic timesources
auto maxed_now = std::max(now, before);
auto elapsed = maxed_now - before;
if (timeout < elapsed) {
return TimedOut{};
}
Duration relative_timeout = timeout - elapsed;
std::cv_status cv_status_value = cv_.wait_for(lock, relative_timeout);
if (cv_status_value == std::cv_status::timeout) {
return TimedOut{};
}
}
auto current_message = std::move(can_receive_.back());
can_receive_.pop_back();
auto m_opt = std::move(current_message).Take<Ms...>();
return std::move(m_opt).value();
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
PromiseKey promise_key{.requester_address = to_address, .request_id = request_id, .replier_address = from_address};
{
std::unique_lock<std::mutex> lock(mu_);
if (promises_.contains(promise_key)) {
// hair-pin local message optimization
spdlog::info("using message to fill local promise");
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
std::any message_any(std::forward<M>(message));
OpaqueMessage opaque_message{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any)};
dop.promise.Fill(std::move(opaque_message));
} else {
spdlog::info("placing message in outbox");
// serialize protobuf message and place it in the outbox
std::string bytes;
bool success = message.SerializeToString(&bytes);
MG_ASSERT(success);
outbox_.emplace_back(std::move(bytes));
}
} // lock dropped
cv_.notify_all();
}
template <Message RequestT, Message ResponseT>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
Duration timeout, ResponsePromise<ResponseT> promise) {
const Time deadline = Now() + timeout;
{
std::unique_lock<std::mutex> lock(mu_);
PromiseKey promise_key{
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
} // lock dropped
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
}
};
} // namespace memgraph::io::protobuf_transport

View File

@@ -1,25 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <map>
#include <mutex>
#include "google/protobuf/message.h"
#include "io/address.hpp"
#include "io/transport.hpp"
namespace memgraph::io::protobuf_transport {}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,24 +0,0 @@
syntax = "proto3";
package memgraph.protobuf;
message Address {
bytes unique_id = 1;
// this is either 4 bytes for ipv4 or 16 bytes for ipv6
bytes last_known_ip = 2;
uint32 last_known_port = 3;
}
message TestRequest {
uint64 request_id = 1;
string content = 2;
}
message UberMessage {
uint64 request_id = 1;
Address to_address = 2;
Address from_address = 3;
oneof specific_message {
TestRequest test_request = 4;
}
}

View File

@@ -434,6 +434,6 @@ target_link_libraries(${test_prefix}local_transport mg-io)
add_unit_test(machine_manager.cpp)
target_link_libraries(${test_prefix}machine_manager mg-io mg-coordinator mg-storage-v3)
# Test ProtobufTransport
add_unit_test(protobuf_transport.cpp)
target_link_libraries(${test_prefix}protobuf_transport mg-io protobuf)
# Test ExpressionEvaluator with expressions
add_unit_test(storage_v3_expr_usage.cpp)
target_link_libraries(${test_prefix}storage_v3_expr_usage mg-io mg-expr mg-storage-v3)

View File

@@ -15,17 +15,18 @@
#include <gtest/gtest.h>
#include "coordinator/coordinator.hpp"
#include "coordinator/coordinator_client.hpp"
#include "coordinator/hybrid_logical_clock.hpp"
#include "coordinator/shard_map.hpp"
#include "io/local_transport/local_system.hpp"
#include "io/local_transport/local_transport.hpp"
#include <coordinator/coordinator.hpp>
#include <coordinator/coordinator_client.hpp>
#include <coordinator/hybrid_logical_clock.hpp>
#include <coordinator/shard_map.hpp>
#include <io/local_transport/local_system.hpp>
#include <io/local_transport/local_transport.hpp>
#include <io/rsm/rsm_client.hpp>
#include <io/transport.hpp>
#include <machine_manager/machine_config.hpp>
#include <machine_manager/machine_manager.hpp>
#include <query/v2/requests.hpp>
#include "io/rsm/rsm_client.hpp"
#include "io/transport.hpp"
#include "machine_manager/machine_config.hpp"
#include "machine_manager/machine_manager.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/schemas.hpp"

View File

@@ -1,85 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <chrono>
#include <limits>
#include <string>
#include <thread>
#include <gtest/gtest.h>
#include "io/address.hpp"
#include "io/protobuf_transport/protobuf_transport.hpp"
#include "io/transport.hpp"
#include "protobuf/messages.pb.cc"
#include "protobuf/messages.pb.h"
namespace memgraph::io::tests {
using memgraph::io::protobuf_transport::ProtobufTransport;
using MgAddress = memgraph::io::Address;
using PbAddress = memgraph::protobuf::Address;
using memgraph::protobuf::TestRequest;
using memgraph::protobuf::UberMessage;
TEST(ProtobufTransport, Echo) {
uint16_t cli_port = 6000;
uint16_t srv_port = 7000;
MgAddress cli_addr = MgAddress::TestAddress(cli_port);
MgAddress srv_addr = MgAddress::TestAddress(srv_port);
ProtobufTransport cli_pt{cli_port};
ProtobufTransport srv_pt{srv_port};
Io<ProtobufTransport> cli_io{cli_pt, cli_addr};
Io<ProtobufTransport> srv_io{srv_pt, srv_addr};
auto response_result_future = cli_io.Request<UberMessage, UberMessage>(srv_addr, UberMessage{});
auto request_result = srv_io.Receive<UberMessage>();
auto request_envelope = request_result.GetValue();
UberMessage request = std::get<UberMessage>(request_envelope.message);
// send it back as an echo
srv_io.Send(request_envelope.from_address, request_envelope.request_id, request);
// client receives it
auto response_result = std::move(response_result_future).Wait();
auto response_envelope = response_result.GetValue();
UberMessage response = response_envelope.message;
PbAddress to_addr;
to_addr.set_last_known_port(1);
PbAddress from_addr;
to_addr.set_last_known_port(2);
auto req = new TestRequest{};
req->set_content("ping");
UberMessage um;
um.set_request_id(1);
um.set_allocated_test_request(req);
std::string out;
bool success = um.SerializeToString(&out);
MG_ASSERT(success);
TestRequest rt;
rt.ParseFromString(out);
}
} // namespace memgraph::io::tests

View File

@@ -0,0 +1,260 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <chrono>
#include <limits>
#include <thread>
#include <gtest/gtest.h>
#include <coordinator/coordinator.hpp>
#include <coordinator/coordinator_client.hpp>
#include <coordinator/hybrid_logical_clock.hpp>
#include <coordinator/shard_map.hpp>
#include <io/local_transport/local_system.hpp>
#include <io/local_transport/local_transport.hpp>
#include <io/rsm/rsm_client.hpp>
#include <io/transport.hpp>
#include <machine_manager/machine_config.hpp>
#include <machine_manager/machine_manager.hpp>
#include "common/types.hpp"
#include "exceptions.hpp"
#include "io/rsm/rsm_client.hpp"
#include "parser/opencypher/parser.hpp"
#include "storage/v3/bindings/ast/ast.hpp"
#include "storage/v3/bindings/bindings.hpp"
#include "storage/v3/bindings/cypher_main_visitor.hpp"
#include "storage/v3/bindings/db_accessor.hpp"
#include "storage/v3/bindings/eval.hpp"
#include "storage/v3/bindings/frame.hpp"
#include "storage/v3/bindings/symbol_generator.hpp"
#include "storage/v3/bindings/symbol_table.hpp"
#include "storage/v3/bindings/typed_value.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/key_store.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/shard.hpp"
#include "utils/string.hpp"
namespace memgraph::storage::v3::test {
class ExpressionEvaluatorUsageTest : public ::testing::Test {
protected:
LabelId primary_label{LabelId::FromInt(1)};
PropertyId primary_property{PropertyId::FromInt(2)};
PrimaryKey min_pk{PropertyValue(0)};
Shard db{primary_label, min_pk, std::nullopt};
Shard::Accessor storage_dba{db.Access(GetNextHlc())};
DbAccessor dba{&storage_dba};
AstStorage storage;
memgraph::utils::MonotonicBufferResource mem{1024};
EvaluationContext ctx{.memory = &mem};
SymbolTable symbol_table_;
Frame frame_{128};
ExpressionEvaluator eval{&frame_, symbol_table_, ctx, &dba, View::NEW};
coordinator::Hlc last_hlc{0, io::Time{}};
void SetUp() override {
db.StoreMapping({{1, "label"}, {2, "property"}});
ASSERT_TRUE(
db.CreateSchema(primary_label, {storage::v3::SchemaProperty{primary_property, common::SchemaType::INT}}));
}
std::vector<PropertyId> NamesToProperties(const std::vector<std::string> &property_names) {
std::vector<PropertyId> properties;
properties.reserve(property_names.size());
for (const auto &name : property_names) {
properties.push_back(dba.NameToProperty(name));
}
return properties;
}
std::vector<LabelId> NamesToLabels(const std::vector<std::string> &label_names) {
std::vector<LabelId> labels;
labels.reserve(label_names.size());
for (const auto &name : label_names) {
labels.push_back(dba.NameToLabel(name));
}
return labels;
}
Identifier *CreateIdentifierWithValue(std::string name, const TypedValue &value) {
auto *id = storage.Create<Identifier>(name, true);
auto symbol = symbol_table_.CreateSymbol(name, true);
id->MapTo(symbol);
frame_[symbol] = value;
return id;
}
template <class TExpression>
auto Eval(TExpression *expr) {
ctx.properties = NamesToProperties(storage.properties_);
ctx.labels = NamesToLabels(storage.labels_);
auto value = expr->Accept(eval);
EXPECT_EQ(value.GetMemoryResource(), &mem) << "ExpressionEvaluator must use the MemoryResource from "
"EvaluationContext for allocations!";
return value;
}
coordinator::Hlc GetNextHlc() {
++last_hlc.logical_id;
last_hlc.coordinator_wall_clock += std::chrono::seconds(1);
return last_hlc;
}
};
class StrippedQuery {
public:
/**
* Strips the input query and stores stripped query, stripped arguments and
* stripped query hash.
*
* @param query Input query.
*/
explicit StrippedQuery(const std::string &query);
/**
* Copy constructor is deleted because we don't want to make unnecessary
* copies of this object (copying of string and vector could be expensive)
*/
StrippedQuery(const StrippedQuery &other) = delete;
StrippedQuery &operator=(const StrippedQuery &other) = delete;
/**
* Move is allowed operation because it is not expensive and we can
* move the object after it was created.
*/
StrippedQuery(StrippedQuery &&other) = default;
StrippedQuery &operator=(StrippedQuery &&other) = default;
const std::string &query() const { return query_; }
const auto &original_query() const { return original_; }
const auto &literals() const { return literals_; }
const auto &named_expressions() const { return named_exprs_; }
const auto &parameters() const { return parameters_; }
uint64_t hash() const { return hash_; }
private:
// Return len of matched keyword if something is matched, otherwise 0.
int MatchKeyword(int start) const;
int MatchString(int start) const;
int MatchSpecial(int start) const;
int MatchDecimalInt(int start) const;
int MatchOctalInt(int start) const;
int MatchHexadecimalInt(int start) const;
int MatchReal(int start) const;
int MatchParameter(int start) const;
int MatchEscapedName(int start) const;
int MatchUnescapedName(int start) const;
int MatchWhitespaceAndComments(int start) const;
// Original query.
std::string original_;
// Stripped query.
std::string query_;
// Token positions of stripped out literals mapped to their values.
// TODO: Parameters class really doesn't provide anything interesting. This
// could be changed to std::unordered_map, but first we need to rewrite (or
// get rid of) hardcoded queries which expect Parameters.
Parameters literals_;
// Token positions of query parameters mapped to their names.
std::unordered_map<int, std::string> parameters_;
// Token positions of nonaliased named expressions in return statement mapped
// to their original (unstripped) string.
std::unordered_map<int, std::string> named_exprs_;
// Hash based on the stripped query.
uint64_t hash_;
};
TEST_F(ExpressionEvaluatorUsageTest, DummyExample) {
// dummy
const std::string expr1{"2+1-3+2"};
// Parse stuff
memgraph::frontend::opencypher::Parser<frontend::opencypher::ParserOpTag::EXPRESSION> parser(expr1);
expr::ParsingContext pc;
CypherMainVisitor visitor(pc, &storage);
auto *ast = parser.tree();
auto ladida = visitor.visit(ast);
auto res1 = Eval(std::any_cast<Expression *>(ladida));
EXPECT_EQ(res1.ValueInt(), 2);
}
TEST_F(ExpressionEvaluatorUsageTest, PropertyLookup) {
db.StoreMapping({{1, "label"}, {2, "property"}, {3, "prop2"}});
const auto prop2 = PropertyId::FromUint(3);
auto v1 = *dba.InsertVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(1)}});
ASSERT_TRUE(v1.SetPropertyAndValidate(prop2, PropertyValue(5)).HasValue());
auto v2 = *dba.InsertVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(2)}});
ASSERT_TRUE(v2.SetPropertyAndValidate(prop2, PropertyValue(5)).HasValue());
auto v3 = *dba.InsertVertexAndValidate(primary_label, {}, {{primary_property, PropertyValue(3)}});
ASSERT_TRUE(v3.SetPropertyAndValidate(prop2, PropertyValue(5)).HasValue());
// Property filtering
// const std::string expr1{"n.prop2 > 0"};
const std::string expr1{"node.prop2 > 0 AND node.prop2 < 10"};
// Parse stuff
memgraph::frontend::opencypher::Parser<memgraph::frontend::opencypher::ParserOpTag::EXPRESSION> parser(expr1);
expr::ParsingContext pc;
CypherMainVisitor visitor(pc, &storage);
auto *ast = parser.tree();
auto expr = visitor.visit(ast);
static constexpr const char *node_name = "node";
static constexpr const char *edge_name = "edge";
static Identifier node_identifier = Identifier(std::string(node_name), false);
bool is_node_identifier_present = false;
static Identifier edge_identifier = Identifier(std::string(edge_name), false);
bool is_edge_identifier_present = false;
std::vector<Identifier *> identifiers;
if (expr1.find(node_name) != std::string::npos) {
is_node_identifier_present = true;
identifiers.push_back(&node_identifier);
}
if (expr1.find(edge_name) != std::string::npos) {
is_edge_identifier_present = true;
identifiers.push_back(&edge_identifier);
}
expr::SymbolGenerator symbol_generator(&symbol_table_, identifiers);
(std::any_cast<Expression *>(expr))->Accept(symbol_generator);
if (is_node_identifier_present) {
frame_[symbol_table_.at(node_identifier)] = v2; // vertex accessor
}
if (is_edge_identifier_present) {
frame_[symbol_table_.at(edge_identifier)] = v1; // edge accessor
}
auto res1 = Eval(std::any_cast<Expression *>(expr));
ASSERT_TRUE(res1.ValueBool());
}
} // namespace memgraph::storage::v3::test