Compare commits
7 Commits
T625-memgr
...
MG-fix-pro
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a29cd40b40 | ||
|
|
67a8263869 | ||
|
|
f046262f0d | ||
|
|
e9fbb7b40b | ||
|
|
d73f5fb781 | ||
|
|
8ef51b876e | ||
|
|
533161d43a |
@@ -279,7 +279,7 @@ if (MG_ENTERPRISE)
|
||||
add_definitions(-DMG_ENTERPRISE)
|
||||
endif()
|
||||
|
||||
set(ENABLE_JEMALLOC ON)
|
||||
set(ENABLE_JEMALLOC OFF)
|
||||
|
||||
if (ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
|
||||
@@ -666,16 +666,11 @@ class InListOperator : public memgraph::query::BinaryOperator {
|
||||
return object;
|
||||
}
|
||||
|
||||
void SetCachedSet(std::unordered_set<size_t> &&set_) { _cached_set.emplace(set_); }
|
||||
|
||||
std::unordered_set<size_t> *GetCachedSet() { return _cached_set.has_value() ? &*_cached_set : nullptr; }
|
||||
|
||||
protected:
|
||||
using BinaryOperator::BinaryOperator;
|
||||
|
||||
private:
|
||||
friend class AstStorage;
|
||||
std::optional<std::unordered_set<size_t>> _cached_set{std::nullopt};
|
||||
};
|
||||
|
||||
class SubscriptOperator : public memgraph::query::BinaryOperator {
|
||||
@@ -2938,7 +2933,6 @@ class LoadCsv : public memgraph::query::Clause {
|
||||
memgraph::query::Expression *file_;
|
||||
bool with_header_;
|
||||
bool ignore_bad_;
|
||||
bool ignore_empty_strings_;
|
||||
memgraph::query::Expression *delimiter_{nullptr};
|
||||
memgraph::query::Expression *quote_{nullptr};
|
||||
memgraph::query::Identifier *row_var_{nullptr};
|
||||
@@ -2948,7 +2942,6 @@ class LoadCsv : public memgraph::query::Clause {
|
||||
object->file_ = file_ ? file_->Clone(storage) : nullptr;
|
||||
object->with_header_ = with_header_;
|
||||
object->ignore_bad_ = ignore_bad_;
|
||||
object->ignore_empty_strings_ = ignore_empty_strings_;
|
||||
object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr;
|
||||
object->quote_ = quote_ ? quote_->Clone(storage) : nullptr;
|
||||
object->row_var_ = row_var_ ? row_var_->Clone(storage) : nullptr;
|
||||
@@ -2956,12 +2949,11 @@ class LoadCsv : public memgraph::query::Clause {
|
||||
}
|
||||
|
||||
protected:
|
||||
explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, bool ignore_empty_strings,
|
||||
Expression *delimiter, Expression *quote, Identifier *row_var)
|
||||
explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, Expression *delimiter, Expression *quote,
|
||||
Identifier *row_var)
|
||||
: file_(file),
|
||||
with_header_(with_header),
|
||||
ignore_bad_(ignore_bad),
|
||||
ignore_empty_strings_(ignore_empty_strings),
|
||||
delimiter_(delimiter),
|
||||
quote_(quote),
|
||||
row_var_(row_var) {
|
||||
|
||||
@@ -357,8 +357,6 @@ antlrcpp::Any CypherMainVisitor::visitLoadCsv(MemgraphCypher::LoadCsvContext *ct
|
||||
// handle skip bad row option
|
||||
load_csv->ignore_bad_ = ctx->IGNORE() && ctx->BAD();
|
||||
|
||||
load_csv->ignore_empty_strings_ = ctx->IGNORE_EMPTY_STRINGS();
|
||||
|
||||
// handle delimiter
|
||||
if (ctx->DELIMITER()) {
|
||||
if (ctx->delimiter()->literal()->StringLiteral()) {
|
||||
|
||||
@@ -58,7 +58,6 @@ memgraphCypherKeyword : cypherKeyword
|
||||
| GRANT
|
||||
| HEADER
|
||||
| IDENTIFIED
|
||||
| IGNORE_EMPTY_STRINGS
|
||||
| ISOLATION
|
||||
| IN_MEMORY_ANALYTICAL
|
||||
| IN_MEMORY_TRANSACTIONAL
|
||||
@@ -223,7 +222,6 @@ loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER
|
||||
( IGNORE BAD ) ?
|
||||
( DELIMITER delimiter ) ?
|
||||
( QUOTE quote ) ?
|
||||
( IGNORE_EMPTY_STRINGS ) ?
|
||||
AS rowVar ;
|
||||
|
||||
csvFile : literal ;
|
||||
|
||||
@@ -68,7 +68,7 @@ GRAPH : G R A P H ;
|
||||
GRANTS : G R A N T S ;
|
||||
HEADER : H E A D E R ;
|
||||
IDENTIFIED : I D E N T I F I E D ;
|
||||
IGNORE_EMPTY_STRINGS : I G N O R E UNDERSCORE E M P T Y UNDERSCORE S T R I N G S;
|
||||
IGNORE : I G N O R E ;
|
||||
ISOLATION : I S O L A T I O N ;
|
||||
IN_MEMORY_ANALYTICAL : I N UNDERSCORE M E M O R Y UNDERSCORE A N A L Y T I C A L ;
|
||||
IN_MEMORY_TRANSACTIONAL : I N UNDERSCORE M E M O R Y UNDERSCORE T R A N S A C T I O N A L ;
|
||||
|
||||
@@ -190,27 +190,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
|
||||
TypedValue Visit(InListOperator &in_list) override {
|
||||
ReferenceExpressionEvaluator reference_expression_evaluator{frame_, symbol_table_, ctx_};
|
||||
|
||||
TypedValue *_list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
|
||||
TypedValue _list;
|
||||
|
||||
if (nullptr == _list_ptr) {
|
||||
_list = in_list.expression2_->Accept(*this);
|
||||
_list_ptr = &_list;
|
||||
}
|
||||
|
||||
auto literal = in_list.expression1_->Accept(*this);
|
||||
|
||||
if (_list_ptr->IsNull()) {
|
||||
auto _list = in_list.expression2_->Accept(*this);
|
||||
if (_list.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
// Exceptions have higher priority than returning nulls when list expression
|
||||
// is not null.
|
||||
if (_list_ptr->type() != TypedValue::Type::List) {
|
||||
if (_list.type() != TypedValue::Type::List) {
|
||||
throw QueryRuntimeException("IN expected a list, got {}.", _list.type());
|
||||
}
|
||||
const auto &list = _list_ptr->ValueList();
|
||||
const auto &list = _list.ValueList();
|
||||
|
||||
// If literal is NULL there is no need to try to compare it with every
|
||||
// element in the list since result of every comparison will be NULL. There
|
||||
@@ -219,24 +209,16 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
if (list.empty()) return TypedValue(false, ctx_->memory);
|
||||
if (literal.IsNull()) return TypedValue(ctx_->memory);
|
||||
|
||||
if (in_list.GetCachedSet() == nullptr) {
|
||||
std::unordered_set<size_t> _cached_set;
|
||||
TypedValue::Hash hash{};
|
||||
for (const TypedValue &element : list) {
|
||||
_cached_set.insert(hash(element));
|
||||
auto has_null = false;
|
||||
for (const auto &element : list) {
|
||||
auto result = literal == element;
|
||||
if (result.IsNull()) {
|
||||
has_null = true;
|
||||
} else if (result.ValueBool()) {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
|
||||
in_list.SetCachedSet(std::move(_cached_set));
|
||||
}
|
||||
|
||||
const auto &in_list_cached_set = in_list.GetCachedSet();
|
||||
|
||||
TypedValue::Hash hash{};
|
||||
if (in_list_cached_set->contains(hash(literal))) {
|
||||
return TypedValue(true, ctx_->memory);
|
||||
}
|
||||
// has null
|
||||
if (literal.type() == TypedValue::Type::Null || in_list_cached_set->contains(hash(TypedValue(ctx_->memory)))) {
|
||||
if (has_null) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
return TypedValue(false, ctx_->memory);
|
||||
|
||||
@@ -1256,7 +1256,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
contains_csv = true;
|
||||
}
|
||||
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
|
||||
auto use_monotonic_memory = !contains_csv;
|
||||
auto use_monotonic_memory = false;
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
|
||||
parsed_query.parameters,
|
||||
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
|
||||
@@ -1385,7 +1385,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
contains_csv = true;
|
||||
}
|
||||
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
|
||||
auto use_monotonic_memory = !contains_csv;
|
||||
auto use_monotonic_memory = false;
|
||||
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
|
||||
Frame frame(0);
|
||||
@@ -2743,12 +2743,12 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
}
|
||||
|
||||
if (const auto &clauses = cypher_query->single_query_->clauses_;
|
||||
std::any_of(clauses.begin(), clauses.end(),
|
||||
[](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) {
|
||||
true || std::any_of(clauses.begin(), clauses.end(),
|
||||
[](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) {
|
||||
// Using PoolResource without MonotonicMemoryResouce for LOAD CSV reduces memory usage.
|
||||
// QueryExecution MemoryResource is mostly used for allocations done on Frame and storing `row`s
|
||||
query_executions_[query_executions_.size() - 1] = std::make_unique<QueryExecution>(
|
||||
utils::PoolResource(8, kExecutionPoolMaxBlockSize, utils::NewDeleteResource(), utils::NewDeleteResource()));
|
||||
query_executions_[query_executions_.size() - 1] = std::make_unique<QueryExecution>(utils::PoolResource(
|
||||
128, kExecutionPoolMaxBlockSize, utils::NewDeleteResource(), utils::NewDeleteResource()));
|
||||
query_execution_ptr = &query_executions_.back();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@
|
||||
#include "utils/likely.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/pmr/deque.hpp"
|
||||
#include "utils/pmr/list.hpp"
|
||||
#include "utils/pmr/unordered_map.hpp"
|
||||
@@ -4528,7 +4529,14 @@ class CallProcedureCursor : public Cursor {
|
||||
// evaluation of an expression.
|
||||
// TODO: This will probably need to be changed when we add support for
|
||||
// generator like procedures which yield a new result on each invocation.
|
||||
|
||||
// 1. problem je sto se koristi ova memorija
|
||||
// 2. problem je sto je to memorija od EvalContexta
|
||||
// 3. problem je sto se koristi i memorija od QueryExecutiona
|
||||
|
||||
// utils::PoolResource poolResource{128, 1024, utils::NewDeleteResource(), utils::NewDeleteResource()};
|
||||
auto *memory = context.evaluation_context.memory;
|
||||
// utils::OnScopeExit scope_exit([&poolResource] { poolResource.Release(); });
|
||||
auto memory_limit = EvaluateMemoryLimit(&evaluator, self_->memory_limit_, self_->memory_scale_);
|
||||
auto graph = mgp_graph::WritableGraph(*context.db_accessor, graph_view, context);
|
||||
CallCustomProcedure(self_->procedure_name_, *proc, self_->arguments_, graph, &evaluator, memory, memory_limit,
|
||||
@@ -4587,19 +4595,18 @@ UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const {
|
||||
}
|
||||
|
||||
LoadCsv::LoadCsv(std::shared_ptr<LogicalOperator> input, Expression *file, bool with_header, bool ignore_bad,
|
||||
bool ignore_empty_strings, Expression *delimiter, Expression *quote, Symbol row_var)
|
||||
Expression *delimiter, Expression *quote, Symbol row_var)
|
||||
: input_(input ? input : (std::make_shared<Once>())),
|
||||
file_(file),
|
||||
with_header_(with_header),
|
||||
ignore_bad_(ignore_bad),
|
||||
ignore_empty_strings_(ignore_empty_strings),
|
||||
delimiter_(delimiter),
|
||||
quote_(quote),
|
||||
row_var_(row_var) {
|
||||
MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__);
|
||||
}
|
||||
|
||||
ACCEPT_WITH_INPUT(LoadCsv)
|
||||
bool LoadCsv::Accept(HierarchicalLogicalOperatorVisitor &visitor) { return false; };
|
||||
|
||||
class LoadCsvCursor;
|
||||
|
||||
@@ -4625,30 +4632,22 @@ auto ToOptionalString(ExpressionEvaluator *evaluator, Expression *expression) ->
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
TypedValue CsvRowToTypedList(csv::Reader::Row &row, bool ignore_empty_strings = false) {
|
||||
TypedValue CsvRowToTypedList(csv::Reader::Row &row) {
|
||||
auto *mem = row.get_allocator().GetMemoryResource();
|
||||
auto typed_columns = utils::pmr::vector<TypedValue>(mem);
|
||||
typed_columns.reserve(row.size());
|
||||
for (auto &column : row) {
|
||||
if (!ignore_empty_strings || column.empty()) {
|
||||
typed_columns.emplace_back(std::move(column));
|
||||
} else {
|
||||
typed_columns.emplace_back(mem);
|
||||
}
|
||||
typed_columns.emplace_back(std::move(column));
|
||||
}
|
||||
return {std::move(typed_columns), mem};
|
||||
}
|
||||
|
||||
TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header, bool ignore_empty_strings = false) {
|
||||
TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header) {
|
||||
// a valid row has the same number of elements as the header
|
||||
auto *mem = row.get_allocator().GetMemoryResource();
|
||||
utils::pmr::map<utils::pmr::string, TypedValue> m(mem);
|
||||
for (auto i = 0; i < row.size(); ++i) {
|
||||
if (!ignore_empty_strings || !row[i].empty()) {
|
||||
m.emplace(std::move(header[i]), std::move(row[i]));
|
||||
} else {
|
||||
m.emplace(std::move(header[i]), mem);
|
||||
}
|
||||
m.emplace(std::move(header[i]), std::move(row[i]));
|
||||
}
|
||||
return {std::move(m), mem};
|
||||
}
|
||||
@@ -4658,12 +4657,14 @@ TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header, b
|
||||
class LoadCsvCursor : public Cursor {
|
||||
const LoadCsv *self_;
|
||||
const UniqueCursorPtr input_cursor_;
|
||||
bool did_pull_;
|
||||
bool input_is_once_;
|
||||
std::optional<csv::Reader> reader_{};
|
||||
|
||||
public:
|
||||
LoadCsvCursor(const LoadCsv *self, utils::MemoryResource *mem)
|
||||
: self_(self), input_cursor_(self_->input_->MakeCursor(mem)), did_pull_{false} {}
|
||||
: self_(self), input_cursor_(self_->input_->MakeCursor(mem)) {
|
||||
input_is_once_ = dynamic_cast<Once *>(self_->input_.get());
|
||||
}
|
||||
|
||||
bool Pull(Frame &frame, ExecutionContext &context) override {
|
||||
SCOPED_PROFILE_OP("LoadCsv");
|
||||
@@ -4681,14 +4682,12 @@ class LoadCsvCursor : public Cursor {
|
||||
|
||||
bool input_pulled = input_cursor_->Pull(frame, context);
|
||||
|
||||
if (input_pulled) {
|
||||
if (did_pull_) {
|
||||
throw QueryRuntimeException(
|
||||
"LOAD CSV can be executed only once, please check if the cardinality of the operator before LOAD CSV is 1");
|
||||
}
|
||||
did_pull_ = true;
|
||||
}
|
||||
|
||||
// If the input is Once, we have to keep going until we read all the rows,
|
||||
// regardless of whether the pull on Once returned false.
|
||||
// If we have e.g. MATCH(n) LOAD CSV ... AS x SET n.name = x.name, then we
|
||||
// have to read at most cardinality(n) rows (but we can read less and stop
|
||||
// pulling MATCH).
|
||||
if (!input_is_once_ && !input_pulled) return false;
|
||||
auto row = reader_->GetNextRow(context.evaluation_context.memory);
|
||||
if (!row) {
|
||||
return false;
|
||||
@@ -4697,8 +4696,7 @@ class LoadCsvCursor : public Cursor {
|
||||
frame[self_->row_var_] = CsvRowToTypedList(*row);
|
||||
} else {
|
||||
frame[self_->row_var_] =
|
||||
CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory),
|
||||
self_->ignore_empty_strings_);
|
||||
CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -4722,10 +4720,10 @@ class LoadCsvCursor : public Cursor {
|
||||
// Note that the reader has to be given its own memory resource, as it
|
||||
// persists between pulls, so it can't use the evalutation context memory
|
||||
// resource.
|
||||
return csv::Reader(*maybe_file,
|
||||
csv::Reader::Config(self_->with_header_, self_->ignore_bad_, self_->ignore_empty_strings_,
|
||||
std::move(maybe_delim), std::move(maybe_quote)),
|
||||
utils::NewDeleteResource());
|
||||
return csv::Reader(
|
||||
*maybe_file,
|
||||
csv::Reader::Config(self_->with_header_, self_->ignore_bad_, std::move(maybe_delim), std::move(maybe_quote)),
|
||||
utils::NewDeleteResource());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -2227,7 +2227,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator {
|
||||
|
||||
LoadCsv() = default;
|
||||
LoadCsv(std::shared_ptr<LogicalOperator> input, Expression *file, bool with_header, bool ignore_bad,
|
||||
bool ignore_empty_strings, Expression *delimiter, Expression *quote, Symbol row_var);
|
||||
Expression *delimiter, Expression *quote, Symbol row_var);
|
||||
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
|
||||
UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override;
|
||||
std::vector<Symbol> OutputSymbols(const SymbolTable &) const override;
|
||||
@@ -2241,7 +2241,6 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator {
|
||||
Expression *file_;
|
||||
bool with_header_;
|
||||
bool ignore_bad_;
|
||||
bool ignore_empty_strings_;
|
||||
Expression *delimiter_{nullptr};
|
||||
Expression *quote_{nullptr};
|
||||
Symbol row_var_;
|
||||
@@ -2252,7 +2251,6 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator {
|
||||
object->file_ = file_ ? file_->Clone(storage) : nullptr;
|
||||
object->with_header_ = with_header_;
|
||||
object->ignore_bad_ = ignore_bad_;
|
||||
object->ignore_empty_strings_ = ignore_empty_strings_;
|
||||
object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr;
|
||||
object->quote_ = quote_ ? quote_->Clone(storage) : nullptr;
|
||||
object->row_var_ = row_var_;
|
||||
|
||||
@@ -874,27 +874,11 @@ bool PlanToJsonVisitor::PreVisit(query::plan::CallProcedure &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(query::plan::LoadCsv &op) {
|
||||
json self;
|
||||
self["name"] = "LoadCsv";
|
||||
|
||||
if (op.file_) {
|
||||
self["file"] = ToJson(op.file_);
|
||||
}
|
||||
|
||||
if (op.with_header_) {
|
||||
self["with_header"] = op.with_header_;
|
||||
}
|
||||
|
||||
if (op.ignore_bad_) {
|
||||
self["ignore_bad"] = op.ignore_bad_;
|
||||
}
|
||||
|
||||
if (op.delimiter_) {
|
||||
self["delimiter"] = ToJson(op.delimiter_);
|
||||
}
|
||||
|
||||
if (op.quote_) {
|
||||
self["quote"] = ToJson(op.quote_);
|
||||
}
|
||||
|
||||
self["file"] = ToJson(op.file_);
|
||||
self["with_header"] = op.with_header_;
|
||||
self["ignore_bad"] = op.ignore_bad_;
|
||||
self["delimiter"] = ToJson(op.delimiter_);
|
||||
self["quote"] = ToJson(op.quote_);
|
||||
self["row_variable"] = ToJson(op.row_var_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
|
||||
@@ -477,16 +477,6 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PreVisit(LoadCsv &op) override {
|
||||
prev_ops_.push_back(&op);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PostVisit(LoadCsv & /*op*/) override {
|
||||
prev_ops_.pop_back();
|
||||
return true;
|
||||
}
|
||||
|
||||
std::shared_ptr<LogicalOperator> new_root_;
|
||||
|
||||
private:
|
||||
|
||||
@@ -226,9 +226,10 @@ class RuleBasedPlanner {
|
||||
const auto &row_sym = context.symbol_table->at(*load_csv->row_var_);
|
||||
context.bound_symbols.insert(row_sym);
|
||||
|
||||
input_op = std::make_unique<plan::LoadCsv>(std::move(input_op), load_csv->file_, load_csv->with_header_,
|
||||
load_csv->ignore_bad_, load_csv->ignore_empty_strings_,
|
||||
load_csv->delimiter_, load_csv->quote_, row_sym);
|
||||
input_op =
|
||||
std::make_unique<plan::LoadCsv>(std::move(input_op), load_csv->file_, load_csv->with_header_,
|
||||
load_csv->ignore_bad_, load_csv->delimiter_, load_csv->quote_, row_sym);
|
||||
|
||||
} else if (auto *foreach = utils::Downcast<query::Foreach>(clause)) {
|
||||
context.is_write_query = true;
|
||||
input_op = HandleForeachClause(foreach, std::move(input_op), *context.symbol_table, context.bound_symbols,
|
||||
|
||||
@@ -1504,6 +1504,7 @@ mgp_error mgp_result_new_record(mgp_result *res, mgp_result_record **result) {
|
||||
|
||||
mgp_error mgp_result_record_insert(mgp_result_record *record, const char *field_name, mgp_value *val) {
|
||||
return WrapExceptions([=] {
|
||||
// MEMORY RESOURCE: QueryExecution memory resource
|
||||
auto *memory = record->values.get_allocator().GetMemoryResource();
|
||||
// Validate field_name & val satisfy the procedure's result signature.
|
||||
MG_ASSERT(record->signature, "Expected to have a valid signature");
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <datetime.h>
|
||||
#include <pyerrors.h>
|
||||
#include <array>
|
||||
#include <optional>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
@@ -860,7 +861,7 @@ py::Object MgpListToPyTuple(mgp_list *list, PyObject *py_graph) {
|
||||
}
|
||||
|
||||
namespace {
|
||||
std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Object py_record) {
|
||||
std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Object py_record, mgp_memory *memory) {
|
||||
py::Object py_mgp(PyImport_ImportModule("mgp"));
|
||||
if (!py_mgp) return py::FetchError();
|
||||
auto record_cls = py_mgp.GetAttr("Record");
|
||||
@@ -902,8 +903,9 @@ std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Obj
|
||||
if (!field_name) return py::FetchError();
|
||||
auto *val = PyTuple_GetItem(item, 1);
|
||||
if (!val) return py::FetchError();
|
||||
mgp_memory memory{result->rows.get_allocator().GetMemoryResource()};
|
||||
mgp_value *field_val = PyObjectToMgpValueWithPythonExceptions(val, &memory);
|
||||
// ova memorija je od query executiona
|
||||
// mgp_memory memory{result->rows.get_allocator().GetMemoryResource()};
|
||||
mgp_value *field_val = PyObjectToMgpValueWithPythonExceptions(val, memory);
|
||||
if (field_val == nullptr) {
|
||||
return py::FetchError();
|
||||
}
|
||||
@@ -921,14 +923,20 @@ std::optional<py::ExceptionInfo> AddRecordFromPython(mgp_result *result, py::Obj
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::optional<py::ExceptionInfo> AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq) {
|
||||
std::optional<py::ExceptionInfo> AddMultipleRecordsFromPython(mgp_result *result, py::Object py_seq,
|
||||
mgp_memory *memory) {
|
||||
Py_ssize_t len = PySequence_Size(py_seq.Ptr());
|
||||
if (len == -1) return py::FetchError();
|
||||
for (Py_ssize_t i = 0; i < len; ++i) {
|
||||
py::Object py_record(PySequence_GetItem(py_seq.Ptr(), i));
|
||||
|
||||
for (Py_ssize_t i = 0, curr_loc = 0; i < len; ++i, ++curr_loc) {
|
||||
py::Object py_record(PySequence_GetItem(py_seq.Ptr(), curr_loc));
|
||||
if (!py_record) return py::FetchError();
|
||||
auto maybe_exc = AddRecordFromPython(result, py_record);
|
||||
auto maybe_exc = AddRecordFromPython(result, py_record, memory);
|
||||
if (maybe_exc) return maybe_exc;
|
||||
if (i && i % 100000 == 0) {
|
||||
PySequence_DelSlice(py_seq.Ptr(), 0, 100000);
|
||||
curr_loc = -1;
|
||||
}
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -962,6 +970,7 @@ std::function<void()> PyObjectCleanup(py::Object &py_object) {
|
||||
|
||||
void CallPythonProcedure(const py::Object &py_cb, mgp_list *args, mgp_graph *graph, mgp_result *result,
|
||||
mgp_memory *memory) {
|
||||
// ovaj *memory tu je memorija od EvalContexta
|
||||
auto gil = py::EnsureGIL();
|
||||
|
||||
auto error_to_msg = [](const std::optional<py::ExceptionInfo> &exc_info) -> std::optional<std::string> {
|
||||
@@ -979,9 +988,9 @@ void CallPythonProcedure(const py::Object &py_cb, mgp_list *args, mgp_graph *gra
|
||||
auto py_res = py_cb.Call(py_graph, py_args);
|
||||
if (!py_res) return py::FetchError();
|
||||
if (PySequence_Check(py_res.Ptr())) {
|
||||
return AddMultipleRecordsFromPython(result, py_res);
|
||||
return AddMultipleRecordsFromPython(result, py_res, memory);
|
||||
} else {
|
||||
return AddRecordFromPython(result, py_res);
|
||||
return AddRecordFromPython(result, py_res, memory);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1027,9 +1036,9 @@ void CallPythonTransformation(const py::Object &py_cb, mgp_messages *msgs, mgp_g
|
||||
auto py_res = py_cb.Call(py_graph, py_messages);
|
||||
if (!py_res) return py::FetchError();
|
||||
if (PySequence_Check(py_res.Ptr())) {
|
||||
return AddMultipleRecordsFromPython(result, py_res);
|
||||
return AddMultipleRecordsFromPython(result, py_res, memory);
|
||||
}
|
||||
return AddRecordFromPython(result, py_res);
|
||||
return AddRecordFromPython(result, py_res, memory);
|
||||
};
|
||||
|
||||
// It is *VERY IMPORTANT* to note that this code takes great care not to keep
|
||||
@@ -1131,6 +1140,7 @@ PyObject *PyQueryModuleAddProcedure(PyQueryModule *self, PyObject *cb, bool is_w
|
||||
PyErr_SetString(PyExc_ValueError, "Procedure name is not a valid identifier");
|
||||
return nullptr;
|
||||
}
|
||||
//
|
||||
auto *memory = self->module->procedures.get_allocator().GetMemoryResource();
|
||||
mgp_proc proc(name,
|
||||
[py_cb](mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memory *memory) {
|
||||
|
||||
@@ -170,12 +170,7 @@ bool CurrentVersionHasLabel(const Vertex &vertex, LabelId label, Transaction *tr
|
||||
deleted = vertex.deleted;
|
||||
has_label = utils::Contains(vertex.labels, label);
|
||||
delta = vertex.delta;
|
||||
|
||||
if (!vertex.label_changed) {
|
||||
return !deleted && has_label;
|
||||
}
|
||||
}
|
||||
|
||||
ApplyDeltasForRead(transaction, delta, view, [&deleted, &has_label, label](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
case Delta::Action::REMOVE_LABEL: {
|
||||
@@ -228,12 +223,7 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
|
||||
has_label = utils::Contains(vertex.labels, label);
|
||||
current_value_equal_to_value = vertex.properties.IsPropertyEqual(key, value);
|
||||
delta = vertex.delta;
|
||||
|
||||
if (!vertex.label_changed && !vertex.property_changed) {
|
||||
return !deleted && has_label && current_value_equal_to_value;
|
||||
}
|
||||
}
|
||||
|
||||
ApplyDeltasForRead(transaction, delta, view,
|
||||
[&deleted, &has_label, ¤t_value_equal_to_value, key, label, &value](const Delta &delta) {
|
||||
switch (delta.action) {
|
||||
@@ -507,31 +497,15 @@ bool LabelPropertyIndex::Entry::operator<(const PropertyValue &rhs) { return val
|
||||
bool LabelPropertyIndex::Entry::operator==(const PropertyValue &rhs) { return value == rhs; }
|
||||
|
||||
void LabelPropertyIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
|
||||
auto vertex_properties = vertex->properties.Properties();
|
||||
|
||||
if (index_.size() > vertex_properties.size()) {
|
||||
for (auto &[prop_id, prop_value] : vertex_properties) {
|
||||
auto label_property_pair = std::make_pair(label, prop_id);
|
||||
if (index_.find(label_property_pair) == index_.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &storage = index_[label_property_pair];
|
||||
for (auto &[label_prop, storage] : index_) {
|
||||
if (label_prop.first != label) {
|
||||
continue;
|
||||
}
|
||||
auto prop_value = vertex->properties.GetProperty(label_prop.second);
|
||||
if (!prop_value.IsNull()) {
|
||||
auto acc = storage.access();
|
||||
acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp});
|
||||
}
|
||||
} else {
|
||||
for (auto &[label_prop, storage] : index_) {
|
||||
if (label_prop.first != label) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (vertex_properties.find(label_prop.second) != vertex_properties.end()) {
|
||||
auto prop_value = vertex_properties[label_prop.second];
|
||||
auto acc = storage.access();
|
||||
acc.insert(Entry{std::move(prop_value), vertex, tx.start_timestamp});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -540,31 +514,14 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
|
||||
if (value.IsNull()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto index_size = index_.size();
|
||||
auto properties_size = vertex->properties.Properties().size();
|
||||
|
||||
if (index_size > properties_size) {
|
||||
for (auto &label : vertex->labels) {
|
||||
auto label_property_pair = std::make_pair(label, property);
|
||||
if (index_.find(label_property_pair) == index_.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto &storage = index_[label_property_pair];
|
||||
for (auto &[label_prop, storage] : index_) {
|
||||
if (label_prop.second != property) {
|
||||
continue;
|
||||
}
|
||||
if (utils::Contains(vertex->labels, label_prop.first)) {
|
||||
auto acc = storage.access();
|
||||
acc.insert(Entry{value, vertex, tx.start_timestamp});
|
||||
}
|
||||
} else {
|
||||
for (auto &[label_prop, storage] : index_) {
|
||||
if (label_prop.second != property) {
|
||||
continue;
|
||||
}
|
||||
if (utils::Contains(vertex->labels, label_prop.first)) {
|
||||
auto acc = storage.access();
|
||||
acc.insert(Entry{value, vertex, tx.start_timestamp});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1015,8 +1015,6 @@ void Storage::Accessor::Abort() {
|
||||
auto vertex = prev.vertex;
|
||||
std::lock_guard<utils::SpinLock> guard(vertex->lock);
|
||||
Delta *current = vertex->delta;
|
||||
vertex->label_changed = false;
|
||||
vertex->property_changed = false;
|
||||
while (current != nullptr && current->timestamp->load(std::memory_order_acquire) ==
|
||||
transaction_.transaction_id.load(std::memory_order_acquire)) {
|
||||
switch (current->action) {
|
||||
@@ -1553,8 +1551,6 @@ void Storage::CollectGarbage() {
|
||||
continue;
|
||||
}
|
||||
vertex->delta = nullptr;
|
||||
vertex->label_changed = false;
|
||||
vertex->property_changed = false;
|
||||
if (vertex->deleted) {
|
||||
current_deleted_vertices.push_back(vertex->gid);
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -39,9 +39,8 @@ struct Vertex {
|
||||
|
||||
mutable utils::SpinLock lock;
|
||||
bool deleted;
|
||||
bool label_changed;
|
||||
bool property_changed;
|
||||
// uint8_t PAD;
|
||||
// uint16_t PAD;
|
||||
|
||||
Delta *delta;
|
||||
};
|
||||
|
||||
@@ -85,7 +85,6 @@ Result<bool> VertexAccessor::AddLabel(LabelId label) {
|
||||
if (std::find(vertex_->labels.begin(), vertex_->labels.end(), label) != vertex_->labels.end()) return false;
|
||||
|
||||
CreateAndLinkDelta(transaction_, vertex_, Delta::RemoveLabelTag(), label);
|
||||
vertex_->label_changed = true;
|
||||
|
||||
vertex_->labels.push_back(label);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -123,12 +123,7 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
|
||||
line_string_view.remove_prefix(read_config_.quote->size());
|
||||
} else if (utils::StartsWith(line_string_view, *read_config_.delimiter)) {
|
||||
// The current field has an empty value.
|
||||
if (read_config_.ignore_empty_strings) {
|
||||
row.emplace_back("\0");
|
||||
} else {
|
||||
row.emplace_back("");
|
||||
}
|
||||
|
||||
row.emplace_back("");
|
||||
state = CsvParserState::NEXT_FIELD;
|
||||
line_string_view.remove_prefix(read_config_.delimiter->size());
|
||||
} else {
|
||||
@@ -188,11 +183,7 @@ Reader::ParsingResult Reader::ParseRow(utils::MemoryResource *mem) {
|
||||
case CsvParserState::EXPECT_DELIMITER:
|
||||
break;
|
||||
case CsvParserState::NEXT_FIELD:
|
||||
if (read_config_.ignore_empty_strings) {
|
||||
row.emplace_back("\0");
|
||||
} else {
|
||||
row.emplace_back("");
|
||||
}
|
||||
row.emplace_back("");
|
||||
break;
|
||||
case CsvParserState::QUOTING: {
|
||||
return ParseError(ParseError::ErrorCode::NO_CLOSING_QUOTE,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -40,17 +40,12 @@ class Reader {
|
||||
public:
|
||||
struct Config {
|
||||
Config() = default;
|
||||
Config(const bool with_header, const bool ignore_bad, const bool ignore_empty_strings,
|
||||
std::optional<utils::pmr::string> delim, std::optional<utils::pmr::string> qt)
|
||||
: with_header(with_header),
|
||||
ignore_bad(ignore_bad),
|
||||
ignore_empty_strings(ignore_empty_strings),
|
||||
delimiter(std::move(delim)),
|
||||
quote(std::move(qt)) {}
|
||||
Config(const bool with_header, const bool ignore_bad, std::optional<utils::pmr::string> delim,
|
||||
std::optional<utils::pmr::string> qt)
|
||||
: with_header(with_header), ignore_bad(ignore_bad), delimiter(std::move(delim)), quote(std::move(qt)) {}
|
||||
|
||||
bool with_header{false};
|
||||
bool ignore_bad{false};
|
||||
bool ignore_empty_strings{false};
|
||||
std::optional<utils::pmr::string> delimiter{};
|
||||
std::optional<utils::pmr::string> quote{};
|
||||
};
|
||||
@@ -63,7 +58,6 @@ class Reader {
|
||||
: memory_(mem), path_(std::move(path)) {
|
||||
read_config_.with_header = cfg.with_header;
|
||||
read_config_.ignore_bad = cfg.ignore_bad;
|
||||
read_config_.ignore_empty_strings = cfg.ignore_empty_strings;
|
||||
read_config_.delimiter = cfg.delimiter ? std::move(*cfg.delimiter) : utils::pmr::string{",", memory_};
|
||||
read_config_.quote = cfg.quote ? std::move(*cfg.quote) : utils::pmr::string{"\"", memory_};
|
||||
InitializeStream();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -97,11 +97,10 @@ TEST_P(CsvReaderTest, CommaDelimiter) {
|
||||
|
||||
bool with_header = false;
|
||||
bool ignore_bad = false;
|
||||
bool ignore_empty_strings = false;
|
||||
memgraph::utils::pmr::string delimiter{",", mem};
|
||||
memgraph::utils::pmr::string quote{"\"", mem};
|
||||
|
||||
memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg, mem);
|
||||
|
||||
auto parsed_row = reader.GetNextRow(mem);
|
||||
@@ -124,8 +123,7 @@ TEST_P(CsvReaderTest, SemicolonDelimiter) {
|
||||
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = false;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg, mem);
|
||||
|
||||
auto parsed_row = reader.GetNextRow(mem);
|
||||
@@ -158,8 +156,7 @@ TEST_P(CsvReaderTest, SkipBad) {
|
||||
// parser's output should be solely the valid row;
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = true;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg, mem);
|
||||
|
||||
auto parsed_row = reader.GetNextRow(mem);
|
||||
@@ -171,8 +168,7 @@ TEST_P(CsvReaderTest, SkipBad) {
|
||||
// an exception must be thrown;
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = false;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg, mem);
|
||||
|
||||
EXPECT_THROW(reader.GetNextRow(mem), memgraph::csv::CsvReadException);
|
||||
@@ -199,8 +195,7 @@ TEST_P(CsvReaderTest, AllRowsValid) {
|
||||
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = false;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg);
|
||||
|
||||
const auto pmr_columns = ToPmrColumns(columns);
|
||||
@@ -229,8 +224,7 @@ TEST_P(CsvReaderTest, SkipAllRows) {
|
||||
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = true;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg);
|
||||
|
||||
auto parsed_row = reader.GetNextRow(mem);
|
||||
@@ -257,8 +251,7 @@ TEST_P(CsvReaderTest, WithHeader) {
|
||||
|
||||
const bool with_header = true;
|
||||
const bool ignore_bad = false;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg(with_header, ignore_bad, ignore_empty_strings, delimiter, quote);
|
||||
const memgraph::csv::Reader::Config cfg(with_header, ignore_bad, delimiter, quote);
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg);
|
||||
|
||||
const auto pmr_header = ToPmrColumns(header);
|
||||
@@ -294,8 +287,7 @@ TEST_P(CsvReaderTest, MultilineQuotedString) {
|
||||
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = true;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg);
|
||||
|
||||
auto parsed_row = reader.GetNextRow(mem);
|
||||
@@ -327,8 +319,7 @@ TEST_P(CsvReaderTest, EmptyColumns) {
|
||||
|
||||
const bool with_header = false;
|
||||
const bool ignore_bad = false;
|
||||
const bool ignore_empty_strings = false;
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, ignore_empty_strings, delimiter, quote};
|
||||
const memgraph::csv::Reader::Config cfg{with_header, ignore_bad, delimiter, quote};
|
||||
auto reader = memgraph::csv::Reader(filepath, cfg);
|
||||
|
||||
for (const auto &expected_row : expected_rows) {
|
||||
|
||||
Reference in New Issue
Block a user