Merge branch 'MG-benchmark-pullmultiple-expand' into MG-benchmark-expand

.clang-tidy

@@ -91,12 +91,4 @@ CheckOptions:
     value:           llvm
   - key:             modernize-use-nullptr.NullMacros
     value:           'NULL'
-  - key:             readability-identifier-length.MinimumVariableNameLength
-    value:           '0'
-  - key:             readability-identifier-length.MinimumParameterNameLength
-    value:           '0'
-  - key:             readability-identifier-length.MinimumLoopCounterNameLength
-    value:           '0'
-  - key:             readability-identifier-length.MinimumExceptionNameLength
-    value:           '0'
 ...

.pre-commit-config.yaml

@@ -1,21 +1,21 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v2.3.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 22.10.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.10.1
     hooks:
       - id: isort
         name: isort (python)
         args: ["--profile", "black"]
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v15.0.7
+    rev: v13.0.0
     hooks:
       - id: clang-format

src/query/v2/interpreter.cpp

@@ -64,11 +64,7 @@
 #include "utils/tsc.hpp"
 #include "utils/variant_helpers.hpp"
 
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
-
 namespace EventCounter {
-
 extern Event ReadQuery;
 extern Event WriteQuery;
 extern Event ReadWriteQuery;
@@ -78,7 +74,6 @@ extern const Event LabelPropertyIndexCreated;
 
 extern const Event StreamsCreated;
 extern const Event TriggersCreated;
-
 }  // namespace EventCounter
 
 namespace memgraph::query::v2 {
@@ -693,7 +688,7 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
     : plan_(plan),
       cursor_(plan->plan().MakeCursor(execution_memory)),
       frame_(plan->symbol_table().max_position(), execution_memory),
-      multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
+      multi_frame_(plan->symbol_table().max_position(), kNumberOfFramesInMultiframe, execution_memory),
       memory_limit_(memory_limit) {
   ctx_.db_accessor = dba;
   ctx_.symbol_table = plan->symbol_table();
@@ -817,7 +812,8 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStrea
 std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
                                                                 const std::vector<Symbol> &output_symbols,
                                                                 std::map<std::string, TypedValue> *summary) {
-  if (FLAGS_use_multi_frame) {
+  auto should_pull_multiple = true;  // TODO on the long term, we will only use PullMultiple
+  if (should_pull_multiple) {
     return PullMultiple(stream, n, output_symbols, summary);
   }
   // Set up temporary memory for a single Pull. Initial memory comes from the

src/query/v2/multiframe.cpp

@@ -17,9 +17,6 @@
 #include "query/v2/bindings/frame.hpp"
 #include "utils/pmr/vector.hpp"
 
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
-
 namespace memgraph::query::v2 {
 
 static_assert(std::forward_iterator<ValidFramesReader::Iterator>);

src/query/v2/multiframe.hpp

@@ -13,14 +13,10 @@
 
 #include <iterator>
 
-#include <gflags/gflags.h>
-
 #include "query/v2/bindings/frame.hpp"
 
-// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
-DECLARE_uint64(default_multi_frame_size);
-
 namespace memgraph::query::v2 {
+constexpr uint64_t kNumberOfFramesInMultiframe = 1000;  // TODO have it configurable
 
 class ValidFramesConsumer;
 class ValidFramesModifier;

src/query/v2/plan/operator.cpp

@@ -509,7 +509,7 @@ class DistributedScanAllAndFilterCursor : public Cursor {
 
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
+                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }
@@ -705,7 +705,7 @@ class DistributedScanByPrimaryKeyCursor : public Cursor {
   void EnsureOwnMultiFrameIsGood(MultiFrame &output_multi_frame) {
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
+                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }
@@ -1347,14 +1347,15 @@ bool ContainsSameEdge(const TypedValue &a, const TypedValue &b) {
 }
 
 bool IsExpansionOk(Frame &frame, const Symbol &expand_symbol, const std::vector<Symbol> &previous_symbols) {
-  // This shouldn't raise a TypedValueException, because the planner
-  // makes sure these are all of the expected type. In case they are not
-  // an error should be raised long before this code is executed.
-  return std::ranges::all_of(previous_symbols,
-                             [&frame, &expand_value = frame[expand_symbol]](const auto &previous_symbol) {
-                               const auto &previous_value = frame[previous_symbol];
-                               return !ContainsSameEdge(previous_value, expand_value);
-                             });
+  const auto &expand_value = frame[expand_symbol];
+  for (const auto &previous_symbol : previous_symbols) {
+    const auto &previous_value = frame[previous_symbol];
+    // This shouldn't raise a TypedValueException, because the planner
+    // makes sure these are all of the expected type. In case they are not
+    // an error should be raised long before this code is executed.
+    if (ContainsSameEdge(previous_value, expand_value)) return false;
+  }
+  return true;
 }
 
 }  // namespace
@@ -2213,7 +2214,7 @@ class UnwindCursor : public Cursor {
 
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
+                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }
@@ -3382,7 +3383,7 @@ class DistributedExpandCursor : public Cursor {
   void EnsureOwnMultiFrameIsGood(MultiFrame &output_multi_frame) {
     if (!own_multi_frame_.has_value()) {
       own_multi_frame_.emplace(MultiFrame(output_multi_frame.GetFirstFrame().elems().size(),
-                                          FLAGS_default_multi_frame_size, output_multi_frame.GetMemoryResource()));
+                                          kNumberOfFramesInMultiframe, output_multi_frame.GetMemoryResource()));
       own_frames_consumer_.emplace(own_multi_frame_->GetValidFramesConsumer());
       own_frames_it_ = own_frames_consumer_->begin();
     }

src/query/v2/request_router.hpp

@@ -106,7 +106,6 @@ class RequestRouterInterface {
   virtual std::vector<msgs::ExpandOneResultRow> ExpandOne(msgs::ExpandOneRequest request) = 0;
   virtual std::vector<msgs::CreateExpandResponse> CreateExpand(std::vector<msgs::NewExpand> new_edges) = 0;
   virtual std::vector<msgs::GetPropertiesResultRow> GetProperties(msgs::GetPropertiesRequest request) = 0;
-  virtual std::vector<msgs::GraphResponse> GetGraph(msgs::GraphRequest req) = 0;
 
   virtual storage::v3::EdgeTypeId NameToEdgeType(const std::string &name) const = 0;
   virtual storage::v3::PropertyId NameToProperty(const std::string &name) const = 0;
@@ -404,44 +403,6 @@ class RequestRouter : public RequestRouterInterface {
     return result_rows;
   }
 
-  std::vector<msgs::GraphResponse> GetGraph(msgs::GraphRequest req) override {
-    SPDLOG_WARN("RequestRouter::GetGraph(GraphRequest) not fully implemented");
-
-    // InitializeRequests
-    // TODO(gitbuda): This seems quite expensive because potentially a lot of requests has to be initialized.
-    auto multi_shards = shards_map_.GetAllShards();
-    std::vector<ShardRequestState<msgs::GraphRequest>> requests = {};
-    for (auto &shards : multi_shards) {
-      for (auto &[key, shard] : shards) {
-        MG_ASSERT(!shard.peers.empty());
-        msgs::GraphRequest request;
-        request.transaction_id = transaction_id_;
-        ShardRequestState<msgs::GraphRequest> shard_request_state{
-            .shard = shard,
-            .request = std::move(request),
-        };
-        requests.emplace_back(std::move(shard_request_state));
-      }
-    }
-    spdlog::trace("created {} Graph requests", requests.size());
-
-    // SendRequests and CollectResponses
-    RunningRequests<msgs::GraphRequest> running_requests = {};
-    running_requests.reserve(requests.size());
-    for (size_t i = 0; i < requests.size(); i++) {
-      auto &request = requests[i];
-      io::ReadinessToken readiness_token{i};
-      auto &storage_client = GetStorageClientForShard(request.shard);
-      storage_client.SendAsyncReadRequest(request.request, notifier_, readiness_token);
-      running_requests.emplace(readiness_token.GetId(), request);
-    }
-    spdlog::trace("sent {} Graph requests in parallel", running_requests.size());
-    auto responses = DriveReadResponses<msgs::GraphRequest, msgs::GraphResponse>(running_requests);
-    spdlog::trace("got back {} Graph responses after driving to completion", responses.size());
-
-    return responses;
-  }
-
   std::optional<storage::v3::PropertyId> MaybeNameToProperty(const std::string &name) const override {
     return shards_map_.GetPropertyId(name);
   }

src/query/v2/requests.hpp

@@ -79,18 +79,6 @@ struct Vertex {
   friend bool operator==(const Vertex &lhs, const Vertex &rhs) { return lhs.id == rhs.id; }
 };
 
-struct Path {
-  // TODO(gitbuda): Define how a path should look like.
-};
-struct Graph {
-  std::vector<Vertex> vertices;
-  std::vector<Edge> edges;
-  friend bool operator==(const Graph &lhs, const Graph &rhs) {
-    LOG_FATAL("Implement memgraph::msgs::Graph::operator==");
-  }
-};
-// TODO(gitbuda): Figure out how to serialize memgraph::msgs::Graph/Path.
-
 struct Null {};
 
 struct Value {
@@ -574,21 +562,6 @@ struct UpdateEdgesResponse {
   std::optional<ShardError> error;
 };
 
-// TODO(gitbuda): Add more filtering options.
-struct GraphRequest {
-  Hlc transaction_id;
-  std::optional<VertexId> maybe_start_id;
-  //  The empty optional means return all of the properties, while an empty list means do not return any properties
-  std::optional<std::vector<PropertyId>> props_to_return;
-  std::optional<size_t> batch_limit;
-  StorageView storage_view{StorageView::NEW};
-};
-
-struct GraphResponse {
-  std::optional<ShardError> error;
-  Graph data;
-};
-
 struct CommitRequest {
   Hlc transaction_id;
   Hlc commit_timestamp;
@@ -598,8 +571,8 @@ struct CommitResponse {
   std::optional<ShardError> error;
 };
 
-using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest, GraphRequest>;
-using ReadResponses = std::variant<ExpandOneResponse, GetPropertiesResponse, ScanVerticesResponse, GraphResponse>;
+using ReadRequests = std::variant<ExpandOneRequest, GetPropertiesRequest, ScanVerticesRequest>;
+using ReadResponses = std::variant<ExpandOneResponse, GetPropertiesResponse, ScanVerticesResponse>;
 
 using WriteRequests = std::variant<CreateVerticesRequest, DeleteVerticesRequest, UpdateVerticesRequest,
                                    CreateExpandRequest, DeleteEdgesRequest, UpdateEdgesRequest, CommitRequest>;
@@ -609,6 +582,7 @@ using WriteResponses = std::variant<CreateVerticesResponse, DeleteVerticesRespon
 }  // namespace memgraph::msgs
 
 namespace std {
+
 template <>
 struct hash<memgraph::msgs::Value>;
 

src/storage/v3/request_helper.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -25,12 +25,10 @@
 #include "utils/template_utils.hpp"
 
 namespace memgraph::storage::v3 {
-
 using EdgeAccessors = std::vector<storage::v3::EdgeAccessor>;
 using EdgeUniquenessFunction = std::function<EdgeAccessors(EdgeAccessors &&, msgs::EdgeDirection)>;
 using EdgeFiller =
     std::function<ShardResult<void>(const EdgeAccessor &edge, bool is_in_edge, msgs::ExpandOneResultRow &result_row)>;
-using conversions::FromPropertyValueToValue;
 using msgs::Value;
 
 template <typename T>

src/storage/v3/shard_rsm.cpp

@@ -50,13 +50,17 @@
 #include "utils/logging.hpp"
 
 namespace memgraph::storage::v3 {
+using msgs::Label;
+using msgs::PropertyId;
+using msgs::Value;
+
 using conversions::ConvertPropertyMap;
 using conversions::ConvertPropertyVector;
 using conversions::ConvertValueVector;
 using conversions::FromMap;
+using conversions::FromPropertyValueToValue;
+using conversions::ToMsgsVertexId;
 using conversions::ToPropertyValue;
-using msgs::PropertyId;
-using msgs::Value;
 
 auto CreateErrorResponse(const ShardError &shard_error, const auto transaction_id, const std::string_view action) {
   msgs::ShardError message_shard_error{shard_error.code, shard_error.message};
@@ -526,7 +530,7 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::GetPropertiesRequest &&req) {
     std::vector<std::pair<PropertyId, Value>> result;
     result.reserve(value.size());
     for (auto &[id, val] : value) {
-      result.emplace_back(id, std::move(val));
+      result.emplace_back(std::make_pair(id, std::move(val)));
     }
     return result;
   };
@@ -702,12 +706,4 @@ msgs::ReadResponses ShardRsm::HandleRead(msgs::GetPropertiesRequest &&req) {
   });
 }
 
-msgs::ReadResponses ShardRsm::HandleRead(msgs::GraphRequest &&req) {
-  shard_->Access(req.transaction_id);
-  SPDLOG_WARN("ShardRsm::HandleRead(GraphRequest) not fully implemented");
-  msgs::GraphResponse response;
-  response.data = msgs::Graph{.vertices = {}, .edges = {}};
-  return response;
-}
-
 }  // namespace memgraph::storage::v3

src/storage/v3/shard_rsm.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -27,7 +27,6 @@ class ShardRsm {
   msgs::ReadResponses HandleRead(msgs::ExpandOneRequest &&req);
   msgs::ReadResponses HandleRead(msgs::GetPropertiesRequest &&req);
   msgs::ReadResponses HandleRead(msgs::ScanVerticesRequest &&req);
-  msgs::ReadResponses HandleRead(msgs::GraphRequest &&req);
 
   msgs::WriteResponses ApplyWrite(msgs::CreateVerticesRequest &&req);
   msgs::WriteResponses ApplyWrite(msgs::DeleteVerticesRequest &&req);

tests/mgbench/compare_results.py

@@ -14,6 +14,7 @@
 import argparse
 import json
 
+
 FIELDS = [
     {
         "name": "throughput",
@@ -84,32 +85,39 @@ def compare_results(results_from, results_to, fields):
                 if group == "__import__":
                     continue
                 for scenario, summary_to in scenarios.items():
-                    summary_from = recursive_get(results_from, dataset, variant, group, scenario, value={})
-                    if (
-                        len(summary_from) > 0
-                        and summary_to["count"] != summary_from["count"]
-                        or summary_to["num_workers"] != summary_from["num_workers"]
-                    ):
+                    summary_from = recursive_get(
+                        results_from, dataset, variant, group, scenario,
+                        value={})
+                    if len(summary_from) > 0 and \
+                            summary_to["count"] != summary_from["count"] or \
+                            summary_to["num_workers"] != \
+                            summary_from["num_workers"]:
                         raise Exception("Incompatible results!")
-                    testcode = "/".join([dataset, variant, group, scenario, "{:02d}".format(summary_to["num_workers"])])
+                    testcode = "/".join([dataset, variant, group, scenario,
+                                         "{:02d}".format(
+                                             summary_to["num_workers"])])
                     row = {}
                     performance_changed = False
                     for field in fields:
                         key = field["name"]
                         if key in summary_to:
-                            row[key] = compute_diff(summary_from.get(key, None), summary_to[key])
+                            row[key] = compute_diff(
+                                summary_from.get(key, None),
+                                summary_to[key])
                         elif key in summary_to["database"]:
                             row[key] = compute_diff(
-                                recursive_get(summary_from, "database", key, value=None), summary_to["database"][key]
-                            )
+                                recursive_get(summary_from, "database", key,
+                                              value=None),
+                                summary_to["database"][key])
                         else:
                             row[key] = compute_diff(
-                                recursive_get(summary_from, "metadata", key, "average", value=None),
-                                summary_to["metadata"][key]["average"],
-                            )
-                        if "diff" not in row[key] or (
-                            "diff_treshold" in field and abs(row[key]["diff"]) >= field["diff_treshold"]
-                        ):
+                                recursive_get(summary_from, "metadata", key,
+                                              "average", value=None),
+                                summary_to["metadata"][key]["average"])
+                        if "diff" not in row[key] or \
+                                ("diff_treshold" in field and
+                                 abs(row[key]["diff"]) >=
+                                 field["diff_treshold"]):
                             performance_changed = True
                     if performance_changed:
                         ret[testcode] = row
@@ -122,36 +130,29 @@ def generate_remarkup(fields, data):
         ret += "<table>\n"
         ret += "  <tr>\n"
         ret += "    <th>Testcode</th>\n"
-        ret += (
-            "\n".join(
-                map(
-                    lambda x: "    <th>{}</th>".format(x["name"].replace("_", " ").capitalize()),
-                    fields,
-                )
-            )
-            + "\n"
-        )
+        ret += "\n".join(map(lambda x: "    <th>{}</th>".format(
+            x["name"].replace("_", " ").capitalize()), fields)) + "\n"
         ret += "  </tr>\n"
         for testcode in sorted(data.keys()):
             ret += "  <tr>\n"
             ret += "    <td>{}</td>\n".format(testcode)
             for field in fields:
-                result = data[testcode].get(field["name"])
-                if result != None:
-                    value = result["value"] * field["scaling"]
-                    if "diff" in result:
-                        diff = result["diff"]
-                        arrow = "arrow-up" if diff >= 0 else "arrow-down"
-                        if not (field["positive_diff_better"] ^ (diff >= 0)):
-                            color = "green"
-                        else:
-                            color = "red"
-                        sign = "{{icon {} color={}}}".format(arrow, color)
-                        ret += '    <td bgcolor="{}">{:.3f}{} ({:+.2%})</td>\n'.format(
-                            color, value, field["unit"], diff
-                        )
+                result = data[testcode][field["name"]]
+                value = result["value"] * field["scaling"]
+                if "diff" in result:
+                    diff = result["diff"]
+                    arrow = "arrow-up" if diff >= 0 else "arrow-down"
+                    if not (field["positive_diff_better"] ^ (diff >= 0)):
+                        color = "green"
                     else:
-                        ret += '<td bgcolor="blue">{:.3f}{} //(new)// </td>\n'.format(value, field["unit"])
+                        color = "red"
+                    sign = "{{icon {} color={}}}".format(arrow, color)
+                    ret += "    <td>{:.3f}{} //({:+.2%})// {}</td>\n".format(
+                        value, field["unit"], diff, sign)
+                else:
+                    ret += "    <td>{:.3f}{} //(new)// " \
+                           "{{icon plus color=blue}}</td>\n".format(
+                               value, field["unit"])
             ret += "  </tr>\n"
         ret += "</table>\n"
     else:
@@ -160,14 +161,11 @@ def generate_remarkup(fields, data):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Compare results of multiple benchmark runs.")
-    parser.add_argument(
-        "--compare",
-        action="append",
-        nargs=2,
-        metavar=("from", "to"),
-        help="compare results between `from` and `to` files",
-    )
+    parser = argparse.ArgumentParser(
+        description="Compare results of multiple benchmark runs.")
+    parser.add_argument("--compare", action="append", nargs=2,
+                        metavar=("from", "to"),
+                        help="compare results between `from` and `to` files")
     parser.add_argument("--output", default="", help="output file name")
     args = parser.parse_args()
 

tests/mgbench/dataset_creator_unwind.py

@@ -55,6 +55,7 @@ def main():
     parser.add_argument("--number_of_files", type=int, default=10)
     parser.add_argument("--percentage_of_permissions", type=float, default=1.0)
     parser.add_argument("--filename", default="dataset.cypher")
+    parser.add_argument("--expand_filename", default="expand_output.txt")
 
     args = parser.parse_args()
 
@@ -62,6 +63,7 @@ def main():
     number_of_files = args.number_of_files
     percentage_of_permissions = args.percentage_of_permissions
     filename = args.filename
+    expand_filename = args.expand_filename
 
     assert number_of_identities >= 0
     assert number_of_files >= 0
@@ -106,33 +108,48 @@ def main():
             uuid += 1
         f.write("] AS props CREATE (:Identity {uuid: props.uuid, name: props.name});\n")
 
-        f.write("UNWIND [")
-        created = 0
-        for outer_index in range(0, number_of_files):
-            for inner_index in range(0, number_of_identities):
+        with open(expand_filename, "w") as ef:
 
-                file_uuid = outer_index + 1
-                identity_uuid = number_of_files + inner_index + 1
+            f.write("UNWIND [")
+            ef.write("UNWIND [")
 
-                if random.random() <= percentage_of_permissions:
+            created = 0
+            for outer_index in range(0, number_of_files):
+                for inner_index in range(0, number_of_identities):
 
-                    if created > 0:
-                        f.write(",")
+                    file_uuid = outer_index + 1
+                    identity_uuid = number_of_files + inner_index + 1
 
-                    f.write(
-                        f' {{permUuid: {uuid}, permName: "name_permission_{uuid}", fileUuid: {file_uuid}, identityUuid: {identity_uuid}}}'
-                    )
-                    created += 1
-                    uuid += 1
+                    if random.random() <= percentage_of_permissions:
 
-                    if created == 5000:
-                        f.write(
-                            "] AS props MATCH (file:File {uuid:props.fileUuid}), (identity:Identity {uuid: props.identityUuid}) CREATE (permission:Permission {uuid: props.permUuid, name: props.permName}) CREATE (permission)-[: IS_FOR_FILE]->(file) CREATE (permission)-[: IS_FOR_IDENTITY]->(identity);\nUNWIND ["
+                        if created > 0:
+                            ef.write(",")
+                            f.write(",")
+
+                        ef.write(
+                            f' {{permUuid: {uuid}, permName: "name_permission_{uuid}", fileUuid: {file_uuid}, identityUuid: {identity_uuid}}}'
                         )
-                        created = 0
-        f.write(
-            "] AS props MATCH (file:File {uuid:props.fileUuid}), (identity:Identity {uuid: props.identityUuid}) CREATE (permission:Permission {uuid: props.permUuid, name: props.permName}) CREATE (permission)-[: IS_FOR_FILE]->(file) CREATE (permission)-[: IS_FOR_IDENTITY]->(identity);\n"
-        )
+
+                        f.write(
+                            f' {{permUuid: {uuid}, permName: "name_permission_{uuid}", fileUuid: {file_uuid}, identityUuid: {identity_uuid}}}'
+                        )
+                        created += 1
+                        uuid += 1
+
+                        if created == 5000:
+                            ef.write(
+                                "] AS props MATCH (permission:Permission {uuid: props.permUuid, name: props.permName})-[: IS_FOR_FILE]->(file:File {uuid:props.fileUuid}), (permission:Permission {uuid: props.permUuid, name: props.permName})-[: IS_FOR_IDENTITY]->(identity:Identity {uuid: props.identityUuid}) RETURN permission, file, identity;\nUNWIND ["
+                            )
+                            f.write(
+                                "] AS props MATCH (file:File {uuid:props.fileUuid}), (identity:Identity {uuid: props.identityUuid}) CREATE (permission:Permission {uuid: props.permUuid, name: props.permName}) CREATE (permission)-[: IS_FOR_FILE]->(file) CREATE (permission)-[: IS_FOR_IDENTITY]->(identity);\nUNWIND ["
+                            )
+                            created = 0
+            ef.write(
+                "] AS props MATCH (permission:Permission {uuid: props.permUuid, name: props.permName})-[: IS_FOR_FILE]->(file:File {uuid:props.fileUuid}), (permission:Permission {uuid: props.permUuid, name: props.permName})-[: IS_FOR_IDENTITY]->(identity:Identity {uuid: props.identityUuid}) RETURN permission, file, identity;"
+            )
+            f.write(
+                "] AS props MATCH (file:File {uuid:props.fileUuid}), (identity:Identity {uuid: props.identityUuid}) CREATE (permission:Permission {uuid: props.permUuid, name: props.permName}) CREATE (permission)-[: IS_FOR_FILE]->(file) CREATE (permission)-[: IS_FOR_IDENTITY]->(identity);"
+            )
 
 
 if __name__ == "__main__":

tests/mgbench/datasets.py

@@ -353,7 +353,7 @@ class AccessControl(Dataset):
 
     def benchmark__create__vertex(self):
         self.next_value_idx += 1
-        query = ("CREATE (:File {uuid: $uuid})", {"uuid": self.next_value_idx})
+        query = (f"CREATE (:File {{uuid: {self.next_value_idx}}});", {})
         return query
 
     def benchmark__create__edges(self):
@@ -379,24 +379,6 @@ class AccessControl(Dataset):
         return query
 
     def benchmark__match__match_all_vertices_with_edges(self):
+        self.next_value_idx += 1
         query = ("MATCH (permission:Permission)-[e:IS_FOR_FILE]->(file:File) RETURN *", {})
         return query
-
-    def benchmark__match__match_users_with_permission_for_files(self):
-        file_uuid_1 = self._get_random_uuid("File")
-        file_uuid_2 = self._get_random_uuid("File")
-        min_file_uuid = min(file_uuid_1, file_uuid_2)
-        max_file_uuid = max(file_uuid_1, file_uuid_2)
-        query = (
-            "MATCH (f:File)<-[ff:IS_FOR_FILE]-(p:Permission)-[fi:IS_FOR_IDENTITY]->(i:Identity) WHERE f.uuid >= $min_file_uuid  AND f.uuid <= $max_file_uuid RETURN *",
-            {"min_file_uuid": min_file_uuid, "max_file_uuid": max_file_uuid},
-        )
-        return query
-
-    def benchmark__match__match_users_with_permission_for_specific_file(self):
-        file_uuid = self._get_random_uuid("File")
-        query = (
-            "MATCH (f:File {uuid: $file_uuid})<-[ff:IS_FOR_FILE]-(p:Permission)-[fi:IS_FOR_IDENTITY]->(i:Identity) RETURN *",
-            {"file_uuid": file_uuid},
-        )
-        return query

tests/mgbench/runners.py

@@ -68,15 +68,6 @@ class Memgraph:
         self._cleanup()
         atexit.unregister(self._cleanup)
 
-    # Returns None if string_value is not true or false, casing doesn't matter
-    def _get_bool_value(self, string_value):
-        lower_string_value = string_value.lower()
-        if lower_string_value == "true":
-            return True
-        if lower_string_value == "false":
-            return False
-        return None
-
     def _get_args(self, **kwargs):
         data_directory = os.path.join(self._directory.name, "memgraph")
         if self._memgraph_version >= (0, 50, 0):
@@ -92,13 +83,7 @@ class Memgraph:
             args_list = self._extra_args.split(" ")
             assert len(args_list) % 2 == 0
             for i in range(0, len(args_list), 2):
-                key = args_list[i]
-                value = args_list[i + 1]
-                maybe_bool_value = self._get_bool_value(value)
-                if maybe_bool_value is not None:
-                    kwargs[key] = maybe_bool_value
-                else:
-                    kwargs[key] = value
+                kwargs[args_list[i]] = args_list[i + 1]
 
         return _convert_args_to_flags(self._memgraph_binary, **kwargs)
 

tests/simulation/common.hpp

@@ -1,4 +1,4 @@
-// Copyright 2023 Memgraph Ltd.
+// Copyright 2022 Memgraph Ltd.
 //
 // Use of this software is governed by the Business Source License
 // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -118,12 +118,6 @@ class MockedShardRsm {
     return resp;
   }
 
-  msgs::GraphResponse ReadImpl(msgs::GraphRequest rqst) {
-    msgs::GraphResponse resp;
-    SPDLOG_INFO("MockedShardRsm::ReadImpl");
-    return resp;
-  }
-
   ReadResponses Read(ReadRequests read_requests) {
     return {std::visit([this]<typename T>(T &&request) { return ReadResponses{ReadImpl(std::forward<T>(request))}; },
                        std::move(read_requests))};

tests/simulation/request_router.cpp

@@ -44,23 +44,40 @@ using CompoundKey = coordinator::PrimaryKey;
 using coordinator::Coordinator;
 using coordinator::CoordinatorClient;
 using coordinator::CoordinatorRsm;
+using coordinator::HlcRequest;
+using coordinator::HlcResponse;
 using coordinator::ShardMap;
 using coordinator::ShardMetadata;
 using coordinator::Shards;
 using coordinator::Status;
 using io::Address;
 using io::Io;
+using io::ResponseEnvelope;
 using io::ResponseFuture;
 using io::Time;
+using io::TimedOut;
 using io::rsm::Raft;
+using io::rsm::ReadRequest;
+using io::rsm::ReadResponse;
+using io::rsm::StorageReadRequest;
+using io::rsm::StorageReadResponse;
+using io::rsm::StorageWriteRequest;
+using io::rsm::StorageWriteResponse;
+using io::rsm::WriteRequest;
+using io::rsm::WriteResponse;
 using io::simulator::Simulator;
 using io::simulator::SimulatorConfig;
 using io::simulator::SimulatorStats;
 using io::simulator::SimulatorTransport;
+using msgs::CreateVerticesRequest;
+using msgs::CreateVerticesResponse;
+using msgs::ScanVerticesRequest;
+using msgs::ScanVerticesResponse;
 using msgs::VertexId;
 using storage::v3::LabelId;
 using storage::v3::SchemaProperty;
 using storage::v3::tests::MockedShardRsm;
+using utils::BasicResult;
 
 namespace {
 
@@ -213,19 +230,10 @@ void TestGetProperties(query::v2::RequestRouterInterface &request_router) {
   auto result = request_router.GetProperties(std::move(request));
   MG_ASSERT(result.size() == 3);
 }
+
 template <typename RequestRouter>
 void TestAggregate(RequestRouter &request_router) {}
 
-void TestGetGraph(query::v2::RequestRouterInterface &rr) {
-  msgs::GraphRequest req;
-  auto graphs = rr.GetGraph(req);
-  MG_ASSERT(graphs.size() == 2);
-  for (const auto &graph : graphs) {
-    MG_ASSERT(graph.data.vertices.size() == 0);
-    MG_ASSERT(graph.data.edges.size() == 0);
-  }
-}
-
 void DoTest() {
   SimulatorConfig config{
       .drop_percent = 0,
@@ -348,7 +356,6 @@ void DoTest() {
   TestCreateVertices(request_router);
   TestCreateExpand(request_router);
   TestGetProperties(request_router);
-  TestGetGraph(request_router);
 
   simulator.ShutDown();
 

tests/simulation/shard_rsm.cpp

@@ -33,21 +33,32 @@
 #include "utils/result.hpp"
 
 namespace memgraph::storage::v3::tests {
+
 using io::Address;
 using io::Io;
+using io::ResponseEnvelope;
 using io::ResponseFuture;
 using io::Time;
+using io::TimedOut;
 using io::rsm::Raft;
+using io::rsm::ReadRequest;
+using io::rsm::ReadResponse;
 using io::rsm::RsmClient;
+using io::rsm::WriteRequest;
+using io::rsm::WriteResponse;
 using io::simulator::Simulator;
 using io::simulator::SimulatorConfig;
 using io::simulator::SimulatorStats;
 using io::simulator::SimulatorTransport;
+using utils::BasicResult;
+
 using msgs::ReadRequests;
 using msgs::ReadResponses;
 using msgs::WriteRequests;
 using msgs::WriteResponses;
+
 using ShardClient = RsmClient<SimulatorTransport, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
+
 using ConcreteShardRsm = Raft<SimulatorTransport, ShardRsm, WriteRequests, WriteResponses, ReadRequests, ReadResponses>;
 
 // TODO(gvolfing) test vertex deletion with DETACH_DELETE as well
@@ -1451,21 +1462,6 @@ void TestGetProperties(ShardClient &client) {
   }
 }
 
-void TestGetGraph(ShardClient &client) {
-  SPDLOG_WARN("shard_rsm.cpp:TestGetGraph(ShardClient) not yet implemented");
-  msgs::GraphRequest req{};
-  while (true) {
-    auto read_res = client.SendReadRequest(req);
-    if (read_res.HasError()) {
-      continue;
-    }
-    auto res = read_res.GetValue();
-    auto graph_res = std::get<msgs::GraphResponse>(res);
-    MG_ASSERT(graph_res.error == std::nullopt);
-    return;
-  }
-}
-
 }  // namespace
 
 int TestMessages() {
@@ -1553,11 +1549,8 @@ int TestMessages() {
 
   // GetProperties tests
   TestGetProperties(client);
-
-  // GetGraph tests
-  TestGetGraph(client);
-
   simulator.ShutDown();
+
   SimulatorStats stats = simulator.Stats();
 
   std::cout << "total messages:     " << stats.total_messages << std::endl;

tests/unit/mock_helpers.hpp

@@ -27,7 +27,6 @@ class MockedRequestRouter : public RequestRouterInterface {
   MOCK_METHOD(std::vector<msgs::ExpandOneResultRow>, ExpandOne, (msgs::ExpandOneRequest));
   MOCK_METHOD(std::vector<msgs::CreateExpandResponse>, CreateExpand, (std::vector<msgs::NewExpand>));
   MOCK_METHOD(std::vector<msgs::GetPropertiesResultRow>, GetProperties, (msgs::GetPropertiesRequest));
-  MOCK_METHOD(std::vector<msgs::GraphResponse>, GetGraph, (msgs::GraphRequest));
   MOCK_METHOD(void, StartTransaction, ());
   MOCK_METHOD(void, Commit, ());
 

tests/unit/query_v2_expression_evaluator.cpp

@@ -97,8 +97,6 @@ class MockedRequestRouter : public RequestRouterInterface {
 
   std::vector<GetPropertiesResultRow> GetProperties(GetPropertiesRequest rqst) override { return {}; }
 
-  std::vector<msgs::GraphResponse> GetGraph(msgs::GraphRequest rqst) override { return {}; }
-
   const std::string &PropertyToName(memgraph::storage::v3::PropertyId id) const override {
     return properties_.IdToName(id.AsUint());
   }