Remove unused function

Add new query "COMPACT MEMORY" and implement the underlying
functionality for it. This proof of concept is bulding on the snapshot
creation/reloading features we already have.

src/auth/models.cpp

@@ -61,7 +61,8 @@ const std::vector<Permission> kPermissionsAll = {Permission::MATCH,
                                                  Permission::TRANSACTION_MANAGEMENT,
                                                  Permission::STORAGE_MODE,
                                                  Permission::MULTI_DATABASE_EDIT,
-                                                 Permission::MULTI_DATABASE_USE};
+                                                 Permission::MULTI_DATABASE_USE,
+                                                 Permission::COMPACT_MEMORY};
 
 }  // namespace
 
@@ -117,6 +118,8 @@ std::string PermissionToString(Permission permission) {
       return "MULTI_DATABASE_EDIT";
     case Permission::MULTI_DATABASE_USE:
       return "MULTI_DATABASE_USE";
+    case Permission::COMPACT_MEMORY:
+      return "COMPACT_MEMORY";
   }
 }
 

src/auth/models.hpp

@@ -47,6 +47,7 @@ enum class Permission : uint64_t {
   STORAGE_MODE = 1U << 22U,
   MULTI_DATABASE_EDIT = 1U << 23U,
   MULTI_DATABASE_USE  = 1U << 24U,
+  COMPACT_MEMORY = 1U << 25U,
 };
 // clang-format on
 

src/glue/auth.cpp

@@ -66,6 +66,8 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
       return auth::Permission::MULTI_DATABASE_EDIT;
     case query::AuthQuery::Privilege::MULTI_DATABASE_USE:
       return auth::Permission::MULTI_DATABASE_USE;
+    case query::AuthQuery::Privilege::COMPACT_MEMORY:
+      return auth::Permission::COMPACT_MEMORY;
   }
 }
 

src/query/exceptions.hpp

@@ -230,6 +230,12 @@ class FreeMemoryModificationInMulticommandTxException : public QueryException {
       : QueryException("Free memory query not allowed in multicommand transactions.") {}
 };
 
+class CompactMemoryModificationInMulticommandTxException : public QueryException {
+ public:
+  CompactMemoryModificationInMulticommandTxException()
+      : QueryException("COMPACT MEMORY is query not allowed in multicommand transactions.") {}
+};
+
 class FreeMemoryDisabledOnDiskStorage : public QueryException {
  public:
   FreeMemoryDisabledOnDiskStorage() : QueryException("Free memory does nothing when using disk storage. ") {}

src/query/frontend/ast/ast.cpp

@@ -243,6 +243,9 @@ constexpr utils::TypeInfo query::LoadCsv::kType{utils::TypeId::AST_LOAD_CSV, "Lo
 constexpr utils::TypeInfo query::FreeMemoryQuery::kType{utils::TypeId::AST_FREE_MEMORY_QUERY, "FreeMemoryQuery",
                                                         &query::Query::kType};
 
+constexpr utils::TypeInfo query::CompactMemoryQuery::kType{utils::TypeId::AST_COMPACT_MEMORY_QUERY,
+                                                           "CompactMemoryQuery", &query::Query::kType};
+
 constexpr utils::TypeInfo query::TriggerQuery::kType{utils::TypeId::AST_TRIGGER_QUERY, "TriggerQuery",
                                                      &query::Query::kType};
 

src/query/frontend/ast/ast.hpp

@@ -2813,6 +2813,7 @@ class AuthQuery : public memgraph::query::Query {
     TRANSACTION_MANAGEMENT,
     MULTI_DATABASE_EDIT,
     MULTI_DATABASE_USE,
+    COMPACT_MEMORY,
   };
 
   enum class FineGrainedPrivilege { NOTHING, READ, UPDATE, CREATE_DELETE };
@@ -2891,7 +2892,8 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {AuthQuery::Privilege::
                                                           AuthQuery::Privilege::TRANSACTION_MANAGEMENT,
                                                           AuthQuery::Privilege::STORAGE_MODE,
                                                           AuthQuery::Privilege::MULTI_DATABASE_EDIT,
-                                                          AuthQuery::Privilege::MULTI_DATABASE_USE};
+                                                          AuthQuery::Privilege::MULTI_DATABASE_USE,
+                                                          AuthQuery::Privilege::COMPACT_MEMORY};
 
 class InfoQuery : public memgraph::query::Query {
  public:
@@ -3115,6 +3117,19 @@ class FreeMemoryQuery : public memgraph::query::Query {
   }
 };
 
+class CompactMemoryQuery : public memgraph::query::Query {
+ public:
+  static const utils::TypeInfo kType;
+  const utils::TypeInfo &GetTypeInfo() const override { return kType; }
+
+  DEFVISITABLE(QueryVisitor<void>);
+
+  CompactMemoryQuery *Clone(AstStorage *storage) const override {
+    CompactMemoryQuery *object = storage->Create<CompactMemoryQuery>();
+    return object;
+  }
+};
+
 class TriggerQuery : public memgraph::query::Query {
  public:
   static const utils::TypeInfo kType;

src/query/frontend/ast/ast_visitor.hpp

@@ -106,6 +106,7 @@ class Exists;
 class MultiDatabaseQuery;
 class ShowDatabasesQuery;
 class EdgeImportModeQuery;
+class CompactMemoryQuery;
 
 using TreeCompositeVisitor = utils::CompositeVisitor<
     SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@@ -144,6 +145,6 @@ class QueryVisitor
                             ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery, FreeMemoryQuery, TriggerQuery,
                             IsolationLevelQuery, CreateSnapshotQuery, StreamQuery, SettingQuery, VersionQuery,
                             ShowConfigQuery, TransactionQueueQuery, StorageModeQuery, AnalyzeGraphQuery,
-                            MultiDatabaseQuery, ShowDatabasesQuery, EdgeImportModeQuery> {};
+                            MultiDatabaseQuery, ShowDatabasesQuery, EdgeImportModeQuery, CompactMemoryQuery> {};
 
 }  // namespace memgraph::query

src/query/frontend/ast/cypher_main_visitor.cpp

@@ -410,6 +410,12 @@ antlrcpp::Any CypherMainVisitor::visitFreeMemoryQuery(MemgraphCypher::FreeMemory
   return free_memory_query;
 }
 
+antlrcpp::Any CypherMainVisitor::visitCompactMemoryQuery(MemgraphCypher::CompactMemoryQueryContext *ctx) {
+  auto *compact_memory_query = storage_->Create<CompactMemoryQuery>();
+  query_ = compact_memory_query;
+  return compact_memory_query;
+}
+
 antlrcpp::Any CypherMainVisitor::visitTriggerQuery(MemgraphCypher::TriggerQueryContext *ctx) {
   MG_ASSERT(ctx->children.size() == 1, "TriggerQuery should have exactly one child!");
   auto *trigger_query = std::any_cast<TriggerQuery *>(ctx->children[0]->accept(this));
@@ -1576,6 +1582,7 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
   if (ctx->STORAGE_MODE()) return AuthQuery::Privilege::STORAGE_MODE;
   if (ctx->MULTI_DATABASE_EDIT()) return AuthQuery::Privilege::MULTI_DATABASE_EDIT;
   if (ctx->MULTI_DATABASE_USE()) return AuthQuery::Privilege::MULTI_DATABASE_USE;
+  if (ctx->COMPACT_MEMORY()) return AuthQuery::Privilege::COMPACT_MEMORY;
   LOG_FATAL("Should not get here - unknown privilege!");
 }
 

src/query/frontend/ast/cypher_main_visitor.hpp

@@ -241,6 +241,11 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
    */
   antlrcpp::Any visitFreeMemoryQuery(MemgraphCypher::FreeMemoryQueryContext *ctx) override;
 
+  /**
+   * @return CompactMemoryQuery*
+   */
+  antlrcpp::Any visitCompactMemoryQuery(MemgraphCypher::CompactMemoryQueryContext *ctx) override;
+
   /**
    * @return TriggerQuery*
    */

src/query/frontend/opencypher/grammar/MemgraphCypher.g4

@@ -37,6 +37,7 @@ memgraphCypherKeyword : cypherKeyword
                       | CLEAR
                       | COMMIT
                       | COMMITTED
+                      | COMPACT
                       | CONFIG
                       | CONFIGS
                       | CONSUMER_GROUP
@@ -148,6 +149,7 @@ query : cypherQuery
       | multiDatabaseQuery
       | showDatabases
       | edgeImportModeQuery
+      | compactMemoryQuery
       ;
 
 authQuery : createRole
@@ -314,6 +316,7 @@ privilege : CREATE
           | STORAGE_MODE
           | MULTI_DATABASE_EDIT
           | MULTI_DATABASE_USE
+          | COMPACT_MEMORY
           ;
 
 granularPrivilege : NOTHING | READ | UPDATE | CREATE_DELETE ;
@@ -482,3 +485,5 @@ dropDatabase : DROP DATABASE databaseName ;
 showDatabases: SHOW DATABASES ;
 
 edgeImportModeQuery : EDGE IMPORT MODE ( ACTIVE | INACTIVE ) ;
+
+compactMemoryQuery : COMPACT MEMORY ;

src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4

@@ -41,6 +41,7 @@ CHECK                   : C H E C K ;
 CLEAR                   : C L E A R ;
 COMMIT                  : C O M M I T ;
 COMMITTED               : C O M M I T T E D ;
+COMPACT                 : C O M P A C T ;
 CONFIG                  : C O N F I G ;
 CONFIGS                 : C O N F I G S;
 CONSUMER_GROUP          : C O N S U M E R UNDERSCORE G R O U P ;

src/query/frontend/semantic/required_privileges.cpp

@@ -69,6 +69,10 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
 
   void Visit(FreeMemoryQuery &free_memory_query) override { AddPrivilege(AuthQuery::Privilege::FREE_MEMORY); }
 
+  void Visit(CompactMemoryQuery & /*compact_memory_query*/) override {
+    AddPrivilege(AuthQuery::Privilege::COMPACT_MEMORY);
+  }
+
   void Visit(ShowConfigQuery & /*show_config_query*/) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
 
   void Visit(TriggerQuery &trigger_query) override { AddPrivilege(AuthQuery::Privilege::TRIGGER); }

src/query/frontend/stripped_lexer_constants.hpp

@@ -219,7 +219,8 @@ const trie::Trie kKeywords = {"union",
                               "directory",
                               "lock",
                               "unlock",
-                              "build"};
+                              "build",
+                              "compact"};
 
 // Unicode codepoints that are allowed at the start of the unescaped name.
 const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(

src/query/interpreter.cpp

@@ -2255,6 +2255,23 @@ PreparedQuery PrepareFreeMemoryQuery(ParsedQuery parsed_query, bool in_explicit_
       RWType::NONE};
 }
 
+PreparedQuery PrepareCompactMemoryQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
+                                        InterpreterContext *interpreter_context) {
+  if (in_explicit_transaction) {
+    throw CompactMemoryModificationInMulticommandTxException();
+  }
+
+  return PreparedQuery{
+      {},
+      std::move(parsed_query.required_privileges),
+      [interpreter_context](AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
+        interpreter_context->db->CompactMemory();
+        // memory::PurgeUnusedMemory();
+        return QueryHandlerResult::COMMIT;
+      },
+      RWType::NONE};
+}
+
 PreparedQuery PrepareShowConfigQuery(ParsedQuery parsed_query, bool in_explicit_transaction) {
   if (in_explicit_transaction) {
     throw ShowConfigModificationInMulticommandTxException();
@@ -3654,6 +3671,9 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
       prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
     } else if (utils::Downcast<FreeMemoryQuery>(parsed_query.query)) {
       prepared_query = PrepareFreeMemoryQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
+    } else if (utils::Downcast<CompactMemoryQuery>(parsed_query.query)) {
+      prepared_query =
+          PrepareCompactMemoryQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
     } else if (utils::Downcast<ShowConfigQuery>(parsed_query.query)) {
       prepared_query = PrepareShowConfigQuery(std::move(parsed_query), in_explicit_transaction_);
     } else if (utils::Downcast<TriggerQuery>(parsed_query.query)) {

src/query/procedure/module.cpp

@@ -849,7 +849,7 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) {
   handle_ = dlopen(file_path.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND);
   if (!handle_) {
     spdlog::error(
-        utils::MessageWithLink("Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules"));
+        utils::MessageWithLink("1 Unable to load module {}; {}.", file_path, dlerror(), "https://memgr.ph/modules"));
     return false;
   }
   // Get required mgp_init_module
@@ -857,7 +857,7 @@ bool SharedLibraryModule::Load(const std::filesystem::path &file_path) {
   char *dl_errored = dlerror();
   if (!init_fn_ || dl_errored) {
     spdlog::error(
-        utils::MessageWithLink("Unable to load module {}; {}.", file_path, dl_errored, "https://memgr.ph/modules"));
+        utils::MessageWithLink("2 Unable to load module {}; {}.", file_path, dl_errored, "https://memgr.ph/modules"));
     dlclose(handle_);
     handle_ = nullptr;
     return false;

src/storage/v2/disk/storage.hpp

@@ -367,6 +367,7 @@ class DiskStorage final : public Storage {
   StorageInfo GetInfo() const override;
 
   void FreeMemory(std::unique_lock<utils::RWLock> /*lock*/) override {}
+  void CompactMemory(std::unique_lock<utils::RWLock> /*lock*/) override {}
 
   void EstablishNewEpoch() override { throw utils::BasicException("Disk storage mode does not support replication."); }
 

src/storage/v2/inmemory/storage.cpp

@@ -10,6 +10,7 @@
 // licenses/APL.txt.
 
 #include "storage/v2/inmemory/storage.hpp"
+#include <memory>
 #include "storage/v2/durability/durability.hpp"
 #include "storage/v2/durability/snapshot.hpp"
 
@@ -17,6 +18,7 @@
 #include "storage/v2/inmemory/replication/replication_client.hpp"
 #include "storage/v2/inmemory/replication/replication_server.hpp"
 #include "storage/v2/inmemory/unique_constraints.hpp"
+#include "utils/exceptions.hpp"
 
 namespace memgraph::storage {
 
@@ -1785,6 +1787,55 @@ void InMemoryStorage::FreeMemory(std::unique_lock<utils::RWLock> main_guard) {
   static_cast<InMemoryLabelPropertyIndex *>(indices_.label_property_index_.get())->RunGC();
 }
 
+void InMemoryStorage::CompactMemory(std::unique_lock<utils::RWLock> main_guard) {
+  // Version 1
+  // 1. Create snapshot.
+  auto snap_err = CreateSnapshot(false);
+  if (snap_err.HasError()) {
+    throw utils::BasicException("Failed to create snapshot.");
+  }
+
+  // 2. Clear Storage.
+  edges_.clear();
+  vertices_.clear();
+  indices_.label_index_ = std::make_unique<InMemoryLabelIndex>(&indices_, config_);
+  indices_.label_property_index_ = std::make_unique<InMemoryLabelPropertyIndex>(&indices_, config_);
+  constraints_.existence_constraints_ = std::make_unique<ExistenceConstraints>();
+  constraints_.unique_constraints_ = std::make_unique<InMemoryUniqueConstraints>();
+  edge_count_.store(0);
+
+  // 3. Recover
+  auto &epoch = replication_state_.GetEpoch();
+  auto info = durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, &epoch.id,
+                                      &replication_state_.history, &vertices_, &edges_, &edge_count_,
+                                      name_id_mapper_.get(), &indices_, &constraints_, config_, &wal_seq_num_);
+  if (info) {
+    vertex_id_ = info->next_vertex_id;
+    edge_id_ = info->next_edge_id;
+    timestamp_ = std::max(timestamp_, info->next_timestamp);
+    if (info->last_commit_timestamp) {
+      replication_state_.last_commit_timestamp_ = *info->last_commit_timestamp;
+    }
+  }
+
+  // Version 2
+  // 1. Take complete control, block everything else.
+  // 2. Force-call garbage collection in both storage modes.
+  // 3. Recreate the skip lists and indices. Order may matter.
+  // 4. Delete old skiplists.
+  // 5. Force-call garbage collection again.
+  // 6. Profit.
+
+  // Version 3
+  // Same as version 1 but intsead revoery ops use rocksdb
+  // to temporarly store the data.
+
+  // MISC
+  // - How does this play together with replication?
+  // - How does this play together with the MemoryChecker?
+  // - How does this play together with multi-tenancy?
+}
+
 uint64_t InMemoryStorage::CommitTimestamp(const std::optional<uint64_t> desired_commit_timestamp) {
   if (!desired_commit_timestamp) {
     return timestamp_++;

src/storage/v2/inmemory/storage.hpp

@@ -361,6 +361,7 @@ class InMemoryStorage final : public Storage {
       LabelId label, const std::set<PropertyId> &properties, std::optional<uint64_t> desired_commit_timestamp) override;
 
   void FreeMemory(std::unique_lock<utils::RWLock> main_guard) override;
+  void CompactMemory(std::unique_lock<utils::RWLock> main_guard) override;
 
   utils::FileRetainer::FileLockerAccessor::ret_type IsPathLocked();
   utils::FileRetainer::FileLockerAccessor::ret_type LockPath();

src/storage/v2/storage.hpp

@@ -254,6 +254,10 @@ class Storage {
 
   void FreeMemory() { FreeMemory({}); }
 
+  virtual void CompactMemory(std::unique_lock<utils::RWLock> main_guard) = 0;
+
+  void CompactMemory() { CompactMemory({}); }
+
   virtual std::unique_ptr<Accessor> Access(std::optional<IsolationLevel> override_isolation_level) = 0;
   std::unique_ptr<Accessor> Access() { return Access(std::optional<IsolationLevel>{}); }
 

src/utils/typeinfo.hpp

@@ -171,6 +171,7 @@ enum class TypeId : uint64_t {
   AST_LOCK_PATH_QUERY,
   AST_LOAD_CSV,
   AST_FREE_MEMORY_QUERY,
+  AST_COMPACT_MEMORY_QUERY,
   AST_TRIGGER_QUERY,
   AST_ISOLATION_LEVEL_QUERY,
   AST_STORAGE_MODE_QUERY,

tests/unit/query_required_privileges.cpp

@@ -169,6 +169,11 @@ TEST_F(TestPrivilegeExtractor, FreeMemoryQuery) {
   EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::FREE_MEMORY));
 }
 
+TEST_F(TestPrivilegeExtractor, CompactMemoryQuery) {
+  auto *query = storage.Create<CompactMemoryQuery>();
+  EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::COMPACT_MEMORY));
+}
+
 TEST_F(TestPrivilegeExtractor, TriggerQuery) {
   auto *query = storage.Create<TriggerQuery>();
   EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::TRIGGER));