Add multiple license support (#618)

Make license info available through LicenseChecker
Add LicenseInfoSender
Move license library from utils
Rename telemetry_lib to mg-telemetry

tests/integration/CMakeLists.txt

@@ -13,24 +13,6 @@ add_subdirectory(auth)
 # lba test binaries
 add_subdirectory(fine_grained_access)
 
-## distributed ha/basic binaries
-#add_subdirectory(ha/basic)
-#
-## distributed ha/constraints binaries
-#add_subdirectory(ha/constraints)
-#
-## distributed ha/index binaries
-#add_subdirectory(ha/index)
-#
-## distributed ha/large_log_entries binaries
-#add_subdirectory(ha/large_log_entries)
-#
-## distributed ha/leader_election binaries
-#add_subdirectory(ha/leader_election)
-#
-## distributed ha/term_updates binaries
-#add_subdirectory(ha/term_updates)
-
 # audit test binaries
 add_subdirectory(audit)
 
@@ -39,3 +21,6 @@ add_subdirectory(ldap)
 
 # mg_import_csv test binaries
 add_subdirectory(mg_import_csv)
+
+# license_check test binaries
+add_subdirectory(license_info)

tests/integration/license_info/CMakeLists.txt

@@ -0,0 +1,6 @@
+set(target_name memgraph__integration__license_info)
+set(client_target_name ${target_name}__client)
+
+add_executable(${client_target_name} client.cpp)
+set_target_properties(${client_target_name} PROPERTIES OUTPUT_NAME client)
+target_link_libraries(${client_target_name} mg-requests mg-license mg-utils)

tests/integration/license_info/client.cpp

@@ -0,0 +1,62 @@
+// Copyright 2022 Memgraph Ltd.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+// License, and you may not use this file except in compliance with the Business Source License.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+#include <exception>
+#include <string>
+
+#include <gflags/gflags.h>
+
+#include "license/license.hpp"
+#include "license/license_sender.hpp"
+#include "requests/requests.hpp"
+#include "spdlog/spdlog.h"
+#include "utils/logging.hpp"
+#include "utils/synchronized.hpp"
+#include "utils/system_info.hpp"
+#include "utils/uuid.hpp"
+
+DEFINE_string(endpoint, "http://127.0.0.1:5500/", "Endpoint that should be used for the test.");
+DEFINE_string(license_type, "enterprise", "License type; can be oem or enterprise.");
+DEFINE_int64(interval, 1, "Interval used for reporting telemetry in seconds.");
+DEFINE_int64(duration, 10, "Duration of the test in seconds.");
+
+memgraph::license::LicenseType StringToLicenseType(const std::string_view license_type) {
+  if (license_type == "enterprise") {
+    return memgraph::license::LicenseType::ENTERPRISE;
+  }
+  if (license_type == "oem") {
+    return memgraph::license::LicenseType::OEM;
+  }
+  spdlog::critical("Invalid license type!");
+  std::terminate();
+}
+
+int main(int argc, char **argv) {
+  gflags::SetVersionString("license-info");
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+
+  memgraph::requests::Init();
+
+  memgraph::license::License license{"Memgraph", 0, 0, StringToLicenseType(FLAGS_license_type)};
+  memgraph::utils::Synchronized<std::optional<memgraph::license::LicenseInfo>, memgraph::utils::SpinLock> license_info{
+      memgraph::license::LicenseInfo{"mg-testkey", "Memgraph"}};
+  license_info.WithLock([license = std::move(license)](auto &license_info) {
+    license_info->license = license;
+    license_info->is_valid = true;
+  });
+
+  memgraph::license::LicenseInfoSender license_sender(FLAGS_endpoint, memgraph::utils::GenerateUUID(),
+                                                      memgraph::utils::GetMachineId(), 10000000, license_info,
+                                                      std::chrono::seconds(FLAGS_interval));
+  std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration));
+
+  return 0;
+}

tests/integration/license_info/runner.py

@@ -0,0 +1,96 @@
+#!/usr/bin/python3 -u
+
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+PROJECT_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", ".."))
+
+
+def execute_test(**kwargs):
+    client_binary = kwargs.pop("client")
+    server_binary = kwargs.pop("server")
+
+    start_server = kwargs.pop("start_server", True)
+    interval = kwargs.pop("interval", 1)
+    duration = kwargs.pop("duration", 5)
+    license_type = kwargs.pop("license-type", "enterprise")
+
+    timeout = duration * 2 if "hang" not in kwargs else duration * 2 + 60
+    success = False
+
+    client_args = [client_binary, "--interval", interval, "--duration", duration, "--license-type", license_type]
+
+    server = None
+    if start_server:
+        server = subprocess.Popen(server_binary)
+        time.sleep(0.4)
+        assert server.poll() is None, "Server process died prematurely!"
+
+    try:
+        subprocess.run(list(map(str, client_args)), timeout=timeout, check=True)
+    finally:
+        if server is None:
+            success = True
+        else:
+            server.terminate()
+            try:
+                success = server.wait(timeout=5) == 0
+                success = True
+            except subprocess.TimeoutExpired:
+                server.kill()
+    return success
+
+
+def main():
+    server_binary = os.path.join(SCRIPT_DIR, "server.py")
+    client_binary = os.path.join(PROJECT_DIR, "build", "tests", "integration", "license_info", "client")
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--client", default=client_binary)
+    parser.add_argument("--server", default=server_binary)
+    parser.add_argument("--server-url", default="127.0.0.1")
+    parser.add_argument("--server-port", default="5500")
+    args = parser.parse_args()
+
+    tests = [
+        {"interval": 2},
+        {"duration": 10},
+        {"interval": 2, "duration": 10},
+        {"license-type": "oem"},
+        {"license-type": "enterprise"},
+    ]
+    for test in tests:
+        print("\033[1;36m~~ Executing test with arguments:", json.dumps(test, sort_keys=True), "~~\033[0m")
+
+        try:
+            success = execute_test(client=args.client, server=args.server, **test)
+        except Exception as e:
+            print("\033[1;33m", e, "\033[0m", sep="")
+            success = False
+
+        if not success:
+            print("\033[1;31m~~", "Test failed!", "~~\033[0m")
+            sys.exit(1)
+        else:
+            print("\033[1;32m~~", "Test ok!", "~~\033[0m")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

tests/integration/license_info/server.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python3 -u
+
+# Copyright 2021 Memgraph Ltd.
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
+# License, and you may not use this file except in compliance with the Business Source License.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+import argparse
+import json
+from http.server import HTTPServer, SimpleHTTPRequestHandler
+
+EXPECTED_LICENSE_INFO_FIELDS = {
+    "run_id": str,
+    "machine_id": str,
+    "type": str,
+    "license_type": str,
+    "license_key": str,
+    "organization": str,
+    "valid": bool,
+    "physical_memory_size": int,
+    "swap_memory_size": int,
+    "memory_used": int,
+    "runtime_memory_limit": int,
+    "license_memory_limit": int,
+    "timestamp": float,
+}
+
+
+class ServerHandler(SimpleHTTPRequestHandler):
+    def do_POST(self):
+        assert self.headers["user-agent"] == "memgraph/license-info", f"The header is {self.headers['user-agent']}"
+        assert self.headers["accept"] == "application/json", f"The header is {self.headers['accept']}"
+        assert self.headers["content-type"] == "application/json", f"The header is {self.headers['content-type']}"
+
+        content_len = int(self.headers.get("content-length", 0))
+        data = json.loads(self.rfile.read(content_len).decode("utf-8"))
+
+        assert isinstance(data, dict)
+
+        for expected_field, expected_type in EXPECTED_LICENSE_INFO_FIELDS.items():
+            assert expected_field in data, f"Field {expected_field} not found in received data"
+            assert isinstance(
+                data[expected_field], expected_type
+            ), f"Field {expected_field} is not correct type: expected {expected_type} got {type(data[expected_field])}"
+        assert len(EXPECTED_LICENSE_INFO_FIELDS) == len(data), "Expected data size does not match received"
+
+        self.send_response(200)
+        self.end_headers()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--address", type=str, default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=5500)
+    args = parser.parse_args()
+
+    with HTTPServer((args.address, args.port), ServerHandler) as srv:
+        print(f"Serving HTTP server at {args.address}:{args.port}")
+        srv.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

tests/integration/telemetry/CMakeLists.txt

@@ -3,4 +3,4 @@ set(client_target_name ${target_name}__client)
 
 add_executable(${client_target_name} client.cpp)
 set_target_properties(${client_target_name} PROPERTIES OUTPUT_NAME client)
-target_link_libraries(${client_target_name} mg-requests telemetry_lib)
+target_link_libraries(${client_target_name} mg-requests mg-telemetry)

tests/integration/telemetry/client.cpp

@@ -13,6 +13,8 @@
 
 #include "requests/requests.hpp"
 #include "telemetry/telemetry.hpp"
+#include "utils/system_info.hpp"
+#include "utils/uuid.hpp"
 
 DEFINE_string(endpoint, "http://127.0.0.1:9000/", "Endpoint that should be used for the test.");
 DEFINE_int64(interval, 1, "Interval used for reporting telemetry in seconds.");
@@ -24,8 +26,8 @@ int main(int argc, char **argv) {
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
   memgraph::requests::Init();
-  memgraph::telemetry::Telemetry telemetry(FLAGS_endpoint, FLAGS_storage_directory,
-                                           std::chrono::seconds(FLAGS_interval), 1);
+  memgraph::telemetry::Telemetry telemetry(FLAGS_endpoint, FLAGS_storage_directory, memgraph::utils::GenerateUUID(),
+                                           memgraph::utils::GetMachineId(), std::chrono::seconds(FLAGS_interval), 1);
 
   uint64_t counter = 0;
   telemetry.AddCollector("db", [&counter]() -> nlohmann::json {

tests/integration/telemetry/runner.py

@@ -36,8 +36,7 @@ def execute_test(**kwargs):
     timeout = duration * 2 if "hang" not in kwargs else duration * 2 + 60
     success = False
 
-    server_args = [server_binary, "--interval", interval,
-                   "--duration", duration]
+    server_args = [server_binary, "--interval", interval, "--duration", duration]
     for flag, value in kwargs.items():
         flag = "--" + flag.replace("_", "-")
         # We handle boolean flags here. The type of value must be `bool`, and
@@ -48,9 +47,15 @@ def execute_test(**kwargs):
         else:
             server_args.extend([flag, value])
 
-    client_args = [client_binary, "--interval", interval,
-                   "--duration", duration,
-                   "--storage-directory", storage_directory]
+    client_args = [
+        client_binary,
+        "--interval",
+        interval,
+        "--duration",
+        duration,
+        "--storage-directory",
+        storage_directory,
+    ]
     if endpoint:
         client_args.extend(["--endpoint", endpoint])
 
@@ -61,8 +66,7 @@ def execute_test(**kwargs):
         assert server.poll() is None, "Server process died prematurely!"
 
     try:
-        subprocess.run(list(map(str, client_args)), timeout=timeout,
-                       check=True)
+        subprocess.run(list(map(str, client_args)), timeout=timeout, check=True)
     finally:
         if server is None:
             success = True
@@ -88,16 +92,14 @@ TESTS = [
     {"endpoint": "http://127.0.0.1:9000/nonexistant/", "no_check": True},
     {"start_server": False},
     {"startups": 4, "no_check_duration": True},  # the last 3 tests failed
-                                                 # to send any data + this test
-    {"add_garbage": True}
+    # to send any data + this test
+    {"add_garbage": True},
 ]
 
 if __name__ == "__main__":
     server_binary = os.path.join(SCRIPT_DIR, "server.py")
-    client_binary = os.path.join(PROJECT_DIR, "build", "tests",
-                                 "integration", "telemetry", "client")
-    kvstore_console_binary = os.path.join(PROJECT_DIR, "build", "tests",
-                                          "manual", "kvstore_console")
+    client_binary = os.path.join(PROJECT_DIR, "build", "tests", "integration", "telemetry", "client")
+    kvstore_console_binary = os.path.join(PROJECT_DIR, "build", "tests", "manual", "kvstore_console")
 
     parser = argparse.ArgumentParser()
     parser.add_argument("--client", default=client_binary)
@@ -108,19 +110,17 @@ if __name__ == "__main__":
     storage = tempfile.TemporaryDirectory()
 
     for test in TESTS:
-        print("\033[1;36m~~ Executing test with arguments:",
-              json.dumps(test, sort_keys=True), "~~\033[0m")
+        print("\033[1;36m~~ Executing test with arguments:", json.dumps(test, sort_keys=True), "~~\033[0m")
 
         if test.pop("add_garbage", False):
-            proc = subprocess.Popen([args.kvstore_console, "--path",
-                                     storage.name], stdin=subprocess.PIPE,
-                                    stdout=subprocess.DEVNULL)
+            proc = subprocess.Popen(
+                [args.kvstore_console, "--path", storage.name], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL
+            )
             proc.communicate("put garbage garbage".encode("utf-8"))
             assert proc.wait() == 0
 
         try:
-            success = execute_test(client=args.client, server=args.server,
-                                   storage=storage.name, **test)
+            success = execute_test(client=args.client, server=args.server, storage=storage.name, **test)
         except Exception as e:
             print("\033[1;33m", e, "\033[0m", sep="")
             success = False

tests/integration/telemetry/server.py

@@ -12,13 +12,12 @@
 # licenses/APL.txt.
 
 import argparse
+import itertools
 import json
 import os
 import signal
 import sys
 import time
-import itertools
-
 from http.server import BaseHTTPRequestHandler, HTTPServer
 
 
@@ -46,7 +45,7 @@ def build_handler(storage, args):
             assert self.headers["accept"] == "application/json"
             assert self.headers["content-type"] == "application/json"
 
-            content_len = int(self.headers.get('content-length', 0))
+            content_len = int(self.headers.get("content-length", 0))
             data = json.loads(self.rfile.read(content_len).decode("utf-8"))
 
             if self.path not in [args.path, args.redirect_path]:
@@ -70,6 +69,7 @@ def build_handler(storage, args):
                 assert type(item) == dict
                 assert "event" in item
                 assert "run_id" in item
+                assert "type" in item
                 assert "machine_id" in item
                 assert "data" in item
                 assert "timestamp" in item
@@ -188,11 +188,11 @@ if __name__ == "__main__":
         startups[-1].append(item)
 
     # Check that there were the correct number of startups.
-    assert len(startups) == args.startups
+    assert len(startups) == args.startups, f"Expected: {args.startups}, actual: {len(startups)}"
 
     # Verify each startup.
     for startup in startups:
         verify_storage(startup, args)
 
     # machine id has to be same for every run on the same machine
-    assert len(set(map(lambda x: x['machine_id'], itertools.chain(*startups)))) == 1
+    assert len(set(map(lambda x: x["machine_id"], itertools.chain(*startups)))) == 1