tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

System replication experimental flag (#1702)

Browse Source 
- Remove the compile time control
- Introduce the runtime control flag

New flag `--experimental-enabled=system-replication`
This commit is contained in:
Gareth Andrew Lloyd
2024-02-13 12:57:18 +00:00
committed by GitHub
parent 4a7c7f0898
commit f48151576b
27 changed files with 389 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/e2e/CMakeLists.txt
View File

@@ -81,10 +81,7 @@ if (MG_EXPERIMENTAL_HIGH_AVAILABILITY)
add_subdirectory(high_availability_experimental)
endif ()
if (MG_EXPERIMENTAL_REPLICATION_MULTITENANCY)
add_subdirectory(replication_experimental)
endif ()
add_subdirectory(replication_experimental)
copy_e2e_python_files(pytest_runner pytest_runner.sh "")
copy_e2e_python_files(x x.sh "")

5
tests/e2e/configuration/default_config.py
View File

@@ -222,4 +222,9 @@ startup_config_dict = {
"128",
"The threshold for when to cache long delta chains. This is used for heavy read + write workloads where repeated processing of delta chains can become costly.",
),
"experimental_enabled": (
"",
"",
"Experimental features to be used, comma seperated. Options [system-replication]",
),
}

18
tests/e2e/replication_experimental/auth.py
View File

@@ -147,6 +147,7 @@ def test_auth_queries_on_replica(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -160,6 +161,7 @@ def test_auth_queries_on_replica(connection):
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -173,6 +175,7 @@ def test_auth_queries_on_replica(connection):
},
"main": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
@@ -213,6 +216,7 @@ def test_manual_users_recovery(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -227,6 +231,7 @@ def test_manual_users_recovery(connection):
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -241,6 +246,7 @@ def test_manual_users_recovery(connection):
},
"main": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
@@ -287,6 +293,7 @@ def test_env_users_recovery(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -300,6 +307,7 @@ def test_env_users_recovery(connection):
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -315,6 +323,7 @@ def test_env_users_recovery(connection):
"username": "user1",
"password": "password",
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
@@ -368,6 +377,7 @@ def test_manual_roles_recovery(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -382,6 +392,7 @@ def test_manual_roles_recovery(connection):
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -398,6 +409,7 @@ def test_manual_roles_recovery(connection):
},
"main": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
@@ -455,6 +467,7 @@ def test_auth_config_recovery(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -475,6 +488,7 @@ def test_auth_config_recovery(connection):
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -497,6 +511,7 @@ def test_auth_config_recovery(connection):
},
"main": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
@@ -550,6 +565,7 @@ def test_auth_replication(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -563,6 +579,7 @@ def test_auth_replication(connection):
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -576,6 +593,7 @@ def test_auth_replication(connection):
},
"main": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",

124
tests/e2e/replication_experimental/multitenancy.py
View File

@@ -39,6 +39,7 @@ def create_memgraph_instances_with_role_recovery(data_directory: Any) -> Dict[st
return {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level",
@@ -53,6 +54,7 @@ def create_memgraph_instances_with_role_recovery(data_directory: Any) -> Dict[st
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -65,7 +67,12 @@ def create_memgraph_instances_with_role_recovery(data_directory: Any) -> Dict[st
"data_directory": f"{data_directory}/replica_2",
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [],
},
@@ -98,6 +105,7 @@ TEMP_DIR = tempfile.TemporaryDirectory().name
MEMGRAPH_INSTANCES_DESCRIPTION_WITH_RECOVERY = {
"replica_1": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
@@ -109,6 +117,7 @@ MEMGRAPH_INSTANCES_DESCRIPTION_WITH_RECOVERY = {
},
"replica_2": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
@@ -120,6 +129,7 @@ MEMGRAPH_INSTANCES_DESCRIPTION_WITH_RECOVERY = {
},
"main": {
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
@@ -206,7 +216,12 @@ def test_manual_databases_create_multitenancy_replication(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_1']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
],
"log_file": "replica1.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -215,7 +230,12 @@ def test_manual_databases_create_multitenancy_replication(connection):
],
},
"replica_2": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_2']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
],
"log_file": "replica2.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -224,7 +244,12 @@ def test_manual_databases_create_multitenancy_replication(connection):
],
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -281,7 +306,12 @@ def test_manual_databases_create_multitenancy_replication_branching(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_1']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
],
"log_file": "replica1.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -294,7 +324,12 @@ def test_manual_databases_create_multitenancy_replication_branching(connection):
],
},
"replica_2": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_2']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
],
"log_file": "replica2.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -307,7 +342,12 @@ def test_manual_databases_create_multitenancy_replication_branching(connection):
],
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -354,7 +394,12 @@ def test_manual_databases_create_multitenancy_replication_dirty_replica(connecti
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_1']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
],
"log_file": "replica1.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -364,7 +409,12 @@ def test_manual_databases_create_multitenancy_replication_dirty_replica(connecti
],
},
"replica_2": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_2']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
],
"log_file": "replica2.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -374,7 +424,12 @@ def test_manual_databases_create_multitenancy_replication_dirty_replica(connecti
],
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -416,7 +471,12 @@ def test_manual_databases_create_multitenancy_replication_main_behind(connection
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_1']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
],
"log_file": "replica1.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -426,7 +486,12 @@ def test_manual_databases_create_multitenancy_replication_main_behind(connection
],
},
"replica_2": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_2']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_2']}",
"--log-level=TRACE",
],
"log_file": "replica2.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -436,7 +501,12 @@ def test_manual_databases_create_multitenancy_replication_main_behind(connection
],
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [
f"REGISTER REPLICA replica_1 SYNC TO '127.0.0.1:{REPLICATION_PORTS['replica_1']}';",
@@ -532,14 +602,24 @@ def test_automatic_databases_multitenancy_replication_predefined(connection):
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_1']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
],
"log_file": "replica1.log",
"setup_queries": [
f"SET REPLICATION ROLE TO REPLICA WITH PORT {REPLICATION_PORTS['replica_1']};",
],
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [
"CREATE DATABASE A;",
@@ -585,14 +665,24 @@ def test_automatic_databases_create_multitenancy_replication_dirty_main(connecti
MEMGRAPH_INSTANCES_DESCRIPTION_MANUAL = {
"replica_1": {
"args": ["--bolt-port", f"{BOLT_PORTS['replica_1']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['replica_1']}",
"--log-level=TRACE",
],
"log_file": "replica1.log",
"setup_queries": [
f"SET REPLICATION ROLE TO REPLICA WITH PORT {REPLICATION_PORTS['replica_1']};",
],
},
"main": {
"args": ["--bolt-port", f"{BOLT_PORTS['main']}", "--log-level=TRACE"],
"args": [
"--experimental-enabled=system-replication",
"--bolt-port",
f"{BOLT_PORTS['main']}",
"--log-level=TRACE",
],
"log_file": "main.log",
"setup_queries": [
"CREATE DATABASE A;",