A python script that visualizes latency results from LDBC.

Summary:
  * extension of run_benchmark script
  * tools/plot_latency.py

Reviewers: teon.banek

Reviewed By: teon.banek

Subscribers: mislav.bradac, pullbot

Differential Revision: https://phabricator.memgraph.io/D743

tests/public_benchmark/ldbc/ldbc-snb-impls-allshortandupdate.properties

@@ -0,0 +1,71 @@
+# Linked Data Benchmark Council
+# Social Network Benchmark
+# Interactive Workload
+
+# *** workload-related driver properties ***
+workload=com.ldbc.driver.workloads.ldbc.snb.interactive.LdbcSnbInteractiveWorkload
+
+# *** vendor-related driver properties ***
+# database=
+
+# *** workload-specific properties ***
+
+# Directory containing query parameter files
+# ldbc.snb.interactive.parameters_dir=
+
+# Directory containing data update stream files
+# ldbc.snb.interactive.updates_dir=
+
+# Short reads random walk dissipation rate, in the interval [1.0-0.0]
+# Higher values translate to shorter walks and therefore fewer short reads
+ldbc.snb.interactive.short_read_dissipation=0.2
+
+## frequency of read queries (number of update queries per one read query)
+ldbc.snb.interactive.LdbcQuery1_freq=26
+ldbc.snb.interactive.LdbcQuery2_freq=37
+ldbc.snb.interactive.LdbcQuery3_freq=69
+ldbc.snb.interactive.LdbcQuery4_freq=36
+ldbc.snb.interactive.LdbcQuery5_freq=57
+ldbc.snb.interactive.LdbcQuery6_freq=129
+ldbc.snb.interactive.LdbcQuery7_freq=87
+ldbc.snb.interactive.LdbcQuery8_freq=45
+ldbc.snb.interactive.LdbcQuery9_freq=157
+ldbc.snb.interactive.LdbcQuery10_freq=30
+ldbc.snb.interactive.LdbcQuery11_freq=16
+ldbc.snb.interactive.LdbcQuery12_freq=44
+ldbc.snb.interactive.LdbcQuery13_freq=19
+ldbc.snb.interactive.LdbcQuery14_freq=49
+
+# *** For debugging purposes ***
+
+ldbc.snb.interactive.LdbcQuery1_enable=false
+ldbc.snb.interactive.LdbcQuery2_enable=false
+ldbc.snb.interactive.LdbcQuery3_enable=false
+ldbc.snb.interactive.LdbcQuery4_enable=false
+ldbc.snb.interactive.LdbcQuery5_enable=false
+ldbc.snb.interactive.LdbcQuery6_enable=false
+ldbc.snb.interactive.LdbcQuery7_enable=false
+ldbc.snb.interactive.LdbcQuery8_enable=true
+ldbc.snb.interactive.LdbcQuery9_enable=false
+ldbc.snb.interactive.LdbcQuery10_enable=false
+ldbc.snb.interactive.LdbcQuery11_enable=false
+ldbc.snb.interactive.LdbcQuery12_enable=false
+ldbc.snb.interactive.LdbcQuery13_enable=false
+ldbc.snb.interactive.LdbcQuery14_enable=false
+
+ldbc.snb.interactive.LdbcShortQuery1PersonProfile_enable=true
+ldbc.snb.interactive.LdbcShortQuery2PersonPosts_enable=true
+ldbc.snb.interactive.LdbcShortQuery3PersonFriends_enable=true
+ldbc.snb.interactive.LdbcShortQuery4MessageContent_enable=true
+ldbc.snb.interactive.LdbcShortQuery5MessageCreator_enable=true
+ldbc.snb.interactive.LdbcShortQuery6MessageForum_enable=true
+ldbc.snb.interactive.LdbcShortQuery7MessageReplies_enable=true
+
+ldbc.snb.interactive.LdbcUpdate1AddPerson_enable=true
+ldbc.snb.interactive.LdbcUpdate2AddPostLike_enable=true
+ldbc.snb.interactive.LdbcUpdate3AddCommentLike_enable=true
+ldbc.snb.interactive.LdbcUpdate4AddForum_enable=true
+ldbc.snb.interactive.LdbcUpdate5AddForumMembership_enable=true
+ldbc.snb.interactive.LdbcUpdate6AddPost_enable=true
+ldbc.snb.interactive.LdbcUpdate7AddComment_enable=true
+ldbc.snb.interactive.LdbcUpdate8AddFriendship_enable=true

tests/public_benchmark/ldbc/ldbc-snb-impls-allshorts.properties

@@ -62,10 +62,10 @@ ldbc.snb.interactive.LdbcShortQuery6MessageForum_enable=true
 ldbc.snb.interactive.LdbcShortQuery7MessageReplies_enable=true
 
 ldbc.snb.interactive.LdbcUpdate1AddPerson_enable=false
-ldbc.snb.interactive.LdbcUpdate2AddPostLike_enable=false
+ldbc.snb.interactive.LdbcUpdate2AddPostLike_enable=true
-ldbc.snb.interactive.LdbcUpdate3AddCommentLike_enable=false
+ldbc.snb.interactive.LdbcUpdate3AddCommentLike_enable=true
 ldbc.snb.interactive.LdbcUpdate4AddForum_enable=false
-ldbc.snb.interactive.LdbcUpdate5AddForumMembership_enable=false
+ldbc.snb.interactive.LdbcUpdate5AddForumMembership_enable=true
 ldbc.snb.interactive.LdbcUpdate6AddPost_enable=false
 ldbc.snb.interactive.LdbcUpdate7AddComment_enable=false
-ldbc.snb.interactive.LdbcUpdate8AddFriendship_enable=false
+ldbc.snb.interactive.LdbcUpdate8AddFriendship_enable=true

tests/public_benchmark/ldbc/plots/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tests/public_benchmark/ldbc/results/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tests/public_benchmark/ldbc/run_benchmark

@@ -16,14 +16,16 @@ function print_help () {
     echo -e "  --time-compression-ratio |"
     echo -e "  --operation-count        | -> https://github.com/ldbc/ldbc_driver/wiki/Driver-Configuration"
     echo -e "  --thread-count           |"
+    echo -e "  --result-file-prefix -> Result file prefix."
 }
 
 # Default parameters.
 host=127.0.0.1
 port=7687
 time_compression_ratio=0.01
-operation_count=100
+operation_count=200
 thread_count=8
+result_file_prefix="undefined"
 
 # Read the arguments.
 while [[ $# -gt 0 ]]
@@ -41,6 +43,10 @@ do
         port=$2
         shift
         ;;
+        --result-file-prefix)
+        result_file_prefix=$2
+        shift
+        ;;
         --time-compression-ratio)
         time_compression_ratio=$2
         shift
@@ -64,4 +70,6 @@ cd ${script_dir}/ldbc-snb-impls
 mvn clean compile assembly:single
 
 cd ${script_dir}/ldbc_driver
-java -cp target/jeeves-0.3-SNAPSHOT.jar:${script_dir}/ldbc-snb-impls/snb-interactive-neo4j/target/snb-interactive-neo4j-1.0.0-jar-with-dependencies.jar com.ldbc.driver.Client -P ${script_dir}/ldbc_driver/configuration/ldbc_driver_default.properties -P ${script_dir}/ldbc-snb-impls-test.properties -P ${script_dir}/ldbc_snb_datagen/social_network/updateStream.properties -p host ${host} -p port ${port} -db net.ellitron.ldbcsnbimpls.interactive.neo4j.Neo4jDb -p ldbc.snb.interactive.parameters_dir ${script_dir}/ldbc_snb_datagen/substitution_parameters --time_compression_ratio ${time_compression_ratio} --operation_count ${operation_count} --thread_count ${thread_count}
+java -cp target/jeeves-0.3-SNAPSHOT.jar:${script_dir}/ldbc-snb-impls/snb-interactive-neo4j/target/snb-interactive-neo4j-1.0.0-jar-with-dependencies.jar com.ldbc.driver.Client -P ${script_dir}/ldbc_driver/configuration/ldbc_driver_default.properties -P ${script_dir}/ldbc-snb-impls-allshortandupdate.properties -P ${script_dir}/ldbc_snb_datagen/social_network/updateStream.properties -p host ${host} -p port ${port} -db net.ellitron.ldbcsnbimpls.interactive.neo4j.Neo4jDb -p ldbc.snb.interactive.parameters_dir ${script_dir}/ldbc_snb_datagen/substitution_parameters --time_compression_ratio ${time_compression_ratio} --operation_count ${operation_count} --thread_count ${thread_count}
+
+cp ${script_dir}/ldbc_driver/results/LDBC-results.json ${script_dir}/results/${result_file_prefix}-LDBC-results.json

tools/plot_ldbc_latency

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+'''
+Latency Barchart (Based on LDBC JSON output).
+'''
+
+import json
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+from matplotlib.cbook import get_sample_data
+from argparse import ArgumentParser
+
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+COLORS = {
+    'memgraph': '#ff7300',
+    'neo4j': '#008cc2'
+}
+
+
+def parse_args():
+    argp = ArgumentParser(description=__doc__)
+    argp.add_argument("--vendor-references", nargs="+",
+                      help="Short references that represent all the "
+                           "vendors that are going to be "
+                           "visualized on the plot.")
+    argp.add_argument("--vendor-titles", nargs="+",
+                      help="Vender titles that are going to appear "
+                           "on the plot, e.g. legend titles.")
+    argp.add_argument("--plot-title", default="{{Plot title placeholder}}",
+                      help="Plot title.")
+    argp.add_argument("--logo-path", default=None,
+                      help="Path to the logo that is going to be presented"
+                           " instead of title.")
+    argp.add_argument("--results-dir",
+                      default=os.path.join(SCRIPT_DIR,
+                                           "../tests/public_benchmark"
+                                           "/ldbc/results"),
+                      help="Path to the folder with result files in format "
+                           "{{vendor-reference}}-LDBC-results.json")
+    return argp.parse_args()
+
+
+def autolabel(ax, rects):
+    """
+    Attach a text label above each bar displaying its height
+    """
+    for rect in rects:
+        height = rect.get_height()
+        # TODO: adjust more vendors
+        ax.text(rect.get_x() + rect.get_width()/2., 1.00*height,
+                '%d' % int(height),
+                ha='center', va='bottom')
+
+
+def main():
+    # Read the arguments.
+    args = parse_args()
+
+    # Prepare the datastructure.
+    vendors = {}
+    for vendor_reference, vendor_title in zip(args.vendor_references,
+                                              args.vendor_titles):
+        vendors[vendor_reference] = {}
+        vendors[vendor_reference]['title'] = vendor_title
+        vendors[vendor_reference]['results_path'] = os.path.join(
+            args.results_dir, "%s-LDBC-results.json" % vendor_reference)
+        vendors[vendor_reference]['color'] = COLORS[vendor_reference]
+        vendors[vendor_reference]['latencies'] = []
+        vendors[vendor_reference]['query_names'] = []
+    assert len(vendors) == 2, "The graph is tailored for only 2 vendors."
+
+    # Collect the benchmark data.
+    print("LDBC Latency Data")
+    for vendor_reference, vendor_data in vendors.items():
+        print("Vendor: %s" % vendor_reference)
+        with open(vendor_data['results_path']) as results_file:
+            results_data = json.load(results_file)
+            for query_data in results_data["all_metrics"]:
+                mean_runtime = query_data["run_time"]["mean"]
+                query_name = query_data['name']
+                print("%s -> %sms" % (query_name, str(mean_runtime)))
+                vendor_data['latencies'].append(mean_runtime)
+                vendor_data['query_names'].append(query_name)
+
+    # Consistency check.
+    all_query_names = [tuple(vd['query_names']) for vd in vendors.values()]
+    assert len(set(all_query_names)) == 1, \
+        "Queries between different vendors are different!"
+    query_names = all_query_names[0]
+
+    # Plot.
+    ind = np.arange(len(query_names))   # the x locations for the groups
+    width = 0.40                        # the width of the bars
+    fig, ax = plt.subplots()            # figure setup
+    ax.set_ylabel('Mean Latency (ms)')  # YAxis title
+    ax.set_facecolor('#dcdcdc')         # plot bg color (light gray)
+    ax.set_xticks(ind + width / len(vendors))  # TODO: adjust (more vendors)
+    # IMPORTANT! Long query names on the XAxis don't look compelling.
+    ax.set_xticklabels(['Q%s' % x for x in range(len(query_names))])
+    # set only horizontal grid lines
+    for line in ax.get_xgridlines():
+        line.set_linestyle(' ')
+    for line in ax.get_ygridlines():
+        line.set_linestyle('--')
+    ax.set_axisbelow(True)              # put the grid below all other elements
+    plt.grid(True)                      # show grid
+    # Draw logo or plot title
+    if args.logo_path is None:
+        ax.set_title(args.plot_title)
+    else:
+        # TODO: improve the logo positioning
+        im = plt.imread(get_sample_data(args.logo_path))
+        plt.gcf().subplots_adjust(top=0.85)
+        newax = fig.add_axes([0.4, 0.75, 0.2, 0.25], anchor='N')
+        newax.imshow(im)
+        newax.axis('off')
+    # Draw bars
+    for index, vendor_data in enumerate(vendors.values()):
+        rects = ax.bar(ind + index * width, vendor_data['latencies'], width,
+                       color=vendor_data['color'])
+        vendor_data['rects'] = rects
+        autolabel(ax, rects)
+    rects = [vd['rects'][0] for vd in vendors.values()]
+    titles = [vd['title'] for vd in vendors.values()]
+    ax.legend(rects, titles)           # Draw the legend.
+    plt.show()
+
+if __name__ == '__main__':
+    main()