From 4788a633a66963c54306aa950a6cfa7e8c2ade83 Mon Sep 17 00:00:00 2001
From: Andi <andi8647@gmail.com>
Date: Tue, 2 Jan 2024 13:29:25 +0100
Subject: [PATCH] Improve e2e Kafka and Pulsar testing (#1604)

---
 .github/workflows/diff.yaml                 | 19 ++++++++++++++----
 tests/e2e/streams/kafka/docker-compose.yml  | 22 +++++++++++++++++++++
 tests/e2e/streams/pulsar/docker-compose.yml |  9 +++++++++
 3 files changed, 46 insertions(+), 4 deletions(-)
 create mode 100755 tests/e2e/streams/kafka/docker-compose.yml
 create mode 100644 tests/e2e/streams/pulsar/docker-compose.yml

diff --git a/.github/workflows/diff.yaml b/.github/workflows/diff.yaml
index 340d11711..ba00941b8 100644
--- a/.github/workflows/diff.yaml
+++ b/.github/workflows/diff.yaml
@@ -229,10 +229,6 @@ jobs:
           # branches and tags. (default: 1)
           fetch-depth: 0
 
-      - name: Check e2e service dependencies
-        run: |
-          cd tests/e2e
-          ./dependency_check.sh
 
       - name: Build release binaries
         run: |
@@ -271,6 +267,13 @@ jobs:
           cd build
           ctest -R memgraph__unit --output-on-failure -j$THREADS
 
+      - name: Ensure Kafka and Pulsar are up
+        run: |
+          cd tests/e2e/streams/kafka
+          docker-compose up -d
+          cd ../pulsar
+          docker-compose up -d
+
       - name: Run e2e tests
         run: |
           cd tests
@@ -279,6 +282,14 @@ jobs:
           cd e2e
           ./run.sh
 
+      - name: Ensure Kafka and Pulsar are down
+        if: always()
+        run: |
+          cd tests/e2e/streams/kafka
+          docker-compose down
+          cd ../pulsar
+          docker-compose down
+
       - name: Run stress test (plain)
         run: |
           cd tests/stress
diff --git a/tests/e2e/streams/kafka/docker-compose.yml b/tests/e2e/streams/kafka/docker-compose.yml
new file mode 100755
index 000000000..3ab2166af
--- /dev/null
+++ b/tests/e2e/streams/kafka/docker-compose.yml
@@ -0,0 +1,22 @@
+version: '3'
+services:
+  zookeeper:
+    image: 'bitnami/zookeeper:3.9.1'
+    ports:
+      - '2181:2181'
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+  kafka:
+    image: 'bitnami/kafka:3.6.1'
+    ports:
+      - '9092:9092'
+      - '29092:29092'
+    environment:
+      - KAFKA_BROKER_ID=1
+      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,PLAINTEXT_HOST://:29092
+      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092,PLAINTEXT_HOST://localhost:29092
+      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    depends_on:
+      - zookeeper
diff --git a/tests/e2e/streams/pulsar/docker-compose.yml b/tests/e2e/streams/pulsar/docker-compose.yml
new file mode 100644
index 000000000..a490fbdb1
--- /dev/null
+++ b/tests/e2e/streams/pulsar/docker-compose.yml
@@ -0,0 +1,9 @@
+version: "3"
+
+services:
+  pulsar:
+    image: 'apachepulsar/pulsar:2.8.1'
+    ports:
+      - '6652:8080'
+      - '6650:6650'
+    entrypoint: [ 'bin/pulsar', 'standalone' ]