integrating data streamer stack and clickhouse stack

2025-12-06 09:04:06 +00:00 · 2024-11-27 10:07:30 -07:00 · 2024-11-27 10:07:30 -07:00 · 4caeda48c2
commit 4caeda48c2
parent b701e239d2
4 changed files with 59 additions and 194 deletions
--- a/clickhouse/docker-compose-old.yaml
+++ b/clickhouse/docker-compose-old.yaml
@ -4,7 +4,7 @@ services:
    container_name: clickhouse-keeper1
    command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
    volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
      - clickhouse_keeper1_data:/var/lib/clickhouse

    networks:
@ -17,7 +17,7 @@ services:
    container_name: clickhouse-keeper2
    command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
    volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
      - clickhouse_keeper2_data:/var/lib/clickhouse
    networks:
      clickhouse-keeper-network:
@ -29,7 +29,7 @@ services:
    container_name: clickhouse-keeper3
    command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
    volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
      - clickhouse_keeper3_data:/var/lib/clickhouse
    networks:
      clickhouse-keeper-network:
@ -40,7 +40,7 @@ services:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-server1
    volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
      - clickhouse_server1_data:/var/lib/clickhouse
      - clickhouse_server1_TTL:/clickhouse_data/server1
    networks:
@ -50,6 +50,9 @@ services:
      clickhouse-keeper-network:
        aliases:
          - clickhouse-server1
+      data_network:
+        aliases:
+          - clickhouse-server1
    deploy:
      replicas: 1
      # placement:
@ -64,15 +67,16 @@ services:
      - clickhouse-keeper1
      - clickhouse-keeper2
      - clickhouse-keeper3
+      - data-streamer
    ports:
-      - "9001:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
+      - "9001:9000" # Native client port
+      - "8123:8123" # HTTP interface

  clickhouse-server2:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-server2
    volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
      - clickhouse_server2_data:/var/lib/clickhouse
      - clickhouse_server2_TTL:/clickhouse_data/server2
    networks:
@ -97,8 +101,8 @@ services:
      - clickhouse-keeper2
      - clickhouse-keeper3
    ports:
-      - "9002:9000"  # Native client port
-      - "8124:8123"  # HTTP interface
+      - "9002:9000" # Native client port
+      - "8124:8123" # HTTP interface

 networks:
  clickhouse-server-network:
--- a/clickhouse/docker-compose.yml
+++ b/clickhouse/docker-compose.yml
@ -1,155 +0,0 @@
-services:
-  clickhouse-keeper1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper1
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper1
-
-  clickhouse-keeper2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper2
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper2
-
-  clickhouse-keeper3:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper3
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper3
-
-  clickhouse-server1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server1
-    volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data1:/var/lib/clickhouse
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-server1
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9001:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
-
-  clickhouse-server2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server2
-    volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data2:/var/lib/clickhouse
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-server2
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9002:9000"  # Native client port
-      - "8125:8123"  # HTTP interface
-
-  zookeeper:
-    image: confluentinc/cp-zookeeper:latest
-    networks:
-      common-network:
-        aliases:
-          - zookeeper
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2182
-    ports:
-      - "2182:2181"
-
-  kafka:
-    image: confluentinc/cp-kafka:latest
-    depends_on:
-      - zookeeper
-    environment:
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2182
-      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      KAFKA_BROKER_ID: 1
-      KAFKA_MESSAGE_MAX_BYTES: 200000000
-      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    networks:
-      common-network:
-        aliases:
-          - kafka
-    ports:
-      - "9092:9092"
-    volumes:
-      - kafka_data_new:/var/lib/kafka/data
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-
-  pcap_streamer:
-    image: levenshtein/streamer_test7:latest
-    depends_on:
-      - kafka
-    networks:
-      common-network:
-        aliases:
-          - pcap_streamer
-    volumes:
-      #- "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project:/data/pcap"
-      - "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project/project_github/real-time-traffic-analysis-clickhouse/preprocessing:/data/pcap"
-    command: ["sh", "-c", "sleep 60 && python /app/pcap_processor.py -c /data/pcap/sample_output.csv -s --stream_size 1000"]
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-
-networks:
-  common-network:
-    driver: overlay
-    attachable: true
-
-volumes:
-  clickhouse_data1:
-    driver: local
-  clickhouse_data2:
-    driver: local
-  kafka_data_new:
-    driver: local
--- a/preprocessing/docker-compose.yml
+++ b/preprocessing/docker-compose.yml
@ -2,7 +2,7 @@ services:
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    networks:
-      kafka_network:
+      data-network:
        aliases:
          - zookeeper
    deploy:
@ -13,6 +13,8 @@ services:
      ZOOKEEPER_CLIENT_PORT: 2181
    ports:
      - "2181:2181"
+    volumes:
+      - zk_data:/var/lib/zookeeper/data

  kafka:
    image: confluentinc/cp-kafka:latest
@ -27,7 +29,7 @@ services:
      KAFKA_MESSAGE_MAX_BYTES: 200000000
      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
    networks:
-      kafka_network:
+      data-network:
        aliases:
          - kafka
    ports:
@ -39,16 +41,16 @@ services:
      restart_policy:
        condition: on-failure

-  data_streamer:
-    image: 127.0.0.1:5000/data_streamer:latest
+  data-streamer:
+    image: 127.0.0.1:5000/data-streamer:latest
    depends_on:
      - kafka
    networks:
-      kafka_network:
+      data-network:
        aliases:
-          - data_streamer
+          - data-streamer
    volumes:
-      - "./10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
+      - "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
    command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'"
    deploy:
      replicas: 1
@ -56,10 +58,12 @@ services:
        condition: on-failure

 networks:
-  kafka_network:
+  data-network:
    driver: overlay
    attachable: true

 volumes:
+  zk_data:
+    driver: local
  kafka_data:
    driver: local
--- a/scripts/deploy.ps1
+++ b/scripts/deploy.ps1
@ -1,34 +1,46 @@
 param (
 	[switch] $MasterNode,
+	[switch] $downStack,
 	[string] $SwarmToken,
 	[string] $ManagerAddr
 )

-$script_dir = $PSScriptRoot
-# Write-Output $script_dir # ===> \Project\scripts
+$scriptDir = $PSScriptRoot
+# Write-Output $scriptDir # ===> \Project\scripts

-if ($MasterNode) {
-	Write-Output "Initializing Docker Swarm..."
+$stackName = "TheWebFarm"

-	docker stack rm test_datastreamer_automated
+if ($downStack)	{
+	Write-Output "[+] Removing stack..."
+	docker stack rm $stackName
+	docker service rm registry
+}
+elseif ($MasterNode) {
+	Write-Output "[+] swarm master"
+
+	# cleanup
+	docker stack rm $stackName
 	docker service rm registry

-	# registry
-	Set-Location $script_dir/../preprocessing
-	
-	docker service create --name registry -p 5000:5000 registry:2
-	docker build -t 127.0.0.1:5000/data_streamer:latest --no-cache --push -f Dockerfile.python .
-	
-	docker stack deploy -d -c docker-compose.yml test_datastreamer_automated
-	
-	Set-Location $script_dir
-	
 	# data streaming
+	Set-Location $scriptDir/../preprocessing
+	docker service create --name registry -p 5000:5000 registry:2
+	docker build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python .

-	# pip install -r "$script_dir/../final/config_update_scripts/requirements.txt"
+	# execute
+	Set-Location $scriptDir
+	docker stack deploy -d `
+		-c ../preprocessing/docker-compose.yml `
+		-c ../clickhouse/docker-compose-old.yaml `
+		$stackName
+
+	# scripts
+	# pip install -r "$scriptDir/../final/config_update_scripts/requirements.txt"
+	# Set-Location $scriptDir/../preprocessing
+	# python3 update_trigger.py
 }
 else {
-	Write-Output "swarm follower"
-	Write-Output "joining swarm with token $SwarmToken" 
+	Write-Output "[+] swarm follower"
+	Write-Output "[+] joining swarm with token $SwarmToken" 
 	docker swarm join --token $SwarmToken $ManagerAddr
 }