integrating data streamer stack and clickhouse stack

clickhouse/docker-compose.yaml

@@ -4,7 +4,7 @@ services:
     container_name: clickhouse-keeper1
     command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
       - clickhouse_keeper1_data:/var/lib/clickhouse
 
     networks:
@@ -17,7 +17,7 @@ services:
     container_name: clickhouse-keeper2
     command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
       - clickhouse_keeper2_data:/var/lib/clickhouse
     networks:
       clickhouse-keeper-network:
@@ -29,7 +29,7 @@ services:
     container_name: clickhouse-keeper3
     command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
       - clickhouse_keeper3_data:/var/lib/clickhouse
     networks:
       clickhouse-keeper-network:
@@ -40,7 +40,7 @@ services:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server1
     volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
@@ -50,6 +50,9 @@ services:
       clickhouse-keeper-network:
         aliases:
           - clickhouse-server1
+      data_network:
+        aliases:
+          - clickhouse-server1
     deploy:
       replicas: 1
       # placement:
@@ -64,15 +67,16 @@ services:
       - clickhouse-keeper1
       - clickhouse-keeper2
       - clickhouse-keeper3
+      - data-streamer
     ports:
-      - "9001:9000"  # Native client port
+      - "9001:9000" # Native client port
-      - "8123:8123"  # HTTP interface
+      - "8123:8123" # HTTP interface
 
   clickhouse-server2:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server2
     volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
       - clickhouse_server2_data:/var/lib/clickhouse
       - clickhouse_server2_TTL:/clickhouse_data/server2
     networks:
@@ -97,8 +101,8 @@ services:
       - clickhouse-keeper2
       - clickhouse-keeper3
     ports:
-      - "9002:9000"  # Native client port
+      - "9002:9000" # Native client port
-      - "8124:8123"  # HTTP interface
+      - "8124:8123" # HTTP interface
 
 networks:
   clickhouse-server-network:
@@ -122,4 +126,4 @@ volumes:
   clickhouse_server1_TTL:
     driver: local
   clickhouse_server2_TTL:
-    driver: local
+    driver: local

clickhouse/docker-compose.yml

@@ -1,155 +0,0 @@
-services:
-  clickhouse-keeper1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper1
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper1
-
-  clickhouse-keeper2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper2
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper2
-
-  clickhouse-keeper3:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper3
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper3
-
-  clickhouse-server1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server1
-    volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data1:/var/lib/clickhouse
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-server1
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9001:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
-
-  clickhouse-server2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server2
-    volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data2:/var/lib/clickhouse
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-server2
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9002:9000"  # Native client port
-      - "8125:8123"  # HTTP interface
-
-  zookeeper:
-    image: confluentinc/cp-zookeeper:latest
-    networks:
-      common-network:
-        aliases:
-          - zookeeper
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2182
-    ports:
-      - "2182:2181"
-
-  kafka:
-    image: confluentinc/cp-kafka:latest
-    depends_on:
-      - zookeeper
-    environment:
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2182
-      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      KAFKA_BROKER_ID: 1
-      KAFKA_MESSAGE_MAX_BYTES: 200000000
-      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    networks:
-      common-network:
-        aliases:
-          - kafka
-    ports:
-      - "9092:9092"
-    volumes:
-      - kafka_data_new:/var/lib/kafka/data
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-
-  pcap_streamer:
-    image: levenshtein/streamer_test7:latest
-    depends_on:
-      - kafka
-    networks:
-      common-network:
-        aliases:
-          - pcap_streamer
-    volumes:
-      #- "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project:/data/pcap"
-      - "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project/project_github/real-time-traffic-analysis-clickhouse/preprocessing:/data/pcap"
-    command: ["sh", "-c", "sleep 60 && python /app/pcap_processor.py -c /data/pcap/sample_output.csv -s --stream_size 1000"]
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-
-networks:
-  common-network:
-    driver: overlay
-    attachable: true
-
-volumes:
-  clickhouse_data1:
-    driver: local
-  clickhouse_data2:
-    driver: local
-  kafka_data_new:
-    driver: local

preprocessing/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   zookeeper:
     image: confluentinc/cp-zookeeper:latest
     networks:
-      kafka_network:
+      data-network:
         aliases:
           - zookeeper
     deploy:
@@ -13,6 +13,8 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
     ports:
       - "2181:2181"
+    volumes:
+      - zk_data:/var/lib/zookeeper/data
 
   kafka:
     image: confluentinc/cp-kafka:latest
@@ -27,7 +29,7 @@ services:
       KAFKA_MESSAGE_MAX_BYTES: 200000000
       KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
     networks:
-      kafka_network:
+      data-network:
         aliases:
           - kafka
     ports:
@@ -39,16 +41,16 @@ services:
       restart_policy:
         condition: on-failure
 
-  data_streamer:
+  data-streamer:
-    image: 127.0.0.1:5000/data_streamer:latest
+    image: 127.0.0.1:5000/data-streamer:latest
     depends_on:
       - kafka
     networks:
-      kafka_network:
+      data-network:
         aliases:
-          - data_streamer
+          - data-streamer
     volumes:
-      - "./10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
+      - "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
     command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'"
     deploy:
       replicas: 1
@@ -56,10 +58,12 @@ services:
         condition: on-failure
 
 networks:
-  kafka_network:
+  data-network:
     driver: overlay
     attachable: true
 
 volumes:
+  zk_data:
+    driver: local
   kafka_data:
     driver: local

scripts/deploy.ps1

@@ -1,34 +1,46 @@
 param (
 	[switch] $MasterNode,
+	[switch] $downStack,
 	[string] $SwarmToken,
 	[string] $ManagerAddr
 )
 
-$script_dir = $PSScriptRoot
+$scriptDir = $PSScriptRoot
-# Write-Output $script_dir # ===> \Project\scripts
+# Write-Output $scriptDir # ===> \Project\scripts
 
-if ($MasterNode) {
+$stackName = "TheWebFarm"
-	Write-Output "Initializing Docker Swarm..."
+
-	
+if ($downStack)	{
-	docker stack rm test_datastreamer_automated
+	Write-Output "[+] Removing stack..."
+	docker stack rm $stackName
 	docker service rm registry
-	
+}
-	# registry
+elseif ($MasterNode) {
-	Set-Location $script_dir/../preprocessing
+	Write-Output "[+] swarm master"
-	
+
-	docker service create --name registry -p 5000:5000 registry:2
+	# cleanup
-	docker build -t 127.0.0.1:5000/data_streamer:latest --no-cache --push -f Dockerfile.python .
+	docker stack rm $stackName
-	
+	docker service rm registry
-	docker stack deploy -d -c docker-compose.yml test_datastreamer_automated
+
-	
-	Set-Location $script_dir
-	
 	# data streaming
-	
+	Set-Location $scriptDir/../preprocessing
-	# pip install -r "$script_dir/../final/config_update_scripts/requirements.txt"
+	docker service create --name registry -p 5000:5000 registry:2
+	docker build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python .
+
+	# execute
+	Set-Location $scriptDir
+	docker stack deploy -d `
+		-c ../preprocessing/docker-compose.yml `
+		-c ../clickhouse/docker-compose-old.yaml `
+		$stackName
+
+	# scripts
+	# pip install -r "$scriptDir/../final/config_update_scripts/requirements.txt"
+	# Set-Location $scriptDir/../preprocessing
+	# python3 update_trigger.py
 }
 else {
-	Write-Output "swarm follower"
+	Write-Output "[+] swarm follower"
-	Write-Output "joining swarm with token $SwarmToken" 
+	Write-Output "[+] joining swarm with token $SwarmToken" 
 	docker swarm join --token $SwarmToken $ManagerAddr
 }