integrating data streamer stack and clickhouse stack

clickhouse/docker-compose.yaml

@@ -4,7 +4,7 @@ services:
     container_name: clickhouse-keeper1
     command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
       - clickhouse_keeper1_data:/var/lib/clickhouse
 
     networks:
@@ -17,7 +17,7 @@ services:
     container_name: clickhouse-keeper2
     command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
       - clickhouse_keeper2_data:/var/lib/clickhouse
     networks:
       clickhouse-keeper-network:
@@ -29,7 +29,7 @@ services:
     container_name: clickhouse-keeper3
     command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - ../clickhouse/clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
       - clickhouse_keeper3_data:/var/lib/clickhouse
     networks:
       clickhouse-keeper-network:
@@ -40,7 +40,7 @@ services:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server1
     volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
@@ -50,6 +50,9 @@ services:
       clickhouse-keeper-network:
         aliases:
           - clickhouse-server1
+      data_network:
+        aliases:
+          - clickhouse-server1
     deploy:
       replicas: 1
       # placement:
@@ -64,15 +67,16 @@ services:
       - clickhouse-keeper1
       - clickhouse-keeper2
       - clickhouse-keeper3
+      - data-streamer
     ports:
-      - "9001:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
+      - "9001:9000" # Native client port
+      - "8123:8123" # HTTP interface
 
   clickhouse-server2:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server2
     volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
       - clickhouse_server2_data:/var/lib/clickhouse
       - clickhouse_server2_TTL:/clickhouse_data/server2
     networks:
@@ -97,8 +101,8 @@ services:
       - clickhouse-keeper2
       - clickhouse-keeper3
     ports:
-      - "9002:9000"  # Native client port
-      - "8124:8123"  # HTTP interface
+      - "9002:9000" # Native client port
+      - "8124:8123" # HTTP interface
 
 networks:
   clickhouse-server-network:
@@ -122,4 +126,4 @@ volumes:
   clickhouse_server1_TTL:
     driver: local
   clickhouse_server2_TTL:
-    driver: local
+    driver: local

clickhouse/docker-compose.yml

@@ -1,155 +0,0 @@
-services:
-  clickhouse-keeper1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper1
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper1
-
-  clickhouse-keeper2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper2
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper2
-
-  clickhouse-keeper3:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper3
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-keeper3
-
-  clickhouse-server1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server1
-    volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data1:/var/lib/clickhouse
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-server1
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9001:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
-
-  clickhouse-server2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server2
-    volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data2:/var/lib/clickhouse
-    networks:
-      common-network:
-        aliases:
-          - clickhouse-server2
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9002:9000"  # Native client port
-      - "8125:8123"  # HTTP interface
-
-  zookeeper:
-    image: confluentinc/cp-zookeeper:latest
-    networks:
-      common-network:
-        aliases:
-          - zookeeper
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2182
-    ports:
-      - "2182:2181"
-
-  kafka:
-    image: confluentinc/cp-kafka:latest
-    depends_on:
-      - zookeeper
-    environment:
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2182
-      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
-      KAFKA_BROKER_ID: 1
-      KAFKA_MESSAGE_MAX_BYTES: 200000000
-      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    networks:
-      common-network:
-        aliases:
-          - kafka
-    ports:
-      - "9092:9092"
-    volumes:
-      - kafka_data_new:/var/lib/kafka/data
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-
-  pcap_streamer:
-    image: levenshtein/streamer_test7:latest
-    depends_on:
-      - kafka
-    networks:
-      common-network:
-        aliases:
-          - pcap_streamer
-    volumes:
-      #- "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project:/data/pcap"
-      - "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project/project_github/real-time-traffic-analysis-clickhouse/preprocessing:/data/pcap"
-    command: ["sh", "-c", "sleep 60 && python /app/pcap_processor.py -c /data/pcap/sample_output.csv -s --stream_size 1000"]
-    deploy:
-      replicas: 1
-      restart_policy:
-        condition: on-failure
-
-networks:
-  common-network:
-    driver: overlay
-    attachable: true
-
-volumes:
-  clickhouse_data1:
-    driver: local
-  clickhouse_data2:
-    driver: local
-  kafka_data_new:
-    driver: local