real-time-traffic-analysis-clickhouse/clickhouse/docker-compose.yml

services:
  clickhouse-keeper1:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-keeper1
    command: >
      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
    volumes:
      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
      - ./clickhouse_data/data:/var/lib/clickhouse/data
      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
    networks:
      common-network:
        aliases:
          - clickhouse-keeper1

  clickhouse-keeper2:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-keeper2
    command: >
      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
    volumes:
      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
      - ./clickhouse_data/data:/var/lib/clickhouse/data
      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
    networks:
      common-network:
        aliases:
          - clickhouse-keeper2

  clickhouse-keeper3:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-keeper3
    command: >
      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
    volumes:
      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
      - ./clickhouse_data/data:/var/lib/clickhouse/data
      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
    networks:
      common-network:
        aliases:
          - clickhouse-keeper3

  clickhouse-server1:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-server1
    volumes:
      - ./node1-config/:/etc/clickhouse-server/config.d/
      - clickhouse_data1:/var/lib/clickhouse
    networks:
      common-network:
        aliases:
          - clickhouse-server1
    depends_on:
      - clickhouse-keeper1
      - clickhouse-keeper2
      - clickhouse-keeper3
    ports:
      - "9001:9000"  # Native client port
      - "8123:8123"  # HTTP interface

  clickhouse-server2:
    image: clickhouse/clickhouse-server:latest
    container_name: clickhouse-server2
    volumes:
      - ./node2-config/:/etc/clickhouse-server/config.d/
      - clickhouse_data2:/var/lib/clickhouse
    networks:
      common-network:
        aliases:
          - clickhouse-server2
    depends_on:
      - clickhouse-keeper1
      - clickhouse-keeper2
      - clickhouse-keeper3
    ports:
      - "9002:9000"  # Native client port
      - "8125:8123"  # HTTP interface

  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    networks:
      common-network:
        aliases:
          - zookeeper
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure
    environment:
      ZOOKEEPER_CLIENT_PORT: 2182
    ports:
      - "2182:2181"

  kafka:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2182
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
      KAFKA_BROKER_ID: 1
      KAFKA_MESSAGE_MAX_BYTES: 200000000
      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    networks:
      common-network:
        aliases:
          - kafka
    ports:
      - "9092:9092"
    volumes:
      - kafka_data_new:/var/lib/kafka/data
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure

  pcap_streamer:
    image: levenshtein/streamer_test4:latest
    depends_on:
      - kafka
    networks:
      common-network:
        aliases:
          - pcap_streamer
    volumes:
      - "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project:/data/pcap"
    environment:
      PCAP_FILE: /data/pcap/202310081400.pcap
    command: ["sh", "-c", "sleep 30 && python /app/pcap_processor.py -f /data/pcap/202310081400.pcap -s --stream_size 1000"]
    deploy:
      replicas: 1
      restart_policy:
        condition: on-failure

networks:
  common-network:
    driver: overlay
    attachable: true

volumes:
  clickhouse_data1:
    driver: local
  clickhouse_data2:
    driver: local
  kafka_data_new:
    driver: local