real-time-traffic-analysis-clickhouse/preprocessing/docker-compose.yml

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    networks:
      data-network:
        aliases:
          - zookeeper
    deploy:
      replicas: 1
      placement:
        constraints: [node.labels.worker == true]
      restart_policy:
        condition: on-failure
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
    ports:
      - "2181:2181"
    volumes:
      - zk_data:/var/lib/zookeeper/data

  kafka:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT
      KAFKA_BROKER_ID: 1
      KAFKA_MESSAGE_MAX_BYTES: 200000000
      KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    command: sh -c "/etc/confluent/docker/run && kafka-topics --create --bootstrap-server kafka:9092 --replication-factor 1 --partitions 1 --topic traffic_records_stream"
    networks:
      data-network:
        aliases:
          - kafka
    ports:
      - "9092:9092"
    volumes:
      - kafka_data:/var/lib/kafka/data
    deploy:
      replicas: 1
      placement:
        constraints: [node.labels.worker == true]
      restart_policy:
        condition: on-failure

  data-streamer:
    image: 127.0.0.1:5000/data-streamer:latest
    depends_on:
      - kafka
    networks:
      data-network:
        aliases:
          - data-streamer
    volumes:
      - "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
    command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'"
    deploy:
      replicas: 1
      placement:
        constraints: [node.labels.worker == true]
      restart_policy:
        condition: on-failure

networks:
  data-network:
    driver: overlay
    attachable: true

volumes:
  zk_data:
    driver: local
  kafka_data:
    driver: local