diff --git a/clickhouse/docker-compose-old.yaml b/clickhouse/docker-compose.yaml similarity index 81% rename from clickhouse/docker-compose-old.yaml rename to clickhouse/docker-compose.yaml index f7c95bc..3afb91b 100644 --- a/clickhouse/docker-compose-old.yaml +++ b/clickhouse/docker-compose.yaml @@ -4,7 +4,7 @@ services: container_name: clickhouse-keeper1 command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml volumes: - - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml + - ../clickhouse/clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml - clickhouse_keeper1_data:/var/lib/clickhouse networks: @@ -17,7 +17,7 @@ services: container_name: clickhouse-keeper2 command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml volumes: - - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml + - ../clickhouse/clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml - clickhouse_keeper2_data:/var/lib/clickhouse networks: clickhouse-keeper-network: @@ -29,7 +29,7 @@ services: container_name: clickhouse-keeper3 command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml volumes: - - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml + - ../clickhouse/clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml - clickhouse_keeper3_data:/var/lib/clickhouse networks: clickhouse-keeper-network: @@ -40,7 +40,7 @@ services: image: clickhouse/clickhouse-server:latest container_name: clickhouse-server1 volumes: - - ./node1-config/:/etc/clickhouse-server/config.d/ + - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/ - clickhouse_server1_data:/var/lib/clickhouse - clickhouse_server1_TTL:/clickhouse_data/server1 networks: @@ -50,6 +50,9 @@ services: clickhouse-keeper-network: aliases: - clickhouse-server1 + data_network: + aliases: + - clickhouse-server1 deploy: replicas: 1 # placement: @@ -64,15 +67,16 @@ services: - clickhouse-keeper1 - clickhouse-keeper2 - clickhouse-keeper3 + - data-streamer ports: - - "9001:9000" # Native client port - - "8123:8123" # HTTP interface + - "9001:9000" # Native client port + - "8123:8123" # HTTP interface clickhouse-server2: image: clickhouse/clickhouse-server:latest container_name: clickhouse-server2 volumes: - - ./node2-config/:/etc/clickhouse-server/config.d/ + - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/ - clickhouse_server2_data:/var/lib/clickhouse - clickhouse_server2_TTL:/clickhouse_data/server2 networks: @@ -97,8 +101,8 @@ services: - clickhouse-keeper2 - clickhouse-keeper3 ports: - - "9002:9000" # Native client port - - "8124:8123" # HTTP interface + - "9002:9000" # Native client port + - "8124:8123" # HTTP interface networks: clickhouse-server-network: @@ -122,4 +126,4 @@ volumes: clickhouse_server1_TTL: driver: local clickhouse_server2_TTL: - driver: local \ No newline at end of file + driver: local diff --git a/clickhouse/docker-compose.yml b/clickhouse/docker-compose.yml deleted file mode 100644 index d38b576..0000000 --- a/clickhouse/docker-compose.yml +++ /dev/null @@ -1,155 +0,0 @@ -services: - clickhouse-keeper1: - image: clickhouse/clickhouse-server:latest - container_name: clickhouse-keeper1 - command: > - /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml - volumes: - - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml - - ./clickhouse_data/data:/var/lib/clickhouse/data - - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp - - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files - - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas - networks: - common-network: - aliases: - - clickhouse-keeper1 - - clickhouse-keeper2: - image: clickhouse/clickhouse-server:latest - container_name: clickhouse-keeper2 - command: > - /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml - volumes: - - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml - - ./clickhouse_data/data:/var/lib/clickhouse/data - - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp - - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files - - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas - networks: - common-network: - aliases: - - clickhouse-keeper2 - - clickhouse-keeper3: - image: clickhouse/clickhouse-server:latest - container_name: clickhouse-keeper3 - command: > - /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml - volumes: - - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml - - ./clickhouse_data/data:/var/lib/clickhouse/data - - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp - - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files - - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas - networks: - common-network: - aliases: - - clickhouse-keeper3 - - clickhouse-server1: - image: clickhouse/clickhouse-server:latest - container_name: clickhouse-server1 - volumes: - - ./node1-config/:/etc/clickhouse-server/config.d/ - - clickhouse_data1:/var/lib/clickhouse - networks: - common-network: - aliases: - - clickhouse-server1 - depends_on: - - clickhouse-keeper1 - - clickhouse-keeper2 - - clickhouse-keeper3 - ports: - - "9001:9000" # Native client port - - "8123:8123" # HTTP interface - - clickhouse-server2: - image: clickhouse/clickhouse-server:latest - container_name: clickhouse-server2 - volumes: - - ./node2-config/:/etc/clickhouse-server/config.d/ - - clickhouse_data2:/var/lib/clickhouse - networks: - common-network: - aliases: - - clickhouse-server2 - depends_on: - - clickhouse-keeper1 - - clickhouse-keeper2 - - clickhouse-keeper3 - ports: - - "9002:9000" # Native client port - - "8125:8123" # HTTP interface - - zookeeper: - image: confluentinc/cp-zookeeper:latest - networks: - common-network: - aliases: - - zookeeper - deploy: - replicas: 1 - restart_policy: - condition: on-failure - environment: - ZOOKEEPER_CLIENT_PORT: 2182 - ports: - - "2182:2181" - - kafka: - image: confluentinc/cp-kafka:latest - depends_on: - - zookeeper - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2182 - KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT - KAFKA_BROKER_ID: 1 - KAFKA_MESSAGE_MAX_BYTES: 200000000 - KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - networks: - common-network: - aliases: - - kafka - ports: - - "9092:9092" - volumes: - - kafka_data_new:/var/lib/kafka/data - deploy: - replicas: 1 - restart_policy: - condition: on-failure - - pcap_streamer: - image: levenshtein/streamer_test7:latest - depends_on: - - kafka - networks: - common-network: - aliases: - - pcap_streamer - volumes: - #- "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project:/data/pcap" - - "/host_mnt/c/Users/akash/storage/Asu/sem3/dds/project/project_github/real-time-traffic-analysis-clickhouse/preprocessing:/data/pcap" - command: ["sh", "-c", "sleep 60 && python /app/pcap_processor.py -c /data/pcap/sample_output.csv -s --stream_size 1000"] - deploy: - replicas: 1 - restart_policy: - condition: on-failure - -networks: - common-network: - driver: overlay - attachable: true - -volumes: - clickhouse_data1: - driver: local - clickhouse_data2: - driver: local - kafka_data_new: - driver: local diff --git a/preprocessing/docker-compose.yml b/preprocessing/docker-compose.yml index 6ed8638..72515ea 100644 --- a/preprocessing/docker-compose.yml +++ b/preprocessing/docker-compose.yml @@ -2,7 +2,7 @@ services: zookeeper: image: confluentinc/cp-zookeeper:latest networks: - kafka_network: + data-network: aliases: - zookeeper deploy: @@ -13,6 +13,8 @@ services: ZOOKEEPER_CLIENT_PORT: 2181 ports: - "2181:2181" + volumes: + - zk_data:/var/lib/zookeeper/data kafka: image: confluentinc/cp-kafka:latest @@ -27,7 +29,7 @@ services: KAFKA_MESSAGE_MAX_BYTES: 200000000 KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000 networks: - kafka_network: + data-network: aliases: - kafka ports: @@ -39,16 +41,16 @@ services: restart_policy: condition: on-failure - data_streamer: - image: 127.0.0.1:5000/data_streamer:latest + data-streamer: + image: 127.0.0.1:5000/data-streamer:latest depends_on: - kafka networks: - kafka_network: + data-network: aliases: - - data_streamer + - data-streamer volumes: - - "./10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro" + - "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro" command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'" deploy: replicas: 1 @@ -56,10 +58,12 @@ services: condition: on-failure networks: - kafka_network: + data-network: driver: overlay attachable: true volumes: + zk_data: + driver: local kafka_data: driver: local diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index e8554c1..8951b82 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -1,34 +1,46 @@ param ( [switch] $MasterNode, + [switch] $downStack, [string] $SwarmToken, [string] $ManagerAddr ) -$script_dir = $PSScriptRoot -# Write-Output $script_dir # ===> \Project\scripts +$scriptDir = $PSScriptRoot +# Write-Output $scriptDir # ===> \Project\scripts -if ($MasterNode) { - Write-Output "Initializing Docker Swarm..." - - docker stack rm test_datastreamer_automated +$stackName = "TheWebFarm" + +if ($downStack) { + Write-Output "[+] Removing stack..." + docker stack rm $stackName docker service rm registry - - # registry - Set-Location $script_dir/../preprocessing - - docker service create --name registry -p 5000:5000 registry:2 - docker build -t 127.0.0.1:5000/data_streamer:latest --no-cache --push -f Dockerfile.python . - - docker stack deploy -d -c docker-compose.yml test_datastreamer_automated - - Set-Location $script_dir - +} +elseif ($MasterNode) { + Write-Output "[+] swarm master" + + # cleanup + docker stack rm $stackName + docker service rm registry + # data streaming - - # pip install -r "$script_dir/../final/config_update_scripts/requirements.txt" + Set-Location $scriptDir/../preprocessing + docker service create --name registry -p 5000:5000 registry:2 + docker build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python . + + # execute + Set-Location $scriptDir + docker stack deploy -d ` + -c ../preprocessing/docker-compose.yml ` + -c ../clickhouse/docker-compose-old.yaml ` + $stackName + + # scripts + # pip install -r "$scriptDir/../final/config_update_scripts/requirements.txt" + # Set-Location $scriptDir/../preprocessing + # python3 update_trigger.py } else { - Write-Output "swarm follower" - Write-Output "joining swarm with token $SwarmToken" + Write-Output "[+] swarm follower" + Write-Output "[+] joining swarm with token $SwarmToken" docker swarm join --token $SwarmToken $ManagerAddr }