diff --git a/clickhouse/clickhouse_keeper/keeper1-config.xml b/clickhouse/clickhouse_keeper/keeper1-config.xml index c5e5d4d..976fa3e 100644 --- a/clickhouse/clickhouse_keeper/keeper1-config.xml +++ b/clickhouse/clickhouse_keeper/keeper1-config.xml @@ -18,6 +18,12 @@ 1 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots + + 9182 + + /ready + + 10000 30000 diff --git a/clickhouse/clickhouse_keeper/keeper2-config.xml b/clickhouse/clickhouse_keeper/keeper2-config.xml index bd2914e..cc78c0b 100644 --- a/clickhouse/clickhouse_keeper/keeper2-config.xml +++ b/clickhouse/clickhouse_keeper/keeper2-config.xml @@ -18,6 +18,12 @@ 2 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots + + 9182 + + /ready + + 10000 30000 diff --git a/clickhouse/clickhouse_keeper/keeper3-config.xml b/clickhouse/clickhouse_keeper/keeper3-config.xml index 383f9d9..37411e7 100644 --- a/clickhouse/clickhouse_keeper/keeper3-config.xml +++ b/clickhouse/clickhouse_keeper/keeper3-config.xml @@ -18,6 +18,12 @@ 3 /var/lib/clickhouse/coordination/log /var/lib/clickhouse/coordination/snapshots + + 9182 + + /ready + + 10000 30000 diff --git a/clickhouse/node-entrypoints/common/00_wait_for_keeper.sh b/clickhouse/node-entrypoints/common/00_wait_for_keeper.sh new file mode 100755 index 0000000..a229651 --- /dev/null +++ b/clickhouse/node-entrypoints/common/00_wait_for_keeper.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +keeper_hostnames=( + "clickhouse-keeper1" + "clickhouse-keeper2" + "clickhouse-keeper3" +) +keeper_healthy=(false false false) + +can_proceed=false + +while ! $can_proceed ; do + for keeper_idx in "${!keeper_hostnames[@]}"; do + if wget -q --tries=1 --spider "http://${keeper_hostnames[$keeper_idx]}:9182/ready" ; then + echo "keeper healthy" + keeper_healthy[$keeper_idx]=true + fi + done + can_proceed=true + for keeper_idx in "${!keeper_hostnames[@]}"; do + if ! ${keeper_healthy[$keeper_idx]} ; then + can_proceed=false + sleep 5 + break + fi + done +done diff --git a/clickhouse/ddl/common/01_table_create.sql b/clickhouse/node-entrypoints/common/01_table_create.sql similarity index 100% rename from clickhouse/ddl/common/01_table_create.sql rename to clickhouse/node-entrypoints/common/01_table_create.sql diff --git a/clickhouse/node-entrypoints/main/00_wait_for_keeper.sh b/clickhouse/node-entrypoints/main/00_wait_for_keeper.sh new file mode 100755 index 0000000..a229651 --- /dev/null +++ b/clickhouse/node-entrypoints/main/00_wait_for_keeper.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +keeper_hostnames=( + "clickhouse-keeper1" + "clickhouse-keeper2" + "clickhouse-keeper3" +) +keeper_healthy=(false false false) + +can_proceed=false + +while ! $can_proceed ; do + for keeper_idx in "${!keeper_hostnames[@]}"; do + if wget -q --tries=1 --spider "http://${keeper_hostnames[$keeper_idx]}:9182/ready" ; then + echo "keeper healthy" + keeper_healthy[$keeper_idx]=true + fi + done + can_proceed=true + for keeper_idx in "${!keeper_hostnames[@]}"; do + if ! ${keeper_healthy[$keeper_idx]} ; then + can_proceed=false + sleep 5 + break + fi + done +done diff --git a/clickhouse/ddl/main/01_table_create.sql b/clickhouse/node-entrypoints/main/01_table_create.sql similarity index 100% rename from clickhouse/ddl/main/01_table_create.sql rename to clickhouse/node-entrypoints/main/01_table_create.sql diff --git a/clickhouse/ddl/main/02_dist_table_create.sql b/clickhouse/node-entrypoints/main/02_dist_table_create.sql similarity index 100% rename from clickhouse/ddl/main/02_dist_table_create.sql rename to clickhouse/node-entrypoints/main/02_dist_table_create.sql diff --git a/clickhouse/ddl/main/03_create_kafka_table.sql b/clickhouse/node-entrypoints/main/04_kafka_table_ingest.sql similarity index 94% rename from clickhouse/ddl/main/03_create_kafka_table.sql rename to clickhouse/node-entrypoints/main/04_kafka_table_ingest.sql index b320fe0..f6e2e14 100644 --- a/clickhouse/ddl/main/03_create_kafka_table.sql +++ b/clickhouse/node-entrypoints/main/04_kafka_table_ingest.sql @@ -11,6 +11,7 @@ kafka_topic_list = 'traffic_records_stream', kafka_group_name = 'clickhouse_consumer', kafka_format = 'JSONEachRow', kafka_num_consumers = 1; + CREATE MATERIALIZED VIEW traffic_records_kafka_view TO traffic_records_all AS SELECT time AS time_stamp, l4_proto AS l4_protocol, @@ -19,4 +20,4 @@ SELECT time AS time_stamp, src_port, dst_port, pkt_len -FROM traffic_records_kafka_queue; \ No newline at end of file +FROM traffic_records_kafka_queue; diff --git a/scripts/README.md b/scripts/README.md index e28e606..1aa037a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,4 +1,13 @@ -# Full setup +# End-to-end stack management + +## Windows (Powershell) `deploy.ps1 -MasterNode` to deploy stack with current node as manager + `deploy.ps1 -downStack` to bring down stack (run from manager node) + +## Linux/macOS (Bash) + +`deploy.sh -M` and `deploy.sh -D` for the same + +Add `-S` if Docker requires `sudo` privileges