From 75bca2d704a2e223abc062565fb9e6042a28edc1 Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Wed, 27 Nov 2024 16:24:14 -0700 Subject: [PATCH 1/4] kafka ch integration done --- .../{table_create.sql => 01_table_create.sql} | 4 +- clickhouse/ddl/main/01_table_create.sql | 28 ++++++++++ .../02_dist_table_create.sql} | 0 clickhouse/ddl/main/03_create_kafka_table.sql | 22 ++++++++ clickhouse/docker-compose.yaml | 5 +- preprocessing/docker-compose.yml | 2 + preprocessing/pcap_processor.py | 56 ++++++++++--------- scripts/deploy.ps1 | 7 +-- 8 files changed, 87 insertions(+), 37 deletions(-) rename clickhouse/ddl/common/{table_create.sql => 01_table_create.sql} (93%) create mode 100644 clickhouse/ddl/main/01_table_create.sql rename clickhouse/ddl/{distr/table_create.sql => main/02_dist_table_create.sql} (100%) create mode 100644 clickhouse/ddl/main/03_create_kafka_table.sql diff --git a/clickhouse/ddl/common/table_create.sql b/clickhouse/ddl/common/01_table_create.sql similarity index 93% rename from clickhouse/ddl/common/table_create.sql rename to clickhouse/ddl/common/01_table_create.sql index 06fc528..0f90004 100644 --- a/clickhouse/ddl/common/table_create.sql +++ b/clickhouse/ddl/common/01_table_create.sql @@ -18,8 +18,8 @@ SETTINGS storage_policy = 'hot_cold'; CREATE TABLE ip_region_map ( ip_range_start IPv4, - ip_range_end IPv4, - region String, + ip_range_end IPv4, + region LowCardinality(String), INDEX region_idx region TYPE bloom_filter ) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/ip_region_map', diff --git a/clickhouse/ddl/main/01_table_create.sql b/clickhouse/ddl/main/01_table_create.sql new file mode 100644 index 0000000..0f90004 --- /dev/null +++ b/clickhouse/ddl/main/01_table_create.sql @@ -0,0 +1,28 @@ +-- local table creation +CREATE TABLE traffic_records ( + time_stamp DateTime64 (6, 'Japan') CODEC (Delta, ZSTD), + l4_protocol Enum8 ('TCP' = 1, 'UDP' = 2), + src_ip IPv4, + dst_ip IPv4, + src_port UInt16 CODEC (ZSTD), + dst_port UInt16 CODEC (ZSTD), + pkt_len UInt16 CODEC (ZSTD), + INDEX port_idx src_port TYPE bloom_filter GRANULARITY 10 +) ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/traffic_records', + '{replica}' +) +ORDER BY time_stamp +TTL toDateTime(time_stamp) + INTERVAL 15 DAY TO VOLUME 'cold_vol' +SETTINGS storage_policy = 'hot_cold'; + +CREATE TABLE ip_region_map ( + ip_range_start IPv4, + ip_range_end IPv4, + region LowCardinality(String), + INDEX region_idx region TYPE bloom_filter +) ENGINE = ReplicatedMergeTree( + '/clickhouse/tables/{shard}/ip_region_map', + '{replica}' +) +ORDER BY ip_range_start; \ No newline at end of file diff --git a/clickhouse/ddl/distr/table_create.sql b/clickhouse/ddl/main/02_dist_table_create.sql similarity index 100% rename from clickhouse/ddl/distr/table_create.sql rename to clickhouse/ddl/main/02_dist_table_create.sql diff --git a/clickhouse/ddl/main/03_create_kafka_table.sql b/clickhouse/ddl/main/03_create_kafka_table.sql new file mode 100644 index 0000000..b320fe0 --- /dev/null +++ b/clickhouse/ddl/main/03_create_kafka_table.sql @@ -0,0 +1,22 @@ +CREATE TABLE traffic_records_kafka_queue ( + time Float64, + l4_proto String, + src_addr String, + dst_addr String, + src_port UInt16, + dst_port UInt16, + pkt_len UInt32 +) ENGINE = Kafka() SETTINGS kafka_broker_list = 'kafka:9092', +kafka_topic_list = 'traffic_records_stream', +kafka_group_name = 'clickhouse_consumer', +kafka_format = 'JSONEachRow', +kafka_num_consumers = 1; +CREATE MATERIALIZED VIEW traffic_records_kafka_view TO traffic_records_all AS +SELECT time AS time_stamp, + l4_proto AS l4_protocol, + src_addr AS src_ip, + dst_addr AS dst_ip, + src_port, + dst_port, + pkt_len +FROM traffic_records_kafka_queue; \ No newline at end of file diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml index 5eee4a4..4ba1a62 100644 --- a/clickhouse/docker-compose.yaml +++ b/clickhouse/docker-compose.yaml @@ -41,8 +41,7 @@ services: container_name: clickhouse-server1 volumes: - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/ - - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql - - ../clickhouse/ddl/distr/table_create.sql:/docker-entrypoint-initdb.d/distr_table_create.sql + - ../clickhouse/ddl/main:/docker-entrypoint-initdb.d - clickhouse_server1_data:/var/lib/clickhouse - clickhouse_server1_TTL:/clickhouse_data/server1 networks: @@ -79,7 +78,7 @@ services: container_name: clickhouse-server2 volumes: - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/ - - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql + - ../clickhouse/ddl/common:/docker-entrypoint-initdb.d - clickhouse_server2_data:/var/lib/clickhouse - clickhouse_server2_TTL:/clickhouse_data/server2 networks: diff --git a/preprocessing/docker-compose.yml b/preprocessing/docker-compose.yml index 72515ea..a587882 100644 --- a/preprocessing/docker-compose.yml +++ b/preprocessing/docker-compose.yml @@ -28,6 +28,8 @@ services: KAFKA_BROKER_ID: 1 KAFKA_MESSAGE_MAX_BYTES: 200000000 KAFKA_REPLICA_FETCH_MAX_BYTES: 200000000 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + command: sh -c "/etc/confluent/docker/run && kafka-topics --create --bootstrap-server kafka:9092 --replication-factor 1 --partitions 1 --topic traffic_records_stream" networks: data-network: aliases: diff --git a/preprocessing/pcap_processor.py b/preprocessing/pcap_processor.py index 0fd40af..11a15fd 100644 --- a/preprocessing/pcap_processor.py +++ b/preprocessing/pcap_processor.py @@ -12,36 +12,33 @@ import json dbg_print = lambda *x: DEBUG and print(f"[DEBUG] {x}") +# Kafka Configuration +KAFKA_TOPIC = "traffic_records_stream" +KAFKA_SERVER = "kafka:9092" # Adjust to your Kafka server + class KafkaClient: - def __init__(self, topic_name=None, mode='producer'): + def __init__(self, topic_name=None, mode="producer"): self.mode = mode self.topic_name = topic_name - if mode == 'producer': + if mode == "producer": self.client = KafkaProducer( - bootstrap_servers=['kafka:9092'], - max_request_size = 200000000, - #api_version=(0,11,5), - value_serializer=lambda x: json.dumps(x).encode('utf-8')) - elif mode == 'consumer' and topic_name is not None: + bootstrap_servers=[KAFKA_SERVER], + max_request_size=200000000, + # api_version=(0,11,5), + value_serializer=lambda x: json.dumps(x).encode("utf-8"), + ) + elif mode == "consumer" and topic_name is not None: self.client = KafkaConsumer( - topic_name, - bootstrap_servers=['localhost:9092'], - api_version=(0,11,5), - value_deserializer=lambda x: json.loads(x.decode('utf-8'))) + topic_name, + bootstrap_servers=["localhost:9092"], + api_version=(0, 11, 5), + value_deserializer=lambda x: json.loads(x.decode("utf-8")), + ) else: raise ValueError("Consumer mode requires a topic_name") -# Kafka Configuration -KAFKA_TOPIC = 'pcap_stream_new' -KAFKA_SERVER = 'kafka:9092' # Adjust to your Kafka server -#KAFKA_SERVER = 'kafka_service:9092' -# Initialize Kafka Producer -# producer = KafkaProducer( -# bootstrap_servers=KAFKA_SERVER, -# value_serializer=lambda v: v.encode('utf-8') if isinstance(v, str) else str(v).encode('utf-8') #remove intermediate JSON encoding -# ) producer = KafkaClient(topic_name=KAFKA_TOPIC) @@ -108,7 +105,7 @@ def create_pkt_object(pkt: Packet) -> dict: "dst_addr": pkt[IP].dst, "src_port": pkt[l4_proto].sport, "dst_port": pkt[l4_proto].dport, - "pkt_len": len(pkt) + "pkt_len": len(pkt), } return res_json @@ -157,7 +154,9 @@ if __name__ == "__main__": argp.add_argument("-f", "--pcap_file", required=False, dest="_pcap") argp.add_argument("-c", "--csv_file", required=False, dest="_csv") argp.add_argument("-o", "--out_file", required=False, dest="_out") - argp.add_argument("--stream_size", required=False, default=10000, dest="_streamsize") + argp.add_argument( + "--stream_size", required=False, default=10000, dest="_streamsize" + ) argp.add_argument( "-x", "--sample", @@ -193,21 +192,21 @@ if __name__ == "__main__": DEBUG = args._debug sample_size = int(args._streamsize) # 100000 - batch_size = 100 #100000 + batch_size = 100 # 100000 # if preprocessed data ready for streaming if csv_file: - #print("true") with open(csv_file, newline="") as f: csv_rdr = csv.reader(f) next(csv_rdr) # skip headers pkts = [] + print("started stream from csv") for idx, row in enumerate(csv_rdr): # direct streaming to kafka goes here producer.client.send(KAFKA_TOPIC, row_to_dict(row)) dbg_print(row_to_dict(row)) - print("streamed packet", idx) + dbg_print("streamed packet", idx) if sample and idx > sample_size: break print(f"total streamed: {idx}") @@ -222,6 +221,8 @@ if __name__ == "__main__": pkts = [] cnt = 0 seen_count = 0 + + print("started stream from pcap") for idx, pkt in enumerate(pcap_rdr): seen_count += 1 # filter packets @@ -243,8 +244,9 @@ if __name__ == "__main__": packet_data = create_pkt_object(pkt) producer.client.send(KAFKA_TOPIC, packet_data) cnt += 1 - #print(f"streamed packet at index {idx} ") - if idx > sample_size: break + # print(f"streamed packet at index {idx} ") + if idx > sample_size: + break print(f"total seen: {seen_count-1}") print(f"total streamed: {cnt}") diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index cea301b..c5fe9cd 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -18,14 +18,11 @@ if ($downStack) { elseif ($MasterNode) { Write-Output "[+] swarm master" - # cleanup - docker stack rm $stackName - docker service rm registry - # data streaming Set-Location $scriptDir/../preprocessing docker service create --name registry -p 5000:5000 registry:2 - docker build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python . + # docker build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python . + docker build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python . # execute Set-Location $scriptDir From 0a6f0bb6f4806682d18c0e4c31ea8c5bd11b5822 Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Wed, 27 Nov 2024 17:40:20 -0700 Subject: [PATCH 2/4] added bash script, integrating ui --- scripts/deploy.ps1 | 1 + scripts/deploy.sh | 51 ++++++++++++++++++++++++++++++++++++++++++ ui/docker-compose.yaml | 3 ++- 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 scripts/deploy.sh diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index c5fe9cd..259392e 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -29,6 +29,7 @@ elseif ($MasterNode) { docker stack deploy -d ` -c ../preprocessing/docker-compose.yml ` -c ../clickhouse/docker-compose.yaml ` + -c ../ui/docker-compose.yaml ` $stackName # scripts diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 0000000..b65bea0 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +while getopts "M:D:T:A" flag; do + case "${flag}" in + M) masterNode=true ;; + D) downStack=true ;; + T) swarmToken=$OPTARG ;; + A) managerAddr=$OPTARG ;; + esac +done + +echo "masterNode: $masterNode" +echo "downStack: $downStack" +echo "swarmToken: $swarmToken" +echo "managerAddr: $managerAddr" + +$scriptDir = $(readlink -f "$0") +# echo $scriptDir # ===> /Project/scripts + +$stackName="TheWebFarm" + +if [[ $downStack ]]; then + echo "[+] Removing stack..." + docker stack rm $stackName + docker service rm registry +elif ($MasterNode); then + echo "[+] swarm master" + + # data streaming + cd $scriptDir/../preprocessing + docker service create --name registry -p 5000:5000 registry:2 + # docker build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python . + docker build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python . + + # execute + cd $scriptDir + docker stack deploy -d \ + -c ../preprocessing/docker-compose.yml \ + -c ../clickhouse/docker-compose.yaml \ + -c ../ui/docker-compose.yaml \ + $stackName + + # scripts + # pip install -r "$scriptDir/../final/config_update_scripts/requirements.txt" + # cd $scriptDir/../preprocessing + # python3 update_trigger.py +else + echo "[+] swarm follower" + echo "[+] joining swarm with token $swarmToken" + docker swarm join --token $swarmToken $managerAddr +fi diff --git a/ui/docker-compose.yaml b/ui/docker-compose.yaml index 3958e5c..544b881 100644 --- a/ui/docker-compose.yaml +++ b/ui/docker-compose.yaml @@ -4,7 +4,7 @@ services: container_name: grafana volumes: - grafana-storage:/var/lib/grafana - - ../ui/grafana_clickhouse_datasource.yaml:/etc/grafana/provisioning/datasources/clickhouse.yaml # relative to clickhouse directory + - ../ui/grafana_clickhouse_datasource.yaml:/etc/grafana/provisioning/datasources/clickhouse.yaml networks: outside_net: aliases: @@ -27,3 +27,4 @@ networks: volumes: grafana-storage: + driver: local From d5bacff369512dd9ca66cf16f1ad668ffcf4df8d Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Wed, 27 Nov 2024 23:26:51 -0700 Subject: [PATCH 3/4] dashboard WiP --- scripts/deploy.ps1 | 2 + scripts/deploy.sh | 2 + ui/Internet traffic capture analysis.json | 508 ++++++++++++++++++++++ 3 files changed, 512 insertions(+) create mode 100644 ui/Internet traffic capture analysis.json diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index 259392e..5d31f2c 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -14,6 +14,8 @@ if ($downStack) { Write-Output "[+] Removing stack..." docker stack rm $stackName docker service rm registry + Start-Sleep 15 + docker volume rm $(docker volume ls --filter name=$stackName -q) } elseif ($MasterNode) { Write-Output "[+] swarm master" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index b65bea0..f50d382 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -23,6 +23,8 @@ if [[ $downStack ]]; then echo "[+] Removing stack..." docker stack rm $stackName docker service rm registry + sleep 10 + docker volume rm $(docker volume ls --filter name=$stackName -q) elif ($MasterNode); then echo "[+] swarm master" diff --git a/ui/Internet traffic capture analysis.json b/ui/Internet traffic capture analysis.json new file mode 100644 index 0000000..79d6e5c --- /dev/null +++ b/ui/Internet traffic capture analysis.json @@ -0,0 +1,508 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.9, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "Port", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT Port,\r\n src_bw/1024.0/1024.0 AS \"Source Port Bandwidth (MB)\",\r\n dst_bw/1024.0/1024.0 AS \"Destination Port Bandwidth (MB)\"\r\nFROM (\r\n SELECT src_port AS Port,\r\n SUM(pkt_len) AS src_bw\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY src_bw DESC\r\n LIMIT 40\r\n ) AS src\r\n INNER JOIN (\r\n SELECT dst_port AS Port,\r\n SUM(pkt_len) AS dst_bw\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY dst_bw DESC\r\n LIMIT 40\r\n ) AS dst USING (Port)\r\nORDER BY (src_bw + dst_bw) DESC\r\nLIMIT 40;", + "refId": "A" + } + ], + "title": "Top ports (by bandwidth)", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 1, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^ProtocolCount$/", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\r\n l4_protocol as Protocol,\r\n COUNT(Protocol) as ProtocolCount\r\n FROM traffic_records_all\r\n GROUP BY Protocol", + "refId": "A" + } + ], + "title": "Distribution of L4 protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "barRadius": 0, + "barWidth": 0.9, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "Port", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT \r\n Port, \r\n SourcePortCount, \r\n DestPortCount\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n", + "refId": "A" + } + ], + "title": "Top ports (by count)", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "SourcePort", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\r\n src_port as SourcePort,\r\n COUNT(SourcePort) as SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY SourcePort\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 10", + "refId": "A" + } + ], + "title": "Top 10 source ports (by count)", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "PDEE91DDB90597936" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "DestPort", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "editorType": "sql", + "format": 1, + "meta": { + "builderOptions": { + "columns": [], + "database": "", + "limit": 1000, + "mode": "list", + "queryType": "table", + "table": "" + } + }, + "pluginVersion": "4.5.1", + "queryType": "table", + "rawSql": "SELECT\r\n dst_port as DestPort,\r\n COUNT(DestPort) as DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY DestPort\r\n ORDER BY DestPortCount DESC\r\n LIMIT 10", + "refId": "A" + } + ], + "title": "Top 10 destination ports (by count)", + "type": "barchart" + } + ], + "preload": false, + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Internet traffic capture analysis", + "uid": "be59fkbp3zs3kc", + "version": 11, + "weekStart": "" +} \ No newline at end of file From 6a4cdd6b5066a0627bd51f882ca8693c12d6c6cd Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Thu, 28 Nov 2024 01:22:59 -0700 Subject: [PATCH 4/4] some more dashboard stuff --- scripts/deploy.ps1 | 2 +- scripts/deploy.sh | 2 +- ...capture analysis.json => dashboard_1.json} | 0 ui/docker-compose.yaml | 4 +++- ui/grafana_dashboards.yaml | 19 +++++++++++++++++++ 5 files changed, 24 insertions(+), 3 deletions(-) rename ui/{Internet traffic capture analysis.json => dashboard_1.json} (100%) create mode 100644 ui/grafana_dashboards.yaml diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index 5d31f2c..eee7cd8 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -14,7 +14,7 @@ if ($downStack) { Write-Output "[+] Removing stack..." docker stack rm $stackName docker service rm registry - Start-Sleep 15 + Start-Sleep 20 docker volume rm $(docker volume ls --filter name=$stackName -q) } elseif ($MasterNode) { diff --git a/scripts/deploy.sh b/scripts/deploy.sh index f50d382..d1cc529 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -23,7 +23,7 @@ if [[ $downStack ]]; then echo "[+] Removing stack..." docker stack rm $stackName docker service rm registry - sleep 10 + sleep 20 docker volume rm $(docker volume ls --filter name=$stackName -q) elif ($MasterNode); then echo "[+] swarm master" diff --git a/ui/Internet traffic capture analysis.json b/ui/dashboard_1.json similarity index 100% rename from ui/Internet traffic capture analysis.json rename to ui/dashboard_1.json diff --git a/ui/docker-compose.yaml b/ui/docker-compose.yaml index 544b881..3698694 100644 --- a/ui/docker-compose.yaml +++ b/ui/docker-compose.yaml @@ -4,7 +4,9 @@ services: container_name: grafana volumes: - grafana-storage:/var/lib/grafana - - ../ui/grafana_clickhouse_datasource.yaml:/etc/grafana/provisioning/datasources/clickhouse.yaml + - ../ui/grafana_clickhouse_datasource.yaml:/etc/grafana/provisioning/datasources/clickhouse.yaml:ro + - ../ui/grafana_dashboards.yaml:/etc/grafana/provisioning/dashboards/grafana_dashboards.yaml:ro + - ../ui/dashboard_1.json:/var/lib/grafana/dashboards/dashboard_1.json networks: outside_net: aliases: diff --git a/ui/grafana_dashboards.yaml b/ui/grafana_dashboards.yaml new file mode 100644 index 0000000..1e729dc --- /dev/null +++ b/ui/grafana_dashboards.yaml @@ -0,0 +1,19 @@ +apiVersion: 1 + +providers: + - name: 'The Web Farm' + orgId: 1 + # name of the dashboard folder. + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: false + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /var/lib/grafana/dashboards + # use folder names from filesystem to create folders in Grafana + foldersFromFilesStructure: true