From 8b04cbdadb41451bbb45149e91232ec6c379c820 Mon Sep 17 00:00:00 2001 From: Kaushik Narayan R Date: Thu, 28 Nov 2024 18:39:04 -0700 Subject: [PATCH] back to ui, WiP --- clickhouse/docker-compose.yaml | 33 ++++-- preprocessing/docker-compose.yml | 12 +- scripts/deploy.sh | 4 +- ui/dashboard_1.json | 183 ++++++------------------------- ui/docker-compose.yaml | 7 +- 5 files changed, 71 insertions(+), 168 deletions(-) diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml index 8c17f35..d0fd34e 100644 --- a/clickhouse/docker-compose.yaml +++ b/clickhouse/docker-compose.yaml @@ -12,8 +12,11 @@ services: aliases: - clickhouse-keeper1 deploy: - placement: - constraints: [node.labels.main == true] + replicas: 1 + # placement: + # constraints: [node.labels.main == true] + restart_policy: + condition: on-failure clickhouse-keeper2: image: clickhouse/clickhouse-server:latest @@ -27,8 +30,11 @@ services: aliases: - clickhouse-keeper2 deploy: - placement: - constraints: [node.labels.main == true] + replicas: 1 + # placement: + # constraints: [node.labels.main == true] + restart_policy: + condition: on-failure clickhouse-keeper3: image: clickhouse/clickhouse-server:latest @@ -42,8 +48,11 @@ services: aliases: - clickhouse-keeper3 deploy: - placement: - constraints: [node.labels.main == true] + replicas: 1 + # placement: + # constraints: [node.labels.main == true] + restart_policy: + condition: on-failure clickhouse-server1: image: clickhouse/clickhouse-server:latest @@ -65,8 +74,10 @@ services: - clickhouse-server1 deploy: replicas: 1 - placement: - constraints: [node.labels.main == true] + # placement: + # constraints: [node.labels.main == true] + restart_policy: + condition: on-failure update_config: delay: 10s resources: @@ -99,8 +110,10 @@ services: - clickhouse-server2 deploy: replicas: 1 - placement: - constraints: [node.labels.main == true] + # placement: + # constraints: [node.labels.main == true] + restart_policy: + condition: on-failure update_config: delay: 10s resources: diff --git a/preprocessing/docker-compose.yml b/preprocessing/docker-compose.yml index 634b231..0973767 100644 --- a/preprocessing/docker-compose.yml +++ b/preprocessing/docker-compose.yml @@ -7,8 +7,8 @@ services: - zookeeper deploy: replicas: 1 - placement: - constraints: [node.labels.worker == true] + # placement: + # constraints: [node.labels.worker == true] restart_policy: condition: on-failure environment: @@ -42,8 +42,8 @@ services: - kafka_data:/var/lib/kafka/data deploy: replicas: 1 - placement: - constraints: [node.labels.worker == true] + # placement: + # constraints: [node.labels.worker == true] restart_policy: condition: on-failure @@ -60,8 +60,8 @@ services: command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'" deploy: replicas: 1 - placement: - constraints: [node.labels.worker == true] + # placement: + # constraints: [node.labels.worker == true] restart_policy: condition: on-failure diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 2ce11d8..3409b90 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -34,8 +34,8 @@ elif ($masterNode); then # data streaming cd $scriptDir/../preprocessing $dockerCmd service create --name registry -p 5000:5000 registry:2 - $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python . - # $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python . + # $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python . + $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python . # execute cd $scriptDir diff --git a/ui/dashboard_1.json b/ui/dashboard_1.json index 79d6e5c..97e5ba7 100644 --- a/ui/dashboard_1.json +++ b/ui/dashboard_1.json @@ -154,14 +154,17 @@ "legend": { "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": true, + "values": [ + "percent" + ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], - "fields": "/^ProtocolCount$/", + "fields": "/^Protocol frequency$/", "values": true }, "tooltip": { @@ -190,11 +193,11 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\r\n l4_protocol as Protocol,\r\n COUNT(Protocol) as ProtocolCount\r\n FROM traffic_records_all\r\n GROUP BY Protocol", + "rawSql": "SELECT\r\n l4_protocol as Protocol,\r\n COUNT(Protocol) as \"Protocol frequency\"\r\n FROM traffic_records_all\r\n GROUP BY Protocol", "refId": "A" } ], - "title": "Distribution of L4 protocol", + "title": "Distribution of L4 protocol (frequency)", "type": "piechart" }, { @@ -287,11 +290,11 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT \r\n Port, \r\n SourcePortCount, \r\n DestPortCount\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n", + "rawSql": "SELECT \r\n Port, \r\n SourcePortCount AS \"Source port frequency\",\r\n DestPortCount AS \"Destination port frequency\"\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n", "refId": "A" } ], - "title": "Top ports (by count)", + "title": "Top ports (frequency)", "type": "barchart" }, { @@ -305,67 +308,48 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } + "mappings": [] }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 16 + "w": 6, + "x": 18, + "y": 8 }, - "id": 2, + "id": 6, "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, + "displayLabels": [ + "percent", + "name" + ], "legend": { - "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Protocol bandwidth$/", + "values": true }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", "tooltip": { "mode": "single", "sort": "none" - }, - "xField": "SourcePort", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 100 + } }, "pluginVersion": "11.3.1", "targets": [ @@ -384,109 +368,12 @@ }, "pluginVersion": "4.5.1", "queryType": "table", - "rawSql": "SELECT\r\n src_port as SourcePort,\r\n COUNT(SourcePort) as SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY SourcePort\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 10", + "rawSql": "SELECT\n l4_protocol as Protocol,\n SUM(pkt_len)/1024.0/1024.0 as \"Protocol bandwidth\"\n FROM traffic_records_all\n GROUP BY Protocol", "refId": "A" } ], - "title": "Top 10 source ports (by count)", - "type": "barchart" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "PDEE91DDB90597936" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 3, - "options": { - "barRadius": 0, - "barWidth": 0.97, - "fullHighlight": false, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "orientation": "horizontal", - "showValue": "auto", - "stacking": "none", - "tooltip": { - "mode": "single", - "sort": "none" - }, - "xField": "DestPort", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 100 - }, - "pluginVersion": "11.3.1", - "targets": [ - { - "editorType": "sql", - "format": 1, - "meta": { - "builderOptions": { - "columns": [], - "database": "", - "limit": 1000, - "mode": "list", - "queryType": "table", - "table": "" - } - }, - "pluginVersion": "4.5.1", - "queryType": "table", - "rawSql": "SELECT\r\n dst_port as DestPort,\r\n COUNT(DestPort) as DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY DestPort\r\n ORDER BY DestPortCount DESC\r\n LIMIT 10", - "refId": "A" - } - ], - "title": "Top 10 destination ports (by count)", - "type": "barchart" + "title": "Distribution of L4 protocol (bandwidth)", + "type": "piechart" } ], "preload": false, @@ -503,6 +390,6 @@ "timezone": "browser", "title": "Internet traffic capture analysis", "uid": "be59fkbp3zs3kc", - "version": 11, + "version": 1, "weekStart": "" } \ No newline at end of file diff --git a/ui/docker-compose.yaml b/ui/docker-compose.yaml index 7ac73c8..ce6f69e 100644 --- a/ui/docker-compose.yaml +++ b/ui/docker-compose.yaml @@ -15,8 +15,11 @@ services: aliases: - grafana deploy: - placement: - constraints: [node.labels.worker == true] + replicas: 1 + # placement: + # constraints: [node.labels.worker == true] + restart_policy: + condition: on-failure depends_on: - clickhouse-server1 environment: