my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 08:04:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

back to ui, WiP

Browse Source
This commit is contained in:
Kaushik Narayan R 2024-11-28 18:39:04 -07:00
parent 84e501cf4b
commit 8b04cbdadb
5 changed files with 71 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
clickhouse/docker-compose.yaml
View File

@ -12,8 +12,11 @@ services:
aliases:
- clickhouse-keeper1
deploy:
placement:
constraints: [node.labels.main == true]
replicas: 1
# placement:
# constraints: [node.labels.main == true]
restart_policy:
condition: on-failure
clickhouse-keeper2:
image: clickhouse/clickhouse-server:latest
@ -27,8 +30,11 @@ services:
aliases:
- clickhouse-keeper2
deploy:
placement:
constraints: [node.labels.main == true]
replicas: 1
# placement:
# constraints: [node.labels.main == true]
restart_policy:
condition: on-failure
clickhouse-keeper3:
image: clickhouse/clickhouse-server:latest
@ -42,8 +48,11 @@ services:
aliases:
- clickhouse-keeper3
deploy:
placement:
constraints: [node.labels.main == true]
replicas: 1
# placement:
# constraints: [node.labels.main == true]
restart_policy:
condition: on-failure
clickhouse-server1:
image: clickhouse/clickhouse-server:latest
@ -65,8 +74,10 @@ services:
- clickhouse-server1
deploy:
replicas: 1
placement:
constraints: [node.labels.main == true]
# placement:
# constraints: [node.labels.main == true]
restart_policy:
condition: on-failure
update_config:
delay: 10s
resources:
@ -99,8 +110,10 @@ services:
- clickhouse-server2
deploy:
replicas: 1
placement:
constraints: [node.labels.main == true]
# placement:
# constraints: [node.labels.main == true]
restart_policy:
condition: on-failure
update_config:
delay: 10s
resources:

12
preprocessing/docker-compose.yml
View File

@ -7,8 +7,8 @@ services:
- zookeeper
deploy:
replicas: 1
placement:
constraints: [node.labels.worker == true]
# placement:
# constraints: [node.labels.worker == true]
restart_policy:
condition: on-failure
environment:
@ -42,8 +42,8 @@ services:
- kafka_data:/var/lib/kafka/data
deploy:
replicas: 1
placement:
constraints: [node.labels.worker == true]
# placement:
# constraints: [node.labels.worker == true]
restart_policy:
condition: on-failure
@ -60,8 +60,8 @@ services:
command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'"
deploy:
replicas: 1
placement:
constraints: [node.labels.worker == true]
# placement:
# constraints: [node.labels.worker == true]
restart_policy:
condition: on-failure

4
scripts/deploy.sh
View File

@ -34,8 +34,8 @@ elif ($masterNode); then
# data streaming
cd $scriptDir/../preprocessing
$dockerCmd service create --name registry -p 5000:5000 registry:2
$dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python .
# $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python .
# $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python .
$dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python .
# execute
cd $scriptDir

183
ui/dashboard_1.json
View File

@ -154,14 +154,17 @@
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": true,
"values": [
"percent"
]
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "/^ProtocolCount$/",
"fields": "/^Protocol frequency$/",
"values": true
},
"tooltip": {
@ -190,11 +193,11 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\r\n l4_protocol as Protocol,\r\n COUNT(Protocol) as ProtocolCount\r\n FROM traffic_records_all\r\n GROUP BY Protocol",
"rawSql": "SELECT\r\n l4_protocol as Protocol,\r\n COUNT(Protocol) as \"Protocol frequency\"\r\n FROM traffic_records_all\r\n GROUP BY Protocol",
"refId": "A"
}
],
"title": "Distribution of L4 protocol",
"title": "Distribution of L4 protocol (frequency)",
"type": "piechart"
},
{
@ -287,11 +290,11 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT \r\n Port, \r\n SourcePortCount, \r\n DestPortCount\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n",
"rawSql": "SELECT \r\n Port, \r\n SourcePortCount AS \"Source port frequency\",\r\n DestPortCount AS \"Destination port frequency\"\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n",
"refId": "A"
}
],
"title": "Top ports (by count)",
"title": "Top ports (frequency)",
"type": "barchart"
},
{
@ -305,67 +308,48 @@
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
"mappings": []
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
"w": 6,
"x": 18,
"y": 8
},
"id": 2,
"id": 6,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"displayLabels": [
"percent",
"name"
],
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
"showLegend": true,
"values": [
"percent"
]
},
"pieType": "pie",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "/^Protocol bandwidth$/",
"values": true
},
"orientation": "horizontal",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"mode": "single",
"sort": "none"
},
"xField": "SourcePort",
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
}
},
"pluginVersion": "11.3.1",
"targets": [
@ -384,109 +368,12 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\r\n src_port as SourcePort,\r\n COUNT(SourcePort) as SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY SourcePort\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 10",
"rawSql": "SELECT\n l4_protocol as Protocol,\n SUM(pkt_len)/1024.0/1024.0 as \"Protocol bandwidth\"\n FROM traffic_records_all\n GROUP BY Protocol",
"refId": "A"
}
],
"title": "Top 10 source ports (by count)",
"type": "barchart"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "PDEE91DDB90597936"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineWidth": 1,
"scaleDistribution": {
"type": "linear"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 16
},
"id": 3,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"fullHighlight": false,
"groupWidth": 0.7,
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"orientation": "horizontal",
"showValue": "auto",
"stacking": "none",
"tooltip": {
"mode": "single",
"sort": "none"
},
"xField": "DestPort",
"xTickLabelRotation": 0,
"xTickLabelSpacing": 100
},
"pluginVersion": "11.3.1",
"targets": [
{
"editorType": "sql",
"format": 1,
"meta": {
"builderOptions": {
"columns": [],
"database": "",
"limit": 1000,
"mode": "list",
"queryType": "table",
"table": ""
}
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\r\n dst_port as DestPort,\r\n COUNT(DestPort) as DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY DestPort\r\n ORDER BY DestPortCount DESC\r\n LIMIT 10",
"refId": "A"
}
],
"title": "Top 10 destination ports (by count)",
"type": "barchart"
"title": "Distribution of L4 protocol (bandwidth)",
"type": "piechart"
}
],
"preload": false,
@ -503,6 +390,6 @@
"timezone": "browser",
"title": "Internet traffic capture analysis",
"uid": "be59fkbp3zs3kc",
"version": 11,
"version": 1,
"weekStart": ""
}

7
ui/docker-compose.yaml
View File

@ -15,8 +15,11 @@ services:
aliases:
- grafana
deploy:
placement:
constraints: [node.labels.worker == true]
replicas: 1
# placement:
# constraints: [node.labels.worker == true]
restart_policy:
condition: on-failure
depends_on:
- clickhouse-server1
environment: