my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2026-01-25 08:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

cc map, full scale data testing

Browse Source
This commit is contained in:
Kaushik Narayan R
2024-11-29 20:11:36 -07:00
parent 11abeb5337
commit a5887f30f5
10 changed files with 445 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

405
ui/dashboard_1.json
View File

@@ -21,6 +21,247 @@
"id": 1,
"links": [],
"panels": [
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "PDEE91DDB90597936"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "dashed"
}
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 10,
"options": {
"legend": {
"calcs": [
"mean",
"sum"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"timezone": [
"Asia/Tokyo"
],
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"editorType": "sql",
"format": 1,
"meta": {
"builderOptions": {
"columns": [],
"database": "",
"limit": 1000,
"mode": "list",
"queryType": "table",
"table": ""
}
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\n toDate(time_stamp) AS \"Day\",\n l4_protocol AS \"IP Protocol\",\n SUM(pkt_len)/1024.0/1024.0 AS \"Bandwidth (MB)\"\nFROM traffic_records_all\nGROUP BY \"Day\", l4_protocol\nORDER BY \"Day\" ASC;\n",
"refId": "A"
}
],
"title": "Daily bandwidth trend",
"transformations": [
{
"id": "prepareTimeSeries",
"options": {
"format": "multi"
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "PDEE91DDB90597936"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "smooth",
"lineStyle": {
"fill": "solid"
},
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "dashed"
}
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 60
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 9,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"timezone": [
"Asia/Tokyo"
],
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"editorType": "sql",
"format": 1,
"meta": {
"builderOptions": {
"columns": [],
"database": "",
"limit": 1000,
"mode": "list",
"queryType": "table",
"table": ""
}
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\n toDate(time_stamp) AS \"Day\",\n l4_protocol AS \"IP Protocol\",\n COUNT(time_stamp) AS \"Packet count\"\nFROM traffic_records_all\nGROUP BY \"Day\", l4_protocol\nORDER BY \"Day\" ASC;\n",
"refId": "A"
}
],
"title": "Daily traffic trend",
"transformations": [
{
"id": "prepareTimeSeries",
"options": {
"format": "multi"
}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
@@ -56,7 +297,7 @@
"h": 7,
"w": 9,
"x": 0,
"y": 0
"y": 8
},
"id": 8,
"options": {
@@ -100,7 +341,7 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\n SUM(pkt_len)/1024.0/1024.0 AS \"Regional traffic bandwidth (MB)\",\n dictGet('ip_region_dict', 'region', tuple(src_ip)) AS region\nFROM traffic_records_all\nGROUP BY region\nORDER BY \"Regional traffic bandwidth (MB)\" DESC\nLIMIT 10",
"rawSql": "SELECT\n SUM(pkt_len)/1024.0/1024.0 AS \"Regional traffic bandwidth (MB)\",\n dictGet('ip_region_dict', ('country_code', 'country'), tuple(src_ip)).2 AS region\nFROM traffic_records_all\nGROUP BY region\nORDER BY \"Regional traffic bandwidth (MB)\" DESC\nLIMIT 10",
"refId": "A"
}
],
@@ -142,7 +383,7 @@
"h": 7,
"w": 9,
"x": 9,
"y": 0
"y": 8
},
"id": 7,
"options": {
@@ -186,7 +427,7 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\n COUNT(src_ip) AS \"Regional traffic\",\n dictGet('ip_region_dict', 'region', tuple(src_ip)) AS region\nFROM traffic_records_all\nGROUP BY region\nORDER BY \"Regional traffic\" DESC\nLIMIT 10",
"rawSql": "SELECT\n COUNT(src_ip)/1000.0/1000.0 AS \"Regional traffic\",\n dictGet('ip_region_dict', ('country_code', 'country'), tuple(src_ip)).2 AS region\nFROM traffic_records_all\nGROUP BY region\nORDER BY \"Regional traffic\" DESC\nLIMIT 10",
"refId": "A"
}
],
@@ -218,7 +459,7 @@
"h": 7,
"w": 6,
"x": 18,
"y": 0
"y": 8
},
"id": 6,
"options": {
@@ -319,7 +560,7 @@
"h": 7,
"w": 9,
"x": 0,
"y": 7
"y": 15
},
"id": 5,
"options": {
@@ -361,7 +602,7 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT Port,\r\n src_bw/1024.0/1024.0 AS \"Source port bandwidth (MB)\",\r\n dst_bw/1024.0/1024.0 AS \"Destination port bandwidth (MB)\"\r\nFROM (\r\n SELECT src_port AS Port,\r\n SUM(pkt_len) AS src_bw\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY src_bw DESC\r\n LIMIT 40\r\n ) AS src\r\n INNER JOIN (\r\n SELECT dst_port AS Port,\r\n SUM(pkt_len) AS dst_bw\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY dst_bw DESC\r\n LIMIT 40\r\n ) AS dst USING (Port)\r\nORDER BY (src_bw + dst_bw) DESC\r\nLIMIT 40;",
"rawSql": "SELECT Port,\r\n src_bw/1024.0/1024.0 AS \"Source port bandwidth (MB)\",\r\n dst_bw/1024.0/1024.0 AS \"Destination port bandwidth (MB)\"\r\nFROM (\r\n SELECT src_port AS Port,\r\n SUM(pkt_len) AS src_bw\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY src_bw DESC\r\n LIMIT 20\r\n ) AS src\r\n INNER JOIN (\r\n SELECT dst_port AS Port,\r\n SUM(pkt_len) AS dst_bw\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY dst_bw DESC\r\n LIMIT 20\r\n ) AS dst USING (Port)\r\nORDER BY (src_bw + dst_bw) DESC\r\nLIMIT 20;",
"refId": "A"
}
],
@@ -416,7 +657,7 @@
"h": 7,
"w": 9,
"x": 9,
"y": 7
"y": 15
},
"id": 4,
"options": {
@@ -458,7 +699,7 @@
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT \r\n Port, \r\n SourcePortCount AS \"Source port frequency\",\r\n DestPortCount AS \"Destination port frequency\"\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n",
"rawSql": "SELECT \r\n Port, \r\n SourcePortCount AS \"Source port frequency\",\r\n DestPortCount AS \"Destination port frequency\"\r\nFROM\r\n(\r\n SELECT \r\n src_port AS Port, \r\n COUNT(*) AS SourcePortCount\r\n FROM traffic_records_all\r\n GROUP BY src_port\r\n ORDER BY SourcePortCount DESC\r\n LIMIT 20\r\n) AS src\r\nINNER JOIN\r\n(\r\n SELECT \r\n dst_port AS Port, \r\n COUNT(*) AS DestPortCount\r\n FROM traffic_records_all\r\n GROUP BY dst_port\r\n ORDER BY DestPortCount DESC\r\n LIMIT 20\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 20;\r\n",
"refId": "A"
}
],
@@ -490,7 +731,7 @@
"h": 7,
"w": 6,
"x": 18,
"y": 7
"y": 15
},
"id": 1,
"options": {
@@ -546,6 +787,144 @@
],
"title": "Distribution of L4 protocol (frequency)",
"type": "piechart"
},
{
"datasource": {
"type": "grafana-clickhouse-datasource",
"uid": "PDEE91DDB90597936"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
}
},
"mappings": [],
"thresholds": {
"mode": "percentage",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 16,
"w": 24,
"x": 0,
"y": 22
},
"id": 11,
"options": {
"basemap": {
"config": {},
"name": "Layer 0",
"type": "default"
},
"controls": {
"mouseWheelZoom": true,
"showAttribution": true,
"showDebug": false,
"showMeasure": false,
"showScale": false,
"showZoom": true
},
"layers": [
{
"config": {
"showLegend": true,
"style": {
"color": {
"field": "cc",
"fixed": "dark-green"
},
"opacity": 0.4,
"rotation": {
"fixed": 0,
"max": 360,
"min": -360,
"mode": "mod"
},
"size": {
"field": "Source",
"fixed": 5,
"max": 25,
"min": 3
},
"symbol": {
"fixed": "img/icons/marker/circle.svg",
"mode": "fixed"
},
"symbolAlign": {
"horizontal": "center",
"vertical": "center"
},
"text": {
"fixed": "",
"mode": "field"
},
"textConfig": {
"fontSize": 8,
"offsetX": 0,
"offsetY": 0,
"textAlign": "center",
"textBaseline": "middle"
}
}
},
"location": {
"lookup": "cc",
"mode": "lookup"
},
"name": "Markers",
"tooltip": false,
"type": "markers"
}
],
"tooltip": {
"mode": "details"
},
"view": {
"allLayers": true,
"id": "oceania",
"lat": -10,
"lon": -140,
"zoom": 3
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"editorType": "sql",
"format": 1,
"meta": {
"builderOptions": {
"columns": [],
"database": "",
"limit": 1000,
"mode": "list",
"queryType": "table",
"table": ""
}
},
"pluginVersion": "4.5.1",
"queryType": "table",
"rawSql": "SELECT\n COUNT(src_ip) AS \"Source\",\n dictGet('ip_region_dict', ('country_code', 'country'), tuple(src_ip)).1 AS cc\nFROM traffic_records_all\nGROUP BY cc\nORDER BY \"Source\" DESC;\n",
"refId": "A"
}
],
"title": "Traffic map",
"type": "geomap"
}
],
"preload": false,
@@ -556,13 +935,13 @@
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
"from": "2023-10-01T05:00:00.000Z",
"to": "2023-10-31T05:00:03.000Z"
},
"timepicker": {},
"timezone": "browser",
"title": "Internet traffic capture analysis",
"uid": "be59fkbp3zs3kc",
"version": 1,
"version": 4,
"weekStart": ""
}