back to ui, WiP

clickhouse/docker-compose.yaml

@@ -12,8 +12,11 @@ services:
         aliases:
           - clickhouse-keeper1
     deploy:
-      placement:
+      replicas: 1
-        constraints: [node.labels.main == true]
+      # placement:
+      #   constraints: [node.labels.main == true]
+      restart_policy:
+          condition: on-failure
 
   clickhouse-keeper2:
     image: clickhouse/clickhouse-server:latest
@@ -27,8 +30,11 @@ services:
         aliases:
           - clickhouse-keeper2
     deploy:
-      placement:
+      replicas: 1
-        constraints: [node.labels.main == true]
+      # placement:
+      #   constraints: [node.labels.main == true]
+      restart_policy:
+          condition: on-failure
 
   clickhouse-keeper3:
     image: clickhouse/clickhouse-server:latest
@@ -42,8 +48,11 @@ services:
         aliases:
           - clickhouse-keeper3
     deploy:
-      placement:
+      replicas: 1
-        constraints: [node.labels.main == true]
+      # placement:
+      #   constraints: [node.labels.main == true]
+      restart_policy:
+          condition: on-failure
 
   clickhouse-server1:
     image: clickhouse/clickhouse-server:latest
@@ -65,8 +74,10 @@ services:
           - clickhouse-server1
     deploy:
       replicas: 1
-      placement:
+      # placement:
-        constraints: [node.labels.main == true]
+      #   constraints: [node.labels.main == true]
+      restart_policy:
+          condition: on-failure
       update_config:
         delay: 10s
       resources:
@@ -99,8 +110,10 @@ services:
           - clickhouse-server2
     deploy:
       replicas: 1
-      placement:
+      # placement:
-        constraints: [node.labels.main == true]
+      #   constraints: [node.labels.main == true]
+      restart_policy:
+          condition: on-failure
       update_config:
         delay: 10s
       resources:

preprocessing/docker-compose.yml

@@ -7,8 +7,8 @@ services:
           - zookeeper
     deploy:
       replicas: 1
-      placement:
+      # placement:
-        constraints: [node.labels.worker == true]
+      #   constraints: [node.labels.worker == true]
       restart_policy:
         condition: on-failure
     environment:
@@ -42,8 +42,8 @@ services:
       - kafka_data:/var/lib/kafka/data
     deploy:
       replicas: 1
-      placement:
+      # placement:
-        constraints: [node.labels.worker == true]
+      #   constraints: [node.labels.worker == true]
       restart_policy:
         condition: on-failure
 
@@ -60,8 +60,8 @@ services:
     command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000'"
     deploy:
       replicas: 1
-      placement:
+      # placement:
-        constraints: [node.labels.worker == true]
+      #   constraints: [node.labels.worker == true]
       restart_policy:
         condition: on-failure
 

scripts/deploy.sh

@@ -34,8 +34,8 @@ elif ($masterNode); then
 	# data streaming
 	cd $scriptDir/../preprocessing
 	$dockerCmd service create --name registry -p 5000:5000 registry:2
-	$dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python .
+	# $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --no-cache --push -f Dockerfile.python .
-	# $dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python .
+	$dockerCmd build -t 127.0.0.1:5000/data-streamer:latest --push -f Dockerfile.python .
 
 	# execute
 	cd $scriptDir

ui/dashboard_1.json

@@ -154,14 +154,17 @@
         "legend": {
           "displayMode": "list",
           "placement": "bottom",
-          "showLegend": true
+          "showLegend": true,
+          "values": [
+            "percent"
+          ]
         },
         "pieType": "pie",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
           ],
-          "fields": "/^ProtocolCount$/",
+          "fields": "/^Protocol frequency$/",
           "values": true
         },
         "tooltip": {
@@ -190,11 +193,11 @@
           },
           "pluginVersion": "4.5.1",
           "queryType": "table",
-          "rawSql": "SELECT\r\n  l4_protocol as Protocol,\r\n  COUNT(Protocol) as ProtocolCount\r\n  FROM traffic_records_all\r\n  GROUP BY Protocol",
+          "rawSql": "SELECT\r\n  l4_protocol as Protocol,\r\n  COUNT(Protocol) as \"Protocol frequency\"\r\n  FROM traffic_records_all\r\n  GROUP BY Protocol",
           "refId": "A"
         }
       ],
-      "title": "Distribution of L4 protocol",
+      "title": "Distribution of L4 protocol (frequency)",
       "type": "piechart"
     },
     {
@@ -287,11 +290,11 @@
           },
           "pluginVersion": "4.5.1",
           "queryType": "table",
-          "rawSql": "SELECT \r\n    Port, \r\n    SourcePortCount, \r\n    DestPortCount\r\nFROM\r\n(\r\n    SELECT \r\n        src_port AS Port, \r\n        COUNT(*) AS SourcePortCount\r\n    FROM traffic_records_all\r\n    GROUP BY src_port\r\n    ORDER BY SourcePortCount DESC\r\n    LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n    SELECT \r\n        dst_port AS Port, \r\n        COUNT(*) AS DestPortCount\r\n    FROM traffic_records_all\r\n    GROUP BY dst_port\r\n    ORDER BY DestPortCount DESC\r\n    LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n",
+          "rawSql": "SELECT \r\n    Port, \r\n    SourcePortCount AS \"Source port frequency\",\r\n    DestPortCount AS \"Destination port frequency\"\r\nFROM\r\n(\r\n    SELECT \r\n        src_port AS Port, \r\n        COUNT(*) AS SourcePortCount\r\n    FROM traffic_records_all\r\n    GROUP BY src_port\r\n    ORDER BY SourcePortCount DESC\r\n    LIMIT 40\r\n) AS src\r\nINNER JOIN\r\n(\r\n    SELECT \r\n        dst_port AS Port, \r\n        COUNT(*) AS DestPortCount\r\n    FROM traffic_records_all\r\n    GROUP BY dst_port\r\n    ORDER BY DestPortCount DESC\r\n    LIMIT 40\r\n) AS dst\r\nUSING (Port)\r\nORDER BY (SourcePortCount + DestPortCount) DESC\r\nLIMIT 40;\r\n",
           "refId": "A"
         }
       ],
-      "title": "Top ports (by count)",
+      "title": "Top ports (frequency)",
       "type": "barchart"
     },
     {
@@ -305,67 +308,48 @@
             "mode": "palette-classic"
           },
           "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "fillOpacity": 80,
-            "gradientMode": "none",
             "hideFrom": {
               "legend": false,
               "tooltip": false,
               "viz": false
-            },
-            "lineWidth": 1,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
             }
           },
-          "mappings": [],
+          "mappings": []
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 6,
-        "x": 0,
+        "x": 18,
-        "y": 16
+        "y": 8
       },
-      "id": 2,
+      "id": 6,
       "options": {
-        "barRadius": 0,
+        "displayLabels": [
-        "barWidth": 0.97,
+          "percent",
-        "fullHighlight": false,
+          "name"
-        "groupWidth": 0.7,
+        ],
         "legend": {
-          "calcs": [],
           "displayMode": "list",
           "placement": "bottom",
-          "showLegend": true
+          "showLegend": true,
+          "values": [
+            "percent"
+          ]
+        },
+        "pieType": "pie",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "/^Protocol bandwidth$/",
+          "values": true
         },
-        "orientation": "horizontal",
-        "showValue": "auto",
-        "stacking": "none",
         "tooltip": {
           "mode": "single",
           "sort": "none"
-        },
+        }
-        "xField": "SourcePort",
-        "xTickLabelRotation": 0,
-        "xTickLabelSpacing": 100
       },
       "pluginVersion": "11.3.1",
       "targets": [
@@ -384,109 +368,12 @@
           },
           "pluginVersion": "4.5.1",
           "queryType": "table",
-          "rawSql": "SELECT\r\n  src_port as SourcePort,\r\n  COUNT(SourcePort) as SourcePortCount\r\n  FROM traffic_records_all\r\n  GROUP BY SourcePort\r\n  ORDER BY SourcePortCount DESC\r\n  LIMIT 10",
+          "rawSql": "SELECT\n  l4_protocol as Protocol,\n  SUM(pkt_len)/1024.0/1024.0 as \"Protocol bandwidth\"\n  FROM traffic_records_all\n  GROUP BY Protocol",
           "refId": "A"
         }
       ],
-      "title": "Top 10 source ports (by count)",
+      "title": "Distribution of L4 protocol (bandwidth)",
-      "type": "barchart"
+      "type": "piechart"
-    },
-    {
-      "datasource": {
-        "type": "grafana-clickhouse-datasource",
-        "uid": "PDEE91DDB90597936"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "fillOpacity": 80,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineWidth": 1,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 16
-      },
-      "id": 3,
-      "options": {
-        "barRadius": 0,
-        "barWidth": 0.97,
-        "fullHighlight": false,
-        "groupWidth": 0.7,
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "orientation": "horizontal",
-        "showValue": "auto",
-        "stacking": "none",
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        },
-        "xField": "DestPort",
-        "xTickLabelRotation": 0,
-        "xTickLabelSpacing": 100
-      },
-      "pluginVersion": "11.3.1",
-      "targets": [
-        {
-          "editorType": "sql",
-          "format": 1,
-          "meta": {
-            "builderOptions": {
-              "columns": [],
-              "database": "",
-              "limit": 1000,
-              "mode": "list",
-              "queryType": "table",
-              "table": ""
-            }
-          },
-          "pluginVersion": "4.5.1",
-          "queryType": "table",
-          "rawSql": "SELECT\r\n  dst_port as DestPort,\r\n  COUNT(DestPort) as DestPortCount\r\n  FROM traffic_records_all\r\n  GROUP BY DestPort\r\n  ORDER BY DestPortCount DESC\r\n  LIMIT 10",
-          "refId": "A"
-        }
-      ],
-      "title": "Top 10 destination ports (by count)",
-      "type": "barchart"
     }
   ],
   "preload": false,
@@ -503,6 +390,6 @@
   "timezone": "browser",
   "title": "Internet traffic capture analysis",
   "uid": "be59fkbp3zs3kc",
-  "version": 11,
+  "version": 1,
   "weekStart": ""
 }

ui/docker-compose.yaml

@@ -15,8 +15,11 @@ services:
         aliases:
           - grafana
     deploy:
-      placement:
+      replicas: 1
-        constraints: [node.labels.worker == true]
+      # placement:
+      #   constraints: [node.labels.worker == true]
+      restart_policy:
+        condition: on-failure
     depends_on:
       - clickhouse-server1
     environment: