ch entrypoint sql, fixes

clickhouse/ddl/common/table_create.sql

@@ -0,0 +1,28 @@
+-- local table creation
+CREATE TABLE traffic_records (
+	time_stamp DateTime64 (6, 'Japan') CODEC (Delta, ZSTD),
+	l4_protocol Enum8 ('TCP' = 1, 'UDP' = 2),
+	src_ip IPv4,
+	dst_ip IPv4,
+	src_port UInt16 CODEC (ZSTD),
+	dst_port UInt16 CODEC (ZSTD),
+	pkt_len UInt16 CODEC (ZSTD),
+	INDEX port_idx src_port TYPE bloom_filter GRANULARITY 10
+) ENGINE = ReplicatedMergeTree(
+	'/clickhouse/tables/{shard}/traffic_records',
+	'{replica}'
+)
+ORDER BY time_stamp
+TTL toDateTime(time_stamp) + INTERVAL 15 DAY TO VOLUME 'cold_vol'
+SETTINGS storage_policy = 'hot_cold';
+
+CREATE TABLE ip_region_map (
+	ip_range_start IPv4,
+	ip_range_end IPv4,
+	region String,
+	INDEX region_idx region TYPE bloom_filter
+) ENGINE = ReplicatedMergeTree(
+	'/clickhouse/tables/{shard}/ip_region_map',
+	'{replica}'
+)
+ORDER BY ip_range_start;

clickhouse/ddl/distr/table_create.sql

@@ -0,0 +1,3 @@
+CREATE TABLE traffic_records_all
+AS traffic_records
+ENGINE = Distributed ('{cluster}', 'default', 'traffic_records');

clickhouse/dml/test_seed_traffic.sql

@@ -0,0 +1,36 @@
+INSERT INTO
+	traffic_records_all
+VALUES
+	(
+		'1698728400.40122',
+		'UDP',
+		'142.12.217.111',
+		'163.213.146.100',
+		443,
+		47104,
+		74
+	) (
+		'1698728400.401217',
+		'UDP',
+		'45.144.255.42',
+		'131.174.60.217',
+		51820,
+		63998,
+		42
+	) (
+		'1698728400.401218',
+		'TCP',
+		'152.199.153.111',
+		'202.215.192.69',
+		80,
+		65305,
+		66
+	) (
+		'1698728400.401219',
+		'UDP',
+		'45.144.255.42',
+		'131.174.60.217',
+		51820,
+		63998,
+		42
+	)

clickhouse/docker-compose.yaml

@@ -41,6 +41,8 @@ services:
     container_name: clickhouse-server1
     volumes:
       - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql
+      - ../clickhouse/ddl/distr/table_create.sql:/docker-entrypoint-initdb.d/distr_table_create.sql
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
@@ -50,7 +52,7 @@ services:
       clickhouse-keeper-network:
         aliases:
           - clickhouse-server1
-      data_network:
+      data-network:
         aliases:
           - clickhouse-server1
     deploy:
@@ -77,6 +79,7 @@ services:
     container_name: clickhouse-server2
     volumes:
       - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql
       - clickhouse_server2_data:/var/lib/clickhouse
       - clickhouse_server2_TTL:/clickhouse_data/server2
     networks:

scripts/README.md

@@ -0,0 +1,4 @@
+# Full setup
+
+`deploy.ps1 -MasterNode` to deploy stack with current node as manager
+`deploy.ps1 -downStack` to bring down stack (run from manager node)

scripts/deploy.ps1

@@ -31,7 +31,7 @@ elseif ($MasterNode) {
 	Set-Location $scriptDir
 	docker stack deploy -d `
 		-c ../preprocessing/docker-compose.yml `
-		-c ../clickhouse/docker-compose-old.yaml `
+		-c ../clickhouse/docker-compose.yaml `
 		$stackName
 
 	# scripts