From 6fb0ff4b4ce9920eee6c24411df920a6b1ea31cd Mon Sep 17 00:00:00 2001
From: Kaushik Narayan R <rknarayan02@gmail.com>
Date: Wed, 27 Nov 2024 11:08:16 -0700
Subject: [PATCH] ch entrypoint sql, fixes

---
 clickhouse/ddl/common/table_create.sql | 28 ++++++++++++++++++++
 clickhouse/ddl/distr/table_create.sql  |  3 +++
 clickhouse/dml/test_seed_traffic.sql   | 36 ++++++++++++++++++++++++++
 clickhouse/docker-compose.yaml         |  5 +++-
 scripts/README.md                      |  4 +++
 scripts/deploy.ps1                     |  2 +-
 6 files changed, 76 insertions(+), 2 deletions(-)
 create mode 100644 clickhouse/ddl/common/table_create.sql
 create mode 100644 clickhouse/ddl/distr/table_create.sql
 create mode 100644 clickhouse/dml/test_seed_traffic.sql
 create mode 100644 scripts/README.md

diff --git a/clickhouse/ddl/common/table_create.sql b/clickhouse/ddl/common/table_create.sql
new file mode 100644
index 0000000..06fc528
--- /dev/null
+++ b/clickhouse/ddl/common/table_create.sql
@@ -0,0 +1,28 @@
+-- local table creation
+CREATE TABLE traffic_records (
+	time_stamp DateTime64 (6, 'Japan') CODEC (Delta, ZSTD),
+	l4_protocol Enum8 ('TCP' = 1, 'UDP' = 2),
+	src_ip IPv4,
+	dst_ip IPv4,
+	src_port UInt16 CODEC (ZSTD),
+	dst_port UInt16 CODEC (ZSTD),
+	pkt_len UInt16 CODEC (ZSTD),
+	INDEX port_idx src_port TYPE bloom_filter GRANULARITY 10
+) ENGINE = ReplicatedMergeTree(
+	'/clickhouse/tables/{shard}/traffic_records',
+	'{replica}'
+)
+ORDER BY time_stamp
+TTL toDateTime(time_stamp) + INTERVAL 15 DAY TO VOLUME 'cold_vol'
+SETTINGS storage_policy = 'hot_cold';
+
+CREATE TABLE ip_region_map (
+	ip_range_start IPv4,
+	ip_range_end IPv4,
+	region String,
+	INDEX region_idx region TYPE bloom_filter
+) ENGINE = ReplicatedMergeTree(
+	'/clickhouse/tables/{shard}/ip_region_map',
+	'{replica}'
+)
+ORDER BY ip_range_start;
\ No newline at end of file
diff --git a/clickhouse/ddl/distr/table_create.sql b/clickhouse/ddl/distr/table_create.sql
new file mode 100644
index 0000000..8a37e34
--- /dev/null
+++ b/clickhouse/ddl/distr/table_create.sql
@@ -0,0 +1,3 @@
+CREATE TABLE traffic_records_all
+AS traffic_records
+ENGINE = Distributed ('{cluster}', 'default', 'traffic_records');
diff --git a/clickhouse/dml/test_seed_traffic.sql b/clickhouse/dml/test_seed_traffic.sql
new file mode 100644
index 0000000..2399221
--- /dev/null
+++ b/clickhouse/dml/test_seed_traffic.sql
@@ -0,0 +1,36 @@
+INSERT INTO
+	traffic_records_all
+VALUES
+	(
+		'1698728400.40122',
+		'UDP',
+		'142.12.217.111',
+		'163.213.146.100',
+		443,
+		47104,
+		74
+	) (
+		'1698728400.401217',
+		'UDP',
+		'45.144.255.42',
+		'131.174.60.217',
+		51820,
+		63998,
+		42
+	) (
+		'1698728400.401218',
+		'TCP',
+		'152.199.153.111',
+		'202.215.192.69',
+		80,
+		65305,
+		66
+	) (
+		'1698728400.401219',
+		'UDP',
+		'45.144.255.42',
+		'131.174.60.217',
+		51820,
+		63998,
+		42
+	)
\ No newline at end of file
diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml
index 3afb91b..5eee4a4 100644
--- a/clickhouse/docker-compose.yaml
+++ b/clickhouse/docker-compose.yaml
@@ -41,6 +41,8 @@ services:
     container_name: clickhouse-server1
     volumes:
       - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql
+      - ../clickhouse/ddl/distr/table_create.sql:/docker-entrypoint-initdb.d/distr_table_create.sql
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
@@ -50,7 +52,7 @@ services:
       clickhouse-keeper-network:
         aliases:
           - clickhouse-server1
-      data_network:
+      data-network:
         aliases:
           - clickhouse-server1
     deploy:
@@ -77,6 +79,7 @@ services:
     container_name: clickhouse-server2
     volumes:
       - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql
       - clickhouse_server2_data:/var/lib/clickhouse
       - clickhouse_server2_TTL:/clickhouse_data/server2
     networks:
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 0000000..e28e606
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,4 @@
+# Full setup
+
+`deploy.ps1 -MasterNode` to deploy stack with current node as manager
+`deploy.ps1 -downStack` to bring down stack (run from manager node)
diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1
index 8951b82..cea301b 100644
--- a/scripts/deploy.ps1
+++ b/scripts/deploy.ps1
@@ -31,7 +31,7 @@ elseif ($MasterNode) {
 	Set-Location $scriptDir
 	docker stack deploy -d `
 		-c ../preprocessing/docker-compose.yml `
-		-c ../clickhouse/docker-compose-old.yaml `
+		-c ../clickhouse/docker-compose.yaml `
 		$stackName
 
 	# scripts