kafka ch integration done

clickhouse/ddl/common/01_table_create.sql

@@ -18,8 +18,8 @@ SETTINGS storage_policy = 'hot_cold';
 
 CREATE TABLE ip_region_map (
 	ip_range_start IPv4,
-	ip_range_end IPv4,
-	region String,
+	ip_range_end IPv4,	
+	region LowCardinality(String),
 	INDEX region_idx region TYPE bloom_filter
 ) ENGINE = ReplicatedMergeTree(
 	'/clickhouse/tables/{shard}/ip_region_map',

clickhouse/ddl/main/01_table_create.sql

@@ -0,0 +1,28 @@
+-- local table creation
+CREATE TABLE traffic_records (
+	time_stamp DateTime64 (6, 'Japan') CODEC (Delta, ZSTD),
+	l4_protocol Enum8 ('TCP' = 1, 'UDP' = 2),
+	src_ip IPv4,
+	dst_ip IPv4,
+	src_port UInt16 CODEC (ZSTD),
+	dst_port UInt16 CODEC (ZSTD),
+	pkt_len UInt16 CODEC (ZSTD),
+	INDEX port_idx src_port TYPE bloom_filter GRANULARITY 10
+) ENGINE = ReplicatedMergeTree(
+	'/clickhouse/tables/{shard}/traffic_records',
+	'{replica}'
+)
+ORDER BY time_stamp
+TTL toDateTime(time_stamp) + INTERVAL 15 DAY TO VOLUME 'cold_vol'
+SETTINGS storage_policy = 'hot_cold';
+
+CREATE TABLE ip_region_map (
+	ip_range_start IPv4,
+	ip_range_end IPv4,	
+	region LowCardinality(String),
+	INDEX region_idx region TYPE bloom_filter
+) ENGINE = ReplicatedMergeTree(
+	'/clickhouse/tables/{shard}/ip_region_map',
+	'{replica}'
+)
+ORDER BY ip_range_start;

clickhouse/ddl/main/03_create_kafka_table.sql

@@ -0,0 +1,22 @@
+CREATE TABLE traffic_records_kafka_queue (
+	time Float64,
+	l4_proto String,
+	src_addr String,
+	dst_addr String,
+	src_port UInt16,
+	dst_port UInt16,
+	pkt_len UInt32
+) ENGINE = Kafka() SETTINGS kafka_broker_list = 'kafka:9092',
+kafka_topic_list = 'traffic_records_stream',
+kafka_group_name = 'clickhouse_consumer',
+kafka_format = 'JSONEachRow',
+kafka_num_consumers = 1;
+CREATE MATERIALIZED VIEW traffic_records_kafka_view TO traffic_records_all AS
+SELECT time AS time_stamp,
+	l4_proto AS l4_protocol,
+	src_addr AS src_ip,
+	dst_addr AS dst_ip,
+	src_port,
+	dst_port,
+	pkt_len
+FROM traffic_records_kafka_queue;

clickhouse/docker-compose.yaml

@@ -41,8 +41,7 @@ services:
     container_name: clickhouse-server1
     volumes:
       - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
-      - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql
-      - ../clickhouse/ddl/distr/table_create.sql:/docker-entrypoint-initdb.d/distr_table_create.sql
+      - ../clickhouse/ddl/main:/docker-entrypoint-initdb.d
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
@@ -79,7 +78,7 @@ services:
     container_name: clickhouse-server2
     volumes:
       - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
-      - ../clickhouse/ddl/common/table_create.sql:/docker-entrypoint-initdb.d/common_table_create.sql
+      - ../clickhouse/ddl/common:/docker-entrypoint-initdb.d
       - clickhouse_server2_data:/var/lib/clickhouse
       - clickhouse_server2_TTL:/clickhouse_data/server2
     networks: