From 3f15f225f47a7bf2e177c29251d75bf61b544f66 Mon Sep 17 00:00:00 2001
From: Kaushik Narayan R <rknarayan02@gmail.com>
Date: Thu, 28 Nov 2024 22:11:32 -0700
Subject: [PATCH] load geoip data from csv

---
 .../data/preprocessed_configs/config.xml      | 61 -------------------
 clickhouse/dml/test_seed_traffic.sql          | 36 -----------
 clickhouse/docker-compose.yaml                |  5 +-
 .../main/03_insert_geoip_csv.sql              |  3 +
 preprocessing/geoip.csv                       |  2 +-
 5 files changed, 7 insertions(+), 100 deletions(-)
 delete mode 100644 clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
 delete mode 100644 clickhouse/dml/test_seed_traffic.sql
 create mode 100644 clickhouse/node-entrypoints/main/03_insert_geoip_csv.sql

diff --git a/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
deleted file mode 100644
index 08de94b..0000000
--- a/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!-- This file was generated automatically.
-     Do not edit it: it is likely to be discarded and generated again before it's read next time.
-     Files used to generate this file:
-       /etc/clickhouse-server/config.xml
-       /etc/clickhouse-server/config.d/docker_related_config.xml      -->
-
-<yandex>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
-        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <listen_host>::</listen_host>
-    
-    <path>/var/lib/clickhouse/data/</path>
-    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
-    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
-
-    <keeper_server>
-        <tcp_port>9181</tcp_port>
-        <server_id>2</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <operation_timeout_ms>10000</operation_timeout_ms>
-            <session_timeout_ms>30000</session_timeout_ms>
-            <raft_logs_level>trace</raft_logs_level>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>clickhouse-keeper1</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>2</id>
-                <hostname>clickhouse-keeper2</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>3</id>
-                <hostname>clickhouse-keeper3</hostname>
-                <port>9234</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-
-     <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
-    
-    <listen_host>0.0.0.0</listen_host>
-    <listen_try>1</listen_try>
-
-    <!--
-    <logger>
-        <console>1</console>
-    </logger>
-    -->
-</yandex>
diff --git a/clickhouse/dml/test_seed_traffic.sql b/clickhouse/dml/test_seed_traffic.sql
deleted file mode 100644
index 2399221..0000000
--- a/clickhouse/dml/test_seed_traffic.sql
+++ /dev/null
@@ -1,36 +0,0 @@
-INSERT INTO
-	traffic_records_all
-VALUES
-	(
-		'1698728400.40122',
-		'UDP',
-		'142.12.217.111',
-		'163.213.146.100',
-		443,
-		47104,
-		74
-	) (
-		'1698728400.401217',
-		'UDP',
-		'45.144.255.42',
-		'131.174.60.217',
-		51820,
-		63998,
-		42
-	) (
-		'1698728400.401218',
-		'TCP',
-		'152.199.153.111',
-		'202.215.192.69',
-		80,
-		65305,
-		66
-	) (
-		'1698728400.401219',
-		'UDP',
-		'45.144.255.42',
-		'131.174.60.217',
-		51820,
-		63998,
-		42
-	)
\ No newline at end of file
diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml
index d0fd34e..dac8f4b 100644
--- a/clickhouse/docker-compose.yaml
+++ b/clickhouse/docker-compose.yaml
@@ -59,7 +59,8 @@ services:
     container_name: clickhouse-server1
     volumes:
       - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
-      - ../clickhouse/ddl/main:/docker-entrypoint-initdb.d
+      - ../clickhouse/node-entrypoints/main:/docker-entrypoint-initdb.d
+      - ../preprocessing/geoip.csv:/tmp/seedData/csv/ip_region_map.csv
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
@@ -98,7 +99,7 @@ services:
     container_name: clickhouse-server2
     volumes:
       - ../clickhouse/node2-config/:/etc/clickhouse-server/config.d/
-      - ../clickhouse/ddl/common:/docker-entrypoint-initdb.d
+      - ../clickhouse/node-entrypoints/common:/docker-entrypoint-initdb.d
       - clickhouse_server2_data:/var/lib/clickhouse
       - clickhouse_server2_TTL:/clickhouse_data/server2
     networks:
diff --git a/clickhouse/node-entrypoints/main/03_insert_geoip_csv.sql b/clickhouse/node-entrypoints/main/03_insert_geoip_csv.sql
new file mode 100644
index 0000000..9a1ea5c
--- /dev/null
+++ b/clickhouse/node-entrypoints/main/03_insert_geoip_csv.sql
@@ -0,0 +1,3 @@
+INSERT INTO ip_region_map
+FROM INFILE '/tmp/seedData/csv/ip_region_map.csv'
+FORMAT CSVWithNames;
\ No newline at end of file
diff --git a/preprocessing/geoip.csv b/preprocessing/geoip.csv
index db087c0..305b5d4 100644
--- a/preprocessing/geoip.csv
+++ b/preprocessing/geoip.csv
@@ -1,4 +1,4 @@
-ip_range_start,ip_range_end,country
+ip_range_start,ip_range_end,region
 0.0.0.0,0.255.255.255,-
 1.0.0.0,1.0.0.255,Australia
 1.0.1.0,1.0.3.255,China