my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 08:04:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

geoip WiP

Browse Source
This commit is contained in:
Kaushik Narayan R 2024-11-29 01:59:51 -07:00
parent fe7f2e570d
commit c013d9cc67
1 changed files with 36 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
clickhouse/node-entrypoints/main/03_insert_geoip_csv.sql
View File

@ -1,3 +1,37 @@
INSERT INTO ip_region_map
FROM INFILE '/tmp/seedData/csv/ip_region_map.csv'
FROM INFILE '/var/lib/clickhouse/user_files/csv/ip_region_map.csv'
FORMAT CSVWithNames;
-- https://clickhouse.com/blog/geolocating-ips-in-clickhouse-and-grafana#using-bit-functions-to-convert-ip-ranges-to-cidr-notation
CREATE FUNCTION unmatchedBits AS (ip_s, ip_e) -> if(
bitXor(ip_s, ip_e) != 0,
ceil(log2(bitXor(ip_s, ip_e))), 0
);
CREATE FUNCTION cidrSuffix AS (ip_s, ip_e) -> 32 - unmatchedBits(ip_s, ip_e);
CREATE FUNCTION cidrAddress AS (ip_s, ip_e) -> toIPv4(
bitAnd(
bitNot(pow(2, unmatchedBits(ip_s, ip_e)) - 1),
ip_s
)::UInt64
);
CREATE FUNCTION IPv4RangeToCIDRString AS (ip_s, ip_e) -> CONCAT(
toString(cidrAddress(ip_s, ip_e)),
'/',
toString(cidrSuffix(ip_s, ip_e))
);
ALTER TABLE ip_region_map
ADD COLUMN ip_range_cidr String
MATERIALIZED IPv4RangeToCIDRString(ip_range_start, ip_range_end);
CREATE DICTIONARY ip_region_dict (ip_range_cidr String, region String) PRIMARY KEY ip_range_cidr SOURCE(CLICKHOUSE(TABLE 'ip_region_map')) LAYOUT(ip_trie) LIFETIME(3600);
-- SELECT
-- *,
-- dictGet('ip_region_dict', 'region', tuple(src_ip)) AS region
-- FROM traffic_records_all
-- LIMIT 10