my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2026-01-25 08:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

cc map, full scale data testing

Browse Source
This commit is contained in:
Kaushik Narayan R
2024-11-29 20:11:36 -07:00
parent 11abeb5337
commit a5887f30f5
10 changed files with 445 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
preprocessing/docker-compose.yml
View File

@@ -56,8 +56,10 @@ services:
aliases:
- data-streamer
volumes:
- "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -l 0.1'"
# - "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
- "../preprocessing/1M_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv'"
# command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -l 0.1'"
deploy:
replicas: 1
# placement:

4
preprocessing/ip2loc_prep.py
View File

@@ -14,7 +14,7 @@ def int_to_ipv4(num: int) -> str:
# with open("IP2LOCATION-LITE-DB3.csv", "r") as input_file, open(
with open("IP2LOCATION-LITE-DB1.csv", "r") as input_file, open(
"geoip.csv", "w", newline=""
"geoip_cc.csv", "w", newline=""
) as output_file:
reader = csv.reader(input_file)
writer = csv.writer(output_file)
@@ -24,6 +24,7 @@ with open("IP2LOCATION-LITE-DB1.csv", "r") as input_file, open(
[
"ip_range_start",
"ip_range_end",
"country_code",
"country",
# "region",
# "city",
@@ -35,6 +36,7 @@ with open("IP2LOCATION-LITE-DB1.csv", "r") as input_file, open(
new_record = [
int_to_ipv4(int(record[0])),
int_to_ipv4(int(record[1])),
record[2],
record[3],
# record[4],
# record[5],