my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 10:54:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

corrections

Browse Source
This commit is contained in:
Kaushik Narayan R 2024-11-15 18:17:41 -07:00
parent e6c0182724
commit 6150cc6679
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
preprocessing/pcap_aggregation.sh
View File

@ -7,35 +7,40 @@ data_month=10
total_size=0 total_size=0
for data_day in {01..31}; do for data_day in {01..31}; do
pcap_size=$(curl -sI "http://mawi.nezu.wide.ad.jp/mawi/samplepoint-F/${data_year}/${data_year}${data_month}${i}1400.pcap.gz" | pcap_size=$(curl -sI "http://mawi.nezu.wide.ad.jp/mawi/samplepoint-F/${data_year}/${data_year}${data_month}${data_day}1400.pcap.gz" |
grep Content-Length | grep Content-Length |
awk '{printf "%.3f", $2/1024/1024/1024}') awk '{printf "%.3f", $2/1024/1024/1024}')
echo "${data_year}-${data_month}-${data_day} - ${pcap_size} GB" echo "[o] ${data_year}-${data_month}-${data_day} - ${pcap_size} GB"
total_size=$(echo $total_size + $pcap_size | bc -l) total_size=$(echo $total_size + $pcap_size | bc -l)
done done
echo "Total size (compressed) of ${data_year}-${data_month} - ${total_size} GB" echo "[+] Total size (compressed) of ${data_year}-${data_month} - ${total_size} GB"
# Total size (compressed) of 2023-10 - 193.292 GB # Total size (compressed) of 2023-10 - 193.292 GB
# extracting data # extracting data
mkdir -p csv_files mkdir -p csv_files
# for data_day in {08..08}; do
for data_day in {01..31}; do for data_day in {01..31}; do
if [[ ! -f "${data_year}${data_month}${data_day}1400.pcap.gz" ]]; then if [[ ! -f "${data_year}${data_month}${data_day}1400.pcap.gz" ]]; then
wget "http://mawi.nezu.wide.ad.jp/mawi/samplepoint-F/${data_year}/${data_year}${data_month}${data_day}1400.pcap.gz" wget "http://mawi.nezu.wide.ad.jp/mawi/samplepoint-F/${data_year}/${data_year}${data_month}${data_day}1400.pcap.gz"
fi fi
gzip -d "${data_year}${data_month}${data_day}1400.pcap.gz"
echo "[+] decompression..."
# gzip -d "${data_year}${data_month}${data_day}1400.pcap.gz"
gzip -kd "${data_year}${data_month}${data_day}1400.pcap.gz"
echo "[+] packet processing..."
# 10000 packets from each day # 10000 packets from each day
python3 pcap_processor.py \ python3 pcap_processor.py \
--pcap_file "${data_year}${data_month}${data_day}1400.pcap" \ --pcap_file "${data_year}${data_month}${data_day}1400.pcap" \
--out_file csv_files/${data_day}.csv \ --out_file csv_files/${data_day}.csv \
--sample \ --sample \
--stream_size 10000 --stream_size 10000
rm "${data_year}${data_month}${data_day}1400.pcap" rm "${data_year}${data_month}${data_day}1400.pcap"
done done
# merge all CSV together # merge all CSV together
rm csv_files/merged.csv
awk '(NR == 1) || (FNR > 1)' csv_files/*.csv > csv_files/merged.csv awk '(NR == 1) || (FNR > 1)' csv_files/*.csv > csv_files/merged.csv