my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2026-01-25 08:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'preprocessing' into integration_2

Browse Source
This commit is contained in:
Kaushik Narayan R
2024-11-26 17:33:24 -07:00
parent b8d36d0907 6884fc681d
commit 75a75a08d6
3 changed files with 231582 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
preprocessing/pcap_processor.py
View File

@@ -189,11 +189,10 @@ if __name__ == "__main__":
out_file = args._out
streaming = args._stream
sample = args._sample
samplesize = int(args._streamsize)
DEBUG = args._debug
sample_size = samplesize #1000000
sample_size = int(args._samplesize) #1000000
batch_size = 100 #100000
# if preprocessed data ready for streaming
@@ -243,9 +242,8 @@ if __name__ == "__main__":
packet_data = create_pkt_object(pkt)
producer.client.send(KAFKA_TOPIC, packet_data)
cnt += 1
# print(f"streamed packet at index {idx} ")
if idx > sample_size:
break
#print(f"streamed packet at index {idx} ")
if idx > sample_size: break
print(f"total seen: {seen_count-1}")
print(f"total streamed: {cnt}")