my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 09:04:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

pcap check

Browse Source
This commit is contained in:
Akash Sivakumar 2024-11-25 16:53:13 -07:00
parent 5b1add9241
commit 35d48a2cf4
1 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
preprocessing/pcap_processor.py
View File

@ -19,7 +19,7 @@ class KafkaClient:
self.topic_name = topic_name self.topic_name = topic_name
if mode == 'producer': if mode == 'producer':
self.client = KafkaProducer( self.client = KafkaProducer(
bootstrap_servers=['kafka:9092'], bootstrap_servers=['localhost:9092'],
max_request_size = 200000000, max_request_size = 200000000,
#api_version=(0,11,5), #api_version=(0,11,5),
value_serializer=lambda x: json.dumps(x).encode('utf-8')) value_serializer=lambda x: json.dumps(x).encode('utf-8'))
@ -186,7 +186,9 @@ if __name__ == "__main__":
pkts = [] pkts = []
cnt = 0 cnt = 0
seen_count = 0
for idx, pkt in enumerate(pcap_rdr): for idx, pkt in enumerate(pcap_rdr):
seen_count += 1
# filter packets # filter packets
if not pkt_filter(pkt): if not pkt_filter(pkt):
continue continue
@ -209,6 +211,7 @@ if __name__ == "__main__":
#print(f"streamed packet at index {idx} ") #print(f"streamed packet at index {idx} ")
if idx > sample_size: break if idx > sample_size: break
print(f"total seen: {seen_count-1}")
print(f"total streamed: {cnt}") print(f"total streamed: {cnt}")
# flush remaining # flush remaining
if not streaming and len(pkts) > 0: if not streaming and len(pkts) > 0: