mirror of
https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git
synced 2026-01-25 16:04:04 +00:00
Fixed integration issue with csv streaming
This commit is contained in:
@@ -19,7 +19,7 @@ class KafkaClient:
|
||||
self.topic_name = topic_name
|
||||
if mode == 'producer':
|
||||
self.client = KafkaProducer(
|
||||
bootstrap_servers=['localhost:9092'],
|
||||
bootstrap_servers=['kafka:9092'],
|
||||
max_request_size = 200000000,
|
||||
#api_version=(0,11,5),
|
||||
value_serializer=lambda x: json.dumps(x).encode('utf-8'))
|
||||
@@ -198,6 +198,7 @@ if __name__ == "__main__":
|
||||
|
||||
# if preprocessed data ready for streaming
|
||||
if csv_file:
|
||||
#print("true")
|
||||
with open(csv_file, newline="") as f:
|
||||
csv_rdr = csv.reader(f)
|
||||
next(csv_rdr) # skip headers
|
||||
@@ -207,10 +208,10 @@ if __name__ == "__main__":
|
||||
# direct streaming to kafka goes here
|
||||
producer.client.send(KAFKA_TOPIC, row_to_dict(row))
|
||||
dbg_print(row_to_dict(row))
|
||||
dbg_print("streamed packet", idx)
|
||||
print("streamed packet", idx)
|
||||
if idx > sample_size:
|
||||
break
|
||||
dbg_print(f"total streamed: {idx}")
|
||||
print(f"total streamed: {idx}")
|
||||
|
||||
# otherwise, process packets
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user