my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 08:54:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'clickhouse'

Browse Source
This commit is contained in:
Kaushik Narayan R 2024-11-25 23:22:27 -07:00
commit b8d36d0907
17 changed files with 219 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
clickhouse/Queries.sql Normal file
View File

@ -0,0 +1,22 @@
-- TABLE CREATION
CREATE TABLE traffic_records ON cluster_1S_2R
(
time_stamp DateTime64(3, 'Japan') CODEC(Delta, ZSTD),
protocol Enum('TCP' = 1, 'UDP' = 2),
from_IP IPv4,
to_IP IPv4,
port UInt16 CODEC(ZSTD),
INDEX port_idx port TYPE bloom_filter GRANULARITY 10
) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/traffic_records', '{replica}')
ORDER BY time_stamp
SETTINGS storage_policy = 'hot_cold'
TTL time_stamp + INTERVAL 5 DAY TO VOLUME 'cold_disk';
CREATE TABLE ip_region_map ON cluster_1S_2R
(
ip_range_start IPv4,
ip_range_end IPv4,
region String,
INDEX region_ind region TYPE bloom_filter
) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/ip_region_map', '{replica}')
ORDER BY ip_range_start;

9
clickhouse/config_update_scripts/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM python
WORKDIR /update_scripts
COPY . /update_scripts/
RUN pip install --no-cache-dir -r requirements.txt
CMD ["python3","update_trigger.py"]

8
clickhouse/config_update_scripts/requirements.txt Normal file
View File

@ -0,0 +1,8 @@
subprocess
json
jinja2
yaml
re
xml
schedule
time

33
clickhouse/config_update_scripts/update_compose.py
View File

@ -4,6 +4,7 @@ import subprocess
import json
import xml.etree.ElementTree as ET
import os
import re
if __name__ == "__main__":
@ -15,15 +16,11 @@ if __name__ == "__main__":
all_services = [json.loads(s) for s in all_services]
# extracting the name, removing the custom id from it and storing it in a list
all_service_names = [service['Names'].split('.')[0] for service in all_services]
# extracting only 'keeper1', 'server1'...
all_service_names = [service['Names'].split('.')[0] for service in all_services if re.findall(r'clickhouse-server',service['Names'])]
# extracting only 'server1','server2'...
all_service_names = [ name.split('-')[-1] for name in all_service_names]
# removing all keeepers
all_service_names.remove('keeper1')
all_service_names.remove('keeper2')
all_service_names.remove('keeper3')
curr_num_servers = sorted(all_service_names)[-1][-1]
curr_num_servers = int(sorted(all_service_names)[-1][-1])
replication_factor = 2
curr_num_shards = curr_num_servers/replication_factor
@ -31,6 +28,7 @@ if __name__ == "__main__":
# new shard template that is gonna be added to remote servers file of each node
new_shard_str = f'''
<shard>
<weight>{curr_num_shards+1}</weight>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server{curr_num_servers+1}</host>
@ -59,21 +57,27 @@ if __name__ == "__main__":
output_path = f'../node{i}-config/remote-servers.xml'
curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False)
env = Environment(loader=FileSystemLoader('.'))
env = Environment(loader=FileSystemLoader('../jinja-templates'))
service_template = env.get_template('service.yml.jinja')
volume_template = env.get_template('volume.yml.jinja')
# loading existing docker-compose file
with open('../docker-compose.yaml','r') as f:
compose_f = yaml.safe_load(f)
# rendering the new service
new_service1 = service_template.render(server_num=curr_num_servers+1)
new_service2 = service_template.render(server_num=curr_num_servers+2)
new_service1 = yaml.safe_load(service_template.render(server_num=curr_num_servers+1))
new_service2 = yaml.safe_load(service_template.render(server_num=curr_num_servers+2))
new_volume1 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+1))
new_volume2 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+2))
# adding the new service to docker-compose
compose_f['services'].update(new_service1)
compose_f['services'].update(new_service2)
compose_f['volumes'].update(new_volume1)
compose_f['volumes'].update(new_volume2)
if compose_f:
with open('../docker-compose.yaml','w') as yamlfile:
yaml.safe_dump(compose_f, yamlfile)
@ -81,13 +85,14 @@ if __name__ == "__main__":
config_template = env.get_template('config.xml.jinja')
macros_template = env.get_template('macros.xml.jinja')
use_keeper_template = env.get_template('use-keeper.xml.jinja')
storage_policy_template = env.get_template('storage-policy.xml.jinja')
for i in range(1,3):
config_content = config_template.render(node_num=curr_num_servers+i)
with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1:
f1.write(config_content)
macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i)
macros_content = macros_template.render(shard_num="0"+str(int(curr_num_shards+1)),replica_num=i)
with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2:
f2.write(macros_content)
@ -95,3 +100,7 @@ if __name__ == "__main__":
with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3:
f3.write(use_keeper_content)
storage_policy_content = storage_policy_template.render(server_num=curr_num_servers+i)
with open(f'../node{curr_num_servers + i}-config/storage-policy.xml','w') as f4:
f4.write(storage_policy_content)

34
clickhouse/config_update_scripts/update_trigger.py Normal file
View File

@ -0,0 +1,34 @@
import subprocess
import json
import re
import schedule
import time
def check_util_exec():
# extracting details of each running container in json format
try:
all_services = subprocess.check_output(["docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
except subprocess.CalledProcessError as e:
print(f"Command failed with return code {e.returncode}")
all_services = [json.loads(s) for s in all_services]
resource_util_exceed_flag = True # Flag to check if all of the containers have exceeded 80% memory utilization
for service in all_services:
if re.findall(r'clickhouse-server',service['Name']):
if float(service['MemPerc'][:-1]) < 80:
resource_util_exceed_flag = False
if resource_util_exceed_flag:
process = subprocess.Popen(['python3','update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
stdout, stderr = process.communicate() # Wait for the process to finish and capture output
print("Standard Output:", stdout)
print("Standard Error:", stderr)
if __name__ == "__main__":
schedule.every(30).seconds.do(check_util_exec)
while True:
schedule.run_pending()
time.sleep(1)

64
clickhouse/docker-compose-old.yaml
View File

@ -2,14 +2,10 @@ services:
clickhouse-keeper1:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-keeper1
command: >
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
volumes:
- ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
- ./clickhouse_data/data:/var/lib/clickhouse/data
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
- clickhouse_keeper1_data:/var/lib/clickhouse
networks:
clickhouse-keeper-network:
@ -19,15 +15,10 @@ services:
clickhouse-keeper2:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-keeper2
command: >
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
volumes:
- ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
- ./clickhouse_data/data:/var/lib/clickhouse/data
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
- clickhouse_keeper2_data:/var/lib/clickhouse
networks:
clickhouse-keeper-network:
aliases:
@ -36,15 +27,10 @@ services:
clickhouse-keeper3:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-keeper3
command: >
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
volumes:
- ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
- ./clickhouse_data/data:/var/lib/clickhouse/data
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
- clickhouse_keeper3_data:/var/lib/clickhouse
networks:
clickhouse-keeper-network:
aliases:
@ -55,7 +41,8 @@ services:
container_name: clickhouse-server1
volumes:
- ./node1-config/:/etc/clickhouse-server/config.d/
- clickhouse_data1:/var/lib/clickhouse
- clickhouse_server1_data:/var/lib/clickhouse
- clickhouse_server1_TTL:/clickhouse_data/server1
networks:
clickhouse-server-network:
aliases:
@ -69,10 +56,10 @@ services:
# constraints: [node.labels.role == server]
update_config:
delay: 10s
# resources:
# limits:
# cpus: "0.50"
# memory: 100M
resources:
limits:
cpus: "0.50"
memory: 1200M
depends_on:
- clickhouse-keeper1
- clickhouse-keeper2
@ -86,7 +73,8 @@ services:
container_name: clickhouse-server2
volumes:
- ./node2-config/:/etc/clickhouse-server/config.d/
- clickhouse_data2:/var/lib/clickhouse
- clickhouse_server2_data:/var/lib/clickhouse
- clickhouse_server2_TTL:/clickhouse_data/server2
networks:
clickhouse-server-network:
aliases:
@ -100,10 +88,10 @@ services:
# constraints: [node.labels.role == server]
update_config:
delay: 10s
# resources:
# limits:
# cpus: "0.50"
# memory: 100M
resources:
limits:
cpus: "0.50"
memory: 1200M
depends_on:
- clickhouse-keeper1
- clickhouse-keeper2
@ -115,11 +103,23 @@ services:
networks:
clickhouse-server-network:
driver: overlay
attachable: true
clickhouse-keeper-network:
driver: overlay
attachable: true
volumes:
clickhouse_data1:
clickhouse_server1_data:
driver: local
clickhouse_data2:
clickhouse_server2_data:
driver: local
clickhouse_keeper1_data:
driver: local
clickhouse_keeper2_data:
driver: local
clickhouse_keeper3_data:
driver: local
clickhouse_server1_TTL:
driver: local
clickhouse_server2_TTL:
driver: local

1
clickhouse/jinja-templates/config.xml.jinja
View File

@ -20,5 +20,6 @@
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

18
clickhouse/jinja-templates/remote-servers.xml.jinja
View File

@ -1,18 +0,0 @@
<clickhouse>
<remote_servers replace="true">
<cluster_1S_2R>
<secret>mysecretphrase</secret>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server1</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse-server2</host>
<port>9000</port>
</replica>
</shard>
</cluster_1S_2R>
</remote_servers>
</clickhouse>

10
clickhouse/jinja-templates/service.yml.jinja
View File

@ -3,7 +3,7 @@ clickhouse-server{{server_num}}:
container_name: clickhouse-server{{server_num}}
volumes:
- ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/
- clickhouse_data{{server_num}}:/var/lib/clickhouse
- clickhouse_server{{server_num}}_data:/var/lib/clickhouse
networks:
clickhouse-server-network:
aliases:
@ -17,10 +17,10 @@ clickhouse-server{{server_num}}:
# constraints: [node.labels.role == server]
update_config:
delay: 10s
# resources:
# limits:
# cpus: "0.50"
# memory: 100M
resources:
limits:
cpus: "0.50"
memory: 100M
depends_on:
- clickhouse-keeper1
- clickhouse-keeper2

25
clickhouse/jinja-templates/storage-policy.xml.jinja Normal file
View File

@ -0,0 +1,25 @@
<storage_configuration>
<disks>
<hot_disk>
<path>/clickhouse_data{{server_num}}/hot</path>
<keep_free_space_bytes>300000000</keep_free_space_bytes>
</hot_disk>
<cold_disk>
<path>/clickhouse_data{{server_num}}/cold</path>
<keep_free_space_bytes>500000000</keep_free_space_bytes>
</cold_disk>
</disks>
<policies>
<hot_cold>
<volumes>
<hot_vol>
<disk>hot_disk</disk>
</hot_vol>
<volume_name_2>
<disk>cold_disk</disk>
</volume_name_2>
</volumes>
<move_factor>0.2</move_factor>
</hot_cold>
</policies>
</storage_configuration>

4
clickhouse/jinja-templates/volume.yml.jinja Normal file
View File

@ -0,0 +1,4 @@
clickhouse_server{{server_num}}_data:
driver: local
clickhouse_server{{server_num}}_TTL:
driver: local

1
clickhouse/node1-config/config.xml
View File

@ -20,5 +20,6 @@
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

1
clickhouse/node1-config/remote-servers.xml
View File

@ -3,6 +3,7 @@
<cluster_1S_2R>
<secret>mysecretphrase</secret>
<shard>
<weight>1</weight>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server1</host>

27
clickhouse/node1-config/storage-policy.xml Normal file
View File

@ -0,0 +1,27 @@
<clickhouse>
<storage_configuration>
<disks>
<hot_disk>
<path>/clickhouse_data/server1/hot/</path>
</hot_disk>
<cold_disk>
<path>/clickhouse_data/server1/cold/</path>
</cold_disk>
</disks>
<policies>
<hot_cold>
<volumes>
<hot_vol>
<disk>hot_disk</disk>
<max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
</hot_vol>
<cold_vol>
<disk>cold_disk</disk>
<max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
</cold_vol>
</volumes>
<move_factor>0.2</move_factor>
</hot_cold>
</policies>
</storage_configuration>
</clickhouse>

1
clickhouse/node2-config/config.xml
View File

@ -20,5 +20,6 @@
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

1
clickhouse/node2-config/remote-servers.xml
View File

@ -3,6 +3,7 @@
<cluster_1S_2R>
<secret>mysecretphrase</secret>
<shard>
<weight>1</weight>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server1</host>

27
clickhouse/node2-config/storage-policy.xml Normal file
View File

@ -0,0 +1,27 @@
<clickhouse>
<storage_configuration>
<disks>
<hot_disk>
<path>/clickhouse_data/server2/hot/</path>
</hot_disk>
<cold_disk>
<path>/clickhouse_data/server2/cold/</path>
</cold_disk>
</disks>
<policies>
<hot_cold>
<volumes>
<hot_vol>
<disk>hot_disk</disk>
<max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
</hot_vol>
<cold_vol>
<disk>cold_disk</disk>
<max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
</cold_vol>
</volumes>
<move_factor>0.2</move_factor>
</hot_cold>
</policies>
</storage_configuration>
</clickhouse>