my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 08:04:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

added volume template and modified update_compose accordingly. updated service template

Browse Source
This commit is contained in:
Lalit Arvind 2024-11-25 01:11:09 -07:00
parent ee03026d47
commit 351a6473a4
3 changed files with 20 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
clickhouse/config_update_scripts/update_compose.py
View File

@ -20,7 +20,7 @@ if __name__ == "__main__":
# extracting only 'server1','server2'... # extracting only 'server1','server2'...
all_service_names = [ name.split('-')[-1] for name in all_service_names] all_service_names = [ name.split('-')[-1] for name in all_service_names]
curr_num_servers = sorted(all_service_names)[-1][-1] curr_num_servers = int(sorted(all_service_names)[-1][-1])
replication_factor = 2 replication_factor = 2
curr_num_shards = curr_num_servers/replication_factor curr_num_shards = curr_num_servers/replication_factor
@ -57,21 +57,27 @@ if __name__ == "__main__":
output_path = f'../node{i}-config/remote-servers.xml' output_path = f'../node{i}-config/remote-servers.xml'
curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False) curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False)
env = Environment(loader=FileSystemLoader('.')) env = Environment(loader=FileSystemLoader('../jinja-templates'))
service_template = env.get_template('service.yml.jinja') service_template = env.get_template('service.yml.jinja')
volume_template = env.get_template('volume.yml.jinja')
# loading existing docker-compose file # loading existing docker-compose file
with open('../docker-compose.yaml','r') as f: with open('../docker-compose.yaml','r') as f:
compose_f = yaml.safe_load(f) compose_f = yaml.safe_load(f)
# rendering the new service # rendering the new service
new_service1 = service_template.render(server_num=curr_num_servers+1) new_service1 = yaml.safe_load(service_template.render(server_num=curr_num_servers+1))
new_service2 = service_template.render(server_num=curr_num_servers+2) new_service2 = yaml.safe_load(service_template.render(server_num=curr_num_servers+2))
new_volume1 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+1))
new_volume2 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+2))
# adding the new service to docker-compose # adding the new service to docker-compose
compose_f['services'].update(new_service1) compose_f['services'].update(new_service1)
compose_f['services'].update(new_service2) compose_f['services'].update(new_service2)
compose_f['volumes'].update(new_volume1)
compose_f['volumes'].update(new_volume2)
if compose_f: if compose_f:
with open('../docker-compose.yaml','w') as yamlfile: with open('../docker-compose.yaml','w') as yamlfile:
yaml.safe_dump(compose_f, yamlfile) yaml.safe_dump(compose_f, yamlfile)

10
clickhouse/jinja-templates/service.yml.jinja
View File

@ -3,7 +3,7 @@ clickhouse-server{{server_num}}:
container_name: clickhouse-server{{server_num}} container_name: clickhouse-server{{server_num}}
volumes: volumes:
- ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/ - ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/
- clickhouse_data{{server_num}}:/var/lib/clickhouse - clickhouse_server{{server_num}}_data:/var/lib/clickhouse
networks: networks:
clickhouse-server-network: clickhouse-server-network:
aliases: aliases:
@ -17,10 +17,10 @@ clickhouse-server{{server_num}}:
# constraints: [node.labels.role == server] # constraints: [node.labels.role == server]
update_config: update_config:
delay: 10s delay: 10s
# resources: resources:
# limits: limits:
# cpus: "0.50" cpus: "0.50"
# memory: 100M memory: 100M
depends_on: depends_on:
- clickhouse-keeper1 - clickhouse-keeper1
- clickhouse-keeper2 - clickhouse-keeper2

4
clickhouse/jinja-templates/volume.yml.jinja Normal file
View File

@ -0,0 +1,4 @@
clickhouse_server{{server_num}}_data:
driver: local
clickhouse_server{{server_num}}_TTL:
driver: local