my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2026-01-25 08:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added jinja template for each of node config files and created an preliminary update script

Browse Source
This commit is contained in:
Lalit Arvind
2024-11-17 00:15:19 -07:00
parent 078ef06279
commit 8e03c62f72
5 changed files with 126 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
clickhouse/config_update_scripts/update_compose.py
View File

@@ -2,6 +2,9 @@ import yaml
from jinja2 import Environment, FileSystemLoader
import subprocess
import json
import xml.etree.ElementTree as ET
import os
if __name__ == "__main__":
# extracting details of each running container in json format
@@ -24,19 +27,71 @@ if __name__ == "__main__":
replication_factor = 2
curr_num_shards = curr_num_servers/replication_factor
# new shard template that is gonna be added to remote servers file of each node
new_shard_str = f'''
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server{curr_num_servers+1}</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse-server{curr_num_servers+2}</host>
<port>9000</port>
</replica>
</shard>
'''
# extracting existing remote-servers file
with open('../node1-config/remote-servers.xml','r') as f:
curr_remote_servers_xml = ET.parse(f)
cluster_root = curr_remote_servers_xml.find('.//cluster_1S_2R')
new_shard_xml = ET.fromstring(new_shard_str)
cluster_root.append(new_shard_xml)
# creating folders for new servers that contain the configuration files
os.makedirs(f'../node{curr_num_servers+1}-config',exist_ok=True)
os.makedirs(f'../node{curr_num_servers+2}-config',exist_ok=True)
# adding the new shard to each remote-servers file
for i in range(1,curr_num_servers+3):
output_path = f'../node{i}-config/remote-servers.xml'
curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False)
env = Environment(loader=FileSystemLoader('.'))
template = env.get_template('service.yml.jinja')
service_template = env.get_template('service.yml.jinja')
# loading existing docker-compose file
with open('../docker-compose.yaml','r') as f:
compose_f = yaml.safe_load(f)
new_service1 = template.render(server_num=curr_num_servers+1)
new_service2 = template.render(server_num=curr_num_servers+1)
# rendering the new service
new_service1 = service_template.render(server_num=curr_num_servers+1)
new_service2 = service_template.render(server_num=curr_num_servers+2)
# adding the new service to docker-compose
compose_f['services'].update(new_service1)
compose_f['services'].update(new_service2)
if compose_f:
with open('../docker-compose.yaml','w') as yamlfile:
yaml.safe_dump(compose_f, yamlfile)
yaml.safe_dump(compose_f, yamlfile)
config_template = env.get_template('config.xml.jinja')
macros_template = env.get_template('macros.xml.jinja')
use_keeper_template = env.get_template('use-keeper.xml.jinja')
for i in range(1,3):
config_content = config_template.render(node_num=curr_num_servers+i)
with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1:
f1.write(config_content)
macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i)
with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2:
f2.write(macros_content)
use_keeper_content = use_keeper_template.render()
with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3:
f3.write(use_keeper_content)