diff --git a/clickhouse/config_update_scripts/update_compose.py b/clickhouse/config_update_scripts/update_compose.py index 76f72fd..9f29a20 100644 --- a/clickhouse/config_update_scripts/update_compose.py +++ b/clickhouse/config_update_scripts/update_compose.py @@ -2,6 +2,9 @@ import yaml from jinja2 import Environment, FileSystemLoader import subprocess import json +import xml.etree.ElementTree as ET +import os + if __name__ == "__main__": # extracting details of each running container in json format @@ -24,19 +27,71 @@ if __name__ == "__main__": replication_factor = 2 curr_num_shards = curr_num_servers/replication_factor + + # new shard template that is gonna be added to remote servers file of each node + new_shard_str = f''' + + true + + clickhouse-server{curr_num_servers+1} + 9000 + + + clickhouse-server{curr_num_servers+2} + 9000 + + + ''' + # extracting existing remote-servers file + with open('../node1-config/remote-servers.xml','r') as f: + curr_remote_servers_xml = ET.parse(f) + + cluster_root = curr_remote_servers_xml.find('.//cluster_1S_2R') + new_shard_xml = ET.fromstring(new_shard_str) + cluster_root.append(new_shard_xml) + + # creating folders for new servers that contain the configuration files + os.makedirs(f'../node{curr_num_servers+1}-config',exist_ok=True) + os.makedirs(f'../node{curr_num_servers+2}-config',exist_ok=True) + + # adding the new shard to each remote-servers file + for i in range(1,curr_num_servers+3): + output_path = f'../node{i}-config/remote-servers.xml' + curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False) env = Environment(loader=FileSystemLoader('.')) - template = env.get_template('service.yml.jinja') + service_template = env.get_template('service.yml.jinja') + # loading existing docker-compose file with open('../docker-compose.yaml','r') as f: compose_f = yaml.safe_load(f) - new_service1 = template.render(server_num=curr_num_servers+1) - new_service2 = template.render(server_num=curr_num_servers+1) - + # rendering the new service + new_service1 = service_template.render(server_num=curr_num_servers+1) + new_service2 = service_template.render(server_num=curr_num_servers+2) + + # adding the new service to docker-compose compose_f['services'].update(new_service1) compose_f['services'].update(new_service2) if compose_f: with open('../docker-compose.yaml','w') as yamlfile: - yaml.safe_dump(compose_f, yamlfile) \ No newline at end of file + yaml.safe_dump(compose_f, yamlfile) + + config_template = env.get_template('config.xml.jinja') + macros_template = env.get_template('macros.xml.jinja') + use_keeper_template = env.get_template('use-keeper.xml.jinja') + + for i in range(1,3): + config_content = config_template.render(node_num=curr_num_servers+i) + with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1: + f1.write(config_content) + + macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i) + with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2: + f2.write(macros_content) + + use_keeper_content = use_keeper_template.render() + with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3: + f3.write(use_keeper_content) + diff --git a/clickhouse/jinja-templates/config.xml.jinja b/clickhouse/jinja-templates/config.xml.jinja new file mode 100644 index 0000000..cd07efb --- /dev/null +++ b/clickhouse/jinja-templates/config.xml.jinja @@ -0,0 +1,24 @@ + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 3 + + cluster_1S_2R node {{node_num}} + 0.0.0.0 + 8123 + 9000 + + 4096 + 3 + 100 + + + + /etc/clickhouse-server/config.d/macros.xml + /etc/clickhouse-server/config.d/remote-servers.xml + /etc/clickhouse-server/config.d/use-keeper.xml + + \ No newline at end of file diff --git a/clickhouse/jinja-templates/macros.xml.jinja b/clickhouse/jinja-templates/macros.xml.jinja new file mode 100644 index 0000000..f7ade4c --- /dev/null +++ b/clickhouse/jinja-templates/macros.xml.jinja @@ -0,0 +1,7 @@ + + + {{shard_num}} + {{replica_num}} + cluster_1S_2R + + \ No newline at end of file diff --git a/clickhouse/jinja-templates/remote-servers.xml.jinja b/clickhouse/jinja-templates/remote-servers.xml.jinja new file mode 100644 index 0000000..a6a9edd --- /dev/null +++ b/clickhouse/jinja-templates/remote-servers.xml.jinja @@ -0,0 +1,18 @@ + + + + mysecretphrase + + true + + clickhouse-server1 + 9000 + + + clickhouse-server2 + 9000 + + + + + \ No newline at end of file diff --git a/clickhouse/jinja-templates/use-keeper.xml.jinja b/clickhouse/jinja-templates/use-keeper.xml.jinja new file mode 100644 index 0000000..2b384dc --- /dev/null +++ b/clickhouse/jinja-templates/use-keeper.xml.jinja @@ -0,0 +1,17 @@ + + + + + clickhouse-keeper1 + 9181 + + + clickhouse-keeper2 + 9181 + + + clickhouse-keeper3 + 9181 + + + \ No newline at end of file