Added jinja template for each of node config files and created an preliminary update script

clickhouse/config_update_scripts/update_compose.py

@@ -2,6 +2,9 @@ import yaml
 from jinja2 import Environment, FileSystemLoader
 import subprocess
 import json
+import xml.etree.ElementTree as ET
+import os
+
 if __name__ == "__main__":
     
     # extracting details of each running container in json format    
@@ -25,18 +28,70 @@ if __name__ == "__main__":
     replication_factor = 2
     curr_num_shards = curr_num_servers/replication_factor
     
-    env = Environment(loader=FileSystemLoader('.'))
+    # new shard template that is gonna be added to remote servers file of each node
-    template = env.get_template('service.yml.jinja')
+    new_shard_str = f'''
+        <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server{curr_num_servers+1}</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server{curr_num_servers+2}</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+    '''
+    # extracting existing remote-servers file
+    with open('../node1-config/remote-servers.xml','r') as f:
+        curr_remote_servers_xml = ET.parse(f)
     
+    cluster_root = curr_remote_servers_xml.find('.//cluster_1S_2R')
+    new_shard_xml = ET.fromstring(new_shard_str)
+    cluster_root.append(new_shard_xml)
+
+    # creating folders for new servers that contain the configuration files
+    os.makedirs(f'../node{curr_num_servers+1}-config',exist_ok=True)
+    os.makedirs(f'../node{curr_num_servers+2}-config',exist_ok=True)
+
+    # adding the new shard to each remote-servers file
+    for i in range(1,curr_num_servers+3):
+        output_path = f'../node{i}-config/remote-servers.xml' 
+        curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False)
+
+    env = Environment(loader=FileSystemLoader('.'))
+    service_template = env.get_template('service.yml.jinja')
+
+    # loading existing docker-compose file
     with open('../docker-compose.yaml','r') as f:
         compose_f = yaml.safe_load(f)
     
-    new_service1 = template.render(server_num=curr_num_servers+1)
+    # rendering the new service
-    new_service2 = template.render(server_num=curr_num_servers+1)
+    new_service1 = service_template.render(server_num=curr_num_servers+1)
+    new_service2 = service_template.render(server_num=curr_num_servers+2)
     
+    # adding the new service to docker-compose
     compose_f['services'].update(new_service1)
     compose_f['services'].update(new_service2)
 
     if compose_f:
         with open('../docker-compose.yaml','w') as yamlfile:
             yaml.safe_dump(compose_f, yamlfile)
+
+    config_template = env.get_template('config.xml.jinja')
+    macros_template = env.get_template('macros.xml.jinja')
+    use_keeper_template = env.get_template('use-keeper.xml.jinja')
+
+    for i in range(1,3):
+        config_content = config_template.render(node_num=curr_num_servers+i)
+        with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1:
+            f1.write(config_content)
+        
+        macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i)
+        with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2:
+            f2.write(macros_content)
+        
+        use_keeper_content = use_keeper_template.render()
+        with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3:
+            f3.write(use_keeper_content)
+

clickhouse/jinja-templates/config.xml.jinja

@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node {{node_num}}</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>

clickhouse/jinja-templates/macros.xml.jinja

@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>{{shard_num}}</shard>
+        <replica>{{replica_num}}</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>

clickhouse/jinja-templates/remote-servers.xml.jinja

@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>

clickhouse/jinja-templates/use-keeper.xml.jinja

@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>