From 20453ae38863ad74c4166611395bbf2cac3a817b Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Fri, 15 Nov 2024 09:57:53 -0700
Subject: [PATCH 1/3] small updates in docker-compose to execute swarm.
 clickhouse client connection through published port prob.

---
 .../data/preprocessed_configs/config.xml      | 61 -------------------
 clickhouse/docker-compose.yaml                | 32 +++++++++-
 2 files changed, 30 insertions(+), 63 deletions(-)
 delete mode 100644 clickhouse/clickhouse_data/data/preprocessed_configs/config.xml

diff --git a/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
deleted file mode 100644
index 08de94b..0000000
--- a/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!-- This file was generated automatically.
-     Do not edit it: it is likely to be discarded and generated again before it's read next time.
-     Files used to generate this file:
-       /etc/clickhouse-server/config.xml
-       /etc/clickhouse-server/config.d/docker_related_config.xml      -->
-
-<yandex>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
-        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <listen_host>::</listen_host>
-    
-    <path>/var/lib/clickhouse/data/</path>
-    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
-    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
-
-    <keeper_server>
-        <tcp_port>9181</tcp_port>
-        <server_id>2</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <operation_timeout_ms>10000</operation_timeout_ms>
-            <session_timeout_ms>30000</session_timeout_ms>
-            <raft_logs_level>trace</raft_logs_level>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>clickhouse-keeper1</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>2</id>
-                <hostname>clickhouse-keeper2</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>3</id>
-                <hostname>clickhouse-keeper3</hostname>
-                <port>9234</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-
-     <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
-    
-    <listen_host>0.0.0.0</listen_host>
-    <listen_try>1</listen_try>
-
-    <!--
-    <logger>
-        <console>1</console>
-    </logger>
-    -->
-</yandex>
diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml
index 7841015..7ee9aea 100644
--- a/clickhouse/docker-compose.yaml
+++ b/clickhouse/docker-compose.yaml
@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   clickhouse-keeper1:
     image: clickhouse/clickhouse-server:latest
@@ -57,6 +55,7 @@ services:
     container_name: clickhouse-server1
     volumes:
       - ./node1-config/:/etc/clickhouse-server/config.d/
+      - clickhouse_data1:/var/lib/clickhouse
     networks:
       clickhouse-server-network:
         aliases:
@@ -64,6 +63,16 @@ services:
       clickhouse-keeper-network:
         aliases:
           - clickhouse-server1
+    deploy:
+      replicas: 1
+      # placement:
+      #   constraints: [node.labels.role == server]
+      update_config:
+        delay: 10s
+      # resources:
+      #   limits:
+      #     cpus: "0.50"
+      #     memory: 100M
     depends_on:
       - clickhouse-keeper1
       - clickhouse-keeper2
@@ -77,6 +86,7 @@ services:
     container_name: clickhouse-server2
     volumes:
       - ./node2-config/:/etc/clickhouse-server/config.d/
+      - clickhouse_data2:/var/lib/clickhouse
     networks:
       clickhouse-server-network:
         aliases:
@@ -84,6 +94,16 @@ services:
       clickhouse-keeper-network:
         aliases:
           - clickhouse-server2
+    deploy:
+      replicas: 1
+      # placement:
+      #   constraints: [node.labels.role == server]
+      update_config:
+        delay: 10s
+      # resources:
+      #   limits:
+      #     cpus: "0.50"
+      #     memory: 100M
     depends_on:
       - clickhouse-keeper1
       - clickhouse-keeper2
@@ -94,4 +114,12 @@ services:
 
 networks:
   clickhouse-server-network:
+    driver: overlay
   clickhouse-keeper-network:
+    driver: overlay
+
+volumes:
+  clickhouse_data1:
+    driver: local
+  clickhouse_data2:
+    driver: local
\ No newline at end of file

From 078ef0627943c6b2989c3abf543da89868221e92 Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Sat, 16 Nov 2024 15:55:19 -0700
Subject: [PATCH 2/3] Jinja file and update script to add service in docker
 compose file

---
 .../config_update_scripts/update_compose.py   | 42 +++++++++++++++++++
 clickhouse/jinja-templates/service.yml.jinja  | 30 +++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 clickhouse/config_update_scripts/update_compose.py
 create mode 100644 clickhouse/jinja-templates/service.yml.jinja

diff --git a/clickhouse/config_update_scripts/update_compose.py b/clickhouse/config_update_scripts/update_compose.py
new file mode 100644
index 0000000..76f72fd
--- /dev/null
+++ b/clickhouse/config_update_scripts/update_compose.py
@@ -0,0 +1,42 @@
+import yaml
+from jinja2 import Environment, FileSystemLoader
+import subprocess
+import json
+if __name__ == "__main__":
+    
+    # extracting details of each running container in json format    
+    try:
+        all_services = subprocess.check_output(["docker","ps","--format","json"],text=True).split('\n')[:-1]
+    except subprocess.CalledProcessError as e:
+        print(f"Command failed with return code {e.returncode}")
+
+    all_services = [json.loads(s) for s in all_services]
+    # extracting the name, removing the custom id from it and storing it in a list
+    all_service_names = [service['Names'].split('.')[0] for service in all_services]
+    # extracting only 'keeper1', 'server1'...
+    all_service_names = [ name.split('-')[-1] for name in all_service_names]
+
+    # removing all keeepers
+    all_service_names.remove('keeper1')
+    all_service_names.remove('keeper2')
+    all_service_names.remove('keeper3')
+    curr_num_servers = sorted(all_service_names)[-1][-1]
+
+    replication_factor = 2
+    curr_num_shards = curr_num_servers/replication_factor
+
+    env = Environment(loader=FileSystemLoader('.'))
+    template = env.get_template('service.yml.jinja')
+
+    with open('../docker-compose.yaml','r') as f:
+        compose_f = yaml.safe_load(f)
+    
+    new_service1 = template.render(server_num=curr_num_servers+1)
+    new_service2 = template.render(server_num=curr_num_servers+1)
+
+    compose_f['services'].update(new_service1)
+    compose_f['services'].update(new_service2)
+
+    if compose_f:
+        with open('../docker-compose.yaml','w') as yamlfile:
+            yaml.safe_dump(compose_f, yamlfile)
\ No newline at end of file
diff --git a/clickhouse/jinja-templates/service.yml.jinja b/clickhouse/jinja-templates/service.yml.jinja
new file mode 100644
index 0000000..1ae0414
--- /dev/null
+++ b/clickhouse/jinja-templates/service.yml.jinja
@@ -0,0 +1,30 @@
+clickhouse-server{{server_num}}:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server{{server_num}}
+    volumes:
+      - ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/
+      - clickhouse_data{{server_num}}:/var/lib/clickhouse
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server{{server_num}}
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server{{server_num}}
+    deploy:
+      replicas: 1
+      # placement:
+      #   constraints: [node.labels.role == server]
+      update_config:
+        delay: 10s
+      # resources:
+      #   limits:
+      #     cpus: "0.50"
+      #     memory: 100M
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "900{{server_num}}:9000"  # Native client port
+      - "8123:8123"  # HTTP interface

From 8e03c62f7271f75ec3ee3de5b7fa054bf3d3228a Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Sun, 17 Nov 2024 00:15:19 -0700
Subject: [PATCH 3/3] Added jinja template for each of node config files and
 created an preliminary update script

---
 .../config_update_scripts/update_compose.py   | 65 +++++++++++++++++--
 clickhouse/jinja-templates/config.xml.jinja   | 24 +++++++
 clickhouse/jinja-templates/macros.xml.jinja   |  7 ++
 .../jinja-templates/remote-servers.xml.jinja  | 18 +++++
 .../jinja-templates/use-keeper.xml.jinja      | 17 +++++
 5 files changed, 126 insertions(+), 5 deletions(-)
 create mode 100644 clickhouse/jinja-templates/config.xml.jinja
 create mode 100644 clickhouse/jinja-templates/macros.xml.jinja
 create mode 100644 clickhouse/jinja-templates/remote-servers.xml.jinja
 create mode 100644 clickhouse/jinja-templates/use-keeper.xml.jinja

diff --git a/clickhouse/config_update_scripts/update_compose.py b/clickhouse/config_update_scripts/update_compose.py
index 76f72fd..9f29a20 100644
--- a/clickhouse/config_update_scripts/update_compose.py
+++ b/clickhouse/config_update_scripts/update_compose.py
@@ -2,6 +2,9 @@ import yaml
 from jinja2 import Environment, FileSystemLoader
 import subprocess
 import json
+import xml.etree.ElementTree as ET
+import os
+
 if __name__ == "__main__":
     
     # extracting details of each running container in json format    
@@ -24,19 +27,71 @@ if __name__ == "__main__":
 
     replication_factor = 2
     curr_num_shards = curr_num_servers/replication_factor
+    
+    # new shard template that is gonna be added to remote servers file of each node
+    new_shard_str = f'''
+        <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server{curr_num_servers+1}</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server{curr_num_servers+2}</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+    '''
+    # extracting existing remote-servers file
+    with open('../node1-config/remote-servers.xml','r') as f:
+        curr_remote_servers_xml = ET.parse(f)
+    
+    cluster_root = curr_remote_servers_xml.find('.//cluster_1S_2R')
+    new_shard_xml = ET.fromstring(new_shard_str)
+    cluster_root.append(new_shard_xml)
+
+    # creating folders for new servers that contain the configuration files
+    os.makedirs(f'../node{curr_num_servers+1}-config',exist_ok=True)
+    os.makedirs(f'../node{curr_num_servers+2}-config',exist_ok=True)
+
+    # adding the new shard to each remote-servers file
+    for i in range(1,curr_num_servers+3):
+        output_path = f'../node{i}-config/remote-servers.xml' 
+        curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False)
 
     env = Environment(loader=FileSystemLoader('.'))
-    template = env.get_template('service.yml.jinja')
+    service_template = env.get_template('service.yml.jinja')
 
+    # loading existing docker-compose file
     with open('../docker-compose.yaml','r') as f:
         compose_f = yaml.safe_load(f)
     
-    new_service1 = template.render(server_num=curr_num_servers+1)
-    new_service2 = template.render(server_num=curr_num_servers+1)
-
+    # rendering the new service
+    new_service1 = service_template.render(server_num=curr_num_servers+1)
+    new_service2 = service_template.render(server_num=curr_num_servers+2)
+    
+    # adding the new service to docker-compose
     compose_f['services'].update(new_service1)
     compose_f['services'].update(new_service2)
 
     if compose_f:
         with open('../docker-compose.yaml','w') as yamlfile:
-            yaml.safe_dump(compose_f, yamlfile)
\ No newline at end of file
+            yaml.safe_dump(compose_f, yamlfile)
+
+    config_template = env.get_template('config.xml.jinja')
+    macros_template = env.get_template('macros.xml.jinja')
+    use_keeper_template = env.get_template('use-keeper.xml.jinja')
+
+    for i in range(1,3):
+        config_content = config_template.render(node_num=curr_num_servers+i)
+        with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1:
+            f1.write(config_content)
+        
+        macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i)
+        with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2:
+            f2.write(macros_content)
+        
+        use_keeper_content = use_keeper_template.render()
+        with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3:
+            f3.write(use_keeper_content)
+
diff --git a/clickhouse/jinja-templates/config.xml.jinja b/clickhouse/jinja-templates/config.xml.jinja
new file mode 100644
index 0000000..cd07efb
--- /dev/null
+++ b/clickhouse/jinja-templates/config.xml.jinja
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node {{node_num}}</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/jinja-templates/macros.xml.jinja b/clickhouse/jinja-templates/macros.xml.jinja
new file mode 100644
index 0000000..f7ade4c
--- /dev/null
+++ b/clickhouse/jinja-templates/macros.xml.jinja
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>{{shard_num}}</shard>
+        <replica>{{replica_num}}</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/jinja-templates/remote-servers.xml.jinja b/clickhouse/jinja-templates/remote-servers.xml.jinja
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/clickhouse/jinja-templates/remote-servers.xml.jinja
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/jinja-templates/use-keeper.xml.jinja b/clickhouse/jinja-templates/use-keeper.xml.jinja
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/clickhouse/jinja-templates/use-keeper.xml.jinja
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file