From 2265c3023f456fad85cbbaf0b3319c2c57a23b8d Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Sat, 9 Nov 2024 09:43:15 -0700
Subject: [PATCH 1/5] Local docker containers demo config

---
 .../data/preprocessed_configs/config.xml      | 61 ++++++++++++
 clickhouse_keeper/keeper1-config.xml          | 44 +++++++++
 clickhouse_keeper/keeper2-config.xml          | 44 +++++++++
 clickhouse_keeper/keeper3-config.xml          | 44 +++++++++
 docker-compose.yaml                           | 97 +++++++++++++++++++
 node1-config/config.xml                       | 24 +++++
 node1-config/macros.xml                       |  7 ++
 node1-config/remote-servers.xml               | 18 ++++
 node1-config/use-keeper.xml                   | 17 ++++
 node2-config/config.xml                       | 24 +++++
 node2-config/macros.xml                       |  7 ++
 node2-config/remote-servers.xml               | 18 ++++
 node2-config/use-keeper.xml                   | 17 ++++
 13 files changed, 422 insertions(+)
 create mode 100644 clickhouse_data/data/preprocessed_configs/config.xml
 create mode 100644 clickhouse_keeper/keeper1-config.xml
 create mode 100644 clickhouse_keeper/keeper2-config.xml
 create mode 100644 clickhouse_keeper/keeper3-config.xml
 create mode 100644 docker-compose.yaml
 create mode 100644 node1-config/config.xml
 create mode 100644 node1-config/macros.xml
 create mode 100644 node1-config/remote-servers.xml
 create mode 100644 node1-config/use-keeper.xml
 create mode 100644 node2-config/config.xml
 create mode 100644 node2-config/macros.xml
 create mode 100644 node2-config/remote-servers.xml
 create mode 100644 node2-config/use-keeper.xml

diff --git a/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse_data/data/preprocessed_configs/config.xml
new file mode 100644
index 0000000..ae7050b
--- /dev/null
+++ b/clickhouse_data/data/preprocessed_configs/config.xml
@@ -0,0 +1,61 @@
+<!-- This file was generated automatically.
+     Do not edit it: it is likely to be discarded and generated again before it's read next time.
+     Files used to generate this file:
+       /etc/clickhouse-server/config.xml
+       /etc/clickhouse-server/config.d/docker_related_config.xml      -->
+
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>::</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+     <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
+    
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+
+    <!--
+    <logger>
+        <console>1</console>
+    </logger>
+    -->
+</yandex>
diff --git a/clickhouse_keeper/keeper1-config.xml b/clickhouse_keeper/keeper1-config.xml
new file mode 100644
index 0000000..c5e5d4d
--- /dev/null
+++ b/clickhouse_keeper/keeper1-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse_keeper/keeper2-config.xml b/clickhouse_keeper/keeper2-config.xml
new file mode 100644
index 0000000..bd2914e
--- /dev/null
+++ b/clickhouse_keeper/keeper2-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>2</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse_keeper/keeper3-config.xml b/clickhouse_keeper/keeper3-config.xml
new file mode 100644
index 0000000..383f9d9
--- /dev/null
+++ b/clickhouse_keeper/keeper3-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>3</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 0000000..86d8527
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,97 @@
+version: '3.8'
+
+services:
+  clickhouse-keeper1:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper1
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/data:/var/lib/clickhouse/data
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper1
+
+  clickhouse-keeper2:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper2
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/data:/var/lib/clickhouse/data
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper2
+
+  clickhouse-keeper3:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper3
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/data:/var/lib/clickhouse/data
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper3
+
+  clickhouse-server1:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server1
+    volumes:
+      - /Users/lalitarvind/Downloads/DDS_proj/node1-config/:/etc/clickhouse-server/config.d/
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server1
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server1
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "9001:9000"  # Native client port
+      - "8123:8123"  # HTTP interface
+
+  clickhouse-server2:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server2
+    volumes:
+      - /Users/lalitarvind/Downloads/DDS_proj/node2-config/:/etc/clickhouse-server/config.d/
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server2
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server2
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "9002:9000"  # Native client port
+      - "8124:8123"  # HTTP interface
+
+networks:
+  clickhouse-server-network:
+  clickhouse-keeper-network:
diff --git a/node1-config/config.xml b/node1-config/config.xml
new file mode 100644
index 0000000..4ced53d
--- /dev/null
+++ b/node1-config/config.xml
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node 1</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/node1-config/macros.xml b/node1-config/macros.xml
new file mode 100644
index 0000000..e8f48c6
--- /dev/null
+++ b/node1-config/macros.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>01</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/node1-config/remote-servers.xml b/node1-config/remote-servers.xml
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/node1-config/remote-servers.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/node1-config/use-keeper.xml b/node1-config/use-keeper.xml
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/node1-config/use-keeper.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file
diff --git a/node2-config/config.xml b/node2-config/config.xml
new file mode 100644
index 0000000..68d5b06
--- /dev/null
+++ b/node2-config/config.xml
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node 2</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/node2-config/macros.xml b/node2-config/macros.xml
new file mode 100644
index 0000000..52257ed
--- /dev/null
+++ b/node2-config/macros.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>02</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/node2-config/remote-servers.xml b/node2-config/remote-servers.xml
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/node2-config/remote-servers.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/node2-config/use-keeper.xml b/node2-config/use-keeper.xml
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/node2-config/use-keeper.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file

From eb17e0660522f5ce4c6293f74e361f7c6ad4a428 Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Sat, 9 Nov 2024 10:40:46 -0700
Subject: [PATCH 2/5] modify paths for common use

---
 .../data/preprocessed_configs/config.xml      |  2 +-
 docker-compose.yaml                           | 34 +++++++++----------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse_data/data/preprocessed_configs/config.xml
index ae7050b..08de94b 100644
--- a/clickhouse_data/data/preprocessed_configs/config.xml
+++ b/clickhouse_data/data/preprocessed_configs/config.xml
@@ -21,7 +21,7 @@
 
     <keeper_server>
         <tcp_port>9181</tcp_port>
-        <server_id>1</server_id>
+        <server_id>2</server_id>
         <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
         <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
         <coordination_settings>
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 86d8527..7841015 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -7,11 +7,11 @@ services:
     command: >
       /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/data:/var/lib/clickhouse/data
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
 
     networks:
       clickhouse-keeper-network:
@@ -24,11 +24,11 @@ services:
     command: >
       /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/data:/var/lib/clickhouse/data
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
 
     networks:
       clickhouse-keeper-network:
@@ -41,11 +41,11 @@ services:
     command: >
       /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/data:/var/lib/clickhouse/data
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - /Users/lalitarvind/Downloads/DDS_proj/clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
 
     networks:
       clickhouse-keeper-network:
@@ -56,7 +56,7 @@ services:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server1
     volumes:
-      - /Users/lalitarvind/Downloads/DDS_proj/node1-config/:/etc/clickhouse-server/config.d/
+      - ./node1-config/:/etc/clickhouse-server/config.d/
     networks:
       clickhouse-server-network:
         aliases:
@@ -76,7 +76,7 @@ services:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server2
     volumes:
-      - /Users/lalitarvind/Downloads/DDS_proj/node2-config/:/etc/clickhouse-server/config.d/
+      - ./node2-config/:/etc/clickhouse-server/config.d/
     networks:
       clickhouse-server-network:
         aliases:

From 2936f6c0db1dcbde14b515fa1fcda1eba714030f Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Sat, 9 Nov 2024 11:05:48 -0700
Subject: [PATCH 3/5] README

---
 README | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 README

diff --git a/README b/README
new file mode 100644
index 0000000..af20090
--- /dev/null
+++ b/README
@@ -0,0 +1,13 @@
+Execution steps:
+
+ensure to pull clickhouse docker image
+> docker-compose up -d
+identify the custom networks created.
+> docker network ls
+inspect the keeper network and verify whether all the containers are connected to it
+> docker network inspect {{dds_proj_clickhouse-keeper-network}}
+if all the containers are not connected
+> docker-compose restart
+To execute queries
+> docker exec -it clickhouse-server1 clickhouse-client
+> docker exec -it clickhouse-server2 clickhouse-client
\ No newline at end of file

From 90a7a2415dccaa74d549141b19b64a316e4b1429 Mon Sep 17 00:00:00 2001
From: Kaushik Narayan R <rknarayan02@gmail.com>
Date: Thu, 14 Nov 2024 22:33:10 -0700
Subject: [PATCH 4/5] reorg

---
 clickhouse/README                             | 13 +++
 .../data/preprocessed_configs/config.xml      | 61 ++++++++++++
 .../clickhouse_keeper/keeper1-config.xml      | 44 +++++++++
 .../clickhouse_keeper/keeper2-config.xml      | 44 +++++++++
 .../clickhouse_keeper/keeper3-config.xml      | 44 +++++++++
 clickhouse/docker-compose.yaml                | 97 +++++++++++++++++++
 clickhouse/node1-config/config.xml            | 24 +++++
 clickhouse/node1-config/macros.xml            |  7 ++
 clickhouse/node1-config/remote-servers.xml    | 18 ++++
 clickhouse/node1-config/use-keeper.xml        | 17 ++++
 clickhouse/node2-config/config.xml            | 24 +++++
 clickhouse/node2-config/macros.xml            |  7 ++
 clickhouse/node2-config/remote-servers.xml    | 18 ++++
 clickhouse/node2-config/use-keeper.xml        | 17 ++++
 14 files changed, 435 insertions(+)
 create mode 100644 clickhouse/README
 create mode 100644 clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
 create mode 100644 clickhouse/clickhouse_keeper/keeper1-config.xml
 create mode 100644 clickhouse/clickhouse_keeper/keeper2-config.xml
 create mode 100644 clickhouse/clickhouse_keeper/keeper3-config.xml
 create mode 100644 clickhouse/docker-compose.yaml
 create mode 100644 clickhouse/node1-config/config.xml
 create mode 100644 clickhouse/node1-config/macros.xml
 create mode 100644 clickhouse/node1-config/remote-servers.xml
 create mode 100644 clickhouse/node1-config/use-keeper.xml
 create mode 100644 clickhouse/node2-config/config.xml
 create mode 100644 clickhouse/node2-config/macros.xml
 create mode 100644 clickhouse/node2-config/remote-servers.xml
 create mode 100644 clickhouse/node2-config/use-keeper.xml

diff --git a/clickhouse/README b/clickhouse/README
new file mode 100644
index 0000000..af20090
--- /dev/null
+++ b/clickhouse/README
@@ -0,0 +1,13 @@
+Execution steps:
+
+ensure to pull clickhouse docker image
+> docker-compose up -d
+identify the custom networks created.
+> docker network ls
+inspect the keeper network and verify whether all the containers are connected to it
+> docker network inspect {{dds_proj_clickhouse-keeper-network}}
+if all the containers are not connected
+> docker-compose restart
+To execute queries
+> docker exec -it clickhouse-server1 clickhouse-client
+> docker exec -it clickhouse-server2 clickhouse-client
\ No newline at end of file
diff --git a/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
new file mode 100644
index 0000000..08de94b
--- /dev/null
+++ b/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
@@ -0,0 +1,61 @@
+<!-- This file was generated automatically.
+     Do not edit it: it is likely to be discarded and generated again before it's read next time.
+     Files used to generate this file:
+       /etc/clickhouse-server/config.xml
+       /etc/clickhouse-server/config.d/docker_related_config.xml      -->
+
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>::</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>2</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+     <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
+    
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+
+    <!--
+    <logger>
+        <console>1</console>
+    </logger>
+    -->
+</yandex>
diff --git a/clickhouse/clickhouse_keeper/keeper1-config.xml b/clickhouse/clickhouse_keeper/keeper1-config.xml
new file mode 100644
index 0000000..c5e5d4d
--- /dev/null
+++ b/clickhouse/clickhouse_keeper/keeper1-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse/clickhouse_keeper/keeper2-config.xml b/clickhouse/clickhouse_keeper/keeper2-config.xml
new file mode 100644
index 0000000..bd2914e
--- /dev/null
+++ b/clickhouse/clickhouse_keeper/keeper2-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>2</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse/clickhouse_keeper/keeper3-config.xml b/clickhouse/clickhouse_keeper/keeper3-config.xml
new file mode 100644
index 0000000..383f9d9
--- /dev/null
+++ b/clickhouse/clickhouse_keeper/keeper3-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>3</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml
new file mode 100644
index 0000000..7841015
--- /dev/null
+++ b/clickhouse/docker-compose.yaml
@@ -0,0 +1,97 @@
+version: '3.8'
+
+services:
+  clickhouse-keeper1:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper1
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper1
+
+  clickhouse-keeper2:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper2
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper2
+
+  clickhouse-keeper3:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper3
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper3
+
+  clickhouse-server1:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server1
+    volumes:
+      - ./node1-config/:/etc/clickhouse-server/config.d/
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server1
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server1
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "9001:9000"  # Native client port
+      - "8123:8123"  # HTTP interface
+
+  clickhouse-server2:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server2
+    volumes:
+      - ./node2-config/:/etc/clickhouse-server/config.d/
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server2
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server2
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "9002:9000"  # Native client port
+      - "8124:8123"  # HTTP interface
+
+networks:
+  clickhouse-server-network:
+  clickhouse-keeper-network:
diff --git a/clickhouse/node1-config/config.xml b/clickhouse/node1-config/config.xml
new file mode 100644
index 0000000..4ced53d
--- /dev/null
+++ b/clickhouse/node1-config/config.xml
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node 1</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node1-config/macros.xml b/clickhouse/node1-config/macros.xml
new file mode 100644
index 0000000..e8f48c6
--- /dev/null
+++ b/clickhouse/node1-config/macros.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>01</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node1-config/remote-servers.xml b/clickhouse/node1-config/remote-servers.xml
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/clickhouse/node1-config/remote-servers.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node1-config/use-keeper.xml b/clickhouse/node1-config/use-keeper.xml
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/clickhouse/node1-config/use-keeper.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/config.xml b/clickhouse/node2-config/config.xml
new file mode 100644
index 0000000..68d5b06
--- /dev/null
+++ b/clickhouse/node2-config/config.xml
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node 2</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/macros.xml b/clickhouse/node2-config/macros.xml
new file mode 100644
index 0000000..52257ed
--- /dev/null
+++ b/clickhouse/node2-config/macros.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>02</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/remote-servers.xml b/clickhouse/node2-config/remote-servers.xml
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/clickhouse/node2-config/remote-servers.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/use-keeper.xml b/clickhouse/node2-config/use-keeper.xml
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/clickhouse/node2-config/use-keeper.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file

From afdddbaf544d5ccc395b5e9c0cb0069372d1d7d4 Mon Sep 17 00:00:00 2001
From: Kaushik Narayan R <rknarayan02@gmail.com>
Date: Thu, 14 Nov 2024 22:41:18 -0700
Subject: [PATCH 5/5] reorg back into master

---
 README                                        | 13 ---
 .../data/preprocessed_configs/config.xml      | 61 ------------
 clickhouse_keeper/keeper1-config.xml          | 44 ---------
 clickhouse_keeper/keeper2-config.xml          | 44 ---------
 clickhouse_keeper/keeper3-config.xml          | 44 ---------
 docker-compose.yaml                           | 97 -------------------
 node1-config/config.xml                       | 24 -----
 node1-config/macros.xml                       |  7 --
 node1-config/remote-servers.xml               | 18 ----
 node1-config/use-keeper.xml                   | 17 ----
 node2-config/config.xml                       | 24 -----
 node2-config/macros.xml                       |  7 --
 node2-config/remote-servers.xml               | 18 ----
 node2-config/use-keeper.xml                   | 17 ----
 14 files changed, 435 deletions(-)
 delete mode 100644 README
 delete mode 100644 clickhouse_data/data/preprocessed_configs/config.xml
 delete mode 100644 clickhouse_keeper/keeper1-config.xml
 delete mode 100644 clickhouse_keeper/keeper2-config.xml
 delete mode 100644 clickhouse_keeper/keeper3-config.xml
 delete mode 100644 docker-compose.yaml
 delete mode 100644 node1-config/config.xml
 delete mode 100644 node1-config/macros.xml
 delete mode 100644 node1-config/remote-servers.xml
 delete mode 100644 node1-config/use-keeper.xml
 delete mode 100644 node2-config/config.xml
 delete mode 100644 node2-config/macros.xml
 delete mode 100644 node2-config/remote-servers.xml
 delete mode 100644 node2-config/use-keeper.xml

diff --git a/README b/README
deleted file mode 100644
index af20090..0000000
--- a/README
+++ /dev/null
@@ -1,13 +0,0 @@
-Execution steps:
-
-ensure to pull clickhouse docker image
-> docker-compose up -d
-identify the custom networks created.
-> docker network ls
-inspect the keeper network and verify whether all the containers are connected to it
-> docker network inspect {{dds_proj_clickhouse-keeper-network}}
-if all the containers are not connected
-> docker-compose restart
-To execute queries
-> docker exec -it clickhouse-server1 clickhouse-client
-> docker exec -it clickhouse-server2 clickhouse-client
\ No newline at end of file
diff --git a/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse_data/data/preprocessed_configs/config.xml
deleted file mode 100644
index 08de94b..0000000
--- a/clickhouse_data/data/preprocessed_configs/config.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<!-- This file was generated automatically.
-     Do not edit it: it is likely to be discarded and generated again before it's read next time.
-     Files used to generate this file:
-       /etc/clickhouse-server/config.xml
-       /etc/clickhouse-server/config.d/docker_related_config.xml      -->
-
-<yandex>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
-        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <listen_host>::</listen_host>
-    
-    <path>/var/lib/clickhouse/data/</path>
-    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
-    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
-
-    <keeper_server>
-        <tcp_port>9181</tcp_port>
-        <server_id>2</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <operation_timeout_ms>10000</operation_timeout_ms>
-            <session_timeout_ms>30000</session_timeout_ms>
-            <raft_logs_level>trace</raft_logs_level>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>clickhouse-keeper1</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>2</id>
-                <hostname>clickhouse-keeper2</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>3</id>
-                <hostname>clickhouse-keeper3</hostname>
-                <port>9234</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-
-     <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
-    
-    <listen_host>0.0.0.0</listen_host>
-    <listen_try>1</listen_try>
-
-    <!--
-    <logger>
-        <console>1</console>
-    </logger>
-    -->
-</yandex>
diff --git a/clickhouse_keeper/keeper1-config.xml b/clickhouse_keeper/keeper1-config.xml
deleted file mode 100644
index c5e5d4d..0000000
--- a/clickhouse_keeper/keeper1-config.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<yandex>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
-        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <listen_host>0.0.0.0</listen_host>
-    
-    <path>/var/lib/clickhouse/data/</path>
-    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
-    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
-
-    <keeper_server>
-        <tcp_port>9181</tcp_port>
-        <server_id>1</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <operation_timeout_ms>10000</operation_timeout_ms>
-            <session_timeout_ms>30000</session_timeout_ms>
-            <raft_logs_level>trace</raft_logs_level>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>clickhouse-keeper1</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>2</id>
-                <hostname>clickhouse-keeper2</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>3</id>
-                <hostname>clickhouse-keeper3</hostname>
-                <port>9234</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-</yandex>
diff --git a/clickhouse_keeper/keeper2-config.xml b/clickhouse_keeper/keeper2-config.xml
deleted file mode 100644
index bd2914e..0000000
--- a/clickhouse_keeper/keeper2-config.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<yandex>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
-        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <listen_host>0.0.0.0</listen_host>
-    
-    <path>/var/lib/clickhouse/data/</path>
-    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
-    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
-
-    <keeper_server>
-        <tcp_port>9181</tcp_port>
-        <server_id>2</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <operation_timeout_ms>10000</operation_timeout_ms>
-            <session_timeout_ms>30000</session_timeout_ms>
-            <raft_logs_level>trace</raft_logs_level>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>clickhouse-keeper1</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>2</id>
-                <hostname>clickhouse-keeper2</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>3</id>
-                <hostname>clickhouse-keeper3</hostname>
-                <port>9234</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-</yandex>
diff --git a/clickhouse_keeper/keeper3-config.xml b/clickhouse_keeper/keeper3-config.xml
deleted file mode 100644
index 383f9d9..0000000
--- a/clickhouse_keeper/keeper3-config.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<yandex>
-    <logger>
-        <level>trace</level>
-        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
-        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <listen_host>0.0.0.0</listen_host>
-    
-    <path>/var/lib/clickhouse/data/</path>
-    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
-    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
-    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
-
-    <keeper_server>
-        <tcp_port>9181</tcp_port>
-        <server_id>3</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <operation_timeout_ms>10000</operation_timeout_ms>
-            <session_timeout_ms>30000</session_timeout_ms>
-            <raft_logs_level>trace</raft_logs_level>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>clickhouse-keeper1</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>2</id>
-                <hostname>clickhouse-keeper2</hostname>
-                <port>9234</port>
-            </server>
-            <server>
-                <id>3</id>
-                <hostname>clickhouse-keeper3</hostname>
-                <port>9234</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-</yandex>
diff --git a/docker-compose.yaml b/docker-compose.yaml
deleted file mode 100644
index 7841015..0000000
--- a/docker-compose.yaml
+++ /dev/null
@@ -1,97 +0,0 @@
-version: '3.8'
-
-services:
-  clickhouse-keeper1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper1
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-
-    networks:
-      clickhouse-keeper-network:
-        aliases:
-          - clickhouse-keeper1
-
-  clickhouse-keeper2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper2
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-
-    networks:
-      clickhouse-keeper-network:
-        aliases:
-          - clickhouse-keeper2
-
-  clickhouse-keeper3:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-keeper3
-    command: >
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-    volumes:
-      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-
-    networks:
-      clickhouse-keeper-network:
-        aliases:
-          - clickhouse-keeper3
-
-  clickhouse-server1:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server1
-    volumes:
-      - ./node1-config/:/etc/clickhouse-server/config.d/
-    networks:
-      clickhouse-server-network:
-        aliases:
-          - clickhouse-server1
-      clickhouse-keeper-network:
-        aliases:
-          - clickhouse-server1
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9001:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
-
-  clickhouse-server2:
-    image: clickhouse/clickhouse-server:latest
-    container_name: clickhouse-server2
-    volumes:
-      - ./node2-config/:/etc/clickhouse-server/config.d/
-    networks:
-      clickhouse-server-network:
-        aliases:
-          - clickhouse-server2
-      clickhouse-keeper-network:
-        aliases:
-          - clickhouse-server2
-    depends_on:
-      - clickhouse-keeper1
-      - clickhouse-keeper2
-      - clickhouse-keeper3
-    ports:
-      - "9002:9000"  # Native client port
-      - "8124:8123"  # HTTP interface
-
-networks:
-  clickhouse-server-network:
-  clickhouse-keeper-network:
diff --git a/node1-config/config.xml b/node1-config/config.xml
deleted file mode 100644
index 4ced53d..0000000
--- a/node1-config/config.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<clickhouse>
-    <logger>
-        <level>debug</level>
-        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
-        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <display_name>cluster_1S_2R node 1</display_name>
-    <listen_host>0.0.0.0</listen_host>
-    <http_port>8123</http_port>
-    <tcp_port>9000</tcp_port>
-    <!-- Maximum connections and settings -->
-    <max_connections>4096</max_connections>
-    <keep_alive_timeout>3</keep_alive_timeout>
-    <max_concurrent_queries>100</max_concurrent_queries>
-
-
-    <!-- Additional configuration files can be included -->
-    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
-    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
-    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
-    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
-</clickhouse>
\ No newline at end of file
diff --git a/node1-config/macros.xml b/node1-config/macros.xml
deleted file mode 100644
index e8f48c6..0000000
--- a/node1-config/macros.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<clickhouse>
-    <macros>
-        <shard>01</shard>
-        <replica>01</replica>
-        <cluster>cluster_1S_2R</cluster>
-    </macros>
-</clickhouse>
\ No newline at end of file
diff --git a/node1-config/remote-servers.xml b/node1-config/remote-servers.xml
deleted file mode 100644
index a6a9edd..0000000
--- a/node1-config/remote-servers.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-<clickhouse>
-    <remote_servers replace="true">
-        <cluster_1S_2R>
-            <secret>mysecretphrase</secret>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>clickhouse-server1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>clickhouse-server2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </cluster_1S_2R>
-    </remote_servers>
-</clickhouse>
\ No newline at end of file
diff --git a/node1-config/use-keeper.xml b/node1-config/use-keeper.xml
deleted file mode 100644
index 2b384dc..0000000
--- a/node1-config/use-keeper.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-<clickhouse>
-    <zookeeper>
-        <!-- where are the ZK nodes -->
-        <node>
-            <host>clickhouse-keeper1</host>
-            <port>9181</port>
-        </node>
-        <node>
-            <host>clickhouse-keeper2</host>
-            <port>9181</port>
-        </node>
-        <node>
-            <host>clickhouse-keeper3</host>
-            <port>9181</port>
-        </node>
-    </zookeeper>
-</clickhouse>
\ No newline at end of file
diff --git a/node2-config/config.xml b/node2-config/config.xml
deleted file mode 100644
index 68d5b06..0000000
--- a/node2-config/config.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<clickhouse>
-    <logger>
-        <level>debug</level>
-        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
-        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
-        <size>1000M</size>
-        <count>3</count>
-    </logger>
-    <display_name>cluster_1S_2R node 2</display_name>
-    <listen_host>0.0.0.0</listen_host>
-    <http_port>8123</http_port>
-    <tcp_port>9000</tcp_port>
-    <!-- Maximum connections and settings -->
-    <max_connections>4096</max_connections>
-    <keep_alive_timeout>3</keep_alive_timeout>
-    <max_concurrent_queries>100</max_concurrent_queries>
-
-
-    <!-- Additional configuration files can be included -->
-    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
-    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
-    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
-    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
-</clickhouse>
\ No newline at end of file
diff --git a/node2-config/macros.xml b/node2-config/macros.xml
deleted file mode 100644
index 52257ed..0000000
--- a/node2-config/macros.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<clickhouse>
-    <macros>
-        <shard>01</shard>
-        <replica>02</replica>
-        <cluster>cluster_1S_2R</cluster>
-    </macros>
-</clickhouse>
\ No newline at end of file
diff --git a/node2-config/remote-servers.xml b/node2-config/remote-servers.xml
deleted file mode 100644
index a6a9edd..0000000
--- a/node2-config/remote-servers.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-<clickhouse>
-    <remote_servers replace="true">
-        <cluster_1S_2R>
-            <secret>mysecretphrase</secret>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>clickhouse-server1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>clickhouse-server2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </cluster_1S_2R>
-    </remote_servers>
-</clickhouse>
\ No newline at end of file
diff --git a/node2-config/use-keeper.xml b/node2-config/use-keeper.xml
deleted file mode 100644
index 2b384dc..0000000
--- a/node2-config/use-keeper.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-<clickhouse>
-    <zookeeper>
-        <!-- where are the ZK nodes -->
-        <node>
-            <host>clickhouse-keeper1</host>
-            <port>9181</port>
-        </node>
-        <node>
-            <host>clickhouse-keeper2</host>
-            <port>9181</port>
-        </node>
-        <node>
-            <host>clickhouse-keeper3</host>
-            <port>9181</port>
-        </node>
-    </zookeeper>
-</clickhouse>
\ No newline at end of file