From 90a7a2415dccaa74d549141b19b64a316e4b1429 Mon Sep 17 00:00:00 2001
From: Kaushik Narayan R <rknarayan02@gmail.com>
Date: Thu, 14 Nov 2024 22:33:10 -0700
Subject: [PATCH] reorg

---
 clickhouse/README                             | 13 +++
 .../data/preprocessed_configs/config.xml      | 61 ++++++++++++
 .../clickhouse_keeper/keeper1-config.xml      | 44 +++++++++
 .../clickhouse_keeper/keeper2-config.xml      | 44 +++++++++
 .../clickhouse_keeper/keeper3-config.xml      | 44 +++++++++
 clickhouse/docker-compose.yaml                | 97 +++++++++++++++++++
 clickhouse/node1-config/config.xml            | 24 +++++
 clickhouse/node1-config/macros.xml            |  7 ++
 clickhouse/node1-config/remote-servers.xml    | 18 ++++
 clickhouse/node1-config/use-keeper.xml        | 17 ++++
 clickhouse/node2-config/config.xml            | 24 +++++
 clickhouse/node2-config/macros.xml            |  7 ++
 clickhouse/node2-config/remote-servers.xml    | 18 ++++
 clickhouse/node2-config/use-keeper.xml        | 17 ++++
 14 files changed, 435 insertions(+)
 create mode 100644 clickhouse/README
 create mode 100644 clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
 create mode 100644 clickhouse/clickhouse_keeper/keeper1-config.xml
 create mode 100644 clickhouse/clickhouse_keeper/keeper2-config.xml
 create mode 100644 clickhouse/clickhouse_keeper/keeper3-config.xml
 create mode 100644 clickhouse/docker-compose.yaml
 create mode 100644 clickhouse/node1-config/config.xml
 create mode 100644 clickhouse/node1-config/macros.xml
 create mode 100644 clickhouse/node1-config/remote-servers.xml
 create mode 100644 clickhouse/node1-config/use-keeper.xml
 create mode 100644 clickhouse/node2-config/config.xml
 create mode 100644 clickhouse/node2-config/macros.xml
 create mode 100644 clickhouse/node2-config/remote-servers.xml
 create mode 100644 clickhouse/node2-config/use-keeper.xml

diff --git a/clickhouse/README b/clickhouse/README
new file mode 100644
index 0000000..af20090
--- /dev/null
+++ b/clickhouse/README
@@ -0,0 +1,13 @@
+Execution steps:
+
+ensure to pull clickhouse docker image
+> docker-compose up -d
+identify the custom networks created.
+> docker network ls
+inspect the keeper network and verify whether all the containers are connected to it
+> docker network inspect {{dds_proj_clickhouse-keeper-network}}
+if all the containers are not connected
+> docker-compose restart
+To execute queries
+> docker exec -it clickhouse-server1 clickhouse-client
+> docker exec -it clickhouse-server2 clickhouse-client
\ No newline at end of file
diff --git a/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml b/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
new file mode 100644
index 0000000..08de94b
--- /dev/null
+++ b/clickhouse/clickhouse_data/data/preprocessed_configs/config.xml
@@ -0,0 +1,61 @@
+<!-- This file was generated automatically.
+     Do not edit it: it is likely to be discarded and generated again before it's read next time.
+     Files used to generate this file:
+       /etc/clickhouse-server/config.xml
+       /etc/clickhouse-server/config.d/docker_related_config.xml      -->
+
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>::</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>2</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+     <!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
+    
+    <listen_host>0.0.0.0</listen_host>
+    <listen_try>1</listen_try>
+
+    <!--
+    <logger>
+        <console>1</console>
+    </logger>
+    -->
+</yandex>
diff --git a/clickhouse/clickhouse_keeper/keeper1-config.xml b/clickhouse/clickhouse_keeper/keeper1-config.xml
new file mode 100644
index 0000000..c5e5d4d
--- /dev/null
+++ b/clickhouse/clickhouse_keeper/keeper1-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse/clickhouse_keeper/keeper2-config.xml b/clickhouse/clickhouse_keeper/keeper2-config.xml
new file mode 100644
index 0000000..bd2914e
--- /dev/null
+++ b/clickhouse/clickhouse_keeper/keeper2-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>2</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse/clickhouse_keeper/keeper3-config.xml b/clickhouse/clickhouse_keeper/keeper3-config.xml
new file mode 100644
index 0000000..383f9d9
--- /dev/null
+++ b/clickhouse/clickhouse_keeper/keeper3-config.xml
@@ -0,0 +1,44 @@
+<yandex>
+    <logger>
+        <level>trace</level>
+        <log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
+        <errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <listen_host>0.0.0.0</listen_host>
+    
+    <path>/var/lib/clickhouse/data/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
+
+    <keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>3</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <operation_timeout_ms>10000</operation_timeout_ms>
+            <session_timeout_ms>30000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>clickhouse-keeper1</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>clickhouse-keeper2</hostname>
+                <port>9234</port>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>clickhouse-keeper3</hostname>
+                <port>9234</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+</yandex>
diff --git a/clickhouse/docker-compose.yaml b/clickhouse/docker-compose.yaml
new file mode 100644
index 0000000..7841015
--- /dev/null
+++ b/clickhouse/docker-compose.yaml
@@ -0,0 +1,97 @@
+version: '3.8'
+
+services:
+  clickhouse-keeper1:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper1
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper1
+
+  clickhouse-keeper2:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper2
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper2
+
+  clickhouse-keeper3:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-keeper3
+    command: >
+      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
+    volumes:
+      - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
+      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
+      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
+      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
+
+    networks:
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-keeper3
+
+  clickhouse-server1:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server1
+    volumes:
+      - ./node1-config/:/etc/clickhouse-server/config.d/
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server1
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server1
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "9001:9000"  # Native client port
+      - "8123:8123"  # HTTP interface
+
+  clickhouse-server2:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse-server2
+    volumes:
+      - ./node2-config/:/etc/clickhouse-server/config.d/
+    networks:
+      clickhouse-server-network:
+        aliases:
+          - clickhouse-server2
+      clickhouse-keeper-network:
+        aliases:
+          - clickhouse-server2
+    depends_on:
+      - clickhouse-keeper1
+      - clickhouse-keeper2
+      - clickhouse-keeper3
+    ports:
+      - "9002:9000"  # Native client port
+      - "8124:8123"  # HTTP interface
+
+networks:
+  clickhouse-server-network:
+  clickhouse-keeper-network:
diff --git a/clickhouse/node1-config/config.xml b/clickhouse/node1-config/config.xml
new file mode 100644
index 0000000..4ced53d
--- /dev/null
+++ b/clickhouse/node1-config/config.xml
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node 1</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node1-config/macros.xml b/clickhouse/node1-config/macros.xml
new file mode 100644
index 0000000..e8f48c6
--- /dev/null
+++ b/clickhouse/node1-config/macros.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>01</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node1-config/remote-servers.xml b/clickhouse/node1-config/remote-servers.xml
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/clickhouse/node1-config/remote-servers.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node1-config/use-keeper.xml b/clickhouse/node1-config/use-keeper.xml
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/clickhouse/node1-config/use-keeper.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/config.xml b/clickhouse/node2-config/config.xml
new file mode 100644
index 0000000..68d5b06
--- /dev/null
+++ b/clickhouse/node2-config/config.xml
@@ -0,0 +1,24 @@
+<clickhouse>
+    <logger>
+        <level>debug</level>
+        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
+        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
+        <size>1000M</size>
+        <count>3</count>
+    </logger>
+    <display_name>cluster_1S_2R node 2</display_name>
+    <listen_host>0.0.0.0</listen_host>
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+    <!-- Maximum connections and settings -->
+    <max_connections>4096</max_connections>
+    <keep_alive_timeout>3</keep_alive_timeout>
+    <max_concurrent_queries>100</max_concurrent_queries>
+
+
+    <!-- Additional configuration files can be included -->
+    <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/macros.xml b/clickhouse/node2-config/macros.xml
new file mode 100644
index 0000000..52257ed
--- /dev/null
+++ b/clickhouse/node2-config/macros.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <macros>
+        <shard>01</shard>
+        <replica>02</replica>
+        <cluster>cluster_1S_2R</cluster>
+    </macros>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/remote-servers.xml b/clickhouse/node2-config/remote-servers.xml
new file mode 100644
index 0000000..a6a9edd
--- /dev/null
+++ b/clickhouse/node2-config/remote-servers.xml
@@ -0,0 +1,18 @@
+<clickhouse>
+    <remote_servers replace="true">
+        <cluster_1S_2R>
+            <secret>mysecretphrase</secret>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse-server1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse-server2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </cluster_1S_2R>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/clickhouse/node2-config/use-keeper.xml b/clickhouse/node2-config/use-keeper.xml
new file mode 100644
index 0000000..2b384dc
--- /dev/null
+++ b/clickhouse/node2-config/use-keeper.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <zookeeper>
+        <!-- where are the ZK nodes -->
+        <node>
+            <host>clickhouse-keeper1</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper2</host>
+            <port>9181</port>
+        </node>
+        <node>
+            <host>clickhouse-keeper3</host>
+            <port>9181</port>
+        </node>
+    </zookeeper>
+</clickhouse>
\ No newline at end of file