Merge branch 'clickhouse'

clickhouse/Queries.sql

@@ -0,0 +1,22 @@
+-- TABLE CREATION
+CREATE TABLE traffic_records ON cluster_1S_2R
+(
+    time_stamp DateTime64(3, 'Japan') CODEC(Delta, ZSTD),
+    protocol Enum('TCP' = 1, 'UDP' = 2),
+    from_IP IPv4,
+    to_IP IPv4,
+    port UInt16 CODEC(ZSTD),
+    INDEX port_idx port TYPE bloom_filter GRANULARITY 10
+) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/traffic_records', '{replica}')
+ORDER BY time_stamp
+SETTINGS storage_policy = 'hot_cold'
+TTL time_stamp + INTERVAL 5 DAY TO VOLUME 'cold_disk';
+
+CREATE TABLE ip_region_map ON cluster_1S_2R
+(
+    ip_range_start IPv4,
+    ip_range_end IPv4,
+    region String,
+    INDEX region_ind region TYPE bloom_filter
+) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/ip_region_map', '{replica}')
+ORDER BY ip_range_start;

clickhouse/config_update_scripts/Dockerfile

@@ -0,0 +1,9 @@
+FROM python
+
+WORKDIR /update_scripts
+
+COPY . /update_scripts/
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+CMD ["python3","update_trigger.py"]

clickhouse/config_update_scripts/requirements.txt

@@ -0,0 +1,8 @@
+subprocess
+json
+jinja2
+yaml
+re
+xml
+schedule
+time

clickhouse/config_update_scripts/update_compose.py

@@ -4,6 +4,7 @@ import subprocess
 import json
 import xml.etree.ElementTree as ET
 import os
+import re
 
 if __name__ == "__main__":
     
@@ -15,15 +16,11 @@ if __name__ == "__main__":
 
     all_services = [json.loads(s) for s in all_services]
     # extracting the name, removing the custom id from it and storing it in a list
-    all_service_names = [service['Names'].split('.')[0] for service in all_services]
+    all_service_names = [service['Names'].split('.')[0] for service in all_services if re.findall(r'clickhouse-server',service['Names'])]
-    # extracting only 'keeper1', 'server1'...
+    # extracting only 'server1','server2'...
     all_service_names = [ name.split('-')[-1] for name in all_service_names]
 
-    # removing all keeepers
+    curr_num_servers = int(sorted(all_service_names)[-1][-1])
-    all_service_names.remove('keeper1')
-    all_service_names.remove('keeper2')
-    all_service_names.remove('keeper3')
-    curr_num_servers = sorted(all_service_names)[-1][-1]
 
     replication_factor = 2
     curr_num_shards = curr_num_servers/replication_factor
@@ -31,6 +28,7 @@ if __name__ == "__main__":
     # new shard template that is gonna be added to remote servers file of each node
     new_shard_str = f'''
         <shard>
+                <weight>{curr_num_shards+1}</weight>
                 <internal_replication>true</internal_replication>
                 <replica>
                     <host>clickhouse-server{curr_num_servers+1}</host>
@@ -59,20 +57,26 @@ if __name__ == "__main__":
         output_path = f'../node{i}-config/remote-servers.xml' 
         curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False)
 
-    env = Environment(loader=FileSystemLoader('.'))
+    env = Environment(loader=FileSystemLoader('../jinja-templates'))
     service_template = env.get_template('service.yml.jinja')
+    volume_template = env.get_template('volume.yml.jinja')
 
     # loading existing docker-compose file
     with open('../docker-compose.yaml','r') as f:
         compose_f = yaml.safe_load(f)
     
     # rendering the new service
-    new_service1 = service_template.render(server_num=curr_num_servers+1)
+    new_service1 = yaml.safe_load(service_template.render(server_num=curr_num_servers+1))
-    new_service2 = service_template.render(server_num=curr_num_servers+2)
+    new_service2 = yaml.safe_load(service_template.render(server_num=curr_num_servers+2))
+
+    new_volume1 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+1))
+    new_volume2 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+2))
     
     # adding the new service to docker-compose
     compose_f['services'].update(new_service1)
     compose_f['services'].update(new_service2)
+    compose_f['volumes'].update(new_volume1)
+    compose_f['volumes'].update(new_volume2)
     
     if compose_f:
         with open('../docker-compose.yaml','w') as yamlfile:
@@ -81,13 +85,14 @@ if __name__ == "__main__":
     config_template = env.get_template('config.xml.jinja')
     macros_template = env.get_template('macros.xml.jinja')
     use_keeper_template = env.get_template('use-keeper.xml.jinja')
+    storage_policy_template = env.get_template('storage-policy.xml.jinja')
 
     for i in range(1,3):
         config_content = config_template.render(node_num=curr_num_servers+i)
         with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1:
             f1.write(config_content)
         
-        macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i)
+        macros_content = macros_template.render(shard_num="0"+str(int(curr_num_shards+1)),replica_num=i)
         with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2:
             f2.write(macros_content)
         
@@ -95,3 +100,7 @@ if __name__ == "__main__":
         with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3:
             f3.write(use_keeper_content)
 
+        storage_policy_content = storage_policy_template.render(server_num=curr_num_servers+i)
+        with open(f'../node{curr_num_servers + i}-config/storage-policy.xml','w') as f4:
+            f4.write(storage_policy_content)
+

clickhouse/config_update_scripts/update_trigger.py

@@ -0,0 +1,34 @@
+import subprocess
+import json
+import re
+import schedule
+import time
+
+def check_util_exec():
+    # extracting details of each running container in json format    
+    try:
+        all_services = subprocess.check_output(["docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
+    except subprocess.CalledProcessError as e:
+        print(f"Command failed with return code {e.returncode}")
+
+    all_services = [json.loads(s) for s in all_services]
+
+    resource_util_exceed_flag = True            # Flag to check if all of the containers have exceeded 80% memory utilization 
+    for service in all_services:
+        if re.findall(r'clickhouse-server',service['Name']):
+            if float(service['MemPerc'][:-1]) < 80:
+                resource_util_exceed_flag = False
+    
+    if resource_util_exceed_flag:
+        process = subprocess.Popen(['python3','update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+        stdout, stderr = process.communicate()  # Wait for the process to finish and capture output
+        print("Standard Output:", stdout)
+        print("Standard Error:", stderr)
+
+if __name__ == "__main__":
+    schedule.every(30).seconds.do(check_util_exec)
+    while True:
+        schedule.run_pending()
+        time.sleep(1)
+
+        

clickhouse/docker-compose-old.yaml

@@ -2,14 +2,10 @@ services:
   clickhouse-keeper1:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-keeper1
-    command: >
+    command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
       - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - clickhouse_keeper1_data:/var/lib/clickhouse
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
 
     networks:
       clickhouse-keeper-network:
@@ -19,15 +15,10 @@ services:
   clickhouse-keeper2:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-keeper2
-    command: >
+    command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
       - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - clickhouse_keeper2_data:/var/lib/clickhouse
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-
     networks:
       clickhouse-keeper-network:
         aliases:
@@ -36,15 +27,10 @@ services:
   clickhouse-keeper3:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-keeper3
-    command: >
+    command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
-      /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
     volumes:
       - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
-      - ./clickhouse_data/data:/var/lib/clickhouse/data
+      - clickhouse_keeper3_data:/var/lib/clickhouse
-      - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
-      - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
-      - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
-
     networks:
       clickhouse-keeper-network:
         aliases:
@@ -55,7 +41,8 @@ services:
     container_name: clickhouse-server1
     volumes:
       - ./node1-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data1:/var/lib/clickhouse
+      - clickhouse_server1_data:/var/lib/clickhouse
+      - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:
       clickhouse-server-network:
         aliases:
@@ -69,10 +56,10 @@ services:
       #   constraints: [node.labels.role == server]
       update_config:
         delay: 10s
-      # resources:
+      resources:
-      #   limits:
+        limits:
-      #     cpus: "0.50"
+          cpus: "0.50"
-      #     memory: 100M
+          memory: 1200M
     depends_on:
       - clickhouse-keeper1
       - clickhouse-keeper2
@@ -86,7 +73,8 @@ services:
     container_name: clickhouse-server2
     volumes:
       - ./node2-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data2:/var/lib/clickhouse
+      - clickhouse_server2_data:/var/lib/clickhouse
+      - clickhouse_server2_TTL:/clickhouse_data/server2
     networks:
       clickhouse-server-network:
         aliases:
@@ -100,10 +88,10 @@ services:
       #   constraints: [node.labels.role == server]
       update_config:
         delay: 10s
-      # resources:
+      resources:
-      #   limits:
+        limits:
-      #     cpus: "0.50"
+          cpus: "0.50"
-      #     memory: 100M
+          memory: 1200M
     depends_on:
       - clickhouse-keeper1
       - clickhouse-keeper2
@@ -115,11 +103,23 @@ services:
 networks:
   clickhouse-server-network:
     driver: overlay
+    attachable: true
   clickhouse-keeper-network:
     driver: overlay
+    attachable: true
 
 volumes:
-  clickhouse_data1:
+  clickhouse_server1_data:
     driver: local
-  clickhouse_data2:
+  clickhouse_server2_data:
+    driver: local
+  clickhouse_keeper1_data:
+    driver: local
+  clickhouse_keeper2_data:
+    driver: local
+  clickhouse_keeper3_data:
+    driver: local
+  clickhouse_server1_TTL:
+    driver: local
+  clickhouse_server2_TTL:
     driver: local

clickhouse/jinja-templates/config.xml.jinja

@@ -20,5 +20,6 @@
     <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
     <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
     <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
     <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
 </clickhouse>

clickhouse/jinja-templates/remote-servers.xml.jinja

@@ -1,18 +0,0 @@
-<clickhouse>
-    <remote_servers replace="true">
-        <cluster_1S_2R>
-            <secret>mysecretphrase</secret>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>clickhouse-server1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>clickhouse-server2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </cluster_1S_2R>
-    </remote_servers>
-</clickhouse>

clickhouse/jinja-templates/service.yml.jinja

@@ -3,7 +3,7 @@ clickhouse-server{{server_num}}:
     container_name: clickhouse-server{{server_num}}
     volumes:
       - ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/
-      - clickhouse_data{{server_num}}:/var/lib/clickhouse
+      - clickhouse_server{{server_num}}_data:/var/lib/clickhouse
     networks:
       clickhouse-server-network:
         aliases:
@@ -17,10 +17,10 @@ clickhouse-server{{server_num}}:
       #   constraints: [node.labels.role == server]
       update_config:
         delay: 10s
-      # resources:
+      resources:
-      #   limits:
+        limits:
-      #     cpus: "0.50"
+          cpus: "0.50"
-      #     memory: 100M
+          memory: 100M
     depends_on:
       - clickhouse-keeper1
       - clickhouse-keeper2

clickhouse/jinja-templates/storage-policy.xml.jinja

@@ -0,0 +1,25 @@
+<storage_configuration>
+    <disks>
+        <hot_disk> 
+            <path>/clickhouse_data{{server_num}}/hot</path>
+            <keep_free_space_bytes>300000000</keep_free_space_bytes>
+        </hot_disk>
+        <cold_disk> 
+            <path>/clickhouse_data{{server_num}}/cold</path>
+            <keep_free_space_bytes>500000000</keep_free_space_bytes>
+        </cold_disk>
+    </disks>
+    <policies>
+        <hot_cold>
+            <volumes>
+                <hot_vol>
+                    <disk>hot_disk</disk>
+                </hot_vol>
+                <volume_name_2>
+                    <disk>cold_disk</disk>
+                </volume_name_2>
+            </volumes>
+            <move_factor>0.2</move_factor>
+        </hot_cold>
+    </policies>
+</storage_configuration>

clickhouse/jinja-templates/volume.yml.jinja

@@ -0,0 +1,4 @@
+clickhouse_server{{server_num}}_data:
+    driver: local
+clickhouse_server{{server_num}}_TTL:
+    driver: local

clickhouse/node1-config/config.xml

@@ -20,5 +20,6 @@
     <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
     <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
     <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
     <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
 </clickhouse>

clickhouse/node1-config/remote-servers.xml

@@ -3,6 +3,7 @@
         <cluster_1S_2R>
             <secret>mysecretphrase</secret>
             <shard>
+                <weight>1</weight>
                 <internal_replication>true</internal_replication>
                 <replica>
                     <host>clickhouse-server1</host>

clickhouse/node1-config/storage-policy.xml

@@ -0,0 +1,27 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <hot_disk> 
+                <path>/clickhouse_data/server1/hot/</path>
+            </hot_disk>
+            <cold_disk> 
+                <path>/clickhouse_data/server1/cold/</path>
+            </cold_disk>
+        </disks>
+        <policies>
+            <hot_cold>
+                <volumes>
+                    <hot_vol>
+                        <disk>hot_disk</disk>
+                        <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                    </hot_vol>
+                    <cold_vol>
+                        <disk>cold_disk</disk>
+                        <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                    </cold_vol>
+                </volumes>
+                <move_factor>0.2</move_factor>
+            </hot_cold>
+        </policies>
+    </storage_configuration>
+</clickhouse>

clickhouse/node2-config/config.xml

@@ -20,5 +20,6 @@
     <include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
     <include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
     <include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
+    <include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
     <!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
 </clickhouse>

clickhouse/node2-config/remote-servers.xml

@@ -3,6 +3,7 @@
         <cluster_1S_2R>
             <secret>mysecretphrase</secret>
             <shard>
+                <weight>1</weight>
                 <internal_replication>true</internal_replication>
                 <replica>
                     <host>clickhouse-server1</host>

clickhouse/node2-config/storage-policy.xml

@@ -0,0 +1,27 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <hot_disk> 
+                <path>/clickhouse_data/server2/hot/</path>
+            </hot_disk>
+            <cold_disk> 
+                <path>/clickhouse_data/server2/cold/</path>
+            </cold_disk>
+        </disks>
+        <policies>
+            <hot_cold>
+                <volumes>
+                    <hot_vol>
+                        <disk>hot_disk</disk>
+                        <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                    </hot_vol>
+                    <cold_vol>
+                        <disk>cold_disk</disk>
+                        <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                    </cold_vol>
+                </volumes>
+                <move_factor>0.2</move_factor>
+            </hot_cold>
+        </policies>
+    </storage_configuration>
+</clickhouse>