diff --git a/clickhouse/Queries.sql b/clickhouse/Queries.sql new file mode 100644 index 0000000..111c82e --- /dev/null +++ b/clickhouse/Queries.sql @@ -0,0 +1,22 @@ +-- TABLE CREATION +CREATE TABLE traffic_records ON cluster_1S_2R +( + time_stamp DateTime64(3, 'Japan') CODEC(Delta, ZSTD), + protocol Enum('TCP' = 1, 'UDP' = 2), + from_IP IPv4, + to_IP IPv4, + port UInt16 CODEC(ZSTD), + INDEX port_idx port TYPE bloom_filter GRANULARITY 10 +) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/traffic_records', '{replica}') +ORDER BY time_stamp +SETTINGS storage_policy = 'hot_cold' +TTL time_stamp + INTERVAL 5 DAY TO VOLUME 'cold_disk'; + +CREATE TABLE ip_region_map ON cluster_1S_2R +( + ip_range_start IPv4, + ip_range_end IPv4, + region String, + INDEX region_ind region TYPE bloom_filter +) ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/ip_region_map', '{replica}') +ORDER BY ip_range_start; \ No newline at end of file diff --git a/clickhouse/config_update_scripts/Dockerfile b/clickhouse/config_update_scripts/Dockerfile new file mode 100644 index 0000000..79e3995 --- /dev/null +++ b/clickhouse/config_update_scripts/Dockerfile @@ -0,0 +1,9 @@ +FROM python + +WORKDIR /update_scripts + +COPY . /update_scripts/ + +RUN pip install --no-cache-dir -r requirements.txt + +CMD ["python3","update_trigger.py"] \ No newline at end of file diff --git a/clickhouse/config_update_scripts/requirements.txt b/clickhouse/config_update_scripts/requirements.txt new file mode 100644 index 0000000..711648d --- /dev/null +++ b/clickhouse/config_update_scripts/requirements.txt @@ -0,0 +1,8 @@ +subprocess +json +jinja2 +yaml +re +xml +schedule +time \ No newline at end of file diff --git a/clickhouse/config_update_scripts/update_compose.py b/clickhouse/config_update_scripts/update_compose.py index 9f29a20..66440e2 100644 --- a/clickhouse/config_update_scripts/update_compose.py +++ b/clickhouse/config_update_scripts/update_compose.py @@ -4,6 +4,7 @@ import subprocess import json import xml.etree.ElementTree as ET import os +import re if __name__ == "__main__": @@ -15,15 +16,11 @@ if __name__ == "__main__": all_services = [json.loads(s) for s in all_services] # extracting the name, removing the custom id from it and storing it in a list - all_service_names = [service['Names'].split('.')[0] for service in all_services] - # extracting only 'keeper1', 'server1'... + all_service_names = [service['Names'].split('.')[0] for service in all_services if re.findall(r'clickhouse-server',service['Names'])] + # extracting only 'server1','server2'... all_service_names = [ name.split('-')[-1] for name in all_service_names] - # removing all keeepers - all_service_names.remove('keeper1') - all_service_names.remove('keeper2') - all_service_names.remove('keeper3') - curr_num_servers = sorted(all_service_names)[-1][-1] + curr_num_servers = int(sorted(all_service_names)[-1][-1]) replication_factor = 2 curr_num_shards = curr_num_servers/replication_factor @@ -31,6 +28,7 @@ if __name__ == "__main__": # new shard template that is gonna be added to remote servers file of each node new_shard_str = f''' + {curr_num_shards+1} true clickhouse-server{curr_num_servers+1} @@ -59,21 +57,27 @@ if __name__ == "__main__": output_path = f'../node{i}-config/remote-servers.xml' curr_remote_servers_xml.write(output_path, encoding='utf-8', xml_declaration=False) - env = Environment(loader=FileSystemLoader('.')) + env = Environment(loader=FileSystemLoader('../jinja-templates')) service_template = env.get_template('service.yml.jinja') + volume_template = env.get_template('volume.yml.jinja') # loading existing docker-compose file with open('../docker-compose.yaml','r') as f: compose_f = yaml.safe_load(f) # rendering the new service - new_service1 = service_template.render(server_num=curr_num_servers+1) - new_service2 = service_template.render(server_num=curr_num_servers+2) + new_service1 = yaml.safe_load(service_template.render(server_num=curr_num_servers+1)) + new_service2 = yaml.safe_load(service_template.render(server_num=curr_num_servers+2)) + + new_volume1 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+1)) + new_volume2 = yaml.safe_load(volume_template.render(server_num=curr_num_servers+2)) # adding the new service to docker-compose compose_f['services'].update(new_service1) compose_f['services'].update(new_service2) - + compose_f['volumes'].update(new_volume1) + compose_f['volumes'].update(new_volume2) + if compose_f: with open('../docker-compose.yaml','w') as yamlfile: yaml.safe_dump(compose_f, yamlfile) @@ -81,13 +85,14 @@ if __name__ == "__main__": config_template = env.get_template('config.xml.jinja') macros_template = env.get_template('macros.xml.jinja') use_keeper_template = env.get_template('use-keeper.xml.jinja') + storage_policy_template = env.get_template('storage-policy.xml.jinja') for i in range(1,3): config_content = config_template.render(node_num=curr_num_servers+i) with open(f'../node{curr_num_servers + i}-config/config.xml','w') as f1: f1.write(config_content) - macros_content = macros_template.render(shard_num="0{curr_num_shards}",replica_num=i) + macros_content = macros_template.render(shard_num="0"+str(int(curr_num_shards+1)),replica_num=i) with open(f'../node{curr_num_servers + i}-config/macros.xml','w') as f2: f2.write(macros_content) @@ -95,3 +100,7 @@ if __name__ == "__main__": with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3: f3.write(use_keeper_content) + storage_policy_content = storage_policy_template.render(server_num=curr_num_servers+i) + with open(f'../node{curr_num_servers + i}-config/storage-policy.xml','w') as f4: + f4.write(storage_policy_content) + diff --git a/clickhouse/config_update_scripts/update_trigger.py b/clickhouse/config_update_scripts/update_trigger.py new file mode 100644 index 0000000..fe154f5 --- /dev/null +++ b/clickhouse/config_update_scripts/update_trigger.py @@ -0,0 +1,34 @@ +import subprocess +import json +import re +import schedule +import time + +def check_util_exec(): + # extracting details of each running container in json format + try: + all_services = subprocess.check_output(["docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1] + except subprocess.CalledProcessError as e: + print(f"Command failed with return code {e.returncode}") + + all_services = [json.loads(s) for s in all_services] + + resource_util_exceed_flag = True # Flag to check if all of the containers have exceeded 80% memory utilization + for service in all_services: + if re.findall(r'clickhouse-server',service['Name']): + if float(service['MemPerc'][:-1]) < 80: + resource_util_exceed_flag = False + + if resource_util_exceed_flag: + process = subprocess.Popen(['python3','update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + stdout, stderr = process.communicate() # Wait for the process to finish and capture output + print("Standard Output:", stdout) + print("Standard Error:", stderr) + +if __name__ == "__main__": + schedule.every(30).seconds.do(check_util_exec) + while True: + schedule.run_pending() + time.sleep(1) + + \ No newline at end of file diff --git a/clickhouse/docker-compose-old.yaml b/clickhouse/docker-compose-old.yaml index 7ee9aea..f7c95bc 100644 --- a/clickhouse/docker-compose-old.yaml +++ b/clickhouse/docker-compose-old.yaml @@ -2,14 +2,10 @@ services: clickhouse-keeper1: image: clickhouse/clickhouse-server:latest container_name: clickhouse-keeper1 - command: > - /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml + command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml volumes: - ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml - - ./clickhouse_data/data:/var/lib/clickhouse/data - - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp - - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files - - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas + - clickhouse_keeper1_data:/var/lib/clickhouse networks: clickhouse-keeper-network: @@ -19,15 +15,10 @@ services: clickhouse-keeper2: image: clickhouse/clickhouse-server:latest container_name: clickhouse-keeper2 - command: > - /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml + command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml volumes: - ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml - - ./clickhouse_data/data:/var/lib/clickhouse/data - - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp - - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files - - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas - + - clickhouse_keeper2_data:/var/lib/clickhouse networks: clickhouse-keeper-network: aliases: @@ -36,15 +27,10 @@ services: clickhouse-keeper3: image: clickhouse/clickhouse-server:latest container_name: clickhouse-keeper3 - command: > - /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml + command: /usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml volumes: - ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml - - ./clickhouse_data/data:/var/lib/clickhouse/data - - ./clickhouse_data/tmp:/var/lib/clickhouse/tmp - - ./clickhouse_data/user_files:/var/lib/clickhouse/user_files - - ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas - + - clickhouse_keeper3_data:/var/lib/clickhouse networks: clickhouse-keeper-network: aliases: @@ -55,7 +41,8 @@ services: container_name: clickhouse-server1 volumes: - ./node1-config/:/etc/clickhouse-server/config.d/ - - clickhouse_data1:/var/lib/clickhouse + - clickhouse_server1_data:/var/lib/clickhouse + - clickhouse_server1_TTL:/clickhouse_data/server1 networks: clickhouse-server-network: aliases: @@ -69,10 +56,10 @@ services: # constraints: [node.labels.role == server] update_config: delay: 10s - # resources: - # limits: - # cpus: "0.50" - # memory: 100M + resources: + limits: + cpus: "0.50" + memory: 1200M depends_on: - clickhouse-keeper1 - clickhouse-keeper2 @@ -86,7 +73,8 @@ services: container_name: clickhouse-server2 volumes: - ./node2-config/:/etc/clickhouse-server/config.d/ - - clickhouse_data2:/var/lib/clickhouse + - clickhouse_server2_data:/var/lib/clickhouse + - clickhouse_server2_TTL:/clickhouse_data/server2 networks: clickhouse-server-network: aliases: @@ -100,10 +88,10 @@ services: # constraints: [node.labels.role == server] update_config: delay: 10s - # resources: - # limits: - # cpus: "0.50" - # memory: 100M + resources: + limits: + cpus: "0.50" + memory: 1200M depends_on: - clickhouse-keeper1 - clickhouse-keeper2 @@ -115,11 +103,23 @@ services: networks: clickhouse-server-network: driver: overlay + attachable: true clickhouse-keeper-network: driver: overlay + attachable: true volumes: - clickhouse_data1: + clickhouse_server1_data: driver: local - clickhouse_data2: + clickhouse_server2_data: + driver: local + clickhouse_keeper1_data: + driver: local + clickhouse_keeper2_data: + driver: local + clickhouse_keeper3_data: + driver: local + clickhouse_server1_TTL: + driver: local + clickhouse_server2_TTL: driver: local \ No newline at end of file diff --git a/clickhouse/jinja-templates/config.xml.jinja b/clickhouse/jinja-templates/config.xml.jinja index cd07efb..4a0f2ba 100644 --- a/clickhouse/jinja-templates/config.xml.jinja +++ b/clickhouse/jinja-templates/config.xml.jinja @@ -20,5 +20,6 @@ /etc/clickhouse-server/config.d/macros.xml /etc/clickhouse-server/config.d/remote-servers.xml /etc/clickhouse-server/config.d/use-keeper.xml + /etc/clickhouse-server/config.d/storage-policy.xml \ No newline at end of file diff --git a/clickhouse/jinja-templates/remote-servers.xml.jinja b/clickhouse/jinja-templates/remote-servers.xml.jinja deleted file mode 100644 index a6a9edd..0000000 --- a/clickhouse/jinja-templates/remote-servers.xml.jinja +++ /dev/null @@ -1,18 +0,0 @@ - - - - mysecretphrase - - true - - clickhouse-server1 - 9000 - - - clickhouse-server2 - 9000 - - - - - \ No newline at end of file diff --git a/clickhouse/jinja-templates/service.yml.jinja b/clickhouse/jinja-templates/service.yml.jinja index 1ae0414..1ee1631 100644 --- a/clickhouse/jinja-templates/service.yml.jinja +++ b/clickhouse/jinja-templates/service.yml.jinja @@ -3,7 +3,7 @@ clickhouse-server{{server_num}}: container_name: clickhouse-server{{server_num}} volumes: - ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/ - - clickhouse_data{{server_num}}:/var/lib/clickhouse + - clickhouse_server{{server_num}}_data:/var/lib/clickhouse networks: clickhouse-server-network: aliases: @@ -17,10 +17,10 @@ clickhouse-server{{server_num}}: # constraints: [node.labels.role == server] update_config: delay: 10s - # resources: - # limits: - # cpus: "0.50" - # memory: 100M + resources: + limits: + cpus: "0.50" + memory: 100M depends_on: - clickhouse-keeper1 - clickhouse-keeper2 diff --git a/clickhouse/jinja-templates/storage-policy.xml.jinja b/clickhouse/jinja-templates/storage-policy.xml.jinja new file mode 100644 index 0000000..440d7e7 --- /dev/null +++ b/clickhouse/jinja-templates/storage-policy.xml.jinja @@ -0,0 +1,25 @@ + + + + /clickhouse_data{{server_num}}/hot + 300000000 + + + /clickhouse_data{{server_num}}/cold + 500000000 + + + + + + + hot_disk + + + cold_disk + + + 0.2 + + + \ No newline at end of file diff --git a/clickhouse/jinja-templates/volume.yml.jinja b/clickhouse/jinja-templates/volume.yml.jinja new file mode 100644 index 0000000..e9324d0 --- /dev/null +++ b/clickhouse/jinja-templates/volume.yml.jinja @@ -0,0 +1,4 @@ +clickhouse_server{{server_num}}_data: + driver: local +clickhouse_server{{server_num}}_TTL: + driver: local \ No newline at end of file diff --git a/clickhouse/node1-config/config.xml b/clickhouse/node1-config/config.xml index 4ced53d..261fe8d 100644 --- a/clickhouse/node1-config/config.xml +++ b/clickhouse/node1-config/config.xml @@ -20,5 +20,6 @@ /etc/clickhouse-server/config.d/macros.xml /etc/clickhouse-server/config.d/remote-servers.xml /etc/clickhouse-server/config.d/use-keeper.xml + /etc/clickhouse-server/config.d/storage-policy.xml \ No newline at end of file diff --git a/clickhouse/node1-config/remote-servers.xml b/clickhouse/node1-config/remote-servers.xml index a6a9edd..e708fb7 100644 --- a/clickhouse/node1-config/remote-servers.xml +++ b/clickhouse/node1-config/remote-servers.xml @@ -3,6 +3,7 @@ mysecretphrase + 1 true clickhouse-server1 diff --git a/clickhouse/node1-config/storage-policy.xml b/clickhouse/node1-config/storage-policy.xml new file mode 100644 index 0000000..41054e4 --- /dev/null +++ b/clickhouse/node1-config/storage-policy.xml @@ -0,0 +1,27 @@ + + + + + /clickhouse_data/server1/hot/ + + + /clickhouse_data/server1/cold/ + + + + + + + hot_disk + 1073741824 + + + cold_disk + 1073741824 + + + 0.2 + + + + \ No newline at end of file diff --git a/clickhouse/node2-config/config.xml b/clickhouse/node2-config/config.xml index 68d5b06..f2928c0 100644 --- a/clickhouse/node2-config/config.xml +++ b/clickhouse/node2-config/config.xml @@ -20,5 +20,6 @@ /etc/clickhouse-server/config.d/macros.xml /etc/clickhouse-server/config.d/remote-servers.xml /etc/clickhouse-server/config.d/use-keeper.xml + /etc/clickhouse-server/config.d/storage-policy.xml \ No newline at end of file diff --git a/clickhouse/node2-config/remote-servers.xml b/clickhouse/node2-config/remote-servers.xml index a6a9edd..e708fb7 100644 --- a/clickhouse/node2-config/remote-servers.xml +++ b/clickhouse/node2-config/remote-servers.xml @@ -3,6 +3,7 @@ mysecretphrase + 1 true clickhouse-server1 diff --git a/clickhouse/node2-config/storage-policy.xml b/clickhouse/node2-config/storage-policy.xml new file mode 100644 index 0000000..0691d22 --- /dev/null +++ b/clickhouse/node2-config/storage-policy.xml @@ -0,0 +1,27 @@ + + + + + /clickhouse_data/server2/hot/ + + + /clickhouse_data/server2/cold/ + + + + + + + hot_disk + 1073741824 + + + cold_disk + 1073741824 + + + 0.2 + + + + \ No newline at end of file