my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 06:44:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added storage policy for TTL in each server's config files and jinja templates. Testing needed

Browse Source
This commit is contained in:
Lalit Arvind 2024-11-24 15:21:35 -07:00
parent 52fc788eee
commit 745870bb9c
7 changed files with 83 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
clickhouse/config_update_scripts/update_compose.py
View File

@ -79,6 +79,7 @@ if __name__ == "__main__":
config_template = env.get_template('config.xml.jinja')
macros_template = env.get_template('macros.xml.jinja')
use_keeper_template = env.get_template('use-keeper.xml.jinja')
storage_policy_template = env.get_template('storage-policy.xml.jinja')
for i in range(1,3):
config_content = config_template.render(node_num=curr_num_servers+i)
@ -93,3 +94,7 @@ if __name__ == "__main__":
with open(f'../node{curr_num_servers + i}-config/use-keeper.xml','w') as f3:
f3.write(use_keeper_content)
storage_policy_content = storage_policy_template.render(server_num=curr_num_servers+i)
with open(f'../node{curr_num_servers + i}-config/storage-policy.xml','w') as f4:
f4.write(storage_policy_content)

1
clickhouse/jinja-templates/config.xml.jinja
View File

@ -20,5 +20,6 @@
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

25
clickhouse/jinja-templates/storage-policy.xml.jinja Normal file
View File

@ -0,0 +1,25 @@
<storage_configuration>
<disks>
<hot_disk>
<path>../clickhouse_data{{server_num}}/hot</path>
<keep_free_space_bytes>300000000</keep_free_space_bytes>
</hot_disk>
<cold_disk>
<path>../clickhouse_data{{server_num}}/cold</path>
<keep_free_space_bytes>500000000</keep_free_space_bytes>
</cold_disk>
</disks>
<policies>
<hot_cold>
<volumes>
<hot_vol>
<disk>hot_disk</disk>
</hot_vol>
<volume_name_2>
<disk>cold_disk</disk>
</volume_name_2>
</volumes>
<move_factor>0.2</move_factor>
</hot_cold>
</policies>
</storage_configuration>

1
clickhouse/node1-config/config.xml
View File

@ -20,5 +20,6 @@
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

25
clickhouse/node1-config/storage-policy.xml Normal file
View File

@ -0,0 +1,25 @@
<storage_configuration>
<disks>
<hot_disk>
<path>../clickhouse_data1/hot</path>
<keep_free_space_bytes>300000000</keep_free_space_bytes>
</hot_disk>
<cold_disk>
<path>../clickhouse_data1/cold</path>
<keep_free_space_bytes>500000000</keep_free_space_bytes>
</cold_disk>
</disks>
<policies>
<hot_cold>
<volumes>
<hot_vol>
<disk>hot_disk</disk>
</hot_vol>
<volume_name_2>
<disk>cold_disk</disk>
</volume_name_2>
</volumes>
<move_factor>0.2</move_factor>
</hot_cold>
</policies>
</storage_configuration>

1
clickhouse/node2-config/config.xml
View File

@ -20,5 +20,6 @@
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/storage-policy.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

25
clickhouse/node2-config/storage-policy.xml Normal file
View File

@ -0,0 +1,25 @@
<storage_configuration>
<disks>
<hot_disk>
<path>../clickhouse_data2/hot</path>
<keep_free_space_bytes>300000000</keep_free_space_bytes>
</hot_disk>
<cold_disk>
<path>../clickhouse_data2/cold</path>
<keep_free_space_bytes>500000000</keep_free_space_bytes>
</cold_disk>
</disks>
<policies>
<hot_cold>
<volumes>
<hot_vol>
<disk>hot_disk</disk>
</hot_vol>
<volume_name_2>
<disk>cold_disk</disk>
</volume_name_2>
</volumes>
<move_factor>0.2</move_factor>
</hot_cold>
</policies>
</storage_configuration>