ttl tested

clickhouse/config_update_scripts/requirements.txt

@@ -1,8 +1,3 @@
-subprocess
-json
 jinja2
-yaml
-re
-xml
+pyyaml
 schedule
-time

clickhouse/docker-compose.yaml

@@ -60,7 +60,7 @@ services:
     volumes:
       - ../clickhouse/node1-config/:/etc/clickhouse-server/config.d/
       - ../clickhouse/node-entrypoints/main:/docker-entrypoint-initdb.d
-      - ../preprocessing/geoip.csv:/tmp/seedData/csv/ip_region_map.csv
+      - ../preprocessing/geoip.csv:/var/lib/clickhouse/user_files/csv/ip_region_map.csv
       - clickhouse_server1_data:/var/lib/clickhouse
       - clickhouse_server1_TTL:/clickhouse_data/server1
     networks:

clickhouse/jinja-templates/service.yml.jinja

@@ -2,8 +2,10 @@ clickhouse-server{{server_num}}:
     image: clickhouse/clickhouse-server:latest
     container_name: clickhouse-server{{server_num}}
     volumes:
-      - ./node{{server_num}}-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node{{server_num}}-config/:/etc/clickhouse-server/config.d/
+      - ../clickhouse/node-entrypoints/common:/docker-entrypoint-initdb.d
       - clickhouse_server{{server_num}}_data:/var/lib/clickhouse
+      - clickhouse_server{{server_num}}_TTL:/clickhouse_data/server{{server_num}}
     networks:
       clickhouse-server-network:
         aliases:
@@ -15,6 +17,8 @@ clickhouse-server{{server_num}}:
       replicas: 1
       # placement:
       #   constraints: [node.labels.role == server]
+      restart_policy:
+          condition: on-failure
       update_config:
         delay: 10s
       resources:
@@ -26,5 +30,5 @@ clickhouse-server{{server_num}}:
       - clickhouse-keeper2
       - clickhouse-keeper3
     ports:
-      - "900{{server_num}}:9000"  # Native client port
-      - "8123:8123"  # HTTP interface
+      - "{{9000+server_num}}:9000"  # Native client port
+      - "{{8123+server_num}}:8123"  # HTTP interface

clickhouse/jinja-templates/storage-policy.xml.jinja

@@ -1,25 +1,27 @@
-<storage_configuration>
-    <disks>
-        <hot_disk> 
-            <path>/clickhouse_data{{server_num}}/hot</path>
-            <keep_free_space_bytes>300000000</keep_free_space_bytes>
-        </hot_disk>
-        <cold_disk> 
-            <path>/clickhouse_data{{server_num}}/cold</path>
-            <keep_free_space_bytes>500000000</keep_free_space_bytes>
-        </cold_disk>
-    </disks>
-    <policies>
-        <hot_cold>
-            <volumes>
-                <hot_vol>
-                    <disk>hot_disk</disk>
-                </hot_vol>
-                <volume_name_2>
-                    <disk>cold_disk</disk>
-                </volume_name_2>
-            </volumes>
-            <move_factor>0.2</move_factor>
-        </hot_cold>
-    </policies>
-</storage_configuration>
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <hot_disk> 
+                <path>/clickhouse_data/server{{server_num}}/hot</path>
+            </hot_disk>
+            <cold_disk> 
+                <path>/clickhouse_data/server{{server_num}}/cold</path>
+            </cold_disk>
+        </disks>
+        <policies>
+            <hot_cold>
+                <volumes>
+                    <hot_vol>
+                        <disk>hot_disk</disk>
+                        <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                    </hot_vol>
+                    <cold_vol>
+                        <disk>cold_disk</disk>
+                        <max_data_part_size_bytes>1073741824</max_data_part_size_bytes>
+                    </cold_vol>
+                </volumes>
+                <move_factor>0.2</move_factor>
+            </hot_cold>
+        </policies>
+    </storage_configuration>
+</clickhouse>

clickhouse/node-entrypoints/common/01_table_create.sql

@@ -13,7 +13,7 @@ CREATE TABLE traffic_records (
 	'{replica}'
 )
 ORDER BY time_stamp
-TTL toDateTime(time_stamp) + INTERVAL 15 DAY TO VOLUME 'cold_vol'
+TTL toDateTime(time_stamp) + INTERVAL 410 DAY TO VOLUME 'cold_vol' -- october 15
 SETTINGS storage_policy = 'hot_cold';
 
 CREATE TABLE ip_region_map (

clickhouse/node-entrypoints/main/01_table_create.sql

@@ -13,7 +13,7 @@ CREATE TABLE traffic_records (
 	'{replica}'
 )
 ORDER BY time_stamp
-TTL toDateTime(time_stamp) + INTERVAL 15 DAY TO VOLUME 'cold_vol'
+TTL toDateTime(time_stamp) + INTERVAL 410 DAY TO VOLUME 'cold_vol' -- october 15
 SETTINGS storage_policy = 'hot_cold';
 
 CREATE TABLE ip_region_map (

preprocessing/docker-compose.yml

@@ -57,7 +57,7 @@ services:
           - data-streamer
     volumes:
       - "../preprocessing/10k_sample_2023_10_01-2023_10_31.csv:/data/csv/main.csv:ro"
-    command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -x --stream_size 100000 -l 0.1'"
+    command: "sh -c 'sleep 30 && python /app/pcap_processor.py -c /data/csv/main.csv -l 0.1'"
     deploy:
       replicas: 1
       # placement: