From 7a79530633cab6331e15a89739fb65ae46d64ee5 Mon Sep 17 00:00:00 2001
From: Lalit Arvind <lalitarvind.balaji@gmail.com>
Date: Sat, 30 Nov 2024 21:12:38 -0700
Subject: [PATCH] error correction wrt autonomous sharding script

---
 clickhouse/config_update_scripts/update_compose.py |  1 +
 clickhouse/config_update_scripts/update_trigger.py | 11 +++++++++--
 scripts/deploy.sh                                  |  8 +++++---
 ui/dashboard_1.json                                |  4 ++--
 4 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/clickhouse/config_update_scripts/update_compose.py b/clickhouse/config_update_scripts/update_compose.py
index 96e4e85..020a29e 100644
--- a/clickhouse/config_update_scripts/update_compose.py
+++ b/clickhouse/config_update_scripts/update_compose.py
@@ -105,3 +105,4 @@ if __name__ == "__main__":
         with open(f'../clickhouse/node{curr_num_servers + i}-config/storage-policy.xml','w') as f4:
             f4.write(storage_policy_content)
 
+    print("Config Files Updated!")
\ No newline at end of file
diff --git a/clickhouse/config_update_scripts/update_trigger.py b/clickhouse/config_update_scripts/update_trigger.py
index 41706f5..4fe7001 100644
--- a/clickhouse/config_update_scripts/update_trigger.py
+++ b/clickhouse/config_update_scripts/update_trigger.py
@@ -5,6 +5,7 @@ import schedule
 import time
 
 def check_util_exec():
+    print("Performing check")
     # extracting details of each running container in json format    
     try:
         all_services = subprocess.check_output(["sudo", "docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
@@ -16,14 +17,20 @@ def check_util_exec():
     resource_util_exceed_flag = True # Flag to check if all of the containers have exceeded 80% memory utilization 
     for service in all_services:
         if re.findall(r'clickhouse-server',service['Name']):
-            if float(service['MemPerc'][:-1]) < 60:
+            if float(service['MemPerc'][:-1]) < 50:
                 resource_util_exceed_flag = False
     
     if resource_util_exceed_flag:
-        process = subprocess.Popen(['python3','../clickhouse/update_config_scripts/update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+        print("Config update triggered")
+        process = subprocess.Popen(['python3','../clickhouse/config_update_scripts/update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()  # Wait for the process to finish and capture output
         print("Standard Output:", stdout)
         print("Standard Error:", stderr)
+        if stdout:
+            redeploy  = subprocess.Popen(['docker','stack','deploy','-d','-c','../preprocessing/docker-compose.yml','-c','../clickhouse/docker-compose.yaml','-c','../ui/docker-compose.yaml','TheWebFarm'])
+            stdout1, stderr1= redeploy.communicate()  # Wait for the process to finish and capture output
+            print("Standard Output:", stdout1)
+            print("Standard Error:", stderr1)
         # try:
         #     all_services = subprocess.check_output(["sudo", "docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
         # except subprocess.CalledProcessError as e:
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index ef97e1d..07ccfce 100755
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -46,11 +46,13 @@ elif [[ $masterNode ]]; then
 		-c ../clickhouse/docker-compose.yaml \
 		-c ../ui/docker-compose.yaml \
 		$stackName
-elif [[ $autoShard ]]; then
-	cd $scriptDir
-	python3 $scriptDir/../clickhouse/config_update_scripts/update_trigger.py
 else
 	echo "[+] swarm follower"
 	echo "[+] joining swarm with token $swarmToken"
 	$dockerCmd swarm join --token $swarmToken $managerAddr
 fi
+
+if [[ $autoShard ]]; then
+	cd $scriptDir
+	python3 ../clickhouse/config_update_scripts/update_trigger.py
+fi
\ No newline at end of file
diff --git a/ui/dashboard_1.json b/ui/dashboard_1.json
index 446799d..26bff44 100644
--- a/ui/dashboard_1.json
+++ b/ui/dashboard_1.json
@@ -431,7 +431,7 @@
           "refId": "A"
         }
       ],
-      "title": "Top regions (packet count)",
+      "title": "Top regions (packet count in millions)",
       "type": "bargauge"
     },
     {
@@ -942,6 +942,6 @@
   "timezone": "browser",
   "title": "Internet traffic capture analysis",
   "uid": "be59fkbp3zs3kc",
-  "version": 4,
+  "version": 1,
   "weekStart": ""
 }
\ No newline at end of file