error correction wrt autonomous sharding script

clickhouse/config_update_scripts/update_compose.py

@@ -105,3 +105,4 @@ if __name__ == "__main__":
         with open(f'../clickhouse/node{curr_num_servers + i}-config/storage-policy.xml','w') as f4:
             f4.write(storage_policy_content)
 
+    print("Config Files Updated!")

clickhouse/config_update_scripts/update_trigger.py

@@ -5,6 +5,7 @@ import schedule
 import time
 
 def check_util_exec():
+    print("Performing check")
     # extracting details of each running container in json format    
     try:
         all_services = subprocess.check_output(["sudo", "docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
@@ -16,14 +17,20 @@ def check_util_exec():
     resource_util_exceed_flag = True # Flag to check if all of the containers have exceeded 80% memory utilization 
     for service in all_services:
         if re.findall(r'clickhouse-server',service['Name']):
-            if float(service['MemPerc'][:-1]) < 60:
+            if float(service['MemPerc'][:-1]) < 50:
                 resource_util_exceed_flag = False
     
     if resource_util_exceed_flag:
-        process = subprocess.Popen(['python3','../clickhouse/update_config_scripts/update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
+        print("Config update triggered")
+        process = subprocess.Popen(['python3','../clickhouse/config_update_scripts/update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
         stdout, stderr = process.communicate()  # Wait for the process to finish and capture output
         print("Standard Output:", stdout)
         print("Standard Error:", stderr)
+        if stdout:
+            redeploy  = subprocess.Popen(['docker','stack','deploy','-d','-c','../preprocessing/docker-compose.yml','-c','../clickhouse/docker-compose.yaml','-c','../ui/docker-compose.yaml','TheWebFarm'])
+            stdout1, stderr1= redeploy.communicate()  # Wait for the process to finish and capture output
+            print("Standard Output:", stdout1)
+            print("Standard Error:", stderr1)
         # try:
         #     all_services = subprocess.check_output(["sudo", "docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
         # except subprocess.CalledProcessError as e:

scripts/deploy.sh

@@ -46,11 +46,13 @@ elif [[ $masterNode ]]; then
 		-c ../clickhouse/docker-compose.yaml \
 		-c ../ui/docker-compose.yaml \
 		$stackName
-elif [[ $autoShard ]]; then
-	cd $scriptDir
-	python3 $scriptDir/../clickhouse/config_update_scripts/update_trigger.py
 else
 	echo "[+] swarm follower"
 	echo "[+] joining swarm with token $swarmToken"
 	$dockerCmd swarm join --token $swarmToken $managerAddr
 fi
+
+if [[ $autoShard ]]; then
+	cd $scriptDir
+	python3 ../clickhouse/config_update_scripts/update_trigger.py
+fi

ui/dashboard_1.json

@@ -431,7 +431,7 @@
           "refId": "A"
         }
       ],
-      "title": "Top regions (packet count)",
+      "title": "Top regions (packet count in millions)",
       "type": "bargauge"
     },
     {
@@ -942,6 +942,6 @@
   "timezone": "browser",
   "title": "Internet traffic capture analysis",
   "uid": "be59fkbp3zs3kc",
-  "version": 4,
+  "version": 1,
   "weekStart": ""
 }