my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2026-01-25 08:04:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

error correction wrt autonomous sharding script

Browse Source
This commit is contained in:
Lalit Arvind
2024-11-30 21:12:38 -07:00
parent fd1ad9b641
commit 7a79530633
4 changed files with 17 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
clickhouse/config_update_scripts/update_compose.py
View File

@@ -105,3 +105,4 @@ if __name__ == "__main__":
with open(f'../clickhouse/node{curr_num_servers + i}-config/storage-policy.xml','w') as f4:
f4.write(storage_policy_content)
print("Config Files Updated!")

11
clickhouse/config_update_scripts/update_trigger.py
View File

@@ -5,6 +5,7 @@ import schedule
import time
def check_util_exec():
print("Performing check")
# extracting details of each running container in json format
try:
all_services = subprocess.check_output(["sudo", "docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
@@ -16,14 +17,20 @@ def check_util_exec():
resource_util_exceed_flag = True # Flag to check if all of the containers have exceeded 80% memory utilization
for service in all_services:
if re.findall(r'clickhouse-server',service['Name']):
if float(service['MemPerc'][:-1]) < 60:
if float(service['MemPerc'][:-1]) < 50:
resource_util_exceed_flag = False
if resource_util_exceed_flag:
process = subprocess.Popen(['python3','../clickhouse/update_config_scripts/update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
print("Config update triggered")
process = subprocess.Popen(['python3','../clickhouse/config_update_scripts/update_compose.py'],text=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
stdout, stderr = process.communicate() # Wait for the process to finish and capture output
print("Standard Output:", stdout)
print("Standard Error:", stderr)
if stdout:
redeploy = subprocess.Popen(['docker','stack','deploy','-d','-c','../preprocessing/docker-compose.yml','-c','../clickhouse/docker-compose.yaml','-c','../ui/docker-compose.yaml','TheWebFarm'])
stdout1, stderr1= redeploy.communicate() # Wait for the process to finish and capture output
print("Standard Output:", stdout1)
print("Standard Error:", stderr1)
# try:
# all_services = subprocess.check_output(["sudo", "docker","stats","--no-stream","--format","json"],text=True).split('\n')[:-1]
# except subprocess.CalledProcessError as e: