my-asu-archive/real-time-traffic-analysis-clickhouse
1
0
Fork 0
mirror of https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git synced 2025-12-06 10:44:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

reorg

Browse Source
This commit is contained in:
Kaushik Narayan R 2024-11-14 22:33:10 -07:00
parent 2936f6c0db
commit 90a7a2415d
14 changed files with 435 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
clickhouse/README Normal file
View File

@ -0,0 +1,13 @@
Execution steps:
ensure to pull clickhouse docker image
> docker-compose up -d
identify the custom networks created.
> docker network ls
inspect the keeper network and verify whether all the containers are connected to it
> docker network inspect {{dds_proj_clickhouse-keeper-network}}
if all the containers are not connected
> docker-compose restart
To execute queries
> docker exec -it clickhouse-server1 clickhouse-client
> docker exec -it clickhouse-server2 clickhouse-client

61
clickhouse/clickhouse_data/data/preprocessed_configs/config.xml Normal file
View File

@ -0,0 +1,61 @@
<!-- This file was generated automatically.
Do not edit it: it is likely to be discarded and generated again before it's read next time.
Files used to generate this file:
/etc/clickhouse-server/config.xml
/etc/clickhouse-server/config.d/docker_related_config.xml -->
<yandex>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
<size>1000M</size>
<count>3</count>
</logger>
<listen_host>::</listen_host>
<path>/var/lib/clickhouse/data/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>2</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>clickhouse-keeper1</hostname>
<port>9234</port>
</server>
<server>
<id>2</id>
<hostname>clickhouse-keeper2</hostname>
<port>9234</port>
</server>
<server>
<id>3</id>
<hostname>clickhouse-keeper3</hostname>
<port>9234</port>
</server>
</raft_configuration>
</keeper_server>
<!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
<listen_host>0.0.0.0</listen_host>
<listen_try>1</listen_try>
<!--
<logger>
<console>1</console>
</logger>
-->
</yandex>

44
clickhouse/clickhouse_keeper/keeper1-config.xml Normal file
View File

@ -0,0 +1,44 @@
<yandex>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
<size>1000M</size>
<count>3</count>
</logger>
<listen_host>0.0.0.0</listen_host>
<path>/var/lib/clickhouse/data/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>1</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>clickhouse-keeper1</hostname>
<port>9234</port>
</server>
<server>
<id>2</id>
<hostname>clickhouse-keeper2</hostname>
<port>9234</port>
</server>
<server>
<id>3</id>
<hostname>clickhouse-keeper3</hostname>
<port>9234</port>
</server>
</raft_configuration>
</keeper_server>
</yandex>

44
clickhouse/clickhouse_keeper/keeper2-config.xml Normal file
View File

@ -0,0 +1,44 @@
<yandex>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
<size>1000M</size>
<count>3</count>
</logger>
<listen_host>0.0.0.0</listen_host>
<path>/var/lib/clickhouse/data/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>2</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>clickhouse-keeper1</hostname>
<port>9234</port>
</server>
<server>
<id>2</id>
<hostname>clickhouse-keeper2</hostname>
<port>9234</port>
</server>
<server>
<id>3</id>
<hostname>clickhouse-keeper3</hostname>
<port>9234</port>
</server>
</raft_configuration>
</keeper_server>
</yandex>

44
clickhouse/clickhouse_keeper/keeper3-config.xml Normal file
View File

@ -0,0 +1,44 @@
<yandex>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
<size>1000M</size>
<count>3</count>
</logger>
<listen_host>0.0.0.0</listen_host>
<path>/var/lib/clickhouse/data/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>3</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>clickhouse-keeper1</hostname>
<port>9234</port>
</server>
<server>
<id>2</id>
<hostname>clickhouse-keeper2</hostname>
<port>9234</port>
</server>
<server>
<id>3</id>
<hostname>clickhouse-keeper3</hostname>
<port>9234</port>
</server>
</raft_configuration>
</keeper_server>
</yandex>

97
clickhouse/docker-compose.yaml Normal file
View File

@ -0,0 +1,97 @@
version: '3.8'
services:
clickhouse-keeper1:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-keeper1
command: >
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
volumes:
- ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
- ./clickhouse_data/data:/var/lib/clickhouse/data
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
networks:
clickhouse-keeper-network:
aliases:
- clickhouse-keeper1
clickhouse-keeper2:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-keeper2
command: >
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
volumes:
- ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
- ./clickhouse_data/data:/var/lib/clickhouse/data
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
networks:
clickhouse-keeper-network:
aliases:
- clickhouse-keeper2
clickhouse-keeper3:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-keeper3
command: >
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
volumes:
- ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
- ./clickhouse_data/data:/var/lib/clickhouse/data
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
networks:
clickhouse-keeper-network:
aliases:
- clickhouse-keeper3
clickhouse-server1:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-server1
volumes:
- ./node1-config/:/etc/clickhouse-server/config.d/
networks:
clickhouse-server-network:
aliases:
- clickhouse-server1
clickhouse-keeper-network:
aliases:
- clickhouse-server1
depends_on:
- clickhouse-keeper1
- clickhouse-keeper2
- clickhouse-keeper3
ports:
- "9001:9000" # Native client port
- "8123:8123" # HTTP interface
clickhouse-server2:
image: clickhouse/clickhouse-server:latest
container_name: clickhouse-server2
volumes:
- ./node2-config/:/etc/clickhouse-server/config.d/
networks:
clickhouse-server-network:
aliases:
- clickhouse-server2
clickhouse-keeper-network:
aliases:
- clickhouse-server2
depends_on:
- clickhouse-keeper1
- clickhouse-keeper2
- clickhouse-keeper3
ports:
- "9002:9000" # Native client port
- "8124:8123" # HTTP interface
networks:
clickhouse-server-network:
clickhouse-keeper-network:

24
clickhouse/node1-config/config.xml Normal file
View File

@ -0,0 +1,24 @@
<clickhouse>
<logger>
<level>debug</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>3</count>
</logger>
<display_name>cluster_1S_2R node 1</display_name>
<listen_host>0.0.0.0</listen_host>
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<!-- Maximum connections and settings -->
<max_connections>4096</max_connections>
<keep_alive_timeout>3</keep_alive_timeout>
<max_concurrent_queries>100</max_concurrent_queries>
<!-- Additional configuration files can be included -->
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

7
clickhouse/node1-config/macros.xml Normal file
View File

@ -0,0 +1,7 @@
<clickhouse>
<macros>
<shard>01</shard>
<replica>01</replica>
<cluster>cluster_1S_2R</cluster>
</macros>
</clickhouse>

18
clickhouse/node1-config/remote-servers.xml Normal file
View File

@ -0,0 +1,18 @@
<clickhouse>
<remote_servers replace="true">
<cluster_1S_2R>
<secret>mysecretphrase</secret>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server1</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse-server2</host>
<port>9000</port>
</replica>
</shard>
</cluster_1S_2R>
</remote_servers>
</clickhouse>

17
clickhouse/node1-config/use-keeper.xml Normal file
View File

@ -0,0 +1,17 @@
<clickhouse>
<zookeeper>
<!-- where are the ZK nodes -->
<node>
<host>clickhouse-keeper1</host>
<port>9181</port>
</node>
<node>
<host>clickhouse-keeper2</host>
<port>9181</port>
</node>
<node>
<host>clickhouse-keeper3</host>
<port>9181</port>
</node>
</zookeeper>
</clickhouse>

24
clickhouse/node2-config/config.xml Normal file
View File

@ -0,0 +1,24 @@
<clickhouse>
<logger>
<level>debug</level>
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
<size>1000M</size>
<count>3</count>
</logger>
<display_name>cluster_1S_2R node 2</display_name>
<listen_host>0.0.0.0</listen_host>
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<!-- Maximum connections and settings -->
<max_connections>4096</max_connections>
<keep_alive_timeout>3</keep_alive_timeout>
<max_concurrent_queries>100</max_concurrent_queries>
<!-- Additional configuration files can be included -->
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
</clickhouse>

7
clickhouse/node2-config/macros.xml Normal file
View File

@ -0,0 +1,7 @@
<clickhouse>
<macros>
<shard>01</shard>
<replica>02</replica>
<cluster>cluster_1S_2R</cluster>
</macros>
</clickhouse>

18
clickhouse/node2-config/remote-servers.xml Normal file
View File

@ -0,0 +1,18 @@
<clickhouse>
<remote_servers replace="true">
<cluster_1S_2R>
<secret>mysecretphrase</secret>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>clickhouse-server1</host>
<port>9000</port>
</replica>
<replica>
<host>clickhouse-server2</host>
<port>9000</port>
</replica>
</shard>
</cluster_1S_2R>
</remote_servers>
</clickhouse>

17
clickhouse/node2-config/use-keeper.xml Normal file
View File

@ -0,0 +1,17 @@
<clickhouse>
<zookeeper>
<!-- where are the ZK nodes -->
<node>
<host>clickhouse-keeper1</host>
<port>9181</port>
</node>
<node>
<host>clickhouse-keeper2</host>
<port>9181</port>
</node>
<node>
<host>clickhouse-keeper3</host>
<port>9181</port>
</node>
</zookeeper>
</clickhouse>