mirror of
https://github.com/20kaushik02/real-time-traffic-analysis-clickhouse.git
synced 2026-01-25 16:04:04 +00:00
reorg back into master
This commit is contained in:
13
README
13
README
@@ -1,13 +0,0 @@
|
|||||||
Execution steps:
|
|
||||||
|
|
||||||
ensure to pull clickhouse docker image
|
|
||||||
> docker-compose up -d
|
|
||||||
identify the custom networks created.
|
|
||||||
> docker network ls
|
|
||||||
inspect the keeper network and verify whether all the containers are connected to it
|
|
||||||
> docker network inspect {{dds_proj_clickhouse-keeper-network}}
|
|
||||||
if all the containers are not connected
|
|
||||||
> docker-compose restart
|
|
||||||
To execute queries
|
|
||||||
> docker exec -it clickhouse-server1 clickhouse-client
|
|
||||||
> docker exec -it clickhouse-server2 clickhouse-client
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
<!-- This file was generated automatically.
|
|
||||||
Do not edit it: it is likely to be discarded and generated again before it's read next time.
|
|
||||||
Files used to generate this file:
|
|
||||||
/etc/clickhouse-server/config.xml
|
|
||||||
/etc/clickhouse-server/config.d/docker_related_config.xml -->
|
|
||||||
|
|
||||||
<yandex>
|
|
||||||
<logger>
|
|
||||||
<level>trace</level>
|
|
||||||
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
|
|
||||||
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
|
|
||||||
<size>1000M</size>
|
|
||||||
<count>3</count>
|
|
||||||
</logger>
|
|
||||||
<listen_host>::</listen_host>
|
|
||||||
|
|
||||||
<path>/var/lib/clickhouse/data/</path>
|
|
||||||
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
||||||
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
||||||
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
|
|
||||||
|
|
||||||
<keeper_server>
|
|
||||||
<tcp_port>9181</tcp_port>
|
|
||||||
<server_id>2</server_id>
|
|
||||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
||||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
||||||
<coordination_settings>
|
|
||||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
||||||
<session_timeout_ms>30000</session_timeout_ms>
|
|
||||||
<raft_logs_level>trace</raft_logs_level>
|
|
||||||
</coordination_settings>
|
|
||||||
<raft_configuration>
|
|
||||||
<server>
|
|
||||||
<id>1</id>
|
|
||||||
<hostname>clickhouse-keeper1</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>2</id>
|
|
||||||
<hostname>clickhouse-keeper2</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>3</id>
|
|
||||||
<hostname>clickhouse-keeper3</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
</raft_configuration>
|
|
||||||
</keeper_server>
|
|
||||||
|
|
||||||
<!-- Listen wildcard address to allow accepting connections from other containers and host network. -->
|
|
||||||
|
|
||||||
<listen_host>0.0.0.0</listen_host>
|
|
||||||
<listen_try>1</listen_try>
|
|
||||||
|
|
||||||
<!--
|
|
||||||
<logger>
|
|
||||||
<console>1</console>
|
|
||||||
</logger>
|
|
||||||
-->
|
|
||||||
</yandex>
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
<yandex>
|
|
||||||
<logger>
|
|
||||||
<level>trace</level>
|
|
||||||
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
|
|
||||||
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
|
|
||||||
<size>1000M</size>
|
|
||||||
<count>3</count>
|
|
||||||
</logger>
|
|
||||||
<listen_host>0.0.0.0</listen_host>
|
|
||||||
|
|
||||||
<path>/var/lib/clickhouse/data/</path>
|
|
||||||
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
||||||
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
||||||
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
|
|
||||||
|
|
||||||
<keeper_server>
|
|
||||||
<tcp_port>9181</tcp_port>
|
|
||||||
<server_id>1</server_id>
|
|
||||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
||||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
||||||
<coordination_settings>
|
|
||||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
||||||
<session_timeout_ms>30000</session_timeout_ms>
|
|
||||||
<raft_logs_level>trace</raft_logs_level>
|
|
||||||
</coordination_settings>
|
|
||||||
<raft_configuration>
|
|
||||||
<server>
|
|
||||||
<id>1</id>
|
|
||||||
<hostname>clickhouse-keeper1</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>2</id>
|
|
||||||
<hostname>clickhouse-keeper2</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>3</id>
|
|
||||||
<hostname>clickhouse-keeper3</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
</raft_configuration>
|
|
||||||
</keeper_server>
|
|
||||||
</yandex>
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
<yandex>
|
|
||||||
<logger>
|
|
||||||
<level>trace</level>
|
|
||||||
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
|
|
||||||
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
|
|
||||||
<size>1000M</size>
|
|
||||||
<count>3</count>
|
|
||||||
</logger>
|
|
||||||
<listen_host>0.0.0.0</listen_host>
|
|
||||||
|
|
||||||
<path>/var/lib/clickhouse/data/</path>
|
|
||||||
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
||||||
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
||||||
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
|
|
||||||
|
|
||||||
<keeper_server>
|
|
||||||
<tcp_port>9181</tcp_port>
|
|
||||||
<server_id>2</server_id>
|
|
||||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
||||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
||||||
<coordination_settings>
|
|
||||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
||||||
<session_timeout_ms>30000</session_timeout_ms>
|
|
||||||
<raft_logs_level>trace</raft_logs_level>
|
|
||||||
</coordination_settings>
|
|
||||||
<raft_configuration>
|
|
||||||
<server>
|
|
||||||
<id>1</id>
|
|
||||||
<hostname>clickhouse-keeper1</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>2</id>
|
|
||||||
<hostname>clickhouse-keeper2</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>3</id>
|
|
||||||
<hostname>clickhouse-keeper3</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
</raft_configuration>
|
|
||||||
</keeper_server>
|
|
||||||
</yandex>
|
|
||||||
@@ -1,44 +0,0 @@
|
|||||||
<yandex>
|
|
||||||
<logger>
|
|
||||||
<level>trace</level>
|
|
||||||
<log>/var/log/clickhouse-keeper/clickhouse-keeper.log</log>
|
|
||||||
<errorlog>/var/log/clickhouse-keeper/clickhouse-keeper.err.log</errorlog>
|
|
||||||
<size>1000M</size>
|
|
||||||
<count>3</count>
|
|
||||||
</logger>
|
|
||||||
<listen_host>0.0.0.0</listen_host>
|
|
||||||
|
|
||||||
<path>/var/lib/clickhouse/data/</path>
|
|
||||||
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
|
|
||||||
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
|
|
||||||
<format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>
|
|
||||||
|
|
||||||
<keeper_server>
|
|
||||||
<tcp_port>9181</tcp_port>
|
|
||||||
<server_id>3</server_id>
|
|
||||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
|
||||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
|
||||||
<coordination_settings>
|
|
||||||
<operation_timeout_ms>10000</operation_timeout_ms>
|
|
||||||
<session_timeout_ms>30000</session_timeout_ms>
|
|
||||||
<raft_logs_level>trace</raft_logs_level>
|
|
||||||
</coordination_settings>
|
|
||||||
<raft_configuration>
|
|
||||||
<server>
|
|
||||||
<id>1</id>
|
|
||||||
<hostname>clickhouse-keeper1</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>2</id>
|
|
||||||
<hostname>clickhouse-keeper2</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
<server>
|
|
||||||
<id>3</id>
|
|
||||||
<hostname>clickhouse-keeper3</hostname>
|
|
||||||
<port>9234</port>
|
|
||||||
</server>
|
|
||||||
</raft_configuration>
|
|
||||||
</keeper_server>
|
|
||||||
</yandex>
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
version: '3.8'
|
|
||||||
|
|
||||||
services:
|
|
||||||
clickhouse-keeper1:
|
|
||||||
image: clickhouse/clickhouse-server:latest
|
|
||||||
container_name: clickhouse-keeper1
|
|
||||||
command: >
|
|
||||||
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
|
|
||||||
volumes:
|
|
||||||
- ./clickhouse_keeper/keeper1-config.xml:/etc/clickhouse-server/config.xml
|
|
||||||
- ./clickhouse_data/data:/var/lib/clickhouse/data
|
|
||||||
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
|
|
||||||
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
|
|
||||||
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
|
|
||||||
|
|
||||||
networks:
|
|
||||||
clickhouse-keeper-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-keeper1
|
|
||||||
|
|
||||||
clickhouse-keeper2:
|
|
||||||
image: clickhouse/clickhouse-server:latest
|
|
||||||
container_name: clickhouse-keeper2
|
|
||||||
command: >
|
|
||||||
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
|
|
||||||
volumes:
|
|
||||||
- ./clickhouse_keeper/keeper2-config.xml:/etc/clickhouse-server/config.xml
|
|
||||||
- ./clickhouse_data/data:/var/lib/clickhouse/data
|
|
||||||
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
|
|
||||||
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
|
|
||||||
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
|
|
||||||
|
|
||||||
networks:
|
|
||||||
clickhouse-keeper-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-keeper2
|
|
||||||
|
|
||||||
clickhouse-keeper3:
|
|
||||||
image: clickhouse/clickhouse-server:latest
|
|
||||||
container_name: clickhouse-keeper3
|
|
||||||
command: >
|
|
||||||
/usr/bin/clickhouse-keeper --config-file=/etc/clickhouse-server/config.xml
|
|
||||||
volumes:
|
|
||||||
- ./clickhouse_keeper/keeper3-config.xml:/etc/clickhouse-server/config.xml
|
|
||||||
- ./clickhouse_data/data:/var/lib/clickhouse/data
|
|
||||||
- ./clickhouse_data/tmp:/var/lib/clickhouse/tmp
|
|
||||||
- ./clickhouse_data/user_files:/var/lib/clickhouse/user_files
|
|
||||||
- ./clickhouse_data/format_schemas:/var/lib/clickhouse/format_schemas
|
|
||||||
|
|
||||||
networks:
|
|
||||||
clickhouse-keeper-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-keeper3
|
|
||||||
|
|
||||||
clickhouse-server1:
|
|
||||||
image: clickhouse/clickhouse-server:latest
|
|
||||||
container_name: clickhouse-server1
|
|
||||||
volumes:
|
|
||||||
- ./node1-config/:/etc/clickhouse-server/config.d/
|
|
||||||
networks:
|
|
||||||
clickhouse-server-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-server1
|
|
||||||
clickhouse-keeper-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-server1
|
|
||||||
depends_on:
|
|
||||||
- clickhouse-keeper1
|
|
||||||
- clickhouse-keeper2
|
|
||||||
- clickhouse-keeper3
|
|
||||||
ports:
|
|
||||||
- "9001:9000" # Native client port
|
|
||||||
- "8123:8123" # HTTP interface
|
|
||||||
|
|
||||||
clickhouse-server2:
|
|
||||||
image: clickhouse/clickhouse-server:latest
|
|
||||||
container_name: clickhouse-server2
|
|
||||||
volumes:
|
|
||||||
- ./node2-config/:/etc/clickhouse-server/config.d/
|
|
||||||
networks:
|
|
||||||
clickhouse-server-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-server2
|
|
||||||
clickhouse-keeper-network:
|
|
||||||
aliases:
|
|
||||||
- clickhouse-server2
|
|
||||||
depends_on:
|
|
||||||
- clickhouse-keeper1
|
|
||||||
- clickhouse-keeper2
|
|
||||||
- clickhouse-keeper3
|
|
||||||
ports:
|
|
||||||
- "9002:9000" # Native client port
|
|
||||||
- "8124:8123" # HTTP interface
|
|
||||||
|
|
||||||
networks:
|
|
||||||
clickhouse-server-network:
|
|
||||||
clickhouse-keeper-network:
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<logger>
|
|
||||||
<level>debug</level>
|
|
||||||
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
||||||
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
||||||
<size>1000M</size>
|
|
||||||
<count>3</count>
|
|
||||||
</logger>
|
|
||||||
<display_name>cluster_1S_2R node 1</display_name>
|
|
||||||
<listen_host>0.0.0.0</listen_host>
|
|
||||||
<http_port>8123</http_port>
|
|
||||||
<tcp_port>9000</tcp_port>
|
|
||||||
<!-- Maximum connections and settings -->
|
|
||||||
<max_connections>4096</max_connections>
|
|
||||||
<keep_alive_timeout>3</keep_alive_timeout>
|
|
||||||
<max_concurrent_queries>100</max_concurrent_queries>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- Additional configuration files can be included -->
|
|
||||||
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
|
|
||||||
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
|
|
||||||
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
|
|
||||||
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<macros>
|
|
||||||
<shard>01</shard>
|
|
||||||
<replica>01</replica>
|
|
||||||
<cluster>cluster_1S_2R</cluster>
|
|
||||||
</macros>
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<remote_servers replace="true">
|
|
||||||
<cluster_1S_2R>
|
|
||||||
<secret>mysecretphrase</secret>
|
|
||||||
<shard>
|
|
||||||
<internal_replication>true</internal_replication>
|
|
||||||
<replica>
|
|
||||||
<host>clickhouse-server1</host>
|
|
||||||
<port>9000</port>
|
|
||||||
</replica>
|
|
||||||
<replica>
|
|
||||||
<host>clickhouse-server2</host>
|
|
||||||
<port>9000</port>
|
|
||||||
</replica>
|
|
||||||
</shard>
|
|
||||||
</cluster_1S_2R>
|
|
||||||
</remote_servers>
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<zookeeper>
|
|
||||||
<!-- where are the ZK nodes -->
|
|
||||||
<node>
|
|
||||||
<host>clickhouse-keeper1</host>
|
|
||||||
<port>9181</port>
|
|
||||||
</node>
|
|
||||||
<node>
|
|
||||||
<host>clickhouse-keeper2</host>
|
|
||||||
<port>9181</port>
|
|
||||||
</node>
|
|
||||||
<node>
|
|
||||||
<host>clickhouse-keeper3</host>
|
|
||||||
<port>9181</port>
|
|
||||||
</node>
|
|
||||||
</zookeeper>
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<logger>
|
|
||||||
<level>debug</level>
|
|
||||||
<log>/var/log/clickhouse-server/clickhouse-server.log</log>
|
|
||||||
<errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
|
|
||||||
<size>1000M</size>
|
|
||||||
<count>3</count>
|
|
||||||
</logger>
|
|
||||||
<display_name>cluster_1S_2R node 2</display_name>
|
|
||||||
<listen_host>0.0.0.0</listen_host>
|
|
||||||
<http_port>8123</http_port>
|
|
||||||
<tcp_port>9000</tcp_port>
|
|
||||||
<!-- Maximum connections and settings -->
|
|
||||||
<max_connections>4096</max_connections>
|
|
||||||
<keep_alive_timeout>3</keep_alive_timeout>
|
|
||||||
<max_concurrent_queries>100</max_concurrent_queries>
|
|
||||||
|
|
||||||
|
|
||||||
<!-- Additional configuration files can be included -->
|
|
||||||
<include_from>/etc/clickhouse-server/config.d/macros.xml</include_from>
|
|
||||||
<include_from>/etc/clickhouse-server/config.d/remote-servers.xml</include_from>
|
|
||||||
<include_from>/etc/clickhouse-server/config.d/use-keeper.xml</include_from>
|
|
||||||
<!-- <include_from>/etc/clickhouse-server/config.d/keeper-config.xml</include_from> -->
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<macros>
|
|
||||||
<shard>01</shard>
|
|
||||||
<replica>02</replica>
|
|
||||||
<cluster>cluster_1S_2R</cluster>
|
|
||||||
</macros>
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<remote_servers replace="true">
|
|
||||||
<cluster_1S_2R>
|
|
||||||
<secret>mysecretphrase</secret>
|
|
||||||
<shard>
|
|
||||||
<internal_replication>true</internal_replication>
|
|
||||||
<replica>
|
|
||||||
<host>clickhouse-server1</host>
|
|
||||||
<port>9000</port>
|
|
||||||
</replica>
|
|
||||||
<replica>
|
|
||||||
<host>clickhouse-server2</host>
|
|
||||||
<port>9000</port>
|
|
||||||
</replica>
|
|
||||||
</shard>
|
|
||||||
</cluster_1S_2R>
|
|
||||||
</remote_servers>
|
|
||||||
</clickhouse>
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
<clickhouse>
|
|
||||||
<zookeeper>
|
|
||||||
<!-- where are the ZK nodes -->
|
|
||||||
<node>
|
|
||||||
<host>clickhouse-keeper1</host>
|
|
||||||
<port>9181</port>
|
|
||||||
</node>
|
|
||||||
<node>
|
|
||||||
<host>clickhouse-keeper2</host>
|
|
||||||
<port>9181</port>
|
|
||||||
</node>
|
|
||||||
<node>
|
|
||||||
<host>clickhouse-keeper3</host>
|
|
||||||
<port>9181</port>
|
|
||||||
</node>
|
|
||||||
</zookeeper>
|
|
||||||
</clickhouse>
|
|
||||||
Reference in New Issue
Block a user