docker-compose快速搭建elk

416次阅读
共计 4649 个字符，预计需要花费 12 分钟才能阅读完成。
这种方式最快，但是不适用生产，只适合测试或者开发环境。
docker pull zookeeper:3.4
docker pull wurstmeister/kafka:2.12-2.4.1
docker network create --subnet 172.30.1.0/16 --gateway 172.30.0.1 kafka


[root@yx-ops-001.g6k.cn]$mkdir -p /data/zk/zoo1/data
[root@yx-ops-001.g6k.cn]$mkdir -p /data/zk/zoo2/data
[root@yx-ops-001.g6k.cn]$mkdir -p /data/zk/zoo3/data
[root@yx-ops-001.g6k.cn]$mkdir -p /data/zk/zoo1/datalog
[root@yx-ops-001.g6k.cn]$mkdir -p /data/zk/zoo2/datalog
[root@yx-ops-001.g6k.cn]$mkdir -p /data/zk/zoo3/datalog
[root@yx-ops-001.g6k.cn]$mkdir -p /data/kafka/kafkaCluster/kafka1/logs
[root@yx-ops-001.g6k.cn]$mkdir -p /data/kafka/kafkaCluster/kafka2/logs
[root@yx-ops-001.g6k.cn]$mkdir -p /data/kafka/kafkaCluster/kafka3/logs


version: '3.4'


services: 
    zoo1:
        image: zookeeper:3.4
        restart: always
        hostname: zoo1
        container_name: zoo1
        ports:
            - 2184:2181
        volumes: 
            - "/data/zk/zoo1/data:/data"
            - "/data/zk/zoo1/datalog:/datalog"
        environment: 
            TZ: Asia/Shanghai
            ZOO_MY_ID: 1
            ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
        networks:
            kafka:
                ipv4_address: 172.30.0.11


    zoo2:
        image: zookeeper:3.4
        restart: always
        hostname: zoo2
        container_name: zoo2
        ports:
            - 2185:2181
        volumes: 
            - "/data/zk/zoo2/data:/data"
            - "/data/zk/zoo2/datalog:/datalog"
        environment: 
            TZ: Asia/Shanghai
            ZOO_MY_ID: 2
            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zoo3:2888:3888
        networks:
            kafka:
                ipv4_address: 172.30.0.12


    zoo3:
        image: zookeeper:3.4
        restart: always
        hostname: zoo3
        container_name: zoo3
        ports:
            - 2186:2181
        volumes: 
            - "/data/zk/zoo3/data:/data"
            - "/data/zk/zoo3/datalog:/datalog"
        environment: 
            TZ: Asia/Shanghai
            ZOO_MY_ID: 3
            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=0.0.0.0:2888:3888
        networks:
            kafka:
                ipv4_address: 172.30.0.13


    kafka1:
        image: wurstmeister/kafka:2.12-2.4.1
        restart: always
        hostname: kafka1.g6k.cn
        container_name: kafka1
        privileged: true
        ports:
            - 9092:9092
        environment:
              KAFKA_ADVERTISED_HOST_NAME: kafka1.g6k.cn
              KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
              KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1.g6k.cn:9092
              KAFKA_ADVERTISED_PORT: 9092
              KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
              KAFKA_BROKER_ID: 1
              KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: ture
        volumes:
            - /data/kafka/kafkaCluster/kafka1/logs:/kafka
            - /etc/localtime:/etc/localtime:ro
            - /etc/timezone:/etc/timezone:ro
        networks:
            kafka:
                ipv4_address: 172.30.1.11
        extra_hosts: 
            - "zoo1:172.30.0.11"
            - "zoo2:172.30.0.12"
            - "zoo3:172.30.0.13"
        depends_on: 
            - zoo1
            - zoo2
            - zoo3
        external_links: 
            - zoo1
            - zoo2
            - zoo3


    kafka2:
        image: wurstmeister/kafka:2.12-2.4.1
        restart: always
        hostname: kafka2.g6k.cn
        container_name: kafka2
        privileged: true
        ports:
            - 9093:9093
        environment:
              KAFKA_ADVERTISED_HOST_NAME: kafka2.g6k.cn
              KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9093
              KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2.g6k.cn:9093
              KAFKA_ADVERTISED_PORT: 9093
              KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
              KAFKA_BROKER_ID: 2
              KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: ture
        volumes:
            - /data/kafka/kafkaCluster/kafka2/logs:/kafka
            - /etc/localtime:/etc/localtime:ro
            - /etc/timezone:/etc/timezone:ro
        networks:
            kafka:
                ipv4_address: 172.30.1.12
        extra_hosts: 
            - "zoo1:172.30.0.11"
            - "zoo2:172.30.0.12"
            - "zoo3:172.30.0.13"                
        depends_on: 
            - zoo1
            - zoo2
            - zoo3                
        external_links: 
            - zoo1
            - zoo2
            - zoo3           


    kafka3:
        image: wurstmeister/kafka:2.12-2.4.1
        restart: always
        hostname: kafka3.g6k.cn
        container_name: kafka3
        privileged: true
        ports:
            - 9094:9094
        environment:
              KAFKA_ADVERTISED_HOST_NAME: kafka3.g6k.cn
              KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9094
              KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3.g6k.cn:9094
              KAFKA_ADVERTISED_PORT: 9094
              KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
              KAFKA_BROKER_ID: 3
              KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: ture
        volumes:
            - /data/kafka/kafkaCluster/kafka3/logs:/kafka
            - /etc/localtime:/etc/localtime:ro
            - /etc/timezone:/etc/timezone:ro
        networks:
            kafka:
                ipv4_address: 172.30.1.13
        extra_hosts: 
            - "zoo1:172.30.0.11"
            - "zoo2:172.30.0.12"
            - "zoo3:172.30.0.13"                
        depends_on: 
            - zoo1
            - zoo2
            - zoo3                
        external_links: 
            - zoo1
            - zoo2
            - zoo3


networks: 
    kafka:
        external: 
            name: kafka


es 搭建参考 https://github.com/hellxz/docker-es-cluster


[root@yx-ops-001.g6k.cn]$cat config/logstash-me.yml
input {
 kafka {
    bootstrap_servers => "kafka1.g6k.cn:9092,kafka2.g6k.cn:9092,kafka3.g6k.cn:9092"
    #topics => ["web-logs-g6k-php"]
    topics => ["web-logs-g6k-php"]
    group_id => "php-logs"
    type => "web-logs-g6k-php"
    client_id => "web-logs-g6k-php"
    consumer_threads => 3
    decorate_events => true
    auto_offset_reset => "earliest"
    codec => json
   }
}
#filter {
#  json {
#    source => "message"
#  }
#}
output {if [type] == "web-logs-g6k-php"{
    elasticsearch {hosts => ["http://192.168.0.88:9205"]
      #index => "web-logs-g6k-php-%{+YYYY.MM.dd}"
      index => "web_%{[@metadata][kafka][topic]}-%{+YYYY.MM.dd}"
     }
  }
}


[root@myblog filebeat]# cat filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/*.log
  json.keys_under_root: true
  json.overwrite_keys: true
output.kafka:
  hosts: ["kafka1.g6k.cn:9092","kafka2.g6k.cn:9093","kafka3.g6k.cn:9094"]
  topic: web-logs-g6k-php
logging.level: info
正文完