[ django部署ELK ]

发表评论
193 views

目录:

ELK日志部署

一.第一种[单机部署]：elastic + kibana + logstash

1.docker-compose.yml

version: '3'
services:
  elasticsearch:
    image: elasticsearch:7.13.4
    container_name: es
    environment:
      - "cluster.name=elasticsearch" #设置集群名称为elasticsearch
      - "discovery.type=single-node" #以单一节点模式启动
      - "ES_JAVA_OPTS=-Xms256m -Xmx1024m" #设置使用jvm内存大小
      - "ELASTIC_PASSWORD=chenhaifeng"    #密码
      - "xpack.security.enabled=true"
    ports:
      - 9200:9200
  kibana:
    image: kibana:7.13.4
    container_name: kibana
    depends_on:
      - elasticsearch #kibana在elasticsearch启动之后再启动
    environment:
      - "elasticsearch.hosts=http://es:9200" #设置访问elasticsearch的地址
      - "ELASTICSEARCH_USERNAME=elastic"    
      - "ELASTICSEARCH_PASSWORD=chenhaifeng"    #elastic密码
      - "I18N_LOCALE=zh-CN"                 #中文
    ports:    
      - 5601:5601
  logstash:
    image: logstash:7.13.4
    container_name: logstash
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - ./logstash.yml:/usr/share/logstash/config/logstash.yml
    depends_on:
      - elasticsearch #logstash在elasticsearch启动之后再启动
    ports:
      - 4560:4560 # 这里是使用到通过请求来推送日志才开放的端口， 当只使用日志文件来获取数据就可以不用开启

2.logstash.conf通过django发起请求到input获取日志

input {
  # tcp请求输入日志
  tcp {
    mode => "server"
    host => "0.0.0.0"
    port => 4560
    codec => json_lines
  }
  # 文件输入日志
  file {
    path => "/path/to/your/logs/*.log"  # 这里指定你的日志文件路径，可以是通配符
    start_position => "beginning"  # 如果需要从文件的开始读取
    sincedb_path => "/dev/null"  # 防止 Logstash 记录处理进度（适用于开发环境）
  }
}

output {
  elasticsearch {
    hosts => ["es:9200"]
    user => "elastic"
    password => "chenhaifeng"
    index => "django-%{+YYYY.MM.dd}"
  }
}

3.logstash.yml

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: chenhaifeng
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://es:9200" ]

Django案例代码推送日志请求tcp到logstash

二.第二种: elastic+logstash+kibana+kafka+filebeat

1.docker-compose.yml

version: '3'
services:
  elasticsearch:
    image: elasticsearch:7.13.4
    container_name: es
    environment:
      - "cluster.name=elasticsearch" #设置集群名称为elasticsearch
      - "discovery.type=single-node" #以单一节点模式启动
      - "ES_JAVA_OPTS=-Xms1g -Xmx2g" #设置使用jvm内存大小
      - "ELASTIC_PASSWORD=chenhaifeng"    #密码
      - "xpack.security.enabled=true"
    depends_on:
      - kafka #kibana在elasticsearch启动之后再启动
    ports:
      - 9200:9200
  kibana:
    image: kibana:7.13.4
    container_name: kibana
    depends_on:
      - elasticsearch #kibana在elasticsearch启动之后再启动
    environment:
      - "elasticsearch.hosts=http://es:9200" #设置访问elasticsearch的地址
      - "ELASTICSEARCH_USERNAME=elastic"    
      - "ELASTICSEARCH_PASSWORD=chenhaifeng"    #elastic密码
      - "I18N_LOCALE=zh-CN"                 #中文
    ports:    
      - 5601:5601
  logstash:
    image: logstash:7.13.4
    container_name: logstash
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - ./logstash.yml:/usr/share/logstash/config/logstash.yml
      #- ./logs:/logs
    depends_on:
      - elasticsearch #logstash在elasticsearch启动之后再启动
    ports:
      - 4560:4560
  filebeat:
    image: elastic/filebeat:7.13.4
    container_name: filebeat
    environment:
      # 时区上海 - Time zone Shanghai (Change if needed)
      TZ: Asia/Shanghai
    user: root
    restart: always
    entrypoint: "filebeat -e -strict.perms=false"  #解决配置文件权限问题 - Solving the configuration file permissions
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      # 此处需指定docker的containers目录，取决于你docker的配置 - The containers directory of docker needs to be specified here, depending on your docker configuration
      # 如snap安装的docker，则为/var/snap/docker/common/var-lib-docker/containers - Example if docker is installed by Snap /var/snap/docker/common/var-lib-docker/containers
      # - /var/snap/docker/common/var-lib-docker/containers:/var/lib/docker/containers
      - ./logs:/logs
    depends_on:
      - kafka
  kafka:
    image: apache/kafka:latest
    container_name: kafka
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_NUM_PARTITIONS: 3
    ports:
      - 9092:9092

2.filebeat.yml

filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /logs/*.log

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

output.kafka:
  enabled: true
  hosts: ["kafka:9092"]
  #要提前创建topic
  topic: "log_topic"
  partition.hash:
    reachable_only: true
  compression: gzip
  max_message_bytes: 1000000
  required_acks: 1

3.logstash.conf

input {
  kafka {
    bootstrap_servers => "kafka:9092"
    topics => ["log_topic"]
  }
}

output {
  elasticsearch {
    hosts => ["es:9200"]
    user => "elastic"
    password => "chenhaifeng"
    index => "django-%{+YYYY.MM.dd}"
  }
}

4.logstash.yml

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: chenhaifeng
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://es:9200" ]

三.主服务器部署[kibana,logstash,es, kafka]

主服务器收集日志部署: - kibana:7.13.4 - logstash:7.13.4 - elasticsearch:7.13.4 - apache/kafka:latest

1.docker-compose.yml

version: '3'
services:
  elasticsearch:
    image: elasticsearch:7.13.4
    container_name: es
    environment:
      - "cluster.name=elasticsearch" 
      - "discovery.type=single-node"
      - "ES_JAVA_OPTS=-Xms1g -Xmx2g"
      - "ELASTIC_PASSWORD=chenhaifeng"
      - "xpack.security.enabled=true"
    depends_on:
      - kafka 
    ports:
      - 9200:9200
  kibana:
    image: kibana:7.13.4
    container_name: kibana
    depends_on:
      - elasticsearch 
    environment:
      - "elasticsearch.hosts=http://192.168.1.2:9200"
      - "ELASTICSEARCH_USERNAME=elastic"    
      - "ELASTICSEARCH_PASSWORD=chenhaifeng"
      - "I18N_LOCALE=zh-CN"                
    ports:    
      - 5601:5601
  logstash:
    image: logstash:7.13.4
    container_name: logstash
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
      - ./logstash.yml:/usr/share/logstash/config/logstash.yml
    depends_on:
      - elasticsearch 
    ports:    
      - 5044:5044
  kafka:
    image: apache/kafka:latest
    container_name: kafka
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.2:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_NUM_PARTITIONS: 3
    ports:
      - 9092:9092

2.logstash.conf

input {
  kafka {
    bootstrap_servers => "kafka:9092"
    topics => ["chf_cews_topic", "test"]
    decorate_events => true
    type => "chf_cews"
  }
}

filter {
  json {
    source => "message"
    target => "parsed_message"  
    remove_field => ["message"]
  }
}

output {

  if[type] == "chf_cews" {
    elasticsearch {
      hosts => ["192.168.1.2:9200"]
      user => "elastic"
      password => "chenhaifeng"
      index => "chf-cews-%{+YYYY.MM.dd}"
    }
  } else {
    elasticsearch {
      hosts => ["192.168.1.2:9200"]
      user => "elastic"
      password => "chenhaifeng"
      index => "default-%{+YYYY.MM.dd}"
    }
  }
}

3.logstash.yml

xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: chenhaifeng
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://192.168.1.2:9200" ]

4.其他服务器需要收集日志的服务器部署[filebeat]

5.docker-compose.yml

version: '3'
services:
  filebeat:
    image: elastic/filebeat:7.13.4
    container_name: filebeat
    environment:
      TZ: Asia/Shanghai
    user: root
    entrypoint: "filebeat -e -strict.perms=false"  
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /var/run/docker.sock:/var/run/docker.sock
      - /var/lib/docker/containers:/var/lib/docker/containers:ro

6.filebeat.yml

filebeat.inputs:
  - type: docker
    containers.ids: '*'
    paths:
      - /var/lib/docker/containers/*/*.log

processors:
  - add_docker_metadata: ~

output.kafka:
  enabled: true
  hosts: ["192.168.1.2:9092"]
  topic: "chf_cews_topic"
  partition.hash:
    reachable_only: true
  compression: gzip
  max_message_bytes: 1000000
  required_acks: 1

输入ip:5601登录查看