From 7b4a7ae25025e65f3b613847231ac64e38beaef0 Mon Sep 17 00:00:00 2001 From: vanzhiganov Date: Sun, 2 Feb 2025 15:05:12 +0300 Subject: [PATCH] init --- README.md | 4 ++ elk.yml | 10 ++++ inv.ini | 4 ++ roles/elk/README.md | 38 ++++++++++++++ roles/elk/defaults/main.yml | 2 + roles/elk/files/.env | 0 roles/elk/files/compose.yml | 50 +++++++++++++++++++ .../files/configs/elasticsearch/config.yml | 5 ++ roles/elk/files/configs/filebeat/config.yml | 15 ++++++ roles/elk/files/configs/kibana/config.yml | 8 +++ roles/elk/files/configs/logstash/config.yml | 1 + .../pipelines/service_stamped_json_logs.conf | 42 ++++++++++++++++ roles/elk/handlers/main.yml | 2 + roles/elk/meta/main.yml | 34 +++++++++++++ roles/elk/tasks/main.yml | 28 +++++++++++ roles/elk/tests/inventory | 2 + roles/elk/tests/test.yml | 5 ++ roles/elk/vars/main.yml | 2 + 18 files changed, 252 insertions(+) create mode 100644 README.md create mode 100644 elk.yml create mode 100644 inv.ini create mode 100644 roles/elk/README.md create mode 100644 roles/elk/defaults/main.yml create mode 100644 roles/elk/files/.env create mode 100644 roles/elk/files/compose.yml create mode 100644 roles/elk/files/configs/elasticsearch/config.yml create mode 100644 roles/elk/files/configs/filebeat/config.yml create mode 100644 roles/elk/files/configs/kibana/config.yml create mode 100644 roles/elk/files/configs/logstash/config.yml create mode 100644 roles/elk/files/configs/logstash/pipelines/service_stamped_json_logs.conf create mode 100644 roles/elk/handlers/main.yml create mode 100644 roles/elk/meta/main.yml create mode 100644 roles/elk/tasks/main.yml create mode 100644 roles/elk/tests/inventory create mode 100644 roles/elk/tests/test.yml create mode 100644 roles/elk/vars/main.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..530521e --- /dev/null +++ b/README.md @@ -0,0 +1,4 @@ +# elk stack for opeenstack env + +based on article +https://habr.com/ru/articles/671344/ \ No newline at end of file diff --git a/elk.yml b/elk.yml new file mode 100644 index 0000000..11f1db2 --- /dev/null +++ b/elk.yml @@ -0,0 +1,10 @@ +- name: elk + hosts: + - elk + # vars: + # SERVER_NAME: www.procdn.net + gather_facts: no + roles: + - elk + tags: + - elk \ No newline at end of file diff --git a/inv.ini b/inv.ini new file mode 100644 index 0000000..284e9f6 --- /dev/null +++ b/inv.ini @@ -0,0 +1,4 @@ +host192-168-71-44 ansible_host=192.168.71.44 ansible_user=root ansible_port=22 ansible_python_interpreter=python3 + +[elk] +host192-168-71-44 diff --git a/roles/elk/README.md b/roles/elk/README.md new file mode 100644 index 0000000..225dd44 --- /dev/null +++ b/roles/elk/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/roles/elk/defaults/main.yml b/roles/elk/defaults/main.yml new file mode 100644 index 0000000..dd7f326 --- /dev/null +++ b/roles/elk/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for roles/elk diff --git a/roles/elk/files/.env b/roles/elk/files/.env new file mode 100644 index 0000000..e69de29 diff --git a/roles/elk/files/compose.yml b/roles/elk/files/compose.yml new file mode 100644 index 0000000..0b7197a --- /dev/null +++ b/roles/elk/files/compose.yml @@ -0,0 +1,50 @@ +services: + elasticsearch: + image: elasticsearch:7.16.1 + volumes: + - ./configs/elasticsearch/config.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + - ./docker_volumes/elasticsearch/data:/usr/share/elasticsearch/data + environment: + ES_JAVA_OPTS: "-Xmx512m -Xms512m" + ELASTIC_USERNAME: "elastic" + ELASTIC_PASSWORD: "MyPw123" + discovery.type: single-node + # networks: + # - elk + ports: + - "9200:9200" + - "9300:9300" + + logstash: + image: logstash:7.16.2 + volumes: + - ./configs/logstash/config.yml:/usr/share/logstash/config/logstash.yml:ro + # volumes: + # - ./configs/logstash/config.yml:/usr/share/logstash/config/logstash.yml:ro + # - ./configs/logstash/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro + # - ./configs/logstash/pipelines:/usr/share/logstash/config/pipelines:ro + environment: + LS_JAVA_OPTS: "-Xmx512m -Xms512m" + ports: + - "5044:5044" + - "5000:5000" + - "9600:9600" + # networks: + # - elk + depends_on: + - elasticsearch + + kibana: + image: kibana:7.16.1 + depends_on: + - elasticsearch + volumes: + - ./configs/kibana/config.yml:/usr/share/kibana/config/kibana.yml:ro + # networks: + # - elk + ports: + - "5601:5601" + +# networks: +# elk: +# driver: bridge \ No newline at end of file diff --git a/roles/elk/files/configs/elasticsearch/config.yml b/roles/elk/files/configs/elasticsearch/config.yml new file mode 100644 index 0000000..1bad5d9 --- /dev/null +++ b/roles/elk/files/configs/elasticsearch/config.yml @@ -0,0 +1,5 @@ +cluster.name: "elk" +network.host: 0.0.0.0 # Для корректной работы внутри контейнера + +xpack.security.enabled: true # Для поддержки функционала, обеспечивающего безопасность кластера +xpack.license.self_generated.type: basic # Типа лицензии "basic" для наших нужд хватит с головой \ No newline at end of file diff --git a/roles/elk/files/configs/filebeat/config.yml b/roles/elk/files/configs/filebeat/config.yml new file mode 100644 index 0000000..8a24d1d --- /dev/null +++ b/roles/elk/files/configs/filebeat/config.yml @@ -0,0 +1,15 @@ +filebeat.inputs: + - type: log + enabled: true + # Я запущу filebeat в докере и проброшу логи приложения по данному пути + paths: + - /host_metrics_app/host_metrics_app.log + # В fields мы можем указать дополнительные поля, а затем в logstash вытаскивать их + # и делать какую-нибудь дополнительную работу с логами + fields: + # Название нашего сервиса + service: host_metrics_app + +output.logstash: + # Будьте внимательны при запуске вне докера и вместо logstash укажите правильный адрес хоста с logstash. + hosts: ["logstash:5044"] \ No newline at end of file diff --git a/roles/elk/files/configs/kibana/config.yml b/roles/elk/files/configs/kibana/config.yml new file mode 100644 index 0000000..62787b7 --- /dev/null +++ b/roles/elk/files/configs/kibana/config.yml @@ -0,0 +1,8 @@ +server.name: kibana +server.host: 0.0.0.0 +server.publicBaseUrl: "http://localhost:5601" +monitoring.ui.container.elasticsearch.enabled: true # Для корректного сбора метрик с elastic search, запущенного в контейнере + +elasticsearch.hosts: [ "http://elasticsearch:9200" ] +elasticsearch.username: elastic +elasticsearch.password: MyPw123 \ No newline at end of file diff --git a/roles/elk/files/configs/logstash/config.yml b/roles/elk/files/configs/logstash/config.yml new file mode 100644 index 0000000..7141ff8 --- /dev/null +++ b/roles/elk/files/configs/logstash/config.yml @@ -0,0 +1 @@ +http.host: "0.0.0.0" \ No newline at end of file diff --git a/roles/elk/files/configs/logstash/pipelines/service_stamped_json_logs.conf b/roles/elk/files/configs/logstash/pipelines/service_stamped_json_logs.conf new file mode 100644 index 0000000..1dd0d48 --- /dev/null +++ b/roles/elk/files/configs/logstash/pipelines/service_stamped_json_logs.conf @@ -0,0 +1,42 @@ +# Логи будут прилетать из beats'ов по порту 5044 +input { + beats { + port => 5044 + } +} + +filter { + # Дропаем лог, если он пришел от неизвестного нам сервиса (по желанию) + # Ниже я два раза указал host_metrics_app в списке - это не опечатка. Какого-то лешего в условии, в массиве должно быть минимум 2 элемента. + # Так как приложение у нас одно - просто дублируем + # Поле service у нас появится благодаря конфигурированию Filebeat + if [fields][service] not in ["host_metrics_app", "host_metrics_app"] { + drop {} + } + # Оригинальный json-лог, который был сгенерирован вашим приложением, будет лежать по ключу message + # (из filebeat'а логи прилетают не в чистом виде) + json { + source => "message" + } + # Говорим logstash'у, чтобы в качестве timestamp'а лога он брал именно наш timestamp + # (в моем случае поле asctime в теле сообщения в формате "yyyy-MM-dd HH:mm:ss.SSS" и часовом поясе UTC) + # и затем подтирал поле asctime. + date { + match => ["asctime", "yyyy-MM-dd HH:mm:ss.SSS"] + timezone => "UTC" + target => "@timestamp" + remove_field => ["asctime"] + } +} + +output { + # Отображаем лог в stdout (поиграйтесь и удалите данную строку) + stdout {} + # Пушим лог в elasticsearch, индекс будет создан автоматически по названию сервиса и текущей дате + elasticsearch { + hosts => "elasticsearch:9200" + index => "logs_%{[fields][service]}-%{+YYYY.MM.dd}" + user => "elastic" + password => "MyPw123" + } +} \ No newline at end of file diff --git a/roles/elk/handlers/main.yml b/roles/elk/handlers/main.yml new file mode 100644 index 0000000..b7477ca --- /dev/null +++ b/roles/elk/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for roles/elk diff --git a/roles/elk/meta/main.yml b/roles/elk/meta/main.yml new file mode 100644 index 0000000..ea68190 --- /dev/null +++ b/roles/elk/meta/main.yml @@ -0,0 +1,34 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/roles/elk/tasks/main.yml b/roles/elk/tasks/main.yml new file mode 100644 index 0000000..7d25213 --- /dev/null +++ b/roles/elk/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Creates directory + file: + path: /opt/elk/{{ item }} + state: directory + recurse: yes + loop: + - configs/elasticsearch + - configs/kibana + - configs/logstash + +- name: copy Docker Compose files + copy: + src: files/{{ item }} + dest: /opt/elk/{{ item }} + loop: + - compose.yml + - configs/elasticsearch/config.yml + - configs/kibana/config.yml + - configs/logstash/config.yml + +# use files parameter to use multiple docker-compose.yml files +# mind the _v2 suffix +- name: deploy Docker Compose stack + community.docker.docker_compose_v2: + project_src: /opt/elk + files: + - compose.yml \ No newline at end of file diff --git a/roles/elk/tests/inventory b/roles/elk/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/roles/elk/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/roles/elk/tests/test.yml b/roles/elk/tests/test.yml new file mode 100644 index 0000000..29ee50e --- /dev/null +++ b/roles/elk/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - roles/elk diff --git a/roles/elk/vars/main.yml b/roles/elk/vars/main.yml new file mode 100644 index 0000000..534b113 --- /dev/null +++ b/roles/elk/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for roles/elk