From e0eeab18d064baf0efd28968d4a3aa250d01619a Mon Sep 17 00:00:00 2001 From: Lino Silva Date: Wed, 12 Mar 2025 15:05:50 +0000 Subject: [PATCH] feat: Add graylog --- inventory/my-cluster/group_vars/all.yml | 21 ++++++ inventory/my-cluster/host_vars/graylog | 6 ++ inventory/my-cluster/host_vars/sidon | 7 ++ inventory/my-cluster/hosts.ini | 14 ++-- playbook-41-graylog.yml | 27 ++++++++ playbook-lxc-install-rsyslog.yml | 6 ++ roles/41-graylog/enable-ssh/tasks/main.yml | 12 ++++ roles/41-graylog/install-app/tasks/main.yml | 28 ++++++++ roles/41-graylog/install-app/templates/.env | 14 ++++ .../install-app/templates/docker-compose.yml | 64 +++++++++++++++++++ .../41-graylog/install-docker/tasks/main.yml | 27 ++++++++ .../provision/create/tasks/main.yml | 30 +++++++++ .../provision/delete/tasks/main.yml | 27 ++++++++ .../41-graylog/provision/start/tasks/main.yml | 8 +++ roles/41-graylog/update/tasks/main.yml | 6 ++ .../common/lxc/install-rsyslog/tasks/main.yml | 28 ++++++++ .../templates/10-graylog.conf.j2 | 1 + .../templates/graylog.subdomain.conf | 49 ++++++++++++++ 18 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 inventory/my-cluster/host_vars/graylog create mode 100644 inventory/my-cluster/host_vars/sidon create mode 100644 playbook-41-graylog.yml create mode 100644 playbook-lxc-install-rsyslog.yml create mode 100644 roles/41-graylog/enable-ssh/tasks/main.yml create mode 100644 roles/41-graylog/install-app/tasks/main.yml create mode 100644 roles/41-graylog/install-app/templates/.env create mode 100644 roles/41-graylog/install-app/templates/docker-compose.yml create mode 100644 roles/41-graylog/install-docker/tasks/main.yml create mode 100644 roles/41-graylog/provision/create/tasks/main.yml create mode 100644 roles/41-graylog/provision/delete/tasks/main.yml create mode 100644 roles/41-graylog/provision/start/tasks/main.yml create mode 100644 roles/41-graylog/update/tasks/main.yml create mode 100644 roles/common/lxc/install-rsyslog/tasks/main.yml create mode 100644 roles/common/lxc/install-rsyslog/templates/10-graylog.conf.j2 create mode 100755 roles/swag/install-app/templates/graylog.subdomain.conf diff --git a/inventory/my-cluster/group_vars/all.yml b/inventory/my-cluster/group_vars/all.yml index 879bdf3..2413389 100644 --- a/inventory/my-cluster/group_vars/all.yml +++ b/inventory/my-cluster/group_vars/all.yml @@ -398,3 +398,24 @@ heartbeat_user_pwd: !vault | 33663438636431313163666234323738316131323263643966613136386537303634303537333036 35623363313731343734613036383731306439663661326465363335633433356533333837303038 636262643139396462643966383465376263 + +graylog_root_password: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 33626163626133356532353362323261613661636635653631636334633438303935333239346132 + 6130313764356433383937303862393334376433363538620a623065636637613231343762666435 + 33323933353630626437393936366664656437306532396562353862373862623165616331643064 + 3136396465653563640a316137376233363935666134666161613265383131633766343164613834 + 31393363613538313238326336643331643632363139653339306433373732663030663864613364 + 34333537313535373637613439653032373836646666646561386365616439633035326133313433 + 31303536373338656230616265643038623136643461363437386464633738376661613038623666 + 39376264333362636364 + +graylog_password_secret: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 63303066396633316531326435656137313261613835356435373037326239306331343065643964 + 6132346632363164666363323565386464316263323336610a656366643761316565343935383262 + 33663561363131323662353364323832616564373264363461616264383563356662366261336138 + 6565613961666561630a313466613931373366636662373137333061333236616539666333363038 + 34663031333031396239386361323539613837383230366165653664396135633563633265376333 + 64346366386136383233613533386437376263653736333437373466353465333339373365376663 + 336434663137336232636133623132653237 diff --git a/inventory/my-cluster/host_vars/graylog b/inventory/my-cluster/host_vars/graylog new file mode 100644 index 0000000..cb4843f --- /dev/null +++ b/inventory/my-cluster/host_vars/graylog @@ -0,0 +1,6 @@ +--- + +ansible_user: root +ansible_host: 10.0.2.41 +ansible_ssh_pass: "{{ proxmox_api_password }}" +vmid: 641 diff --git a/inventory/my-cluster/host_vars/sidon b/inventory/my-cluster/host_vars/sidon new file mode 100644 index 0000000..2c0d88f --- /dev/null +++ b/inventory/my-cluster/host_vars/sidon @@ -0,0 +1,7 @@ +--- +ansible_user: root +ansible_host: 10.0.2.2 +ansible_ssh_pass: "{{ proxmox_api_password }}" +ip_addr: 10.0.2.2 +# interface which will be used for flannel +flannel_iface: "vmbr0" diff --git a/inventory/my-cluster/hosts.ini b/inventory/my-cluster/hosts.ini index 9dc1603..bba6d3d 100644 --- a/inventory/my-cluster/hosts.ini +++ b/inventory/my-cluster/hosts.ini @@ -2,25 +2,26 @@ frigate swag cloudflare-ddns -dahua-to-mqtt -immich +#dahua-to-mqtt +#immich mastodon tautulli openvpn -youtube-downloader +#youtube-downloader changedetection arr mealie vaultwarden gitea paperless -nextcloud +#nextcloud minio outline -nginx-proxy-manager +#nginx-proxy-manager upsnap -geoguessr +#geoguessr ghostfolio +graylog [baremetal] mipha @@ -28,6 +29,7 @@ purah revali yuga impa +sidon [pihole] epona-pihole diff --git a/playbook-41-graylog.yml b/playbook-41-graylog.yml new file mode 100644 index 0000000..a864b70 --- /dev/null +++ b/playbook-41-graylog.yml @@ -0,0 +1,27 @@ +--- +# - hosts: localhost +# become: yes +# roles: +# - role: 41-graylog/provision/delete +# vars: +# vmid: 641 +# - role: 41-graylog/provision/create +# vars: +# vmid: 641 +# - role: 41-graylog/provision/start +# vars: +# vmid: 641 + +- hosts: sidon + become: yes + roles: + - role: 41-graylog/enable-ssh + vars: + vmid: 641 + +- hosts: graylog + become: yes + roles: + - role: 41-graylog/update + - role: 41-graylog/install-docker + - role: 41-graylog/install-app diff --git a/playbook-lxc-install-rsyslog.yml b/playbook-lxc-install-rsyslog.yml new file mode 100644 index 0000000..df81aac --- /dev/null +++ b/playbook-lxc-install-rsyslog.yml @@ -0,0 +1,6 @@ +--- +- hosts: lxc + become: yes + roles: + - role: common/lxc/install-rsyslog + diff --git a/roles/41-graylog/enable-ssh/tasks/main.yml b/roles/41-graylog/enable-ssh/tasks/main.yml new file mode 100644 index 0000000..f2f7925 --- /dev/null +++ b/roles/41-graylog/enable-ssh/tasks/main.yml @@ -0,0 +1,12 @@ +--- +# Unable to use ansible.builtin.lineinfile, because we need to run this through the proxmox host (because SSH is not enabled duh) + +- name: Pause for 10 seconds to wait for SSH server + ansible.builtin.pause: + seconds: 10 + +- name: Allow SSH into LXC + ansible.builtin.command: lxc-attach -n 641 -- sed -i "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/g" /etc/ssh/sshd_config + +- name: Restart SSH Service + ansible.builtin.command: lxc-attach -n 641 service ssh restart diff --git a/roles/41-graylog/install-app/tasks/main.yml b/roles/41-graylog/install-app/tasks/main.yml new file mode 100644 index 0000000..98d60c2 --- /dev/null +++ b/roles/41-graylog/install-app/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Create directory for docker-compose + ansible.builtin.file: + path: /root/docker/ + state: directory + mode: "0755" + +- name: Copy .env file + template: + src: ".env" + dest: /root/docker/.env + owner: root + group: root + mode: 0755 + +- name: Copy docker-compose file + template: + src: "docker-compose.yml" + dest: /root/docker/docker-compose.yml + owner: root + group: root + mode: 0755 + +- name: Run docker-compose + ansible.builtin.shell: + args: + cmd: docker compose up -d + chdir: /root/docker/ diff --git a/roles/41-graylog/install-app/templates/.env b/roles/41-graylog/install-app/templates/.env new file mode 100644 index 0000000..2d03af2 --- /dev/null +++ b/roles/41-graylog/install-app/templates/.env @@ -0,0 +1,14 @@ +# You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters. +# Generate one by using for example: pwgen -N 1 -s 96 +# ATTENTION: This value must be the same on all Graylog nodes in the cluster. +# Changing this value after installation will render all user sessions and encrypted values in the database invalid. (e.g. encrypted access tokens) +GRAYLOG_PASSWORD_SECRET={{ graylog_password_secret }} + +# You MUST specify a hash password for the root user (which you only need to initially set up the +# system and in case you lose connectivity to your authentication backend) +# This password cannot be changed using the API or via the web interface. If you need to change it, +# modify it in this file. +# Create one by using for example: echo -n yourpassword | shasum -a 256 +# and put the resulting hash value into the following line +# CHANGE THIS! +GRAYLOG_ROOT_PASSWORD_SHA2={{ graylog_root_password }} \ No newline at end of file diff --git a/roles/41-graylog/install-app/templates/docker-compose.yml b/roles/41-graylog/install-app/templates/docker-compose.yml new file mode 100644 index 0000000..9c96f2a --- /dev/null +++ b/roles/41-graylog/install-app/templates/docker-compose.yml @@ -0,0 +1,64 @@ +services: + mongodb: + image: "mongo:5.0" + volumes: + - "/data/mongodb:/data/db" + restart: "on-failure" + + # For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch. + # Please take a look at the README at the top of this repo or the regular docs for more info. + datanode: + image: "${DATANODE_IMAGE:-graylog/graylog-datanode:6.1}" + hostname: "datanode" + environment: + GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id" + # GRAYLOG_DATANODE_PASSWORD_SECRET and GRAYLOG_PASSWORD_SECRET MUST be the same value + GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}" + GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ulimits: + memlock: + hard: -1 + soft: -1 + nofile: + soft: 65536 + hard: 65536 + ports: + - "8999:8999/tcp" # DataNode API + - "9200:9200/tcp" + - "9300:9300/tcp" + volumes: + - "/data/datanode:/var/lib/graylog-datanode" + restart: "on-failure" + + graylog: + hostname: "server" + image: "${GRAYLOG_IMAGE:-graylog/graylog:6.1}" + depends_on: + mongodb: + condition: "service_started" + entrypoint: "/usr/bin/tini -- /docker-entrypoint.sh" + environment: + GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id" + # GRAYLOG_DATANODE_PASSWORD_SECRET and GRAYLOG_PASSWORD_SECRET MUST be the same value + GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}" + GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2}" + GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000" + GRAYLOG_HTTP_EXTERNAL_URI: "https://graylog.lino.cooking/" + GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog" + ports: + - "5044:5044/tcp" # Beats + - "5140:5140/udp" # Syslog + - "5140:5140/tcp" # Syslog + - "5555:5555/tcp" # RAW TCP + - "5555:5555/udp" # RAW UDP + - "9000:9000/tcp" # Server API + - "12201:12201/tcp" # GELF TCP + - "12201:12201/udp" # GELF UDP + #- "10000:10000/tcp" # Custom TCP port + #- "10000:10000/udp" # Custom UDP port + - "13301:13301/tcp" # Forwarder data + - "13302:13302/tcp" # Forwarder config + volumes: + - "/data/data:/usr/share/graylog/data/data" + - "/data/journal:/usr/share/graylog/data/journal" + restart: "on-failure" diff --git a/roles/41-graylog/install-docker/tasks/main.yml b/roles/41-graylog/install-docker/tasks/main.yml new file mode 100644 index 0000000..d5baba9 --- /dev/null +++ b/roles/41-graylog/install-docker/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Get convenience script + uri: + url: "https://get.docker.com" + method: GET + dest: /tmp/get-docker.sh + mode: a+x + creates: /tmp/get-docker.sh + +- name: Execute script + ansible.builtin.shell: /tmp/get-docker.sh + +- name: Ensure group "docker" exists + ansible.builtin.group: + name: docker + state: present + +- name: Add root user to docker group + ansible.builtin.user: + name: root + groups: docker + append: yes + +- name: Enable docker on startup + ansible.builtin.shell: | + systemctl enable docker.service + systemctl enable containerd.service diff --git a/roles/41-graylog/provision/create/tasks/main.yml b/roles/41-graylog/provision/create/tasks/main.yml new file mode 100644 index 0000000..97daecc --- /dev/null +++ b/roles/41-graylog/provision/create/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: Create container + community.general.proxmox: + vmid: 641 + node: sidon + api_user: root@pam + api_password: "{{ proxmox_api_password }}" + api_host: 10.0.2.2 + password: "{{ lxc_password }}" + hostname: graylog + ostemplate: "local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst" + netif: "{'net0':'name=eth0,\ + gw=10.0.0.1,\ + ip=10.0.2.41/21,\ + hwaddr=cc:c6:cf:de:20:41,\ + bridge=vmbr0'}" + cores: 2 + memory: 2048 + unprivileged: no + swap: 0 + searchdomain: "home" + onboot: 1 + features: + - nesting=1 + - keyctl=1 + mounts: '{ + "mp0":"rest:20,mp=/data" + }' + disk: rest:10 + force: yes diff --git a/roles/41-graylog/provision/delete/tasks/main.yml b/roles/41-graylog/provision/delete/tasks/main.yml new file mode 100644 index 0000000..213dd6f --- /dev/null +++ b/roles/41-graylog/provision/delete/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Stop container + community.general.proxmox: + vmid: "{{ vmid }}" + api_user: root@pam + api_password: "{{ proxmox_api_password }}" + api_host: 10.0.2.2 + state: stopped + ignore_errors: true + timeout: 90 + +- name: Remove containers + community.general.proxmox: + vmid: "{{ vmid }}" + api_user: root@pam + api_password: "{{ proxmox_api_password }}" + api_host: 10.0.2.2 + state: absent + ignore_errors: true + timeout: 90 + +- name: Remove .ssh/known_hosts lines + ansible.builtin.lineinfile: + path: /Users/lino.silva/.ssh/known_hosts + state: absent + regexp: "^10.0.2.41" + diff --git a/roles/41-graylog/provision/start/tasks/main.yml b/roles/41-graylog/provision/start/tasks/main.yml new file mode 100644 index 0000000..de86b9b --- /dev/null +++ b/roles/41-graylog/provision/start/tasks/main.yml @@ -0,0 +1,8 @@ +--- +- name: Start deployments + community.general.proxmox: + vmid: "{{ vmid }}" + api_user: root@pam + api_password: "{{ proxmox_api_password }}" + api_host: 10.0.2.2 + state: started diff --git a/roles/41-graylog/update/tasks/main.yml b/roles/41-graylog/update/tasks/main.yml new file mode 100644 index 0000000..8227bf4 --- /dev/null +++ b/roles/41-graylog/update/tasks/main.yml @@ -0,0 +1,6 @@ +--- +- name: Update all packages to their latest version + become: true + ansible.builtin.apt: + update_cache: yes + upgrade: full diff --git a/roles/common/lxc/install-rsyslog/tasks/main.yml b/roles/common/lxc/install-rsyslog/tasks/main.yml new file mode 100644 index 0000000..fb8f600 --- /dev/null +++ b/roles/common/lxc/install-rsyslog/tasks/main.yml @@ -0,0 +1,28 @@ +--- +- name: Install rsyslog + become: true + ansible.builtin.apt: + name: rsyslog + state: present + update_cache: yes + +- name: Create directory for configuration files + ansible.builtin.file: + path: /etc/rsyslog.d/ + state: directory + mode: "0755" + +- name: Copy rsyslog configuration and create rsyslog.d folder if not exists + template: + src: "10-graylog.conf.j2" + dest: /etc/rsyslog.d/10-graylog.conf + owner: root + group: root + mode: 0755 + + +- name: Restart rsyslog service + service: + name: rsyslog + state: restarted + enabled: yes diff --git a/roles/common/lxc/install-rsyslog/templates/10-graylog.conf.j2 b/roles/common/lxc/install-rsyslog/templates/10-graylog.conf.j2 new file mode 100644 index 0000000..77fd451 --- /dev/null +++ b/roles/common/lxc/install-rsyslog/templates/10-graylog.conf.j2 @@ -0,0 +1 @@ +*.* action(type="omfwd" target="10.0.2.41" port="5140" protocol="udp") \ No newline at end of file diff --git a/roles/swag/install-app/templates/graylog.subdomain.conf b/roles/swag/install-app/templates/graylog.subdomain.conf new file mode 100755 index 0000000..16dc8b4 --- /dev/null +++ b/roles/swag/install-app/templates/graylog.subdomain.conf @@ -0,0 +1,49 @@ +## Version 2024/07/16 +# make sure that your graylog container is named graylog +# make sure that your dns has a cname set for graylog +# Ensure the upstream_port matches your GRAYLOG_HTTP_BIND_ADDRESS port +# This conf assumes GRAYLOG_HTTP_BIND_ADDRESS=0.0.0.0:9000 + +server { + listen 443 ssl; + listen [::]:443 ssl; + + server_name graylog.*; + + include /config/nginx/ssl.conf; + + client_max_body_size 0; + + # enable for ldap auth (requires ldap-location.conf in the location block) + #include /config/nginx/ldap-server.conf; + + # enable for Authelia (requires authelia-location.conf in the location block) + include /config/nginx/authelia-server.conf; + + # enable for Authentik (requires authentik-location.conf in the location block) + #include /config/nginx/authentik-server.conf; + + location / { + # enable the next two lines for http auth + #auth_basic "Restricted"; + #auth_basic_user_file /config/nginx/.htpasswd; + + # enable for ldap auth (requires ldap-server.conf in the server block) + #include /config/nginx/ldap-location.conf; + + # enable for Authelia (requires authelia-server.conf in the server block) + include /config/nginx/authelia-location.conf; + + # enable for Authentik (requires authentik-server.conf in the server block) + #include /config/nginx/authentik-location.conf; + + include /config/nginx/proxy.conf; + include /config/nginx/resolver.conf; + set $upstream_app 10.0.2.41; + set $upstream_port 9000; + set $upstream_proto http; + proxy_pass $upstream_proto://$upstream_app:$upstream_port; + + } + +} \ No newline at end of file