feat: Add graylog

inventory/my-cluster/group_vars/all.yml

@@ -398,3 +398,24 @@ heartbeat_user_pwd: !vault |
   33663438636431313163666234323738316131323263643966613136386537303634303537333036
   35623363313731343734613036383731306439663661326465363335633433356533333837303038
   636262643139396462643966383465376263
+
+graylog_root_password: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  33626163626133356532353362323261613661636635653631636334633438303935333239346132
+  6130313764356433383937303862393334376433363538620a623065636637613231343762666435
+  33323933353630626437393936366664656437306532396562353862373862623165616331643064
+  3136396465653563640a316137376233363935666134666161613265383131633766343164613834
+  31393363613538313238326336643331643632363139653339306433373732663030663864613364
+  34333537313535373637613439653032373836646666646561386365616439633035326133313433
+  31303536373338656230616265643038623136643461363437386464633738376661613038623666
+  39376264333362636364
+
+graylog_password_secret: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  63303066396633316531326435656137313261613835356435373037326239306331343065643964
+  6132346632363164666363323565386464316263323336610a656366643761316565343935383262
+  33663561363131323662353364323832616564373264363461616264383563356662366261336138
+  6565613961666561630a313466613931373366636662373137333061333236616539666333363038
+  34663031333031396239386361323539613837383230366165653664396135633563633265376333
+  64346366386136383233613533386437376263653736333437373466353465333339373365376663
+  336434663137336232636133623132653237

inventory/my-cluster/host_vars/graylog

@@ -0,0 +1,6 @@
+---
+
+ansible_user: root
+ansible_host: 10.0.2.41
+ansible_ssh_pass: "{{ proxmox_api_password }}"
+vmid: 641

inventory/my-cluster/host_vars/sidon

@@ -0,0 +1,7 @@
+---
+ansible_user: root
+ansible_host: 10.0.2.2
+ansible_ssh_pass: "{{ proxmox_api_password }}"
+ip_addr: 10.0.2.2
+# interface which will be used for flannel
+flannel_iface: "vmbr0"

inventory/my-cluster/hosts.ini

@@ -2,25 +2,26 @@
 frigate
 swag
 cloudflare-ddns
-dahua-to-mqtt
+#dahua-to-mqtt
-immich
+#immich
 mastodon
 tautulli
 openvpn
-youtube-downloader
+#youtube-downloader
 changedetection
 arr
 mealie
 vaultwarden
 gitea
 paperless
-nextcloud
+#nextcloud
 minio
 outline
-nginx-proxy-manager
+#nginx-proxy-manager
 upsnap
-geoguessr
+#geoguessr
 ghostfolio
+graylog
 
 [baremetal]
 mipha
@@ -28,6 +29,7 @@ purah
 revali
 yuga
 impa
+sidon
 
 [pihole]
 epona-pihole

playbook-41-graylog.yml

@@ -0,0 +1,27 @@
+---
+# - hosts: localhost
+#   become: yes
+#   roles:
+#     - role: 41-graylog/provision/delete
+#       vars:
+#         vmid: 641
+#     - role: 41-graylog/provision/create
+#       vars:
+#         vmid: 641
+#     - role: 41-graylog/provision/start
+#       vars:
+#         vmid: 641
+
+- hosts: sidon
+  become: yes
+  roles:
+    - role: 41-graylog/enable-ssh
+      vars:
+        vmid: 641
+
+- hosts: graylog
+  become: yes
+  roles:
+    - role: 41-graylog/update
+    - role: 41-graylog/install-docker
+    - role: 41-graylog/install-app

playbook-lxc-install-rsyslog.yml

@@ -0,0 +1,6 @@
+---
+- hosts: lxc
+  become: yes
+  roles: 
+    - role: common/lxc/install-rsyslog
+

roles/41-graylog/enable-ssh/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+# Unable to use ansible.builtin.lineinfile, because we need to run this through the proxmox host (because SSH is not enabled duh)
+
+- name: Pause for 10 seconds to wait for SSH server
+  ansible.builtin.pause:
+    seconds: 10
+
+- name: Allow SSH into LXC
+  ansible.builtin.command: lxc-attach -n 641 -- sed -i "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/g" /etc/ssh/sshd_config
+
+- name: Restart SSH Service
+  ansible.builtin.command: lxc-attach -n 641 service ssh restart

roles/41-graylog/install-app/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Create directory for docker-compose
+  ansible.builtin.file:
+    path: /root/docker/
+    state: directory
+    mode: "0755"
+
+- name: Copy .env file
+  template:
+    src: ".env"
+    dest: /root/docker/.env
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Copy docker-compose file
+  template:
+    src: "docker-compose.yml"
+    dest: /root/docker/docker-compose.yml
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Run docker-compose
+  ansible.builtin.shell:
+  args:
+    cmd: docker compose up -d
+    chdir: /root/docker/

roles/41-graylog/install-app/templates/.env

@@ -0,0 +1,14 @@
+# You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters.
+# Generate one by using for example: pwgen -N 1 -s 96
+# ATTENTION: This value must be the same on all Graylog nodes in the cluster.
+# Changing this value after installation will render all user sessions and encrypted values in the database invalid. (e.g. encrypted access tokens)
+GRAYLOG_PASSWORD_SECRET={{ graylog_password_secret }}
+
+# You MUST specify a hash password for the root user (which you only need to initially set up the
+# system and in case you lose connectivity to your authentication backend)
+# This password cannot be changed using the API or via the web interface. If you need to change it,
+# modify it in this file.
+# Create one by using for example: echo -n yourpassword | shasum -a 256
+# and put the resulting hash value into the following line
+# CHANGE THIS!
+GRAYLOG_ROOT_PASSWORD_SHA2={{ graylog_root_password }}

roles/41-graylog/install-app/templates/docker-compose.yml

@@ -0,0 +1,64 @@
+services:
+  mongodb:
+    image: "mongo:5.0"
+    volumes:
+      - "/data/mongodb:/data/db"
+    restart: "on-failure"
+
+  # For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
+  # Please take a look at the README at the top of this repo or the regular docs for more info.
+  datanode:
+    image: "${DATANODE_IMAGE:-graylog/graylog-datanode:6.1}"
+    hostname: "datanode"
+    environment:
+      GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id"
+      # GRAYLOG_DATANODE_PASSWORD_SECRET and GRAYLOG_PASSWORD_SECRET MUST be the same value
+      GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}"
+      GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog"
+    ulimits:
+      memlock:
+        hard: -1
+        soft: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "8999:8999/tcp"   # DataNode API
+      - "9200:9200/tcp"
+      - "9300:9300/tcp"
+    volumes:
+      - "/data/datanode:/var/lib/graylog-datanode"
+    restart: "on-failure"
+
+  graylog:
+    hostname: "server"
+    image: "${GRAYLOG_IMAGE:-graylog/graylog:6.1}"
+    depends_on:
+      mongodb:
+        condition: "service_started"
+    entrypoint: "/usr/bin/tini --  /docker-entrypoint.sh"
+    environment:
+      GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id"
+      # GRAYLOG_DATANODE_PASSWORD_SECRET and GRAYLOG_PASSWORD_SECRET MUST be the same value
+      GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET}"
+      GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2}"
+      GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
+      GRAYLOG_HTTP_EXTERNAL_URI: "https://graylog.lino.cooking/"
+      GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
+    ports:
+    - "5044:5044/tcp"   # Beats
+    - "5140:5140/udp"   # Syslog
+    - "5140:5140/tcp"   # Syslog
+    - "5555:5555/tcp"   # RAW TCP
+    - "5555:5555/udp"   # RAW UDP
+    - "9000:9000/tcp"   # Server API
+    - "12201:12201/tcp" # GELF TCP
+    - "12201:12201/udp" # GELF UDP
+    #- "10000:10000/tcp" # Custom TCP port
+    #- "10000:10000/udp" # Custom UDP port
+    - "13301:13301/tcp" # Forwarder data
+    - "13302:13302/tcp" # Forwarder config
+    volumes:
+      - "/data/data:/usr/share/graylog/data/data"
+      - "/data/journal:/usr/share/graylog/data/journal"
+    restart: "on-failure"

roles/41-graylog/install-docker/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Get convenience script
+  uri:
+    url: "https://get.docker.com"
+    method: GET
+    dest: /tmp/get-docker.sh
+    mode: a+x
+    creates: /tmp/get-docker.sh
+
+- name: Execute script
+  ansible.builtin.shell: /tmp/get-docker.sh
+
+- name: Ensure group "docker" exists
+  ansible.builtin.group:
+    name: docker
+    state: present
+
+- name: Add root user to docker group
+  ansible.builtin.user:
+    name: root
+    groups: docker
+    append: yes
+
+- name: Enable docker on startup
+  ansible.builtin.shell: |
+    systemctl enable docker.service
+    systemctl enable containerd.service

roles/41-graylog/provision/create/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: Create container
+  community.general.proxmox:
+    vmid: 641
+    node: sidon
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    password: "{{ lxc_password }}"
+    hostname: graylog
+    ostemplate: "local:vztmpl/debian-12-standard_12.7-1_amd64.tar.zst"
+    netif: "{'net0':'name=eth0,\
+      gw=10.0.0.1,\
+      ip=10.0.2.41/21,\
+      hwaddr=cc:c6:cf:de:20:41,\
+      bridge=vmbr0'}"
+    cores: 2
+    memory: 2048
+    unprivileged: no
+    swap: 0
+    searchdomain: "home"
+    onboot: 1
+    features:
+      - nesting=1
+      - keyctl=1
+    mounts: '{
+      "mp0":"rest:20,mp=/data"
+      }'
+    disk: rest:10
+    force: yes

roles/41-graylog/provision/delete/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Stop container
+  community.general.proxmox:
+    vmid: "{{ vmid }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    state: stopped
+  ignore_errors: true
+  timeout: 90
+
+- name: Remove containers
+  community.general.proxmox:
+    vmid: "{{ vmid }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    state: absent
+  ignore_errors: true
+  timeout: 90
+
+- name: Remove .ssh/known_hosts lines
+  ansible.builtin.lineinfile:
+    path: /Users/lino.silva/.ssh/known_hosts
+    state: absent
+    regexp: "^10.0.2.41"
+

roles/41-graylog/provision/start/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Start deployments
+  community.general.proxmox:
+    vmid: "{{ vmid }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    state: started

roles/41-graylog/update/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Update all packages to their latest version
+  become: true
+  ansible.builtin.apt:
+    update_cache: yes
+    upgrade: full

roles/common/lxc/install-rsyslog/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Install rsyslog
+  become: true
+  ansible.builtin.apt:
+    name: rsyslog
+    state: present
+    update_cache: yes
+
+- name: Create directory for configuration files
+  ansible.builtin.file:
+    path: /etc/rsyslog.d/
+    state: directory
+    mode: "0755"
+
+- name: Copy rsyslog configuration and create rsyslog.d folder if not exists
+  template:
+    src: "10-graylog.conf.j2"
+    dest: /etc/rsyslog.d/10-graylog.conf
+    owner: root
+    group: root
+    mode: 0755
+
+
+- name: Restart rsyslog service
+  service:
+    name: rsyslog
+    state: restarted
+    enabled: yes

roles/common/lxc/install-rsyslog/templates/10-graylog.conf.j2

@@ -0,0 +1 @@
+*.* action(type="omfwd" target="10.0.2.41" port="5140" protocol="udp")

roles/swag/install-app/templates/graylog.subdomain.conf

@@ -0,0 +1,49 @@
+## Version 2024/07/16
+# make sure that your graylog container is named graylog
+# make sure that your dns has a cname set for graylog
+# Ensure the upstream_port matches your GRAYLOG_HTTP_BIND_ADDRESS port
+# This conf assumes GRAYLOG_HTTP_BIND_ADDRESS=0.0.0.0:9000
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+
+    server_name graylog.*;
+
+    include /config/nginx/ssl.conf;
+
+    client_max_body_size 0;
+
+    # enable for ldap auth (requires ldap-location.conf in the location block)
+    #include /config/nginx/ldap-server.conf;
+
+    # enable for Authelia (requires authelia-location.conf in the location block)
+    include /config/nginx/authelia-server.conf;
+
+    # enable for Authentik (requires authentik-location.conf in the location block)
+    #include /config/nginx/authentik-server.conf;
+
+    location / {
+        # enable the next two lines for http auth
+        #auth_basic "Restricted";
+        #auth_basic_user_file /config/nginx/.htpasswd;
+
+        # enable for ldap auth (requires ldap-server.conf in the server block)
+        #include /config/nginx/ldap-location.conf;
+
+        # enable for Authelia (requires authelia-server.conf in the server block)
+        include /config/nginx/authelia-location.conf;
+
+        # enable for Authentik (requires authentik-server.conf in the server block)
+        #include /config/nginx/authentik-location.conf;
+
+        include /config/nginx/proxy.conf;
+        include /config/nginx/resolver.conf;
+        set $upstream_app 10.0.2.41;
+        set $upstream_port 9000;
+        set $upstream_proto http;
+        proxy_pass $upstream_proto://$upstream_app:$upstream_port;
+
+    }
+
+}