feat: Added k3s, metallb, tried adding traefik

.ansible-lint

@@ -8,3 +8,4 @@ exclude_paths:
 
 skip_list:
   - "fqcn-builtins"
+  - "name[play]"

README.md

@@ -76,7 +76,7 @@ If needed, you can also edit `inventory/my-cluster/group_vars/all.yml` to match
 Start provisioning of the cluster using the following command:
 
 ```bash
-ansible-playbook site.yml -i inventory/my-cluster/hosts.ini
+ansible-playbook site.yml -i inventory/my-cluster/hosts.ini --ask-become-pass --ask-vault-pass
 ```
 
 After deployment control plane will be accessible via virtual ip-address which is defined in inventory/group_vars/all.yml as `apiserver_endpoint`

inventory/my-cluster/group_vars/all.yml

@@ -11,7 +11,7 @@ system_timezone: "Europe/Lisbon"
 flannel_iface: "eth0"
 
 # apiserver_endpoint is virtual ip-address which will be configured on each master
-apiserver_endpoint: "10.0.2.102"
+apiserver_endpoint: "10.0.3.1"
 
 # k3s_token is required  masters can talk together securely
 # this token should be alpha numeric only
@@ -23,7 +23,7 @@ k3s_token: "7qXiuKpSY9uLwdVSNSnEF5RkttoERixCpc2EVJW7vh7Ws4NMN3"
 k3s_node_ip: '{{ ansible_facts[flannel_iface]["ipv4"]["address"] }}'
 
 # Disable the taint manually by setting: k3s_master_taint = false
-k3s_master_taint: "{{ true if groups['node'] | default([]) | length >= 1 else false }}"
+k3s_master_taint: false
 
 # these arguments are recommended for servers as well as agents:
 extra_args: >-
@@ -51,17 +51,36 @@ metal_lb_controller_tag_version: "v0.13.6"
 metal_lb_ip_range: "10.1.1.2-10.1.1.254"
 
 lxc_password: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
+  $ANSIBLE_VAULT;1.1;AES256
-          38303735306236303463613632623161643633663631303931396564346565666236643562316264
+  38303735306236303463613632623161643633663631303931396564346565666236643562316264
-          6533643331306364653564653763356537303932313531350a393261643137636232616335376461
+  6533643331306364653564653763356537303932313531350a393261643137636232616335376461
-          66383966333765626539363561613361393665616333303964373761356166623766663232303063
+  66383966333765626539363561613361393665616333303964373761356166623766663232303063
-          3138353333373935660a383230393330646538303933336366383736643333623663333934663131
+  3138353333373935660a383230393330646538303933336366383736643333623663333934663131
-          3064
+  3064
 
 proxmox_api_password: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
+  $ANSIBLE_VAULT;1.1;AES256
-          35376334616332386130656335663736343337396532663266383934643632363664646631653935
+  35376334616332386130656335663736343337396532663266383934643632363664646631653935
-          6533343936353734343761343465646365616130643130360a316234333036303738663566666364
+  6533343936353734343761343465646365616130643130360a316234333036303738663566666364
-          61653638373830383733323563373862346662363339656632643661336533363162616435616531
+  61653638373830383733323563373862346662363339656632643661336533363162616435616531
-          6331326462356366320a303331616366356333306638386130666538633833623162653934616338
+  6331326462356366320a303331616366356333306638386130666538633833623162653934616338
-          3566
+  3566
+
+traefik_http_auth_user: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  38323532616336373939646333613338626431363466633631343162636235623563393135653231
+  3961383965356631613164303566393632323938386664360a373037616335643662613564353130
+  30353832376431633834336234386161313062373437613132623733646166303639313364373637
+  3933626639646536320a303163353835633837356530613931346165353939363235373561333836
+  39366266303064393334383835323330353934643862323330343337393761353166393333376131
+  33303439393531303031653361393530313930363039646566613831373366326432653634653165
+  313735383263623836363030386531613033
+
+cloudflare_api_key: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  38363363386466666266613930386237623430646531303734613863306530666530376433633339
+  3166373361393839363439326661396136616637393865630a666637366132643035343832666335
+  33376139643533313730313135653064393239316162376339653965313366643565643664666534
+  6631393564333230370a303634643030346166383235643666356164393232643832333238313664
+  38346161306138653735303861646638653830633938326566663136393862643264353437623963
+  3462616435653132623563316231343739333761653365333437

inventory/my-cluster/host_vars/epona

@@ -1,11 +1,12 @@
 ---
-
-mac_addr: de:05:ff:02:47:d7
-hostname: k3s-master-epona
-ip_addr: 10.0.2.2
-lxc_host: 10.0.3.2
-cores: 4
-memory: 8192
-disk: 75
-vmid: 601
 ansible_user: root
+ansible_host: 10.0.2.2
+ansible_ssh_pass: "{{ proxmox_api_password }}"
+ip_addr: 10.0.2.2
+k3s_mac_addr: de:05:ff:02:47:d7
+k3s_hostname: k3s-master-epona
+k3s_lxc_host: 10.0.3.2
+k3s_cores: 4
+k3s_memory: 8192
+k3s_disk: 75
+k3s_vmid: 601

inventory/my-cluster/host_vars/frigate

@@ -1,9 +0,0 @@
----
-
-mac_addr: ee:36:d5:79:f8:ff
-node: urbosa
-lxc_host: 10.0.3.3
-cores: 3
-memory: 2048
-disk: 8
-vmid: 604

inventory/my-cluster/host_vars/k3s-agent-revali

@@ -0,0 +1,4 @@
+---
+ansible_user: root
+ansible_host: 10.0.3.4
+ansible_ssh_pass: "{{ proxmox_api_password }}"

inventory/my-cluster/host_vars/k3s-agent-urbosa

@@ -0,0 +1,4 @@
+---
+ansible_user: root
+ansible_host: 10.0.3.3
+ansible_ssh_pass: "{{ proxmox_api_password }}"

inventory/my-cluster/host_vars/k3s-master-epona

@@ -0,0 +1,4 @@
+---
+ansible_user: root
+ansible_host: 10.0.3.2
+ansible_ssh_pass: "{{ proxmox_api_password }}"

inventory/my-cluster/host_vars/k3s-master-mipha

@@ -0,0 +1,4 @@
+---
+ansible_user: root
+ansible_host: 10.0.3.1
+ansible_ssh_pass: "{{ proxmox_api_password }}"

inventory/my-cluster/host_vars/mipha

@@ -1,11 +1,12 @@
 ---
-
-mac_addr: 0e:a0:ff:8c:70:df
-hostname: k3s-master-mipha
-ip_addr: 10.0.2.3
-lxc_host: 10.0.3.1
-cores: 4
-memory: 6144
-disk: 75
-vmid: 602
 ansible_user: root
+ansible_host: 10.0.2.3
+ansible_ssh_pass: "{{ proxmox_api_password }}"
+ip_addr: 10.0.2.3
+k3s_mac_addr: 0e:a0:ff:8c:70:df
+k3s_hostname: k3s-master-mipha
+k3s_lxc_host: 10.0.3.1
+k3s_cores: 4
+k3s_memory: 6144
+k3s_disk: 75
+k3s_vmid: 602

inventory/my-cluster/host_vars/revali

@@ -1,11 +1,12 @@
 ---
-
-mac_addr: 32:47:89:3f:1a:e2
-hostname: k3s-agent-revali
-ip_addr: 10.0.2.4
-lxc_host: 10.0.3.4
-cores: 2
-memory: 4096
-disk: 200
-vmid: 603
 ansible_user: root
+ansible_host: 10.0.2.4
+ansible_ssh_pass: "{{ proxmox_api_password }}"
+ip_addr: 10.0.2.4
+k3s_mac_addr: 32:47:89:3f:1a:e2
+k3s_hostname: k3s-agent-revali
+k3s_lxc_host: 10.0.3.4
+k3s_cores: 2
+k3s_memory: 4096
+k3s_disk: 200
+k3s_vmid: 603

inventory/my-cluster/host_vars/urbosa

@@ -1,11 +1,12 @@
 ---
-
-mac_addr: ee:36:d5:79:f8:ff
-hostname: k3s-agent-urbosa
-ip_addr: 10.0.2.5
-lxc_host: 10.0.3.3
-cores: 3
-memory: 2048
-disk: 80
-vmid: 604
 ansible_user: root
+ansible_host: 10.0.2.5
+ansible_ssh_pass: "{{ proxmox_api_password }}"
+ip_addr: 10.0.2.5
+k3s_mac_addr: ee:36:d5:79:f8:ff
+k3s_hostname: k3s-agent-urbosa
+k3s_lxc_host: 10.0.3.3
+k3s_cores: 3
+k3s_memory: 2048
+k3s_disk: 80
+k3s_vmid: 604

inventory/my-cluster/hosts.ini

@@ -1,14 +1,17 @@
-[frigate]
-frigate
-
 [master]
-mipha
+k3s-master-mipha
-epona
+k3s-master-epona
 
 [node]
-urbosa
+k3s-agent-urbosa
-revali
+k3s-agent-revali
 
 [k3s_cluster:children]
 master
 node
+
+[baremetal]
+mipha
+epona
+urbosa
+revali

roles/authelia/tasks/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Add helm chart
+  ansible.builtin.shell: |
+    helm repo add authelia https://charts.authelia.com
+    helm repo update

roles/cert-manager/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+# From repository
+- name: Add traefik helm repo
+  kubernetes.core.helm_repository:
+    name: jetstack
+    repo_url: "https://charts.jetstack.io"
+
+- name: Update the repository cache
+  kubernetes.core.helm:
+    kubeconfig: /Users/lino.silva/.kube/config
+    name: dummy
+    namespace: kube-system
+    state: absent
+    update_repo_cache: true
+
+- name: Download cert-manager.crds manifest to the cluster.
+  ansible.builtin.get_url:
+    url: https://github.com/cert-manager/cert-manager/releases/download/v1.9.1/cert-manager.crds.yaml
+    dest: /tmp/cert-manager.crds.yaml
+    mode: "0664"
+
+- name: Apply cert-manager.crds manifest to the cluster.
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    src: /tmp/cert-manager.crds.yaml
+
+- name: Deploy latest version of cert-manager chart inside cert-manager namespace (and create it)
+  kubernetes.core.helm:
+    kubeconfig: /Users/lino.silva/.kube/config
+    name: cert-manager
+    chart_ref: jetstack/cert-manager
+    release_namespace: cert-manager
+    create_namespace: true
+    values: "{{ lookup('template', 'values.yml') | from_yaml }}"
+
+- name: Deploy cert-manager secret - Cloudflare
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'secret-cf-token.yml') | from_yaml }}"
+
+- name: Deploy lets encrypt staging
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'letsencrypt-staging.yml') | from_yaml }}"
+
+- name: Deploy cert-manager staging
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'lino-cooking.staging.yml') | from_yaml }}"
+
+- name: Deploy lets encrypt production
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'letsencrypt-production.yml') | from_yaml }}"
+
+- name: Deploy cert-manager production
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'lino-cooking.prod.yml') | from_yaml }}"

roles/cert-manager/templates/letsencrypt-production.yml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-production
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: letsencrypt@lino.cooking
+    privateKeySecretRef:
+      name: letsencrypt-production
+    solvers:
+      - dns01:
+          cloudflare:
+            email: D5&YbHe&oKx82uuTQ^AfW#$*D8GsDE#K3x^446S^wvH#8T@W2C
+            apiTokenSecretRef:
+              name: cloudflare-token-secret
+              key: cloudflare-token
+        selector:
+          dnsZones:
+            - "lino.cooking"

roles/cert-manager/templates/letsencrypt-staging.yml

@@ -0,0 +1,21 @@
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-staging
+spec:
+  acme:
+    server: https://acme-staging-v02.api.letsencrypt.org/directory
+    email: letsencrypt@lino.cooking
+    privateKeySecretRef:
+      name: letsencrypt-staging
+    solvers:
+      - dns01:
+          cloudflare:
+            email: okulto+cloudflare@gmail.com
+            apiTokenSecretRef:
+              name: cloudflare-api-token-secret
+              key: api-token
+        selector:
+          dnsZones:
+            - "lino.cooking"

roles/cert-manager/templates/lino-cooking.prod.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: lino-cooking
+  namespace: default
+spec:
+  secretName: lino-cooking-tls
+  issuerRef:
+    name: letsencrypt-production
+    kind: ClusterIssuer
+  commonName: "*.lino.cooking"
+  dnsNames:
+    - "lino.cooking"
+    - "*.lino.cooking"

roles/cert-manager/templates/lino-cooking.staging.yml

@@ -0,0 +1,15 @@
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: lino-cooking
+  namespace: default
+spec:
+  secretName: lino-cooking-staging-tls
+  issuerRef:
+    name: letsencrypt-staging
+    kind: ClusterIssuer
+  commonName: "*.lino.cooking"
+  dnsNames:
+    - "lino.cooking"
+    - "*.lino.cooking"

roles/cert-manager/templates/secret-cf-token.yml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudflare-api-token-secret
+  namespace: cert-manager
+type: Opaque
+stringData:
+  api-token: "{{ cloudflare_api_key }}"

roles/cert-manager/templates/values.yml

@@ -0,0 +1,10 @@
+installCRDs: false
+replicaCount: 3
+extraArgs:
+  - --dns01-recursive-nameservers=1.1.1.1:53,9.9.9.9:53
+  - --dns01-recursive-nameservers-only
+podDnsPolicy: None
+podDnsConfig:
+  nameservers:
+    - "1.1.1.1"
+    - "9.9.9.9"

roles/frigate/main.yml

@@ -1,56 +0,0 @@
----
-- name: Create LXC for frigate
-  hosts: localhost
-  gather_facts: yes
-  tasks:
-    - name: Stop container
-      community.general.proxmox:
-        vmid: 200
-        api_user: root@pam
-        api_password: {{ proxmox_api_password }}
-        api_host: 10.0.2.2
-        state: stopped
-      ignore_errors: yes
-
-    - name: Remove container
-      community.general.proxmox:
-        vmid: 200
-        api_user: root@pam
-        api_password: {{ proxmox_api_password }}
-        api_host: 10.0.2.2
-        state: absent
-      ignore_errors: yes
-
-    - name: Create container
-      community.general.proxmox:
-        vmid: 200
-        node: epona
-        api_user: root@pam
-        api_password: {{ proxmox_api_password }}
-        api_host: 10.0.2.2
-        password: {{ lxc_password }}
-        hostname: frigate
-        ostemplate: "hyrule-8tb-nfs:vztmpl/debian-11-standard_11.3-1_amd64.tar.zst"
-        netif: "{'net0':'name=eth0,\
-          gw=10.0.0.1,\
-          ip=10.0.2.14/21,\
-          hwaddr=62:67:fc:7a:58:01,\
-          bridge=vmbr0'}"
-        cores: "2"
-        memory: "2048"
-        unprivileged: no
-        swap: 0
-        searchdomain: "home"
-        onboot: 1
-        disk: local-lvm:8
-        mounts: '{"mp0":"hyrule-8tb-nfs:500,mp=/media/frigate"}'
-        mounts: '{"mp1":"/dev/bus/usb,mp=/dev/bus/usb"}'
-        force: yes
-
-    - name: Start deployment
-      community.general.proxmox:
-        vmid: 200
-        api_user: root@pam
-        api_password: {{ proxmox_api_password }}
-        api_host: 10.0.2.2
-        state: started

roles/k3s/copy-config/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Store kube configuration
+  ansible.builtin.fetch:
+    src: ~/.kube/config
+    dest: ~/.kube/config
+    flat: true
+  when: ansible_hostname == hostvars[groups['master'][0]]['ansible_hostname']

roles/k3s/master/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-
 - name: Clean previous runs of k3s-init
   systemd:
     name: k3s-init
@@ -11,7 +10,7 @@
   failed_when: false
   changed_when: false
   args:
-    warn: false  # The ansible systemd module does not support reset-failed
+    warn: false # The ansible systemd module does not support reset-failed
 
 - name: Create manifests directory on first master
   file:
@@ -62,12 +61,12 @@
 - name: Init cluster inside the transient k3s-init service
   command:
     cmd: "systemd-run -p RestartSec=2 \
-                      -p Restart=on-failure \
+      -p Restart=on-failure \
-                      --unit=k3s-init \
+      --unit=k3s-init \
-                      k3s server {{ server_init_args }}"
+      k3s server {{ server_init_args }}"
     creates: "{{ systemd_dir }}/k3s.service"
   args:
-    warn: false  # The ansible systemd module does not support transient units
+    warn: false # The ansible systemd module does not support transient units
 
 - name: Verification
   block:

roles/k3s/pre/tasks/main.yml

@@ -1,57 +0,0 @@
----
-- name: Get uptime information
-  ansible.builtin.shell: /usr/bin/uptime
-
-- name: Stop containers
-  community.general.proxmox:
-    vmid: "{{ hostvars[item]['vmid'] }}"
-    api_user: root@pam
-    api_password: "{{ proxmox_api_password }}"
-    api_host: 10.0.2.2
-    state: stopped
-  loop: "{{ groups['k3s_cluster'] }}"
-  ignore_errors: true
-
-- name: Remove containers
-  community.general.proxmox:
-    vmid: "{{ hostvars[item]['vmid'] }}"
-    api_user: root@pam
-    api_password: "{{ proxmox_api_password }}"
-    api_host: "{{ hostvars[item]['ip_addr'] }}"
-    state: absent
-  loop: "{{ groups['k3s_cluster'] }}"
-  ignore_errors: true
-
-- name: Create containers
-  community.general.proxmox:
-    vmid: "{{ hostvars[item]['vmid'] }}"
-    node: "{{ item }}"
-    api_user: root@pam
-    api_password: "{{ proxmox_api_password }}"
-    api_host: 10.0.2.2
-    password: "{{ lxc_password }}"
-    hostname: "{{ hostvars[item]['hostname'] }}"
-    ostemplate: "hyrule-8tb-nfs:vztmpl/debian-11-standard_11.3-1_amd64.tar.zst"
-    netif: "{'net0':'name=eth0,\
-      gw=10.0.0.1,\
-      ip={{ hostvars[item]['lxc_host'] }}/21,\
-      hwaddr={{ hostvars[item]['mac_addr'] }},\
-      bridge=vmbr0'}"
-    cores: "{{ hostvars[item]['cores'] }}"
-    memory: "{{ hostvars[item]['memory'] }}"
-    unprivileged: no
-    swap: 0
-    searchdomain: "home"
-    onboot: 1
-    disk: local-lvm:{{ hostvars[item]['disk'] }}
-    force: yes
-  loop: "{{ groups['k3s_cluster'] }}"
-
-- name: Start deployments
-  community.general.proxmox:
-    vmid: "{{ hostvars[item]['vmid'] }}"
-    api_user: root@pam
-    api_password: "{{ proxmox_api_password }}"
-    api_host: 10.0.2.2
-    state: started
-  loop: "{{ groups['k3s_cluster'] }}"

roles/nginx/tasks/main.yml

@@ -0,0 +1,18 @@
+---
+- name: Deploy nginx - deployment
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'deployment.yml') | from_yaml }}"
+
+- name: Deploy nginx - service
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'service.yml') | from_yaml }}"
+
+- name: Deploy nginx - ingress
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'ingress.yml') | from_yaml }}"

roles/nginx/templates/deployment.yml

@@ -1,13 +1,20 @@
 ---
-apiVersion: apps/v1
 kind: Deployment
+apiVersion: apps/v1
 metadata:
   name: nginx
+  namespace: default
+  labels:
+    app: nginx
 spec:
+  replicas: 3
+  progressDeadlineSeconds: 600
+  revisionHistoryLimit: 2
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: nginx
-  replicas: 3
   template:
     metadata:
       labels:
@@ -15,6 +22,4 @@ spec:
     spec:
       containers:
         - name: nginx
-          image: nginx:alpine
+          image: nginx:latest
-          ports:
-            - containerPort: 80

roles/nginx/templates/ingress.yml

@@ -0,0 +1,26 @@
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: nginx
+  namespace: default
+  annotations:
+    kubernetes.io/ingress.class: traefik-external
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`www.nginx.lino.cooking`)
+      kind: Rule
+      services:
+        - name: nginx
+          port: 80
+    - match: Host(`nginx.lino.cooking`)
+      kind: Rule
+      services:
+        - name: nginx
+          port: 80
+      middlewares:
+        - name: default-headers
+  tls:
+    secretName: lino-cooking-tls

roles/nginx/templates/service.yml

@@ -3,11 +3,11 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginx
+  namespace: default
 spec:
-  ipFamilyPolicy: PreferDualStack
   selector:
     app: nginx
   ports:
-    - port: 80
+    - name: http
       targetPort: 80
-  type: LoadBalancer
+      port: 80

roles/prereq/tasks/main.yml

@@ -63,3 +63,14 @@
     path: /etc/sudoers
     validate: "visudo -cf %s"
   when: ansible_os_family == "RedHat"
+
+- name: Copy /etc/rc.local file
+  template:
+    src: "rclocal.j2"
+    dest: "/etc/rc.local"
+    owner: root
+    group: root
+    mode: a+x
+
+- name: Reboot
+  ansible.builtin.reboot:

roles/prereq/templates/rclocal.j2

@@ -0,0 +1,3 @@
+#!/bin/sh -e
+ln -s /dev/console /dev/kmsg
+mount --make-rshared /

roles/provision/cgroup/tasks/main.yml

@@ -0,0 +1,28 @@
+---
+- name: Add cgroup rule
+  ansible.builtin.lineinfile:
+    path: /etc/pve/nodes/{{ ansible_hostname }}/lxc/"{{ hostvars[ansible_hostname]['k3s_vmid'] }}".conf
+    state: present
+    line: lxc.apparmor.profile{{":"}} unconfined
+    validate: /usr/sbin/visudo -cf %s
+
+- name: Add cgroup rule
+  ansible.builtin.lineinfile:
+    path: /etc/pve/nodes/{{ ansible_hostname }}/lxc/"{{ hostvars[ansible_hostname]['k3s_vmid'] }}".conf
+    state: present
+    line: lxc.cap.drop{{":"}}
+    validate: /usr/sbin/visudo -cf %s
+
+- name: Add cgroup rule
+  ansible.builtin.lineinfile:
+    path: /etc/pve/nodes/{{ ansible_hostname }}/lxc/"{{ hostvars[ansible_hostname]['k3s_vmid'] }}".conf
+    state: present
+    line: lxc.mount.auto"{{":"}}" "proc{{":"}}rw sys{{":"}}rw"
+    validate: /usr/sbin/visudo -cf %s
+
+- name: Add cgroup rule
+  ansible.builtin.lineinfile:
+    path: /etc/pve/nodes/{{ ansible_hostname }}/lxc/"{{ hostvars[ansible_hostname]['k3s_vmid'] }}".conf
+    state: present
+    line: lxc.cgroup2.devices.allow{{":"}} c 10{{":"}}200 rwm
+    validate: /usr/sbin/visudo -cf %s

roles/provision/create/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Create containers
+  community.general.proxmox:
+    vmid: "{{ hostvars[item]['k3s_vmid'] }}"
+    node: "{{ item }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    password: "{{ lxc_password }}"
+    hostname: "{{ hostvars[item]['k3s_hostname'] }}"
+    ostemplate: "hyrule-8tb-nfs:vztmpl/debian-11-standard_11.3-1_amd64.tar.zst"
+    netif: "{'net0':'name=eth0,\
+      gw=10.0.0.1,\
+      ip={{ hostvars[item]['k3s_lxc_host'] }}/21,\
+      hwaddr={{ hostvars[item]['k3s_mac_addr'] }},\
+      bridge=vmbr0'}"
+    cores: "{{ hostvars[item]['k3s_cores'] }}"
+    memory: "{{ hostvars[item]['k3s_memory'] }}"
+    unprivileged: no
+    swap: 0
+    searchdomain: "home"
+    onboot: 1
+    features:
+      - nesting=1
+    disk: local-lvm:{{ hostvars[item]['k3s_disk'] }}
+    force: yes
+  loop: "{{ groups['baremetal'] }}"

roles/provision/delete/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Stop containers
+  community.general.proxmox:
+    vmid: "{{ hostvars[item]['k3s_vmid'] }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    state: stopped
+  loop: "{{ groups['baremetal'] }}"
+  ignore_errors: true
+
+- name: Remove containers
+  community.general.proxmox:
+    vmid: "{{ hostvars[item]['k3s_vmid'] }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: "{{ hostvars[item]['ip_addr'] }}"
+    state: absent
+  loop: "{{ groups['baremetal'] }}"
+  ignore_errors: true
+
+- name: Remove .ssh/known_hosts lines
+  ansible.builtin.lineinfile:
+    path: /Users/lino.silva/.ssh/known_hosts
+    state: absent
+    regexp: '^{{ hostvars[item]["k3s_lxc_host"] }}'
+  loop: "{{ groups['baremetal'] }}"

roles/provision/enable-ssh/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+# Unable to use ansible.builtin.lineinfile, because we need to run this through the proxmox host (because SSH is not enabled duh)
+
+- name: Allow SSH into LXC
+  ansible.builtin.command: lxc-attach -n "{{ k3s_vmid }}" -- sed -i "s/#PermitRootLogin prohibit-password/PermitRootLogin yes/g" /etc/ssh/sshd_config
+
+- name: Restart SSH Service
+  ansible.builtin.command: lxc-attach -n "{{ k3s_vmid }}" service ssh restart

roles/provision/pre/tasks/main.yml

@@ -0,0 +1,12 @@
+---
+- name: Allow ipv4 forwarding
+  ansible.builtin.shell: "sysctl net.ipv4.ip_forward=1"
+
+- name: Allow ipv6 forwarding
+  ansible.builtin.shell: "sysctl net.ipv6.conf.all.forwarding=1"
+
+- name: Uncomment ipv4 forward line on /etc/sysctl.conf
+  ansible.builtin.shell: "sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/g' /etc/sysctl.conf"
+
+- name: Uncomment ipv6 forward line on /etc/sysctl.conf
+  ansible.builtin.shell: "sed -i 's/#net.ipv6.conf.all.forwarding=1/net.ipv6.conf.all.forwarding=1/g' /etc/sysctl.conf"

roles/provision/start/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Start deployments
+  community.general.proxmox:
+    vmid: "{{ hostvars[item]['k3s_vmid'] }}"
+    api_user: root@pam
+    api_password: "{{ proxmox_api_password }}"
+    api_host: 10.0.2.2
+    state: started
+  loop: "{{ groups['baremetal'] }}"

roles/traefik/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+# From repository
+- name: Add traefik helm repo
+  kubernetes.core.helm_repository:
+    name: traefik
+    repo_url: "https://helm.traefik.io/traefik"
+
+- name: Update the repository cache
+  kubernetes.core.helm:
+    kubeconfig: /Users/lino.silva/.kube/config
+    name: dummy
+    namespace: kube-system
+    state: absent
+    update_repo_cache: true
+
+- name: Deploy latest version of Traefik chart inside traefik namespace (and create it)
+  kubernetes.core.helm:
+    kubeconfig: /Users/lino.silva/.kube/config
+    name: traefik
+    chart_ref: traefik/traefik
+    release_namespace: traefik
+    create_namespace: true
+    values: "{{ lookup('template', 'values.yml') | from_yaml }}"
+
+- name: Create a Deployment by reading the definition from a local file
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'default-headers.yml') | from_yaml }}"
+
+- name: Create a Deployment by reading the definition from a local file
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'secret-dashboard.yml') | from_yaml }}"
+
+- name: Create a Deployment by reading the definition from a local file
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'dashboard-middleware.yml') | from_yaml }}"
+
+- name: Create a Deployment by reading the definition from a local file
+  kubernetes.core.k8s:
+    kubeconfig: /Users/lino.silva/.kube/config
+    state: present
+    definition: "{{ lookup('template', 'dashboard-ingress.yml') | from_yaml }}"

roles/traefik/templates/dashboard-ingress.yml

@@ -0,0 +1,21 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: traefik-dashboard
+  namespace: traefik
+  annotations:
+    kubernetes.io/ingress.class: traefik-external
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`traefik-dash.lino.cooking`)
+      kind: Rule
+      middlewares:
+        - name: traefik-dashboard-basicauth
+          namespace: traefik
+      services:
+        - name: api@internal
+          kind: TraefikService
+ tls:
+   secretName: lino-cooking-staging-tls

roles/traefik/templates/dashboard-middleware.yml

@@ -0,0 +1,8 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: traefik-dashboard-basicauth
+  namespace: traefik
+spec:
+  basicAuth:
+    secret: traefik-dashboard-auth

roles/traefik/templates/default-headers.yml

@@ -0,0 +1,16 @@
+apiVersion: traefik.containo.us/v1alpha1
+kind: Middleware
+metadata:
+  name: default-headers
+  namespace: default
+spec:
+  headers:
+    browserXssFilter: true
+    contentTypeNosniff: true
+    forceSTSHeader: true
+    stsIncludeSubdomains: true
+    stsPreload: true
+    stsSeconds: 15552000
+    customFrameOptionsValue: SAMEORIGIN
+    customRequestHeaders:
+      X-Forwarded-Proto: https

roles/traefik/templates/secret-dashboard.yml

@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: traefik-dashboard-auth
+  namespace: traefik
+type: Opaque
+data:
+  users: "{{ traefik_http_auth_user }}"

roles/traefik/templates/values.yml

@@ -0,0 +1,50 @@
+globalArguments:
+  - "--global.sendanonymoususage=false"
+  - "--global.checknewversion=false"
+
+additionalArguments:
+  - "--serversTransport.insecureSkipVerify=true"
+  - "--log.level=INFO"
+
+deployment:
+  enabled: true
+  replicas: 3
+  annotations: {}
+  podAnnotations: {}
+  additionalContainers: []
+  initContainers: []
+
+ports:
+  web:
+    redirectTo: websecure
+  websecure:
+    tls:
+      enabled: true
+
+ingressRoute:
+  dashboard:
+    enabled: false
+
+providers:
+  kubernetesCRD:
+    enabled: true
+    ingressClass: traefik-external
+    allowExternalNameServices: true
+  kubernetesIngress:
+    enabled: true
+    publishedService:
+      enabled: false
+    allowExternalNameServices: true
+
+rbac:
+  enabled: true
+
+service:
+  enabled: true
+  type: LoadBalancer
+  annotations: {}
+  labels: {}
+  spec:
+    loadBalancerIP: 10.1.1.3 # this should be an IP in the MetalLB range
+  loadBalancerSourceRanges: []
+  externalIPs: []

site.yml

@@ -1,6 +1,66 @@
 ---
+# - hosts: localhost
+#   gather_facts: no
+#   become: yes
+#   roles:
+#     - role: provision/delete
+
+# - hosts: localhost
+#   gather_facts: no
+#   become: yes
+#   roles:
+#     - role: provision/create
+
+# - hosts: baremetal
+#   gather_facts: yes
+#   become: yes
+#   roles:
+#     - role: provision/pre
+#     - role: provision/cgroup
+
+# - hosts: localhost
+#   gather_facts: no
+#   become: yes
+#   roles:
+#     - role: provision/start
+
+# - hosts: baremetal
+#   gather_facts: yes
+#   become: yes
+#   roles:
+#     - role: provision/enable-ssh
+
+# - hosts: k3s_cluster
+#   gather_facts: yes
+#   become: yes
+#   roles:
+#     - role: prereq
+#     - role: download
+
+# - hosts: master
+#   become: yes
+#   roles:
+#     - role: k3s/master
+
+# - hosts: node
+#   become: yes
+#   roles:
+#     - role: k3s/node
+
+# - hosts: master
+#   become: yes
+#   roles:
+#     - role: k3s/post
+
+# - hosts: master
+#   become: yes
+#   roles:
+#     - role: k3s/copy-config
+
 - hosts: localhost
-  gather_facts: no
   become: yes
   roles:
-    - role: k3s/pre
+    #    - role: traefik
+    #    - role: nginx
+    - role: cert-manager
+#    - role: authelia