Removing accidental tear-down step that is clearly a typo (#117)

Co-authored-by: Techno Tim <timothystewart6@gmail.com>

.github/download-boxes.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# download-boxes.sh
+# Check all molecule.yml files for required Vagrant boxes and download the ones that are not
+# already present on the system.
+
+set -euo pipefail
+
+GIT_ROOT=$(git rev-parse --show-toplevel)
+PROVIDER=virtualbox
+
+# Read all boxes for all platforms from the "molecule.yml" files
+all_boxes=$(cat "${GIT_ROOT}"/molecule/*/molecule.yml |
+    yq -r '.platforms[].box' |         # Read the "box" property of each node under "platforms"
+    grep --invert-match --regexp=--- | # Filter out file separators
+    sort |
+    uniq)
+
+# Read the boxes that are currently present on the system (for the current provider)
+present_boxes=$(
+    (vagrant box list |
+        grep "${PROVIDER}" | # Filter by boxes available for the current provider
+        awk '{print $1;}' |  # The box name is the first word in each line
+        sort |
+        uniq) ||
+        echo "" # In case any of these commands fails, just use an empty list
+)
+
+# The boxes that we need to download are the ones present in $all_boxes, but not $present_boxes.
+download_boxes=$(comm -2 -3 <(echo "${all_boxes}") <(echo "${present_boxes}"))
+
+# Actually download the necessary boxes
+if [ -n "${download_boxes}" ]; then
+    echo "${download_boxes}" | while IFS= read -r box; do
+        vagrant box add --provider "${PROVIDER}" "${box}"
+    done
+fi

.github/workflows/lint.yml

@@ -18,10 +18,21 @@ jobs:
       - name: Set up Python 3.x
         uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 #4.0.2
         with:
-          python-version: '3.x'
+          python-version: "3.x"
 
-      - name: Install test dependencies
+      - name: Install dependencies
-        run: pip3 install yamllint ansible-lint ansible
+        run: |
+          echo "::group::Upgrade pip"
+          python3 -m pip install --upgrade pip
+          echo "::endgroup::"
+
+          echo "::group::Install Python requirements from requirements.txt"
+          python3 -m pip install -r requirements.txt
+          echo "::endgroup::"
+
+          echo "::group::Install Ansible role requirements from collections/requirements.yml"
+          ansible-galaxy install -r collections/requirements.yml
+          echo "::endgroup::"
 
       - name: Run yamllint
         run: yamllint .

.github/workflows/test.yml

@@ -5,7 +5,8 @@ on:
   push:
     branches:
       - master
-
+    paths-ignore:
+      - '**/README.md'
 jobs:
   molecule:
     name: Molecule
@@ -42,6 +43,12 @@ jobs:
           restore-keys: |
             vagrant-boxes
 
+      - name: Download Vagrant boxes for all scenarios
+        # To save some cache space, all scenarios share the same cache key.
+        # On the other hand, this means that the cache contents should be
+        # the same across all scenarios. This step ensures that.
+        run: ./.github/download-boxes.sh
+
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v2
         with:
@@ -58,6 +65,8 @@ jobs:
           ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
           ANSIBLE_SSH_RETRIES: 4
           ANSIBLE_TIMEOUT: 60
+          PY_COLORS: 1
+          ANSIBLE_FORCE_COLOR: 1
 
       - name: Upload log files
         if: always() # do this even if a step before has failed

README.md

@@ -16,9 +16,9 @@ If you want more context on how this works, see:
 
 Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a HA Kubernetes cluster on machines running:
 
-- [X] Debian
+- [x] Debian (tested on version 11)
-- [X] Ubuntu
+- [x] Ubuntu (tested on version 22.04)
-- [X] CentOS
+- [x] Rocky (tested on version 9)
 
 on processor architecture:
 
@@ -29,9 +29,13 @@ on processor architecture:
 ## ✅ System requirements
 
 - Deployment environment must have Ansible 2.4.0+.  If you need a quick primer on Ansible [you can check out my docs and setting up Ansible](https://docs.technotim.live/posts/ansible-automation/).
-  Furthermore, the [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment.
+
+- [`netaddr` package](https://pypi.org/project/netaddr/) must be available to Ansible. If you have installed Ansible via apt, this is already taken care of. If you have installed Ansible via `pip`, make sure to install `netaddr` into the respective virtual environment.
+
 - `server` and `agent` nodes should have passwordless SSH access, if not you can supply arguments to provide credentials `--ask-pass --ask-become-pass` to each command.
 
+- You will also need to install collections that this playbook uses by running `ansible-galaxy collection install -r ./collections/requirements.yml`
+
 ## 🚀 Getting Started
 
 ### 🍴 Preparation
@@ -110,9 +114,7 @@ You can find more information about it [here](molecule/README.md).
 
 ## Thanks 🤝
 
-This repo is really standing on the shoulders of giants.  To all those who have contributed.
+This repo is really standing on the shoulders of giants. Thank you to all those who have contributed and tanks to these repos for code and ideas:
-
-Thanks to these repos for code and ideas:
 
 - [k3s-io/k3s-ansible](https://github.com/k3s-io/k3s-ansible)
 - [geerlingguy/turing-pi-cluster](https://github.com/geerlingguy/turing-pi-cluster)

inventory/sample/group_vars/all.yml

@@ -1,5 +1,5 @@
 ---
-k3s_version: v1.24.4+k3s1
+k3s_version: v1.24.6+k3s1
 # this is the user that has ssh access to these machines
 ansible_user: ansibleuser
 systemd_dir: /etc/systemd/system
@@ -22,25 +22,30 @@ k3s_token: "some-SUPER-DEDEUPER-secret-password"
 # it for each of your hosts, though.
 k3s_node_ip: '{{ ansible_facts[flannel_iface]["ipv4"]["address"] }}'
 
+# Disable the taint manually by setting: k3s_master_taint = false
+k3s_master_taint: "{{ true if groups['node'] | default([]) | length >= 1 else false }}"
+
 # these arguments are recommended for servers as well as agents:
 extra_args: >-
   --flannel-iface={{ flannel_iface }}
   --node-ip={{ k3s_node_ip }}
 
-# change these to your liking, the only required one is --disable servicelb
+# change these to your liking, the only required are: --disable servicelb, --tls-san {{ apiserver_endpoint }}
 extra_server_args: >-
   {{ extra_args }}
+  {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
+  --tls-san {{ apiserver_endpoint }}
   --disable servicelb
   --disable traefik
 extra_agent_args: >-
   {{ extra_args }}
 
 # image tag for kube-vip
-kube_vip_tag_version: "v0.5.0"
+kube_vip_tag_version: "v0.5.5"
 
 # image tag for metal lb
-metal_lb_speaker_tag_version: "v0.13.5"
+metal_lb_speaker_tag_version: "v0.13.6"
-metal_lb_controller_tag_version: "v0.13.5"
+metal_lb_controller_tag_version: "v0.13.6"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"

molecule/default/molecule.yml

@@ -3,43 +3,52 @@ dependency:
   name: galaxy
 driver:
   name: vagrant
-platforms:
+.platform_presets:
   - &control
-    name: control1
-    box: generic/ubuntu2204
     memory: 2048
     cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+  - &node
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
+  - &debian
+    box: generic/debian11
+  - &rocky
+    box: generic/rocky9
+  - &ubuntu
+    box: generic/ubuntu2204
     config_options:
       # We currently can not use public-key based authentication on Ubuntu 22.04,
       # see: https://github.com/chef/bento/issues/1405
       ssh.username: "vagrant"
       ssh.password: "vagrant"
-    groups:
+platforms:
-      - k3s_cluster
+  - <<: [*control, *ubuntu]
-      - master
+    name: control1
     interfaces:
       - network_name: private_network
         ip: 192.168.30.38
-  - <<: *control
+  - <<: [*control, *debian]
     name: control2
     interfaces:
       - network_name: private_network
         ip: 192.168.30.39
-  - <<: *control
+  - <<: [*control, *rocky]
     name: control3
     interfaces:
       - network_name: private_network
         ip: 192.168.30.40
-  - &node
+  - <<: [*node, *ubuntu]
-    <<: *control
     name: node1
-    groups:
-      - k3s_cluster
-      - node
     interfaces:
       - network_name: private_network
         ip: 192.168.30.41
-  - <<: *node
+  - <<: [*node, *rocky]
     name: node2
     interfaces:
       - network_name: private_network

molecule/default/prepare.yml

@@ -0,0 +1,22 @@
+---
+- name: Apply overrides
+  ansible.builtin.import_playbook: >-
+    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
+
+- name: Network setup
+  hosts: all
+  tasks:
+    - name: Disable firewalld
+      when: ansible_distribution == "Rocky"
+      # Rocky Linux comes with firewalld enabled. It blocks some of the network
+      # connections needed for our k3s cluster. For our test setup, we just disable
+      # it since the VM host's firewall is still active for connections to and from
+      # the Internet.
+      # When building your own cluster, please DO NOT blindly copy this. Instead,
+      # please create a custom firewall configuration that fits your network design
+      # and security needs.
+      ansible.builtin.systemd:
+        name: firewalld
+        enabled: no
+        state: stopped
+      become: true

molecule/ipv6/molecule.yml

@@ -3,28 +3,34 @@ dependency:
   name: galaxy
 driver:
   name: vagrant
-platforms:
+.platform_presets:
   - &control
-    name: control1
-    box: generic/ubuntu2204
     memory: 2048
     cpus: 2
+    groups:
+      - k3s_cluster
+      - master
+  - &node
+    memory: 2048
+    cpus: 2
+    groups:
+      - k3s_cluster
+      - node
+  - &ubuntu
+    box: generic/ubuntu2204
     config_options:
       # We currently can not use public-key based authentication on Ubuntu 22.04,
       # see: https://github.com/chef/bento/issues/1405
       ssh.username: "vagrant"
       ssh.password: "vagrant"
-    groups:
+platforms:
-      - k3s_cluster
+  - <<: [*control, *ubuntu]
-      - master
+    name: control1
     interfaces:
       - network_name: private_network
         ip: fdad:bad:ba55::de:11
-  - <<: *control
+  - <<: [*node, *ubuntu]
     name: node1
-    groups:
-      - k3s_cluster
-      - node
     interfaces:
       - network_name: private_network
         ip: fdad:bad:ba55::de:21

molecule/ipv6/overrides.yml

@@ -36,6 +36,8 @@
         #    the default has IPv4 ranges only.
         extra_server_args: >-
           {{ extra_args }}
+          --tls-san {{ apiserver_endpoint }}
+          {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
           --disable servicelb
           --disable traefik
           --disable-network-policy

molecule/resources/verify/from_outside/tasks/main.yml

@@ -3,10 +3,12 @@
   run_once: true
   delegate_to: "{{ outside_host }}"
   block:
-    - ansible.builtin.import_tasks: kubecfg-fetch.yml
+    - name: "Test CASE: Get kube config"
+      ansible.builtin.import_tasks: kubecfg-fetch.yml
     - name: "TEST CASE: Get nodes"
       ansible.builtin.include_tasks: test/get-nodes.yml
     - name: "TEST CASE: Deploy example"
       ansible.builtin.include_tasks: test/deploy-example.yml
   always:
-    - ansible.builtin.import_tasks: kubecfg-cleanup.yml
+    - name: "TEST CASE: Cleanup"
+      ansible.builtin.import_tasks: kubecfg-cleanup.yml

molecule/resources/verify/from_outside/tasks/test/deploy-example.yml

@@ -43,6 +43,10 @@
           {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
         port: >-
           {{ nginx_services.resources[0].spec.ports[0].port }}
+      # Deactivated linter rules:
+      #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
+      #                     would be undefined. This will not be the case during playbook execution.
+      # noqa jinja[invalid]
 
   always:
     - name: "Remove namespace: {{ testing_namespace }}"

molecule/resources/verify/from_outside/tasks/test/get-nodes.yml

@@ -22,3 +22,7 @@
         | unique
         | sort
       }}
+  # Deactivated linter rules:
+  #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
+  #                     would be undefined. This will not be the case during playbook execution.
+  # noqa jinja[invalid]

requirements.txt

@@ -1,8 +1,71 @@
-ansible-core>=2.13.2
+ansible-compat==2.2.1
-jmespath
+ansible-core==2.13.5
-jsonpatch
+ansible-lint==6.8.2
-kubernetes>=12.0.0
+arrow==1.2.3
-molecule-vagrant>=1.0.0
+attrs==22.1.0
-molecule>=4.0.1
+binaryornot==0.4.4
-netaddr>=0.8.0
+black==22.10.0
-pyyaml>=3.11
+bracex==2.3.post1
+cachetools==5.2.0
+Cerberus==1.3.2
+certifi==2022.9.24
+cffi==1.15.1
+chardet==5.0.0
+charset-normalizer==2.1.1
+click==8.1.3
+click-help-colors==0.9.1
+commonmark==0.9.1
+cookiecutter==2.1.1
+cryptography==38.0.1
+distro==1.8.0
+enrich==1.2.7
+filelock==3.8.0
+google-auth==2.12.0
+idna==3.4
+importlib-resources==5.10.0
+Jinja2==3.1.2
+jinja2-time==0.2.0
+jmespath==1.0.1
+jsonpatch==1.32
+jsonpointer==2.3
+jsonschema==4.16.0
+kubernetes==24.2.0
+MarkupSafe==2.1.1
+molecule==4.0.1
+molecule-vagrant==1.0.0
+mypy-extensions==0.4.3
+netaddr==0.8.0
+oauthlib==3.2.1
+packaging==21.3
+pathspec==0.10.1
+pkgutil-resolve-name==1.3.10
+platformdirs==2.5.2
+pluggy==1.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.21
+Pygments==2.13.0
+pyparsing==3.0.9
+pyrsistent==0.18.1
+python-dateutil==2.8.2
+python-slugify==6.1.2
+python-vagrant==1.0.0
+PyYAML==6.0
+requests==2.28.1
+requests-oauthlib==1.3.1
+resolvelib==0.8.1
+rich==12.6.0
+rsa==4.9
+ruamel.yaml==0.17.21
+ruamel.yaml.clib==0.2.6
+selinux==0.2.1
+six==1.16.0
+subprocess-tee==0.3.5
+text-unidecode==1.3
+tomli==2.0.1
+typing-extensions==4.4.0
+urllib3==1.26.12
+wcmatch==8.4.1
+websocket-client==1.4.1
+yamllint==1.28.0
+zipp==3.9.0

reset.yml

@@ -5,3 +5,9 @@
   become: yes
   roles:
     - role: reset
+    - role: raspberrypi
+      vars: {state: absent}
+  post_tasks:
+    - name: Reboot and wait for node to come back up
+      reboot:
+        reboot_timeout: 3600

roles/k3s/master/tasks/main.yml

@@ -152,12 +152,19 @@
     owner: "{{ ansible_user }}"
     mode: "u=rw,g=,o="
 
-- name: Configure kubectl cluster to https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443
+- name: Configure kubectl cluster to {{ endpoint_url }}
   command: >-
     k3s kubectl config set-cluster default
-      --server=https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443
+      --server={{ endpoint_url }}
       --kubeconfig ~{{ ansible_user }}/.kube/config
   changed_when: true
+  vars:
+    endpoint_url: >-
+      https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443
+  # Deactivated linter rules:
+  #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
+  #                     would be undefined. This will not be the case during playbook execution.
+  # noqa jinja[invalid]
 
 - name: Create kubectl symlink
   file:

roles/k3s/post/defaults/main.yml

@@ -1,3 +1,3 @@
 ---
 # Timeout to wait for MetalLB services to come up
-metal_lb_available_timeout: 60s
+metal_lb_available_timeout: 120s

roles/k3s/post/tasks/main.yml

@@ -28,9 +28,9 @@
   command: >-
     k3s kubectl wait {{ item.resource }}
     --namespace='metallb-system'
-    {% if item.name | default(False) -%} {{ item.name }} {%- endif %}
+    {% if item.name | default(False) -%}{{ item.name }}{%- endif %}
-    {% if item.selector | default(False) -%} --selector='{{ item.selector }}' {%- endif %}
+    {% if item.selector | default(False) -%}--selector='{{ item.selector }}'{%- endif %}
-    {% if item.condition | default(False) -%} {{ item.condition }} {%- endif %}
+    {% if item.condition | default(False) -%}{{ item.condition }}{%- endif %}
     --timeout='{{ metal_lb_available_timeout }}'
   changed_when: false
   run_once: true

roles/raspberrypi/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+# Indicates whether the k3s prerequisites for Raspberry Pi should be set up
+# Possible values:
+#   - present
+#   - absent
+state: present

roles/raspberrypi/tasks/main.yml

@@ -47,13 +47,20 @@
     - raspberry_pi|default(false)
     - ansible_facts.lsb.description|default("") is match("Debian.*bullseye")
 
-- name: execute OS related tasks on the Raspberry Pi
+- name: execute OS related tasks on the Raspberry Pi - {{ action }}
   include_tasks: "{{ item }}"
   with_first_found:
-    - "prereq/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
+    - "{{ action }}/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
-    - "prereq/{{ detected_distribution }}.yml"
+    - "{{ action }}/{{ detected_distribution }}.yml"
-    - "prereq/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
+    - "{{ action }}/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
-    - "prereq/{{ ansible_distribution }}.yml"
+    - "{{ action }}/{{ ansible_distribution }}.yml"
-    - "prereq/default.yml"
+    - "{{ action }}/default.yml"
+  vars:
+    action: >-
+      {% if state == "present" -%}
+      setup
+      {%- else -%}
+      teardown
+      {%- endif %}
   when:
     - raspberry_pi|default(false)

roles/raspberrypi/tasks/setup/Rocky.yml

@@ -1,5 +1,5 @@
 ---
-- name: Enable cgroup via boot commandline if not already enabled for Centos
+- name: Enable cgroup via boot commandline if not already enabled for Rocky
   lineinfile:
     path: /boot/cmdline.txt
     backrefs: yes

roles/raspberrypi/tasks/setup/Ubuntu.yml

@@ -6,8 +6,8 @@
     regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
     line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
   notify: reboot
-  when: not ansible_check_mode
 
 - name: Install linux-modules-extra-raspi
-  apt: name=linux-modules-extra-raspi state=present
+  apt:
-  when: (raspberry_pi) and (not ansible_check_mode)
+    name: linux-modules-extra-raspi
+    state: present

roles/raspberrypi/tasks/teardown/Raspbian.yml

@@ -0,0 +1 @@
+---

roles/raspberrypi/tasks/teardown/Rocky.yml

@@ -0,0 +1 @@
+---

roles/raspberrypi/tasks/teardown/Ubuntu.yml

@@ -0,0 +1,5 @@
+---
+- name: Remove linux-modules-extra-raspi
+  apt:
+    name: linux-modules-extra-raspi
+    state: absent

roles/raspberrypi/tasks/teardown/default.yml

@@ -0,0 +1 @@
+---

roles/reset/tasks/main.yml

@@ -50,14 +50,7 @@
   systemd:
     daemon_reload: yes
 
-- name: Remove linux-modules-extra-raspi
+- name: Remove tmp directory used for manifests
-  apt: name=linux-modules-extra-raspi state=absent
-
-- name: Remove tmp director used for manifests
   file:
     path: /tmp/k3s
     state: absent
-
-- name: Reboot and wait for node to come back up
-  reboot:
-    reboot_timeout: 3600