k3s lxc playbook done

2022-11-07 21:18:02 +00:00
parent f892029fcf
commit 6753d614ee
44 changed files with 2296 additions and 935 deletions
@@ -1,17 +1,10 @@
 ---
 exclude_paths:
  # default paths
-  - '.cache/'
+  - ".cache/"
-  - '.github/'
+  - ".github/"
-  - 'test/fixtures/formatting-before/'
+  - "test/fixtures/formatting-before/"
-  - 'test/fixtures/formatting-prettier/'
+  - "test/fixtures/formatting-prettier/"
  # The "converge" and "reset" playbooks use import_playbook in
  # conjunction with the "env" lookup plugin, which lets the
  # syntax check of ansible-lint fail.
  - 'molecule/**/converge.yml'
  - 'molecule/**/prepare.yml'
  - 'molecule/**/reset.yml'
 skip_list:
-  - 'fqcn-builtins'
+  - "fqcn-builtins"
@@ -1,37 +0,0 @@
 #!/bin/bash
 # download-boxes.sh
 # Check all molecule.yml files for required Vagrant boxes and download the ones that are not
 # already present on the system.
 set -euo pipefail
 GIT_ROOT=$(git rev-parse --show-toplevel)
 PROVIDER=virtualbox
 # Read all boxes for all platforms from the "molecule.yml" files
 all_boxes=$(cat "${GIT_ROOT}"/molecule/*/molecule.yml |
    yq -r '.platforms[].box' |         # Read the "box" property of each node under "platforms"
    grep --invert-match --regexp=--- | # Filter out file separators
    sort |
    uniq)
 # Read the boxes that are currently present on the system (for the current provider)
 present_boxes=$(
    (vagrant box list |
        grep "${PROVIDER}" | # Filter by boxes available for the current provider
        awk '{print $1;}' |  # The box name is the first word in each line
        sort |
        uniq) ||
        echo "" # In case any of these commands fails, just use an empty list
 )
 # The boxes that we need to download are the ones present in $all_boxes, but not $present_boxes.
 download_boxes=$(comm -2 -3 <(echo "${all_boxes}") <(echo "${present_boxes}"))
 # Actually download the necessary boxes
 if [ -n "${download_boxes}" ]; then
    echo "${download_boxes}" | while IFS= read -r box; do
        vagrant box add --provider "${PROVIDER}" "${box}"
    done
 fi
@@ -1,97 +0,0 @@
 ---
 name: Test
 on:
  pull_request:
  push:
    branches:
      - master
    paths-ignore:
      - '**/README.md'
 jobs:
  molecule:
    name: Molecule
    runs-on: macos-12
    strategy:
      matrix:
        scenario:
          - default
          - ipv6
          - single_node
      fail-fast: false
    env:
      PYTHON_VERSION: "3.10"
    steps:
      - name: Check out the codebase
        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # 3.0.2
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Configure VirtualBox
        run: |-
          sudo mkdir -p /etc/vbox
          cat <<EOF | sudo tee -a /etc/vbox/networks.conf > /dev/null
          * 192.168.30.0/24
          * fdad:bad:ba55::/64
          EOF
      - name: Cache pip
        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('./requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Cache Vagrant boxes
        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # 3.0.11
        with:
          path: |
            ~/.vagrant.d/boxes
          key: vagrant-boxes-${{ hashFiles('**/molecule.yml') }}
          restore-keys: |
            vagrant-boxes
      - name: Download Vagrant boxes for all scenarios
        # To save some cache space, all scenarios share the same cache key.
        # On the other hand, this means that the cache contents should be
        # the same across all scenarios. This step ensures that.
        run: ./.github/download-boxes.sh
      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # 4.3.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip' # caching pip dependencies
      - name: Install dependencies
        run: |
          echo "::group::Upgrade pip"
          python3 -m pip install --upgrade pip
          echo "::endgroup::"
          echo "::group::Install Python requirements from requirements.txt"
          python3 -m pip install -r requirements.txt
          echo "::endgroup::"
      - name: Test with molecule
        run: molecule test --scenario-name ${{ matrix.scenario }}
        env:
          ANSIBLE_K3S_LOG_DIR: ${{ runner.temp }}/logs/k3s-ansible/${{ matrix.scenario }}
          ANSIBLE_SSH_RETRIES: 4
          ANSIBLE_TIMEOUT: 60
          PY_COLORS: 1
          ANSIBLE_FORCE_COLOR: 1
      - name: Upload log files
        if: always() # do this even if a step before has failed
        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # 3.1.1
        with:
          name: logs
          path: |
            ${{ runner.temp }}/logs
      - name: Delete old box versions
        if: always() # do this even if a step before has failed
        run: vagrant box prune --force
@@ -22,9 +22,9 @@ Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a
 on processor architecture:
- [X] x64
+- [x] x64
- [X] arm64
+- [x] arm64
- [X] armhf
+- [x] armhf
 ## ✅ System requirements
@@ -105,13 +105,6 @@ See the commands [here](https://docs.technotim.live/posts/k3s-etcd-ansible/#test
 Be sure to see [this post](https://github.com/techno-tim/k3s-ansible/discussions/20) on how to troubleshoot common problems
 ### Testing the playbook using molecule
 This playbook includes a [molecule](https://molecule.rtfd.io/)-based test setup.
 It is run automatically in CI, but you can also run the tests locally.
 This might be helpful for quick feedback in a few cases.
 You can find more information about it [here](molecule/README.md).
 ## Thanks 🤝
 This repo is really standing on the shoulders of giants. Thank you to all those who have contributed and tanks to these repos for code and ideas:
@@ -1,73 +0,0 @@
 # Test suites for `k3s-ansible`
 This folder contains the [molecule](https://molecule.rtfd.io/)-based test setup for this playbook.
 ## Scenarios
 We have these scenarios:
 - **default**:
  A 3 control + 2 worker node cluster based very closely on the [sample inventory](../inventory/sample/).
 - **ipv6**:
  A cluster that is externally accessible via IPv6 ([more information](ipv6/README.md))
  To save a bit of test time, this cluster is _not_ highly available, it consists of only one control and one worker node.
 - **single_node**:
  Very similar to the default scenario, but uses only a single node for all cluster functionality.
 ## How to execute
 To test on your local machine, follow these steps:
 ### System requirements
 Make sure that the following software packages are available on your system:
 - [Python 3](https://www.python.org/downloads)
 - [Vagrant](https://www.vagrantup.com/downloads)
 - [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
 ### Set up VirtualBox networking on Linux and macOS
 _You can safely skip this if you are working on Windows._
 Furthermore, the test cluster uses the `192.168.30.0/24` subnet which is [not set up by VirtualBox automatically](https://www.virtualbox.org/manual/ch06.html#network_hostonly).
 To set the subnet up for use with VirtualBox, please make sure that `/etc/vbox/networks.conf` exists and that it contains this line:
 ```
 * 192.168.30.0/24
 * fdad:bad:ba55::/64
 ```
 ### Install Python dependencies
 You will get [Molecule, Ansible and a few extra dependencies](../requirements.txt) via [pip](https://pip.pypa.io/).
 Usually, it is advisable to work in a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for this:
 ```bash
 cd /path/to/k3s-ansible
 # Create a virtualenv at ".env". You only need to do this once.
 python3 -m venv .env
 # Activate the virtualenv for your current shell session.
 # If you start a new session, you will have to repeat this.
 source .env/bin/activate
 # Install the required packages into the virtualenv.
 # These remain installed across shell sessions.
 python3 -m pip install -r requirements.txt
 ```
 ### Run molecule
 With the virtual environment from the previous step active in your shell session, you can now use molecule to test the playbook.
 Interesting commands are:
 - `molecule create`: Create virtual machines for the test cluster nodes.
 - `molecule destroy`: Delete the virtual machines for the test cluster nodes.
 - `molecule converge`: Run the `site` playbook on the nodes of the test cluster.
 - `molecule side_effect`: Run the `reset` playbook on the nodes of the test cluster.
 - `molecule verify`: Verify that the cluster works correctly.
 - `molecule test`: The "all-in-one" sequence of steps that is executed in CI.
  This includes the `create`, `converge`, `verify`, `side_effect` and `destroy` steps.
  See [`molecule.yml`](default/molecule.yml) for more details.
@@ -1,99 +0,0 @@
 ---
 dependency:
  name: galaxy
 driver:
  name: vagrant
 platforms:
  - name: control1
    box: generic/ubuntu2204
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - master
    interfaces:
      - network_name: private_network
        ip: 192.168.30.38
    config_options:
      # We currently can not use public-key based authentication on Ubuntu 22.04,
      # see: https://github.com/chef/bento/issues/1405
      ssh.username: "vagrant"
      ssh.password: "vagrant"
  - name: control2
    box: generic/debian11
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - master
    interfaces:
      - network_name: private_network
        ip: 192.168.30.39
  - name: control3
    box: generic/rocky9
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - master
    interfaces:
      - network_name: private_network
        ip: 192.168.30.40
  - name: node1
    box: generic/ubuntu2204
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - node
    interfaces:
      - network_name: private_network
        ip: 192.168.30.41
    config_options:
      # We currently can not use public-key based authentication on Ubuntu 22.04,
      # see: https://github.com/chef/bento/issues/1405
      ssh.username: "vagrant"
      ssh.password: "vagrant"
  - name: node2
    box: generic/rocky9
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - node
    interfaces:
      - network_name: private_network
        ip: 192.168.30.42
 provisioner:
  name: ansible
  playbooks:
    converge: ../resources/converge.yml
    side_effect: ../resources/reset.yml
    verify: ../resources/verify.yml
  inventory:
    links:
      group_vars: ../../inventory/sample/group_vars
 scenario:
  test_sequence:
    - dependency
    - lint
    - cleanup
    - destroy
    - syntax
    - create
    - prepare
    - converge
    # idempotence is not possible with the playbook in its current form.
    - verify
    # We are repurposing side_effect here to test the reset playbook.
    # This is why we do not run it before verify (which tests the cluster),
    # but after the verify step.
    - side_effect
    - cleanup
    - destroy
@@ -1,11 +0,0 @@
 ---
 - name: Apply overrides
  hosts: all
  tasks:
    - name: Override host variables
      ansible.builtin.set_fact:
        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
        flannel_iface: eth1
        # The test VMs might be a bit slow, so we give them more time to join the cluster:
        retry_count: 45
@@ -1,22 +0,0 @@
 ---
 - name: Apply overrides
  ansible.builtin.import_playbook: >-
    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
 - name: Network setup
  hosts: all
  tasks:
    - name: Disable firewalld
      when: ansible_distribution == "Rocky"
      # Rocky Linux comes with firewalld enabled. It blocks some of the network
      # connections needed for our k3s cluster. For our test setup, we just disable
      # it since the VM host's firewall is still active for connections to and from
      # the Internet.
      # When building your own cluster, please DO NOT blindly copy this. Instead,
      # please create a custom firewall configuration that fits your network design
      # and security needs.
      ansible.builtin.systemd:
        name: firewalld
        enabled: no
        state: stopped
      become: true
@@ -1,35 +0,0 @@
 # Sample IPv6 configuration for `k3s-ansible`
 This scenario contains a cluster configuration which is _IPv6 first_, but still supports dual-stack networking with IPv4 for most things.
 This means:
 - The API server VIP is an IPv6 address.
 - The MetalLB pool consists of both IPv4 and IPv4 addresses.
 - Nodes as well as cluster-internal resources (pods and services) are accessible via IPv4 as well as IPv6.
 ## Network design
 All IPv6 addresses used in this scenario share a single `/48` prefix: `fdad:bad:ba55`.
 The following subnets are used:
 - `fdad:bad:ba55:`**`0`**`::/64` is the subnet which contains the cluster components meant for external access.
  That includes:
  - The VIP for the Kubernetes API server: `fdad:bad:ba55::333`
  - Services load-balanced by MetalLB: `fdad:bad:ba55::1b:0/112`
  - Cluster nodes: `fdad:bad:ba55::de:0/112`
  - The host executing Vagrant: `fdad:bad:ba55::1`
  In a home lab setup, this might be your LAN.
 - `fdad:bad:ba55:`**`4200`**`::/56` is used internally by the cluster for pods.
 - `fdad:bad:ba55:`**`4300`**`::/108` is used internally by the cluster for services.
 IPv4 networking is also available:
 - The nodes have addresses inside `192.168.123.0/24`.
  MetalLB also has a bit of address space in this range: `192.168.123.80-192.168.123.90`
 - For pods and services, the k3s defaults (`10.42.0.0/16` and `10.43.0.0/16)` are used.
 Note that the host running Vagrant is not part any of these IPv4 networks.
@@ -1,3 +0,0 @@
 ---
 node_ipv4: 192.168.123.11
 node_ipv6: fdad:bad:ba55::de:11
@@ -1,3 +0,0 @@
 ---
 node_ipv4: 192.168.123.21
 node_ipv6: fdad:bad:ba55::de:21
@@ -1,65 +0,0 @@
 ---
 dependency:
  name: galaxy
 driver:
  name: vagrant
 platforms:
  - name: control1
    box: generic/ubuntu2204
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - master
    interfaces:
      - network_name: private_network
        ip: fdad:bad:ba55::de:11
    config_options:
      # We currently can not use public-key based authentication on Ubuntu 22.04,
      # see: https://github.com/chef/bento/issues/1405
      ssh.username: "vagrant"
      ssh.password: "vagrant"
  - name: node1
    box: generic/ubuntu2204
    memory: 2048
    cpus: 2
    groups:
      - k3s_cluster
      - node
    interfaces:
      - network_name: private_network
        ip: fdad:bad:ba55::de:21
    config_options:
      # We currently can not use public-key based authentication on Ubuntu 22.04,
      # see: https://github.com/chef/bento/issues/1405
      ssh.username: "vagrant"
      ssh.password: "vagrant"
 provisioner:
  name: ansible
  playbooks:
    converge: ../resources/converge.yml
    side_effect: ../resources/reset.yml
    verify: ../resources/verify.yml
  inventory:
    links:
      group_vars: ../../inventory/sample/group_vars
 scenario:
  test_sequence:
    - dependency
    - lint
    - cleanup
    - destroy
    - syntax
    - create
    - prepare
    - converge
    # idempotence is not possible with the playbook in its current form.
    - verify
    # We are repurposing side_effect here to test the reset playbook.
    # This is why we do not run it before verify (which tests the cluster),
    # but after the verify step.
    - side_effect
    - cleanup
    - destroy
@@ -1,45 +0,0 @@
 ---
 - name: Apply overrides
  hosts: all
  tasks:
    - name: Override host variables (1/2)
      ansible.builtin.set_fact:
        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
        flannel_iface: eth1
        # The test VMs might be a bit slow, so we give them more time to join the cluster:
        retry_count: 45
        # IPv6 configuration
        # ######################################################################
        # The API server will be reachable on IPv6 only
        apiserver_endpoint: fdad:bad:ba55::333
        # We give MetalLB address space for both IPv4 and IPv6
        metal_lb_ip_range:
          - fdad:bad:ba55::1b:0/112
          - 192.168.123.80-192.168.123.90
        # k3s_node_ip is by default set to the IPv4 address of flannel_iface.
        # We want IPv6 addresses here of course, so we just specify them
        # manually below.
        k3s_node_ip: "{{ node_ipv4 }},{{ node_ipv6 }}"
    - name: Override host variables (2/2)
      # Since "extra_args" depends on "k3s_node_ip" and "flannel_iface" we have
      # to set this AFTER overriding the both of them.
      ansible.builtin.set_fact:
        # A few extra server args are necessary:
        #  - the network policy needs to be disabled.
        #  - we need to manually specify the subnets for services and pods, as
        #    the default has IPv4 ranges only.
        extra_server_args: >-
          {{ extra_args }}
          --tls-san {{ apiserver_endpoint }}
          {{ '--node-taint node-role.kubernetes.io/master=true:NoSchedule' if k3s_master_taint else '' }}
          --disable servicelb
          --disable traefik
          --disable-network-policy
          --cluster-cidr=10.42.0.0/16,fdad:bad:ba55:4200::/56
          --service-cidr=10.43.0.0/16,fdad:bad:ba55:4300::/108
@@ -1,51 +0,0 @@
 ---
 - name: Apply overrides
  ansible.builtin.import_playbook: >-
    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
 - name: Configure dual-stack networking
  hosts: all
  become: true
  # Unfortunately, as of 2022-09, Vagrant does not support the configuration
  # of both IPv4 and IPv6 addresses for a single network adapter. So we have
  # to configure that ourselves.
  # Moreover, we have to explicitly enable IPv6 for the loopback interface.
  tasks:
    - name: Enable IPv6 for network interfaces
      ansible.posix.sysctl:
        name: net.ipv6.conf.{{ item }}.disable_ipv6
        value: "0"
      with_items:
        - all
        - default
        - lo
    - name: Disable duplicate address detection
      # Duplicate address detection did repeatedly fail within the virtual
      # network. But since this setup does not use SLAAC anyway, we can safely
      # disable it.
      ansible.posix.sysctl:
        name: net.ipv6.conf.{{ item }}.accept_dad
        value: "0"
      with_items:
        - "{{ flannel_iface }}"
    - name: Write IPv4 configuration
      ansible.builtin.template:
        src: 55-flannel-ipv4.yaml.j2
        dest: /etc/netplan/55-flannel-ipv4.yaml
        owner: root
        group: root
        mode: 0644
      register: netplan_template
    - name: Apply netplan configuration
      # Conceptually, this should be a handler rather than a task.
      # However, we are currently not in a role context - creating
      # one just for this seemed overkill.
      when: netplan_template.changed
      ansible.builtin.command:
        cmd: netplan apply
      changed_when: true
@@ -1,8 +0,0 @@
 ---
 network:
  version: 2
  renderer: networkd
  ethernets:
    {{ flannel_iface }}:
      addresses:
      - {{ node_ipv4 }}/24
@@ -1,7 +0,0 @@
 ---
 - name: Apply overrides
  ansible.builtin.import_playbook: >-
    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
 - name: Converge
  ansible.builtin.import_playbook: ../../site.yml
@@ -1,7 +0,0 @@
 ---
 - name: Apply overrides
  ansible.builtin.import_playbook: >-
    {{ lookup("ansible.builtin.env", "MOLECULE_SCENARIO_DIRECTORY") }}/overrides.yml
 - name: Reset
  ansible.builtin.import_playbook: ../../reset.yml
@@ -1,5 +0,0 @@
 ---
 - name: Verify
  hosts: all
  roles:
    - verify/from_outside
@@ -1,9 +0,0 @@
 ---
 # A host outside of the cluster from which the checks shall be performed
 outside_host: localhost
 # This kubernetes namespace will be used for testing
 testing_namespace: molecule-verify-from-outside
 # The directory in which the example manifests reside
 example_manifests_path: ../../../../example
@@ -1,5 +0,0 @@
 ---
 - name: Clean up kubecfg
  ansible.builtin.file:
    path: "{{ kubecfg.path }}"
    state: absent
@@ -1,19 +0,0 @@
 ---
 - name: Create temporary directory for kubecfg
  ansible.builtin.tempfile:
    state: directory
    suffix: kubecfg
  register: kubecfg
 - name: Gathering facts
  delegate_to: "{{ groups['master'][0] }}"
  ansible.builtin.gather_facts:
 - name: Download kubecfg
  ansible.builtin.fetch:
    src: "{{ ansible_env.HOME }}/.kube/config"
    dest: "{{ kubecfg.path }}/"
    flat: true
  delegate_to: "{{ groups['master'][0] }}"
  delegate_facts: true
 - name: Store path to kubecfg
  ansible.builtin.set_fact:
    kubecfg_path: "{{ kubecfg.path }}/config"
@@ -1,14 +0,0 @@
 ---
 - name: Verify
  run_once: true
  delegate_to: "{{ outside_host }}"
  block:
    - name: "Test CASE: Get kube config"
      ansible.builtin.import_tasks: kubecfg-fetch.yml
    - name: "TEST CASE: Get nodes"
      ansible.builtin.include_tasks: test/get-nodes.yml
    - name: "TEST CASE: Deploy example"
      ansible.builtin.include_tasks: test/deploy-example.yml
  always:
    - name: "TEST CASE: Cleanup"
      ansible.builtin.import_tasks: kubecfg-cleanup.yml
@@ -1,58 +0,0 @@
 ---
 - name: Deploy example
  block:
    - name: "Create namespace: {{ testing_namespace }}"
      kubernetes.core.k8s:
        api_version: v1
        kind: Namespace
        name: "{{ testing_namespace }}"
        state: present
        wait: true
        kubeconfig: "{{ kubecfg_path }}"
    - name: Apply example manifests
      kubernetes.core.k8s:
        src: "{{ example_manifests_path }}/{{ item }}"
        namespace: "{{ testing_namespace }}"
        state: present
        wait: true
        kubeconfig: "{{ kubecfg_path }}"
      with_items:
        - deployment.yml
        - service.yml
    - name: Get info about nginx service
      kubernetes.core.k8s_info:
        kind: service
        name: nginx
        namespace: "{{ testing_namespace }}"
        kubeconfig: "{{ kubecfg_path }}"
      vars: &load_balancer_metadata
        metallb_ip: status.loadBalancer.ingress[0].ip
        metallb_port: spec.ports[0].port
      register: nginx_services
    - name: Assert that the nginx welcome page is available
      ansible.builtin.uri:
        url: http://{{ ip | ansible.utils.ipwrap }}:{{ port }}/
        return_content: yes
      register: result
      failed_when: "'Welcome to nginx!' not in result.content"
      vars:
        ip: >-
          {{ nginx_services.resources[0].status.loadBalancer.ingress[0].ip }}
        port: >-
          {{ nginx_services.resources[0].spec.ports[0].port }}
      # Deactivated linter rules:
      #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
      #                     would be undefined. This will not be the case during playbook execution.
      # noqa jinja[invalid]
  always:
    - name: "Remove namespace: {{ testing_namespace }}"
      kubernetes.core.k8s:
        api_version: v1
        kind: Namespace
        name: "{{ testing_namespace }}"
        state: absent
        kubeconfig: "{{ kubecfg_path }}"
@@ -1,28 +0,0 @@
 ---
 - name: Get all nodes in cluster
  kubernetes.core.k8s_info:
    kind: node
    kubeconfig: "{{ kubecfg_path }}"
  register: cluster_nodes
 - name: Assert that the cluster contains exactly the expected nodes
  ansible.builtin.assert:
    that: found_nodes == expected_nodes
    success_msg: "Found nodes as expected: {{ found_nodes }}"
    fail_msg: "Expected nodes {{ expected_nodes }}, but found nodes {{ found_nodes }}"
  vars:
    found_nodes: >-
      {{ cluster_nodes | json_query('resources[*].metadata.name') | unique | sort }}
    expected_nodes: |-
      {{
        (
          ( groups['master'] | default([]) ) +
          ( groups['node'] | default([]) )
        )
        | unique
        | sort
      }}
  # Deactivated linter rules:
  #   - jinja[invalid]: As of version 6.6.0, ansible-lint complains that the input to ipwrap
  #                     would be undefined. This will not be the case during playbook execution.
  # noqa jinja[invalid]
@@ -1,48 +0,0 @@
 ---
 dependency:
  name: galaxy
 driver:
  name: vagrant
 platforms:
  - name: control1
    box: generic/ubuntu2204
    memory: 4096
    cpus: 4
    config_options:
      # We currently can not use public-key based authentication on Ubuntu 22.04,
      # see: https://github.com/chef/bento/issues/1405
      ssh.username: "vagrant"
      ssh.password: "vagrant"
    groups:
      - k3s_cluster
      - master
    interfaces:
      - network_name: private_network
        ip: 192.168.30.50
 provisioner:
  name: ansible
  playbooks:
    converge: ../resources/converge.yml
    side_effect: ../resources/reset.yml
    verify: ../resources/verify.yml
  inventory:
    links:
      group_vars: ../../inventory/sample/group_vars
 scenario:
  test_sequence:
    - dependency
    - lint
    - cleanup
    - destroy
    - syntax
    - create
    - prepare
    - converge
    # idempotence is not possible with the playbook in its current form.
    - verify
    # We are repurposing side_effect here to test the reset playbook.
    # This is why we do not run it before verify (which tests the cluster),
    # but after the verify step.
    - side_effect
    - cleanup
    - destroy
@@ -1,15 +0,0 @@
 ---
 - name: Apply overrides
  hosts: all
  tasks:
    - name: Override host variables
      ansible.builtin.set_fact:
        # See: https://github.com/flannel-io/flannel/blob/67d603aaf45ef80f5dd39f43714fc5e6f8a637eb/Documentation/troubleshooting.md#Vagrant  # noqa yaml[line-length]
        flannel_iface: eth1
        # The test VMs might be a bit slow, so we give them more time to join the cluster:
        retry_count: 45
        # Make sure that our IP ranges do not collide with those of the default scenario
        apiserver_endpoint: "192.168.30.223"
        metal_lb_ip_range: "192.168.30.91-192.168.30.99"
@@ -1,9 +1,6 @@
 molecule>=4.0.1
 ansible-core>=2.13.2
 ansible-lint>=6.6.0
 kubernetes>=12.0.0
 molecule-vagrant>=1.0.0
 molecule>=4.0.1
 netaddr>=0.8.0
 pyyaml>=3.11
 yamllint>=1.28.0
@@ -7,7 +7,6 @@
 ansible-compat==2.2.4
    # via
    #   ansible-lint
    #   molecule
 ansible-core==2.13.5
    # via
    #   -r requirements.in
@@ -43,13 +42,10 @@ click==8.1.3
    #   black
    #   click-help-colors
    #   cookiecutter
    #   molecule
 click-help-colors==0.9.1
    # via molecule
 commonmark==0.9.1
    # via rich
 cookiecutter==2.1.1
    # via molecule
 cryptography==38.0.3
    # via ansible-core
 distlib==0.3.6
@@ -1,12 +1,9 @@
 ---
 - hosts: k3s_cluster
  gather_facts: yes
  become: yes
  roles:
    - role: reset
    - role: raspberrypi
      vars: {state: absent}
  post_tasks:
    - name: Reboot and wait for node to come back up
      reboot:
@@ -0,0 +1,58 @@
 ---
 - name: Create LXC for k3s
  hosts: localhost
  gather_facts: no
  tasks:
    - name: Stop containers
      community.general.proxmox:
        vmid: "{{ hostvars[item]['vmid'] }}"
        api_user: root@pam
        api_password: bemjogado
        api_host: 10.0.2.2
        state: stopped
      loop: "{{ groups['all'] }}"
      ignore_errors: yes
    - name: Remove containers
      community.general.proxmox:
        vmid: "{{ hostvars[item]['vmid'] }}"
        api_user: root@pam
        api_password: bemjogado
        api_host: 10.0.2.2
        state: absent
      loop: "{{ groups['all'] }}"
      ignore_errors: yes
    - name: Create containers
      community.general.proxmox:
        vmid: "{{ hostvars[item]['vmid'] }}"
        node: "{{ hostvars[item]['node'] }}"
        api_user: root@pam
        api_password: bemjogado
        api_host: 10.0.2.2
        password: bemjogado
        hostname: "{{ item }}"
        ostemplate: "hyrule-8tb-nfs:vztmpl/debian-11-standard_11.3-1_amd64.tar.zst"
        netif: "{'net0':'name=eth0,\
          gw=10.0.0.1,\
          ip={{ hostvars[item]['ansible_host'] }}/21,\
          hwaddr={{ hostvars[item]['mac_addr'] }},\
          bridge=vmbr0'}"
        cores: "{{ hostvars[item]['cores'] }}"
        memory: "{{ hostvars[item]['memory'] }}"
        unprivileged: no
        swap: 0
        searchdomain: "home"
        onboot: 1
        disk: local-lvm:8
        force: yes
      loop: "{{ groups['all'] }}"
    - name: Start deployments
      community.general.proxmox:
        vmid: "{{ hostvars[item]['vmid'] }}"
        api_user: root@pam
        api_password: bemjogado
        api_host: 10.0.2.2
        state: started
      loop: "{{ groups['all'] }}"
@@ -56,10 +56,10 @@
 - name: Add /usr/local/bin to sudo secure_path
  lineinfile:
-    line: 'Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin'
+    line: "Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin"
    regexp: "Defaults(\\s)*secure_path(\\s)*="
    state: present
    insertafter: EOF
    path: /etc/sudoers
-    validate: 'visudo -cf %s'
+    validate: "visudo -cf %s"
  when: ansible_os_family == "RedHat"
@@ -1,6 +0,0 @@
 ---
 # Indicates whether the k3s prerequisites for Raspberry Pi should be set up
 # Possible values:
 #   - present
 #   - absent
 state: present
@@ -1,3 +0,0 @@
 ---
 - name: Reboot
  reboot:
@@ -1,66 +0,0 @@
 ---
 - name: Test for raspberry pi /proc/cpuinfo
  command: grep -E "Raspberry Pi|BCM2708|BCM2709|BCM2835|BCM2836" /proc/cpuinfo
  register: grep_cpuinfo_raspberrypi
  failed_when: false
  changed_when: false
 - name: Test for raspberry pi /proc/device-tree/model
  command: grep -E "Raspberry Pi" /proc/device-tree/model
  register: grep_device_tree_model_raspberrypi
  failed_when: false
  changed_when: false
 - name: Set raspberry_pi fact to true
  set_fact:
    raspberry_pi: true
  when:
    grep_cpuinfo_raspberrypi.rc == 0 or grep_device_tree_model_raspberrypi.rc == 0
 - name: Set detected_distribution to Raspbian
  set_fact:
    detected_distribution: Raspbian
  when: >
    raspberry_pi|default(false) and
    ( ansible_facts.lsb.id|default("") == "Raspbian" or
      ansible_facts.lsb.description|default("") is match("[Rr]aspbian.*") )
 - name: Set detected_distribution to Raspbian (ARM64 on Debian Buster)
  set_fact:
    detected_distribution: Raspbian
  when:
    - ansible_facts.architecture is search("aarch64")
    - raspberry_pi|default(false)
    - ansible_facts.lsb.description|default("") is match("Debian.*buster")
 - name: Set detected_distribution_major_version
  set_fact:
    detected_distribution_major_version: "{{ ansible_facts.lsb.major_release }}"
  when:
    - detected_distribution | default("") == "Raspbian"
 - name: Set detected_distribution to Raspbian (ARM64 on Debian Bullseye)
  set_fact:
    detected_distribution: Raspbian
  when:
    - ansible_facts.architecture is search("aarch64")
    - raspberry_pi|default(false)
    - ansible_facts.lsb.description|default("") is match("Debian.*bullseye")
 - name: execute OS related tasks on the Raspberry Pi - {{ action }}
  include_tasks: "{{ item }}"
  with_first_found:
    - "{{ action }}/{{ detected_distribution }}-{{ detected_distribution_major_version }}.yml"
    - "{{ action }}/{{ detected_distribution }}.yml"
    - "{{ action }}/{{ ansible_distribution }}-{{ ansible_distribution_major_version }}.yml"
    - "{{ action }}/{{ ansible_distribution }}.yml"
    - "{{ action }}/default.yml"
  vars:
    action: >-
      {% if state == "present" -%}
      setup
      {%- else -%}
      teardown
      {%- endif %}
  when:
    - raspberry_pi|default(false)
@@ -1,27 +0,0 @@
 ---
 - name: Activating cgroup support
  lineinfile:
    path: /boot/cmdline.txt
    regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
    line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
    backrefs: true
  notify: reboot
 - name: Install iptables
  apt: name=iptables state=present
 - name: Flush iptables before changing to iptables-legacy
  iptables:
    flush: true
 - name: Changing to iptables-legacy
  alternatives:
    path: /usr/sbin/iptables-legacy
    name: iptables
  register: ip4_legacy
 - name: Changing to ip6tables-legacy
  alternatives:
    path: /usr/sbin/ip6tables-legacy
    name: ip6tables
  register: ip6_legacy
@@ -1,9 +0,0 @@
 ---
 - name: Enable cgroup via boot commandline if not already enabled for Rocky
  lineinfile:
    path: /boot/cmdline.txt
    backrefs: yes
    regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
    line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
  notify: reboot
  when: not ansible_check_mode
@@ -1,13 +0,0 @@
 ---
 - name: Enable cgroup via boot commandline if not already enabled for Ubuntu on a Raspberry Pi
  lineinfile:
    path: /boot/firmware/cmdline.txt
    backrefs: yes
    regexp: '^((?!.*\bcgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory\b).*)$'
    line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory'
  notify: reboot
 - name: Install linux-modules-extra-raspi
  apt:
    name: linux-modules-extra-raspi
    state: present
@@ -1 +0,0 @@
 ---
@@ -1 +0,0 @@
 ---
@@ -1 +0,0 @@
 ---
@@ -1,5 +0,0 @@
 ---
 - name: Remove linux-modules-extra-raspi
  apt:
    name: linux-modules-extra-raspi
    state: absent
@@ -1 +0,0 @@
 ---
@@ -1,12 +1,10 @@
 ---
 - hosts: k3s_cluster
  gather_facts: yes
  become: yes
  roles:
    - role: prereq
    - role: download
    - role: raspberrypi
 - hosts: master
  become: yes