Simplify codebase: modernize loops, split templates, improve CI (#14889)

* Simplify codebase: modernize loops, split templates, improve CI

This PR consolidates several simplification phases:

## Ansible Modernization
- Modernize `with_items` to `loop` across ~50 task files
- Add OS detection facts (is_ubuntu, os_family_lowercase)
- Condense inline YAML syntax where appropriate

## Template Splitting
- Split 568-line dnscrypt-proxy.toml.j2 into focused partials:
  - global.toml.j2 (core settings)
  - sources.toml.j2 (resolver sources)
  - filters.toml.j2 (blocking rules)
  - cache.toml.j2 (caching config)

## CI Workflow Improvements
- Create setup-algo composite action for shared CI setup
- Re-enable integration tests with health checks
- Fix smart-tests.yml silent lint failures (remove || true)
- Use env variables for GitHub SHAs (security)

## server.yml Async Simplification
- Reorganize VPN service configuration with clear sections
- Add performance_parallel_services toggle
- Simplify status display from json_query to inline conditionals
- Keep services explicit for readability

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

* Fix with_items to loop conversion: preserve list flattening

with_items automatically flattens nested lists, but loop does NOT.
The mechanical conversion broke iteration over list variables.

Wrong:
  loop:
    - "{{ users }}"  # ['alice', 'bob'] treated as ONE item

Fixed:
  loop: "{{ users }}"  # Iterates over alice, bob correctly

For combined lists (users + server):
  loop: "{{ users + [IP_subject_alt_name] }}"

Fixes IPsec certificate generation creating files named literally
'['alice', 'bob'].key' instead of separate alice.key and bob.key.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

* Fix integration test: use strongswan-starter service name on Ubuntu 20.04+

The StrongSwan service is named 'strongswan-starter' on Ubuntu 20.04+,
not 'strongswan'. The test was checking the wrong service name, causing
false failures even when StrongSwan was actually running.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

* Fix IPsec path issues: remove trailing slashes and fix test paths

1. Remove trailing slashes from ipsec_config_path and ipsec_pki_path
   in roles/strongswan/defaults/main.yml (causes double slashes)

2. Fix integration test to check correct subdirectories:
   - .p12 files are in ipsec/manual/
   - .mobileconfig files are in ipsec/apple/

3. Fix strongswan service name check (strongswan-starter on Ubuntu 20.04+)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

---------

Co-authored-by: Claude <[email protected]>

Author: dguido

.github/actions/setup-algo/action.yml

@@ -0,0 +1,36 @@
+---
+name: 'Setup Algo Environment'
+description: 'Setup Python, uv, and dependencies for Algo VPN CI'
+inputs:
+  python-version:
+    description: 'Python version to use'
+    required: false
+    default: '3.11'
+  install-shellcheck:
+    description: 'Install shellcheck for shell script linting'
+    required: false
+    default: 'false'
+  install-ansible-collections:
+    description: 'Install Ansible Galaxy collections'
+    required: false
+    default: 'false'
+runs:
+  using: composite
+  steps:
+    - name: Setup Python
+      uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548  # v6.1.0
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Setup uv environment
+      uses: ./.github/actions/setup-uv
+
+    - name: Install shellcheck
+      if: inputs.install-shellcheck == 'true'
+      run: sudo apt-get update && sudo apt-get install -y shellcheck
+      shell: bash
+
+    - name: Install Ansible collections
+      if: inputs.install-ansible-collections == 'true'
+      run: uv run ansible-galaxy collection install -r requirements.yml
+      shell: bash

.github/workflows/integration-tests.yml

@@ -21,8 +21,8 @@ jobs:
     name: Localhost VPN Deployment Test
     runs-on: ubuntu-22.04
     timeout-minutes: 30
-    if: false  # Disabled until we fix the ansible issues
     strategy:
+      fail-fast: false
       matrix:
         vpn_type: ['wireguard', 'ipsec', 'both']
     steps:
@@ -78,7 +78,6 @@ jobs:
           tests: true
           no_log: false
           ansible_connection: local
-          ansible_python_interpreter: /usr/bin/python3
           dns_encryption: true
           algo_dns_adblocking: true
           algo_ssh_tunneling: false
@@ -88,11 +87,15 @@ jobs:
           pki_in_tmpfs: true
           endpoint: 127.0.0.1
           ssh_port: 4160
+          local_service_ip: 172.16.0.1
+          local_service_ipv6: "fd00::1"
           EOF
 
       - name: Run Algo deployment
         run: |
-          sudo ansible-playbook main.yml \
+          # Run ansible-playbook via uv - become: true in playbook handles root
+          # GitHub runners have passwordless sudo for become escalation
+          uv run ansible-playbook main.yml \
             -i "localhost," \
             -c local \
             -e @integration-test.cfg \
@@ -112,11 +115,11 @@ jobs:
             echo "✓ WireGuard is running"
           fi
 
-          # Check StrongSwan
+          # Check StrongSwan (service name is strongswan-starter on Ubuntu 20.04+)
           if [[ "${{ matrix.vpn_type }}" == "ipsec" || "${{ matrix.vpn_type }}" == "both" ]]; then
             echo "Checking StrongSwan..."
             sudo ipsec statusall
-            if ! sudo systemctl is-active --quiet strongswan; then
+            if ! sudo systemctl is-active --quiet strongswan-starter; then
               echo "✗ StrongSwan service not running"
               exit 1
             fi
@@ -130,6 +133,21 @@ jobs:
             echo "✓ dnsmasq is running"
           fi
 
+          # Check dnscrypt-proxy
+          if sudo systemctl is-active --quiet dnscrypt-proxy; then
+            echo "✓ dnscrypt-proxy is running"
+          else
+            echo "⚠️  dnscrypt-proxy not running"
+          fi
+
+          # DNS health check - verify DNS resolution works
+          echo "Testing DNS resolution via local_service_ip (172.16.0.1)..."
+          if dig @172.16.0.1 google.com +short +timeout=5 | grep -q .; then
+            echo "✓ DNS resolution working"
+          else
+            echo "⚠️  DNS resolution failed (service may still be starting)"
+          fi
+
       - name: Verify generated configs
         run: |
           echo "Checking generated configuration files..."
@@ -149,14 +167,14 @@ jobs:
             echo "✓ All WireGuard configs generated"
           fi
 
-          # IPsec configs
+          # IPsec configs (p12 in manual/, mobileconfig in apple/)
           if [[ "${{ matrix.vpn_type }}" == "ipsec" || "${{ matrix.vpn_type }}" == "both" ]]; then
             for user in alice bob; do
-              if [ ! -f "configs/localhost/ipsec/${user}.p12" ]; then
+              if [ ! -f "configs/localhost/ipsec/manual/${user}.p12" ]; then
                 echo "✗ Missing IPsec certificate for ${user}"
                 exit 1
               fi
-              if [ ! -f "configs/localhost/ipsec/${user}.mobileconfig" ]; then
+              if [ ! -f "configs/localhost/ipsec/apple/${user}.mobileconfig" ]; then
                 echo "✗ Missing IPsec mobile config for ${user}"
                 exit 1
               fi
@@ -196,12 +214,22 @@ jobs:
       - name: Upload logs on failure
         if: failure()
         run: |
-          echo "=== Ansible Log ==="
-          sudo journalctl -u ansible --no-pager || true
+          echo "=== Network Interfaces ==="
+          ip addr || true
+          echo "=== Listening Ports ==="
+          sudo ss -tulnp || true
+          echo "=== WireGuard Status ==="
+          sudo wg show || true
+          echo "=== IPsec Status ==="
+          sudo ipsec statusall || true
+          echo "=== DNS Services ==="
+          sudo systemctl status dnscrypt-proxy dnscrypt-proxy.socket dnsmasq --no-pager || true
           echo "=== WireGuard Log ==="
-          sudo journalctl -u wg-quick@wg0 --no-pager || true
+          sudo journalctl -u wg-quick@wg0 -n 50 --no-pager || true
           echo "=== StrongSwan Log ==="
-          sudo journalctl -u strongswan --no-pager || true
+          sudo journalctl -u strongswan -n 50 --no-pager || true
+          echo "=== dnscrypt-proxy Log ==="
+          sudo journalctl -u dnscrypt-proxy -n 50 --no-pager || true
           echo "=== System Log (last 100 lines) ==="
           sudo journalctl -n 100 --no-pager || true
 

.github/workflows/lint.yml

@@ -14,15 +14,11 @@ jobs:
       - uses: actions/checkout@c2d88d3ecc89a9ef08eebf45d9637801dcee7eb5  # v5.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548  # v6.1.0
-        with:
-          python-version: '3.11'
 
-      - name: Setup uv environment
-        uses: ./.github/actions/setup-uv
-
-      - name: Install Ansible collections
-        run: uv run --with ansible-lint --with ansible ansible-galaxy collection install -r requirements.yml
+      - name: Setup Algo environment
+        uses: ./.github/actions/setup-algo
+        with:
+          install-ansible-collections: 'true'
 
       - name: Run ansible-lint
         run: |
@@ -59,12 +55,9 @@ jobs:
       - uses: actions/checkout@c2d88d3ecc89a9ef08eebf45d9637801dcee7eb5  # v5.0.1
         with:
           persist-credentials: false
-      - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548  # v6.1.0
-        with:
-          python-version: '3.11'
 
-      - name: Setup uv environment
-        uses: ./.github/actions/setup-uv
+      - name: Setup Algo environment
+        uses: ./.github/actions/setup-algo
 
       - name: Run ruff
         run: |
@@ -79,9 +72,13 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Setup Algo environment
+        uses: ./.github/actions/setup-algo
+        with:
+          install-shellcheck: 'true'
+
       - name: Run shellcheck
         run: |
-          sudo apt-get update && sudo apt-get install -y shellcheck
           # Check all shell scripts, not just algo and install.sh
           find . -type f -name "*.sh" -not -path "./.git/*" -exec shellcheck {} \;
 

.github/workflows/smart-tests.yml

@@ -61,7 +61,6 @@ jobs:
               - '**/*.sh'
               - '.ansible-lint'
               - '.yamllint'
-              - 'ruff.toml'
               - 'pyproject.toml'
             integration:
               - 'main.yml'
@@ -239,17 +238,25 @@ jobs:
       - name: Run relevant linters
         env:
           RUN_LINT: ${{ needs.changed-files.outputs.run_lint }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.sha }}
         run: |
-          # Always run if lint files changed
+          # Run linters if lint-related files changed
           if [[ "${RUN_LINT}" == "true" ]]; then
-            # Run all linters
-            uv run --with ruff ruff check . || true
-            uv run --with yamllint yamllint . || true
-            uv run --with ansible-lint ansible-lint || true
+            echo "Running linters..."
+
+            # Run Python linter
+            uv run --with ruff ruff check .
+
+            # Run YAML linter
+            uv run --with yamllint yamllint -c .yamllint .
+
+            # Run Ansible linter
+            uv run --with ansible-lint ansible-lint
 
             # Check shell scripts if any changed
-            if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q '\.sh$'; then
-              find . -name "*.sh" -type f -exec shellcheck {} + || true
+            if git diff --name-only "${BASE_SHA}" "${HEAD_SHA}" | grep -q '\.sh$'; then
+              find . -name "*.sh" -type f -not -path "./.git/*" -exec shellcheck {} +
             fi
           fi
 

roles/client/tasks/main.yml

@@ -6,8 +6,7 @@
 
 - name: Install prerequisites
   package: name="{{ item }}" state=present
-  with_items:
-    - "{{ prerequisites }}"
+  loop: "{{ prerequisites }}"
   register: result
   until: result is succeeded
   retries: 10
@@ -25,8 +24,6 @@
     src: roles/strongswan/templates/client_ipsec.conf.j2
     dest: "{{ configs_prefix }}/ipsec.{{ IP_subject_alt_name }}.conf"
     mode: "0644"
-  with_items:
-    - "{{ vpn_user }}"
   notify:
     - restart strongswan
 
@@ -35,8 +32,6 @@
     src: roles/strongswan/templates/client_ipsec.secrets.j2
     dest: "{{ configs_prefix }}/ipsec.{{ IP_subject_alt_name }}.secrets"
     mode: "0600"
-  with_items:
-    - "{{ vpn_user }}"
   notify:
     - restart strongswan
 
@@ -46,7 +41,7 @@
     line: "{{ item.line }}"
     create: true
     mode: "{{ item.mode }}"
-  with_items:
+  loop:
     - dest: "{{ configs_prefix }}/ipsec.conf"
       line: include ipsec.{{ IP_subject_alt_name }}.conf
       mode: '0644'
@@ -69,7 +64,7 @@
     src: "{{ item.src }}"
     dest: "{{ item.dest }}"
     mode: "{{ item.mode }}"
-  with_items:
+  loop:
     - src: configs/{{ IP_subject_alt_name }}/ipsec/.pki/certs/{{ vpn_user }}.crt
       dest: "{{ configs_prefix }}/ipsec.d/certs/{{ vpn_user }}.crt"
       mode: '0644'

roles/cloud-cloudstack/tasks/main.yml

@@ -23,7 +23,7 @@
         start_port: "{{ item.start_port }}"
         end_port: "{{ item.end_port }}"
         cidr: "{{ item.range }}"
-      with_items:
+      loop:
         - { proto: tcp, start_port: "{{ ssh_port }}", end_port: "{{ ssh_port }}", range: 0.0.0.0/0 }
         - { proto: udp, start_port: 4500, end_port: 4500, range: 0.0.0.0/0 }
         - { proto: udp, start_port: 500, end_port: 500, range: 0.0.0.0/0 }

roles/cloud-openstack/tasks/main.yml

@@ -23,7 +23,7 @@
     port_range_min: "{{ item.port_min }}"
     port_range_max: "{{ item.port_max }}"
     remote_ip_prefix: "{{ item.range }}"
-  with_items:
+  loop:
     - { proto: tcp, port_min: "{{ ssh_port }}", port_max: "{{ ssh_port }}", range: 0.0.0.0/0 }
     - { proto: icmp, port_min: -1, port_max: -1, range: 0.0.0.0/0 }
     - { proto: udp, port_min: 4500, port_max: 4500, range: 0.0.0.0/0 }
@@ -58,7 +58,7 @@
     - item['router:external']|default(omit)
     - item['admin_state_up']|default(omit)
     - item['status'] == 'ACTIVE'
-  with_items: "{{ os_network.openstack_networks }}"
+  loop: "{{ os_network.openstack_networks }}"
 
 - name: Set facts
   set_fact:

roles/cloud-vultr/tasks/main.yml

@@ -19,7 +19,7 @@
         ip_type: "{{ item.ip }}"
         subnet: "{{ item.cidr.split('/')[0] }}"
         subnet_size: "{{ item.cidr.split('/')[1] }}"
-      with_items:
+      loop:
         - { protocol: tcp, port: "{{ ssh_port }}", ip: v4, cidr: 0.0.0.0/0 }
         - { protocol: tcp, port: "{{ ssh_port }}", ip: v6, cidr: "::/0" }
         - { protocol: udp, port: 500, ip: v4, cidr: 0.0.0.0/0 }

roles/common/tasks/facts.yml

@@ -1,4 +1,11 @@
 ---
+- name: Set OS platform facts
+  set_fact:
+    is_debian_based: "{{ ansible_distribution in ['Debian', 'Ubuntu'] }}"
+    uses_systemd_socket: "{{ ansible_distribution in ['Debian', 'Ubuntu'] }}"
+    is_ubuntu_22_plus: "{{ ansible_distribution == 'Ubuntu' and ansible_distribution_version is version('22.04', '>=') }}"
+  tags: always
+
 - name: Define facts
   set_fact:
     p12_export_password: "{{ p12_password | default(lookup('password', '/dev/null length=9 chars=ascii_letters,digits,_,@')) }}"

roles/common/tasks/iptables.yml

@@ -6,7 +6,7 @@
     owner: root
     group: root
     mode: '0640'
-  with_items:
+  loop:
     - { src: rules.v4.j2, dest: /etc/iptables/rules.v4 }
   notify:
     - restart iptables
@@ -19,7 +19,7 @@
     group: root
     mode: '0640'
   when: ipv6_support
-  with_items:
+  loop:
     - { src: rules.v6.j2, dest: /etc/iptables/rules.v6 }
   notify:
     - restart iptables