Borg Backups

ansible/playbook.yml

@@ -68,6 +68,7 @@
     - postgres
     - prometheus-postgres-exporter
     - mongodb
+    - borg
 
 - name: Deploy our LDAP server environment to the LDAP host
   hosts: ldap

ansible/roles/borg/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - systemd

ansible/roles/borg/tasks/main.yml

@@ -0,0 +1,117 @@
+---
+- name: Install borg dependencies
+  package:
+    name:
+      - liblz4-dev
+      - liblz4-1
+      - libzstd-dev
+      - libzstd1
+      - libacl1-dev
+      - libacl1
+      - libxxhash0
+    state: present
+  tags:
+    - role::borg
+
+- name: Create borg service user account
+  user:
+    state: present
+    system: true
+    name: "{{ borg_user }}"
+    home: "{{ borg_home }}"
+    shell: /usr/sbin/nologin
+  tags:
+    - role::borg
+
+- name: Create borgmatic virtual environment
+  command:
+    cmd: python3 -m venv {{ borg_virtualenv }}
+    creates: "{{ borg_virtualenv }}/bin/activate"
+  tags:
+    - role::borg
+
+- name: Install borg & dependencies to virtual environment
+  pip:
+    virtualenv: "{{ borg_virtualenv }}"
+    name:
+      - borgbackup[s3] @ git+https://github.com/borgbackup/borg.git@{{ borg_git_version }}
+      - borgmatic=={{ borgmatic_version }}
+    state: present
+  tags:
+    - role::borg
+
+- name: Template Borg environment variables
+  template:
+    src: borgmatic.env.j2
+    dest: /etc/default/borgmatic
+    mode: "0444"
+    owner: root
+    group: root
+  tags:
+    - role::borg
+
+- name: Create borgmatic configuration directory
+  file:
+    state: directory
+    owner: borg
+    group: root
+    mode: "0755"
+    path: /etc/borgmatic
+  tags:
+    - role::borg
+
+- name: Template Borg configuration file
+  template:
+    src: config.yaml.j2
+    dest: /etc/borgmatic/config.yaml
+    mode: "0444"
+    owner: borg
+    group: root
+  tags:
+    - role::borg
+
+- name: Create borgmatic service
+  template:
+    src: borgmatic.service.j2
+    dest: /etc/systemd/system/borgmatic.service
+    owner: root
+    group: root
+    mode: "0444"
+  tags:
+    - role::borg
+  notify:
+    - Reload the systemd daemon
+
+- name: Create borgmatic timer
+  template:
+    src: borgmatic.timer.j2
+    dest: /etc/systemd/system/borgmatic.timer
+    owner: root
+    group: root
+    mode: "0444"
+  tags:
+    - role::borg
+  notify:
+    - Reload the systemd daemon
+
+- name: Enable borgmatic timer
+  systemd:
+    name: borgmatic.timer
+    state: started
+    enabled: true
+    masked: false
+  tags:
+    - role::borg
+  notify:
+    - Reload the systemd daemon
+
+- name: Create borgmatic repositories
+  become_user: "{{ borg_user }}"
+  command: "{{ borg_virtualenv }}/bin/borgmatic repo-create --encryption=none"
+  environment:
+    POSTGRES_PASSWORD: "{{ vault_postgres_user_passwords.borg }}"
+    BORG_S3_KEY_ID: "{{ borg_s3_access_key_id }}"
+    BORG_S3_KEY_SECRET: "{{ borg_s3_access_key_secret }}"
+  #   creates: "{{ borg_home }}/keys/repositories/borg2"
+  tags:
+    - role::borg

ansible/roles/borg/templates/borgmatic.env.j2

@@ -0,0 +1,3 @@
+POSTGRES_PASSWORD="{{ vault_postgres_user_passwords.borg }}"
+BORG_S3_KEY_ID="{{ borg_s3_access_key_id }}"
+BORG_S3_KEY_SECRET="{{ borg_s3_access_key_secret }}"

ansible/roles/borg/templates/borgmatic.service.j2

@@ -0,0 +1,66 @@
+[Unit]
+Description=borgmatic backup
+Wants=network-online.target
+After=network-online.target
+Documentation=https://torsion.org/borgmatic/
+
+[Service]
+User={{ borg_user }}
+EnvironmentFile=/etc/default/borgmatic
+Type=oneshot
+RuntimeDirectory=borgmatic
+StateDirectory=borgmatic
+
+# Security settings for systemd running as root, optional but recommended to improve security. You
+# can disable individual settings if they cause problems for your use case. For more details, see
+# the systemd manual: https://www.freedesktop.org/software/systemd/man/systemd.exec.html
+LockPersonality=true
+# Certain borgmatic features like Healthchecks integration need MemoryDenyWriteExecute to be off.
+# But you can try setting it to "yes" for improved security if you don't use those features.
+MemoryDenyWriteExecute=no
+NoNewPrivileges=yes
+# Filesystem hooks like ZFS may not work unless PrivateDevices is disabled.
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHostname=yes
+ProtectKernelLogs=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallFilter=@system-service @mount
+SystemCallErrorNumber=EPERM
+# To restrict write access further, change "ProtectSystem" to "strict" and
+# uncomment "ReadWritePaths", "TemporaryFileSystem", "BindPaths" and
+# "BindReadOnlyPaths". Then add any local repository paths to the list of
+# "ReadWritePaths". This leaves most of the filesystem read-only to borgmatic.
+ProtectSystem=full
+# ReadWritePaths=-/mnt/my_backup_drive
+# This will mount a tmpfs on top of /root and pass through needed paths
+# TemporaryFileSystem=/root:ro
+# BindPaths=-/root/.cache/borg -/root/.config/borg -/root/.borgmatic
+# BindReadOnlyPaths=-/root/.ssh
+
+# May interfere with running external programs within borgmatic hooks. This
+# includes, for instance, programs to snapshot filesystems (e.g. ZFS).
+CapabilityBoundingSet=CAP_DAC_READ_SEARCH CAP_NET_RAW
+
+# Lower CPU and I/O priority.
+Nice=19
+CPUSchedulingPolicy=batch
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+IOWeight=100
+
+Restart=no
+# Prevent rate limiting of borgmatic log events. If you are using an older version of systemd that
+# doesn't support this (pre-240 or so), you may have to remove this option.
+LogRateLimitIntervalSec=0
+
+ExecStartPre={{ borg_virtualenv }}/bin/borgmatic config validate
+ExecStart={{ borg_virtualenv }}/bin/borgmatic --config /etc/borgmatic/config.yaml

ansible/roles/borg/templates/borgmatic.timer.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Run borgmatic backup
+
+[Timer]
+OnCalendar={{ borg_timer }}
+Persistent=true
+
+[Install]
+WantedBy=timers.target

ansible/roles/borg/templates/config.yaml.j2

@@ -0,0 +1,138 @@
+repositories:
+    - path: b2:${BORG_S3_KEY_ID}:${BORG_S3_KEY_SECRET}@https://eu-central-1.linodeobjects.com/borg/database-backups/
+      label: frankfurt
+    - path: b2:${BORG_S3_KEY_ID}:${BORG_S3_KEY_SECRET}@https://us-east-1.linodeobjects.com/borg/database-backups/
+      label: newark
+
+# Path for storing temporary runtime data like streaming database
+# dumps and bootstrap metadata. borgmatic automatically creates and
+# uses a "borgmatic" subdirectory here. Defaults to $XDG_RUNTIME_DIR
+# or or $TMPDIR or $TEMP or /run/user/$UID.
+# user_runtime_directory: /run/user/1001
+
+retries: {{ borg_borgmatic_retries }}
+retry_wait: {{ borg_borgmatic_retry_wait }}
+
+working_directory: {{ borg_home }}
+
+temporary_directory: /tmp
+borg_base_directory: /etc/borgmatic
+borg_config_directory: /etc/borgmatic/config
+borg_cache_directory: {{ borg_home }}/cache
+borg_security_directory: {{ borg_home }}/security
+borg_keys_directory: {{ borg_home }}/keys
+
+archive_name_format: '{hostname}-databases-{now}'
+
+keep_daily: {{ borg_borgmatic_keep_daily }}
+
+read_special: true
+
+local_path: {{ borg_virtualenv }}/bin/borg
+
+# List of one or more shell commands or scripts to execute after
+# creating a backup, run once per repository.
+# after_backup:
+    # - echo Finished a backup.
+
+# List of one or more shell commands or scripts to execute when an
+# exception occurs during a "create", "prune", "compact", or "check"
+# action or an associated before/after hook.
+# on_error:
+    # - echo Error during create/prune/compact/check.
+
+# List of one or more shell commands or scripts to execute after
+# running all actions (if one of them is "create"). These are
+# collected from all configuration files and then run once after all
+# of them (after any action).
+# after_everything:
+    # - echo Completed actions.
+
+
+postgresql_databases:
+    # Database name (required if using this hook). Or "all" to
+    # dump all databases on the host. (Also set the "format"
+    # to dump each database to a separate file instead of one
+    # combined file.) Note that using this database hook
+    # implicitly enables read_special (see above) to support
+    # dump and restore streaming.
+    - name: all
+      username: borg
+      password: ${POSTGRES_PASSWORD}
+      format: custom
+      # Disabled since Borg has built-in compression
+      compression: none
+
+      # SSL mode to use to connect to the database server. One
+      # of "disable", "allow", "prefer", "require", "verify-ca"
+      # or "verify-full". Defaults to "disable".
+      # ssl_mode: require
+
+      # Path to a client certificate.
+      # ssl_cert: /root/.postgresql/postgresql.crt
+
+      # Path to a private client key.
+      # ssl_key: /root/.postgresql/postgresql.key
+
+      # Path to a root certificate containing a list of trusted
+      # certificate authorities.
+      # ssl_root_cert: /root/.postgresql/root.crt
+
+      # Path to a certificate revocation list.
+      # ssl_crl: /root/.postgresql/root.crl
+
+
+# mongodb_databases:
+    # - name: all
+      # hostname: database.example.org
+
+      # Username with which to connect to the database. Skip it
+      # if no authentication is needed. Supports the
+      # "{credential ...}" syntax.
+      # username: dbuser
+
+      # Password with which to connect to the database. Skip it
+      # if no authentication is needed. Supports the
+      # "{credential ...}" syntax.
+      # password: trustsome1
+
+
+# Configuration for a monitoring integration with Grafana Loki. You
+# can send the logs to a self-hosted instance or create an account at
+# https://grafana.com/auth/sign-up/create-user. See borgmatic
+# monitoring documentation for details.
+# loki:
+    # Grafana loki log URL to notify when a backup begins,
+    # ends, or fails.
+    # url: http://localhost:3100/loki/api/v1/push
+
+    # Allows setting custom labels for the logging stream. At
+    # least one label is required. "__hostname" gets replaced by
+    # the machine hostname automatically. "__config" gets replaced
+    # by the name of the configuration file. "__config_path" gets
+    # replaced by the full path of the configuration file.
+    # labels:
+        # app: borgmatic
+        # config: __config
+        # hostname: __hostname
+
+# Configuration for a monitoring integration with Sentry. You can use
+# a self-hosted instance via https://develop.sentry.dev/self-hosted/
+# or create a cloud-hosted account at https://sentry.io. See borgmatic
+# monitoring documentation for details.
+# sentry:
+    # Sentry Data Source Name (DSN) URL, associated with a
+    # particular Sentry project. Used to construct a cron URL,
+    # notified when a backup begins, ends, or errors.
+    # data_source_name_url: https://[email protected]/203069
+
+    # Sentry monitor slug, associated with a particular Sentry
+    # project monitor. Used along with the data source name URL to
+    # construct a cron URL.
+    # monitor_slug: mymonitor
+
+    # List of one or more monitoring states to ping for: "start",
+    # "finish", and/or "fail". Defaults to pinging for all states.
+    # states:
+        # - start
+        # - finish

ansible/roles/borg/vars/main/keys.yml

@@ -0,0 +1,11 @@
+$ANSIBLE_VAULT;1.1;AES256
+34326663643266306162376261383566646364376639613137646237313636303430363739646631
+3762303366303532386362356135313462613639323330630a333937623737366231656264623065
+66643731373537326437383465386139373539653030343930646434313262386565323637356539
+3962383331353331310a363163383538636133623137633930393632653930643664306663366363
+62613233383935633533343861343864363131313435383237333738343766306565353463653337
+35336632646538373539333563393433336532393137633062353265636533666134346464383961
+33323562643462646436366133333566653565613537326137393164306165386166633539383663
+38616462386432623531653564623032626632353561656431336634373530633038626665633230
+33636639643037366332613264356463633262653663393531643932636335353163396431393465
+3162666530626433313865656164613434626334353465333738

ansible/roles/borg/vars/main/main.yml

@@ -0,0 +1,22 @@
+---
+borg_user: borg
+borg_home: /var/lib/borg
+borg_virtualenv: "{{ borg_home }}/borgmatic-venv"
+
+# Pinned to a specific commit for S3 compatibility.
+borg_git_version: "1373b30ba8e6b88f6264dc57f6a8da46d8b2e80f"
+borgmatic_version: "2.0.11"
+
+# see https://wiki.archlinux.org/title/Systemd/Timers
+# or for more details https://www.freedesktop.org/software/systemd/man/latest/systemd.time.html
+borg_timer: "*-*-* 09:00:00"
+
+
+# Borgmatic config, see https://torsion.org/borgmatic/reference/configuration/
+borg_borgmatic_retries: "1"
+borg_borgmatic_retry_wait: "3"
+borg_borgmatic_keep_daily: "7"
+
+
+borg_s3_access_key_id: "{{ vault_borg_s3_access_key_id }}"
+borg_s3_access_key_secret: "{{ vault_borg_s3_access_key_secret }}"