Add Attic binary cache and containerize gitea runner

Replace nix-serve-only setup with Attic for managed binary caching with
upstream filtering and GC. Move gitea actions runner from host into an
isolated NixOS container with private networking. nix-serve kept alongside
Attic during migration.

.gitea/workflows/check-flake.yaml

@@ -5,6 +5,7 @@ on: [push]
 env:
   DEBIAN_FRONTEND: noninteractive
   PATH: /run/current-system/sw/bin/
+  XDG_CONFIG_HOME: ${{ runner.temp }}/.config
 
 jobs:
   check-flake:
@@ -15,5 +16,34 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Configure Attic cache
+        run: |
+          attic login local "${{ vars.ATTIC_ENDPOINT }}" "${{ secrets.ATTIC_TOKEN }}"
+          attic use local:nixos
+
       - name: Check Flake
         run: nix flake check --all-systems --print-build-logs --log-format raw --show-trace
+
+      - name: Build all systems
+        run: |
+          nix eval .#nixosConfigurations --apply 'cs: builtins.attrNames cs' --json \
+            | jq -r '.[]' \
+            | xargs -I{} nix build ".#nixosConfigurations.{}.config.system.build.toplevel" --no-link --print-build-logs --log-format raw
+
+      - name: Push to cache
+        run: |
+          set -euo pipefail
+          # Get all system toplevel store paths
+          toplevels=$(nix eval .#nixosConfigurations --apply 'cs: map (n: "${cs.${n}.config.system.build.toplevel}") (builtins.attrNames cs)' --json | jq -r '.[]')
+          echo "Found $(echo "$toplevels" | wc -l) system toplevels"
+          # Expand to full closures, deduplicate, and filter out paths that are:
+          # - already signed by cache.nixos.org (available upstream)
+          # - smaller than 0.5MB (insignificant build artifacts)
+          paths=$(echo "$toplevels" \
+            | xargs nix path-info -r --json \
+            | jq -r '[to_entries[] | select(
+                (.value.signatures | all(startswith("cache.nixos.org") | not))
+                and .value.narSize >= 524288
+              ) | .key] | unique[]')
+          echo "Pushing $(echo "$paths" | wc -l) unique paths to cache"
+          echo "$paths" | xargs attic push local:nixos

common/binary-cache.nix

@@ -1,4 +1,4 @@
-{ ... }:
+{ config, ... }:
 
 {
   nix = {
@@ -6,11 +6,11 @@
       substituters = [
         "https://cache.nixos.org/"
         "https://nix-community.cachix.org"
-        "http://s0.koi-bebop.ts.net:5000"
+        "http://s0.koi-bebop.ts.net:28338/nixos"
       ];
       trusted-public-keys = [
         "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
-        "s0.koi-bebop.ts.net:OjbzD86YjyJZpCp9RWaQKANaflcpKhtzBMNP8I2aPUU="
+        "nixos:SnTTQutdOJbAmxo6AQ3cbRt5w9f4byMXQODCieBH3PQ="
       ];
 
       # Allow substituters to be offline
@@ -19,6 +19,11 @@
       # and use this flag as intended for deciding if it should build missing
       # derivations locally. See https://github.com/NixOS/nix/issues/6901
       fallback = true;
+
+      # Authenticate to private nixos cache
+      netrc-file = config.age.secrets.attic-netrc.path;
     };
   };
+
+  age.secrets.attic-netrc.file = ../secrets/attic-netrc.age;
 }

common/server/atticd.nix

@@ -0,0 +1,31 @@
+{ config, lib, ... }:
+
+{
+  config = lib.mkIf (config.thisMachine.hasRole."binary-cache") {
+    services.atticd = {
+      enable = true;
+      environmentFile = config.age.secrets.atticd-credentials.path;
+      settings = {
+        listen = "[::]:28338";
+
+        chunking = {
+          nar-size-threshold = 64 * 1024; # 64 KiB
+
+          # The preferred minimum size of a chunk, in bytes
+          min-size = 16 * 1024; # 16 KiB
+
+          # The preferred average size of a chunk, in bytes
+          avg-size = 64 * 1024; # 64 KiB
+
+          # The preferred maximum size of a chunk, in bytes
+          max-size = 256 * 1024; # 256 KiB
+        };
+
+        compression.type = "zstd";
+        garbage-collection.default-retention-period = "6 months";
+      };
+    };
+
+    age.secrets.atticd-credentials.file = ../../secrets/atticd-credentials.age;
+  };
+}

common/server/default.nix

@@ -12,6 +12,7 @@
     ./mailserver.nix
     ./nextcloud.nix
     ./gitea-actions-runner.nix
+    ./atticd.nix
     ./librechat.nix
     ./actualbudget.nix
     ./unifi.nix

common/server/gitea-actions-runner.nix

@@ -1,132 +1,78 @@
-{ config, pkgs, lib, ... }:
+{ config, lib, ... }:
 
-# Gitea Actions Runner. Starts 'host' runner that runs directly on the host inside of a nixos container
+# Gitea Actions Runner inside a NixOS container.
-# This is useful for providing a real Nix/OS builder to gitea.
+# The container shares the host's /nix/store (read-only) and nix-daemon socket,
-# Warning, NixOS containers are not secure. For example, the container shares the /nix/store
+# so builds go through the host daemon and outputs land in the host store.
-# Therefore, this should not be used to run untrusted code.
+# Warning: NixOS containers are not fully secure — do not run untrusted code.
-# To enable, assign a machine the 'gitea-actions-runner' system role
+# To enable, assign a machine the 'gitea-actions-runner' system role.
-
-# TODO: skipping running inside of nixos container for now because of issues getting docker/podman running
 
 let
   thisMachineIsARunner = config.thisMachine.hasRole."gitea-actions-runner";
   containerName = "gitea-runner";
+  giteaRunnerUid = 991;
+  giteaRunnerGid = 989;
 in
 {
   config = lib.mkIf (thisMachineIsARunner && !config.boot.isContainer) {
-    # containers.${containerName} = {
-    #   ephemeral = true;
-    #   autoStart = true;
 
-    #   # for podman
+    containers.${containerName} = {
-    #   enableTun = true;
+      autoStart = true;
+      ephemeral = true;
 
-    #   # privateNetwork = true;
+      bindMounts = {
-    #   # hostAddress = "172.16.101.1";
+        "/run/agenix/gitea-actions-runner-token" = {
-    #   # localAddress = "172.16.101.2";
+          hostPath = "/run/agenix/gitea-actions-runner-token";
+          isReadOnly = true;
+        };
+        "/var/lib/gitea-runner" = {
+          hostPath = "/var/lib/gitea-runner";
+          isReadOnly = false;
+        };
+      };
 
-    #   bindMounts =
+      config = { config, lib, pkgs, ... }: {
-    #     {
+        system.stateVersion = "25.11";
-    #       "/run/agenix/gitea-actions-runner-token" = {
-    #         hostPath = "/run/agenix/gitea-actions-runner-token";
-    #         isReadOnly = true;
-    #       };
-    #       "/var/lib/gitea-runner" = {
-    #         hostPath = "/var/lib/gitea-runner";
-    #         isReadOnly = false;
-    #       };
-    #     };
 
-    #   extraFlags = [
+        services.gitea-actions-runner.instances.inst = {
-    #     # Allow podman
+          enable = true;
-    #     ''--system-call-filter=thisystemcalldoesnotexistforsure''
+          name = containerName;
-    #   ];
+          url = "https://git.neet.dev/";
+          tokenFile = "/run/agenix/gitea-actions-runner-token";
+          labels = [ "nixos:host" ];
+        };
 
-    #   additionalCapabilities = [
+        # Disable dynamic user so runner state persists via bind mount
-    #     "CAP_SYS_ADMIN"
+        systemd.services.gitea-runner-inst.serviceConfig.DynamicUser = lib.mkForce false;
-    #   ];
+        users.users.gitea-runner = {
+          uid = giteaRunnerUid;
+          home = "/var/lib/gitea-runner";
+          group = "gitea-runner";
+          isSystemUser = true;
+          createHome = true;
+        };
+        users.groups.gitea-runner.gid = giteaRunnerGid;
 
-    #   config = {
+        nix.settings.experimental-features = [ "nix-command" "flakes" ];
-    #     imports = allModules;
 
-    #     # speeds up evaluation
+        environment.systemPackages = with pkgs; [
-    #     nixpkgs.pkgs = pkgs;
+          git
-
+          nodejs
-    #     networking.hostName = lib.mkForce containerName;
+          jq
-
+          attic-client
-    #     # don't use remote builders
+        ];
-    #     nix.distributedBuilds = lib.mkForce false;
+      };
-
-    #     environment.systemPackages = with pkgs; [
-    #       git
-    #       # Gitea Actions rely heavily on node. Include it because it would be installed anyway.
-    #       nodejs
-    #     ];
-
-    #     services.gitea-actions-runner.instances.inst = {
-    #       enable = true;
-    #       name = config.networking.hostName;
-    #       url = "https://git.neet.dev/";
-    #       tokenFile = "/run/agenix/gitea-actions-runner-token";
-    #       labels = [
-    #         "ubuntu-latest:docker://node:18-bullseye"
-    #         "nixos:host"
-    #       ];
-    #     };
-
-    #     # To allow building on the host, must override the the service's config so it doesn't use a dynamic user
-    #     systemd.services.gitea-runner-inst.serviceConfig.DynamicUser = lib.mkForce false;
-    #     users.users.gitea-runner = {
-    #       home = "/var/lib/gitea-runner";
-    #       group = "gitea-runner";
-    #       isSystemUser = true;
-    #       createHome = true;
-    #     };
-    #     users.groups.gitea-runner = { };
-
-    #     virtualisation.podman.enable = true;
-    #     boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
-    #   };
-    # };
-
-    # networking.nat.enable = true;
-    # networking.nat.internalInterfaces = [
-    #   "ve-${containerName}"
-    # ];
-    # networking.ip_forward = true;
-
-    # don't use remote builders
-    nix.distributedBuilds = lib.mkForce false;
-
-    services.gitea-actions-runner.instances.inst = {
-      enable = true;
-      name = config.networking.hostName;
-      url = "https://git.neet.dev/";
-      tokenFile = "/run/agenix/gitea-actions-runner-token";
-      labels = [
-        "ubuntu-latest:docker://node:18-bullseye"
-        "nixos:host"
-      ];
     };
 
-    environment.systemPackages = with pkgs; [
+    # Matching user on host — the container's gitea-runner UID must be
-      git
+    # recognized by the host's nix-daemon as trusted (shared UID namespace)
-      # Gitea Actions rely heavily on node. Include it because it would be installed anyway.
-      nodejs
-    ];
-
-    # To allow building on the host, must override the the service's config so it doesn't use a dynamic user
-    systemd.services.gitea-runner-inst.serviceConfig.DynamicUser = lib.mkForce false;
     users.users.gitea-runner = {
+      uid = giteaRunnerUid;
       home = "/var/lib/gitea-runner";
       group = "gitea-runner";
       isSystemUser = true;
       createHome = true;
     };
-    users.groups.gitea-runner = { };
+    users.groups.gitea-runner.gid = giteaRunnerGid;
-
-    virtualisation.podman.enable = true;
-    boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 
     age.secrets.gitea-actions-runner-token.file = ../../secrets/gitea-actions-runner-token.age;
   };

secrets/secrets.nix

@@ -7,6 +7,9 @@ let
 
   # nobody is using this secret but I still need to be able to r/w it
   nobody = sshKeys.userKeys;
+
+  # For secrets that all machines need to know
+  everyone = roles.personal ++ roles.server;
 in
 
 with roles;
@@ -22,8 +25,10 @@ with roles;
   # nix binary cache
   # public key: s0.koi-bebop.ts.net:OjbzD86YjyJZpCp9RWaQKANaflcpKhtzBMNP8I2aPUU=
   "binary-cache-private-key.age".publicKeys = binary-cache;
-  # public key: ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINpUZFFL9BpBVqeeU63sFPhR9ewuhEZerTCDIGW1NPSB
+
-  "binary-cache-push-sshkey.age".publicKeys = nobody; # this value is directly given to gitea
+  # attic binary cache
+  "atticd-credentials.age".publicKeys = binary-cache;
+  "attic-netrc.age".publicKeys = everyone;
 
   # vpn
   "pia-login.age".publicKeys = pia;