From cf71b74d6fb862bfb66ebd4881823fea5a65a089 Mon Sep 17 00:00:00 2001 From: Zuckerberg Date: Sun, 8 Feb 2026 14:55:48 -0800 Subject: [PATCH] Add Incus container support to sandboxed workspaces - Add incus.nix module for fully declarative Incus/LXC containers - Build NixOS LXC images using nixpkgs.lib.nixosSystem - Ephemeral containers: recreated on each start, cleaned up on stop - Use flock to serialize concurrent container operations - Deterministic MAC addresses via lib.mkMac to prevent ARP cache issues - Add veth* to NetworkManager unmanaged interfaces - Update CLAUDE.md with coding conventions and shared lib docs --- CLAUDE.md | 22 +++ common/network/sandbox.nix | 1 + common/sandboxed-workspace/base.nix | 1 + common/sandboxed-workspace/default.nix | 24 ++- common/sandboxed-workspace/incus.nix | 173 +++++++++++++++++++++ common/sandboxed-workspace/vm.nix | 33 +++- lib/default.nix | 9 ++ machines/fry/workspaces/test-container.nix | 2 +- 8 files changed, 252 insertions(+), 13 deletions(-) create mode 100644 common/sandboxed-workspace/incus.nix diff --git a/CLAUDE.md b/CLAUDE.md index 4314fda..f6f64c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,9 +48,31 @@ Configuration: `common/sandboxed-workspace/` - `common/` - Shared NixOS modules for all machines - `home/` - Home Manager configurations +- `lib/` - Custom lib functions (extends nixpkgs lib, accessible as `lib.*` in modules) - `machines/` - Per-machine configurations - `skills/` - Claude Code skills for common tasks +## Shared Library + +Custom utility functions go in `lib/default.nix`. The flake extends `nixpkgs.lib` with these functions, so they're accessible as `lib.functionName` in all modules. Add reusable functions here when used in multiple places. + +## Code Comments + +Only add comments that provide value beyond what the code already shows: +- Explain *why* something is done, not *what* is being done +- Document non-obvious constraints or gotchas +- Never add filler comments that repeat the code (e.g. `# Start the service` before a start command) + +## Bash Commands + +Do not redirect stderr to stdout (no `2>&1`). This can hide important output and errors. + +Do not use `doas` or `sudo` - they will fail. Ask the user to run privileged commands themselves. + +## Nix Commands + +Use `--no-link` with `nix build` to avoid creating `result` symlinks in the working directory. + ## Git Commits Do not add "Co-Authored-By" lines to commit messages. diff --git a/common/network/sandbox.nix b/common/network/sandbox.nix index 045f9fb..c9cf662 100644 --- a/common/network/sandbox.nix +++ b/common/network/sandbox.nix @@ -99,6 +99,7 @@ in "interface-name:${cfg.bridgeName}" "interface-name:vm-*" "interface-name:ve-*" + "interface-name:veth*" ]; # Make systemd-resolved listen on the bridge for workspace DNS queries. diff --git a/common/sandboxed-workspace/base.nix b/common/sandboxed-workspace/base.nix index 75d7dfb..11947e0 100644 --- a/common/sandboxed-workspace/base.nix +++ b/common/sandboxed-workspace/base.nix @@ -67,6 +67,7 @@ # Basic system packages environment.systemPackages = with pkgs; [ + claude-code kakoune vim git diff --git a/common/sandboxed-workspace/default.nix b/common/sandboxed-workspace/default.nix index cd8924f..1bc77df 100644 --- a/common/sandboxed-workspace/default.nix +++ b/common/sandboxed-workspace/default.nix @@ -12,6 +12,7 @@ in imports = [ ./vm.nix ./container.nix + ./incus.nix ]; options.sandboxed-workspace = { @@ -21,11 +22,12 @@ in type = types.attrsOf (types.submodule { options = { type = mkOption { - type = types.enum [ "vm" "container" ]; + type = types.enum [ "vm" "container" "incus" ]; description = '' Backend type for this workspace: - "vm": microVM with cloud-hypervisor (more isolation, uses virtiofs) - - "container": systemd-nspawn container (less overhead, uses bind mounts) + - "container": systemd-nspawn via NixOS containers (less overhead, uses bind mounts) + - "incus": Incus/LXD container (unprivileged, better security than NixOS containers) ''; }; @@ -102,11 +104,13 @@ in cfg.workspaces); # Shell aliases for workspace management - # Service names differ by type: microvm@ for VMs, container@ for containers environment.shellAliases = lib.mkMerge (lib.mapAttrsToList (name: ws: let - serviceName = if ws.type == "vm" then "microvm@${name}" else "container@${name}"; + serviceName = + if ws.type == "vm" then "microvm@${name}" + else if ws.type == "incus" then "incus-workspace-${name}" + else "container@${name}"; in { "workspace_${name}" = "ssh googlebot@workspace-${name}"; @@ -144,7 +148,10 @@ in (lib.mapAttrs' (name: ws: let - serviceName = if ws.type == "vm" then "microvm@${name}" else "container@${name}"; + serviceName = + if ws.type == "vm" then "microvm@${name}" + else if ws.type == "incus" then "incus-workspace-${name}" + else "container@${name}"; claudeConfig = builtins.toJSON { hasCompletedOnboarding = true; theme = "dark"; @@ -187,6 +194,13 @@ in echo '${claudeConfig}' > /home/googlebot/sandboxed/${name}/claude-config/.claude.json chown googlebot:users /home/googlebot/sandboxed/${name}/claude-config/.claude.json fi + '' + lib.optionalString (ws.type == "incus") '' + # Copy credentials for incus (can't use bind mount for files inside another mount) + if [ -f /home/googlebot/.claude/.credentials.json ]; then + cp /home/googlebot/.claude/.credentials.json /home/googlebot/sandboxed/${name}/claude-config/.credentials.json + chown googlebot:users /home/googlebot/sandboxed/${name}/claude-config/.credentials.json + chmod 600 /home/googlebot/sandboxed/${name}/claude-config/.credentials.json + fi ''; } ) diff --git a/common/sandboxed-workspace/incus.nix b/common/sandboxed-workspace/incus.nix new file mode 100644 index 0000000..55bc57a --- /dev/null +++ b/common/sandboxed-workspace/incus.nix @@ -0,0 +1,173 @@ +{ config, lib, pkgs, ... }: + +# Incus-specific configuration for sandboxed workspaces +# Creates fully declarative Incus containers from NixOS configurations + +with lib; + +let + cfg = config.sandboxed-workspace; + hostConfig = config; + + incusWorkspaces = filterAttrs (n: ws: ws.type == "incus") cfg.workspaces; + + # Build a NixOS LXC image for a workspace + mkContainerImage = name: ws: + let + nixpkgs = hostConfig.inputs.nixpkgs; + containerSystem = nixpkgs.lib.nixosSystem { + modules = [ + (import ./base.nix { + inherit hostConfig; + workspaceName = name; + ip = ws.ip; + networkInterface = { Name = "eth0"; }; + }) + + (import ws.config) + + ({ config, lib, pkgs, ... }: { + nixpkgs.hostPlatform = hostConfig.currentSystem; + boot.isContainer = true; + networking.useHostResolvConf = false; + nixpkgs.config.allowUnfree = true; + }) + ]; + }; + in + { + rootfs = containerSystem.config.system.build.images.lxc; + metadata = containerSystem.config.system.build.images.lxc-metadata; + toplevel = containerSystem.config.system.build.toplevel; + }; + + mkIncusService = name: ws: + let + images = mkContainerImage name ws; + hash = builtins.substring 0 12 (builtins.hashString "sha256" "${images.rootfs}"); + imageName = "nixos-workspace-${name}-${hash}"; + containerName = "workspace-${name}"; + + bridgeName = config.networking.sandbox.bridgeName; + mac = lib.mkMac "incus-${name}"; + + addDevices = '' + incus config device add ${containerName} eth0 nic nictype=bridged parent=${bridgeName} hwaddr=${mac} + incus config device add ${containerName} workspace disk source=/home/googlebot/sandboxed/${name}/workspace path=/home/googlebot/workspace shift=true + incus config device add ${containerName} ssh-keys disk source=/home/googlebot/sandboxed/${name}/ssh-host-keys path=/etc/ssh-host-keys shift=true + incus config device add ${containerName} claude-config disk source=/home/googlebot/sandboxed/${name}/claude-config path=/home/googlebot/claude-config shift=true + ''; + in + { + description = "Incus workspace ${name}"; + after = [ "incus.service" "incus-preseed.service" "workspace-${name}-setup.service" ]; + requires = [ "incus.service" ]; + wants = [ "workspace-${name}-setup.service" ]; + wantedBy = optional ws.autoStart "multi-user.target"; + + path = [ config.virtualisation.incus.package pkgs.gnutar pkgs.xz pkgs.util-linux ]; + + restartTriggers = [ images.rootfs images.metadata ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + + script = '' + set -euo pipefail + + # Serialize incus operations - concurrent container creation causes race conditions + exec 9>/run/incus-workspace.lock + flock -x 9 + + # Import image if not present + if ! incus image list --format csv | grep -q "${imageName}"; then + metadata_tarball=$(echo ${images.metadata}/tarball/*.tar.xz) + rootfs_tarball=$(echo ${images.rootfs}/tarball/*.tar.xz) + incus image import "$metadata_tarball" "$rootfs_tarball" --alias ${imageName} + + # Clean up old images for this workspace + incus image list --format csv | grep "nixos-workspace-${name}-" | grep -v "${imageName}" | cut -d, -f2 | while read old_image; do + incus image delete "$old_image" || true + done || true + fi + + # Always recreate container for ephemeral behavior + incus stop ${containerName} --force 2>/dev/null || true + incus delete ${containerName} --force 2>/dev/null || true + + incus init ${imageName} ${containerName} + ${addDevices} + incus start ${containerName} + + # Wait for container to start + for i in $(seq 1 30); do + if incus list --format csv | grep -q "^${containerName},RUNNING"; then + exit 0 + fi + sleep 1 + done + + exit 1 + ''; + + preStop = '' + exec 9>/run/incus-workspace.lock + flock -x 9 + + incus stop ${containerName} --force 2>/dev/null || true + incus delete ${containerName} --force 2>/dev/null || true + + # Clean up all images for this workspace + incus image list --format csv 2>/dev/null | grep "nixos-workspace-${name}-" | cut -d, -f2 | while read img; do + incus image delete "$img" 2>/dev/null || true + done + ''; + }; + + +in +{ + config = mkIf (cfg.enable && incusWorkspaces != { }) { + + virtualisation.incus.enable = true; + networking.nftables.enable = true; + + virtualisation.incus.preseed = { + storage_pools = [{ + name = "default"; + driver = "dir"; + config = { + source = "/var/lib/incus/storage-pools/default"; + }; + }]; + + profiles = [{ + name = "default"; + config = { + "security.privileged" = "false"; + "security.idmap.isolated" = "true"; + }; + devices = { + root = { + path = "/"; + pool = "default"; + type = "disk"; + }; + }; + }]; + }; + + systemd.services = mapAttrs' + (name: ws: nameValuePair "incus-workspace-${name}" (mkIncusService name ws)) + incusWorkspaces; + + # Extra alias for incus shell access (ssh is also available via default.nix aliases) + environment.shellAliases = mkMerge (mapAttrsToList + (name: ws: { + "workspace_${name}_shell" = "doas incus exec workspace-${name} -- su -l googlebot"; + }) + incusWorkspaces); + }; +} diff --git a/common/sandboxed-workspace/vm.nix b/common/sandboxed-workspace/vm.nix index 10dc45f..4ca3073 100644 --- a/common/sandboxed-workspace/vm.nix +++ b/common/sandboxed-workspace/vm.nix @@ -1,4 +1,4 @@ -{ config, lib, ... }: +{ config, lib, pkgs, ... }: # VM-specific configuration for sandboxed workspaces using microvm.nix # This module is imported by default.nix for workspaces with type = "vm" @@ -47,6 +47,7 @@ let # Generate VM configuration for a workspace mkVmConfig = name: ws: { + inherit pkgs; # Use host's pkgs (includes allowUnfree) config = import ws.config; specialArgs = { inputs = hostConfig.inputs; }; extraModules = [ @@ -57,6 +58,21 @@ let networkInterface = { Type = "ether"; }; }) { + # Copy credentials from host mount to per-workspace config + systemd.services.claude-credentials = { + description = "Copy Claude credentials from host"; + wantedBy = [ "multi-user.target" ]; + before = [ "multi-user.target" ]; + serviceConfig.Type = "oneshot"; + script = '' + if [ -f /home/googlebot/.claude-credentials/.credentials.json ]; then + install -m 600 -o googlebot -g users \ + /home/googlebot/.claude-credentials/.credentials.json \ + /home/googlebot/claude-config/.credentials.json + fi + ''; + }; + # MicroVM specific configuration microvm = { # Use cloud-hypervisor for better performance @@ -100,6 +116,14 @@ let source = "/home/googlebot/sandboxed/${name}/claude-config"; mountPoint = "/home/googlebot/claude-config"; } + { + # Credentials-only directory (read-only) + # This directory should contain only .credentials.json + proto = "virtiofs"; + tag = "claude-credentials"; + source = "/home/googlebot/.claude-credentials"; + mountPoint = "/home/googlebot/.claude-credentials"; + } ]; # Writeable overlay for /nix/store @@ -111,12 +135,7 @@ let interfaces = [{ type = "tap"; id = "vm-${name}"; - # Generate a deterministic MAC from workspace name (02: prefix = locally administered) - mac = - let - hash = builtins.hashString "sha256" name; - in - "02:${builtins.substring 0 2 hash}:${builtins.substring 2 2 hash}:${builtins.substring 4 2 hash}:${builtins.substring 6 2 hash}:${builtins.substring 8 2 hash}"; + mac = lib.mkMac "vm-${name}"; }]; # Enable vsock for systemd-notify integration diff --git a/lib/default.nix b/lib/default.nix index ddfc106..ed95f19 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -53,4 +53,13 @@ with lib; getElem = x: y: elemAt (elemAt ll y) x; in genList (y: genList (x: f x y (getElem x y)) innerSize) outerSize; + + # Generate a deterministic MAC address from a name + # Uses locally administered unicast range (02:xx:xx:xx:xx:xx) + mkMac = name: + let + hash = builtins.hashString "sha256" name; + octets = map (i: builtins.substring i 2 hash) [ 0 2 4 6 8 ]; + in + "02:${builtins.concatStringsSep ":" octets}"; } diff --git a/machines/fry/workspaces/test-container.nix b/machines/fry/workspaces/test-container.nix index a634e2d..e08984c 100644 --- a/machines/fry/workspaces/test-container.nix +++ b/machines/fry/workspaces/test-container.nix @@ -4,7 +4,7 @@ # # Add to sandboxed-workspace.workspaces in machines/fry/default.nix: # sandboxed-workspace.workspaces.test-container = { -# type = "container"; +# type = "container" OR "incus"; # config = ./workspaces/test-container.nix; # ip = "192.168.83.50"; # };