zuckerberg/nix-config
1
0
Fork 0
Code Issues Pull Requests Actions Activity

Add Incus container support to sandboxed workspaces

Browse Source 
- Add incus.nix module for fully declarative Incus/LXC containers
- Build NixOS LXC images using nixpkgs.lib.nixosSystem
- Ephemeral containers: recreated on each start, cleaned up on stop
- Use flock to serialize concurrent container operations
- Deterministic MAC addresses via lib.mkMac to prevent ARP cache issues
- Add veth* to NetworkManager unmanaged interfaces
- Update CLAUDE.md with coding conventions and shared lib docs
This commit is contained in:
Zuckerberg
2026-02-08 14:55:48 -08:00
parent 5178ea6835
commit cf71b74d6f
8 changed files with 252 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
common/network/sandbox.nix
View File

@@ -99,6 +99,7 @@ in
"interface-name:${cfg.bridgeName}"
"interface-name:vm-*"
"interface-name:ve-*"
"interface-name:veth*"
];
# Make systemd-resolved listen on the bridge for workspace DNS queries.

1
common/sandboxed-workspace/base.nix
View File

@@ -67,6 +67,7 @@
# Basic system packages
environment.systemPackages = with pkgs; [
claude-code
kakoune
vim
git

24
common/sandboxed-workspace/default.nix
View File

@@ -12,6 +12,7 @@ in
imports = [
./vm.nix
./container.nix
./incus.nix
];
options.sandboxed-workspace = {
@@ -21,11 +22,12 @@ in
type = types.attrsOf (types.submodule {
options = {
type = mkOption {
type = types.enum [ "vm" "container" ];
type = types.enum [ "vm" "container" "incus" ];
description = ''
Backend type for this workspace:
- "vm": microVM with cloud-hypervisor (more isolation, uses virtiofs)
- "container": systemd-nspawn container (less overhead, uses bind mounts)
- "container": systemd-nspawn via NixOS containers (less overhead, uses bind mounts)
- "incus": Incus/LXD container (unprivileged, better security than NixOS containers)
'';
};
@@ -102,11 +104,13 @@ in
cfg.workspaces);
# Shell aliases for workspace management
# Service names differ by type: microvm@<name> for VMs, container@<name> for containers
environment.shellAliases = lib.mkMerge (lib.mapAttrsToList
(name: ws:
let
serviceName = if ws.type == "vm" then "microvm@${name}" else "container@${name}";
serviceName =
if ws.type == "vm" then "microvm@${name}"
else if ws.type == "incus" then "incus-workspace-${name}"
else "container@${name}";
in
{
"workspace_${name}" = "ssh googlebot@workspace-${name}";
@@ -144,7 +148,10 @@ in
(lib.mapAttrs'
(name: ws:
let
serviceName = if ws.type == "vm" then "microvm@${name}" else "container@${name}";
serviceName =
if ws.type == "vm" then "microvm@${name}"
else if ws.type == "incus" then "incus-workspace-${name}"
else "container@${name}";
claudeConfig = builtins.toJSON {
hasCompletedOnboarding = true;
theme = "dark";
@@ -187,6 +194,13 @@ in
echo '${claudeConfig}' > /home/googlebot/sandboxed/${name}/claude-config/.claude.json
chown googlebot:users /home/googlebot/sandboxed/${name}/claude-config/.claude.json
fi
'' + lib.optionalString (ws.type == "incus") ''
# Copy credentials for incus (can't use bind mount for files inside another mount)
if [ -f /home/googlebot/.claude/.credentials.json ]; then
cp /home/googlebot/.claude/.credentials.json /home/googlebot/sandboxed/${name}/claude-config/.credentials.json
chown googlebot:users /home/googlebot/sandboxed/${name}/claude-config/.credentials.json
chmod 600 /home/googlebot/sandboxed/${name}/claude-config/.credentials.json
fi
'';
}
)

173
common/sandboxed-workspace/incus.nix Normal file
View File

@@ -0,0 +1,173 @@
{ config, lib, pkgs, ... }:
# Incus-specific configuration for sandboxed workspaces
# Creates fully declarative Incus containers from NixOS configurations
with lib;
let
cfg = config.sandboxed-workspace;
hostConfig = config;
incusWorkspaces = filterAttrs (n: ws: ws.type == "incus") cfg.workspaces;
# Build a NixOS LXC image for a workspace
mkContainerImage = name: ws:
let
nixpkgs = hostConfig.inputs.nixpkgs;
containerSystem = nixpkgs.lib.nixosSystem {
modules = [
(import ./base.nix {
inherit hostConfig;
workspaceName = name;
ip = ws.ip;
networkInterface = { Name = "eth0"; };
})
(import ws.config)
({ config, lib, pkgs, ... }: {
nixpkgs.hostPlatform = hostConfig.currentSystem;
boot.isContainer = true;
networking.useHostResolvConf = false;
nixpkgs.config.allowUnfree = true;
})
];
};
in
{
rootfs = containerSystem.config.system.build.images.lxc;
metadata = containerSystem.config.system.build.images.lxc-metadata;
toplevel = containerSystem.config.system.build.toplevel;
};
mkIncusService = name: ws:
let
images = mkContainerImage name ws;
hash = builtins.substring 0 12 (builtins.hashString "sha256" "${images.rootfs}");
imageName = "nixos-workspace-${name}-${hash}";
containerName = "workspace-${name}";
bridgeName = config.networking.sandbox.bridgeName;
mac = lib.mkMac "incus-${name}";
addDevices = ''
incus config device add ${containerName} eth0 nic nictype=bridged parent=${bridgeName} hwaddr=${mac}
incus config device add ${containerName} workspace disk source=/home/googlebot/sandboxed/${name}/workspace path=/home/googlebot/workspace shift=true
incus config device add ${containerName} ssh-keys disk source=/home/googlebot/sandboxed/${name}/ssh-host-keys path=/etc/ssh-host-keys shift=true
incus config device add ${containerName} claude-config disk source=/home/googlebot/sandboxed/${name}/claude-config path=/home/googlebot/claude-config shift=true
'';
in
{
description = "Incus workspace ${name}";
after = [ "incus.service" "incus-preseed.service" "workspace-${name}-setup.service" ];
requires = [ "incus.service" ];
wants = [ "workspace-${name}-setup.service" ];
wantedBy = optional ws.autoStart "multi-user.target";
path = [ config.virtualisation.incus.package pkgs.gnutar pkgs.xz pkgs.util-linux ];
restartTriggers = [ images.rootfs images.metadata ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
script = ''
set -euo pipefail
# Serialize incus operations - concurrent container creation causes race conditions
exec 9>/run/incus-workspace.lock
flock -x 9
# Import image if not present
if ! incus image list --format csv | grep -q "${imageName}"; then
metadata_tarball=$(echo ${images.metadata}/tarball/*.tar.xz)
rootfs_tarball=$(echo ${images.rootfs}/tarball/*.tar.xz)
incus image import "$metadata_tarball" "$rootfs_tarball" --alias ${imageName}
# Clean up old images for this workspace
incus image list --format csv | grep "nixos-workspace-${name}-" | grep -v "${imageName}" | cut -d, -f2 | while read old_image; do
incus image delete "$old_image" || true
done || true
fi
# Always recreate container for ephemeral behavior
incus stop ${containerName} --force 2>/dev/null || true
incus delete ${containerName} --force 2>/dev/null || true
incus init ${imageName} ${containerName}
${addDevices}
incus start ${containerName}
# Wait for container to start
for i in $(seq 1 30); do
if incus list --format csv | grep -q "^${containerName},RUNNING"; then
exit 0
fi
sleep 1
done
exit 1
'';
preStop = ''
exec 9>/run/incus-workspace.lock
flock -x 9
incus stop ${containerName} --force 2>/dev/null || true
incus delete ${containerName} --force 2>/dev/null || true
# Clean up all images for this workspace
incus image list --format csv 2>/dev/null | grep "nixos-workspace-${name}-" | cut -d, -f2 | while read img; do
incus image delete "$img" 2>/dev/null || true
done
'';
};
in
{
config = mkIf (cfg.enable && incusWorkspaces != { }) {
virtualisation.incus.enable = true;
networking.nftables.enable = true;
virtualisation.incus.preseed = {
storage_pools = [{
name = "default";
driver = "dir";
config = {
source = "/var/lib/incus/storage-pools/default";
};
}];
profiles = [{
name = "default";
config = {
"security.privileged" = "false";
"security.idmap.isolated" = "true";
};
devices = {
root = {
path = "/";
pool = "default";
type = "disk";
};
};
}];
};
systemd.services = mapAttrs'
(name: ws: nameValuePair "incus-workspace-${name}" (mkIncusService name ws))
incusWorkspaces;
# Extra alias for incus shell access (ssh is also available via default.nix aliases)
environment.shellAliases = mkMerge (mapAttrsToList
(name: ws: {
"workspace_${name}_shell" = "doas incus exec workspace-${name} -- su -l googlebot";
})
incusWorkspaces);
};
}

33
common/sandboxed-workspace/vm.nix
View File

@@ -1,4 +1,4 @@
{ config, lib, ... }:
{ config, lib, pkgs, ... }:
# VM-specific configuration for sandboxed workspaces using microvm.nix
# This module is imported by default.nix for workspaces with type = "vm"
@@ -47,6 +47,7 @@ let
# Generate VM configuration for a workspace
mkVmConfig = name: ws: {
inherit pkgs; # Use host's pkgs (includes allowUnfree)
config = import ws.config;
specialArgs = { inputs = hostConfig.inputs; };
extraModules = [
@@ -57,6 +58,21 @@ let
networkInterface = { Type = "ether"; };
})
{
# Copy credentials from host mount to per-workspace config
systemd.services.claude-credentials = {
description = "Copy Claude credentials from host";
wantedBy = [ "multi-user.target" ];
before = [ "multi-user.target" ];
serviceConfig.Type = "oneshot";
script = ''
if [ -f /home/googlebot/.claude-credentials/.credentials.json ]; then
install -m 600 -o googlebot -g users \
/home/googlebot/.claude-credentials/.credentials.json \
/home/googlebot/claude-config/.credentials.json
fi
'';
};
# MicroVM specific configuration
microvm = {
# Use cloud-hypervisor for better performance
@@ -100,6 +116,14 @@ let
source = "/home/googlebot/sandboxed/${name}/claude-config";
mountPoint = "/home/googlebot/claude-config";
}
{
# Credentials-only directory (read-only)
# This directory should contain only .credentials.json
proto = "virtiofs";
tag = "claude-credentials";
source = "/home/googlebot/.claude-credentials";
mountPoint = "/home/googlebot/.claude-credentials";
}
];
# Writeable overlay for /nix/store
@@ -111,12 +135,7 @@ let
interfaces = [{
type = "tap";
id = "vm-${name}";
# Generate a deterministic MAC from workspace name (02: prefix = locally administered)
mac =
let
hash = builtins.hashString "sha256" name;
in
"02:${builtins.substring 0 2 hash}:${builtins.substring 2 2 hash}:${builtins.substring 4 2 hash}:${builtins.substring 6 2 hash}:${builtins.substring 8 2 hash}";
mac = lib.mkMac "vm-${name}";
}];
# Enable vsock for systemd-notify integration