Add Incus container support to sandboxed workspaces
- Add incus.nix module for fully declarative Incus/LXC containers - Build NixOS LXC images using nixpkgs.lib.nixosSystem - Ephemeral containers: recreated on each start, cleaned up on stop - Use flock to serialize concurrent container operations - Deterministic MAC addresses via lib.mkMac to prevent ARP cache issues - Add veth* to NetworkManager unmanaged interfaces - Update CLAUDE.md with coding conventions and shared lib docs
This commit is contained in:
@@ -67,6 +67,7 @@
|
||||
|
||||
# Basic system packages
|
||||
environment.systemPackages = with pkgs; [
|
||||
claude-code
|
||||
kakoune
|
||||
vim
|
||||
git
|
||||
|
||||
@@ -12,6 +12,7 @@ in
|
||||
imports = [
|
||||
./vm.nix
|
||||
./container.nix
|
||||
./incus.nix
|
||||
];
|
||||
|
||||
options.sandboxed-workspace = {
|
||||
@@ -21,11 +22,12 @@ in
|
||||
type = types.attrsOf (types.submodule {
|
||||
options = {
|
||||
type = mkOption {
|
||||
type = types.enum [ "vm" "container" ];
|
||||
type = types.enum [ "vm" "container" "incus" ];
|
||||
description = ''
|
||||
Backend type for this workspace:
|
||||
- "vm": microVM with cloud-hypervisor (more isolation, uses virtiofs)
|
||||
- "container": systemd-nspawn container (less overhead, uses bind mounts)
|
||||
- "container": systemd-nspawn via NixOS containers (less overhead, uses bind mounts)
|
||||
- "incus": Incus/LXD container (unprivileged, better security than NixOS containers)
|
||||
'';
|
||||
};
|
||||
|
||||
@@ -102,11 +104,13 @@ in
|
||||
cfg.workspaces);
|
||||
|
||||
# Shell aliases for workspace management
|
||||
# Service names differ by type: microvm@<name> for VMs, container@<name> for containers
|
||||
environment.shellAliases = lib.mkMerge (lib.mapAttrsToList
|
||||
(name: ws:
|
||||
let
|
||||
serviceName = if ws.type == "vm" then "microvm@${name}" else "container@${name}";
|
||||
serviceName =
|
||||
if ws.type == "vm" then "microvm@${name}"
|
||||
else if ws.type == "incus" then "incus-workspace-${name}"
|
||||
else "container@${name}";
|
||||
in
|
||||
{
|
||||
"workspace_${name}" = "ssh googlebot@workspace-${name}";
|
||||
@@ -144,7 +148,10 @@ in
|
||||
(lib.mapAttrs'
|
||||
(name: ws:
|
||||
let
|
||||
serviceName = if ws.type == "vm" then "microvm@${name}" else "container@${name}";
|
||||
serviceName =
|
||||
if ws.type == "vm" then "microvm@${name}"
|
||||
else if ws.type == "incus" then "incus-workspace-${name}"
|
||||
else "container@${name}";
|
||||
claudeConfig = builtins.toJSON {
|
||||
hasCompletedOnboarding = true;
|
||||
theme = "dark";
|
||||
@@ -187,6 +194,13 @@ in
|
||||
echo '${claudeConfig}' > /home/googlebot/sandboxed/${name}/claude-config/.claude.json
|
||||
chown googlebot:users /home/googlebot/sandboxed/${name}/claude-config/.claude.json
|
||||
fi
|
||||
'' + lib.optionalString (ws.type == "incus") ''
|
||||
# Copy credentials for incus (can't use bind mount for files inside another mount)
|
||||
if [ -f /home/googlebot/.claude/.credentials.json ]; then
|
||||
cp /home/googlebot/.claude/.credentials.json /home/googlebot/sandboxed/${name}/claude-config/.credentials.json
|
||||
chown googlebot:users /home/googlebot/sandboxed/${name}/claude-config/.credentials.json
|
||||
chmod 600 /home/googlebot/sandboxed/${name}/claude-config/.credentials.json
|
||||
fi
|
||||
'';
|
||||
}
|
||||
)
|
||||
|
||||
173
common/sandboxed-workspace/incus.nix
Normal file
173
common/sandboxed-workspace/incus.nix
Normal file
@@ -0,0 +1,173 @@
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
# Incus-specific configuration for sandboxed workspaces
|
||||
# Creates fully declarative Incus containers from NixOS configurations
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.sandboxed-workspace;
|
||||
hostConfig = config;
|
||||
|
||||
incusWorkspaces = filterAttrs (n: ws: ws.type == "incus") cfg.workspaces;
|
||||
|
||||
# Build a NixOS LXC image for a workspace
|
||||
mkContainerImage = name: ws:
|
||||
let
|
||||
nixpkgs = hostConfig.inputs.nixpkgs;
|
||||
containerSystem = nixpkgs.lib.nixosSystem {
|
||||
modules = [
|
||||
(import ./base.nix {
|
||||
inherit hostConfig;
|
||||
workspaceName = name;
|
||||
ip = ws.ip;
|
||||
networkInterface = { Name = "eth0"; };
|
||||
})
|
||||
|
||||
(import ws.config)
|
||||
|
||||
({ config, lib, pkgs, ... }: {
|
||||
nixpkgs.hostPlatform = hostConfig.currentSystem;
|
||||
boot.isContainer = true;
|
||||
networking.useHostResolvConf = false;
|
||||
nixpkgs.config.allowUnfree = true;
|
||||
})
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
rootfs = containerSystem.config.system.build.images.lxc;
|
||||
metadata = containerSystem.config.system.build.images.lxc-metadata;
|
||||
toplevel = containerSystem.config.system.build.toplevel;
|
||||
};
|
||||
|
||||
mkIncusService = name: ws:
|
||||
let
|
||||
images = mkContainerImage name ws;
|
||||
hash = builtins.substring 0 12 (builtins.hashString "sha256" "${images.rootfs}");
|
||||
imageName = "nixos-workspace-${name}-${hash}";
|
||||
containerName = "workspace-${name}";
|
||||
|
||||
bridgeName = config.networking.sandbox.bridgeName;
|
||||
mac = lib.mkMac "incus-${name}";
|
||||
|
||||
addDevices = ''
|
||||
incus config device add ${containerName} eth0 nic nictype=bridged parent=${bridgeName} hwaddr=${mac}
|
||||
incus config device add ${containerName} workspace disk source=/home/googlebot/sandboxed/${name}/workspace path=/home/googlebot/workspace shift=true
|
||||
incus config device add ${containerName} ssh-keys disk source=/home/googlebot/sandboxed/${name}/ssh-host-keys path=/etc/ssh-host-keys shift=true
|
||||
incus config device add ${containerName} claude-config disk source=/home/googlebot/sandboxed/${name}/claude-config path=/home/googlebot/claude-config shift=true
|
||||
'';
|
||||
in
|
||||
{
|
||||
description = "Incus workspace ${name}";
|
||||
after = [ "incus.service" "incus-preseed.service" "workspace-${name}-setup.service" ];
|
||||
requires = [ "incus.service" ];
|
||||
wants = [ "workspace-${name}-setup.service" ];
|
||||
wantedBy = optional ws.autoStart "multi-user.target";
|
||||
|
||||
path = [ config.virtualisation.incus.package pkgs.gnutar pkgs.xz pkgs.util-linux ];
|
||||
|
||||
restartTriggers = [ images.rootfs images.metadata ];
|
||||
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
|
||||
# Serialize incus operations - concurrent container creation causes race conditions
|
||||
exec 9>/run/incus-workspace.lock
|
||||
flock -x 9
|
||||
|
||||
# Import image if not present
|
||||
if ! incus image list --format csv | grep -q "${imageName}"; then
|
||||
metadata_tarball=$(echo ${images.metadata}/tarball/*.tar.xz)
|
||||
rootfs_tarball=$(echo ${images.rootfs}/tarball/*.tar.xz)
|
||||
incus image import "$metadata_tarball" "$rootfs_tarball" --alias ${imageName}
|
||||
|
||||
# Clean up old images for this workspace
|
||||
incus image list --format csv | grep "nixos-workspace-${name}-" | grep -v "${imageName}" | cut -d, -f2 | while read old_image; do
|
||||
incus image delete "$old_image" || true
|
||||
done || true
|
||||
fi
|
||||
|
||||
# Always recreate container for ephemeral behavior
|
||||
incus stop ${containerName} --force 2>/dev/null || true
|
||||
incus delete ${containerName} --force 2>/dev/null || true
|
||||
|
||||
incus init ${imageName} ${containerName}
|
||||
${addDevices}
|
||||
incus start ${containerName}
|
||||
|
||||
# Wait for container to start
|
||||
for i in $(seq 1 30); do
|
||||
if incus list --format csv | grep -q "^${containerName},RUNNING"; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
exit 1
|
||||
'';
|
||||
|
||||
preStop = ''
|
||||
exec 9>/run/incus-workspace.lock
|
||||
flock -x 9
|
||||
|
||||
incus stop ${containerName} --force 2>/dev/null || true
|
||||
incus delete ${containerName} --force 2>/dev/null || true
|
||||
|
||||
# Clean up all images for this workspace
|
||||
incus image list --format csv 2>/dev/null | grep "nixos-workspace-${name}-" | cut -d, -f2 | while read img; do
|
||||
incus image delete "$img" 2>/dev/null || true
|
||||
done
|
||||
'';
|
||||
};
|
||||
|
||||
|
||||
in
|
||||
{
|
||||
config = mkIf (cfg.enable && incusWorkspaces != { }) {
|
||||
|
||||
virtualisation.incus.enable = true;
|
||||
networking.nftables.enable = true;
|
||||
|
||||
virtualisation.incus.preseed = {
|
||||
storage_pools = [{
|
||||
name = "default";
|
||||
driver = "dir";
|
||||
config = {
|
||||
source = "/var/lib/incus/storage-pools/default";
|
||||
};
|
||||
}];
|
||||
|
||||
profiles = [{
|
||||
name = "default";
|
||||
config = {
|
||||
"security.privileged" = "false";
|
||||
"security.idmap.isolated" = "true";
|
||||
};
|
||||
devices = {
|
||||
root = {
|
||||
path = "/";
|
||||
pool = "default";
|
||||
type = "disk";
|
||||
};
|
||||
};
|
||||
}];
|
||||
};
|
||||
|
||||
systemd.services = mapAttrs'
|
||||
(name: ws: nameValuePair "incus-workspace-${name}" (mkIncusService name ws))
|
||||
incusWorkspaces;
|
||||
|
||||
# Extra alias for incus shell access (ssh is also available via default.nix aliases)
|
||||
environment.shellAliases = mkMerge (mapAttrsToList
|
||||
(name: ws: {
|
||||
"workspace_${name}_shell" = "doas incus exec workspace-${name} -- su -l googlebot";
|
||||
})
|
||||
incusWorkspaces);
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
{ config, lib, ... }:
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
# VM-specific configuration for sandboxed workspaces using microvm.nix
|
||||
# This module is imported by default.nix for workspaces with type = "vm"
|
||||
@@ -47,6 +47,7 @@ let
|
||||
|
||||
# Generate VM configuration for a workspace
|
||||
mkVmConfig = name: ws: {
|
||||
inherit pkgs; # Use host's pkgs (includes allowUnfree)
|
||||
config = import ws.config;
|
||||
specialArgs = { inputs = hostConfig.inputs; };
|
||||
extraModules = [
|
||||
@@ -57,6 +58,21 @@ let
|
||||
networkInterface = { Type = "ether"; };
|
||||
})
|
||||
{
|
||||
# Copy credentials from host mount to per-workspace config
|
||||
systemd.services.claude-credentials = {
|
||||
description = "Copy Claude credentials from host";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
before = [ "multi-user.target" ];
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = ''
|
||||
if [ -f /home/googlebot/.claude-credentials/.credentials.json ]; then
|
||||
install -m 600 -o googlebot -g users \
|
||||
/home/googlebot/.claude-credentials/.credentials.json \
|
||||
/home/googlebot/claude-config/.credentials.json
|
||||
fi
|
||||
'';
|
||||
};
|
||||
|
||||
# MicroVM specific configuration
|
||||
microvm = {
|
||||
# Use cloud-hypervisor for better performance
|
||||
@@ -100,6 +116,14 @@ let
|
||||
source = "/home/googlebot/sandboxed/${name}/claude-config";
|
||||
mountPoint = "/home/googlebot/claude-config";
|
||||
}
|
||||
{
|
||||
# Credentials-only directory (read-only)
|
||||
# This directory should contain only .credentials.json
|
||||
proto = "virtiofs";
|
||||
tag = "claude-credentials";
|
||||
source = "/home/googlebot/.claude-credentials";
|
||||
mountPoint = "/home/googlebot/.claude-credentials";
|
||||
}
|
||||
];
|
||||
|
||||
# Writeable overlay for /nix/store
|
||||
@@ -111,12 +135,7 @@ let
|
||||
interfaces = [{
|
||||
type = "tap";
|
||||
id = "vm-${name}";
|
||||
# Generate a deterministic MAC from workspace name (02: prefix = locally administered)
|
||||
mac =
|
||||
let
|
||||
hash = builtins.hashString "sha256" name;
|
||||
in
|
||||
"02:${builtins.substring 0 2 hash}:${builtins.substring 2 2 hash}:${builtins.substring 4 2 hash}:${builtins.substring 6 2 hash}:${builtins.substring 8 2 hash}";
|
||||
mac = lib.mkMac "vm-${name}";
|
||||
}];
|
||||
|
||||
# Enable vsock for systemd-notify integration
|
||||
|
||||
Reference in New Issue
Block a user