PoC: Frigate + PCIe Coral + ESPCam, Home Assistant, ESPHome, MQTT, zigbee2mqtt

flake.nix

@@ -51,8 +51,11 @@
             agenix.nixosModules.default
             dailybuild_modules.nixosModule
             nix-index-database.nixosModules.nix-index
+            self.nixosModules.kernel-modules
             ({ lib, ... }: {
               config = {
+                nixpkgs.overlays = [ self.overlays.default ];
+
                 environment.systemPackages = [
                   agenix.packages.${system}.agenix
                 ];
@@ -116,6 +119,9 @@
           "aarch64-linux"."iso" = mkIso "aarch64-linux";
         };
 
+      overlays.default = import ./overlays;
+      nixosModules.kernel-modules = import ./overlays/kernel-modules;
+
       deploy.nodes =
         let
           mkDeploy = configName: arch: hostname: {

machines/storage/s0/default.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ./hardware-configuration.nix
+    ./home-automation.nix
   ];
 
   networking.hostName = "s0";

machines/storage/s0/home-automation.nix

@@ -0,0 +1,135 @@
+{ config, ... }:
+
+{
+  networking.firewall.allowedTCPPorts = [
+    # 1883 # mqtt
+    55834 # mqtt zigbee frontend
+  ];
+
+  services.frigate = {
+    enable = true;
+    hostname = "frigate.s0";
+    settings = {
+      mqtt = {
+        enabled = true;
+        host = "localhost:1883";
+      };
+      cameras = {
+        dahlia-cam = {
+          ffmpeg = {
+            input_args = "";
+            inputs = [{
+              path = "http://dahlia-cam.lan:8080";
+              roles = [ "detect" "record" ];
+            }];
+
+            output_args.record = "-f segment -pix_fmt yuv420p -segment_time 10 -segment_format mp4 -reset_timestamps 1 -strftime 1 -c:v libx264 -preset ultrafast -an ";
+          };
+          rtmp.enabled = false;
+          snapshots = {
+            enabled = true;
+            bounding_box = true;
+          };
+          record = {
+            enabled = false;
+            retain.days = 0; # To not retain any recording if there is no detection of any events 
+            events.retain = {
+              default = 3; # To retain recording for 3 days of only the events that happened
+              mode = "active_objects";
+            };
+          };
+          detect = {
+            enabled = true;
+            width = 800;
+            height = 600;
+            fps = 20;
+          };
+          objects = {
+            track = [ "dog" ];
+            filters.dog.threshold = 0.4;
+          };
+        };
+      };
+      # ffmpeg = {
+      #   hwaccel_args = "preset-vaapi";
+      # };
+      detectors.coral = {
+        type = "edgetpu";
+        device = "pci";
+      };
+    };
+  };
+
+  # AMD GPU for vaapi
+  systemd.services.frigate.environment.LIBVA_DRIVER_NAME = "radeonsi";
+
+  # Coral TPU for frigate
+  services.udev.packages = [ pkgs.libedgetpu ];
+  users.groups.apex = { };
+  systemd.services.frigate.environment.LD_LIBRARY_PATH = "${pkgs.libedgetpu}/lib";
+  systemd.services.frigate.serviceConfig = {
+    SupplementaryGroups = "apex";
+  };
+  # Coral PCIe driver
+  kernel.enableGasketKernelModule = true;
+
+  services.esphome = {
+    enable = true;
+    address = "0.0.0.0";
+    openFirewall = true;
+  };
+  systemd.services.esphome.serviceConfig.ProcSubset = lib.mkForce "all";
+  systemd.services.esphome.serviceConfig.ProtectHostname = lib.mkForce false;
+  systemd.services.esphome.serviceConfig.ProtectKernelLogs = lib.mkForce false;
+  systemd.services.esphome.serviceConfig.ProtectKernelTunables = lib.mkForce false;
+
+  # TODO lock down
+  services.mosquitto = {
+    enable = true;
+    listeners = [
+      {
+        acl = [ "pattern readwrite #" ];
+        omitPasswordAuth = true;
+        settings.allow_anonymous = true;
+      }
+    ];
+  };
+
+  services.zigbee2mqtt = {
+    enable = true;
+    settings = {
+      homeassistant = true;
+      permit_join = false;
+      serial = {
+        port = "/dev/ttyACM0";
+      };
+      mqtt = {
+        server = "mqtt://localhost:1883";
+        # base_topic = "zigbee2mqtt";
+      };
+      frontend = {
+        host = "0.0.0.0";
+        port = 55834;
+      };
+    };
+  };
+
+  services.home-assistant = {
+    enable = true;
+    openFirewall = true;
+    configWritable = true;
+    extraComponents = [
+      "esphome"
+      "met"
+      "radio_browser"
+      "wled"
+      "mqtt"
+    ];
+    # config = null;
+    config = {
+      # Includes dependencies for a basic setup
+      # https://www.home-assistant.io/integrations/default_config/
+      default_config = { };
+    };
+  };
+}

overlays/default.nix

@@ -0,0 +1,5 @@
+final: prev:
+
+{
+  libedgetpu = prev.callPackage ./libedgetpu { };
+}

overlays/kernel-modules/default.nix

@@ -0,0 +1,19 @@
+{ config, lib, ... }:
+
+# Adds additional kernel modules to the nixos system
+# Not actually an overlay but a module. Has to be this way because kernel
+# modules are tightly coupled to the kernel version they were built against.
+# https://nixos.wiki/wiki/Linux_kernel
+
+let
+  cfg = config.kernel;
+
+  gasket = config.boot.kernelPackages.callPackage ./gasket.nix { };
+in
+{
+  options.kernel.enableGasketKernelModule = lib.mkEnableOption "Enable Gasket Kernel Module";
+
+  config = lib.mkIf cfg.enableGasketKernelModule {
+    boot.extraModulePackages = [ gasket ];
+  };
+}

overlays/kernel-modules/gasket.nix

@@ -0,0 +1,35 @@
+{ stdenv, lib, fetchFromGitHub, kernel }:
+
+stdenv.mkDerivation rec {
+  pname = "gasket";
+  version = "1.0-18";
+
+  src = fetchFromGitHub {
+    owner = "google";
+    repo = "gasket-driver";
+    rev = "97aeba584efd18983850c36dcf7384b0185284b3";
+    sha256 = "pJwrrI7jVKFts4+bl2xmPIAD01VKFta2SRuElerQnTo=";
+  };
+
+  makeFlags = [
+    "-C"
+    "${kernel.dev}/lib/modules/${kernel.modDirVersion}/build"
+    "M=$(PWD)"
+  ];
+  buildFlags = [ "modules" ];
+
+  installFlags = [ "INSTALL_MOD_PATH=${placeholder "out"}" ];
+  installTargets = [ "modules_install" ];
+
+  sourceRoot = "source/src";
+  hardeningDisable = [ "pic" "format" ];
+  nativeBuildInputs = kernel.moduleBuildDependencies;
+
+  meta = with lib; {
+    description = "The Coral Gasket Driver allows usage of the Coral EdgeTPU on Linux systems.";
+    homepage = "https://github.com/google/gasket-driver";
+    license = licenses.gpl2;
+    maintainers = [ lib.maintainers.kylehendricks ];
+    platforms = platforms.linux;
+  };
+}

overlays/libedgetpu/default.nix

@@ -0,0 +1,72 @@
+{ stdenv
+, lib
+, fetchFromGitHub
+, libusb1
+, abseil-cpp
+, flatbuffers
+, xxd
+}:
+
+let
+  flatbuffers_1_12 = flatbuffers.overrideAttrs (oldAttrs: rec {
+    version = "1.12.0";
+    NIX_CFLAGS_COMPILE = "-Wno-error=class-memaccess -Wno-error=maybe-uninitialized";
+    cmakeFlags = (oldAttrs.cmakeFlags or [ ]) ++ [ "-DFLATBUFFERS_BUILD_SHAREDLIB=ON" ];
+    NIX_CXXSTDLIB_COMPILE = "-std=c++17";
+    configureFlags = (oldAttrs.configureFlags or [ ]) ++ [ "--enable-shared" ];
+    src = fetchFromGitHub {
+      owner = "google";
+      repo = "flatbuffers";
+      rev = "v${version}";
+      sha256 = "sha256-L1B5Y/c897Jg9fGwT2J3+vaXsZ+lfXnskp8Gto1p/Tg=";
+    };
+  });
+
+in
+stdenv.mkDerivation rec {
+  pname = "libedgetpu";
+  version = "grouper";
+
+  src = fetchFromGitHub {
+    owner = "google-coral";
+    repo = pname;
+    rev = "release-${version}";
+    sha256 = "sha256-73hwItimf88Iqnb40lk4ul/PzmCNIfdt6Afi+xjNiBE=";
+  };
+
+  patches = [ ./libedgetpu-stddef.diff ];
+
+  makeFlags = [ "-f" "makefile_build/Makefile" "libedgetpu" ];
+
+  buildInputs = [
+    libusb1
+    abseil-cpp
+    flatbuffers_1_12
+  ];
+
+  nativeBuildInputs = [
+    xxd
+  ];
+
+  NIX_CXXSTDLIB_COMPILE = "-std=c++17";
+
+  TFROOT = "${fetchFromGitHub {
+    owner = "tensorflow";
+    repo = "tensorflow";
+    rev = "v2.7.4";
+    sha256 = "sha256-liDbUAdaVllB0b74aBeqNxkYNu/zPy7k3CevzRF5dk0=";
+  }}";
+
+  enableParallelBuilding = false;
+
+  installPhase = ''
+    mkdir -p $out/lib
+    cp out/direct/k8/libedgetpu.so.1.0 $out/lib
+    ln -s $out/lib/libedgetpu.so.1.0 $out/lib/libedgetpu.so.1
+    mkdir -p $out/lib/udev/rules.d
+    cp debian/edgetpu-accelerator.rules $out/lib/udev/rules.d/99-edgetpu-accelerator.rules
+
+    # PCIe rule
+    echo 'SUBSYSTEM=="apex", MODE="0660", GROUP="apex"' > $out/lib/udev/rules.d/65-apex.rules
+  '';
+}

overlays/libedgetpu/libedgetpu-stddef.diff

@@ -0,0 +1,12 @@
+diff --git a/api/allocated_buffer.h b/api/allocated_buffer.h
+index 97740f0..7bc0547 100644
+--- a/api/allocated_buffer.h
++++ b/api/allocated_buffer.h
+@@ -16,6 +16,7 @@
+ #define DARWINN_API_ALLOCATED_BUFFER_H_
+ 
+ #include <functional>
++#include <cstddef>
+ 
+ namespace platforms {
+ namespace darwinn {

secrets/secrets.nix

@@ -39,8 +39,8 @@ with roles;
   "wolframalpha.age".publicKeys = dailybot;
 
   # hostapd
-  "hostapd-pw-experimental-tower.age".publicKeys = wireless;
-  "hostapd-pw-CXNK00BF9176.age".publicKeys = wireless;
+  "hostapd-pw-experimental-tower.age".publicKeys = nobody;
+  "hostapd-pw-CXNK00BF9176.age".publicKeys = nobody;
 
   # backups
   "backblaze-s3-backups.age".publicKeys = personal ++ server;