use python smartcrop

.gitignore

@@ -1,3 +1,4 @@
 result
 .pio
 firmware/.vscode
+firmware/.gitignore

firmware/src/main.cpp

@@ -11,6 +11,43 @@ const char* serverName = "http://192.168.3.133:8080/getImage";
 
 Epd epd;
 
+const char* einkDisplayProperties = R"json(
+{
+  "width": 800,
+  "height": 480,
+  "color_space": [
+    {
+      "rgb_color": [0, 0, 0],
+      "color_code": 0
+    },
+    {
+      "rgb_color": [1, 1, 1],
+      "color_code": 1
+    },
+    {
+      "rgb_color": [0.059, 0.329, 0.119],
+      "color_code": 2
+    },
+    {
+      "rgb_color": [0.061, 0.147, 0.336],
+      "color_code": 3
+    },
+    {
+      "rgb_color": [0.574, 0.066, 0.010],
+      "color_code": 4
+    },
+    {
+      "rgb_color": [0.982, 0.756, 0.004],
+      "color_code": 5
+    },
+    {
+      "rgb_color": [0.795, 0.255, 0.018],
+      "color_code": 6
+    }
+  ]
+}
+)json";
+
 void setup()
 {
   // put your setup code here, to run once:
@@ -51,9 +88,10 @@ void fetchAndDrawImage() {
   HTTPClient http;
 
   http.begin(serverName);
-  // http.setAuthorization("username", "password");
+  http.setTimeout(20000); // wait up to 20 seconds for response
-
+  http.setAuthorization("username", "password");
-  int httpCode = http.GET();
+  http.addHeader("Content-Type", "application/json");
+  int httpCode = http.POST(einkDisplayProperties);
 
   if (httpCode > 0) {
       int length = http.getSize();

flake.nix

@@ -10,10 +10,12 @@
     flake-utils.lib.eachDefaultSystem (system:
       let
         pkgs = nixpkgs.legacyPackages.${system};
-        server = pkgs.callPackage ./server/default.nix { };
+        server = pkgs.callPackage ./server/server.nix { };
+        smartcrop = pkgs.callPackage ./server/smartcrop.nix { };
       in
       {
         packages."dynamic-frame-server" = server;
+        packages."dynamic-frame-smartcrop" = smartcrop;
         packages.default = server;
         devShell = pkgs.callPackage ./shell.nix { };
       }

server/bestcrop.go

@@ -1,17 +1,93 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"image"
+	"image/png"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
 
-	"github.com/muesli/smartcrop"
-	"github.com/muesli/smartcrop/nfnt"
 	"github.com/nfnt/resize"
 )
 
+// Writes the file as a PNG to a temporary file for the purpose of processing the image using another program
+func ProcessFindingCrop(img image.Image, width, height int, processFunc func(string, int, int) (image.Rectangle, error)) (image.Rectangle, error) {
+	// Create a temporary directory
+	tmpDir, err := ioutil.TempDir("", "image_processing")
+	if err != nil {
+		return image.Rectangle{}, fmt.Errorf("creating temp dir: %w", err)
+	}
+
+	// Create a temporary file within our temp-directory that follows a particular naming pattern
+	tmpFile, err := ioutil.TempFile(tmpDir, "image-*.png")
+	if err != nil {
+		return image.Rectangle{}, fmt.Errorf("creating temp file: %w", err)
+	}
+
+	// Use png.Encode to encode img to the PNG format with tmpFile as the target writer
+	if err := png.Encode(tmpFile, img); err != nil {
+		return image.Rectangle{}, fmt.Errorf("encoding image: %w", err)
+	}
+
+	// Close the file
+	if err := tmpFile.Close(); err != nil {
+		return image.Rectangle{}, fmt.Errorf("closing temp file: %w", err)
+	}
+
+	// Run the process function
+	res, err := processFunc(tmpFile.Name(), width, height)
+	if err != nil {
+		return image.Rectangle{}, fmt.Errorf("processing image: %w", err)
+	}
+
+	// Delete the temporary file and directory after the function completes
+	if err := os.Remove(tmpFile.Name()); err != nil {
+		return image.Rectangle{}, fmt.Errorf("deleting temp file: %w", err)
+	}
+	if err := os.Remove(tmpDir); err != nil {
+		return image.Rectangle{}, fmt.Errorf("deleting temp dir: %w", err)
+	}
+
+	return res, nil
+}
+
+type PythonResult struct {
+	X      int `json:"x"`
+	Y      int `json:"y"`
+	Width  int `json:"width"`
+	Height int `json:"height"`
+}
+
+// Go's smartcrop selects terrible crops.  Of my test images it never selected the face.
+// But the python smartcrop works great.  So save to a file and execute the python one instead.
+func pythonSmartcrop(filename string, width, height int) (image.Rectangle, error) {
+	cmd := exec.Command("smartcrop-cli", filename, strconv.Itoa(width), strconv.Itoa(height))
+
+	output, err := cmd.Output()
+	if err != nil {
+		return image.Rectangle{}, fmt.Errorf("running python script: %w", err)
+	}
+
+	var result PythonResult
+	err = json.Unmarshal(output, &result)
+	if err != nil {
+		return image.Rectangle{}, fmt.Errorf("parsing output: %w", err)
+	}
+
+	rect := image.Rect(result.X, result.Y, result.X+result.Width, result.Y+result.Height)
+	return rect, nil
+}
+
 func GetBestPieceOfImage(width, height int, img image.Image) image.Image {
-	analyzer := smartcrop.NewAnalyzer(nfnt.NewDefaultResizer())
+	// analyzer := smartcrop.NewAnalyzer(nfnt.NewDefaultResizer())
-	bestCrop, _ := analyzer.FindBestCrop(img, 800, 480)
+	// bestCrop, _ := analyzer.FindBestCrop(img, 800, 480)
+
+	fmt.Println("Finding best image crop...")
+
+	bestCrop, _ := ProcessFindingCrop(img, width, height, pythonSmartcrop)
 
 	fmt.Printf("Best crop: %+v\n", bestCrop)
 

server/einkimage.go

@@ -9,11 +9,12 @@ import (
 
 // Maps RGB colors to the eink display's color values
 type ColorSpace []struct {
-	Color Colorf
+	Color Colorf `json:"rgb_color"`
-	Code  uint8
+	Code  uint8  `json:"color_code"`
 }
 
-func ditherPixel(image [][]Colorf, x int, y int, diff Colorf) {
+func ditherFloydSteinberg(image [][]Colorf, x int, y int, color Colorf) {
+	ditherPixel := func(x int, y int, color Colorf) {
 		height := len(image)
 		width := len(image[0])
 
@@ -21,32 +22,32 @@ func ditherPixel(image [][]Colorf, x int, y int, diff Colorf) {
 			return
 		}
 
-	image[y][x] = clampColor(addColorf(image[y][x], diff), 0, 1)
+		image[y][x] = clampColor(addColorf(image[y][x], color), 0, 1)
-}
+	}
 
-func ditherFloydSteinberg(image [][]Colorf, x int, y int, color Colorf) {
+	ditherPixel(x+1, y, multColor(color, 7.0/16.0))
-	ditherPixel(image, x+1, y, multColor(color, 7.0/16.0))
+	ditherPixel(x-1, y+1, multColor(color, 3.0/16.0))
-	ditherPixel(image, x-1, y+1, multColor(color, 3.0/16.0))
+	ditherPixel(x, y+1, multColor(color, 5.0/16.0))
-	ditherPixel(image, x, y+1, multColor(color, 5.0/16.0))
+	ditherPixel(x+1, y+1, multColor(color, 1.0/16.0))
-	ditherPixel(image, x+1, y+1, multColor(color, 1.0/16.0))
 }
 
 func ConvertToEInkImage(src image.Image, colorSpace ColorSpace) []byte {
 	bounds := src.Bounds()
 	width, height := bounds.Max.X, bounds.Max.Y
 
-	image := convertImageToColors(src)
+	image := ConvertImageToColors(src)
 
 	var einkImage []byte
 
 	for y := 0; y < height; y++ {
 		for x := 0; x < width; x++ {
 			// find closest color
+			imageColor := CorrectGamma(image[y][x])
 			minDist := math.MaxFloat64
 			bestColorCode := uint8(0x0)
 			bestColor := Colorf{0, 0, 0}
 			for _, def := range colorSpace {
-				dist := ciede2000.Diff(convertToRGBA(image[y][x]), convertToRGBA(def.Color))
+				dist := ciede2000.Diff(convertToRGBA(imageColor), convertToRGBA(def.Color))
 				if dist < minDist {
 					minDist = dist
 					bestColorCode = def.Code
@@ -57,7 +58,7 @@ func ConvertToEInkImage(src image.Image, colorSpace ColorSpace) []byte {
 			einkImage = append(einkImage, byte(bestColorCode))
 
 			// dither the remaining color using Floyd-Steinberg to create the illusion of shading
-			diff := subtractColorf(image[y][x], bestColor)
+			diff := subtractColorf(imageColor, bestColor)
 			ditherFloydSteinberg(image, x, y, diff)
 		}
 	}

server/image.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"fmt"
+	"image"
+	"image/color"
+	"image/draw"
+	"math"
+	"os"
+)
+
+func ReadImage(name string) image.Image {
+	file, err := os.Open(name)
+	if err != nil {
+		fmt.Println("Error: File could not be opened")
+		os.Exit(1)
+	}
+	defer file.Close()
+
+	img, _, _ := image.Decode(file)
+
+	return img
+}
+
+func imageToRGBA(src image.Image) *image.RGBA {
+	// No conversion needed if image is an *image.RGBA.
+	if dst, ok := src.(*image.RGBA); ok {
+		return dst
+	}
+
+	// Use the image/draw package to convert to *image.RGBA.
+	b := src.Bounds()
+	dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
+	draw.Draw(dst, dst.Bounds(), src, b.Min, draw.Src)
+	return dst
+}
+
+// Colorf represents a color with float64 RGB values. [R, G, B]
+type Colorf [3]float64
+
+// Convert an image to a 2D array of Colorf.
+func ConvertImageToColors(src image.Image) [][]Colorf {
+	img := imageToRGBA(src)
+
+	bounds := img.Bounds()
+	width, height := bounds.Max.X, bounds.Max.Y
+
+	// Initialize the 2D slice.
+	colorfImg := make([][]Colorf, height)
+	for i := range colorfImg {
+		colorfImg[i] = make([]Colorf, width)
+	}
+
+	// Iterate over each pixel.
+	for y := 0; y < height; y++ {
+		for x := 0; x < width; x++ {
+			r, g, b, _ := img.At(x, y).RGBA()
+
+			// Convert the uint32 color values to float64, normalized to [0, 1].
+			colorf := Colorf{
+				float64(r) / 0xffff,
+				float64(g) / 0xffff,
+				float64(b) / 0xffff,
+			}
+
+			colorfImg[y][x] = colorf
+		}
+	}
+
+	return colorfImg
+}
+
+func CorrectGamma(in Colorf) Colorf {
+	gamma := func(in float64) float64 {
+		return math.Pow(in, 1.2)
+
+		// This is the correct algorithm but looks too dark
+		// if in > 0.04045 {
+		// 	return math.Pow((in+0.055)/(1.0+0.055), 2.4)
+		// } else {
+		// 	return in / 12.92
+		// }
+	}
+
+	return Colorf{
+		gamma(in[0]),
+		gamma(in[1]),
+		gamma(in[2]),
+	}
+}
+
+func convertToRGBA(colorf Colorf) color.RGBA {
+	// Convert the float64 color values to uint8, and ignore the alpha channel.
+	return color.RGBA{
+		R: uint8(colorf[0] * 255),
+		G: uint8(colorf[1] * 255),
+		B: uint8(colorf[2] * 255),
+		A: 255,
+	}
+}
+
+func subtractColorf(c1, c2 Colorf) Colorf {
+	return Colorf{
+		c1[0] - c2[0],
+		c1[1] - c2[1],
+		c1[2] - c2[2],
+	}
+}
+
+func addColorf(c1, c2 Colorf) Colorf {
+	return Colorf{
+		c1[0] + c2[0],
+		c1[1] + c2[1],
+		c1[2] + c2[2],
+	}
+}
+
+func multColor(c Colorf, m float64) Colorf {
+	return Colorf{
+		c[0] * m,
+		c[1] * m,
+		c[2] * m,
+	}
+}
+
+func clampColor(value Colorf, min, max float64) Colorf {
+	return Colorf{
+		clamp(value[0], min, max),
+		clamp(value[1], min, max),
+		clamp(value[2], min, max),
+	}
+}
+
+func clamp(value, min, max float64) float64 {
+	if value < min {
+		return min
+	}
+	if value > max {
+		return max
+	}
+	return value
+}

server/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
@@ -11,6 +12,7 @@ import (
 	_ "image/png"
 
 	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/chi/v5/middleware"
 )
 
 // Define a function to handle incoming requests
@@ -27,22 +29,32 @@ func requestHandler(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprintf(w, "Hello, %s!\n", name)
 }
 
-var colorSpace = ColorSpace{
+type ImageProperties struct {
-	{Colorf{0, 0, 0}, 0x0},
+	Width      int        `json:"width"`
-	{Colorf{1, 1, 1}, 0x1},
+	Height     int        `json:"height"`
-	{Colorf{0.059, 0.329, 0.119}, 0x2},
+	ColorSpace ColorSpace `json:"color_space"`
-	{Colorf{0.061, 0.147, 0.336}, 0x3},
-	{Colorf{0.574, 0.066, 0.010}, 0x4},
-	{Colorf{0.982, 0.756, 0.004}, 0x5},
-	{Colorf{0.795, 0.255, 0.018}, 0x6},
 }
 
-func getImage(w http.ResponseWriter, r *http.Request) {
+func fetchImage(w http.ResponseWriter, r *http.Request) {
+	var imageProps ImageProperties
+	err := json.NewDecoder(r.Body).Decode(&imageProps)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Now you can access the fields of imageProps, e.g.:
+	fmt.Printf("Received request for an image with width: %d, height: %d\n", imageProps.Width, imageProps.Height)
+	for _, color := range imageProps.ColorSpace {
+		fmt.Printf("Color code: %d, RGB: %v\n", color.Code, color.Color)
+	}
+
 	// img := ReadImage("image.jpg")
 	// img := ReadImage("image-out.jpg")
-	img := ReadImage("dahlia-out.jpg")
+	// img := ReadImage("dahlia-out.jpg")
-	bestCrop := GetBestPieceOfImage(800, 480, img)
+	img := ReadImage("dahlia.jpg")
-	data := ConvertToEInkImage(bestCrop, colorSpace)
+	bestCrop := GetBestPieceOfImage(imageProps.Width, imageProps.Height, img)
+	data := ConvertToEInkImage(bestCrop, imageProps.ColorSpace)
 
 	fmt.Printf("Bytes to send: %+v\n", len(data))
 
@@ -67,14 +79,19 @@ func main() {
 	// Create a new Chi router
 	router := chi.NewRouter()
 
+	router.Use(middleware.RequestID)
+	router.Use(middleware.RealIP)
+	router.Use(middleware.Logger)
+	router.Use(middleware.Recoverer)
+
 	// Register the requestHandler function to handle requests at the root path
 	// and a path with the 'name' parameter
 	router.Get("/", requestHandler)
 	router.Get("/{name}", requestHandler)
 
 	router.Group(func(r chi.Router) {
-		// r.Use(basicAuth)
+		r.Use(basicAuth)
-		r.Get("/getImage", getImage)
+		r.Post("/fetchImage", fetchImage)
 	})
 
 	// Start the HTTP server on port 8080 and log any errors

server/smartcrop-cli.py

@@ -0,0 +1,20 @@
+import json
+import sys
+
+import smartcrop
+from PIL import Image
+
+filename = sys.argv[1]
+cropWidth = int(sys.argv[2])
+cropHeight = int(sys.argv[3])
+
+image = Image.open(filename)
+if image.mode != 'RGB' and image.mode != 'RGBA':
+    new_image = Image.new('RGB', image.size)
+    new_image.paste(image)
+    image = new_image
+
+cropper = smartcrop.SmartCrop()
+result = cropper.crop(image, width=100, height=int(cropHeight / cropWidth * 100))
+
+print(json.dumps(result['top_crop'], indent=2))

server/smartcrop.nix

@@ -0,0 +1,66 @@
+{ lib
+, python3
+, fetchFromGitHub
+, makeWrapper
+}:
+
+with python3.pkgs;
+
+let
+  smartcrop = buildPythonPackage rec {
+    pname = "smartcrop.py";
+    version = "v0.3.2";
+
+    src = fetchFromGitHub {
+      owner = "smartcrop";
+      repo = pname;
+      rev = version;
+      hash = "sha256-37ADx72OvORAan51CzdnDFS4uWH8DN/CaXSt5qjnLA4=";
+    };
+
+    propagatedBuildInputs = [
+      pillow
+      numpy
+    ];
+
+    format = "setuptools";
+
+    doCheck = true;
+
+    pythonImportsCheck = [
+      "smartcrop"
+    ];
+  };
+
+  smartcrop-cli = buildPythonApplication {
+    pname = "smartcrop-cli";
+    version = "v1";
+
+    format = "other";
+
+    src = ./.;
+
+    nativeBuildInputs = [ makeWrapper ];
+
+    phases = [ "installPhase" ]; # Removes all phases except installPhase
+
+    installPhase = ''
+      mkdir -p $out
+
+      # copy files
+      cp -r $src/smartcrop-cli.py $out
+
+      mkdir -p $out/bin
+      makeWrapper ${python3}/bin/python3 $out/bin/smartcrop-cli \
+        --prefix PYTHONPATH : ${makePythonPath [smartcrop]} \
+        --add-flags "$out/smartcrop-cli.py"
+    '';
+  };
+
+in
+
+# python3.withPackages (ps: with ps; [
+#   smartcrop
+# ])
+
+smartcrop-cli