diff --git a/.gitignore b/.gitignore index 072c54b..66514bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ result .pio firmware/.vscode +firmware/.gitignore diff --git a/firmware/src/main.cpp b/firmware/src/main.cpp index db57ef5..b4b4c91 100644 --- a/firmware/src/main.cpp +++ b/firmware/src/main.cpp @@ -11,6 +11,43 @@ const char* serverName = "http://192.168.3.133:8080/getImage"; Epd epd; +const char* einkDisplayProperties = R"json( +{ + "width": 800, + "height": 480, + "color_space": [ + { + "rgb_color": [0, 0, 0], + "color_code": 0 + }, + { + "rgb_color": [1, 1, 1], + "color_code": 1 + }, + { + "rgb_color": [0.059, 0.329, 0.119], + "color_code": 2 + }, + { + "rgb_color": [0.061, 0.147, 0.336], + "color_code": 3 + }, + { + "rgb_color": [0.574, 0.066, 0.010], + "color_code": 4 + }, + { + "rgb_color": [0.982, 0.756, 0.004], + "color_code": 5 + }, + { + "rgb_color": [0.795, 0.255, 0.018], + "color_code": 6 + } + ] +} +)json"; + void setup() { // put your setup code here, to run once: @@ -51,9 +88,10 @@ void fetchAndDrawImage() { HTTPClient http; http.begin(serverName); - // http.setAuthorization("username", "password"); - - int httpCode = http.GET(); + http.setTimeout(20000); // wait up to 20 seconds for response + http.setAuthorization("username", "password"); + http.addHeader("Content-Type", "application/json"); + int httpCode = http.POST(einkDisplayProperties); if (httpCode > 0) { int length = http.getSize(); diff --git a/flake.nix b/flake.nix index 44468c4..f145775 100644 --- a/flake.nix +++ b/flake.nix @@ -10,10 +10,12 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; - server = pkgs.callPackage ./server/default.nix { }; + server = pkgs.callPackage ./server/server.nix { }; + smartcrop = pkgs.callPackage ./server/smartcrop.nix { }; in { packages."dynamic-frame-server" = server; + packages."dynamic-frame-smartcrop" = smartcrop; packages.default = server; devShell = pkgs.callPackage ./shell.nix { }; } diff --git a/server/bestcrop.go b/server/bestcrop.go index 4062cfd..7f73e34 100644 --- a/server/bestcrop.go +++ b/server/bestcrop.go @@ -1,17 +1,93 @@ package main import ( + "encoding/json" "fmt" "image" + "image/png" + "io/ioutil" + "os" + "os/exec" + "strconv" - "github.com/muesli/smartcrop" - "github.com/muesli/smartcrop/nfnt" "github.com/nfnt/resize" ) +// Writes the file as a PNG to a temporary file for the purpose of processing the image using another program +func ProcessFindingCrop(img image.Image, width, height int, processFunc func(string, int, int) (image.Rectangle, error)) (image.Rectangle, error) { + // Create a temporary directory + tmpDir, err := ioutil.TempDir("", "image_processing") + if err != nil { + return image.Rectangle{}, fmt.Errorf("creating temp dir: %w", err) + } + + // Create a temporary file within our temp-directory that follows a particular naming pattern + tmpFile, err := ioutil.TempFile(tmpDir, "image-*.png") + if err != nil { + return image.Rectangle{}, fmt.Errorf("creating temp file: %w", err) + } + + // Use png.Encode to encode img to the PNG format with tmpFile as the target writer + if err := png.Encode(tmpFile, img); err != nil { + return image.Rectangle{}, fmt.Errorf("encoding image: %w", err) + } + + // Close the file + if err := tmpFile.Close(); err != nil { + return image.Rectangle{}, fmt.Errorf("closing temp file: %w", err) + } + + // Run the process function + res, err := processFunc(tmpFile.Name(), width, height) + if err != nil { + return image.Rectangle{}, fmt.Errorf("processing image: %w", err) + } + + // Delete the temporary file and directory after the function completes + if err := os.Remove(tmpFile.Name()); err != nil { + return image.Rectangle{}, fmt.Errorf("deleting temp file: %w", err) + } + if err := os.Remove(tmpDir); err != nil { + return image.Rectangle{}, fmt.Errorf("deleting temp dir: %w", err) + } + + return res, nil +} + +type PythonResult struct { + X int `json:"x"` + Y int `json:"y"` + Width int `json:"width"` + Height int `json:"height"` +} + +// Go's smartcrop selects terrible crops. Of my test images it never selected the face. +// But the python smartcrop works great. So save to a file and execute the python one instead. +func pythonSmartcrop(filename string, width, height int) (image.Rectangle, error) { + cmd := exec.Command("smartcrop-cli", filename, strconv.Itoa(width), strconv.Itoa(height)) + + output, err := cmd.Output() + if err != nil { + return image.Rectangle{}, fmt.Errorf("running python script: %w", err) + } + + var result PythonResult + err = json.Unmarshal(output, &result) + if err != nil { + return image.Rectangle{}, fmt.Errorf("parsing output: %w", err) + } + + rect := image.Rect(result.X, result.Y, result.X+result.Width, result.Y+result.Height) + return rect, nil +} + func GetBestPieceOfImage(width, height int, img image.Image) image.Image { - analyzer := smartcrop.NewAnalyzer(nfnt.NewDefaultResizer()) - bestCrop, _ := analyzer.FindBestCrop(img, 800, 480) + // analyzer := smartcrop.NewAnalyzer(nfnt.NewDefaultResizer()) + // bestCrop, _ := analyzer.FindBestCrop(img, 800, 480) + + fmt.Println("Finding best image crop...") + + bestCrop, _ := ProcessFindingCrop(img, width, height, pythonSmartcrop) fmt.Printf("Best crop: %+v\n", bestCrop) diff --git a/server/einkimage.go b/server/einkimage.go index 295fe9b..59e7c5e 100644 --- a/server/einkimage.go +++ b/server/einkimage.go @@ -9,44 +9,45 @@ import ( // Maps RGB colors to the eink display's color values type ColorSpace []struct { - Color Colorf - Code uint8 -} - -func ditherPixel(image [][]Colorf, x int, y int, diff Colorf) { - height := len(image) - width := len(image[0]) - - if x < 0 || y < 0 || x >= width || y >= height { - return - } - - image[y][x] = clampColor(addColorf(image[y][x], diff), 0, 1) + Color Colorf `json:"rgb_color"` + Code uint8 `json:"color_code"` } func ditherFloydSteinberg(image [][]Colorf, x int, y int, color Colorf) { - ditherPixel(image, x+1, y, multColor(color, 7.0/16.0)) - ditherPixel(image, x-1, y+1, multColor(color, 3.0/16.0)) - ditherPixel(image, x, y+1, multColor(color, 5.0/16.0)) - ditherPixel(image, x+1, y+1, multColor(color, 1.0/16.0)) + ditherPixel := func(x int, y int, color Colorf) { + height := len(image) + width := len(image[0]) + + if x < 0 || y < 0 || x >= width || y >= height { + return + } + + image[y][x] = clampColor(addColorf(image[y][x], color), 0, 1) + } + + ditherPixel(x+1, y, multColor(color, 7.0/16.0)) + ditherPixel(x-1, y+1, multColor(color, 3.0/16.0)) + ditherPixel(x, y+1, multColor(color, 5.0/16.0)) + ditherPixel(x+1, y+1, multColor(color, 1.0/16.0)) } func ConvertToEInkImage(src image.Image, colorSpace ColorSpace) []byte { bounds := src.Bounds() width, height := bounds.Max.X, bounds.Max.Y - image := convertImageToColors(src) + image := ConvertImageToColors(src) var einkImage []byte for y := 0; y < height; y++ { for x := 0; x < width; x++ { // find closest color + imageColor := CorrectGamma(image[y][x]) minDist := math.MaxFloat64 bestColorCode := uint8(0x0) bestColor := Colorf{0, 0, 0} for _, def := range colorSpace { - dist := ciede2000.Diff(convertToRGBA(image[y][x]), convertToRGBA(def.Color)) + dist := ciede2000.Diff(convertToRGBA(imageColor), convertToRGBA(def.Color)) if dist < minDist { minDist = dist bestColorCode = def.Code @@ -57,7 +58,7 @@ func ConvertToEInkImage(src image.Image, colorSpace ColorSpace) []byte { einkImage = append(einkImage, byte(bestColorCode)) // dither the remaining color using Floyd-Steinberg to create the illusion of shading - diff := subtractColorf(image[y][x], bestColor) + diff := subtractColorf(imageColor, bestColor) ditherFloydSteinberg(image, x, y, diff) } } diff --git a/server/image.go b/server/image.go new file mode 100644 index 0000000..b5c004c --- /dev/null +++ b/server/image.go @@ -0,0 +1,142 @@ +package main + +import ( + "fmt" + "image" + "image/color" + "image/draw" + "math" + "os" +) + +func ReadImage(name string) image.Image { + file, err := os.Open(name) + if err != nil { + fmt.Println("Error: File could not be opened") + os.Exit(1) + } + defer file.Close() + + img, _, _ := image.Decode(file) + + return img +} + +func imageToRGBA(src image.Image) *image.RGBA { + // No conversion needed if image is an *image.RGBA. + if dst, ok := src.(*image.RGBA); ok { + return dst + } + + // Use the image/draw package to convert to *image.RGBA. + b := src.Bounds() + dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) + draw.Draw(dst, dst.Bounds(), src, b.Min, draw.Src) + return dst +} + +// Colorf represents a color with float64 RGB values. [R, G, B] +type Colorf [3]float64 + +// Convert an image to a 2D array of Colorf. +func ConvertImageToColors(src image.Image) [][]Colorf { + img := imageToRGBA(src) + + bounds := img.Bounds() + width, height := bounds.Max.X, bounds.Max.Y + + // Initialize the 2D slice. + colorfImg := make([][]Colorf, height) + for i := range colorfImg { + colorfImg[i] = make([]Colorf, width) + } + + // Iterate over each pixel. + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + r, g, b, _ := img.At(x, y).RGBA() + + // Convert the uint32 color values to float64, normalized to [0, 1]. + colorf := Colorf{ + float64(r) / 0xffff, + float64(g) / 0xffff, + float64(b) / 0xffff, + } + + colorfImg[y][x] = colorf + } + } + + return colorfImg +} + +func CorrectGamma(in Colorf) Colorf { + gamma := func(in float64) float64 { + return math.Pow(in, 1.2) + + // This is the correct algorithm but looks too dark + // if in > 0.04045 { + // return math.Pow((in+0.055)/(1.0+0.055), 2.4) + // } else { + // return in / 12.92 + // } + } + + return Colorf{ + gamma(in[0]), + gamma(in[1]), + gamma(in[2]), + } +} + +func convertToRGBA(colorf Colorf) color.RGBA { + // Convert the float64 color values to uint8, and ignore the alpha channel. + return color.RGBA{ + R: uint8(colorf[0] * 255), + G: uint8(colorf[1] * 255), + B: uint8(colorf[2] * 255), + A: 255, + } +} + +func subtractColorf(c1, c2 Colorf) Colorf { + return Colorf{ + c1[0] - c2[0], + c1[1] - c2[1], + c1[2] - c2[2], + } +} + +func addColorf(c1, c2 Colorf) Colorf { + return Colorf{ + c1[0] + c2[0], + c1[1] + c2[1], + c1[2] + c2[2], + } +} + +func multColor(c Colorf, m float64) Colorf { + return Colorf{ + c[0] * m, + c[1] * m, + c[2] * m, + } +} + +func clampColor(value Colorf, min, max float64) Colorf { + return Colorf{ + clamp(value[0], min, max), + clamp(value[1], min, max), + clamp(value[2], min, max), + } +} + +func clamp(value, min, max float64) float64 { + if value < min { + return min + } + if value > max { + return max + } + return value +} diff --git a/server/main.go b/server/main.go index 5928fb5..f9fb2ea 100644 --- a/server/main.go +++ b/server/main.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "log" "net/http" @@ -11,6 +12,7 @@ import ( _ "image/png" "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" ) // Define a function to handle incoming requests @@ -27,22 +29,32 @@ func requestHandler(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, %s!\n", name) } -var colorSpace = ColorSpace{ - {Colorf{0, 0, 0}, 0x0}, - {Colorf{1, 1, 1}, 0x1}, - {Colorf{0.059, 0.329, 0.119}, 0x2}, - {Colorf{0.061, 0.147, 0.336}, 0x3}, - {Colorf{0.574, 0.066, 0.010}, 0x4}, - {Colorf{0.982, 0.756, 0.004}, 0x5}, - {Colorf{0.795, 0.255, 0.018}, 0x6}, +type ImageProperties struct { + Width int `json:"width"` + Height int `json:"height"` + ColorSpace ColorSpace `json:"color_space"` } -func getImage(w http.ResponseWriter, r *http.Request) { +func fetchImage(w http.ResponseWriter, r *http.Request) { + var imageProps ImageProperties + err := json.NewDecoder(r.Body).Decode(&imageProps) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Now you can access the fields of imageProps, e.g.: + fmt.Printf("Received request for an image with width: %d, height: %d\n", imageProps.Width, imageProps.Height) + for _, color := range imageProps.ColorSpace { + fmt.Printf("Color code: %d, RGB: %v\n", color.Code, color.Color) + } + // img := ReadImage("image.jpg") // img := ReadImage("image-out.jpg") - img := ReadImage("dahlia-out.jpg") - bestCrop := GetBestPieceOfImage(800, 480, img) - data := ConvertToEInkImage(bestCrop, colorSpace) + // img := ReadImage("dahlia-out.jpg") + img := ReadImage("dahlia.jpg") + bestCrop := GetBestPieceOfImage(imageProps.Width, imageProps.Height, img) + data := ConvertToEInkImage(bestCrop, imageProps.ColorSpace) fmt.Printf("Bytes to send: %+v\n", len(data)) @@ -67,14 +79,19 @@ func main() { // Create a new Chi router router := chi.NewRouter() + router.Use(middleware.RequestID) + router.Use(middleware.RealIP) + router.Use(middleware.Logger) + router.Use(middleware.Recoverer) + // Register the requestHandler function to handle requests at the root path // and a path with the 'name' parameter router.Get("/", requestHandler) router.Get("/{name}", requestHandler) router.Group(func(r chi.Router) { - // r.Use(basicAuth) - r.Get("/getImage", getImage) + r.Use(basicAuth) + r.Post("/fetchImage", fetchImage) }) // Start the HTTP server on port 8080 and log any errors diff --git a/server/default.nix b/server/server.nix similarity index 100% rename from server/default.nix rename to server/server.nix diff --git a/server/smartcrop-cli.py b/server/smartcrop-cli.py new file mode 100644 index 0000000..a94a8d2 --- /dev/null +++ b/server/smartcrop-cli.py @@ -0,0 +1,20 @@ +import json +import sys + +import smartcrop +from PIL import Image + +filename = sys.argv[1] +cropWidth = int(sys.argv[2]) +cropHeight = int(sys.argv[3]) + +image = Image.open(filename) +if image.mode != 'RGB' and image.mode != 'RGBA': + new_image = Image.new('RGB', image.size) + new_image.paste(image) + image = new_image + +cropper = smartcrop.SmartCrop() +result = cropper.crop(image, width=100, height=int(cropHeight / cropWidth * 100)) + +print(json.dumps(result['top_crop'], indent=2)) \ No newline at end of file diff --git a/server/smartcrop.nix b/server/smartcrop.nix new file mode 100644 index 0000000..44cbf49 --- /dev/null +++ b/server/smartcrop.nix @@ -0,0 +1,66 @@ +{ lib +, python3 +, fetchFromGitHub +, makeWrapper +}: + +with python3.pkgs; + +let + smartcrop = buildPythonPackage rec { + pname = "smartcrop.py"; + version = "v0.3.2"; + + src = fetchFromGitHub { + owner = "smartcrop"; + repo = pname; + rev = version; + hash = "sha256-37ADx72OvORAan51CzdnDFS4uWH8DN/CaXSt5qjnLA4="; + }; + + propagatedBuildInputs = [ + pillow + numpy + ]; + + format = "setuptools"; + + doCheck = true; + + pythonImportsCheck = [ + "smartcrop" + ]; + }; + + smartcrop-cli = buildPythonApplication { + pname = "smartcrop-cli"; + version = "v1"; + + format = "other"; + + src = ./.; + + nativeBuildInputs = [ makeWrapper ]; + + phases = [ "installPhase" ]; # Removes all phases except installPhase + + installPhase = '' + mkdir -p $out + + # copy files + cp -r $src/smartcrop-cli.py $out + + mkdir -p $out/bin + makeWrapper ${python3}/bin/python3 $out/bin/smartcrop-cli \ + --prefix PYTHONPATH : ${makePythonPath [smartcrop]} \ + --add-flags "$out/smartcrop-cli.py" + ''; + }; + +in + +# python3.withPackages (ps: with ps; [ +# smartcrop +# ]) + +smartcrop-cli \ No newline at end of file