Basic stitching functionality

- Median filtering

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "env": {},
+            "args": []
+        }
+    ]
+}

.vscode/settings.json

@@ -1,6 +1,7 @@
 {
     "cSpell.words": [
         "Vogel",
+        "hacky",
         "kbinani",
         "noita"
     ]

bin/stitch/imagetile.go

@@ -15,8 +15,8 @@ import (
 type imageTile struct {
 	fileName string
 
-	originalRect image.Rectangle // Rectangle of the original position. Determined by the file name, the real coordinates may differ a few pixels.
+	offset image.Point // Correction offset of the image, so that it aligns pixel perfect with other images. Determined by image matching.
-	image        image.Image     // Either a rectangle or an RGBA image. The bounds of this image represent the real and corrected coordinates.
+	image  image.Image // Either a rectangle or an RGBA image. The bounds of this image are determined by the filename.
 }
 
 func (it *imageTile) loadImage() error {
@@ -55,3 +55,7 @@ func (it *imageTile) loadImage() error {
 func (it *imageTile) unloadImage() {
 	it.image = it.image.Bounds()
 }
+
+func (it *imageTile) String() string {
+	return fmt.Sprintf("<ImageTile \"%v\">", it.fileName)
+}

bin/stitch/imagetiles.go

@@ -8,11 +8,29 @@ package main
 import (
 	"fmt"
 	"image"
+	"image/color"
+	"log"
+	"math"
+	"math/rand"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 )
 
+const tileAlignmentSearchRadius = 5
+
+type tileAlignment struct {
+	offset image.Point // Contains the offset of the tile a, so that it aligns pixel perfect with tile b
+}
+
+type tileAlignmentKeys struct {
+	a, b *imageTile
+}
+
+// tilePairs contains image pairs and their alignment.
+type tilePairs map[tileAlignmentKeys]tileAlignment
+
 var regexFileParse = regexp.MustCompile(`^(-?\d+),(-?\d+).png$`)
 
 func loadImages(path string) ([]imageTile, error) {
@@ -44,11 +62,258 @@ func loadImages(path string) ([]imageTile, error) {
 		}
 
 		imageTiles = append(imageTiles, imageTile{
-			fileName:     file,
+			fileName: file,
-			originalRect: image.Rect(x, y, x+width, y+height),
+			image:    image.Rect(x, y, x+width, y+height),
-			image:        image.Rect(x, y, x+width, y+height),
 		})
 	}
 
 	return imageTiles, nil
 }
+
+// alignTilePair returns the pixel delta for the first tile, so that it aligns perfectly with the second.
+// This function will load images if needed.
+func alignTilePair(tileA, tileB *imageTile, searchRadius int) (image.Point, error) {
+	if err := tileA.loadImage(); err != nil {
+		return image.Point{}, err
+	}
+	if err := tileB.loadImage(); err != nil {
+		return image.Point{}, err
+	}
+
+	// Type assertion.
+	imgA, imgB := *tileA.image.(*image.RGBA), *tileB.image.(*image.RGBA)
+
+	bestPoint := image.Point{}
+	bestValue := math.Inf(1)
+
+	for y := -searchRadius; y <= searchRadius; y++ {
+		for x := -searchRadius; x <= searchRadius; x++ {
+			point := image.Point{x, y} // Offset of the first image.
+
+			value := getImageDifferenceValue(&imgA, &imgB, point)
+			if bestValue > value {
+				bestValue, bestPoint = value, point
+			}
+		}
+	}
+
+	return bestPoint, nil
+}
+
+func (tp tilePairs) alignTiles(tiles []*imageTile) error {
+
+	n := len(tiles)
+	maxOperations, operations := (n-1)*(n)/2, 0
+
+	// Compare all n tiles with each other. (`(n-1)*(n)/2` comparisons)
+	for i, tileA := range tiles {
+		for j := i + 1; j < len(tiles); j++ {
+			tileB := tiles[j]
+
+			_, ok := tp[tileAlignmentKeys{tileA, tileB}]
+			if !ok {
+				// Entry doesn't exist yet. Determine tile pair alignment.
+				offset, err := alignTilePair(tileA, tileB, tileAlignmentSearchRadius)
+				if err != nil {
+					return fmt.Errorf("Failed to align tile pair %v %v: %w", tileA, tileB, err)
+				}
+
+				operations++
+				log.Printf("(%v/%v)Got alignment for pair %v %v. Offset = %v", operations, maxOperations, tileA, tileB, offset)
+
+				// Store tile alignment pair, also reversed.
+				tp[tileAlignmentKeys{tileA, tileB}] = tileAlignment{offset: offset}
+				tp[tileAlignmentKeys{tileB, tileA}] = tileAlignment{offset: offset.Mul(-1)}
+
+			}
+		}
+	}
+
+	// Silly and hacky method to determine the minimal error.
+	// TODO: Use some mixed integer method or something similar to optimize the tile alignment
+
+	// The error function returns the x and y error. The axes are optimized independent of each other later on.
+	errorFunction := func(tiles []*imageTile) (image.Point, error) {
+		errorValue := image.Point{}
+
+		for i, tileA := range tiles {
+			for j := i + 1; j < len(tiles); j++ {
+				tileB := tiles[j]
+
+				tileAlignment, ok := tp[tileAlignmentKeys{tileA, tileB}]
+				if !ok {
+					return image.Point{}, fmt.Errorf("Offset of the tile pair %v %v is missing", tileA, tileB)
+				}
+
+				// The error is the difference between the needed offset, and the actual offsets
+				tempErrorValue := pointAbs(tileAlignment.offset.Sub(tileA.offset).Add(tileB.offset))
+
+				errorValue = errorValue.Add(tempErrorValue)
+			}
+		}
+		return errorValue, nil
+	}
+
+	errorValue, err := errorFunction(tiles)
+	if err != nil {
+		return fmt.Errorf("Failed to calculate error value: %w", err)
+	}
+	// Randomly select tiles, and move them in the direction where the error value is lower.
+	// The "gradient" is basically caluclated by try and error.
+	for i := 0; i < len(tiles)*tileAlignmentSearchRadius*5; i++ {
+		tile := tiles[rand.Intn(len(tiles))]
+
+		// Calculate error value for positive shifting.
+		tile.offset = tile.offset.Add(image.Point{1, 1})
+		plusErrorValue, err := errorFunction(tiles)
+		if err != nil {
+			return fmt.Errorf("Failed to calculate error value: %w", err)
+		}
+
+		// Calculate error value for negative shifting.
+		tile.offset = tile.offset.Add(image.Point{-2, -2})
+		minusErrorValue, err := errorFunction(tiles)
+		if err != nil {
+			return fmt.Errorf("Failed to calculate error value: %w", err)
+		}
+
+		// Reset tile movement.
+		tile.offset = tile.offset.Add(image.Point{1, 1})
+
+		// Move this tile towards the smaller error value.
+		if plusErrorValue.X < errorValue.X {
+			tile.offset = tile.offset.Add(image.Point{1, 0})
+		}
+		if minusErrorValue.X < errorValue.X {
+			tile.offset = tile.offset.Add(image.Point{-1, 0})
+		}
+		if plusErrorValue.Y < errorValue.Y {
+			tile.offset = tile.offset.Add(image.Point{0, 1})
+		}
+		if minusErrorValue.Y < errorValue.Y {
+			tile.offset = tile.offset.Add(image.Point{0, -1})
+		}
+	}
+
+	// TODO: Move images in a way that the majority of images is positioned equal to their original position
+
+	return nil
+}
+
+func (tp tilePairs) stitch(tiles []imageTile, destImage *image.RGBA) error {
+	intersectTiles := []*imageTile{}
+
+	// Get only the tiles that intersect with the destination image bounds.
+	// Ignore alignment here, doesn't matter if an image overlaps a few pixels anyways.
+	for i, tile := range tiles {
+		if tile.image.Bounds().Add(tile.offset).Overlaps(destImage.Bounds()) {
+			intersectTiles = append(intersectTiles, &tiles[i])
+		}
+	}
+
+	//log.Printf("intersectTiles: %v", intersectTiles)
+
+	// Align those tiles
+	/*if err := tp.alignTiles(intersectTiles); err != nil {
+		return fmt.Errorf("Failed to align tiles: %w", err)
+	}*/
+
+	// TODO: Add working aligning algorithm
+
+	/*for _, intersectTile := range intersectTiles {
+		intersectTile.loadImage()
+		draw.Draw(destImage, destImage.Bounds(), intersectTile.image, destImage.Bounds().Min, draw.Over)
+	}*/
+
+	drawMedianBlended(intersectTiles, destImage)
+
+	/*for _, intersectTile := range intersectTiles {
+		drawLabel(destImage, intersectTile.image.Bounds().Min.X, intersectTile.image.Bounds().Min.Y, fmt.Sprintf("%v", intersectTile.fileName))
+	}*/
+
+	return nil
+}
+
+func drawMedianBlended(tiles []*imageTile, destImage *image.RGBA) {
+	bounds := destImage.Bounds()
+
+	// Make sure images are loaded.
+	for _, tile := range tiles {
+		tile.loadImage()
+	}
+
+	for iy := bounds.Min.Y; iy < bounds.Max.Y; iy++ {
+		for ix := bounds.Min.X; ix < bounds.Max.X; ix++ {
+			//colList := []color.RGBA{}
+			rList, gList, bList := []int{}, []int{}, []int{}
+			point := image.Point{ix, iy}
+
+			// Iterate through all tiles, and create a list of colors.
+			for _, tile := range tiles {
+				tilePoint := point.Sub(tile.offset)
+				imageRGBA, ok := tile.image.(*image.RGBA)
+				if ok && tilePoint.In(imageRGBA.Bounds()) {
+					col := imageRGBA.RGBAAt(tilePoint.X, tilePoint.Y)
+					//colList = append(colList, col)
+					rList, gList, bList = append(rList, int(col.R)), append(gList, int(col.G)), append(bList, int(col.B))
+				}
+			}
+
+			// Sort color list.
+			/*sort.Slice(colList, func(i, j int) bool {
+				return rgbToHSV(colList[i]) < rgbToHSV(colList[j])
+			})*/
+
+			// Sort rList.
+			sort.Slice(rList, func(i, j int) bool {
+				return rList[i] < rList[j]
+			})
+
+			// Sort gList.
+			sort.Slice(gList, func(i, j int) bool {
+				return gList[i] < gList[j]
+			})
+
+			// Sort bList.
+			sort.Slice(bList, func(i, j int) bool {
+				return bList[i] < bList[j]
+			})
+
+			//var col color.RGBA
+
+			/*if len(colList)%2 == 0 {
+				// Even
+				a, b := colList[len(colList)/2-1], colList[len(colList)/2]
+				col = color.RGBA{uint8((uint16(a.R) + uint16(b.R)) / 2), uint8((uint16(a.G) + uint16(b.G)) / 2), uint8((uint16(a.B) + uint16(b.B)) / 2), uint8((uint16(a.A) + uint16(b.A)) / 2)}
+			} else if len(colList) > 0 {
+				// Odd
+				col = colList[(len(colList)-1)/2]
+			}*/
+
+			var r, g, b uint8
+			if len(rList)%2 == 0 {
+				// Even
+				r = uint8((rList[len(rList)/2-1] + rList[len(rList)/2]) / 2)
+			} else if len(rList) > 0 {
+				// Odd
+				r = uint8(rList[(len(rList)-1)/2])
+			}
+			if len(gList)%2 == 0 {
+				// Even
+				g = uint8((gList[len(gList)/2-1] + gList[len(gList)/2]) / 2)
+			} else if len(gList) > 0 {
+				// Odd
+				g = uint8(gList[(len(gList)-1)/2])
+			}
+			if len(bList)%2 == 0 {
+				// Even
+				b = uint8((bList[len(bList)/2-1] + bList[len(bList)/2]) / 2)
+			} else if len(bList) > 0 {
+				// Odd
+				b = uint8(bList[(len(bList)-1)/2])
+			}
+
+			destImage.SetRGBA(ix, iy, color.RGBA{r, g, b, 255})
+		}
+	}
+}

bin/stitch/stitch.go

@@ -6,8 +6,10 @@
 package main
 
 import (
-	"fmt"
+	"image"
+	"image/png"
 	"log"
+	"os"
 	"path/filepath"
 )
 
@@ -19,7 +21,34 @@ func main() {
 		log.Panic(err)
 	}
 
-	for _, tile := range tiles {
+	/*f, err := os.Create("cpu.prof")
-		fmt.Printf("%v\n", tile)
+	if err != nil {
+		log.Panicf("could not create CPU profile: %v", err)
+	}
+	defer f.Close()
+	if err := pprof.StartCPUProfile(f); err != nil {
+		log.Panicf("could not start CPU profile: %v", err)
+	}
+	defer pprof.StopCPUProfile()*/
+
+	outputImage := image.NewRGBA(image.Rect(-4000, -4000, 8000, 8000))
+
+	tp := make(tilePairs)
+	if err := tp.stitch(tiles, outputImage); err != nil {
+		log.Panic(err)
+	}
+
+	f, err := os.Create("output.png")
+	if err != nil {
+		log.Panic(err)
+	}
+
+	if err := png.Encode(f, outputImage); err != nil {
+		f.Close()
+		log.Panic(err)
+	}
+
+	if err := f.Close(); err != nil {
+		log.Panic(err)
 	}
 }

bin/stitch/util.go

@@ -8,8 +8,13 @@ package main
 import (
 	"fmt"
 	"image"
+	"image/color"
 	"math"
 	"os"
+
+	"golang.org/x/image/font"
+	"golang.org/x/image/font/basicfont"
+	"golang.org/x/image/math/fixed"
 )
 
 // Source: https://gist.github.com/sergiotapia/7882944
@@ -28,26 +33,61 @@ func getImageFileDimension(imagePath string) (int, int, error) {
 	return image.Width, image.Height, nil
 }
 
-// getImageDifferenceValue returns the average quadratic difference of the (sub)pixels
+// getImageDifferenceValue returns the average quadratic difference of the (sub)pixels.
-func getImageDifferenceValue(a, b *image.RGBA) float64 {
+// 0 means the images are identical, +inf means that the images don't intersect.
-	intersection := a.Bounds().Intersect(b.Bounds())
+func getImageDifferenceValue(a, b *image.RGBA, offsetA image.Point) float64 {
+	intersection := a.Bounds().Add(offsetA).Intersect(b.Bounds())
 
 	if intersection.Empty() {
 		return math.Inf(1)
 	}
 
-	aSub := a.SubImage(intersection).(*image.RGBA)
+	aSub := a.SubImage(intersection.Sub(offsetA)).(*image.RGBA)
 	bSub := b.SubImage(intersection).(*image.RGBA)
 
-	value := 0.0
+	intersectionWidth := intersection.Dx() * 4
+	intersectionHeight := intersection.Dy()
 
-	for iy := 0; iy < intersection.Dy(); iy++ {
+	var value int64
-		for ix := 0; ix < intersection.Dx()*4; ix++ {
+
-			aValue := float64(aSub.Pix[ix+iy*aSub.Stride])
+	for iy := 0; iy < intersectionHeight; iy++ {
-			bValue := float64(bSub.Pix[ix+iy*bSub.Stride])
+		aSlice := aSub.Pix[iy*aSub.Stride : iy*aSub.Stride+intersectionWidth]
-			value += math.Pow(aValue-bValue, 2)
+		bSlice := bSub.Pix[iy*bSub.Stride : iy*bSub.Stride+intersectionWidth]
+		for ix := 0; ix < intersectionWidth; ix += 3 {
+			diff := int64(aSlice[ix]) - int64(bSlice[ix])
+			value += diff * diff
 		}
 	}
 
-	return value / float64(intersection.Dx()*intersection.Dy())
+	return float64(value) / float64(intersectionWidth*intersectionHeight)
+}
+
+func drawLabel(img *image.RGBA, x, y int, label string) {
+	col := color.RGBA{200, 100, 0, 255}
+	point := fixed.Point26_6{fixed.Int26_6(x * 64), fixed.Int26_6(y * 64)}
+
+	d := &font.Drawer{
+		Dst:  img,
+		Src:  image.NewUniform(col),
+		Face: basicfont.Face7x13,
+		Dot:  point,
+	}
+	d.DrawString(label)
+}
+
+func intAbs(x int) int {
+	if x < 0 {
+		return -x
+	}
+	return x
+}
+
+func pointAbs(p image.Point) image.Point {
+	if p.X < 0 {
+		p.X = -p.X
+	}
+	if p.Y < 0 {
+		p.Y = -p.Y
+	}
+	return p
 }

go.mod

@@ -7,4 +7,5 @@ require (
 	github.com/gen2brain/shm v0.0.0-20180314170312-6c18ff7f8b90 // indirect
 	github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc
 	github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8 // indirect
+	golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
 )

go.sum

@@ -6,5 +6,8 @@ github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc h1:kGFotla6Dyr6
 github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw=
 github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8 h1:RVMGIuuNgrpGB7I79f6xfhGCkpN47IaEGh8VTM0p7Xc=
 github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=