From 801fb10f81caee13e02e8bfc90705843184df6ef Mon Sep 17 00:00:00 2001 From: David Vogel Date: Wed, 23 Oct 2019 03:28:37 +0200 Subject: [PATCH] Basic stitching functionality - Median filtering --- .vscode/launch.json | 17 +++ .vscode/settings.json | 1 + bin/stitch/imagetile.go | 8 +- bin/stitch/imagetiles.go | 271 ++++++++++++++++++++++++++++++++++++++- bin/stitch/stitch.go | 35 ++++- bin/stitch/util.go | 62 +++++++-- go.mod | 1 + go.sum | 3 + 8 files changed, 379 insertions(+), 19 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..c23774c --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}", + "env": {}, + "args": [] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 07a98d4..9039a4f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "cSpell.words": [ "Vogel", + "hacky", "kbinani", "noita" ] diff --git a/bin/stitch/imagetile.go b/bin/stitch/imagetile.go index d4e542c..423a319 100644 --- a/bin/stitch/imagetile.go +++ b/bin/stitch/imagetile.go @@ -15,8 +15,8 @@ import ( type imageTile struct { fileName string - originalRect image.Rectangle // Rectangle of the original position. Determined by the file name, the real coordinates may differ a few pixels. - image image.Image // Either a rectangle or an RGBA image. The bounds of this image represent the real and corrected coordinates. + offset image.Point // Correction offset of the image, so that it aligns pixel perfect with other images. Determined by image matching. + image image.Image // Either a rectangle or an RGBA image. The bounds of this image are determined by the filename. } func (it *imageTile) loadImage() error { @@ -55,3 +55,7 @@ func (it *imageTile) loadImage() error { func (it *imageTile) unloadImage() { it.image = it.image.Bounds() } + +func (it *imageTile) String() string { + return fmt.Sprintf("", it.fileName) +} diff --git a/bin/stitch/imagetiles.go b/bin/stitch/imagetiles.go index 274c587..9e856b4 100644 --- a/bin/stitch/imagetiles.go +++ b/bin/stitch/imagetiles.go @@ -8,11 +8,29 @@ package main import ( "fmt" "image" + "image/color" + "log" + "math" + "math/rand" "path/filepath" "regexp" + "sort" "strconv" ) +const tileAlignmentSearchRadius = 5 + +type tileAlignment struct { + offset image.Point // Contains the offset of the tile a, so that it aligns pixel perfect with tile b +} + +type tileAlignmentKeys struct { + a, b *imageTile +} + +// tilePairs contains image pairs and their alignment. +type tilePairs map[tileAlignmentKeys]tileAlignment + var regexFileParse = regexp.MustCompile(`^(-?\d+),(-?\d+).png$`) func loadImages(path string) ([]imageTile, error) { @@ -44,11 +62,258 @@ func loadImages(path string) ([]imageTile, error) { } imageTiles = append(imageTiles, imageTile{ - fileName: file, - originalRect: image.Rect(x, y, x+width, y+height), - image: image.Rect(x, y, x+width, y+height), + fileName: file, + image: image.Rect(x, y, x+width, y+height), }) } return imageTiles, nil } + +// alignTilePair returns the pixel delta for the first tile, so that it aligns perfectly with the second. +// This function will load images if needed. +func alignTilePair(tileA, tileB *imageTile, searchRadius int) (image.Point, error) { + if err := tileA.loadImage(); err != nil { + return image.Point{}, err + } + if err := tileB.loadImage(); err != nil { + return image.Point{}, err + } + + // Type assertion. + imgA, imgB := *tileA.image.(*image.RGBA), *tileB.image.(*image.RGBA) + + bestPoint := image.Point{} + bestValue := math.Inf(1) + + for y := -searchRadius; y <= searchRadius; y++ { + for x := -searchRadius; x <= searchRadius; x++ { + point := image.Point{x, y} // Offset of the first image. + + value := getImageDifferenceValue(&imgA, &imgB, point) + if bestValue > value { + bestValue, bestPoint = value, point + } + } + } + + return bestPoint, nil +} + +func (tp tilePairs) alignTiles(tiles []*imageTile) error { + + n := len(tiles) + maxOperations, operations := (n-1)*(n)/2, 0 + + // Compare all n tiles with each other. (`(n-1)*(n)/2` comparisons) + for i, tileA := range tiles { + for j := i + 1; j < len(tiles); j++ { + tileB := tiles[j] + + _, ok := tp[tileAlignmentKeys{tileA, tileB}] + if !ok { + // Entry doesn't exist yet. Determine tile pair alignment. + offset, err := alignTilePair(tileA, tileB, tileAlignmentSearchRadius) + if err != nil { + return fmt.Errorf("Failed to align tile pair %v %v: %w", tileA, tileB, err) + } + + operations++ + log.Printf("(%v/%v)Got alignment for pair %v %v. Offset = %v", operations, maxOperations, tileA, tileB, offset) + + // Store tile alignment pair, also reversed. + tp[tileAlignmentKeys{tileA, tileB}] = tileAlignment{offset: offset} + tp[tileAlignmentKeys{tileB, tileA}] = tileAlignment{offset: offset.Mul(-1)} + + } + } + } + + // Silly and hacky method to determine the minimal error. + // TODO: Use some mixed integer method or something similar to optimize the tile alignment + + // The error function returns the x and y error. The axes are optimized independent of each other later on. + errorFunction := func(tiles []*imageTile) (image.Point, error) { + errorValue := image.Point{} + + for i, tileA := range tiles { + for j := i + 1; j < len(tiles); j++ { + tileB := tiles[j] + + tileAlignment, ok := tp[tileAlignmentKeys{tileA, tileB}] + if !ok { + return image.Point{}, fmt.Errorf("Offset of the tile pair %v %v is missing", tileA, tileB) + } + + // The error is the difference between the needed offset, and the actual offsets + tempErrorValue := pointAbs(tileAlignment.offset.Sub(tileA.offset).Add(tileB.offset)) + + errorValue = errorValue.Add(tempErrorValue) + } + } + return errorValue, nil + } + + errorValue, err := errorFunction(tiles) + if err != nil { + return fmt.Errorf("Failed to calculate error value: %w", err) + } + // Randomly select tiles, and move them in the direction where the error value is lower. + // The "gradient" is basically caluclated by try and error. + for i := 0; i < len(tiles)*tileAlignmentSearchRadius*5; i++ { + tile := tiles[rand.Intn(len(tiles))] + + // Calculate error value for positive shifting. + tile.offset = tile.offset.Add(image.Point{1, 1}) + plusErrorValue, err := errorFunction(tiles) + if err != nil { + return fmt.Errorf("Failed to calculate error value: %w", err) + } + + // Calculate error value for negative shifting. + tile.offset = tile.offset.Add(image.Point{-2, -2}) + minusErrorValue, err := errorFunction(tiles) + if err != nil { + return fmt.Errorf("Failed to calculate error value: %w", err) + } + + // Reset tile movement. + tile.offset = tile.offset.Add(image.Point{1, 1}) + + // Move this tile towards the smaller error value. + if plusErrorValue.X < errorValue.X { + tile.offset = tile.offset.Add(image.Point{1, 0}) + } + if minusErrorValue.X < errorValue.X { + tile.offset = tile.offset.Add(image.Point{-1, 0}) + } + if plusErrorValue.Y < errorValue.Y { + tile.offset = tile.offset.Add(image.Point{0, 1}) + } + if minusErrorValue.Y < errorValue.Y { + tile.offset = tile.offset.Add(image.Point{0, -1}) + } + } + + // TODO: Move images in a way that the majority of images is positioned equal to their original position + + return nil +} + +func (tp tilePairs) stitch(tiles []imageTile, destImage *image.RGBA) error { + intersectTiles := []*imageTile{} + + // Get only the tiles that intersect with the destination image bounds. + // Ignore alignment here, doesn't matter if an image overlaps a few pixels anyways. + for i, tile := range tiles { + if tile.image.Bounds().Add(tile.offset).Overlaps(destImage.Bounds()) { + intersectTiles = append(intersectTiles, &tiles[i]) + } + } + + //log.Printf("intersectTiles: %v", intersectTiles) + + // Align those tiles + /*if err := tp.alignTiles(intersectTiles); err != nil { + return fmt.Errorf("Failed to align tiles: %w", err) + }*/ + + // TODO: Add working aligning algorithm + + /*for _, intersectTile := range intersectTiles { + intersectTile.loadImage() + draw.Draw(destImage, destImage.Bounds(), intersectTile.image, destImage.Bounds().Min, draw.Over) + }*/ + + drawMedianBlended(intersectTiles, destImage) + + /*for _, intersectTile := range intersectTiles { + drawLabel(destImage, intersectTile.image.Bounds().Min.X, intersectTile.image.Bounds().Min.Y, fmt.Sprintf("%v", intersectTile.fileName)) + }*/ + + return nil +} + +func drawMedianBlended(tiles []*imageTile, destImage *image.RGBA) { + bounds := destImage.Bounds() + + // Make sure images are loaded. + for _, tile := range tiles { + tile.loadImage() + } + + for iy := bounds.Min.Y; iy < bounds.Max.Y; iy++ { + for ix := bounds.Min.X; ix < bounds.Max.X; ix++ { + //colList := []color.RGBA{} + rList, gList, bList := []int{}, []int{}, []int{} + point := image.Point{ix, iy} + + // Iterate through all tiles, and create a list of colors. + for _, tile := range tiles { + tilePoint := point.Sub(tile.offset) + imageRGBA, ok := tile.image.(*image.RGBA) + if ok && tilePoint.In(imageRGBA.Bounds()) { + col := imageRGBA.RGBAAt(tilePoint.X, tilePoint.Y) + //colList = append(colList, col) + rList, gList, bList = append(rList, int(col.R)), append(gList, int(col.G)), append(bList, int(col.B)) + } + } + + // Sort color list. + /*sort.Slice(colList, func(i, j int) bool { + return rgbToHSV(colList[i]) < rgbToHSV(colList[j]) + })*/ + + // Sort rList. + sort.Slice(rList, func(i, j int) bool { + return rList[i] < rList[j] + }) + + // Sort gList. + sort.Slice(gList, func(i, j int) bool { + return gList[i] < gList[j] + }) + + // Sort bList. + sort.Slice(bList, func(i, j int) bool { + return bList[i] < bList[j] + }) + + //var col color.RGBA + + /*if len(colList)%2 == 0 { + // Even + a, b := colList[len(colList)/2-1], colList[len(colList)/2] + col = color.RGBA{uint8((uint16(a.R) + uint16(b.R)) / 2), uint8((uint16(a.G) + uint16(b.G)) / 2), uint8((uint16(a.B) + uint16(b.B)) / 2), uint8((uint16(a.A) + uint16(b.A)) / 2)} + } else if len(colList) > 0 { + // Odd + col = colList[(len(colList)-1)/2] + }*/ + + var r, g, b uint8 + if len(rList)%2 == 0 { + // Even + r = uint8((rList[len(rList)/2-1] + rList[len(rList)/2]) / 2) + } else if len(rList) > 0 { + // Odd + r = uint8(rList[(len(rList)-1)/2]) + } + if len(gList)%2 == 0 { + // Even + g = uint8((gList[len(gList)/2-1] + gList[len(gList)/2]) / 2) + } else if len(gList) > 0 { + // Odd + g = uint8(gList[(len(gList)-1)/2]) + } + if len(bList)%2 == 0 { + // Even + b = uint8((bList[len(bList)/2-1] + bList[len(bList)/2]) / 2) + } else if len(bList) > 0 { + // Odd + b = uint8(bList[(len(bList)-1)/2]) + } + + destImage.SetRGBA(ix, iy, color.RGBA{r, g, b, 255}) + } + } +} diff --git a/bin/stitch/stitch.go b/bin/stitch/stitch.go index cd55205..0cd4f5e 100644 --- a/bin/stitch/stitch.go +++ b/bin/stitch/stitch.go @@ -6,8 +6,10 @@ package main import ( - "fmt" + "image" + "image/png" "log" + "os" "path/filepath" ) @@ -19,7 +21,34 @@ func main() { log.Panic(err) } - for _, tile := range tiles { - fmt.Printf("%v\n", tile) + /*f, err := os.Create("cpu.prof") + if err != nil { + log.Panicf("could not create CPU profile: %v", err) + } + defer f.Close() + if err := pprof.StartCPUProfile(f); err != nil { + log.Panicf("could not start CPU profile: %v", err) + } + defer pprof.StopCPUProfile()*/ + + outputImage := image.NewRGBA(image.Rect(-4000, -4000, 8000, 8000)) + + tp := make(tilePairs) + if err := tp.stitch(tiles, outputImage); err != nil { + log.Panic(err) + } + + f, err := os.Create("output.png") + if err != nil { + log.Panic(err) + } + + if err := png.Encode(f, outputImage); err != nil { + f.Close() + log.Panic(err) + } + + if err := f.Close(); err != nil { + log.Panic(err) } } diff --git a/bin/stitch/util.go b/bin/stitch/util.go index 1f7e52c..c9a02fc 100644 --- a/bin/stitch/util.go +++ b/bin/stitch/util.go @@ -8,8 +8,13 @@ package main import ( "fmt" "image" + "image/color" "math" "os" + + "golang.org/x/image/font" + "golang.org/x/image/font/basicfont" + "golang.org/x/image/math/fixed" ) // Source: https://gist.github.com/sergiotapia/7882944 @@ -28,26 +33,61 @@ func getImageFileDimension(imagePath string) (int, int, error) { return image.Width, image.Height, nil } -// getImageDifferenceValue returns the average quadratic difference of the (sub)pixels -func getImageDifferenceValue(a, b *image.RGBA) float64 { - intersection := a.Bounds().Intersect(b.Bounds()) +// getImageDifferenceValue returns the average quadratic difference of the (sub)pixels. +// 0 means the images are identical, +inf means that the images don't intersect. +func getImageDifferenceValue(a, b *image.RGBA, offsetA image.Point) float64 { + intersection := a.Bounds().Add(offsetA).Intersect(b.Bounds()) if intersection.Empty() { return math.Inf(1) } - aSub := a.SubImage(intersection).(*image.RGBA) + aSub := a.SubImage(intersection.Sub(offsetA)).(*image.RGBA) bSub := b.SubImage(intersection).(*image.RGBA) - value := 0.0 + intersectionWidth := intersection.Dx() * 4 + intersectionHeight := intersection.Dy() - for iy := 0; iy < intersection.Dy(); iy++ { - for ix := 0; ix < intersection.Dx()*4; ix++ { - aValue := float64(aSub.Pix[ix+iy*aSub.Stride]) - bValue := float64(bSub.Pix[ix+iy*bSub.Stride]) - value += math.Pow(aValue-bValue, 2) + var value int64 + + for iy := 0; iy < intersectionHeight; iy++ { + aSlice := aSub.Pix[iy*aSub.Stride : iy*aSub.Stride+intersectionWidth] + bSlice := bSub.Pix[iy*bSub.Stride : iy*bSub.Stride+intersectionWidth] + for ix := 0; ix < intersectionWidth; ix += 3 { + diff := int64(aSlice[ix]) - int64(bSlice[ix]) + value += diff * diff } } - return value / float64(intersection.Dx()*intersection.Dy()) + return float64(value) / float64(intersectionWidth*intersectionHeight) +} + +func drawLabel(img *image.RGBA, x, y int, label string) { + col := color.RGBA{200, 100, 0, 255} + point := fixed.Point26_6{fixed.Int26_6(x * 64), fixed.Int26_6(y * 64)} + + d := &font.Drawer{ + Dst: img, + Src: image.NewUniform(col), + Face: basicfont.Face7x13, + Dot: point, + } + d.DrawString(label) +} + +func intAbs(x int) int { + if x < 0 { + return -x + } + return x +} + +func pointAbs(p image.Point) image.Point { + if p.X < 0 { + p.X = -p.X + } + if p.Y < 0 { + p.Y = -p.Y + } + return p } diff --git a/go.mod b/go.mod index 5905688..f021a36 100644 --- a/go.mod +++ b/go.mod @@ -7,4 +7,5 @@ require ( github.com/gen2brain/shm v0.0.0-20180314170312-6c18ff7f8b90 // indirect github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8 // indirect + golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 ) diff --git a/go.sum b/go.sum index 45d8f07..22f4bf5 100644 --- a/go.sum +++ b/go.sum @@ -6,5 +6,8 @@ github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc h1:kGFotla6Dyr6 github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw= github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8 h1:RVMGIuuNgrpGB7I79f6xfhGCkpN47IaEGh8VTM0p7Xc= github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=