Basic stitching functionality

- Median filtering
This commit is contained in:
David Vogel 2019-10-23 03:28:37 +02:00
parent 117054bdf5
commit 801fb10f81
8 changed files with 379 additions and 19 deletions

17
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,17 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${fileDirname}",
"env": {},
"args": []
}
]
}

View File

@ -1,6 +1,7 @@
{
"cSpell.words": [
"Vogel",
"hacky",
"kbinani",
"noita"
]

View File

@ -15,8 +15,8 @@ import (
type imageTile struct {
fileName string
originalRect image.Rectangle // Rectangle of the original position. Determined by the file name, the real coordinates may differ a few pixels.
image image.Image // Either a rectangle or an RGBA image. The bounds of this image represent the real and corrected coordinates.
offset image.Point // Correction offset of the image, so that it aligns pixel perfect with other images. Determined by image matching.
image image.Image // Either a rectangle or an RGBA image. The bounds of this image are determined by the filename.
}
func (it *imageTile) loadImage() error {
@ -55,3 +55,7 @@ func (it *imageTile) loadImage() error {
func (it *imageTile) unloadImage() {
it.image = it.image.Bounds()
}
func (it *imageTile) String() string {
return fmt.Sprintf("<ImageTile \"%v\">", it.fileName)
}

View File

@ -8,11 +8,29 @@ package main
import (
"fmt"
"image"
"image/color"
"log"
"math"
"math/rand"
"path/filepath"
"regexp"
"sort"
"strconv"
)
const tileAlignmentSearchRadius = 5
type tileAlignment struct {
offset image.Point // Contains the offset of the tile a, so that it aligns pixel perfect with tile b
}
type tileAlignmentKeys struct {
a, b *imageTile
}
// tilePairs contains image pairs and their alignment.
type tilePairs map[tileAlignmentKeys]tileAlignment
var regexFileParse = regexp.MustCompile(`^(-?\d+),(-?\d+).png$`)
func loadImages(path string) ([]imageTile, error) {
@ -45,10 +63,257 @@ func loadImages(path string) ([]imageTile, error) {
imageTiles = append(imageTiles, imageTile{
fileName: file,
originalRect: image.Rect(x, y, x+width, y+height),
image: image.Rect(x, y, x+width, y+height),
})
}
return imageTiles, nil
}
// alignTilePair returns the pixel delta for the first tile, so that it aligns perfectly with the second.
// This function will load images if needed.
func alignTilePair(tileA, tileB *imageTile, searchRadius int) (image.Point, error) {
if err := tileA.loadImage(); err != nil {
return image.Point{}, err
}
if err := tileB.loadImage(); err != nil {
return image.Point{}, err
}
// Type assertion.
imgA, imgB := *tileA.image.(*image.RGBA), *tileB.image.(*image.RGBA)
bestPoint := image.Point{}
bestValue := math.Inf(1)
for y := -searchRadius; y <= searchRadius; y++ {
for x := -searchRadius; x <= searchRadius; x++ {
point := image.Point{x, y} // Offset of the first image.
value := getImageDifferenceValue(&imgA, &imgB, point)
if bestValue > value {
bestValue, bestPoint = value, point
}
}
}
return bestPoint, nil
}
func (tp tilePairs) alignTiles(tiles []*imageTile) error {
n := len(tiles)
maxOperations, operations := (n-1)*(n)/2, 0
// Compare all n tiles with each other. (`(n-1)*(n)/2` comparisons)
for i, tileA := range tiles {
for j := i + 1; j < len(tiles); j++ {
tileB := tiles[j]
_, ok := tp[tileAlignmentKeys{tileA, tileB}]
if !ok {
// Entry doesn't exist yet. Determine tile pair alignment.
offset, err := alignTilePair(tileA, tileB, tileAlignmentSearchRadius)
if err != nil {
return fmt.Errorf("Failed to align tile pair %v %v: %w", tileA, tileB, err)
}
operations++
log.Printf("(%v/%v)Got alignment for pair %v %v. Offset = %v", operations, maxOperations, tileA, tileB, offset)
// Store tile alignment pair, also reversed.
tp[tileAlignmentKeys{tileA, tileB}] = tileAlignment{offset: offset}
tp[tileAlignmentKeys{tileB, tileA}] = tileAlignment{offset: offset.Mul(-1)}
}
}
}
// Silly and hacky method to determine the minimal error.
// TODO: Use some mixed integer method or something similar to optimize the tile alignment
// The error function returns the x and y error. The axes are optimized independent of each other later on.
errorFunction := func(tiles []*imageTile) (image.Point, error) {
errorValue := image.Point{}
for i, tileA := range tiles {
for j := i + 1; j < len(tiles); j++ {
tileB := tiles[j]
tileAlignment, ok := tp[tileAlignmentKeys{tileA, tileB}]
if !ok {
return image.Point{}, fmt.Errorf("Offset of the tile pair %v %v is missing", tileA, tileB)
}
// The error is the difference between the needed offset, and the actual offsets
tempErrorValue := pointAbs(tileAlignment.offset.Sub(tileA.offset).Add(tileB.offset))
errorValue = errorValue.Add(tempErrorValue)
}
}
return errorValue, nil
}
errorValue, err := errorFunction(tiles)
if err != nil {
return fmt.Errorf("Failed to calculate error value: %w", err)
}
// Randomly select tiles, and move them in the direction where the error value is lower.
// The "gradient" is basically caluclated by try and error.
for i := 0; i < len(tiles)*tileAlignmentSearchRadius*5; i++ {
tile := tiles[rand.Intn(len(tiles))]
// Calculate error value for positive shifting.
tile.offset = tile.offset.Add(image.Point{1, 1})
plusErrorValue, err := errorFunction(tiles)
if err != nil {
return fmt.Errorf("Failed to calculate error value: %w", err)
}
// Calculate error value for negative shifting.
tile.offset = tile.offset.Add(image.Point{-2, -2})
minusErrorValue, err := errorFunction(tiles)
if err != nil {
return fmt.Errorf("Failed to calculate error value: %w", err)
}
// Reset tile movement.
tile.offset = tile.offset.Add(image.Point{1, 1})
// Move this tile towards the smaller error value.
if plusErrorValue.X < errorValue.X {
tile.offset = tile.offset.Add(image.Point{1, 0})
}
if minusErrorValue.X < errorValue.X {
tile.offset = tile.offset.Add(image.Point{-1, 0})
}
if plusErrorValue.Y < errorValue.Y {
tile.offset = tile.offset.Add(image.Point{0, 1})
}
if minusErrorValue.Y < errorValue.Y {
tile.offset = tile.offset.Add(image.Point{0, -1})
}
}
// TODO: Move images in a way that the majority of images is positioned equal to their original position
return nil
}
func (tp tilePairs) stitch(tiles []imageTile, destImage *image.RGBA) error {
intersectTiles := []*imageTile{}
// Get only the tiles that intersect with the destination image bounds.
// Ignore alignment here, doesn't matter if an image overlaps a few pixels anyways.
for i, tile := range tiles {
if tile.image.Bounds().Add(tile.offset).Overlaps(destImage.Bounds()) {
intersectTiles = append(intersectTiles, &tiles[i])
}
}
//log.Printf("intersectTiles: %v", intersectTiles)
// Align those tiles
/*if err := tp.alignTiles(intersectTiles); err != nil {
return fmt.Errorf("Failed to align tiles: %w", err)
}*/
// TODO: Add working aligning algorithm
/*for _, intersectTile := range intersectTiles {
intersectTile.loadImage()
draw.Draw(destImage, destImage.Bounds(), intersectTile.image, destImage.Bounds().Min, draw.Over)
}*/
drawMedianBlended(intersectTiles, destImage)
/*for _, intersectTile := range intersectTiles {
drawLabel(destImage, intersectTile.image.Bounds().Min.X, intersectTile.image.Bounds().Min.Y, fmt.Sprintf("%v", intersectTile.fileName))
}*/
return nil
}
func drawMedianBlended(tiles []*imageTile, destImage *image.RGBA) {
bounds := destImage.Bounds()
// Make sure images are loaded.
for _, tile := range tiles {
tile.loadImage()
}
for iy := bounds.Min.Y; iy < bounds.Max.Y; iy++ {
for ix := bounds.Min.X; ix < bounds.Max.X; ix++ {
//colList := []color.RGBA{}
rList, gList, bList := []int{}, []int{}, []int{}
point := image.Point{ix, iy}
// Iterate through all tiles, and create a list of colors.
for _, tile := range tiles {
tilePoint := point.Sub(tile.offset)
imageRGBA, ok := tile.image.(*image.RGBA)
if ok && tilePoint.In(imageRGBA.Bounds()) {
col := imageRGBA.RGBAAt(tilePoint.X, tilePoint.Y)
//colList = append(colList, col)
rList, gList, bList = append(rList, int(col.R)), append(gList, int(col.G)), append(bList, int(col.B))
}
}
// Sort color list.
/*sort.Slice(colList, func(i, j int) bool {
return rgbToHSV(colList[i]) < rgbToHSV(colList[j])
})*/
// Sort rList.
sort.Slice(rList, func(i, j int) bool {
return rList[i] < rList[j]
})
// Sort gList.
sort.Slice(gList, func(i, j int) bool {
return gList[i] < gList[j]
})
// Sort bList.
sort.Slice(bList, func(i, j int) bool {
return bList[i] < bList[j]
})
//var col color.RGBA
/*if len(colList)%2 == 0 {
// Even
a, b := colList[len(colList)/2-1], colList[len(colList)/2]
col = color.RGBA{uint8((uint16(a.R) + uint16(b.R)) / 2), uint8((uint16(a.G) + uint16(b.G)) / 2), uint8((uint16(a.B) + uint16(b.B)) / 2), uint8((uint16(a.A) + uint16(b.A)) / 2)}
} else if len(colList) > 0 {
// Odd
col = colList[(len(colList)-1)/2]
}*/
var r, g, b uint8
if len(rList)%2 == 0 {
// Even
r = uint8((rList[len(rList)/2-1] + rList[len(rList)/2]) / 2)
} else if len(rList) > 0 {
// Odd
r = uint8(rList[(len(rList)-1)/2])
}
if len(gList)%2 == 0 {
// Even
g = uint8((gList[len(gList)/2-1] + gList[len(gList)/2]) / 2)
} else if len(gList) > 0 {
// Odd
g = uint8(gList[(len(gList)-1)/2])
}
if len(bList)%2 == 0 {
// Even
b = uint8((bList[len(bList)/2-1] + bList[len(bList)/2]) / 2)
} else if len(bList) > 0 {
// Odd
b = uint8(bList[(len(bList)-1)/2])
}
destImage.SetRGBA(ix, iy, color.RGBA{r, g, b, 255})
}
}
}

View File

@ -6,8 +6,10 @@
package main
import (
"fmt"
"image"
"image/png"
"log"
"os"
"path/filepath"
)
@ -19,7 +21,34 @@ func main() {
log.Panic(err)
}
for _, tile := range tiles {
fmt.Printf("%v\n", tile)
/*f, err := os.Create("cpu.prof")
if err != nil {
log.Panicf("could not create CPU profile: %v", err)
}
defer f.Close()
if err := pprof.StartCPUProfile(f); err != nil {
log.Panicf("could not start CPU profile: %v", err)
}
defer pprof.StopCPUProfile()*/
outputImage := image.NewRGBA(image.Rect(-4000, -4000, 8000, 8000))
tp := make(tilePairs)
if err := tp.stitch(tiles, outputImage); err != nil {
log.Panic(err)
}
f, err := os.Create("output.png")
if err != nil {
log.Panic(err)
}
if err := png.Encode(f, outputImage); err != nil {
f.Close()
log.Panic(err)
}
if err := f.Close(); err != nil {
log.Panic(err)
}
}

View File

@ -8,8 +8,13 @@ package main
import (
"fmt"
"image"
"image/color"
"math"
"os"
"golang.org/x/image/font"
"golang.org/x/image/font/basicfont"
"golang.org/x/image/math/fixed"
)
// Source: https://gist.github.com/sergiotapia/7882944
@ -28,26 +33,61 @@ func getImageFileDimension(imagePath string) (int, int, error) {
return image.Width, image.Height, nil
}
// getImageDifferenceValue returns the average quadratic difference of the (sub)pixels
func getImageDifferenceValue(a, b *image.RGBA) float64 {
intersection := a.Bounds().Intersect(b.Bounds())
// getImageDifferenceValue returns the average quadratic difference of the (sub)pixels.
// 0 means the images are identical, +inf means that the images don't intersect.
func getImageDifferenceValue(a, b *image.RGBA, offsetA image.Point) float64 {
intersection := a.Bounds().Add(offsetA).Intersect(b.Bounds())
if intersection.Empty() {
return math.Inf(1)
}
aSub := a.SubImage(intersection).(*image.RGBA)
aSub := a.SubImage(intersection.Sub(offsetA)).(*image.RGBA)
bSub := b.SubImage(intersection).(*image.RGBA)
value := 0.0
intersectionWidth := intersection.Dx() * 4
intersectionHeight := intersection.Dy()
for iy := 0; iy < intersection.Dy(); iy++ {
for ix := 0; ix < intersection.Dx()*4; ix++ {
aValue := float64(aSub.Pix[ix+iy*aSub.Stride])
bValue := float64(bSub.Pix[ix+iy*bSub.Stride])
value += math.Pow(aValue-bValue, 2)
var value int64
for iy := 0; iy < intersectionHeight; iy++ {
aSlice := aSub.Pix[iy*aSub.Stride : iy*aSub.Stride+intersectionWidth]
bSlice := bSub.Pix[iy*bSub.Stride : iy*bSub.Stride+intersectionWidth]
for ix := 0; ix < intersectionWidth; ix += 3 {
diff := int64(aSlice[ix]) - int64(bSlice[ix])
value += diff * diff
}
}
return value / float64(intersection.Dx()*intersection.Dy())
return float64(value) / float64(intersectionWidth*intersectionHeight)
}
func drawLabel(img *image.RGBA, x, y int, label string) {
col := color.RGBA{200, 100, 0, 255}
point := fixed.Point26_6{fixed.Int26_6(x * 64), fixed.Int26_6(y * 64)}
d := &font.Drawer{
Dst: img,
Src: image.NewUniform(col),
Face: basicfont.Face7x13,
Dot: point,
}
d.DrawString(label)
}
func intAbs(x int) int {
if x < 0 {
return -x
}
return x
}
func pointAbs(p image.Point) image.Point {
if p.X < 0 {
p.X = -p.X
}
if p.Y < 0 {
p.Y = -p.Y
}
return p
}

1
go.mod
View File

@ -7,4 +7,5 @@ require (
github.com/gen2brain/shm v0.0.0-20180314170312-6c18ff7f8b90 // indirect
github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc
github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8 // indirect
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
)

3
go.sum
View File

@ -6,5 +6,8 @@ github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc h1:kGFotla6Dyr6
github.com/kbinani/screenshot v0.0.0-20190719135742-f06580e30cdc/go.mod h1:f8GY5V3lRzakvEyr49P7hHRYoHtPr8zvj/7JodCoRzw=
github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8 h1:RVMGIuuNgrpGB7I79f6xfhGCkpN47IaEGh8VTM0p7Xc=
github.com/lxn/win v0.0.0-20190919090605-24c5960b03d8/go.mod h1:ouWl4wViUNh8tPSIwxTVMuS014WakR1hqvBc2I0bMoA=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=