SIGN IN SIGN UP
2018-01-11 17:14:52 +02:00
package caire
import (
"fmt"
2018-01-11 17:14:52 +02:00
"image"
2018-01-17 16:39:13 +02:00
"image/color"
"image/draw"
2018-01-11 17:14:52 +02:00
"math"
2018-03-16 16:24:14 +02:00
"github.com/esimov/caire/utils"
2018-06-12 12:59:44 +03:00
pigo "github.com/esimov/pigo/core"
2018-01-11 17:14:52 +02:00
)
2025-04-27 10:40:09 +03:00
// SeamCarver defines the Carve interface method, which have to be
// implemented by the Processor struct.
type SeamCarver interface {
Resize(*image.NRGBA) (image.Image, error)
2025-04-27 10:40:09 +03:00
}
// maxFaceDetAttempts defines the maximum number of attempts of face detections
const maxFaceDetAttempts = 20
var (
detAttempts int
isFaceDetected bool
)
var (
sobel *image.NRGBA
energySeams = make([][]Seam, 0)
)
// Carver is the main entry struct having as parameters the newly generated image width, height and seam points.
2018-01-17 10:52:56 +02:00
type Carver struct {
2018-01-17 16:29:38 +02:00
Points []float64
Seams []Seam
2022-12-09 18:32:59 +02:00
Width int
Height int
2018-01-11 17:14:52 +02:00
}
// Seam struct contains the seam pixel coordinates.
type Seam struct {
X int
Y int
}
2018-01-15 09:59:53 +02:00
// NewCarver returns an initialized Carver structure.
2018-01-17 10:52:56 +02:00
func NewCarver(width, height int) *Carver {
2018-01-15 09:59:53 +02:00
return &Carver{
2025-04-27 10:40:09 +03:00
Points: make([]float64, width*height),
Seams: []Seam{},
Width: width,
Height: height,
2018-01-15 09:59:53 +02:00
}
}
// Get energy pixel value.
func (c *Carver) get(x, y int) float64 {
2018-01-15 10:06:26 +02:00
px := x + y*c.Width
2018-01-15 09:59:53 +02:00
return c.Points[px]
2018-01-11 17:14:52 +02:00
}
2018-01-15 09:59:53 +02:00
// Set energy pixel value.
func (c *Carver) set(x, y int, px float64) {
2018-01-15 10:06:26 +02:00
idx := x + y*c.Width
2018-01-15 09:59:53 +02:00
c.Points[idx] = px
2018-01-11 17:14:52 +02:00
}
2018-02-11 21:14:19 +02:00
// ComputeSeams compute the minimum energy level based on the following logic:
2018-01-11 17:14:52 +02:00
//
2022-12-09 18:32:59 +02:00
// - traverse the image from the second row to the last row
// and compute the cumulative minimum energy M for all possible
// connected seams for each entry (i, j).
//
// - the minimum energy level is calculated by summing up the current pixel value
// with the minimum pixel value of the neighboring pixels from the previous row.
func (c *Carver) ComputeSeams(p *Processor, img *image.NRGBA) (*image.NRGBA, error) {
width, height := img.Bounds().Dx(), img.Bounds().Dy()
sobel = c.SobelDetector(img, float64(p.SobelThreshold))
2018-01-26 17:44:05 +02:00
dets := []pigo.Detection{}
2024-04-24 14:31:06 +03:00
if p.FaceDetector != nil && p.FaceDetect && detAttempts < maxFaceDetAttempts {
var ratio float64
if width < height {
ratio = float64(width) / float64(height)
} else {
ratio = float64(height) / float64(width)
}
minSize := float64(utils.Min(width, height)) * ratio / 3
// Transform the image to pixel array.
2025-04-27 10:40:09 +03:00
pixels := rgbToGrayscale(img)
2018-06-12 12:59:44 +03:00
cParams := pigo.CascadeParams{
MinSize: int(minSize),
MaxSize: utils.Min(width, height),
2018-06-12 12:59:44 +03:00
ShiftFactor: 0.1,
ScaleFactor: 1.1,
2018-03-16 16:24:14 +02:00
2019-01-17 07:54:29 +02:00
ImageParams: pigo.ImageParams{
Pixels: pixels,
2021-11-26 06:46:13 +02:00
Rows: height,
Cols: width,
Dim: width,
2019-01-17 07:54:29 +02:00
},
2018-06-12 12:59:44 +03:00
}
if p.vRes {
p.FaceAngle = 0.2
}
2018-06-12 12:59:44 +03:00
// Run the classifier over the obtained leaf nodes and return the detection results.
// The result contains quadruplets representing the row, column, scale and detection score.
2024-04-24 14:31:06 +03:00
dets = p.FaceDetector.RunCascade(cParams, p.FaceAngle)
2018-06-12 12:59:44 +03:00
// Calculate the intersection over union (IoU) of two clusters.
2024-04-24 14:31:06 +03:00
dets = p.FaceDetector.ClusterDetections(dets, 0.1)
if len(dets) == 0 {
// Retry detecting faces for a certain amount of time.
if detAttempts < maxFaceDetAttempts {
detAttempts++
}
} else {
detAttempts = 0
isFaceDetected = true
2018-03-16 16:24:14 +02:00
}
}
// Traverse the pixel data of the binary file used for protecting the regions
// which we do not want to be altered by the seam carver,
// obtain the white patches and apply it to the sobel image.
if len(p.MaskPath) > 0 && p.Mask != nil {
p.DebugMask = image.NewNRGBA(img.Bounds())
for i := 0; i < width*height; i++ {
x := i % width
y := (i - x) / width
r, g, b, _ := p.Mask.At(x, y).RGBA()
if r>>8 == 0xff && g>>8 == 0xff && b>>8 == 0xff {
if isFaceDetected {
// Reduce the brightness of the mask with a small factor if human faces are detected.
// This way we can avoid the seam carver to remove
// the pixels inside the detected human faces.
sobel.Set(x, y, color.RGBA{R: 225, G: 225, B: 225, A: 255})
} else {
sobel.Set(x, y, color.White)
}
}
}
}
2022-01-13 05:00:59 +02:00
// Traverse the pixel data of the binary file used to remove the image regions
// we do not want to be retained in the final image, obtain the white patches,
// but this time inverse the colors to black and merge it back to the sobel image.
if len(p.RMaskPath) > 0 && p.RMask != nil {
p.DebugMask = image.NewNRGBA(img.Bounds())
for i := 0; i < width*height; i++ {
x := i % width
y := (i - x) / width
2022-10-22 19:08:59 +03:00
r, g, b, _ := p.RMask.At(x, y).RGBA()
// Replace the white pixels with black.
if r>>8 == 0xff && g>>8 == 0xff && b>>8 == 0xff {
2022-10-22 19:08:59 +03:00
if isFaceDetected {
// Reduce the brightness of the mask with a small factor if human faces are detected.
// This way we can avoid the seam carver to remove
// the pixels inside the detected human faces.
sobel.Set(x, y, color.RGBA{R: 25, G: 25, B: 25, A: 255})
} else {
sobel.Set(x, y, color.Black)
}
2025-04-27 10:40:09 +03:00
p.DebugMask.Set(x, y, color.Black)
} else {
2025-04-27 10:40:09 +03:00
p.DebugMask.Set(x, y, color.Transparent)
}
}
}
// Iterate over the detected faces and fill out the rectangles with white.
// We need to trick the sobel detector to consider them as important image parts.
for _, face := range dets {
if (p.NewHeight != 0 && p.NewHeight < face.Scale) ||
(p.NewWidth != 0 && p.NewWidth < face.Scale) {
return nil, fmt.Errorf("%s %s",
"cannot resize the image to the specified dimension without face deformation.\n",
2022-10-23 16:55:07 +03:00
"\tRemove the face detection option in case you still wish to resize the image.")
}
if face.Q > 5.0 {
scale := int(float64(face.Scale) / 1.7)
rect := image.Rect(
face.Col-scale,
face.Row-scale,
face.Col+scale,
face.Row+scale,
)
p.DebugMask = image.NewNRGBA(img.Bounds())
draw.Draw(sobel, rect, &image.Uniform{color.White}, image.Point{}, draw.Src)
2025-04-27 10:40:09 +03:00
draw.Draw(p.DebugMask, rect, &image.Uniform{color.White}, image.Point{}, draw.Src)
}
}
// Increase the energy value for each of the selected seam from the seams table
// in order to avoid picking the same seam over and over again.
// We expand the energy level of the selected seams to have a better redistribution.
if len(energySeams) > 0 {
for i := 0; i < len(energySeams); i++ {
for _, seam := range energySeams[i] {
sobel.Set(seam.X, seam.Y, &image.Uniform{color.White})
}
}
}
var srcImg *image.NRGBA
2018-01-17 10:52:56 +02:00
if p.BlurRadius > 0 {
srcImg = image.NewNRGBA(img.Bounds())
err := Stackblur(srcImg, sobel, uint32(p.BlurRadius))
if err != nil {
return nil, fmt.Errorf("error bluring the image: %w", err)
}
2018-01-13 15:11:54 +02:00
} else {
2018-06-12 12:59:44 +03:00
srcImg = sobel
2018-01-13 15:11:54 +02:00
}
2018-01-15 09:59:53 +02:00
for x := 0; x < c.Width; x++ {
for y := 0; y < c.Height; y++ {
2018-06-12 12:59:44 +03:00
r, _, _, a := srcImg.At(x, y).RGBA()
2018-01-26 16:35:38 +02:00
c.set(x, y, float64(r)/float64(a))
2018-01-11 17:14:52 +02:00
}
}
var left, middle, right float64
2018-01-11 17:14:52 +02:00
2018-01-17 16:25:32 +02:00
// Traverse the image from top to bottom and compute the minimum energy level.
// For each pixel in a row we compute the energy of the current pixel
// plus the energy of one of the three possible pixels above it.
for y := 1; y < c.Height; y++ {
for x := 1; x < c.Width-1; x++ {
left = c.get(x-1, y-1)
2018-01-15 09:59:53 +02:00
middle = c.get(x, y-1)
2018-01-17 16:25:32 +02:00
right = c.get(x+1, y-1)
2018-01-13 15:11:54 +02:00
min := math.Min(math.Min(left, middle), right)
2018-01-17 16:25:32 +02:00
// Set the minimum energy level.
2018-01-17 16:29:38 +02:00
c.set(x, y, c.get(x, y)+min)
2018-01-11 17:14:52 +02:00
}
2018-01-17 16:25:32 +02:00
// Special cases: pixels are far left or far right
left := c.get(0, y) + math.Min(c.get(0, y-1), c.get(1, y-1))
c.set(0, y, left)
right := c.get(0, y) + math.Min(c.get(c.Width-1, y-1), c.get(c.Width-2, y-1))
c.set(c.Width-1, y, right)
2018-01-11 17:14:52 +02:00
}
return srcImg, nil
2018-01-11 17:14:52 +02:00
}
2018-02-11 21:14:19 +02:00
// FindLowestEnergySeams find the lowest vertical energy seam.
func (c *Carver) FindLowestEnergySeams(p *Processor) []Seam {
2018-01-11 17:14:52 +02:00
// Find the lowest cost seam from the energy matrix starting from the last row.
var (
min = math.MaxFloat64
px int
)
2018-01-13 15:11:54 +02:00
seams := make([]Seam, 0)
2018-01-11 17:14:52 +02:00
2018-01-14 14:49:48 +02:00
// Find the pixel on the last row with the minimum cumulative energy and use this as the starting pixel
2018-01-15 09:59:53 +02:00
for x := 0; x < c.Width; x++ {
seam := c.get(x, c.Height-1)
2018-01-17 10:52:56 +02:00
if seam < min {
2018-01-11 17:14:52 +02:00
min = seam
px = x
}
}
2018-01-15 10:06:26 +02:00
seams = append(seams, Seam{X: px, Y: c.Height - 1})
2018-01-13 15:11:54 +02:00
var left, middle, right float64
2018-01-11 17:14:52 +02:00
// Walk up in the matrix table, check the immediate three top pixels seam level
// and add that one which has the lowest cumulative energy.
2018-01-15 10:06:26 +02:00
for y := c.Height - 2; y >= 0; y-- {
2018-02-13 12:14:42 +03:00
middle = c.get(px, y)
2018-01-11 17:14:52 +02:00
// Leftmost seam, no child to the left
if px == 0 {
2018-01-15 09:59:53 +02:00
right = c.get(px+1, y)
2018-01-13 15:11:54 +02:00
if right < middle {
px++
2018-01-11 17:14:52 +02:00
}
2018-01-15 10:06:26 +02:00
// Rightmost seam, no child to the right
2018-01-15 09:59:53 +02:00
} else if px == c.Width-1 {
left = c.get(px-1, y)
2018-01-13 15:11:54 +02:00
if left < middle {
px--
2018-01-11 17:14:52 +02:00
}
} else {
2018-01-15 09:59:53 +02:00
left = c.get(px-1, y)
right = c.get(px+1, y)
2018-01-13 15:11:54 +02:00
min := math.Min(math.Min(left, middle), right)
2018-01-11 17:14:52 +02:00
if min == left {
px--
2018-01-11 17:14:52 +02:00
} else if min == right {
px++
2018-01-11 17:14:52 +02:00
}
}
2018-01-13 15:11:54 +02:00
seams = append(seams, Seam{X: px, Y: y})
2018-01-11 17:14:52 +02:00
}
// compare against c.Width and NOT c.Height, because the image is rotated.
if p.NewWidth > c.Width || (p.NewHeight > 0 && p.NewHeight > c.Width) {
// Include the currently processed energy seam into the seams table,
// but only when an image enlargement operation is commenced.
// We need to take this approach in order to avoid picking the same seam each time.
energySeams = append(energySeams, seams)
}
2018-01-11 17:14:52 +02:00
return seams
}
2018-03-05 12:41:42 +02:00
// RemoveSeam remove the least important columns based on the stored energy (seams) level.
2018-01-17 16:39:13 +02:00
func (c *Carver) RemoveSeam(img *image.NRGBA, seams []Seam, debug bool) *image.NRGBA {
2018-01-13 15:11:54 +02:00
bounds := img.Bounds()
2018-06-12 12:59:44 +03:00
// Reduce the image width with one pixel on each iteration.
2018-01-14 14:49:48 +02:00
dst := image.NewNRGBA(image.Rect(0, 0, bounds.Dx()-1, bounds.Dy()))
2018-01-13 15:11:54 +02:00
for _, seam := range seams {
y := seam.Y
for x := 0; x < bounds.Max.X; x++ {
if seam.X == x {
2018-01-17 16:39:13 +02:00
if debug {
2021-12-16 15:00:45 +02:00
c.Seams = append(c.Seams, Seam{X: x, Y: y})
2018-01-17 16:39:13 +02:00
}
2018-01-13 15:11:54 +02:00
} else if seam.X < x {
dst.Set(x-1, y, img.At(x, y))
} else {
dst.Set(x, y, img.At(x, y))
}
}
}
return dst
}
// AddSeam add a new seam.
func (c *Carver) AddSeam(img *image.NRGBA, seams []Seam, debug bool) *image.NRGBA {
var (
lr, lg, lb uint32
rr, rg, rb uint32
)
2018-01-27 12:57:50 +02:00
bounds := img.Bounds()
dst := image.NewNRGBA(image.Rect(0, 0, bounds.Dx()+1, bounds.Dy()))
for _, seam := range seams {
y := seam.Y
for x := 0; x < bounds.Max.X; x++ {
if seam.X == x {
if debug {
2021-12-16 15:00:45 +02:00
c.Seams = append(c.Seams, Seam{X: x, Y: y})
}
if x > 0 && x != bounds.Max.X {
lr, lg, lb, _ = img.At(x-1, y).RGBA()
2018-01-27 12:57:50 +02:00
} else {
lr, lg, lb, _ = img.At(x, y).RGBA()
}
2018-01-27 16:22:33 +02:00
2018-01-27 12:57:50 +02:00
if x < bounds.Max.X-1 {
rr, rg, rb, _ = img.At(x+1, y).RGBA()
} else if x == bounds.Max.X {
2018-01-27 12:57:50 +02:00
rr, rg, rb, _ = img.At(x, y).RGBA()
}
// calculate the average color of the neighboring pixels
avr, avg, avb := (lr+rr)>>1, (lg+rg)>>1, (lb+rb)>>1
dst.Set(x, y, color.RGBA{uint8(avr >> 8), uint8(avg >> 8), uint8(avb >> 8), 0xff})
dst.Set(x+1, y, img.At(x, y))
} else if seam.X < x {
dst.Set(x, y, img.At(x-1, y))
dst.Set(x+1, y, img.At(x, y))
} else {
dst.Set(x, y, img.At(x, y))
}
}
}
return dst
}