diff --git a/modules/photo/include/opencv2/photo.hpp b/modules/photo/include/opencv2/photo.hpp index df0ccbeb34..ee6b12e9dc 100644 --- a/modules/photo/include/opencv2/photo.hpp +++ b/modules/photo/include/opencv2/photo.hpp @@ -315,7 +315,7 @@ CV_EXPORTS_W void illuminationChange(InputArray src, InputArray mask, OutputArra float alpha = 0.2f, float beta = 0.4f); CV_EXPORTS_W void textureFlattening(InputArray src, InputArray mask, OutputArray dst, - double low_threshold = 30, double high_threshold = 45, + float low_threshold = 30, float high_threshold = 45, int kernel_size = 3); CV_EXPORTS_W void edgePreservingFilter(InputArray src, OutputArray dst, int flags = 1, diff --git a/modules/photo/src/seamless_cloning.cpp b/modules/photo/src/seamless_cloning.cpp index 445c6dae74..2564145e58 100644 --- a/modules/photo/src/seamless_cloning.cpp +++ b/modules/photo/src/seamless_cloning.cpp @@ -41,7 +41,6 @@ #include "precomp.hpp" #include "opencv2/photo.hpp" -#include #include "seamless_cloning.hpp" @@ -50,9 +49,9 @@ using namespace cv; void cv::seamlessClone(InputArray _src, InputArray _dst, InputArray _mask, Point p, OutputArray _blend, int flags) { - Mat src = _src.getMat(); - Mat dest = _dst.getMat(); - Mat mask = _mask.getMat(); + const Mat src = _src.getMat(); + const Mat dest = _dst.getMat(); + const Mat mask = _mask.getMat(); _blend.create(dest.size(), CV_8UC3); Mat blend = _blend.getMat(); @@ -87,6 +86,8 @@ void cv::seamlessClone(InputArray _src, InputArray _dst, InputArray _mask, Point int lenx = maxx - minx; int leny = maxy - miny; + Mat patch = Mat::zeros(Size(leny, lenx), CV_8UC3); + int minxd = p.y - lenx/2; int maxxd = p.y + lenx/2; int minyd = p.x - leny/2; @@ -102,12 +103,14 @@ void cv::seamlessClone(InputArray _src, InputArray _dst, InputArray _mask, Point gray(roi_s).copyTo(destinationROI); src(roi_s).copyTo(sourceROI,gray(roi_s)); + src(roi_s).copyTo(patch, gray(roi_s)); destinationROI = cd_mask(roi_d); cs_mask(roi_s).copyTo(destinationROI); + Cloning obj; - obj.normal_clone(dest,cd_mask,dst_mask,blend,flags); + obj.normalClone(dest,cd_mask,dst_mask,blend,flags); } @@ -134,7 +137,7 @@ void cv::colorChange(InputArray _src, InputArray _mask, OutputArray _dst, float src.copyTo(cs_mask,gray); Cloning obj; - obj.local_color_change(src,cs_mask,gray,blend,red,green,blue); + obj.localColorChange(src,cs_mask,gray,blend,red,green,blue); } void cv::illuminationChange(InputArray _src, InputArray _mask, OutputArray _dst, float a, float b) @@ -159,12 +162,12 @@ void cv::illuminationChange(InputArray _src, InputArray _mask, OutputArray _dst, src.copyTo(cs_mask,gray); Cloning obj; - obj.illum_change(src,cs_mask,gray,blend,alpha,beta); + obj.illuminationChange(src,cs_mask,gray,blend,alpha,beta); } void cv::textureFlattening(InputArray _src, InputArray _mask, OutputArray _dst, - double low_threshold, double high_threshold, int kernel_size) + float low_threshold, float high_threshold, int kernel_size) { Mat src = _src.getMat(); @@ -184,5 +187,5 @@ void cv::textureFlattening(InputArray _src, InputArray _mask, OutputArray _dst, src.copyTo(cs_mask,gray); Cloning obj; - obj.texture_flatten(src,cs_mask,gray,low_threshold,high_threshold,kernel_size,blend); + obj.textureFlatten(src,cs_mask,gray,low_threshold,high_threshold,kernel_size,blend); } diff --git a/modules/photo/src/seamless_cloning.hpp b/modules/photo/src/seamless_cloning.hpp index f0713ceb2e..a0f9d6863c 100644 --- a/modules/photo/src/seamless_cloning.hpp +++ b/modules/photo/src/seamless_cloning.hpp @@ -39,546 +39,52 @@ // //M*/ +#ifndef CV_SEAMLESS_CLONING_HPP___ +#define CV_SEAMLESS_CLONING_HPP___ + #include "precomp.hpp" #include "opencv2/photo.hpp" -#include -#include -#include -#include "math.h" -using namespace std; -using namespace cv; +#include -class Cloning +namespace cv { - public: - - vector rgb_channel, rgbx_channel, rgby_channel, output; - Mat grx, gry, sgx, sgy, srx32, sry32, grx32, gry32, smask, smask1; - void init_var(Mat &I, Mat &wmask); - void initialization(Mat &I, Mat &mask, Mat &wmask); - void scalar_product(Mat mat, float r, float g, float b); - void array_product(Mat mat1, Mat mat2, Mat mat3); - void poisson(Mat &I, Mat &gx, Mat &gy, Mat &sx, Mat &sy); - void evaluate(Mat &I, Mat &wmask, Mat &cloned); - void getGradientx(const Mat &img, Mat &gx); - void getGradienty(const Mat &img, Mat &gy); - void lapx(const Mat &img, Mat &gxx); - void lapy(const Mat &img, Mat &gyy); - void dst(double *mod_diff, double *sineTransform,int h,int w); - void idst(double *mod_diff, double *sineTransform,int h,int w); - void transpose(double *mat, double *mat_t,int h,int w); - void solve(const Mat &img, double *mod_diff, Mat &result); - void poisson_solver(const Mat &img, Mat &gxx , Mat &gyy, Mat &result); - void normal_clone(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, int num); - void local_color_change(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float red_mul, float green_mul, float blue_mul); - void illum_change(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float alpha, float beta); - void texture_flatten(Mat &I, Mat &mask, Mat &wmask, double low_threshold, double high_threhold, int kernel_size, Mat &cloned); -}; - -void Cloning::getGradientx( const Mat &img, Mat &gx) -{ - Mat kernel = Mat::zeros(1, 3, CV_8S); - kernel.at(0,2) = 1; - kernel.at(0,1) = -1; - filter2D(img, gx, CV_32F, kernel); -} - -void Cloning::getGradienty( const Mat &img, Mat &gy) -{ - Mat kernel = Mat::zeros(3, 1, CV_8S); - kernel.at(2,0) = 1; - kernel.at(1,0) = -1; - filter2D(img, gy, CV_32F, kernel); -} - -void Cloning::lapx( const Mat &img, Mat &gxx) -{ - Mat kernel = Mat::zeros(1, 3, CV_8S); - kernel.at(0,0) = -1; - kernel.at(0,1) = 1; - filter2D(img, gxx, CV_32F, kernel); -} - -void Cloning::lapy( const Mat &img, Mat &gyy) -{ - Mat kernel = Mat::zeros(3, 1, CV_8S); - kernel.at(0,0) = -1; - kernel.at(1,0) = 1; - filter2D(img, gyy, CV_32F, kernel); -} - -void Cloning::dst(double *mod_diff, double *sineTransform,int h,int w) -{ - - unsigned long int idx; - - Mat temp = Mat(2*h+2,1,CV_32F); - Mat res = Mat(h,1,CV_32F); - - Mat planes[] = {Mat_(temp), Mat::zeros(temp.size(), CV_32F)}; - - Mat result; - int p=0; - for(int i=0;i(0,0) = 0.0; + public: + void normalClone(const cv::Mat& destination, const cv::Mat &mask, const cv::Mat &wmask, cv::Mat &cloned, int flag); + void illuminationChange(cv::Mat &I, cv::Mat &mask, cv::Mat &wmask, cv::Mat &cloned, float alpha, float beta); + void localColorChange(cv::Mat &I, cv::Mat &mask, cv::Mat &wmask, cv::Mat &cloned, float red_mul, float green_mul, float blue_mul); + void textureFlatten(cv::Mat &I, cv::Mat &mask, cv::Mat &wmask, float low_threshold, float high_threhold, int kernel_size, cv::Mat &cloned); - for(int j=0,r=1;j(r,0) = (float) mod_diff[idx]; - } + protected: - temp.at(h+1,0)=0.0; + void initVariables(const cv::Mat &destination, const cv::Mat &binaryMask); + void computeDerivatives(const cv::Mat &destination, const cv::Mat &patch, const cv::Mat &binaryMask); + void scalarProduct(cv::Mat mat, float r, float g, float b); + void poisson(const cv::Mat &destination); + void evaluate(const cv::Mat &I, const cv::Mat &wmask, const cv::Mat &cloned); + void dst(const Mat& src, Mat& dest, bool invert = false); + void idst(const Mat& src, Mat& dest); + void solve(const Mat &img, Mat& mod_diff, Mat &result); - for(int j=h-1, r=h+2;j>=0;j--,r++) - { - idx = j*w+i; - temp.at(r,0) = (float) (-1.0 * mod_diff[idx]); - } + void poissonSolver(const cv::Mat &img, cv::Mat &gxx , cv::Mat &gyy, cv::Mat &result); - merge(planes, 2, result); + void arrayProduct(const cv::Mat& lhs, const cv::Mat& rhs, cv::Mat& result) const; - dft(result,result,0,0); + void computeGradientX(const cv::Mat &img, cv::Mat &gx); + void computeGradientY(const cv::Mat &img, cv::Mat &gy); + void computeLaplacianX(const cv::Mat &img, cv::Mat &gxx); + void computeLaplacianY(const cv::Mat &img, cv::Mat &gyy); - Mat planes1[] = {Mat::zeros(result.size(), CV_32F), Mat::zeros(result.size(), CV_32F)}; + private: + std::vector rgbx_channel, rgby_channel, output; + cv::Mat destinationGradientX, destinationGradientY; + cv::Mat patchGradientX, patchGradientY; + cv::Mat binaryMaskFloat, binaryMaskFloatInverted; - split(result, planes1); - - std::complex two_i = std::sqrt(std::complex(-1)); - - double factor = -2*imag(two_i); - - for(int c=1,z=0;c(z,0) = (float) (planes1[1].at(c,0)/factor); - } - - for(int q=0,z=0;q(z,0); - } - p++; - } -} - -void Cloning::idst(double *mod_diff, double *sineTransform,int h,int w) -{ - int nn = h+1; - unsigned long int idx; - dst(mod_diff,sineTransform,h,w); - for(int i= 0;i(i,j) = (float) mat[idx]; - } - } - Mat tmp_t = tmp.t(); - - for(int i = 0;i < tmp_t.size().height; i++) - for(int j=0;j(i,j); - } -} - -void Cloning::solve(const Mat &img, double *mod_diff, Mat &result) -{ - int w = img.size().width; - int h = img.size().height; - - unsigned long int idx,idx1; - - double *sineTransform = new double[(h-2)*(w-2)]; - double *sineTransform_t = new double[(h-2)*(w-2)]; - double *denom = new double[(h-2)*(w-2)]; - double *invsineTransform = new double[(h-2)*(w-2)]; - double *invsineTransform_t = new double[(h-2)*(w-2)]; - double *img_d = new double[(h)*(w)]; - - dst(mod_diff,sineTransform,h-2,w-2); - - transpose(sineTransform,sineTransform_t,h-2,w-2); - - dst(sineTransform_t,sineTransform,w-2,h-2); - - transpose(sineTransform,sineTransform_t,w-2,h-2); - - int cy = 1; - - for(int i = 0 ; i < w-2;i++,cy++) - { - for(int j = 0,cx = 1; j < h-2; j++,cx++) - { - idx = j*(w-2) + i; - denom[idx] = (float) 2*cos(CV_PI*cy/( (double) (w-1))) - 2 + 2*cos(CV_PI*cx/((double) (h-1))) - 2; - - } - } - - for(idx = 0 ; idx < (unsigned)(w-2)*(h-2) ;idx++) - { - sineTransform_t[idx] = sineTransform_t[idx]/denom[idx]; - } - - idst(sineTransform_t,invsineTransform,h-2,w-2); - - transpose(invsineTransform,invsineTransform_t,h-2,w-2); - - idst(invsineTransform_t,invsineTransform,w-2,h-2); - - transpose(invsineTransform,invsineTransform_t,w-2,h-2); - - for(int i = 0 ; i < h;i++) - { - for(int j = 0 ; j < w; j++) - { - idx = i*w + j; - img_d[idx] = (double)img.at(i,j); - } - } - for(int i = 1 ; i < h-1;i++) - { - for(int j = 1 ; j < w-1; j++) - { - idx = i*w + j; - img_d[idx] = 0.0; - } - } - for(int i = 1,id1=0 ; i < h-1;i++,id1++) - { - for(int j = 1,id2=0 ; j < w-1; j++,id2++) - { - idx = i*w + j; - idx1= id1*(w-2) + id2; - img_d[idx] = invsineTransform_t[idx1]; - } - } - - for(int i = 0 ; i < h;i++) - { - for(int j = 0 ; j < w; j++) - { - idx = i*w + j; - if(img_d[idx] < 0.0) - result.at(i,j) = 0; - else if(img_d[idx] > 255.0) - result.at(i,j) = 255; - else - result.at(i,j) = (uchar) img_d[idx]; - } - } - - delete [] sineTransform; - delete [] sineTransform_t; - delete [] denom; - delete [] invsineTransform; - delete [] invsineTransform_t; - delete [] img_d; -} - -void Cloning::poisson_solver(const Mat &img, Mat &gxx , Mat &gyy, Mat &result) -{ - - int w = img.size().width; - int h = img.size().height; - - unsigned long int idx; - - Mat lap = Mat(img.size(),CV_32FC1); - - lap = gxx + gyy; - - Mat bound = img.clone(); - - rectangle(bound, Point(1, 1), Point(img.cols-2, img.rows-2), Scalar::all(0), -1); - - double *boundary_point = new double[h*w]; - - for(int i =1;i(i,j) + (int)bound.at(i,(j+1)) + (int)bound.at(i,(j-1)) - + (int)bound.at(i-1,j) + (int)bound.at(i+1,j); - } - - Mat diff = Mat(h,w,CV_32FC1); - for(int i =0;i(i,j) = (float) (lap.at(i,j) - boundary_point[idx]); - } - } - - double *mod_diff = new double[(h-2)*(w-2)]; - for(int i = 0 ; i < h-2;i++) - { - for(int j = 0 ; j < w-2; j++) - { - idx = i*(w-2) + j; - mod_diff[idx] = diff.at(i+1,j+1); - - } - } - ///////////////////////////////////////////////////// Find DST ///////////////////////////////////////////////////// - - solve(img,mod_diff,result); - - delete [] mod_diff; - delete [] boundary_point; -} - -void Cloning::init_var(Mat &I, Mat &wmask) -{ - grx = Mat(I.size(),CV_32FC3); - gry = Mat(I.size(),CV_32FC3); - sgx = Mat(I.size(),CV_32FC3); - sgy = Mat(I.size(),CV_32FC3); - - split(I,rgb_channel); - - smask = Mat(wmask.size(),CV_32FC1); - srx32 = Mat(I.size(),CV_32FC3); - sry32 = Mat(I.size(),CV_32FC3); - smask1 = Mat(wmask.size(),CV_32FC1); - grx32 = Mat(I.size(),CV_32FC3); - gry32 = Mat(I.size(),CV_32FC3); -} - -void Cloning::initialization(Mat &I, Mat &mask, Mat &wmask) -{ - init_var(I,wmask); - - getGradientx(I,grx); - getGradienty(I,gry); - - getGradientx(mask,sgx); - getGradienty(mask,sgy); - - Mat Kernel(Size(3, 3), CV_8UC1); - Kernel.setTo(Scalar(1)); - - erode(wmask, wmask, Kernel, Point(-1,-1), 3); - - wmask.convertTo(smask,CV_32FC1,1.0/255.0); - I.convertTo(srx32,CV_32FC3,1.0/255.0); - I.convertTo(sry32,CV_32FC3,1.0/255.0); -} - -void Cloning::scalar_product(Mat mat, float r, float g, float b) -{ - vector channels; - split(mat,channels); - multiply(channels[2],r,channels[2]); - multiply(channels[1],g,channels[1]); - multiply(channels[0],b,channels[0]); - merge(channels,mat); -} - -void Cloning::array_product(Mat mat1, Mat mat2, Mat mat3) -{ - vector channels_temp1; - vector channels_temp2; - split(mat1,channels_temp1); - split(mat2,channels_temp2); - multiply(channels_temp2[2],mat3,channels_temp1[2]); - multiply(channels_temp2[1],mat3,channels_temp1[1]); - multiply(channels_temp2[0],mat3,channels_temp1[0]); - merge(channels_temp1,mat1); -} - -void Cloning::poisson(Mat &I, Mat &gx, Mat &gy, Mat &sx, Mat &sy) -{ - Mat fx = Mat(I.size(),CV_32FC3); - Mat fy = Mat(I.size(),CV_32FC3); - - fx = gx + sx; - fy = gy + sy; - - Mat gxx = Mat(I.size(),CV_32FC3); - Mat gyy = Mat(I.size(),CV_32FC3); - - lapx(fx,gxx); - lapy(fy,gyy); - - split(gxx,rgbx_channel); - split(gyy,rgby_channel); - - split(I,output); - - poisson_solver(rgb_channel[2],rgbx_channel[2], rgby_channel[2],output[2]); - poisson_solver(rgb_channel[1],rgbx_channel[1], rgby_channel[1],output[1]); - poisson_solver(rgb_channel[0],rgbx_channel[0], rgby_channel[0],output[0]); -} - -void Cloning::evaluate(Mat &I, Mat &wmask, Mat &cloned) -{ - bitwise_not(wmask,wmask); - - wmask.convertTo(smask1,CV_32FC1,1.0/255.0); - I.convertTo(grx32,CV_32FC3,1.0/255.0); - I.convertTo(gry32,CV_32FC3,1.0/255.0); - - array_product(grx32,grx,smask1); - array_product(gry32,gry,smask1); - - poisson(I,grx32,gry32,srx32,sry32); - - merge(output,cloned); -} - -void Cloning::normal_clone(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, int num) -{ - int w = I.size().width; - int h = I.size().height; - int channel = I.channels(); - - - initialization(I,mask,wmask); - - if(num == 1) - { - array_product(srx32,sgx,smask); - array_product(sry32,sgy,smask); - - } - else if(num == 2) - { - - for(int i=0;i < h; i++) - { - for(int j=0; j < w; j++) - { - for(int c=0;c(i,j*channel+c) - sgy.at(i,j*channel+c)) > - abs(grx.at(i,j*channel+c) - gry.at(i,j*channel+c))) - { - - srx32.at(i,j*channel+c) = sgx.at(i,j*channel+c) - * smask.at(i,j); - sry32.at(i,j*channel+c) = sgy.at(i,j*channel+c) - * smask.at(i,j); - } - else - { - srx32.at(i,j*channel+c) = grx.at(i,j*channel+c) - * smask.at(i,j); - sry32.at(i,j*channel+c) = gry.at(i,j*channel+c) - * smask.at(i,j); - } - } - } - } - - } - else if(num == 3) - { - Mat gray = Mat(mask.size(),CV_8UC1); - Mat gray8 = Mat(mask.size(),CV_8UC3); - cvtColor(mask, gray, COLOR_BGR2GRAY ); - vector temp; - split(I,temp); - gray.copyTo(temp[2]); - gray.copyTo(temp[1]); - gray.copyTo(temp[0]); - - merge(temp,gray8); - - getGradientx(gray8,sgx); - getGradienty(gray8,sgy); - - array_product(srx32,sgx,smask); - array_product(sry32,sgy,smask); - - } - - evaluate(I,wmask,cloned); -} - -void Cloning::local_color_change(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float red_mul=1.0, - float green_mul=1.0, float blue_mul=1.0) -{ - initialization(I,mask,wmask); - - array_product(srx32,sgx,smask); - array_product(sry32,sgy,smask); - scalar_product(srx32,red_mul,green_mul,blue_mul); - scalar_product(sry32,red_mul,green_mul,blue_mul); - - evaluate(I,wmask,cloned); -} - -void Cloning::illum_change(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float alpha, float beta) -{ - initialization(I,mask,wmask); - - array_product(srx32,sgx,smask); - array_product(sry32,sgy,smask); - - Mat mag = Mat(I.size(),CV_32FC3); - magnitude(srx32,sry32,mag); - - Mat multX, multY, multx_temp, multy_temp; - - multiply(srx32,pow(alpha,beta),multX); - pow(mag,-1*beta, multx_temp); - multiply(multX,multx_temp,srx32); - patchNaNs(srx32); - - multiply(sry32,pow(alpha,beta),multY); - pow(mag,-1*beta, multy_temp); - multiply(multY,multy_temp,sry32); - patchNaNs(sry32); - - Mat zeroMask = (srx32 != 0); - - srx32.copyTo(srx32, zeroMask); - sry32.copyTo(sry32, zeroMask); - - evaluate(I,wmask,cloned); -} - -void Cloning::texture_flatten(Mat &I, Mat &mask, Mat &wmask, double low_threshold, - double high_threshold, int kernel_size, Mat &cloned) -{ - initialization(I,mask,wmask); - - Mat out = Mat(mask.size(),CV_8UC1); - Canny(mask,out,low_threshold,high_threshold,kernel_size); - - Mat zeros(sgx.size(), CV_32FC3); - zeros.setTo(0); - Mat zerosMask = (out != 255); - zeros.copyTo(sgx, zerosMask); - zeros.copyTo(sgy, zerosMask); - - array_product(srx32,sgx,smask); - array_product(sry32,sgy,smask); - - evaluate(I,wmask,cloned); + std::vector filter_X, filter_Y; + }; } +#endif \ No newline at end of file diff --git a/modules/photo/src/seamless_cloning_impl.cpp b/modules/photo/src/seamless_cloning_impl.cpp new file mode 100644 index 0000000000..fe2751d143 --- /dev/null +++ b/modules/photo/src/seamless_cloning_impl.cpp @@ -0,0 +1,469 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2013, OpenCV Foundation, all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#include "seamless_cloning.hpp" + +using namespace cv; +using namespace std; + + +void Cloning::computeGradientX( const Mat &img, Mat &gx) +{ + Mat kernel = Mat::zeros(1, 3, CV_8S); + kernel.at(0,2) = 1; + kernel.at(0,1) = -1; + + if(img.channels() == 3) + { + filter2D(img, gx, CV_32F, kernel); + } + else if (img.channels() == 1) + { + Mat tmp[3]; + for(int chan = 0 ; chan < 3 ; ++chan) + { + filter2D(img, tmp[chan], CV_32F, kernel); + } + merge(tmp, 3, gx); + } +} + +void Cloning::computeGradientY( const Mat &img, Mat &gy) +{ + Mat kernel = Mat::zeros(3, 1, CV_8S); + kernel.at(2,0) = 1; + kernel.at(1,0) = -1; + + if(img.channels() == 3) + { + filter2D(img, gy, CV_32F, kernel); + } + else if (img.channels() == 1) + { + Mat tmp[3]; + for(int chan = 0 ; chan < 3 ; ++chan) + { + filter2D(img, tmp[chan], CV_32F, kernel); + } + merge(tmp, 3, gy); + } +} + +void Cloning::computeLaplacianX( const Mat &img, Mat &laplacianX) +{ + Mat kernel = Mat::zeros(1, 3, CV_8S); + kernel.at(0,0) = -1; + kernel.at(0,1) = 1; + filter2D(img, laplacianX, CV_32F, kernel); +} + +void Cloning::computeLaplacianY( const Mat &img, Mat &laplacianY) +{ + Mat kernel = Mat::zeros(3, 1, CV_8S); + kernel.at(0,0) = -1; + kernel.at(1,0) = 1; + filter2D(img, laplacianY, CV_32F, kernel); +} + +void Cloning::dst(const Mat& src, Mat& dest, bool invert) +{ + Mat temp = Mat::zeros(src.rows, 2 * src.cols + 2, CV_32F); + + int flag = invert ? DFT_ROWS + DFT_SCALE + DFT_INVERSE: DFT_ROWS; + + src.copyTo(temp(Rect(1,0, src.cols, src.rows))); + + for(int j = 0 ; j < src.rows ; ++j) + { + float * tempLinePtr = temp.ptr(j); + const float * srcLinePtr = src.ptr(j); + for(int i = 0 ; i < src.cols ; ++i) + { + tempLinePtr[src.cols + 2 + i] = - srcLinePtr[src.cols - 1 - i]; + } + } + + Mat planes[] = {temp, Mat::zeros(temp.size(), CV_32F)}; + Mat complex; + + merge(planes, 2, complex); + dft(complex, complex, flag); + split(complex, planes); + temp = Mat::zeros(src.cols, 2 * src.rows + 2, CV_32F); + + for(int j = 0 ; j < src.cols ; ++j) + { + float * tempLinePtr = temp.ptr(j); + for(int i = 0 ; i < src.rows ; ++i) + { + float val = planes[1].ptr(i)[j + 1]; + tempLinePtr[i + 1] = val; + tempLinePtr[temp.cols - 1 - i] = - val; + } + } + + Mat planes2[] = {temp, Mat::zeros(temp.size(), CV_32F)}; + + merge(planes2, 2, complex); + dft(complex, complex, flag); + split(complex, planes2); + + temp = planes2[1].t(); + dest = Mat::zeros(src.size(), CV_32F); + temp(Rect( 0, 1, src.cols, src.rows)).copyTo(dest); +} + +void Cloning::idst(const Mat& src, Mat& dest) +{ + dst(src, dest, true); +} + +void Cloning::solve(const Mat &img, Mat& mod_diff, Mat &result) +{ + const int w = img.cols; + const int h = img.rows; + + Mat res; + dst(mod_diff, res); + + for(int j = 0 ; j < h-2; j++) + { + float * resLinePtr = res.ptr(j); + for(int i = 0 ; i < w-2; i++) + { + resLinePtr[i] /= (filter_X[i] + filter_Y[j] - 4); + } + } + + idst(res, mod_diff); + + unsigned char * resLinePtr = result.ptr(0); + const unsigned char * imgLinePtr = img.ptr(0); + const float * interpLinePtr = NULL; + + //first col + for(int i = 0 ; i < w ; ++i) + result.ptr(0)[i] = img.ptr(0)[i]; + + for(int j = 1 ; j < h-1 ; ++j) + { + resLinePtr = result.ptr(j); + imgLinePtr = img.ptr(j); + interpLinePtr = mod_diff.ptr(j-1); + + //first row + resLinePtr[0] = imgLinePtr[0]; + + for(int i = 1 ; i < w-1 ; ++i) + { + //saturate cast is not used here, because it behaves differently from the previous implementation + //most notable, saturate_cast rounds before truncating, here it's the opposite. + float value = interpLinePtr[i-1]; + if(value < 0.) + resLinePtr[i] = 0; + else if (value > 255.0) + resLinePtr[i] = 255; + else + resLinePtr[i] = static_cast(value); + } + + //last row + resLinePtr[w-1] = imgLinePtr[w-1]; + } + + //last col + resLinePtr = result.ptr(h-1); + imgLinePtr = img.ptr(h-1); + for(int i = 0 ; i < w ; ++i) + resLinePtr[i] = imgLinePtr[i]; +} + +void Cloning::poissonSolver(const Mat &img, Mat &laplacianX , Mat &laplacianY, Mat &result) +{ + const int w = img.cols; + const int h = img.rows; + + Mat lap = Mat(img.size(),CV_32FC1); + + lap = laplacianX + laplacianY; + + Mat bound = img.clone(); + + rectangle(bound, Point(1, 1), Point(img.cols-2, img.rows-2), Scalar::all(0), -1); + Mat boundary_points; + Laplacian(bound, boundary_points, CV_32F); + + boundary_points = lap - boundary_points; + + Mat mod_diff = boundary_points(Rect(1, 1, w-2, h-2)); + + solve(img,mod_diff,result); +} + +void Cloning::initVariables(const Mat &destination, const Mat &binaryMask) +{ + destinationGradientX = Mat(destination.size(),CV_32FC3); + destinationGradientY = Mat(destination.size(),CV_32FC3); + patchGradientX = Mat(destination.size(),CV_32FC3); + patchGradientY = Mat(destination.size(),CV_32FC3); + + binaryMaskFloat = Mat(binaryMask.size(),CV_32FC1); + binaryMaskFloatInverted = Mat(binaryMask.size(),CV_32FC1); + + //init of the filters used in the dst + const int w = destination.cols; + filter_X.resize(w - 2); + for(int i = 0 ; i < w-2 ; ++i) + filter_X[i] = 2.0f * std::cos(static_cast(CV_PI) * (i + 1) / (w - 1)); + + const int h = destination.rows; + filter_Y.resize(h - 2); + for(int j = 0 ; j < h - 2 ; ++j) + filter_Y[j] = 2.0f * std::cos(static_cast(CV_PI) * (j + 1) / (h - 1)); +} + +void Cloning::computeDerivatives(const Mat& destination, const Mat &patch, const Mat &binaryMask) +{ + initVariables(destination,binaryMask); + + computeGradientX(destination,destinationGradientX); + computeGradientY(destination,destinationGradientY); + + computeGradientX(patch,patchGradientX); + computeGradientY(patch,patchGradientY); + + Mat Kernel(Size(3, 3), CV_8UC1); + Kernel.setTo(Scalar(1)); + erode(binaryMask, binaryMask, Kernel, Point(-1,-1), 3); + + binaryMask.convertTo(binaryMaskFloat,CV_32FC1,1.0/255.0); +} + +void Cloning::scalarProduct(Mat mat, float r, float g, float b) +{ + vector channels; + split(mat,channels); + multiply(channels[2],r,channels[2]); + multiply(channels[1],g,channels[1]); + multiply(channels[0],b,channels[0]); + merge(channels,mat); +} + +void Cloning::arrayProduct(const cv::Mat& lhs, const cv::Mat& rhs, cv::Mat& result) const +{ + vector lhs_channels; + vector result_channels; + + split(lhs,lhs_channels); + split(result,result_channels); + + for(int chan = 0 ; chan < 3 ; ++chan) + multiply(lhs_channels[chan],rhs,result_channels[chan]); + + merge(result_channels,result); +} + +void Cloning::poisson(const Mat &destination) +{ + Mat laplacianX = Mat(destination.size(),CV_32FC3); + Mat laplacianY = Mat(destination.size(),CV_32FC3); + + laplacianX = destinationGradientX + patchGradientX; + laplacianY = destinationGradientY + patchGradientY; + + computeLaplacianX(laplacianX,laplacianX); + computeLaplacianY(laplacianY,laplacianY); + + split(laplacianX,rgbx_channel); + split(laplacianY,rgby_channel); + + split(destination,output); + + for(int chan = 0 ; chan < 3 ; ++chan) + { + poissonSolver(output[chan], rgbx_channel[chan], rgby_channel[chan], output[chan]); + } +} + +void Cloning::evaluate(const Mat &I, const Mat &wmask, const Mat &cloned) +{ + bitwise_not(wmask,wmask); + + wmask.convertTo(binaryMaskFloatInverted,CV_32FC1,1.0/255.0); + + arrayProduct(destinationGradientX,binaryMaskFloatInverted, destinationGradientX); + arrayProduct(destinationGradientY,binaryMaskFloatInverted, destinationGradientY); + + poisson(I); + + merge(output,cloned); +} + +void Cloning::normalClone(const Mat &destination, const Mat &patch, const Mat &binaryMask, Mat &cloned, int flag) +{ + const int w = destination.cols; + const int h = destination.rows; + const int channel = destination.channels(); + const int n_elem_in_line = w * channel; + + computeDerivatives(destination,patch,binaryMask); + + switch(flag) + { + case NORMAL_CLONE: + arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); + arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); + break; + + case MIXED_CLONE: + { + AutoBuffer maskIndices(n_elem_in_line); + for (int i = 0; i < n_elem_in_line; ++i) + maskIndices[i] = i / channel; + + for(int i=0;i < h; i++) + { + float * patchXLinePtr = patchGradientX.ptr(i); + float * patchYLinePtr = patchGradientY.ptr(i); + const float * destinationXLinePtr = destinationGradientX.ptr(i); + const float * destinationYLinePtr = destinationGradientY.ptr(i); + const float * binaryMaskLinePtr = binaryMaskFloat.ptr(i); + + for(int j=0; j < n_elem_in_line; j++) + { + int maskIndex = maskIndices[j]; + + if(abs(patchXLinePtr[j] - patchYLinePtr[j]) > + abs(destinationXLinePtr[j] - destinationYLinePtr[j])) + { + patchXLinePtr[j] *= binaryMaskLinePtr[maskIndex]; + patchYLinePtr[j] *= binaryMaskLinePtr[maskIndex]; + } + else + { + patchXLinePtr[j] = destinationXLinePtr[j] + * binaryMaskLinePtr[maskIndex]; + patchYLinePtr[j] = destinationYLinePtr[j] + * binaryMaskLinePtr[maskIndex]; + } + } + } + } + break; + + case MONOCHROME_TRANSFER: + Mat gray = Mat(patch.size(),CV_8UC1); + cvtColor(patch, gray, COLOR_BGR2GRAY ); + + computeGradientX(gray,patchGradientX); + computeGradientY(gray,patchGradientY); + + arrayProduct(patchGradientX, binaryMaskFloat, patchGradientX); + arrayProduct(patchGradientY, binaryMaskFloat, patchGradientY); + break; + + } + + evaluate(destination,binaryMask,cloned); +} + +void Cloning::localColorChange(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float red_mul=1.0, + float green_mul=1.0, float blue_mul=1.0) +{ + computeDerivatives(I,mask,wmask); + + arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); + arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); + scalarProduct(patchGradientX,red_mul,green_mul,blue_mul); + scalarProduct(patchGradientY,red_mul,green_mul,blue_mul); + + evaluate(I,wmask,cloned); +} + +void Cloning::illuminationChange(Mat &I, Mat &mask, Mat &wmask, Mat &cloned, float alpha, float beta) +{ + computeDerivatives(I,mask,wmask); + + arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); + arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); + + Mat mag = Mat(I.size(),CV_32FC3); + magnitude(patchGradientX,patchGradientY,mag); + + Mat multX, multY, multx_temp, multy_temp; + + multiply(patchGradientX,pow(alpha,beta),multX); + pow(mag,-1*beta, multx_temp); + multiply(multX,multx_temp, patchGradientX); + patchNaNs(patchGradientX); + + multiply(patchGradientY,pow(alpha,beta),multY); + pow(mag,-1*beta, multy_temp); + multiply(multY,multy_temp,patchGradientY); + patchNaNs(patchGradientY); + + Mat zeroMask = (patchGradientX != 0); + + patchGradientX.copyTo(patchGradientX, zeroMask); + patchGradientY.copyTo(patchGradientY, zeroMask); + + evaluate(I,wmask,cloned); +} + +void Cloning::textureFlatten(Mat &I, Mat &mask, Mat &wmask, float low_threshold, + float high_threshold, int kernel_size, Mat &cloned) +{ + computeDerivatives(I,mask,wmask); + + Mat out = Mat(mask.size(),CV_8UC1); + Canny(mask,out,low_threshold,high_threshold,kernel_size); + + Mat zeros(patchGradientX.size(), CV_32FC3); + zeros.setTo(0); + Mat zerosMask = (out != 255); + zeros.copyTo(patchGradientX, zerosMask); + zeros.copyTo(patchGradientY, zerosMask); + + arrayProduct(patchGradientX,binaryMaskFloat, patchGradientX); + arrayProduct(patchGradientY,binaryMaskFloat, patchGradientY); + + evaluate(I,wmask,cloned); +} diff --git a/modules/photo/test/test_cloning.cpp b/modules/photo/test/test_cloning.cpp index 62fe243954..56d166205c 100644 --- a/modules/photo/test/test_cloning.cpp +++ b/modules/photo/test/test_cloning.cpp @@ -39,6 +39,15 @@ // //M*/ +#define OUTPUT_SAVING 0 +#if OUTPUT_SAVING +#define SAVE(x) std::vector params;\ + params.push_back(16);\ + params.push_back(0);\ + imwrite(folder + "output.png", x ,params); +#else +#define SAVE(x) +#endif #include "test_precomp.hpp" #include "opencv2/photo.hpp" @@ -47,6 +56,7 @@ using namespace cv; using namespace std; +static const double numerical_precision = 1000.; TEST(Photo_SeamlessClone_normal, regression) { @@ -69,8 +79,13 @@ TEST(Photo_SeamlessClone_normal, regression) p.y = destination.size().height/2; seamlessClone(source, destination, mask, p, result, 1); - imwrite(folder + "cloned.png", result); + Mat reference = imread(folder + "reference.png"); + + SAVE(result); + + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } TEST(Photo_SeamlessClone_mixed, regression) @@ -94,7 +109,11 @@ TEST(Photo_SeamlessClone_mixed, regression) p.y = destination.size().height/2; seamlessClone(source, destination, mask, p, result, 2); - imwrite(folder + "cloned.png", result); + SAVE(result); + + Mat reference = imread(folder + "reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } @@ -119,7 +138,11 @@ TEST(Photo_SeamlessClone_featureExchange, regression) p.y = destination.size().height/2; seamlessClone(source, destination, mask, p, result, 3); - imwrite(folder + "cloned.png", result); + SAVE(result); + + Mat reference = imread(folder + "reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } @@ -138,7 +161,11 @@ TEST(Photo_SeamlessClone_colorChange, regression) Mat result; colorChange(source, mask, result, 1.5, .5, .5); - imwrite(folder + "cloned.png", result); + SAVE(result); + + Mat reference = imread(folder + "reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } @@ -157,7 +184,11 @@ TEST(Photo_SeamlessClone_illuminationChange, regression) Mat result; illuminationChange(source, mask, result, 0.2f, 0.4f); - imwrite(folder + "cloned.png", result); + SAVE(result); + + Mat reference = imread(folder + "reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } @@ -176,6 +207,10 @@ TEST(Photo_SeamlessClone_textureFlattening, regression) Mat result; textureFlattening(source, mask, result, 30, 45, 3); - imwrite(folder + "cloned.png", result); + SAVE(result); + + Mat reference = imread(folder + "reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } diff --git a/modules/photo/test/test_decolor.cpp b/modules/photo/test/test_decolor.cpp index bf21f37384..259f7afd10 100644 --- a/modules/photo/test/test_decolor.cpp +++ b/modules/photo/test/test_decolor.cpp @@ -47,6 +47,7 @@ using namespace cv; using namespace std; +static const double numerical_precision = 10.; TEST(Photo_Decolor, regression) { @@ -61,7 +62,11 @@ TEST(Photo_Decolor, regression) Mat grayscale, color_boost; decolor(original, grayscale, color_boost); - imwrite(folder + "grayscale.png",grayscale); - imwrite(folder + "color_boost.png",color_boost); + Mat reference_grayscale = imread(folder + "grayscale_reference.png", 0 /* == grayscale image*/); + double error_grayscale = cvtest::norm(reference_grayscale, grayscale, NORM_L1); + EXPECT_LE(error_grayscale, numerical_precision); + Mat reference_boost = imread(folder + "boost_reference.png"); + double error_boost = cvtest::norm(reference_boost, color_boost, NORM_L1); + EXPECT_LE(error_boost, numerical_precision); } diff --git a/modules/photo/test/test_npr.cpp b/modules/photo/test/test_npr.cpp index 4d1c027ba1..24f6f886ec 100755 --- a/modules/photo/test/test_npr.cpp +++ b/modules/photo/test/test_npr.cpp @@ -47,6 +47,7 @@ using namespace cv; using namespace std; +static const double numerical_precision = 100.; TEST(Photo_NPR_EdgePreserveSmoothing_RecursiveFilter, regression) { @@ -60,8 +61,9 @@ TEST(Photo_NPR_EdgePreserveSmoothing_RecursiveFilter, regression) Mat result; edgePreservingFilter(source,result,1); - imwrite(folder + "smoothened_RF.png", result); - + Mat reference = imread(folder + "smoothened_RF_reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } TEST(Photo_NPR_EdgePreserveSmoothing_NormConvFilter, regression) @@ -76,7 +78,9 @@ TEST(Photo_NPR_EdgePreserveSmoothing_NormConvFilter, regression) Mat result; edgePreservingFilter(source,result,2); - imwrite(folder + "smoothened_NCF.png", result); + Mat reference = imread(folder + "smoothened_NCF_reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } @@ -92,8 +96,9 @@ TEST(Photo_NPR_DetailEnhance, regression) Mat result; detailEnhance(source,result); - imwrite(folder + "detail_enhanced.png", result); - + Mat reference = imread(folder + "detail_enhanced_reference.png"); + double error = cvtest::norm(reference, result, NORM_L1); + EXPECT_LE(error, numerical_precision); } TEST(Photo_NPR_PencilSketch, regression) @@ -105,12 +110,16 @@ TEST(Photo_NPR_PencilSketch, regression) ASSERT_FALSE(source.empty()) << "Could not load input image " << original_path; - Mat result,result1; - pencilSketch(source,result,result1, 10, 0.1f, 0.03f); + Mat pencil_result, color_pencil_result; + pencilSketch(source,pencil_result, color_pencil_result, 10, 0.1f, 0.03f); - imwrite(folder + "pencil_sketch.png", result); - imwrite(folder + "color_pencil_sketch.png", result1); + Mat pencil_reference = imread(folder + "pencil_sketch_reference.png", 0 /* == grayscale*/); + double pencil_error = norm(pencil_reference, pencil_result, NORM_L1); + EXPECT_LE(pencil_error, numerical_precision); + Mat color_pencil_reference = imread(folder + "color_pencil_sketch_reference.png"); + double color_pencil_error = cvtest::norm(color_pencil_reference, color_pencil_result, NORM_L1); + EXPECT_LE(color_pencil_error, numerical_precision); } TEST(Photo_NPR_Stylization, regression) @@ -125,6 +134,8 @@ TEST(Photo_NPR_Stylization, regression) Mat result; stylization(source,result); - imwrite(folder + "stylized.png", result); + Mat stylized_reference = imread(folder + "stylized_reference.png"); + double stylized_error = cvtest::norm(stylized_reference, result, NORM_L1); + EXPECT_LE(stylized_error, numerical_precision); } diff --git a/samples/cpp/cloning_gui.cpp b/samples/cpp/cloning_gui.cpp index 07671ca8a6..668dd2a959 100644 --- a/samples/cpp/cloning_gui.cpp +++ b/samples/cpp/cloning_gui.cpp @@ -65,7 +65,7 @@ float alpha,beta; float red, green, blue; -double low_t, high_t; +float low_t, high_t; void source(int, int, int, int, void*); void destination(int, int, int, int, void*); diff --git a/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp b/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp index 6edeb3eadb..6ca79a7a04 100644 --- a/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp +++ b/samples/cpp/tutorial_code/photo/seamless_cloning/cloning_gui.cpp @@ -64,7 +64,7 @@ float alpha,beta; float red, green, blue; -double low_t, high_t; +float low_t, high_t; void source(int, int, int, int, void*); void destination(int, int, int, int, void*);